baalajimaestro/ollama
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

fix gemma, command-r layer weights

Browse source
This commit is contained in:
Michael Yang 2024-04-26 15:00:54 -07:00
parent 7fea1ecdf6
commit f81f308118
1 changed files with 8 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
llm/memory.go
View file

@ -102,10 +102,14 @@ func EstimateGPULayers(gpus []gpu.GpuInfo, ggml *GGML, projectors []string, opts
layers := ggml.Tensors().Layers() layers := ggml.Tensors().Layers()
var memoryLayerOutput uint64 var memoryLayerOutput uint64
for k, v := range layers { if layer, ok := layers["output_norm"]; ok {
if k == "output" || k == "output_norm" { memoryLayerOutput += layer.size()
memoryLayerOutput += v.size() }
}
if layer, ok := layers["output"]; ok {
memoryLayerOutput += layer.size()
} else if layer, ok := layers["token_embd"]; ok {
memoryLayerOutput += layer.size()
} }
if gpus[0].Library == "metal" && opts.UseMMap { if gpus[0].Library == "metal" && opts.UseMMap {