baalajimaestro/ollama
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #3964 from ollama/mxyng/weights

Browse source 
fix gemma, command-r layer weights
This commit is contained in:
Michael Yang 2024-04-26 15:23:33 -07:00 committed by GitHub
commit 65fadddc85
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 8 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
llm/memory.go
View file

@ -102,10 +102,14 @@ func EstimateGPULayers(gpus []gpu.GpuInfo, ggml *GGML, projectors []string, opts
layers := ggml.Tensors().Layers() layers := ggml.Tensors().Layers()
var memoryLayerOutput uint64 var memoryLayerOutput uint64
for k, v := range layers { if layer, ok := layers["output_norm"]; ok {
if k == "output" || k == "output_norm" { memoryLayerOutput += layer.size()
memoryLayerOutput += v.size() }
}
if layer, ok := layers["output"]; ok {
memoryLayerOutput += layer.size()
} else if layer, ok := layers["token_embd"]; ok {
memoryLayerOutput += layer.size()
} }
if gpus[0].Library == "metal" && opts.UseMMap { if gpus[0].Library == "metal" && opts.UseMMap {