baalajimaestro/ollama
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

fix metal gpu

Browse source
This commit is contained in:
Michael Yang 2024-04-02 16:06:45 -07:00
parent a57818d93e
commit 80163ebcb5
1 changed files with 2 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
llm/server.go
View file

@ -85,7 +85,7 @@ func NewLlamaServer(model string, adapters, projectors []string, opts *api.Optio
graph := int64(ggml.KV().GQA()) * kv / 6 graph := int64(ggml.KV().GQA()) * kv / 6
usedMemory += graph usedMemory += graph
if usedMemory > availableMemory || slices.Contains(cpuOnlyFamilies, ggml.KV().Architecture()) { if (usedMemory > availableMemory || slices.Contains(cpuOnlyFamilies, ggml.KV().Architecture())) && info.Library != "metal" {
info.Library = "cpu" info.Library = "cpu"
} }
@ -159,7 +159,7 @@ func NewLlamaServer(model string, adapters, projectors []string, opts *api.Optio
params = append(params, "--log-disable") params = append(params, "--log-disable")
} }
if opts.NumGPU > 0 { if opts.NumGPU >= 0 {
params = append(params, "--n-gpu-layers", fmt.Sprintf("%d", opts.NumGPU)) params = append(params, "--n-gpu-layers", fmt.Sprintf("%d", opts.NumGPU))
} }