baalajimaestro/ollama
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

main-gpu argument is not getting passed to llamacpp, fixed. (#1192)

Browse source
This commit is contained in:
Purinda Gunasekara 2023-11-21 02:52:52 +11:00 committed by GitHub
parent 2fdf1b5ff8
commit be61a81758
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 2 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
llm/llama.go
View file

@ -339,6 +339,7 @@ func newLlama(model string, adapters []string, runners []ModelRunner, numLayers
"--model", model,
"--ctx-size", fmt.Sprintf("%d", opts.NumCtx),
"--batch-size", fmt.Sprintf("%d", opts.NumBatch),
"--main-gpu", fmt.Sprintf("%d", opts.MainGPU),
"--n-gpu-layers", fmt.Sprintf("%d", numGPU),
"--embedding",
}
@ -544,6 +545,7 @@ func (llm *llama) Predict(ctx context.Context, prevContext []int, prompt string,
"stream": true,
"n_predict": llm.NumPredict,
"n_keep": llm.NumKeep,
"main_gpu": llm.MainGPU,
"temperature": llm.Temperature,
"top_k": llm.TopK,
"top_p": llm.TopP,