baalajimaestro/ollama
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

no gpu for 70B model

Browse source
This commit is contained in:
Michael Yang 2023-08-01 16:57:14 -07:00
parent 7a1c3e62dc
commit 74a5f7e698
1 changed files with 5 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
llama/llama.go
View file

@ -128,6 +128,11 @@ func New(model string, opts api.Options) (*LLM, error) {
C.llama_backend_init(C.bool(llm.UseNUMA)) C.llama_backend_init(C.bool(llm.UseNUMA))
// TODO: GQA == 8 suggests 70B model which doesn't support metal
if llm.NumGQA == 8 {
llm.NumGPU = 0
}
params := C.llama_context_default_params() params := C.llama_context_default_params()
params.seed = C.uint(llm.Seed) params.seed = C.uint(llm.Seed)
params.n_ctx = C.int(llm.NumCtx) params.n_ctx = C.int(llm.NumCtx)