baalajimaestro/ollama
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

typo

Browse source
This commit is contained in:
Michael Yang 2024-01-09 09:45:42 -08:00
parent 4a33cede20
commit f921e2696e
1 changed files with 1 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
llm/llm.go
View file

@ -61,7 +61,7 @@ func New(workDir, model string, adapters, projectors []string, opts api.Options)
requiredKv := 2 * 2 * int64(opts.NumCtx) * int64(ggml.NumLayers()) * int64(ggml.NumEmbed()) * int64(ggml.NumHeadKv()) / int64(ggml.NumHead()) requiredKv := 2 * 2 * int64(opts.NumCtx) * int64(ggml.NumLayers()) * int64(ggml.NumEmbed()) * int64(ggml.NumHeadKv()) / int64(ggml.NumHead())
// this amount is the overhead + tensors in memory // this amount is the overhead + tensors in memory
// TODO: get this from the llama.cpp's graph calcluations instead of // TODO: get this from the llama.cpp's graph calculations instead of
// estimating it's 1/6 * kv_cache_size * num_gqa // estimating it's 1/6 * kv_cache_size * num_gqa
requiredAlloc := int64(ggml.NumGQA()) * requiredKv / 6 requiredAlloc := int64(ggml.NumGQA()) * requiredKv / 6