baalajimaestro/ollama
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #3712 from ollama/mxyng/mem

Browse source 
add stablelm graph calculation
This commit is contained in:
Michael Yang 2024-04-17 15:57:51 -07:00 committed by GitHub
commit 8645076a71
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 6 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
llm/ggml.go
View file

@ -381,6 +381,12 @@ func (llm GGML) GraphSize(context, batch uint64) (partialOffload, fullOffload ui
) )
partialOffload = 4*batch*(2*embedding+vocab) + embedding*vocab*105/128 partialOffload = 4*batch*(2*embedding+vocab) + embedding*vocab*105/128
case "stablelm":
fullOffload = 4 * batch * (context*(1+heads) + 3*embedding + 2)
partialOffload = max(
4*batch*(vocab+2*embedding),
fullOffload,
)
} }
return return

1
llm/server.go
View file

@ -112,7 +112,6 @@ func NewLlamaServer(model string, adapters, projectors []string, opts api.Option
var memoryLayerOutput uint64 var memoryLayerOutput uint64
for k, v := range layers { for k, v := range layers {
if !strings.HasPrefix(k, "blk.") { if !strings.HasPrefix(k, "blk.") {
slog.Info("aaa", "name", k, "size", format.HumanBytes2(v.size()))
memoryLayerOutput += v.size() memoryLayerOutput += v.size()
} }
} }