From 2003d601594c89931d4741d20ffa730a14bbb4d9 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Thu, 8 Aug 2024 11:18:13 -0700 Subject: [PATCH] llama3.1 memory --- llm/ggml.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/llm/ggml.go b/llm/ggml.go index d7f2eef7..bde9c347 100644 --- a/llm/ggml.go +++ b/llm/ggml.go @@ -344,11 +344,13 @@ func (llm GGML) GraphSize(context, batch uint64) (partialOffload, fullOffload ui switch llm.KV().Architecture() { case "llama": - fullOffload = 4 * batch * (1 + 4*embedding + context*(1+heads)) + fullOffload = max( + 4*batch*(1+4*embedding+context*(1+heads)), + 4*batch*(embedding+vocab), + ) partialOffload = 4 * batch * embedding partialOffload += max( - // 4*batch*(4+6*embedding+context*(2*heads)+llm.KV().GQA()), 4*batch*(1+embedding+max(context, embedding))+embedding*embedding*9/16+4*context*(batch*heads+embeddingHeads*headsKV), 4*batch*(embedding+vocab)+embedding*vocab*105/128, )