90 lines
3.5 KiB
Diff
90 lines
3.5 KiB
Diff
|
From 6145068a6613c37bb43a7408b5496524bdcfc402 Mon Sep 17 00:00:00 2001
|
||
|
|
From: Bruce MacDonald <brucewmacdonald@gmail.com>
|
||
|
|
Date: Mon, 28 Aug 2023 18:08:53 -0400
|
||
|
|
Subject: [PATCH] 34B model support
|
||
|
|
|
||
|
|
---
|
||
|
|
llama.cpp | 10 ++++++++++
|
||
|
|
1 file changed, 10 insertions(+)
|
||
|
|
|
||
|
|
diff --git a/llama.cpp b/llama.cpp
|
||
|
|
index f2cbe76..62c5cdf 100644
|
||
|
|
--- a/llama.cpp
|
||
|
|
+++ b/llama.cpp
|
||
|
|
@@ -79,6 +79,7 @@ enum e_model {
|
||
|
|
MODEL_7B,
|
||
|
|
MODEL_13B,
|
||
|
|
MODEL_30B,
|
||
|
|
+ MODEL_34B,
|
||
|
|
MODEL_65B,
|
||
|
|
MODEL_70B,
|
||
|
|
};
|
||
|
|
@@ -122,6 +123,7 @@ static std::map<e_model, size_t> MEM_REQ_SCRATCH0(int n_ctx)
|
||
|
|
{ MODEL_7B, ((size_t) n_ctx / 16ull + 100ull) * MB },
|
||
|
|
{ MODEL_13B, ((size_t) n_ctx / 12ull + 120ull) * MB },
|
||
|
|
{ MODEL_30B, ((size_t) n_ctx / 9ull + 160ull) * MB },
|
||
|
|
+ { MODEL_34B, ((size_t) n_ctx / 9ull + 160ull) * MB },
|
||
|
|
{ MODEL_65B, ((size_t) n_ctx / 6ull + 256ull) * MB }, // guess
|
||
|
|
{ MODEL_70B, ((size_t) n_ctx / 7ull + 164ull) * MB },
|
||
|
|
};
|
||
|
|
@@ -135,6 +137,7 @@ static const std::map<e_model, size_t> & MEM_REQ_SCRATCH1()
|
||
|
|
{ MODEL_7B, 160ull * MB },
|
||
|
|
{ MODEL_13B, 192ull * MB },
|
||
|
|
{ MODEL_30B, 256ull * MB },
|
||
|
|
+ { MODEL_34B, 256ull * MB },
|
||
|
|
{ MODEL_65B, 384ull * MB }, // guess
|
||
|
|
{ MODEL_70B, 304ull * MB },
|
||
|
|
};
|
||
|
|
@@ -149,6 +152,7 @@ static const std::map<e_model, size_t> & MEM_REQ_EVAL()
|
||
|
|
{ MODEL_7B, 10ull * MB },
|
||
|
|
{ MODEL_13B, 12ull * MB },
|
||
|
|
{ MODEL_30B, 16ull * MB },
|
||
|
|
+ { MODEL_34B, 16ull * MB },
|
||
|
|
{ MODEL_65B, 24ull * MB }, // guess
|
||
|
|
{ MODEL_70B, 24ull * MB },
|
||
|
|
};
|
||
|
|
@@ -164,6 +168,7 @@ static const std::map<e_model, size_t> & VRAM_REQ_SCRATCH_BASE()
|
||
|
|
{ MODEL_7B, 512ull * kB },
|
||
|
|
{ MODEL_13B, 640ull * kB },
|
||
|
|
{ MODEL_30B, 768ull * kB },
|
||
|
|
+ { MODEL_34B, 768ull * kB },
|
||
|
|
{ MODEL_65B, 1280ull * kB },
|
||
|
|
{ MODEL_70B, 1280ull * kB },
|
||
|
|
};
|
||
|
|
@@ -179,6 +184,7 @@ static const std::map<e_model, size_t> & VRAM_REQ_SCRATCH_PER_CONTEXT()
|
||
|
|
{ MODEL_7B, 128ull },
|
||
|
|
{ MODEL_13B, 160ull },
|
||
|
|
{ MODEL_30B, 208ull },
|
||
|
|
+ { MODEL_34B, 208ull },
|
||
|
|
{ MODEL_65B, 256ull },
|
||
|
|
{ MODEL_70B, 256ull },
|
||
|
|
};
|
||
|
|
@@ -1027,6 +1033,7 @@ static const char * llama_model_type_name(e_model type) {
|
||
|
|
case MODEL_7B: return "7B";
|
||
|
|
case MODEL_13B: return "13B";
|
||
|
|
case MODEL_30B: return "30B";
|
||
|
|
+ case MODEL_34B: return "34B";
|
||
|
|
case MODEL_65B: return "65B";
|
||
|
|
case MODEL_70B: return "70B";
|
||
|
|
default: LLAMA_ASSERT(false);
|
||
|
|
@@ -1074,6 +1081,7 @@ static void llama_model_load_internal(
|
||
|
|
case 26: model.type = e_model::MODEL_3B; break;
|
||
|
|
case 32: model.type = e_model::MODEL_7B; break;
|
||
|
|
case 40: model.type = e_model::MODEL_13B; break;
|
||
|
|
+ case 48: model.type = e_model::MODEL_34B; break;
|
||
|
|
case 60: model.type = e_model::MODEL_30B; break;
|
||
|
|
case 80: model.type = e_model::MODEL_65B; break;
|
||
|
|
default:
|
||
|
|
@@ -1094,6 +1102,8 @@ static void llama_model_load_internal(
|
||
|
|
LLAMA_LOG_WARN("%s: warning: assuming 70B model based on GQA == %d\n", __func__, n_gqa);
|
||
|
|
model.type = e_model::MODEL_70B;
|
||
|
|
hparams.f_ffn_mult = 1.3f; // from the params.json of the 70B model
|
||
|
|
+ } else if (model.type == e_model::MODEL_34B && n_gqa == 8) {
|
||
|
|
+ hparams.f_ffn_mult = 1.0f; // from the params.json of the 34B model
|
||
|
|
}
|
||
|
|
|
||
|
|
hparams.rope_freq_base = rope_freq_base;
|
||
|
|
--
|
||
|
|
2.39.2 (Apple Git-143)
|
||
|
|
|