fix(server): Propagate flash_attn to model load. (#1424)

2024-05-03 18:17:07 +02:00 · 2024-05-03 18:17:07 +02:00 · 2138561fab
commit 2138561fab
parent 2117122396
1 changed files with 1 additions and 0 deletions
--- a/llama_cpp/server/model.py
+++ b/llama_cpp/server/model.py
@ -242,6 +242,7 @@ class LlamaProxy:
            logits_all=settings.logits_all,
            embedding=settings.embedding,
            offload_kqv=settings.offload_kqv,
+            flash_attn=settings.flash_attn,
            # Sampling Params
            last_n_tokens_size=settings.last_n_tokens_size,
            # LoRA Params