Merge pull request #108 from eiery/main

Update n_batch default to 512 to match upstream llama.cpp
2023-04-24 13:48:09 -04:00 · 2023-04-24 13:48:09 -04:00 · f37456133a
commit f37456133a
parent 02cf881317 aa12d8a81f
1 changed files with 1 additions and 1 deletions
--- a/llama_cpp/llama.py
+++ b/llama_cpp/llama.py
@ -37,7 +37,7 @@ class Llama:
        use_mlock: bool = False,
        embedding: bool = False,
        n_threads: Optional[int] = None,
-        n_batch: int = 8,
+        n_batch: int = 512,
        last_n_tokens_size: int = 64,
        lora_base: Optional[str] = None,
        lora_path: Optional[str] = None,