baalajimaestro/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Safer calculation of default n_threads

Browse source
This commit is contained in:
Andrei Betlen 2023-04-06 21:22:19 -04:00
parent 9b7526895d
commit da539cc2ee
1 changed files with 1 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
llama_cpp/server/__main__.py
View file

@ -28,7 +28,7 @@ class Settings(BaseSettings):
model: str model: str
n_ctx: int = 2048 n_ctx: int = 2048
n_batch: int = 8 n_batch: int = 8
n_threads: int = int(os.cpu_count() / 2) or 1 n_threads: int = ((os.cpu_count() or 2) // 2) or 1
f16_kv: bool = True f16_kv: bool = True
use_mlock: bool = False # This causes a silent failure on platforms that don't support mlock (e.g. Windows) took forever to figure out... use_mlock: bool = False # This causes a silent failure on platforms that don't support mlock (e.g. Windows) took forever to figure out...
embedding: bool = True embedding: bool = True