Safer calculation of default n_threads

This commit is contained in:
Andrei Betlen 2023-04-06 21:22:19 -04:00
parent 9b7526895d
commit da539cc2ee

View file

@ -28,7 +28,7 @@ class Settings(BaseSettings):
model: str
n_ctx: int = 2048
n_batch: int = 8
n_threads: int = int(os.cpu_count() / 2) or 1
n_threads: int = ((os.cpu_count() or 2) // 2) or 1
f16_kv: bool = True
use_mlock: bool = False # This causes a silent failure on platforms that don't support mlock (e.g. Windows) took forever to figure out...
embedding: bool = True