Set n_batch to the default value of 8. I think this is leftover from when n_ctx was missing and n_batch was 2048.

This commit is contained in:
MillionthOdin16 2023-04-05 17:44:53 -04:00
parent 1e90597983
commit 76a82babef

View file

@ -27,7 +27,7 @@ from sse_starlette.sse import EventSourceResponse
class Settings(BaseSettings): class Settings(BaseSettings):
model: str model: str
n_ctx: int = 2048 n_ctx: int = 2048
n_batch: int = 2048 n_batch: int = 8
n_threads: int = os.cpu_count() or 1 n_threads: int = os.cpu_count() or 1
f16_kv: bool = True f16_kv: bool = True
use_mlock: bool = True use_mlock: bool = True