Set n_batch to the default value of 8. I think this is leftover from when n_ctx was missing and n_batch was 2048.
This commit is contained in:
parent
1e90597983
commit
76a82babef
1 changed files with 1 additions and 1 deletions
|
@ -27,7 +27,7 @@ from sse_starlette.sse import EventSourceResponse
|
||||||
class Settings(BaseSettings):
|
class Settings(BaseSettings):
|
||||||
model: str
|
model: str
|
||||||
n_ctx: int = 2048
|
n_ctx: int = 2048
|
||||||
n_batch: int = 2048
|
n_batch: int = 8
|
||||||
n_threads: int = os.cpu_count() or 1
|
n_threads: int = os.cpu_count() or 1
|
||||||
f16_kv: bool = True
|
f16_kv: bool = True
|
||||||
use_mlock: bool = True
|
use_mlock: bool = True
|
||||||
|
|
Loading…
Reference in a new issue