baalajimaestro/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

expose RoPE param to server start

Browse source
This commit is contained in:
shutup 2023-07-18 16:34:36 +08:00
parent 6d8892fe64
commit 5ed8bf132f
1 changed files with 4 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
llama_cpp/server/app.py
View file

@ -35,6 +35,8 @@ class Settings(BaseSettings):
default=None, default=None,
description="Split layers across multiple GPUs in proportion.", description="Split layers across multiple GPUs in proportion.",
) )
rope_freq_base: float = Field(default=10000, ge=1, description="RoPE base frequency")
rope_freq_scale: float = Field(default=1.0, description="RoPE frequency scaling factor")
seed: int = Field( seed: int = Field(
default=1337, description="Random seed. -1 for random." default=1337, description="Random seed. -1 for random."
) )
@ -118,6 +120,8 @@ def create_app(settings: Optional[Settings] = None):
model_path=settings.model, model_path=settings.model,
n_gpu_layers=settings.n_gpu_layers, n_gpu_layers=settings.n_gpu_layers,
tensor_split=settings.tensor_split, tensor_split=settings.tensor_split,
rope_freq_base=settings.rope_freq_base,
rope_freq_scale=settings.rope_freq_scale,
seed=settings.seed, seed=settings.seed,
f16_kv=settings.f16_kv, f16_kv=settings.f16_kv,
use_mlock=settings.use_mlock, use_mlock=settings.use_mlock,