baalajimaestro/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #485 from callMeMakerRen/main

Browse source 
expose RoPE param to server start
This commit is contained in:
Andrei 2023-07-18 12:30:21 -04:00 committed by GitHub
commit 071ac799d5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 4 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
llama_cpp/server/app.py
View file

@ -35,6 +35,8 @@ class Settings(BaseSettings):
default=None, default=None,
description="Split layers across multiple GPUs in proportion.", description="Split layers across multiple GPUs in proportion.",
) )
rope_freq_base: float = Field(default=10000, ge=1, description="RoPE base frequency")
rope_freq_scale: float = Field(default=1.0, description="RoPE frequency scaling factor")
seed: int = Field( seed: int = Field(
default=1337, description="Random seed. -1 for random." default=1337, description="Random seed. -1 for random."
) )
@ -118,6 +120,8 @@ def create_app(settings: Optional[Settings] = None):
model_path=settings.model, model_path=settings.model,
n_gpu_layers=settings.n_gpu_layers, n_gpu_layers=settings.n_gpu_layers,
tensor_split=settings.tensor_split, tensor_split=settings.tensor_split,
rope_freq_base=settings.rope_freq_base,
rope_freq_scale=settings.rope_freq_scale,
seed=settings.seed, seed=settings.seed,
f16_kv=settings.f16_kv, f16_kv=settings.f16_kv,
use_mlock=settings.use_mlock, use_mlock=settings.use_mlock,