From e40fcb05754d0ec9c65359e245a436794cbfefdb Mon Sep 17 00:00:00 2001 From: Lucas Doyle Date: Sat, 29 Apr 2023 00:47:35 -0700 Subject: [PATCH] llama_cpp server: mark model as required `model` is ignored, but currently marked "optional"... on the one hand could mark "required" to make it explicit in case the server supports multiple llama's at the same time, but also could delete it since its ignored. decision: mark it required for the sake of openai api compatibility. I think out of all parameters, `model` is probably the most important one for people to keep using even if its ignored for now. --- llama_cpp/server/app.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/llama_cpp/server/app.py b/llama_cpp/server/app.py index 640dd3f..5d87e78 100644 --- a/llama_cpp/server/app.py +++ b/llama_cpp/server/app.py @@ -66,6 +66,10 @@ def get_llama(): with llama_lock: yield llama +model_field = Field( + description="The model to use for generating completions." +) + class CreateCompletionRequest(BaseModel): prompt: Union[str, List[str]] suffix: Optional[str] = Field(None) @@ -76,8 +80,9 @@ class CreateCompletionRequest(BaseModel): stop: Optional[List[str]] = [] stream: bool = False - # ignored or currently unsupported - model: Optional[str] = Field(None) + # ignored, but marked as required for the sake of compatibility with openai's api + model: str = model_field + n: Optional[int] = 1 logprobs: Optional[int] = Field(None) presence_penalty: Optional[float] = 0 @@ -133,7 +138,8 @@ def create_completion( class CreateEmbeddingRequest(BaseModel): - model: Optional[str] + # ignored, but marked as required for the sake of compatibility with openai's api + model: str = model_field input: str user: Optional[str] @@ -173,8 +179,9 @@ class CreateChatCompletionRequest(BaseModel): stop: Optional[List[str]] = [] max_tokens: int = 128 - # ignored or currently unsupported - model: Optional[str] = Field(None) + # ignored, but marked as required for the sake of compatibility with openai's api + model: str = model_field + n: Optional[int] = 1 presence_penalty: Optional[float] = 0 frequency_penalty: Optional[float] = 0