baalajimaestro/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

bugfix: fix compatibility bug with openai api on last token

Browse source
This commit is contained in:
Andrei Betlen 2023-07-08 00:06:11 -04:00
parent 4f2b5d0b53
commit d6e6aad927
2 changed files with 36 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
llama_cpp/llama.py
View file

@ -1060,6 +1060,20 @@ class Llama:
].decode("utf-8", errors="ignore"), ].decode("utf-8", errors="ignore"),
"index": 0, "index": 0,
"logprobs": logprobs_or_none, "logprobs": logprobs_or_none,
"finish_reason": None,
}
],
}
yield {
"id": completion_id,
"object": "text_completion",
"created": created,
"model": model_name,
"choices": [
{
"text": "",
"index": 0,
"logprobs": None,
"finish_reason": finish_reason, "finish_reason": finish_reason,
} }
], ],
@ -1078,9 +1092,21 @@ class Llama:
), ),
"index": 0, "index": 0,
"logprobs": logprobs_or_none, "logprobs": logprobs_or_none,
"finish_reason": finish_reason "finish_reason": None,
if returned_tokens == len(completion_tokens) }
else None, ],
}
yield {
"id": completion_id,
"object": "text_completion",
"created": created,
"model": model_name,
"choices": [
{
"text": "",
"index": 0,
"logprobs": None,
"finish_reason": finish_reason,
} }
], ],
} }
@ -1370,7 +1396,9 @@ class Llama:
"index": 0, "index": 0,
"delta": { "delta": {
"content": chunk["choices"][0]["text"], "content": chunk["choices"][0]["text"],
}, }
if chunk["choices"][0]["finish_reason"] is None
else {},
"finish_reason": chunk["choices"][0]["finish_reason"], "finish_reason": chunk["choices"][0]["finish_reason"],
} }
], ],

6
llama_cpp/llama_types.py
View file

@ -1,4 +1,4 @@
from typing import List, Optional, Dict from typing import Any, List, Optional, Dict, Union
from typing_extensions import TypedDict, NotRequired, Literal from typing_extensions import TypedDict, NotRequired, Literal
@ -77,6 +77,8 @@ class ChatCompletion(TypedDict):
choices: List[ChatCompletionChoice] choices: List[ChatCompletionChoice]
usage: CompletionUsage usage: CompletionUsage
class ChatCompletionChunkDeltaEmpty(TypedDict):
pass
class ChatCompletionChunkDelta(TypedDict): class ChatCompletionChunkDelta(TypedDict):
role: NotRequired[Literal["assistant"]] role: NotRequired[Literal["assistant"]]
@ -85,7 +87,7 @@ class ChatCompletionChunkDelta(TypedDict):
class ChatCompletionChunkChoice(TypedDict): class ChatCompletionChunkChoice(TypedDict):
index: int index: int
delta: ChatCompletionChunkDelta delta: Union[ChatCompletionChunkDelta, ChatCompletionChunkDeltaEmpty]
finish_reason: Optional[str] finish_reason: Optional[str]