baalajimaestro/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Fix streaming doesn't return finish reason (#798)

Browse source 
When streaming the yield that contains the finish can be skipped. This change ensures that yield isn't skipped.
This commit is contained in:
gmcgoldr 2023-10-19 02:55:56 -04:00 committed by GitHub
parent 28c2b884e2
commit 09a8406c83
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 14 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
llama_cpp/llama.py
View file

@ -1232,20 +1232,6 @@ class Llama:
} }
], ],
} }
yield {
"id": completion_id,
"object": "text_completion",
"created": created,
"model": model_name,
"choices": [
{
"text": "",
"index": 0,
"logprobs": None,
"finish_reason": finish_reason,
}
],
}
break break
returned_tokens += 1 returned_tokens += 1
yield { yield {
@ -1264,20 +1250,20 @@ class Llama:
} }
], ],
} }
yield { yield {
"id": completion_id, "id": completion_id,
"object": "text_completion", "object": "text_completion",
"created": created, "created": created,
"model": model_name, "model": model_name,
"choices": [ "choices": [
{ {
"text": "", "text": "",
"index": 0, "index": 0,
"logprobs": None, "logprobs": None,
"finish_reason": finish_reason, "finish_reason": finish_reason,
} }
], ],
} }
if self.cache: if self.cache:
if self.verbose: if self.verbose:
print("Llama._create_completion: cache save", file=sys.stderr) print("Llama._create_completion: cache save", file=sys.stderr)