baalajimaestro/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Move workaround to new sample method

Browse source
This commit is contained in:
Andrei Betlen 2023-04-02 00:06:34 -04:00
parent a4a1bbeaa9
commit 353e18a781
1 changed files with 5 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
llama_cpp/llama.py
View file

@ -169,6 +169,11 @@ class Llama:
The sampled token.
"""
assert self.ctx is not None
# Temporary workaround for https://github.com/ggerganov/llama.cpp/issues/684
if temp == 0.0:
temp = 1.0
top_p = 0.0
top_k = 1
return llama_cpp.llama_sample_top_p_top_k(
ctx=self.ctx,
last_n_tokens_data=(llama_cpp.llama_token * self.last_n_tokens_size)(
@ -209,11 +214,6 @@ class Llama:
Yields:
The generated tokens.
"""
# Temporary workaround for https://github.com/ggerganov/llama.cpp/issues/684
if temp == 0.0:
temp = 1.0
top_p = 0.0
top_k = 1
assert self.ctx is not None
self.reset()
while True: