Bugfix: only eval new tokens

2023-04-15 17:32:53 -04:00 · 2023-04-15 17:32:53 -04:00 · 89856ef00d
commit 89856ef00d
parent 887f3b73ac
1 changed files with 1 additions and 0 deletions
--- a/llama_cpp/llama.py
+++ b/llama_cpp/llama.py
@ -280,6 +280,7 @@ class Llama:
            if self.verbose:
                print("generate cache hit", file=sys.stderr)
            reset = False
+            tokens = tokens[len(self.tokens) :]
        ###
        if reset:
            self.reset()