baalajimaestro/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Added tokenizer

Browse source
This commit is contained in:
Andrei Betlen 2023-05-26 03:00:51 -04:00
parent 8fa2ef1959
commit 5be8354e11
1 changed files with 4 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
llama_cpp/llama.py
View file

@ -1416,8 +1416,10 @@ class LlamaTokenizer:
def __init__(self, llama: Llama): def __init__(self, llama: Llama):
self.llama = llama self.llama = llama
def encode(self, text: str) -> List[int]: def encode(self, text: str, add_bos: bool = True) -> List[int]:
return self.llama.tokenize(text.encode("utf-8", errors="ignore")) return self.llama.tokenize(
text.encode("utf-8", errors="ignore"), add_bos=add_bos
)
def decode(self, tokens: List[int]) -> str: def decode(self, tokens: List[int]) -> str:
return self.llama.detokenize(tokens).decode("utf-8", errors="ignore") return self.llama.detokenize(tokens).decode("utf-8", errors="ignore")