baalajimaestro/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

fix: UTF-8 handling with grammars (#1415)

Browse source 
Use Python's built-in UTF-8 handling to get code points
This commit is contained in:
Jonathan Soma 2024-04-30 14:33:23 -04:00 committed by GitHub
parent 945c62c567
commit 3226b3c5ef
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 5 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
llama_cpp/llama_grammar.py
View file

@ -556,17 +556,11 @@ def add_rule(
# } # }
def decode_utf8(src: const_char_p) -> Tuple[int, const_char_p]: def decode_utf8(src: const_char_p) -> Tuple[int, const_char_p]:
"""Decodes a UTF-8 character from the source string.""" """Decodes a UTF-8 character from the source string."""
lookup = (1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4) # Get the codepoint of the first character
first_byte = ord(src[0]) # type: int value = ord(src[0])
highbits = first_byte >> 4 # type: int # Move the pointer ahead one character
len = lookup[highbits] # type: int pos = src + 1
mask = (1 << (8 - len)) - 1 # type: int
value = first_byte & mask # type: int
end = src + len # type: const_char_p # may overrun!
pos = src + 1 # type: const_char_p
while pos < end and pos[0]:
value = (value << 6) + (ord(pos[0]) & 0x3F)
pos += 1
return value, pos return value, pos