feat: Update llama.cpp
This commit is contained in:
Andrei Betlen
parent
d634efcdd9
commit
255e1b4495
3 changed files with 38 additions and 103 deletions
|
|
@ -128,9 +128,9 @@ class _LlamaModel:
|
|||
assert self.model is not None
|
||||
return llama_cpp.llama_token_get_score(self.model, token)
|
||||
|
||||
def token_get_type(self, token: int) -> int:
|
||||
def token_get_attr(self, token: int) -> int:
|
||||
assert self.model is not None
|
||||
return llama_cpp.llama_token_get_type(self.model, token)
|
||||
return llama_cpp.llama_token_get_attr(self.model, token)
|
||||
|
||||
# Special tokens
|
||||
|
||||
|
|
|
|||
|
|
@ -333,7 +333,7 @@ LLAMA_ROPE_TYPE_NEOX = 2
|
|||
LLAMA_ROPE_TYPE_GLM = 4
|
||||
|
||||
|
||||
# enum llama_token_type {
|
||||
# enum llama_token_type { //TODO: remove, required until per token attributes are available from GGUF file
|
||||
# LLAMA_TOKEN_TYPE_UNDEFINED = 0,
|
||||
# LLAMA_TOKEN_TYPE_NORMAL = 1,
|
||||
# LLAMA_TOKEN_TYPE_UNKNOWN = 2,
|
||||
|
|
@ -351,6 +351,32 @@ LLAMA_TOKEN_TYPE_UNUSED = 5
|
|||
LLAMA_TOKEN_TYPE_BYTE = 6
|
||||
|
||||
|
||||
# enum llama_token_attr {
|
||||
# LLAMA_TOKEN_ATTR_UNDEFINED = 0,
|
||||
# LLAMA_TOKEN_ATTR_UNKNOWN = 1 << 0,
|
||||
# LLAMA_TOKEN_ATTR_UNUSED = 1 << 1,
|
||||
# LLAMA_TOKEN_ATTR_NORMAL = 1 << 2,
|
||||
# LLAMA_TOKEN_ATTR_CONTROL = 1 << 3, // SPECIAL?
|
||||
# LLAMA_TOKEN_ATTR_USER_DEFINED = 1 << 4,
|
||||
# LLAMA_TOKEN_ATTR_BYTE = 1 << 5,
|
||||
# LLAMA_TOKEN_ATTR_NORMALIZED = 1 << 6,
|
||||
# LLAMA_TOKEN_ATTR_LSTRIP = 1 << 7,
|
||||
# LLAMA_TOKEN_ATTR_RSTRIP = 1 << 8,
|
||||
# LLAMA_TOKEN_ATTR_SINGLE_WORD = 1 << 9,
|
||||
# };
|
||||
LLAMA_TOKEN_ATTR_UNDEFINED = 0
|
||||
LLAMA_TOKEN_ATTR_UNKNOWN = 1 << 0
|
||||
LLAMA_TOKEN_ATTR_UNUSED = 1 << 1
|
||||
LLAMA_TOKEN_ATTR_NORMAL = 1 << 2
|
||||
LLAMA_TOKEN_ATTR_CONTROL = 1 << 3
|
||||
LLAMA_TOKEN_ATTR_USER_DEFINED = 1 << 4
|
||||
LLAMA_TOKEN_ATTR_BYTE = 1 << 5
|
||||
LLAMA_TOKEN_ATTR_NORMALIZED = 1 << 6
|
||||
LLAMA_TOKEN_ATTR_LSTRIP = 1 << 7
|
||||
LLAMA_TOKEN_ATTR_RSTRIP = 1 << 8
|
||||
LLAMA_TOKEN_ATTR_SINGLE_WORD = 1 << 9
|
||||
|
||||
|
||||
# // model file types
|
||||
# enum llama_ftype {
|
||||
# LLAMA_FTYPE_ALL_F32 = 0,
|
||||
|
|
@ -959,6 +985,9 @@ llama_grammar_p = ctypes.c_void_p
|
|||
# // modifies a preceding LLAMA_GRETYPE_CHAR or
|
||||
# // LLAMA_GRETYPE_CHAR_RNG_UPPER to add an alternate char to match ([ab], [a-zA])
|
||||
# LLAMA_GRETYPE_CHAR_ALT = 6,
|
||||
|
||||
# // any character (.)
|
||||
# LLAMA_GRETYPE_CHAR_ANY = 7,
|
||||
# };
|
||||
LLAMA_GRETYPE_END = 0
|
||||
LLAMA_GRETYPE_ALT = 1
|
||||
|
|
@ -967,6 +996,7 @@ LLAMA_GRETYPE_CHAR = 3
|
|||
LLAMA_GRETYPE_CHAR_NOT = 4
|
||||
LLAMA_GRETYPE_CHAR_RNG_UPPER = 5
|
||||
LLAMA_GRETYPE_CHAR_ALT = 6
|
||||
LLAMA_GRETYPE_CHAR_ANY = 7
|
||||
|
||||
|
||||
# typedef struct llama_grammar_element {
|
||||
|
|
@ -2438,11 +2468,11 @@ def llama_token_get_score(
|
|||
) -> float: ...
|
||||
|
||||
|
||||
# LLAMA_API enum llama_token_type llama_token_get_type(const struct llama_model * model, llama_token token);
|
||||
# LLAMA_API enum llama_token_attr llama_token_get_attr(const struct llama_model * model, llama_token token);
|
||||
@ctypes_function(
|
||||
"llama_token_get_type", [llama_model_p_ctypes, llama_token], ctypes.c_int
|
||||
"llama_token_get_attr", [llama_model_p_ctypes, llama_token], ctypes.c_int
|
||||
)
|
||||
def llama_token_get_type(
|
||||
def llama_token_get_attr(
|
||||
model: llama_model_p, token: Union[llama_token, int], /
|
||||
) -> int: ...
|
||||
|
||||
|
|
@ -3200,104 +3230,9 @@ def llama_grammar_accept_token(
|
|||
|
||||
|
||||
# //
|
||||
# // Beam search
|
||||
# // Model split
|
||||
# //
|
||||
|
||||
# struct llama_beam_view {
|
||||
# const llama_token * tokens;
|
||||
|
||||
|
||||
# size_t n_tokens;
|
||||
# float p; // Cumulative beam probability (renormalized relative to all beams)
|
||||
# bool eob; // Callback should set this to true when a beam is at end-of-beam.
|
||||
# };
|
||||
class llama_beam_view(ctypes.Structure):
|
||||
if TYPE_CHECKING:
|
||||
tokens: CtypesArray[llama_token]
|
||||
n_tokens: int
|
||||
p: float
|
||||
eob: bool
|
||||
|
||||
_fields_ = [
|
||||
("tokens", llama_token_p),
|
||||
("n_tokens", ctypes.c_size_t),
|
||||
("p", ctypes.c_float),
|
||||
("eob", ctypes.c_bool),
|
||||
]
|
||||
|
||||
|
||||
# // Passed to beam_search_callback function.
|
||||
# // Whenever 0 < common_prefix_length, this number of tokens should be copied from any of the beams
|
||||
# // (e.g. beams[0]) as they will be removed (shifted) from all beams in all subsequent callbacks.
|
||||
# // These pointers are valid only during the synchronous callback, so should not be saved.
|
||||
# struct llama_beams_state {
|
||||
# struct llama_beam_view * beam_views;
|
||||
# size_t n_beams; // Number of elements in beam_views[].
|
||||
# size_t common_prefix_length; // Current max length of prefix tokens shared by all beams.
|
||||
# bool last_call; // True iff this is the last callback invocation.
|
||||
# };
|
||||
class llama_beams_state(ctypes.Structure):
|
||||
if TYPE_CHECKING:
|
||||
beam_views: CtypesArray[llama_beam_view]
|
||||
n_beams: int
|
||||
common_prefix_length: int
|
||||
last_call: bool
|
||||
|
||||
_fields_ = [
|
||||
("beam_views", ctypes.POINTER(llama_beam_view)),
|
||||
("n_beams", ctypes.c_size_t),
|
||||
("common_prefix_length", ctypes.c_size_t),
|
||||
("last_call", ctypes.c_bool),
|
||||
]
|
||||
|
||||
|
||||
# // Type of pointer to the beam_search_callback function.
|
||||
# // void* callback_data is any custom data passed to llama_beam_search, that is subsequently
|
||||
# // passed back to beam_search_callback. This avoids having to use global variables in the callback.
|
||||
# typedef void (*llama_beam_search_callback_fn_t)(void * callback_data, struct llama_beams_state);
|
||||
llama_beam_search_callback_fn_t = ctypes.CFUNCTYPE(
|
||||
None, ctypes.c_void_p, llama_beams_state
|
||||
)
|
||||
|
||||
|
||||
# /// @details Deterministically returns entire sentence constructed by a beam search.
|
||||
# /// @param ctx Pointer to the llama_context.
|
||||
# /// @param callback Invoked for each iteration of the beam_search loop, passing in beams_state.
|
||||
# /// @param callback_data A pointer that is simply passed back to callback.
|
||||
# /// @param n_beams Number of beams to use.
|
||||
# /// @param n_past Number of tokens already evaluated.
|
||||
# /// @param n_predict Maximum number of tokens to predict. EOS may occur earlier.
|
||||
# /// @param n_threads Number of threads as passed to llama_eval().
|
||||
# LLAMA_API void llama_beam_search(
|
||||
# struct llama_context * ctx,
|
||||
# llama_beam_search_callback_fn_t callback,
|
||||
# void * callback_data,
|
||||
# size_t n_beams,
|
||||
# int32_t n_past,
|
||||
# int32_t n_predict);
|
||||
@ctypes_function(
|
||||
"llama_beam_search",
|
||||
[
|
||||
llama_context_p_ctypes,
|
||||
llama_beam_search_callback_fn_t,
|
||||
ctypes.c_void_p,
|
||||
ctypes.c_size_t,
|
||||
ctypes.c_int32,
|
||||
ctypes.c_int32,
|
||||
],
|
||||
None,
|
||||
)
|
||||
def llama_beam_search(
|
||||
ctx: llama_context_p,
|
||||
callback: CtypesFuncPointer,
|
||||
callback_data: ctypes.c_void_p,
|
||||
n_beams: Union[ctypes.c_size_t, int],
|
||||
n_past: Union[ctypes.c_int, int],
|
||||
n_predict: Union[ctypes.c_int, int],
|
||||
/,
|
||||
): ...
|
||||
|
||||
|
||||
# /// @details Build a split GGUF final path for this chunk.
|
||||
# /// llama_split_path(split_path, sizeof(split_path), "/models/ggml-model-q4_0", 2, 4) => split_path = "/models/ggml-model-q4_0-00002-of-00004.gguf"
|
||||
# // Returns the split_path length.
|
||||
|
|
|
|||
2
vendor/llama.cpp
vendored
2
vendor/llama.cpp
vendored
|
|
@ -1 +1 @@
|
|||
Subproject commit bde7cd3cd949c1a85d3a199498ac98e78039d46f
|
||||
Subproject commit ee459f40f65810a810151b24eba5b8bd174ceffe
|
||||
Loading…
Reference in a new issue