2023-04-03 01:50:13 +00:00
import sys
import os
2023-03-23 09:33:06 +00:00
import ctypes
2023-04-11 15:59:03 +00:00
from ctypes import (
2023-09-29 02:42:03 +00:00
c_bool ,
2023-04-11 15:59:03 +00:00
c_char_p ,
2023-09-29 02:42:03 +00:00
c_int ,
c_int8 ,
2023-07-05 05:00:14 +00:00
c_int32 ,
2023-09-29 02:42:03 +00:00
c_uint8 ,
2023-07-05 05:00:14 +00:00
c_uint32 ,
2023-12-22 19:52:20 +00:00
c_int64 ,
2023-09-29 02:42:03 +00:00
c_size_t ,
c_float ,
c_double ,
2023-04-11 15:59:03 +00:00
c_void_p ,
POINTER ,
2023-05-05 17:54:22 +00:00
_Pointer , # type: ignore
2023-04-11 15:59:03 +00:00
Structure ,
2023-12-22 19:52:20 +00:00
Union as CtypesUnion ,
2023-04-11 15:59:03 +00:00
Array ,
)
2023-03-23 09:33:06 +00:00
import pathlib
2023-06-26 12:50:38 +00:00
from typing import List , Union
2023-03-23 09:33:06 +00:00
2023-04-11 15:59:03 +00:00
2023-03-23 09:33:06 +00:00
# Load the library
2023-05-05 16:22:27 +00:00
def _load_shared_library ( lib_base_name : str ) :
2023-06-08 04:27:19 +00:00
# Construct the paths to the possible shared library names
2023-09-14 18:51:43 +00:00
_base_path = pathlib . Path ( os . path . abspath ( os . path . dirname ( __file__ ) ) )
2023-06-08 04:27:19 +00:00
# Searching for the library in the current directory under the name "libllama" (default name
# for llamacpp) and "llama" (default name for this repo)
_lib_paths : List [ pathlib . Path ] = [ ]
2023-04-03 01:50:13 +00:00
# Determine the file extension based on the platform
if sys . platform . startswith ( " linux " ) :
2023-06-08 04:27:19 +00:00
_lib_paths + = [
_base_path / f " lib { lib_base_name } .so " ,
]
2023-04-03 01:50:13 +00:00
elif sys . platform == " darwin " :
2023-06-08 04:27:19 +00:00
_lib_paths + = [
_base_path / f " lib { lib_base_name } .so " ,
_base_path / f " lib { lib_base_name } .dylib " ,
]
2023-04-03 01:50:13 +00:00
elif sys . platform == " win32 " :
2023-06-08 04:27:19 +00:00
_lib_paths + = [
_base_path / f " { lib_base_name } .dll " ,
2023-11-01 22:55:57 +00:00
_base_path / f " lib { lib_base_name } .dll " ,
2023-06-08 04:27:19 +00:00
]
2023-04-03 01:50:13 +00:00
else :
raise RuntimeError ( " Unsupported platform " )
2023-04-11 15:59:03 +00:00
if " LLAMA_CPP_LIB " in os . environ :
2023-04-10 15:27:17 +00:00
lib_base_name = os . environ [ " LLAMA_CPP_LIB " ]
2023-04-10 15:12:25 +00:00
_lib = pathlib . Path ( lib_base_name )
_base_path = _lib . parent . resolve ( )
_lib_paths = [ _lib . resolve ( ) ]
2023-04-10 15:00:35 +00:00
2023-05-19 15:59:33 +00:00
cdll_args = dict ( ) # type: ignore
2023-04-03 01:50:13 +00:00
# Add the library directory to the DLL search path on Windows (if needed)
if sys . platform == " win32 " and sys . version_info > = ( 3 , 8 ) :
os . add_dll_directory ( str ( _base_path ) )
2023-05-17 19:26:38 +00:00
if " CUDA_PATH " in os . environ :
2023-05-19 15:59:33 +00:00
os . add_dll_directory ( os . path . join ( os . environ [ " CUDA_PATH " ] , " bin " ) )
os . add_dll_directory ( os . path . join ( os . environ [ " CUDA_PATH " ] , " lib " ) )
2023-12-22 20:29:56 +00:00
if " HIP_PATH " in os . environ :
os . add_dll_directory ( os . path . join ( os . environ [ " HIP_PATH " ] , " bin " ) )
os . add_dll_directory ( os . path . join ( os . environ [ " HIP_PATH " ] , " lib " ) )
2023-09-16 18:57:49 +00:00
cdll_args [ " winmode " ] = ctypes . RTLD_GLOBAL
2023-04-03 01:50:13 +00:00
# Try to load the shared library, handling potential errors
for _lib_path in _lib_paths :
if _lib_path . exists ( ) :
try :
2023-05-15 13:15:01 +00:00
return ctypes . CDLL ( str ( _lib_path ) , * * cdll_args )
2023-04-03 01:50:13 +00:00
except Exception as e :
raise RuntimeError ( f " Failed to load shared library ' { _lib_path } ' : { e } " )
2023-04-11 15:59:03 +00:00
raise FileNotFoundError (
f " Shared library with base name ' { lib_base_name } ' not found "
)
2023-04-03 01:50:13 +00:00
# Specify the base name of the shared library to load
2023-04-03 17:06:50 +00:00
_lib_base_name = " llama "
2023-04-03 01:50:13 +00:00
# Load the library
2023-04-03 17:06:50 +00:00
_lib = _load_shared_library ( _lib_base_name )
2023-03-23 09:33:06 +00:00
2023-05-21 21:47:21 +00:00
# Misc
c_float_p = POINTER ( c_float )
c_uint8_p = POINTER ( c_uint8 )
c_size_t_p = POINTER ( c_size_t )
2024-01-19 02:21:49 +00:00
# from ggml-backend.h
# typedef bool (*ggml_backend_sched_eval_callback)(struct ggml_tensor * t, bool ask, void * user_data);
2024-01-26 16:45:48 +00:00
ggml_backend_sched_eval_callback = ctypes . CFUNCTYPE ( c_bool , c_void_p , c_bool , c_void_p )
2024-01-19 02:21:49 +00:00
2023-05-21 21:47:21 +00:00
# llama.h bindings
2024-01-04 03:04:04 +00:00
_lib . llama_max_devices . argtypes = [ ]
2024-01-31 15:41:42 +00:00
_lib . llama_max_devices . restype = ctypes . c_size_t
2024-01-04 03:04:04 +00:00
2023-12-22 20:19:28 +00:00
LLAMA_MAX_DEVICES = _lib . llama_max_devices ( )
2023-06-06 20:23:55 +00:00
2023-08-24 04:17:00 +00:00
# define LLAMA_DEFAULT_SEED 0xFFFFFFFF
2023-09-14 01:11:52 +00:00
LLAMA_DEFAULT_SEED = 0xFFFFFFFF
2023-08-24 04:17:00 +00:00
2023-09-29 02:42:03 +00:00
# define LLAMA_MAX_RNG_STATE (64*1024)
LLAMA_MAX_RNG_STATE = 64 * 1024
2024-01-08 19:51:29 +00:00
# define LLAMA_FILE_MAGIC_GGLA 0x67676c61u // 'ggla'
2023-12-16 23:57:43 +00:00
LLAMA_FILE_MAGIC_GGLA = 0x67676C61
2023-08-24 04:17:00 +00:00
# define LLAMA_FILE_MAGIC_GGSN 0x6767736eu // 'ggsn'
2023-09-14 01:11:52 +00:00
LLAMA_FILE_MAGIC_GGSN = 0x6767736E
2023-05-21 21:47:21 +00:00
2023-08-24 04:17:00 +00:00
# define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN
2023-05-21 21:47:21 +00:00
LLAMA_SESSION_MAGIC = LLAMA_FILE_MAGIC_GGSN
2024-01-14 03:37:49 +00:00
# define LLAMA_SESSION_VERSION 4
LLAMA_SESSION_VERSION = 4
2023-05-01 14:44:28 +00:00
2023-07-05 05:00:14 +00:00
2023-06-26 12:50:38 +00:00
# struct llama_model;
llama_model_p = c_void_p
2023-05-21 21:47:21 +00:00
# struct llama_context;
2023-03-24 18:58:42 +00:00
llama_context_p = c_void_p
2023-09-29 02:42:03 +00:00
# typedef int32_t llama_pos;
llama_pos = c_int32
# typedef int32_t llama_token;
llama_token = c_int32
2023-03-23 09:33:06 +00:00
llama_token_p = POINTER ( llama_token )
2023-09-29 02:42:03 +00:00
# typedef int32_t llama_seq_id;
llama_seq_id = c_int32
2023-03-23 09:33:06 +00:00
2023-08-24 04:17:00 +00:00
# enum llama_vocab_type {
# LLAMA_VOCAB_TYPE_SPM = 0, // SentencePiece
# LLAMA_VOCAB_TYPE_BPE = 1, // Byte Pair Encoding
2024-02-12 00:00:17 +00:00
# LLAMA_VOCAB_TYPE_WPM = 2, // WordPiece
2023-08-24 04:17:00 +00:00
# };
2023-09-14 01:11:52 +00:00
LLAMA_VOCAB_TYPE_SPM = 0
LLAMA_VOCAB_TYPE_BPE = 1
2024-02-12 00:00:17 +00:00
LLAMA_VOCAB_TYPE_WPM = 2
2023-08-24 04:17:00 +00:00
# enum llama_token_type {
# LLAMA_TOKEN_TYPE_UNDEFINED = 0,
# LLAMA_TOKEN_TYPE_NORMAL = 1,
# LLAMA_TOKEN_TYPE_UNKNOWN = 2,
# LLAMA_TOKEN_TYPE_CONTROL = 3,
# LLAMA_TOKEN_TYPE_USER_DEFINED = 4,
# LLAMA_TOKEN_TYPE_UNUSED = 5,
# LLAMA_TOKEN_TYPE_BYTE = 6,
# };
2023-09-14 01:11:52 +00:00
LLAMA_TOKEN_TYPE_UNDEFINED = 0
LLAMA_TOKEN_TYPE_NORMAL = 1
LLAMA_TOKEN_TYPE_UNKNOWN = 2
LLAMA_TOKEN_TYPE_CONTROL = 3
LLAMA_TOKEN_TYPE_USER_DEFINED = 4
LLAMA_TOKEN_TYPE_UNUSED = 5
LLAMA_TOKEN_TYPE_BYTE = 6
2023-08-24 04:17:00 +00:00
2023-09-29 02:42:03 +00:00
# // model file types
2023-08-24 04:17:00 +00:00
# enum llama_ftype {
# LLAMA_FTYPE_ALL_F32 = 0,
2023-09-29 02:42:03 +00:00
# LLAMA_FTYPE_MOSTLY_F16 = 1, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16
# // LLAMA_FTYPE_MOSTLY_Q4_2 = 5, // support has been removed
# // LLAMA_FTYPE_MOSTLY_Q4_3 = 6, // support has been removed
# LLAMA_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q5_1 = 9, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q2_K = 10, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q3_K_S = 11, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q3_K_M = 12, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q3_K_L = 13, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q4_K_S = 14, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q4_K_M = 15, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q5_K_S = 16, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q5_K_M = 17, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q6_K = 18, // except 1d tensors
2024-01-08 19:51:29 +00:00
# LLAMA_FTYPE_MOSTLY_IQ2_XXS = 19, // except 1d tensors
2024-01-12 03:51:12 +00:00
# LLAMA_FTYPE_MOSTLY_IQ2_XS = 20, // except 1d tensors
# LLAMA_FTYPE_MOSTLY_Q2_K_S = 21, // except 1d tensors
2024-01-22 16:25:11 +00:00
# LLAMA_FTYPE_MOSTLY_Q3_K_XS = 22, // except 1d tensors
2024-01-30 14:48:09 +00:00
# LLAMA_FTYPE_MOSTLY_IQ3_XXS = 23, // except 1d tensors
2023-09-29 02:42:03 +00:00
2023-08-24 04:17:00 +00:00
# LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file
# };
2023-09-14 01:11:52 +00:00
LLAMA_FTYPE_ALL_F32 = 0
LLAMA_FTYPE_MOSTLY_F16 = 1
LLAMA_FTYPE_MOSTLY_Q4_0 = 2
LLAMA_FTYPE_MOSTLY_Q4_1 = 3
LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4
LLAMA_FTYPE_MOSTLY_Q8_0 = 7
LLAMA_FTYPE_MOSTLY_Q5_0 = 8
LLAMA_FTYPE_MOSTLY_Q5_1 = 9
LLAMA_FTYPE_MOSTLY_Q2_K = 10
LLAMA_FTYPE_MOSTLY_Q3_K_S = 11
LLAMA_FTYPE_MOSTLY_Q3_K_M = 12
LLAMA_FTYPE_MOSTLY_Q3_K_L = 13
LLAMA_FTYPE_MOSTLY_Q4_K_S = 14
LLAMA_FTYPE_MOSTLY_Q4_K_M = 15
LLAMA_FTYPE_MOSTLY_Q5_K_S = 16
LLAMA_FTYPE_MOSTLY_Q5_K_M = 17
LLAMA_FTYPE_MOSTLY_Q6_K = 18
2024-01-12 03:51:12 +00:00
LLAMA_FTYPE_MOSTLY_IQ2_XXS = 19
LLAMA_FTYPE_MOSTLY_IQ2_XS = 20
LLAMA_FTYPE_MOSTLY_Q2_K_S = 21
2024-01-22 16:25:11 +00:00
LLAMA_FTYPE_MOSTLY_Q3_K_XS = 22
2024-01-30 14:48:09 +00:00
LLAMA_FTYPE_MOSTLY_IQ3_XXS = 23
2023-09-14 01:11:52 +00:00
LLAMA_FTYPE_GUESSED = 1024
2023-08-24 04:17:00 +00:00
2023-11-02 17:40:20 +00:00
# enum llama_rope_scaling_type {
# LLAMA_ROPE_SCALING_UNSPECIFIED = -1,
# LLAMA_ROPE_SCALING_NONE = 0,
# LLAMA_ROPE_SCALING_LINEAR = 1,
# LLAMA_ROPE_SCALING_YARN = 2,
# LLAMA_ROPE_SCALING_MAX_VALUE = LLAMA_ROPE_SCALING_YARN,
# };
LLAMA_ROPE_SCALING_UNSPECIFIED = - 1
LLAMA_ROPE_SCALING_NONE = 0
LLAMA_ROPE_SCALING_LINEAR = 1
LLAMA_ROPE_SCALING_YARN = 2
LLAMA_ROPE_SCALING_MAX_VALUE = LLAMA_ROPE_SCALING_YARN
2023-03-24 18:35:41 +00:00
2024-02-15 20:17:30 +00:00
# enum llama_pooling_type {
# LLAMA_POOLING_NONE = 0,
# LLAMA_POOLING_MEAN = 1,
# LLAMA_POOLING_CLS = 2,
# };
LLAMA_POOLING_NONE = 0
LLAMA_POOLING_MEAN = 1
LLAMA_POOLING_CLS = 2
2024-01-14 03:37:49 +00:00
# enum llama_split_mode {
# LLAMA_SPLIT_NONE = 0, // single GPU
# LLAMA_SPLIT_LAYER = 1, // split layers and KV across GPUs
# LLAMA_SPLIT_ROW = 2, // split rows across GPUs
# };
LLAMA_SPLIT_NONE = 0
LLAMA_SPLIT_LAYER = 1
LLAMA_SPLIT_ROW = 2
2023-11-20 19:11:33 +00:00
2024-01-15 15:12:10 +00:00
2023-05-21 21:47:21 +00:00
# typedef struct llama_token_data {
# llama_token id; // token id
# float logit; // log-odds of the token
# float p; // probability of the token
# } llama_token_data;
2023-03-23 09:33:06 +00:00
class llama_token_data ( Structure ) :
2023-11-28 00:03:02 +00:00
""" Used to store token data
2024-01-08 19:51:29 +00:00
2023-11-28 00:03:02 +00:00
Attributes :
id ( llama_token ) : token id
logit ( float ) : log - odds of the token
p ( float ) : probability of the token """
2024-01-08 19:51:29 +00:00
2023-03-23 09:33:06 +00:00
_fields_ = [
2023-05-21 21:47:21 +00:00
( " id " , llama_token ) ,
( " logit " , c_float ) ,
( " p " , c_float ) ,
2023-03-23 09:33:06 +00:00
]
2023-03-24 18:35:41 +00:00
2023-03-23 09:33:06 +00:00
llama_token_data_p = POINTER ( llama_token_data )
2023-05-01 14:44:28 +00:00
2023-05-21 21:47:21 +00:00
# typedef struct llama_token_data_array {
# llama_token_data * data;
# size_t size;
# bool sorted;
# } llama_token_data_array;
2023-05-01 14:44:28 +00:00
class llama_token_data_array ( Structure ) :
2023-11-28 00:03:02 +00:00
""" Used to sample tokens given logits
2024-01-08 19:51:29 +00:00
2023-11-28 00:03:02 +00:00
Attributes :
data ( ctypes . Array [ llama_token_data ] ) : token data
size ( int ) : size of the array
sorted ( bool ) : whether the array is sorted """
2024-01-08 19:51:29 +00:00
2023-05-01 14:44:28 +00:00
_fields_ = [
( " data " , llama_token_data_p ) ,
( " size " , c_size_t ) ,
( " sorted " , c_bool ) ,
]
llama_token_data_array_p = POINTER ( llama_token_data_array )
2023-12-22 19:10:34 +00:00
# typedef bool (*llama_progress_callback)(float progress, void *ctx);
llama_progress_callback = ctypes . CFUNCTYPE ( c_bool , c_float , c_void_p )
2023-03-24 18:35:41 +00:00
2023-08-24 04:17:00 +00:00
2023-09-29 02:42:03 +00:00
# // Input data for llama_decode
# // A llama_batch object can contain input about one or many sequences
# // The provided arrays (i.e. token, embd, pos, etc.) must have size of n_tokens
# //
# // - token : the token ids of the input (used when embd is NULL)
# // - embd : token embeddings (i.e. float vector of size n_embd) (used when token is NULL)
# // - pos : the positions of the respective token in the sequence
# // - seq_id : the sequence to which the respective token belongs
# // - logits : if zero, the logits for the respective token will not be output
# //
# typedef struct llama_batch {
# int32_t n_tokens;
2023-10-19 06:55:08 +00:00
# llama_token * token;
# float * embd;
# llama_pos * pos;
2023-11-01 01:29:35 +00:00
# int32_t * n_seq_id;
2023-10-19 06:55:08 +00:00
# llama_seq_id ** seq_id;
# int8_t * logits;
2023-09-29 02:42:03 +00:00
# // NOTE: helpers for smooth API transition - can be deprecated in the future
# // for future-proof code, use the above fields instead and ignore everything below
# //
# // pos[i] = all_pos_0 + i*all_pos_1
# //
# llama_pos all_pos_0; // used if pos == NULL
# llama_pos all_pos_1; // used if pos == NULL
# llama_seq_id all_seq_id; // used if seq_id == NULL
# } llama_batch;
class llama_batch ( Structure ) :
2023-11-23 05:26:26 +00:00
""" Input data for llama_decode
2023-11-23 21:26:00 +00:00
2023-11-23 05:26:26 +00:00
A llama_batch object can contain input about one or many sequences
2023-11-23 21:26:00 +00:00
2023-11-23 05:26:26 +00:00
The provided arrays ( i . e . token , embd , pos , etc . ) must have size of n_tokens
2023-11-23 21:26:00 +00:00
2023-11-23 05:26:26 +00:00
Attributes :
token ( ctypes . Array [ llama_token ] ) : the token ids of the input ( used when embd is NULL )
embd ( ctypes . Array [ ctypes . c_float ] ) : token embeddings ( i . e . float vector of size n_embd ) ( used when token is NULL )
pos ( ctypes . Array [ ctypes . Array [ llama_pos ] ] ) : the positions of the respective token in the sequence
2024-01-08 19:51:29 +00:00
seq_id ( ctypes . Array [ ctypes . Array [ llama_seq_id ] ] ) : the sequence to which the respective token belongs
"""
2023-11-23 05:26:26 +00:00
2023-09-29 02:42:03 +00:00
_fields_ = [
( " n_tokens " , c_int32 ) ,
( " token " , POINTER ( llama_token ) ) ,
( " embd " , c_float_p ) ,
( " pos " , POINTER ( llama_pos ) ) ,
2023-11-01 01:29:35 +00:00
( " n_seq_id " , POINTER ( c_int32 ) ) ,
2023-10-19 06:55:08 +00:00
( " seq_id " , POINTER ( POINTER ( llama_seq_id ) ) ) ,
2023-09-29 02:42:03 +00:00
( " logits " , POINTER ( c_int8 ) ) ,
( " all_pos_0 " , llama_pos ) ,
( " all_pos_1 " , llama_pos ) ,
( " all_seq_id " , llama_seq_id ) ,
]
2023-07-15 19:11:01 +00:00
2024-01-08 19:51:29 +00:00
2023-12-11 15:21:35 +00:00
# enum llama_model_kv_override_type {
# LLAMA_KV_OVERRIDE_INT,
# LLAMA_KV_OVERRIDE_FLOAT,
# LLAMA_KV_OVERRIDE_BOOL,
# };
2023-12-22 19:52:20 +00:00
LLAMA_KV_OVERRIDE_INT = 0
LLAMA_KV_OVERRIDE_FLOAT = 1
LLAMA_KV_OVERRIDE_BOOL = 2
2023-12-11 15:21:35 +00:00
2024-01-08 19:51:29 +00:00
2023-12-11 15:21:35 +00:00
# struct llama_model_kv_override {
# char key[128];
# enum llama_model_kv_override_type tag;
# union {
# int64_t int_value;
# double float_value;
# bool bool_value;
# };
# };
2023-12-22 19:52:20 +00:00
class llama_model_kv_override_value ( CtypesUnion ) :
2023-12-11 15:21:35 +00:00
_fields_ = [
2023-12-22 19:52:20 +00:00
( " int_value " , c_int64 ) ,
2023-12-11 15:21:35 +00:00
( " float_value " , c_double ) ,
( " bool_value " , c_bool ) ,
]
2023-09-29 02:42:03 +00:00
2024-01-08 19:51:29 +00:00
2023-12-22 19:52:20 +00:00
class llama_model_kv_override ( Structure ) :
_fields_ = [
( " key " , ctypes . c_char * 128 ) ,
( " tag " , c_int ) ,
( " value " , llama_model_kv_override_value ) ,
]
2024-01-08 19:51:29 +00:00
2023-09-29 02:42:03 +00:00
# struct llama_model_params {
# int32_t n_gpu_layers; // number of layers to store in VRAM
2024-01-14 03:37:49 +00:00
# enum llama_split_mode split_mode; // how to split the model across multiple GPUs
# // main_gpu interpretation depends on split_mode:
# // LLAMA_SPLIT_NONE: the GPU that is used for the entire model
# // LLAMA_SPLIT_ROW: the GPU that is used for small tensors and intermediate results
# // LLAMA_SPLIT_LAYER: ignored
# int32_t main_gpu;
2024-01-31 15:41:42 +00:00
# // proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices()
2024-01-14 03:37:49 +00:00
# const float * tensor_split;
2023-07-15 19:11:01 +00:00
2023-12-22 19:10:34 +00:00
# // Called with a progress value between 0.0 and 1.0. Pass NULL to disable.
# // If the provided progress_callback returns true, model loading continues.
# // If it returns false, model loading is immediately aborted.
2023-06-20 15:25:10 +00:00
# llama_progress_callback progress_callback;
2024-01-14 03:37:49 +00:00
2023-06-20 15:25:10 +00:00
# // context pointer passed to the progress callback
# void * progress_callback_user_data;
2023-05-21 21:47:21 +00:00
2023-12-11 15:21:35 +00:00
# // override key-value pairs of the model meta data
# const struct llama_model_kv_override * kv_overrides;
2023-07-20 22:51:53 +00:00
2024-01-15 15:12:10 +00:00
2023-06-20 15:25:10 +00:00
# // Keep the booleans together to avoid misalignment during copy-by-value.
2023-05-21 21:47:21 +00:00
# bool vocab_only; // only load the vocabulary, no weights
# bool use_mmap; // use mmap if possible
# bool use_mlock; // force system to keep model in RAM
# };
2023-09-29 02:42:03 +00:00
class llama_model_params ( Structure ) :
2023-11-28 00:03:02 +00:00
""" Parameters for llama_model
2024-01-08 19:51:29 +00:00
2023-11-28 00:03:02 +00:00
Attributes :
n_gpu_layers ( int ) : number of layers to store in VRAM
2024-01-14 03:37:49 +00:00
split_mode ( int ) : how to split the model across multiple GPUs
main_gpu ( int ) : the GPU that is used for the entire model . main_gpu interpretation depends on split_mode : LLAMA_SPLIT_NONE : the GPU that is used for the entire model LLAMA_SPLIT_ROW : the GPU that is used for small tensors and intermediate results LLAMA_SPLIT_LAYER : ignored
2024-01-31 15:41:42 +00:00
tensor_split ( ctypes . Array [ ctypes . c_float ] ) : proportion of the model ( layers or rows ) to offload to each GPU , size : llama_max_devices ( )
2023-12-22 19:10:34 +00:00
progress_callback ( llama_progress_callback ) : called with a progress value between 0.0 and 1.0 . Pass NULL to disable . If the provided progress_callback returns true , model loading continues . If it returns false , model loading is immediately aborted .
2023-11-28 00:03:02 +00:00
progress_callback_user_data ( ctypes . c_void_p ) : context pointer passed to the progress callback
2023-12-11 15:21:35 +00:00
kv_overrides ( ctypes . Array [ llama_model_kv_override ] ) : override key - value pairs of the model meta data
2023-11-28 00:03:02 +00:00
vocab_only ( bool ) : only load the vocabulary , no weights
use_mmap ( bool ) : use mmap if possible
use_mlock ( bool ) : force system to keep model in RAM """
2024-01-08 19:51:29 +00:00
2023-03-23 09:33:06 +00:00
_fields_ = [
2023-07-05 05:00:14 +00:00
( " n_gpu_layers " , c_int32 ) ,
2024-01-14 03:37:49 +00:00
( " split_mode " , c_int ) ,
2023-07-05 05:00:14 +00:00
( " main_gpu " , c_int32 ) ,
2023-08-24 04:17:00 +00:00
( " tensor_split " , c_float_p ) ,
2023-06-20 15:25:10 +00:00
( " progress_callback " , llama_progress_callback ) ,
( " progress_callback_user_data " , c_void_p ) ,
2023-12-11 15:21:35 +00:00
( " kv_overrides " , POINTER ( llama_model_kv_override ) ) ,
2023-05-21 21:47:21 +00:00
( " vocab_only " , c_bool ) ,
( " use_mmap " , c_bool ) ,
( " use_mlock " , c_bool ) ,
2023-03-23 09:33:06 +00:00
]
2023-03-24 18:35:41 +00:00
2023-09-29 02:42:03 +00:00
# struct llama_context_params {
2023-11-03 15:34:50 +00:00
# uint32_t seed; // RNG seed, -1 for random
# uint32_t n_ctx; // text context, 0 = from model
# uint32_t n_batch; // prompt processing maximum batch size
# uint32_t n_threads; // number of threads to use for generation
# uint32_t n_threads_batch; // number of threads to use for batch processing
2024-02-05 18:26:50 +00:00
# int32_t rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type`
2023-09-29 02:42:03 +00:00
# // ref: https://github.com/ggerganov/llama.cpp/pull/2054
2023-11-02 17:40:20 +00:00
# float rope_freq_base; // RoPE base frequency, 0 = from model
# float rope_freq_scale; // RoPE frequency scaling factor, 0 = from model
2023-11-26 20:38:22 +00:00
# float yarn_ext_factor; // YaRN extrapolation mix factor, negative = from model
2023-11-02 17:40:20 +00:00
# float yarn_attn_factor; // YaRN magnitude scaling factor
# float yarn_beta_fast; // YaRN low correction dim
# float yarn_beta_slow; // YaRN high correction dim
# uint32_t yarn_orig_ctx; // YaRN original context size
2023-09-29 02:42:03 +00:00
2024-01-19 02:21:49 +00:00
# ggml_backend_sched_eval_callback cb_eval;
# void * cb_eval_user_data;
2023-12-11 15:21:35 +00:00
# enum ggml_type type_k; // data type for K cache
# enum ggml_type type_v; // data type for V cache
2023-09-29 02:42:03 +00:00
2024-01-08 19:51:29 +00:00
2023-09-29 02:42:03 +00:00
# // Keep the booleans together to avoid misalignment during copy-by-value.
2023-12-11 15:21:35 +00:00
# bool mul_mat_q; // if true, use experimental mul_mat_q kernels (DEPRECATED - always true)
2023-12-14 02:43:16 +00:00
# bool logits_all; // the llama_eval() call computes all logits, not just the last one (DEPRECATED - set llama_batch.logits instead)
2023-12-11 15:21:35 +00:00
# bool embedding; // embedding mode only
# bool offload_kqv; // whether to offload the KQV ops (including the KV cache) to GPU
2024-02-13 17:24:00 +00:00
# bool do_pooling; // whether to pool (sum) embedding results by sequence id (ignored if no pooling layer)
2023-09-29 02:42:03 +00:00
# };
class llama_context_params ( Structure ) :
2023-11-28 00:03:02 +00:00
""" Parameters for llama_context
2024-01-08 19:51:29 +00:00
2023-11-28 00:03:02 +00:00
Attributes :
seed ( int ) : RNG seed , - 1 for random
n_ctx ( int ) : text context , 0 = from model
n_batch ( int ) : prompt processing maximum batch size
n_threads ( int ) : number of threads to use for generation
n_threads_batch ( int ) : number of threads to use for batch processing
rope_scaling_type ( int ) : RoPE scaling type , from ` enum llama_rope_scaling_type `
rope_freq_base ( float ) : RoPE base frequency , 0 = from model
rope_freq_scale ( float ) : RoPE frequency scaling factor , 0 = from model
yarn_ext_factor ( float ) : YaRN extrapolation mix factor , negative = from model
yarn_attn_factor ( float ) : YaRN magnitude scaling factor
yarn_beta_fast ( float ) : YaRN low correction dim
yarn_beta_slow ( float ) : YaRN high correction dim
yarn_orig_ctx ( int ) : YaRN original context size
2024-01-19 02:21:49 +00:00
cb_eval ( ggml_backend_sched_eval_callback ) : callback for scheduling eval
cb_eval_user_data ( ctypes . c_void_p ) : user data for cb_eval
2023-12-11 15:21:35 +00:00
type_k ( int ) : data type for K cache
type_v ( int ) : data type for V cache
2023-11-28 00:03:02 +00:00
mul_mat_q ( bool ) : if true , use experimental mul_mat_q kernels ( DEPRECATED - always true )
2023-12-14 02:43:16 +00:00
logits_all ( bool ) : the llama_eval ( ) call computes all logits , not just the last one ( DEPRECATED - set llama_batch . logits instead )
2023-12-18 19:27:11 +00:00
embedding ( bool ) : embedding mode only
2024-01-08 19:51:29 +00:00
offload_kqv ( bool ) : whether to offload the KQV ops ( including the KV cache ) to GPU
2024-02-13 17:24:00 +00:00
do_pooling ( bool ) : whether to pool ( sum ) embedding results by sequence id ( ignored if no pooling layer )
2024-01-08 19:51:29 +00:00
"""
2023-09-29 02:42:03 +00:00
_fields_ = [
( " seed " , c_uint32 ) ,
( " n_ctx " , c_uint32 ) ,
( " n_batch " , c_uint32 ) ,
( " n_threads " , c_uint32 ) ,
( " n_threads_batch " , c_uint32 ) ,
2024-02-05 18:26:50 +00:00
( " rope_scaling_type " , c_int32 ) ,
2023-09-29 02:42:03 +00:00
( " rope_freq_base " , c_float ) ,
( " rope_freq_scale " , c_float ) ,
2023-11-02 17:40:20 +00:00
( " yarn_ext_factor " , c_float ) ,
( " yarn_attn_factor " , c_float ) ,
( " yarn_beta_fast " , c_float ) ,
( " yarn_beta_slow " , c_float ) ,
( " yarn_orig_ctx " , c_uint32 ) ,
2024-01-19 02:21:49 +00:00
( " cb_eval " , ggml_backend_sched_eval_callback ) ,
( " cb_eval_user_data " , c_void_p ) ,
2023-12-11 15:21:35 +00:00
( " type_k " , c_int ) ,
( " type_v " , c_int ) ,
2023-09-29 02:42:03 +00:00
( " mul_mat_q " , c_bool ) ,
( " logits_all " , c_bool ) ,
( " embedding " , c_bool ) ,
2023-12-18 19:27:11 +00:00
( " offload_kqv " , c_bool ) ,
2024-02-13 17:24:00 +00:00
( " do_pooling " , c_bool ) ,
2023-09-29 02:42:03 +00:00
]
2023-03-23 09:33:06 +00:00
2023-08-24 04:17:00 +00:00
# // Signature for logging events
# // Note that text includes the new line character at the end for most events.
# // If your logging mechanism cannot handle that, check if the last character is '\n' and strip it
# // if it exists.
# // It might not exist for progress report where '.' is output repeatedly.
# typedef void (*llama_log_callback)(enum llama_log_level level, const char * text, void * user_data);
llama_log_callback = ctypes . CFUNCTYPE ( None , c_int , c_char_p , c_void_p )
2023-11-23 05:26:26 +00:00
""" Signature for logging events
Note that text includes the new line character at the end for most events .
If your logging mechanism cannot handle that , check if the last character is ' \n ' and strip it
if it exists .
It might not exist for progress report where ' . ' is output repeatedly . """
2023-03-23 09:33:06 +00:00
2023-03-24 18:35:41 +00:00
2023-06-10 16:17:38 +00:00
# // model quantization parameters
# typedef struct llama_model_quantize_params {
2024-01-04 03:04:04 +00:00
# int32_t nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency()
2023-08-24 04:17:00 +00:00
# enum llama_ftype ftype; // quantize to this llama_ftype
2023-06-10 16:17:38 +00:00
# bool allow_requantize; // allow quantizing non-f32/f16 tensors
# bool quantize_output_tensor; // quantize output.weight
2023-09-01 18:26:13 +00:00
# bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored
2023-11-01 01:29:35 +00:00
# bool pure; // disable k-quant mixtures and quantize all tensors to the same type
2024-01-14 13:17:22 +00:00
# void * imatrix; // pointer to importance matrix data
2023-06-10 16:17:38 +00:00
# } llama_model_quantize_params;
class llama_model_quantize_params ( Structure ) :
2023-11-28 00:03:02 +00:00
""" Parameters for llama_model_quantize
2024-01-08 19:51:29 +00:00
2023-11-28 00:03:02 +00:00
Attributes :
nthread ( int ) : number of threads to use for quantizing , if < = 0 will use std : : thread : : hardware_concurrency ( )
ftype ( int ) : quantize to this llama_ftype
allow_requantize ( bool ) : allow quantizing non - f32 / f16 tensors
quantize_output_tensor ( bool ) : quantize output . weight
only_copy ( bool ) : only copy tensors - ftype , allow_requantize and quantize_output_tensor are ignored
2024-01-08 19:51:29 +00:00
pure ( bool ) : disable k - quant mixtures and quantize all tensors to the same type
2024-01-14 13:17:22 +00:00
imatrix ( ctypes . c_void_p ) : pointer to importance matrix data
2024-01-08 19:51:29 +00:00
"""
2023-06-10 16:17:38 +00:00
_fields_ = [
2024-01-04 03:04:04 +00:00
( " nthread " , c_int32 ) ,
2023-06-10 16:17:38 +00:00
( " ftype " , c_int ) ,
( " allow_requantize " , c_bool ) ,
( " quantize_output_tensor " , c_bool ) ,
2023-09-01 18:26:13 +00:00
( " only_copy " , c_bool ) ,
2024-01-14 13:17:22 +00:00
( " pure " , c_bool ) ,
( " imatrix " , c_void_p ) ,
2023-06-10 16:17:38 +00:00
]
2023-07-24 17:04:34 +00:00
# // grammar types
# struct llama_grammar;
llama_grammar_p = c_void_p
# // grammar element type
# enum llama_gretype {
# // end of rule definition
# LLAMA_GRETYPE_END = 0,
# // start of alternate definition for rule
# LLAMA_GRETYPE_ALT = 1,
# // non-terminal element: reference to rule
# LLAMA_GRETYPE_RULE_REF = 2,
# // terminal element: character (code point)
# LLAMA_GRETYPE_CHAR = 3,
# // inverse char(s) ([^a], [^a-b] [^abc])
# LLAMA_GRETYPE_CHAR_NOT = 4,
# // modifies a preceding LLAMA_GRETYPE_CHAR or LLAMA_GRETYPE_CHAR_ALT to
# // be an inclusive range ([a-z])
# LLAMA_GRETYPE_CHAR_RNG_UPPER = 5,
# // modifies a preceding LLAMA_GRETYPE_CHAR or
# // LLAMA_GRETYPE_CHAR_RNG_UPPER to add an alternate char to match ([ab], [a-zA])
# LLAMA_GRETYPE_CHAR_ALT = 6,
# };
2023-07-24 19:42:07 +00:00
LLAMA_GRETYPE_END = 0
LLAMA_GRETYPE_ALT = 1
LLAMA_GRETYPE_RULE_REF = 2
LLAMA_GRETYPE_CHAR = 3
LLAMA_GRETYPE_CHAR_NOT = 4
LLAMA_GRETYPE_CHAR_RNG_UPPER = 5
LLAMA_GRETYPE_CHAR_ALT = 6
2023-07-24 17:04:34 +00:00
# typedef struct llama_grammar_element {
# enum llama_gretype type;
# uint32_t value; // Unicode code point or rule ID
# } llama_grammar_element;
class llama_grammar_element ( Structure ) :
_fields_ = [
( " type " , c_int ) ,
( " value " , c_uint32 ) ,
]
llama_grammar_element_p = POINTER ( llama_grammar_element )
2023-07-06 21:57:56 +00:00
# // performance timing information
# struct llama_timings {
# double t_start_ms;
# double t_end_ms;
# double t_load_ms;
# double t_sample_ms;
# double t_p_eval_ms;
# double t_eval_ms;
# int32_t n_sample;
# int32_t n_p_eval;
# int32_t n_eval;
# };
class llama_timings ( Structure ) :
_fields_ = [
( " t_start_ms " , c_double ) ,
( " t_end_ms " , c_double ) ,
( " t_load_ms " , c_double ) ,
( " t_sample_ms " , c_double ) ,
( " t_p_eval_ms " , c_double ) ,
( " t_eval_ms " , c_double ) ,
( " n_sample " , c_int32 ) ,
( " n_p_eval " , c_int32 ) ,
( " n_eval " , c_int32 ) ,
]
2023-09-29 02:42:03 +00:00
# // Helpers for getting default parameters
# LLAMA_API struct llama_model_params llama_model_default_params(void);
def llama_model_default_params ( ) - > llama_model_params :
2023-11-28 00:03:02 +00:00
""" Get default parameters for llama_model """
2023-09-29 02:42:03 +00:00
return _lib . llama_model_default_params ( )
_lib . llama_model_default_params . argtypes = [ ]
_lib . llama_model_default_params . restype = llama_model_params
2023-08-24 04:17:00 +00:00
# LLAMA_API struct llama_context_params llama_context_default_params(void);
2023-03-23 09:33:06 +00:00
def llama_context_default_params ( ) - > llama_context_params :
2023-11-28 00:03:02 +00:00
""" Get default parameters for llama_context """
2023-03-29 01:10:23 +00:00
return _lib . llama_context_default_params ( )
2023-03-23 09:33:06 +00:00
2023-03-24 18:35:41 +00:00
2023-03-24 22:43:29 +00:00
_lib . llama_context_default_params . argtypes = [ ]
_lib . llama_context_default_params . restype = llama_context_params
2023-03-24 18:58:42 +00:00
2023-04-11 15:59:03 +00:00
2023-08-24 04:17:00 +00:00
# LLAMA_API struct llama_model_quantize_params llama_model_quantize_default_params(void);
2023-06-10 16:17:38 +00:00
def llama_model_quantize_default_params ( ) - > llama_model_quantize_params :
2023-11-28 00:03:02 +00:00
""" Get default parameters for llama_model_quantize """
2023-06-10 16:17:38 +00:00
return _lib . llama_model_quantize_default_params ( )
_lib . llama_model_quantize_default_params . argtypes = [ ]
_lib . llama_model_quantize_default_params . restype = llama_model_quantize_params
2023-05-21 21:47:21 +00:00
# // Initialize the llama + ggml backend
2023-06-29 05:08:15 +00:00
# // If numa is true, use NUMA optimizations
2023-05-21 21:47:21 +00:00
# // Call once at the start of the program
2023-07-15 19:11:01 +00:00
# LLAMA_API void llama_backend_init(bool numa);
2024-02-17 05:37:51 +00:00
# LLAMA_API void llama_backend_init(void);
def llama_backend_init ( ) :
2023-11-23 05:26:26 +00:00
""" Initialize the llama + ggml backend
If numa is true , use NUMA optimizations
Call once at the start of the program """
2024-02-17 05:37:51 +00:00
return _lib . llama_backend_init ( )
2023-07-15 19:11:01 +00:00
2024-02-17 05:37:51 +00:00
_lib . llama_backend_init . argtypes = [ ]
2023-07-15 19:11:01 +00:00
_lib . llama_backend_init . restype = None
2023-05-21 21:47:21 +00:00
2024-02-17 05:37:51 +00:00
# // numa strategies
# enum ggml_numa_strategy {
# GGML_NUMA_STRATEGY_DISABLED = 0,
# GGML_NUMA_STRATEGY_DISTRIBUTE = 1,
# GGML_NUMA_STRATEGY_ISOLATE = 2,
# GGML_NUMA_STRATEGY_NUMACTL = 3,
# GGML_NUMA_STRATEGY_MIRROR = 4,
# GGML_NUMA_STRATEGY_COUNT
# };
GGML_NUMA_STRATEGY_DISABLED = 0
GGML_NUMA_STRATEGY_DISTRIBUTE = 1
GGML_NUMA_STRATEGY_ISOLATE = 2
GGML_NUMA_STRATEGY_NUMACTL = 3
GGML_NUMA_STRATEGY_MIRROR = 4
GGML_NUMA_STRATEGY_COUNT = 5
# //optional:
# LLAMA_API void llama_numa_init(enum ggml_numa_strategy numa);
def llama_numa_init ( numa : int ) :
return _lib . llama_numa_init ( numa )
_lib . llama_numa_init . argtypes = [ c_int ]
_lib . llama_numa_init . restype = None
2023-07-15 19:11:01 +00:00
# // Call once at the end of the program - currently only used for MPI
2023-08-24 04:17:00 +00:00
# LLAMA_API void llama_backend_free(void);
2023-07-15 19:11:01 +00:00
def llama_backend_free ( ) :
2023-11-23 05:26:26 +00:00
""" Call once at the end of the program - currently only used for MPI """
2023-07-15 19:11:01 +00:00
return _lib . llama_backend_free ( )
2023-05-21 21:47:21 +00:00
2023-07-15 19:11:01 +00:00
_lib . llama_backend_free . argtypes = [ ]
_lib . llama_backend_free . restype = None
2023-05-21 21:47:21 +00:00
2023-06-26 12:50:38 +00:00
# LLAMA_API struct llama_model * llama_load_model_from_file(
2023-08-24 04:17:00 +00:00
# const char * path_model,
2023-10-05 20:07:49 +00:00
# struct llama_model_params params);
2023-06-26 12:50:38 +00:00
def llama_load_model_from_file (
2023-09-29 02:42:03 +00:00
path_model : bytes , params : llama_model_params
2023-06-26 12:50:38 +00:00
) - > llama_model_p :
2023-07-24 17:11:10 +00:00
return _lib . llama_load_model_from_file ( path_model , params )
2023-06-26 12:50:38 +00:00
2023-09-29 02:42:03 +00:00
_lib . llama_load_model_from_file . argtypes = [ c_char_p , llama_model_params ]
2023-06-26 12:50:38 +00:00
_lib . llama_load_model_from_file . restype = llama_model_p
# LLAMA_API void llama_free_model(struct llama_model * model);
def llama_free_model ( model : llama_model_p ) :
return _lib . llama_free_model ( model )
_lib . llama_free_model . argtypes = [ llama_model_p ]
_lib . llama_free_model . restype = None
# LLAMA_API struct llama_context * llama_new_context_with_model(
2023-08-24 04:17:00 +00:00
# struct llama_model * model,
2023-06-26 12:50:38 +00:00
# struct llama_context_params params);
def llama_new_context_with_model (
model : llama_model_p , params : llama_context_params
) - > llama_context_p :
return _lib . llama_new_context_with_model ( model , params )
_lib . llama_new_context_with_model . argtypes = [ llama_model_p , llama_context_params ]
_lib . llama_new_context_with_model . restype = llama_context_p
2023-08-24 04:17:00 +00:00
# // Frees all allocated memory
# LLAMA_API void llama_free(struct llama_context * ctx);
def llama_free ( ctx : llama_context_p ) :
2023-11-23 05:26:26 +00:00
""" Frees all allocated memory """
2023-08-24 04:17:00 +00:00
return _lib . llama_free ( ctx )
_lib . llama_free . argtypes = [ llama_context_p ]
_lib . llama_free . restype = None
# LLAMA_API int64_t llama_time_us(void);
2023-05-21 21:47:21 +00:00
def llama_time_us ( ) - > int :
return _lib . llama_time_us ( )
_lib . llama_time_us . argtypes = [ ]
_lib . llama_time_us . restype = ctypes . c_int64
2024-01-31 15:41:42 +00:00
# LLAMA_API size_t llama_max_devices(void);
2023-08-24 04:17:00 +00:00
def llama_max_devices ( ) - > int :
return _lib . llama_max_devices ( )
2023-03-23 09:33:06 +00:00
2023-03-24 18:35:41 +00:00
2023-08-24 04:17:00 +00:00
_lib . llama_max_devices . argtypes = [ ]
2024-01-31 15:41:42 +00:00
_lib . llama_max_devices . restype = ctypes . c_size_t
2023-03-24 18:58:42 +00:00
2023-03-24 18:59:29 +00:00
2024-01-31 15:41:42 +00:00
# LLAMA_API bool llama_supports_mmap (void);
def llama_supports_mmap ( ) - > bool :
return _lib . llama_supports_mmap ( )
_lib . llama_supports_mmap . argtypes = [ ]
_lib . llama_supports_mmap . restype = c_bool
# LLAMA_API bool llama_supports_mlock (void);
def llama_supports_mlock ( ) - > bool :
return _lib . llama_supports_mlock ( )
_lib . llama_supports_mlock . argtypes = [ ]
_lib . llama_supports_mlock . restype = c_bool
# LLAMA_API bool llama_supports_gpu_offload(void);
def llama_supports_gpu_offload ( ) - > bool :
return _lib . llama_supports_gpu_offload ( )
_lib . llama_supports_gpu_offload . argtypes = [ ]
_lib . llama_supports_gpu_offload . restype = c_bool
# LLAMA_API DEPRECATED(bool llama_mmap_supported (void), "use llama_supports_mmap() instead");
2023-08-24 04:17:00 +00:00
def llama_mmap_supported ( ) - > bool :
return _lib . llama_mmap_supported ( )
2023-03-23 09:33:06 +00:00
2023-03-24 18:35:41 +00:00
2023-08-24 04:17:00 +00:00
_lib . llama_mmap_supported . argtypes = [ ]
_lib . llama_mmap_supported . restype = c_bool
2024-01-31 15:41:42 +00:00
# LLAMA_API DEPRECATED(bool llama_mlock_supported(void), "use llama_supports_mlock() instead");
2023-08-24 04:17:00 +00:00
def llama_mlock_supported ( ) - > bool :
return _lib . llama_mlock_supported ( )
_lib . llama_mlock_supported . argtypes = [ ]
_lib . llama_mlock_supported . restype = c_bool
2023-09-29 02:42:03 +00:00
# LLAMA_API const struct llama_model * llama_get_model(const struct llama_context * ctx);
def llama_get_model ( ctx : llama_context_p ) - > llama_model_p :
return _lib . llama_get_model ( ctx )
2023-08-24 04:17:00 +00:00
2023-09-29 02:42:03 +00:00
_lib . llama_get_model . argtypes = [ llama_context_p ]
_lib . llama_get_model . restype = llama_model_p
2023-08-24 04:17:00 +00:00
2024-01-04 03:04:04 +00:00
# LLAMA_API uint32_t llama_n_ctx (const struct llama_context * ctx);
2023-08-24 04:17:00 +00:00
def llama_n_ctx ( ctx : llama_context_p ) - > int :
return _lib . llama_n_ctx ( ctx )
_lib . llama_n_ctx . argtypes = [ llama_context_p ]
2023-12-22 05:12:37 +00:00
_lib . llama_n_ctx . restype = c_uint32
2023-08-24 04:17:00 +00:00
2024-01-08 19:51:29 +00:00
2023-12-22 05:12:37 +00:00
# LLAMA_API uint32_t llama_n_batch (const struct llama_context * ctx);
def llama_n_batch ( ctx : llama_context_p ) - > int :
return _lib . llama_n_batch ( ctx )
2024-01-08 19:51:29 +00:00
2023-12-22 05:12:37 +00:00
_lib . llama_n_batch . argtypes = [ llama_context_p ]
_lib . llama_n_batch . restype = c_uint32
2023-08-24 04:17:00 +00:00
2024-01-08 19:51:29 +00:00
2023-09-29 02:42:03 +00:00
# LLAMA_API enum llama_vocab_type llama_vocab_type(const struct llama_model * model);
def llama_vocab_type ( model : llama_model_p ) - > int :
return _lib . llama_vocab_type ( model )
2023-09-09 16:12:32 +00:00
2023-08-24 04:17:00 +00:00
2023-09-29 02:42:03 +00:00
_lib . llama_vocab_type . argtypes = [ llama_model_p ]
2023-08-24 22:01:42 +00:00
_lib . llama_vocab_type . restype = c_int
2024-01-04 03:04:04 +00:00
# LLAMA_API int32_t llama_n_vocab (const struct llama_model * model);
2023-09-29 02:42:03 +00:00
def llama_n_vocab ( model : llama_model_p ) - > int :
return _lib . llama_n_vocab ( model )
2023-08-24 04:17:00 +00:00
2023-09-29 02:42:03 +00:00
_lib . llama_n_vocab . argtypes = [ llama_model_p ]
2024-01-04 03:04:04 +00:00
_lib . llama_n_vocab . restype = c_int32
2023-08-24 04:17:00 +00:00
2024-01-04 03:04:04 +00:00
# LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model);
2023-09-29 02:42:03 +00:00
def llama_n_ctx_train ( model : llama_model_p ) - > int :
return _lib . llama_n_ctx_train ( model )
2023-09-09 16:12:32 +00:00
2023-09-29 02:42:03 +00:00
_lib . llama_n_ctx_train . argtypes = [ llama_model_p ]
2024-01-04 03:04:04 +00:00
_lib . llama_n_ctx_train . restype = c_int32
2023-09-09 16:12:32 +00:00
2024-01-04 03:04:04 +00:00
# LLAMA_API int32_t llama_n_embd (const struct llama_model * model);
2023-09-29 02:42:03 +00:00
def llama_n_embd ( model : llama_model_p ) - > int :
return _lib . llama_n_embd ( model )
2023-08-24 04:17:00 +00:00
2023-09-29 02:42:03 +00:00
_lib . llama_n_embd . argtypes = [ llama_model_p ]
2024-01-04 03:04:04 +00:00
_lib . llama_n_embd . restype = c_int32
2023-08-24 04:17:00 +00:00
2023-10-03 19:23:35 +00:00
# // Get the model's RoPE frequency scaling factor
# LLAMA_API float llama_rope_freq_scale_train(const struct llama_model * model);
def llama_rope_freq_scale_train ( model : llama_model_p ) - > float :
2023-11-23 05:26:26 +00:00
""" Get the model ' s RoPE frequency scaling factor """
2023-10-03 19:23:35 +00:00
return _lib . llama_rope_freq_scale_train ( model )
_lib . llama_rope_freq_scale_train . argtypes = [ llama_model_p ]
_lib . llama_rope_freq_scale_train . restype = c_float
2023-11-20 19:11:33 +00:00
# // Functions to access the model's GGUF metadata scalar values
# // - The functions return the length of the string on success, or -1 on failure
# // - The output string is always null-terminated and cleared on failure
# // - GGUF array values are not supported by these functions
# // Get metadata value as a string by key name
2024-01-04 03:04:04 +00:00
# LLAMA_API int32_t llama_model_meta_val_str(const struct llama_model * model, const char * key, char * buf, size_t buf_size);
2023-11-20 19:11:33 +00:00
def llama_model_meta_val_str (
model : llama_model_p , key : Union [ c_char_p , bytes ] , buf : bytes , buf_size : int
) - > int :
2023-11-23 05:26:26 +00:00
""" Get metadata value as a string by key name """
2023-11-20 19:11:33 +00:00
return _lib . llama_model_meta_val_str ( model , key , buf , buf_size )
_lib . llama_model_meta_val_str . argtypes = [ llama_model_p , c_char_p , c_char_p , c_size_t ]
2024-01-04 03:04:04 +00:00
_lib . llama_model_meta_val_str . restype = c_int32
2023-11-20 19:11:33 +00:00
# // Get the number of metadata key/value pairs
2024-01-04 03:04:04 +00:00
# LLAMA_API int32_t llama_model_meta_count(const struct llama_model * model);
2023-11-20 19:11:33 +00:00
def llama_model_meta_count ( model : llama_model_p ) - > int :
2023-11-23 05:26:26 +00:00
""" Get the number of metadata key/value pairs """
2023-11-20 19:11:33 +00:00
return _lib . llama_model_meta_count ( model )
_lib . llama_model_meta_count . argtypes = [ llama_model_p ]
2024-01-04 03:04:04 +00:00
_lib . llama_model_meta_count . restype = c_int32
2023-11-20 19:11:33 +00:00
# // Get metadata key name by index
2024-01-04 03:04:04 +00:00
# LLAMA_API int32_t llama_model_meta_key_by_index(const struct llama_model * model, int32_t i, char * buf, size_t buf_size);
2023-11-20 19:11:33 +00:00
def llama_model_meta_key_by_index (
model : llama_model_p , i : Union [ c_int , int ] , buf : bytes , buf_size : int
) - > int :
2023-11-23 05:26:26 +00:00
""" Get metadata key name by index """
2023-11-20 19:11:33 +00:00
return _lib . llama_model_meta_key_by_index ( model , i , buf , buf_size )
2024-01-04 03:04:04 +00:00
_lib . llama_model_meta_key_by_index . argtypes = [
llama_model_p ,
c_int32 ,
c_char_p ,
c_size_t ,
]
_lib . llama_model_meta_key_by_index . restype = c_int32
2023-11-20 19:11:33 +00:00
# // Get metadata value as a string by index
2024-01-04 03:04:04 +00:00
# LLAMA_API int32_t llama_model_meta_val_str_by_index(const struct llama_model * model, int32_t i, char * buf, size_t buf_size);
2023-11-20 19:11:33 +00:00
def llama_model_meta_val_str_by_index (
model : llama_model_p , i : Union [ c_int , int ] , buf : bytes , buf_size : int
) - > int :
2023-11-23 05:26:26 +00:00
""" Get metadata value as a string by index """
2023-11-20 19:11:33 +00:00
return _lib . llama_model_meta_val_str_by_index ( model , i , buf , buf_size )
_lib . llama_model_meta_val_str_by_index . argtypes = [
llama_model_p ,
2024-01-04 03:04:04 +00:00
c_int32 ,
2023-11-20 19:11:33 +00:00
c_char_p ,
c_size_t ,
]
2024-01-04 03:04:04 +00:00
_lib . llama_model_meta_val_str_by_index . restype = c_int32
2023-11-20 19:11:33 +00:00
2023-10-03 19:23:35 +00:00
2023-08-24 04:17:00 +00:00
# // Get a string describing the model type
2024-01-04 03:04:04 +00:00
# LLAMA_API int32_t llama_model_desc(const struct llama_model * model, char * buf, size_t buf_size);
2023-09-29 02:42:03 +00:00
def llama_model_desc (
model : llama_model_p , buf : bytes , buf_size : Union [ c_size_t , int ]
) - > int :
2023-11-23 05:26:26 +00:00
""" Get a string describing the model type """
2023-08-25 18:35:53 +00:00
return _lib . llama_model_desc ( model , buf , buf_size )
2023-08-24 04:17:00 +00:00
2023-08-25 18:35:53 +00:00
_lib . llama_model_desc . argtypes = [ llama_model_p , c_char_p , c_size_t ]
2024-01-04 03:04:04 +00:00
_lib . llama_model_desc . restype = c_int32
2023-08-25 18:35:53 +00:00
# // Returns the total size of all the tensors in the model in bytes
# LLAMA_API uint64_t llama_model_size(const struct llama_model * model);
def llama_model_size ( model : llama_model_p ) - > int :
2023-11-23 05:26:26 +00:00
""" Returns the total size of all the tensors in the model in bytes """
2023-08-25 18:35:53 +00:00
return _lib . llama_model_size ( model )
2023-08-24 04:17:00 +00:00
2023-08-25 18:35:53 +00:00
_lib . llama_model_size . argtypes = [ llama_model_p ]
_lib . llama_model_size . restype = ctypes . c_uint64
2023-08-24 04:17:00 +00:00
2023-08-25 18:35:53 +00:00
# // Returns the total number of parameters in the model
# LLAMA_API uint64_t llama_model_n_params(const struct llama_model * model);
def llama_model_n_params ( model : llama_model_p ) - > int :
2023-11-23 05:26:26 +00:00
""" Returns the total number of parameters in the model """
2023-08-25 18:35:53 +00:00
return _lib . llama_model_n_params ( model )
_lib . llama_model_n_params . argtypes = [ llama_model_p ]
_lib . llama_model_n_params . restype = ctypes . c_uint64
2023-03-24 18:58:42 +00:00
2023-03-24 18:59:29 +00:00
2023-09-29 02:42:03 +00:00
# // Get a llama model tensor
# LLAMA_API struct ggml_tensor * llama_get_model_tensor(struct llama_model * model, const char * name);
def llama_get_model_tensor (
model : llama_model_p , name : Union [ c_char_p , bytes ]
) - > c_void_p :
2023-11-23 05:26:26 +00:00
""" Get a llama model tensor """
2023-09-29 02:42:03 +00:00
return _lib . llama_get_model_tensor ( model , name )
_lib . llama_get_model_tensor . argtypes = [ llama_model_p , c_char_p ]
_lib . llama_get_model_tensor . restype = c_void_p
2023-06-10 16:17:38 +00:00
# // Returns 0 on success
2024-01-04 03:04:04 +00:00
# LLAMA_API uint32_t llama_model_quantize(
2023-05-21 21:47:21 +00:00
# const char * fname_inp,
# const char * fname_out,
2023-06-10 16:17:38 +00:00
# const llama_model_quantize_params * params);
2023-04-22 23:50:28 +00:00
def llama_model_quantize (
2023-06-10 16:17:38 +00:00
fname_inp : bytes ,
fname_out : bytes ,
params , # type: POINTER(llama_model_quantize_params) # type: ignore
2023-05-19 15:59:33 +00:00
) - > int :
2023-11-23 05:26:26 +00:00
""" Returns 0 on success """
2023-06-10 16:17:38 +00:00
return _lib . llama_model_quantize ( fname_inp , fname_out , params )
2023-03-23 09:33:06 +00:00
2023-03-24 18:35:41 +00:00
2023-06-10 16:17:38 +00:00
_lib . llama_model_quantize . argtypes = [
c_char_p ,
c_char_p ,
POINTER ( llama_model_quantize_params ) ,
]
2024-01-04 03:04:04 +00:00
_lib . llama_model_quantize . restype = c_uint32
2023-03-24 18:58:42 +00:00
2023-04-11 15:59:03 +00:00
2023-09-29 02:42:03 +00:00
# // Apply a LoRA adapter to a loaded model
# // path_base_model is the path to a higher quality model to use as a base for
# // the layers modified by the adapter. Can be NULL to use the current loaded model.
# // The model needs to be reloaded before applying a new adapter, otherwise the adapter
# // will be applied on top of the previous one
# // Returns 0 on success
2024-01-04 03:04:04 +00:00
# LLAMA_API DEPRECATED(int32_t llama_apply_lora_from_file(
2023-05-21 21:47:21 +00:00
# struct llama_context * ctx,
# const char * path_lora,
2023-09-29 02:42:03 +00:00
# float scale,
2023-05-21 21:47:21 +00:00
# const char * path_base_model,
2024-01-04 03:04:04 +00:00
# int32_t n_threads),
2023-09-29 02:42:03 +00:00
# "use llama_model_apply_lora_from_file instead");
2023-04-18 05:30:04 +00:00
def llama_apply_lora_from_file (
2023-04-22 23:50:28 +00:00
ctx : llama_context_p ,
2023-07-24 19:42:07 +00:00
path_lora : Union [ c_char_p , bytes ] ,
2023-09-29 02:42:03 +00:00
scale : Union [ c_float , float ] ,
2023-07-24 19:42:07 +00:00
path_base_model : Union [ c_char_p , bytes ] ,
n_threads : Union [ c_int , int ] ,
2023-05-19 15:59:33 +00:00
) - > int :
2023-11-23 05:26:26 +00:00
""" Apply a LoRA adapter to a loaded model
path_base_model is the path to a higher quality model to use as a base for
the layers modified by the adapter . Can be NULL to use the current loaded model .
The model needs to be reloaded before applying a new adapter , otherwise the adapter
will be applied on top of the previous one
Returns 0 on success """
2023-09-29 02:42:03 +00:00
return _lib . llama_apply_lora_from_file (
ctx , path_lora , scale , path_base_model , n_threads
)
2023-04-18 05:30:04 +00:00
2023-09-29 02:42:03 +00:00
_lib . llama_apply_lora_from_file . argtypes = [
llama_context_p ,
c_char_p ,
c_float ,
c_char_p ,
2024-01-04 03:04:04 +00:00
c_int32 ,
2023-09-29 02:42:03 +00:00
]
2024-01-04 03:04:04 +00:00
_lib . llama_apply_lora_from_file . restype = c_int32
2023-06-26 12:50:38 +00:00
2024-01-04 03:04:04 +00:00
# LLAMA_API int32_t llama_model_apply_lora_from_file(
2023-06-26 12:50:38 +00:00
# const struct llama_model * model,
2023-09-29 02:42:03 +00:00
# const char * path_lora,
# float scale,
# const char * path_base_model,
2024-01-04 03:04:04 +00:00
# int32_t n_threads);
2023-06-26 12:50:38 +00:00
def llama_model_apply_lora_from_file (
model : llama_model_p ,
path_lora : Union [ c_char_p , bytes ] ,
2023-09-29 02:42:03 +00:00
scale : Union [ c_float , float ] ,
2023-06-26 12:50:38 +00:00
path_base_model : Union [ c_char_p , bytes ] ,
2023-07-24 19:42:07 +00:00
n_threads : Union [ c_int , int ] ,
2023-06-26 12:50:38 +00:00
) - > int :
return _lib . llama_model_apply_lora_from_file (
2023-09-29 02:42:03 +00:00
model , path_lora , scale , path_base_model , n_threads
2023-06-26 12:50:38 +00:00
)
_lib . llama_model_apply_lora_from_file . argtypes = [
llama_model_p ,
c_char_p ,
2023-09-29 02:42:03 +00:00
c_float ,
2023-06-26 12:50:38 +00:00
c_char_p ,
2024-01-04 03:04:04 +00:00
c_int32 ,
2023-06-26 12:50:38 +00:00
]
2024-01-04 03:04:04 +00:00
_lib . llama_model_apply_lora_from_file . restype = c_int32
2023-04-18 05:30:04 +00:00
2023-09-29 02:42:03 +00:00
# //
# // KV cache
# //
2023-04-18 05:30:04 +00:00
2023-11-23 21:26:00 +00:00
# // Information associated with an individual cell in the KV cache view.
# struct llama_kv_cache_view_cell {
# // The position for this cell. Takes KV cache shifts into account.
# // May be negative if the cell is not populated.
# llama_pos pos;
# };
class llama_kv_cache_view_cell ( Structure ) :
_fields_ = [ ( " pos " , llama_pos ) ]
# // An updateable view of the KV cache.
# struct llama_kv_cache_view {
# // Number of KV cache cells. This will be the same as the context size.
# int32_t n_cells;
# // Maximum number of sequences that can exist in a cell. It's not an error
# // if there are more sequences in a cell than this value, however they will
# // not be visible in the view cells_sequences.
# int32_t n_max_seq;
# // Number of tokens in the cache. For example, if there are two populated
# // cells, the first with 1 sequence id in it and the second with 2 sequence
# // ids then you'll have 3 tokens.
# int32_t token_count;
# // Number of populated cache cells.
# int32_t used_cells;
# // Maximum contiguous empty slots in the cache.
# int32_t max_contiguous;
# // Index to the start of the max_contiguous slot range. Can be negative
# // when cache is full.
# int32_t max_contiguous_idx;
# // Information for an individual cell.
# struct llama_kv_cache_view_cell * cells;
# // The sequences for each cell. There will be n_max_seq items per cell.
# llama_seq_id * cells_sequences;
# };
class llama_kv_cache_view ( Structure ) :
_fields_ = [
( " n_cells " , c_int32 ) ,
( " n_max_seq " , c_int32 ) ,
( " token_count " , c_int32 ) ,
( " used_cells " , c_int32 ) ,
( " max_contiguous " , c_int32 ) ,
( " max_contiguous_idx " , c_int32 ) ,
( " cells " , POINTER ( llama_kv_cache_view_cell ) ) ,
( " cells_sequences " , POINTER ( llama_seq_id ) ) ,
]
2023-12-18 23:11:26 +00:00
llama_kv_cache_view_p = POINTER ( llama_kv_cache_view )
2023-11-23 21:26:00 +00:00
# // Create an empty KV cache view. (use only for debugging purposes)
# LLAMA_API struct llama_kv_cache_view llama_kv_cache_view_init(const struct llama_context * ctx, int32_t n_max_seq);
def llama_kv_cache_view_init (
ctx : llama_context_p , n_max_seq : Union [ c_int32 , int ]
) - > llama_kv_cache_view :
""" Create an empty KV cache view. (use only for debugging purposes) """
return _lib . llama_kv_cache_view_init ( ctx , n_max_seq )
_lib . llama_kv_cache_view_init . argtypes = [ llama_context_p , c_int32 ]
_lib . llama_kv_cache_view_init . restype = llama_kv_cache_view
# // Free a KV cache view. (use only for debugging purposes)
# LLAMA_API void llama_kv_cache_view_free(struct llama_kv_cache_view * view);
2024-01-08 19:51:29 +00:00
def llama_kv_cache_view_free ( view : " ctypes.pointer[llama_kv_cache_view] " ) : # type: ignore
2023-11-23 21:26:00 +00:00
""" Free a KV cache view. (use only for debugging purposes) """
return _lib . llama_kv_cache_view_free ( view )
2023-12-18 23:11:26 +00:00
_lib . llama_kv_cache_view_free . argtypes = [ llama_kv_cache_view_p ]
2023-11-23 21:26:00 +00:00
_lib . llama_kv_cache_view_free . restype = None
# // Update the KV cache view structure with the current state of the KV cache. (use only for debugging purposes)
# LLAMA_API void llama_kv_cache_view_update(const struct llama_context * ctx, struct llama_kv_cache_view * view);
2024-01-08 19:51:29 +00:00
def llama_kv_cache_view_update ( ctx : llama_context_p , view : " ctypes.pointer[llama_kv_cache_view] " ) : # type: ignore
2023-11-23 21:26:00 +00:00
""" Update the KV cache view structure with the current state of the KV cache. (use only for debugging purposes) """
return _lib . llama_kv_cache_view_update ( ctx , view )
2023-12-18 23:11:26 +00:00
_lib . llama_kv_cache_view_update . argtypes = [ llama_context_p , llama_kv_cache_view_p ]
2023-11-23 21:26:00 +00:00
_lib . llama_kv_cache_view_update . restype = None
# // Returns the number of tokens in the KV cache (slow, use only for debug)
# // If a KV cell has multiple sequences assigned to it, it will be counted multiple times
2024-01-04 03:04:04 +00:00
# LLAMA_API int32_t llama_get_kv_cache_token_count(const struct llama_context * ctx);
2023-05-19 15:59:33 +00:00
def llama_get_kv_cache_token_count ( ctx : llama_context_p ) - > int :
2023-11-23 21:26:00 +00:00
""" Returns the number of tokens in the KV cache (slow, use only for debug)
If a KV cell has multiple sequences assigned to it , it will be counted multiple times
"""
2023-04-02 17:33:49 +00:00
return _lib . llama_get_kv_cache_token_count ( ctx )
2023-04-11 15:59:03 +00:00
2023-04-02 17:33:49 +00:00
_lib . llama_get_kv_cache_token_count . argtypes = [ llama_context_p ]
2024-01-04 03:04:04 +00:00
_lib . llama_get_kv_cache_token_count . restype = c_int32
2023-04-02 17:33:49 +00:00
2023-04-28 19:32:43 +00:00
2023-11-23 21:26:00 +00:00
# // Returns the number of used KV cells (i.e. have at least one sequence assigned to them)
2024-01-04 03:04:04 +00:00
# LLAMA_API int32_t llama_get_kv_cache_used_cells(const struct llama_context * ctx);
2023-11-23 21:26:00 +00:00
def llama_get_kv_cache_used_cells ( ctx : llama_context_p ) - > int :
""" Returns the number of used KV cells (i.e. have at least one sequence assigned to them) """
return _lib . llama_get_kv_cache_used_cells ( ctx )
_lib . llama_get_kv_cache_used_cells . argtypes = [ llama_context_p ]
2024-01-04 03:04:04 +00:00
_lib . llama_get_kv_cache_used_cells . restype = c_int32
2023-11-23 21:26:00 +00:00
2023-11-01 01:29:35 +00:00
# // Clear the KV cache
# LLAMA_API void llama_kv_cache_clear(
# struct llama_context * ctx);
def llama_kv_cache_clear ( ctx : llama_context_p ) :
2023-11-23 05:26:26 +00:00
""" Clear the KV cache """
2023-11-01 01:29:35 +00:00
return _lib . llama_kv_cache_clear ( ctx )
2023-04-27 00:00:54 +00:00
2023-04-28 19:32:43 +00:00
2023-11-01 01:29:35 +00:00
_lib . llama_kv_cache_clear . argtypes = [ llama_context_p ]
_lib . llama_kv_cache_clear . restype = None
2023-09-29 02:42:03 +00:00
# // Removes all tokens that belong to the specified sequence and have positions in [p0, p1)
2023-11-01 01:29:35 +00:00
# // seq_id < 0 : match any sequence
# // p0 < 0 : [0, p1]
# // p1 < 0 : [p0, inf)
2023-09-29 02:42:03 +00:00
# LLAMA_API void llama_kv_cache_seq_rm(
# struct llama_context * ctx,
# llama_seq_id seq_id,
# llama_pos p0,
# llama_pos p1);
def llama_kv_cache_seq_rm (
ctx : llama_context_p ,
2023-11-05 21:57:10 +00:00
seq_id : Union [ llama_seq_id , int ] ,
2023-09-29 02:42:03 +00:00
p0 : Union [ llama_pos , int ] ,
p1 : Union [ llama_pos , int ] ,
) :
2023-11-23 05:26:26 +00:00
""" Removes all tokens that belong to the specified sequence and have positions in [p0, p1)
seq_id < 0 : match any sequence
p0 < 0 : [ 0 , p1 ]
p1 < 0 : [ p0 , inf ) """
2023-09-29 02:42:03 +00:00
return _lib . llama_kv_cache_seq_rm ( ctx , seq_id , p0 , p1 )
_lib . llama_kv_cache_seq_rm . argtypes = [
llama_context_p ,
llama_seq_id ,
llama_pos ,
llama_pos ,
]
_lib . llama_kv_cache_seq_rm . restype = None
# // Copy all tokens that belong to the specified sequence to another sequence
# // Note that this does not allocate extra KV cache memory - it simply assigns the tokens to the new sequence
2023-10-03 19:23:35 +00:00
# // p0 < 0 : [0, p1]
# // p1 < 0 : [p0, inf)
2023-09-29 02:42:03 +00:00
# LLAMA_API void llama_kv_cache_seq_cp(
# struct llama_context * ctx,
# llama_seq_id seq_id_src,
# llama_seq_id seq_id_dst,
# llama_pos p0,
# llama_pos p1);
def llama_kv_cache_seq_cp (
ctx : llama_context_p ,
2023-11-05 21:57:10 +00:00
seq_id_src : Union [ llama_seq_id , int ] ,
seq_id_dst : Union [ llama_seq_id , int ] ,
2023-09-29 02:42:03 +00:00
p0 : Union [ llama_pos , int ] ,
p1 : Union [ llama_pos , int ] ,
) :
2023-11-23 05:26:26 +00:00
""" Copy all tokens that belong to the specified sequence to another sequence
Note that this does not allocate extra KV cache memory - it simply assigns the tokens to the new sequence
p0 < 0 : [ 0 , p1 ]
p1 < 0 : [ p0 , inf ) """
2023-09-29 02:42:03 +00:00
return _lib . llama_kv_cache_seq_cp ( ctx , seq_id_src , seq_id_dst , p0 , p1 )
_lib . llama_kv_cache_seq_cp . argtypes = [
llama_context_p ,
llama_seq_id ,
llama_seq_id ,
llama_pos ,
llama_pos ,
]
_lib . llama_kv_cache_seq_cp . restype = None
# // Removes all tokens that do not belong to the specified sequence
# LLAMA_API void llama_kv_cache_seq_keep(
# struct llama_context * ctx,
# llama_seq_id seq_id);
def llama_kv_cache_seq_keep (
ctx : llama_context_p ,
2023-11-05 21:57:10 +00:00
seq_id : Union [ llama_seq_id , int ] ,
2023-09-29 02:42:03 +00:00
) :
2023-11-23 05:26:26 +00:00
""" Removes all tokens that do not belong to the specified sequence """
2023-09-29 02:42:03 +00:00
return _lib . llama_kv_cache_seq_keep ( ctx , seq_id )
_lib . llama_kv_cache_seq_keep . argtypes = [ llama_context_p , llama_seq_id ]
_lib . llama_kv_cache_seq_keep . restype = None
# // Adds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1)
# // If the KV cache is RoPEd, the KV data is updated accordingly
2023-10-03 19:23:35 +00:00
# // p0 < 0 : [0, p1]
# // p1 < 0 : [p0, inf)
2023-09-29 02:42:03 +00:00
# LLAMA_API void llama_kv_cache_seq_shift(
# struct llama_context * ctx,
# llama_seq_id seq_id,
# llama_pos p0,
# llama_pos p1,
# llama_pos delta);
def llama_kv_cache_seq_shift (
ctx : llama_context_p ,
2023-11-05 21:57:10 +00:00
seq_id : Union [ llama_seq_id , int ] ,
2023-09-29 02:42:03 +00:00
p0 : Union [ llama_pos , int ] ,
p1 : Union [ llama_pos , int ] ,
delta : Union [ llama_pos , int ] ,
) :
2023-11-23 05:26:26 +00:00
""" Adds relative position " delta " to all tokens that belong to the specified sequence and have positions in [p0, p1)
If the KV cache is RoPEd , the KV data is updated accordingly
p0 < 0 : [ 0 , p1 ]
p1 < 0 : [ p0 , inf ) """
2023-09-29 02:42:03 +00:00
return _lib . llama_kv_cache_seq_shift ( ctx , seq_id , p0 , p1 , delta )
_lib . llama_kv_cache_seq_shift . argtypes = [
llama_context_p ,
llama_seq_id ,
llama_pos ,
llama_pos ,
llama_pos ,
]
_lib . llama_kv_cache_seq_shift . restype = None
2024-01-08 19:51:29 +00:00
# // Integer division of the positions by factor of `d > 1`
# // If the KV cache is RoPEd, the KV data is updated accordingly
# // p0 < 0 : [0, p1]
# // p1 < 0 : [p0, inf)
# LLAMA_API void llama_kv_cache_seq_div(
# struct llama_context * ctx,
# llama_seq_id seq_id,
# llama_pos p0,
# llama_pos p1,
# int d);
def llama_kv_cache_seq_div (
ctx : llama_context_p ,
seq_id : Union [ llama_seq_id , int ] ,
p0 : Union [ llama_pos , int ] ,
p1 : Union [ llama_pos , int ] ,
d : Union [ c_int , int ] ,
) :
""" Integer division of the positions by factor of `d > 1`
If the KV cache is RoPEd , the KV data is updated accordingly
p0 < 0 : [ 0 , p1 ]
p1 < 0 : [ p0 , inf ) """
return _lib . llama_kv_cache_seq_div ( ctx , seq_id , p0 , p1 , d )
_lib . llama_kv_cache_seq_div . argtypes = [
llama_context_p ,
llama_seq_id ,
llama_pos ,
llama_pos ,
c_int ,
]
_lib . llama_kv_cache_seq_div . restype = None
2023-09-29 02:42:03 +00:00
# //
# // State / sessions
# //
2023-04-02 17:33:49 +00:00
2023-04-28 19:32:43 +00:00
2023-05-03 13:33:30 +00:00
# Returns the maximum size in bytes of the state (rng, logits, embedding
# and kv_cache) - will often be smaller after compacting tokens
2023-05-21 21:47:21 +00:00
# LLAMA_API size_t llama_get_state_size(const struct llama_context * ctx);
2023-05-19 15:59:33 +00:00
def llama_get_state_size ( ctx : llama_context_p ) - > int :
2023-11-23 05:26:26 +00:00
""" Returns the maximum size in bytes of the state (rng, logits, embedding
and kv_cache ) - will often be smaller after compacting tokens """
2023-04-22 23:50:28 +00:00
return _lib . llama_get_state_size ( ctx )
_lib . llama_get_state_size . argtypes = [ llama_context_p ]
_lib . llama_get_state_size . restype = c_size_t
# Copies the state to the specified destination address.
# Destination needs to have allocated enough memory.
# Returns the number of bytes copied
2023-09-29 02:42:03 +00:00
# LLAMA_API size_t llama_copy_state_data(
# struct llama_context * ctx,
# uint8_t * dst);
2023-05-05 18:12:26 +00:00
def llama_copy_state_data (
2023-05-14 04:04:22 +00:00
ctx : llama_context_p , dst # type: Array[c_uint8]
2023-05-07 23:30:14 +00:00
) - > int :
2023-11-23 05:26:26 +00:00
""" Copies the state to the specified destination address.
Destination needs to have allocated enough memory .
Returns the number of bytes copied """
2023-05-14 04:04:22 +00:00
return _lib . llama_copy_state_data ( ctx , dst )
2023-04-22 23:50:28 +00:00
2023-05-05 18:04:12 +00:00
_lib . llama_copy_state_data . argtypes = [ llama_context_p , c_uint8_p ]
2023-04-22 23:50:28 +00:00
_lib . llama_copy_state_data . restype = c_size_t
# Set the state reading from the specified address
# Returns the number of bytes read
2023-09-29 02:42:03 +00:00
# LLAMA_API size_t llama_set_state_data(
# struct llama_context * ctx,
# uint8_t * src);
2023-05-05 16:22:27 +00:00
def llama_set_state_data (
ctx : llama_context_p , src # type: Array[c_uint8]
2023-05-07 23:30:14 +00:00
) - > int :
2023-11-23 05:26:26 +00:00
""" Set the state reading from the specified address """
2023-04-22 23:50:28 +00:00
return _lib . llama_set_state_data ( ctx , src )
2023-05-05 18:04:12 +00:00
_lib . llama_set_state_data . argtypes = [ llama_context_p , c_uint8_p ]
2023-04-22 23:50:28 +00:00
_lib . llama_set_state_data . restype = c_size_t
2023-04-28 19:32:43 +00:00
# Save/load session file
2023-09-29 02:42:03 +00:00
# LLAMA_API bool llama_load_session_file(
# struct llama_context * ctx,
# const char * path_session,
# llama_token * tokens_out,
# size_t n_token_capacity,
# size_t * n_token_count_out);
2023-04-28 19:32:43 +00:00
def llama_load_session_file (
ctx : llama_context_p ,
path_session : bytes ,
2023-05-05 18:12:26 +00:00
tokens_out , # type: Array[llama_token]
2023-07-24 19:42:07 +00:00
n_token_capacity : Union [ c_size_t , int ] ,
2023-05-05 18:12:26 +00:00
n_token_count_out , # type: _Pointer[c_size_t]
2023-05-19 15:59:33 +00:00
) - > int :
2023-04-28 19:32:43 +00:00
return _lib . llama_load_session_file (
ctx , path_session , tokens_out , n_token_capacity , n_token_count_out
)
_lib . llama_load_session_file . argtypes = [
llama_context_p ,
c_char_p ,
llama_token_p ,
c_size_t ,
2023-05-05 18:04:12 +00:00
c_size_t_p ,
2023-04-28 19:32:43 +00:00
]
_lib . llama_load_session_file . restype = c_size_t
2023-09-29 02:42:03 +00:00
# LLAMA_API bool llama_save_session_file(
# struct llama_context * ctx,
# const char * path_session,
# const llama_token * tokens,
# size_t n_token_count);
2023-04-28 19:32:43 +00:00
def llama_save_session_file (
2023-05-05 16:22:27 +00:00
ctx : llama_context_p ,
path_session : bytes ,
2023-05-05 18:12:26 +00:00
tokens , # type: Array[llama_token]
2023-07-24 19:42:07 +00:00
n_token_count : Union [ c_size_t , int ] ,
2023-05-19 15:59:33 +00:00
) - > int :
2023-04-28 19:32:43 +00:00
return _lib . llama_save_session_file ( ctx , path_session , tokens , n_token_count )
_lib . llama_save_session_file . argtypes = [
llama_context_p ,
c_char_p ,
llama_token_p ,
c_size_t ,
]
_lib . llama_save_session_file . restype = c_size_t
2023-09-29 02:42:03 +00:00
# //
# // Decoding
# //
2023-04-28 19:32:43 +00:00
2023-09-29 02:42:03 +00:00
# // Run the llama inference to obtain the logits and probabilities for the next token(s).
# // tokens + n_tokens is the provided batch of new tokens to process
# // n_past is the number of tokens to use from previous eval calls
# // Returns 0 on success
# // DEPRECATED: use llama_decode() instead
# LLAMA_API DEPRECATED(int llama_eval(
2023-05-21 21:47:21 +00:00
# struct llama_context * ctx,
2023-09-29 02:42:03 +00:00
# llama_token * tokens,
# int32_t n_tokens,
2024-01-04 03:04:04 +00:00
# int32_t n_past),
2023-09-29 02:42:03 +00:00
# "use llama_decode() instead");
2023-03-24 18:35:41 +00:00
def llama_eval (
ctx : llama_context_p ,
2023-05-05 18:12:26 +00:00
tokens , # type: Array[llama_token]
2023-07-24 19:42:07 +00:00
n_tokens : Union [ c_int , int ] ,
n_past : Union [ c_int , int ] ,
2023-05-19 15:59:33 +00:00
) - > int :
2023-11-23 05:26:26 +00:00
""" Run the llama inference to obtain the logits and probabilities for the next token(s).
tokens + n_tokens is the provided batch of new tokens to process
n_past is the number of tokens to use from previous eval calls
Returns 0 on success
DEPRECATED : use llama_decode ( ) instead """
2023-09-29 02:42:03 +00:00
return _lib . llama_eval ( ctx , tokens , n_tokens , n_past )
2023-03-23 09:33:06 +00:00
2023-03-24 18:35:41 +00:00
2024-01-04 03:04:04 +00:00
_lib . llama_eval . argtypes = [ llama_context_p , llama_token_p , c_int32 , c_int32 ]
2023-03-24 22:43:29 +00:00
_lib . llama_eval . restype = c_int
2023-03-24 18:58:42 +00:00
2023-06-29 05:08:15 +00:00
# // Same as llama_eval, but use float matrix input directly.
2023-09-29 02:42:03 +00:00
# // DEPRECATED: use llama_decode() instead
# LLAMA_API DEPRECATED(int llama_eval_embd(
2023-06-29 05:08:15 +00:00
# struct llama_context * ctx,
2023-09-29 02:42:03 +00:00
# float * embd,
# int32_t n_tokens,
2024-01-04 03:04:04 +00:00
# int32_t n_past),
2023-09-29 02:42:03 +00:00
# "use llama_decode() instead");
2023-06-29 05:08:15 +00:00
def llama_eval_embd (
ctx : llama_context_p ,
embd , # type: Array[c_float]
2023-07-24 19:42:07 +00:00
n_tokens : Union [ c_int , int ] ,
n_past : Union [ c_int , int ] ,
2023-06-29 05:08:15 +00:00
) - > int :
2023-11-23 05:26:26 +00:00
""" Same as llama_eval, but use float matrix input directly.
DEPRECATED : use llama_decode ( ) instead """
2023-09-29 02:42:03 +00:00
return _lib . llama_eval_embd ( ctx , embd , n_tokens , n_past )
2023-06-29 05:08:15 +00:00
2024-01-04 03:04:04 +00:00
_lib . llama_eval_embd . argtypes = [ llama_context_p , c_float_p , c_int32 , c_int32 ]
2023-06-29 05:08:15 +00:00
_lib . llama_eval_embd . restype = c_int
2023-09-29 02:42:03 +00:00
# // Return batch for single sequence of tokens starting at pos_0
# //
# // NOTE: this is a helper function to facilitate transition to the new batch API - avoid using it
# //
# LLAMA_API struct llama_batch llama_batch_get_one(
# llama_token * tokens,
# int32_t n_tokens,
# llama_pos pos_0,
# llama_seq_id seq_id);
def llama_batch_get_one (
tokens , # type: Array[llama_token]
n_tokens : Union [ c_int , int ] ,
pos_0 : Union [ llama_pos , int ] ,
seq_id : llama_seq_id ,
) - > llama_batch :
2023-11-23 05:26:26 +00:00
""" Return batch for single sequence of tokens starting at pos_0
2023-11-23 21:26:00 +00:00
NOTE : this is a helper function to facilitate transition to the new batch API - avoid using it
"""
2023-09-29 02:42:03 +00:00
return _lib . llama_batch_get_one ( tokens , n_tokens , pos_0 , seq_id )
_lib . llama_batch_get_one . argtypes = [
llama_token_p ,
c_int ,
llama_pos ,
llama_seq_id ,
]
_lib . llama_batch_get_one . restype = llama_batch
2023-10-19 06:55:08 +00:00
# // Allocates a batch of tokens on the heap that can hold a maximum of n_tokens
# // Each token can be assigned up to n_seq_max sequence ids
2023-09-29 02:42:03 +00:00
# // The batch has to be freed with llama_batch_free()
# // If embd != 0, llama_batch.embd will be allocated with size of n_tokens * embd * sizeof(float)
# // Otherwise, llama_batch.token will be allocated to store n_tokens llama_token
# // The rest of the llama_batch members are allocated with size n_tokens
# // All members are left uninitialized
# LLAMA_API struct llama_batch llama_batch_init(
# int32_t n_tokens,
2023-10-19 06:55:08 +00:00
# int32_t embd,
# int32_t n_seq_max);
2023-09-29 02:42:03 +00:00
def llama_batch_init (
2023-10-19 06:55:08 +00:00
n_tokens : Union [ c_int32 , int ] ,
embd : Union [ c_int32 , int ] ,
n_seq_max : Union [ c_int32 , int ] ,
2023-09-29 02:42:03 +00:00
) - > llama_batch :
2023-11-23 05:26:26 +00:00
""" Allocates a batch of tokens on the heap that can hold a maximum of n_tokens
Each token can be assigned up to n_seq_max sequence ids
The batch has to be freed with llama_batch_free ( )
If embd != 0 , llama_batch . embd will be allocated with size of n_tokens * embd * sizeof ( float )
Otherwise , llama_batch . token will be allocated to store n_tokens llama_token
The rest of the llama_batch members are allocated with size n_tokens
All members are left uninitialized """
2023-10-19 06:55:08 +00:00
return _lib . llama_batch_init ( n_tokens , embd , n_seq_max )
2023-09-29 02:42:03 +00:00
2023-10-19 06:55:08 +00:00
_lib . llama_batch_init . argtypes = [ c_int32 , c_int32 , c_int32 ]
2023-09-29 02:42:03 +00:00
_lib . llama_batch_init . restype = llama_batch
# // Frees a batch of tokens allocated with llama_batch_init()
# LLAMA_API void llama_batch_free(struct llama_batch batch);
def llama_batch_free ( batch : llama_batch ) :
2023-11-23 05:26:26 +00:00
""" Frees a batch of tokens allocated with llama_batch_init() """
2023-09-29 02:42:03 +00:00
return _lib . llama_batch_free ( batch )
_lib . llama_batch_free . argtypes = [ llama_batch ]
_lib . llama_batch_free . restype = None
2023-08-24 04:17:00 +00:00
2023-09-29 02:42:03 +00:00
# // Positive return values does not mean a fatal error, but rather a warning.
# // 0 - success
# // 1 - could not find a KV slot for the batch (try reducing the size of the batch or increase the context)
# // < 0 - error
2024-01-04 03:04:04 +00:00
# LLAMA_API int32_t llama_decode(
2023-09-29 02:42:03 +00:00
# struct llama_context * ctx,
# struct llama_batch batch);
def llama_decode ( ctx : llama_context_p , batch : llama_batch ) - > int :
2023-11-23 05:26:26 +00:00
""" Positive return values does not mean a fatal error, but rather a warning.
0 - success
1 - could not find a KV slot for the batch ( try reducing the size of the batch or increase the context )
< 0 - error """
2023-09-29 02:42:03 +00:00
return _lib . llama_decode ( ctx , batch )
_lib . llama_decode . argtypes = [ llama_context_p , llama_batch ]
2024-01-04 03:04:04 +00:00
_lib . llama_decode . restype = c_int32
2023-09-29 02:42:03 +00:00
# // Set the number of threads used for decoding
# // n_threads is the number of threads used for generation (single token)
# // n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens)
# LLAMA_API void llama_set_n_threads(struct llama_context * ctx, uint32_t n_threads, uint32_t n_threads_batch);
def llama_set_n_threads (
ctx : llama_context_p ,
n_threads : Union [ c_uint32 , int ] ,
n_threads_batch : Union [ c_uint32 , int ] ,
) :
2023-11-23 05:26:26 +00:00
""" Set the number of threads used for decoding
n_threads is the number of threads used for generation ( single token )
2023-11-23 21:26:00 +00:00
n_threads_batch is the number of threads used for prompt and batch processing ( multiple tokens )
"""
2023-09-29 02:42:03 +00:00
return _lib . llama_set_n_threads ( ctx , n_threads , n_threads_batch )
2023-08-24 04:17:00 +00:00
2023-09-29 02:42:03 +00:00
_lib . llama_set_n_threads . argtypes = [ llama_context_p , c_uint32 , c_uint32 ]
_lib . llama_set_n_threads . restype = None
2023-08-24 04:17:00 +00:00
2023-09-29 02:42:03 +00:00
# // Token logits obtained from the last call to llama_eval()
# // The logits for the last token are stored in the last row
# // Logits for which llama_batch.logits[i] == 0 are undefined
# // Rows: n_tokens provided with llama_batch
# // Cols: n_vocab
2023-08-24 04:17:00 +00:00
# LLAMA_API float * llama_get_logits(struct llama_context * ctx);
def llama_get_logits (
2023-03-24 18:35:41 +00:00
ctx : llama_context_p ,
2023-08-24 04:17:00 +00:00
) : # type: (...) -> Array[float] # type: ignore
2023-11-23 05:26:26 +00:00
""" Token logits obtained from the last call to llama_eval()
The logits for the last token are stored in the last row
Logits for which llama_batch . logits [ i ] == 0 are undefined
Rows : n_tokens provided with llama_batch
Cols : n_vocab """
2023-08-24 04:17:00 +00:00
return _lib . llama_get_logits ( ctx )
2023-03-23 09:33:06 +00:00
2023-03-24 18:35:41 +00:00
2023-08-24 04:17:00 +00:00
_lib . llama_get_logits . argtypes = [ llama_context_p ]
_lib . llama_get_logits . restype = c_float_p
2023-03-24 18:58:42 +00:00
2023-09-29 02:42:03 +00:00
# // Logits for the ith token. Equivalent to:
# // llama_get_logits(ctx) + i*n_vocab
# LLAMA_API float * llama_get_logits_ith(struct llama_context * ctx, int32_t i);
def llama_get_logits_ith (
ctx : llama_context_p , i : Union [ c_int32 , int ]
) : # type: (...) -> Array[float] # type: ignore
2023-11-23 05:26:26 +00:00
""" Logits for the ith token. Equivalent to:
llama_get_logits ( ctx ) + i * n_vocab """
2023-09-29 02:42:03 +00:00
return _lib . llama_get_logits_ith ( ctx , i )
_lib . llama_get_logits_ith . argtypes = [ llama_context_p , c_int32 ]
_lib . llama_get_logits_ith . restype = c_float_p
2023-08-24 04:17:00 +00:00
# Get the embeddings for the input
# shape: [n_embd] (1-dimensional)
# LLAMA_API float * llama_get_embeddings(struct llama_context * ctx);
def llama_get_embeddings (
ctx : llama_context_p ,
) : # type: (...) -> Array[float] # type: ignore
2023-11-23 05:26:26 +00:00
""" Get the embeddings for the input
shape : [ n_embd ] ( 1 - dimensional ) """
2023-08-24 04:17:00 +00:00
return _lib . llama_get_embeddings ( ctx )
2023-07-15 19:11:01 +00:00
2023-08-24 04:17:00 +00:00
_lib . llama_get_embeddings . argtypes = [ llama_context_p ]
_lib . llama_get_embeddings . restype = c_float_p
2023-03-23 09:33:06 +00:00
2023-03-24 18:35:41 +00:00
2024-02-13 17:24:00 +00:00
# // Get the embeddings for the ith sequence
# // llama_get_embeddings(ctx) + i*n_embd
# LLAMA_API float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i);
def llama_get_embeddings_ith (
ctx : llama_context_p , i : Union [ c_int32 , int ]
) : # type: (...) -> Array[float] # type: ignore
""" Get the embeddings for the ith sequence
llama_get_embeddings ( ctx ) + i * n_embd """
return _lib . llama_get_embeddings_ith ( ctx , i )
_lib . llama_get_embeddings_ith . argtypes = [ llama_context_p , c_int32 ]
_lib . llama_get_embeddings_ith . restype = c_float_p
2023-08-24 04:17:00 +00:00
# //
# // Vocab
# //
2023-03-24 18:58:42 +00:00
2023-10-24 07:13:32 +00:00
# LLAMA_API const char * llama_token_get_text(const struct llama_model * model, llama_token token);
2023-11-05 21:57:10 +00:00
def llama_token_get_text ( model : llama_model_p , token : Union [ llama_token , int ] ) - > bytes :
2023-10-24 07:13:32 +00:00
return _lib . llama_token_get_text ( model , token )
2023-03-23 09:33:06 +00:00
2023-03-24 18:35:41 +00:00
2023-10-24 07:13:32 +00:00
_lib . llama_token_get_text . argtypes = [ llama_model_p , llama_token ]
2023-08-24 04:17:00 +00:00
_lib . llama_token_get_text . restype = c_char_p
2023-03-24 18:58:42 +00:00
2023-03-24 18:59:29 +00:00
2023-10-24 07:13:32 +00:00
# LLAMA_API float llama_token_get_score(const struct llama_model * model, llama_token token);
2023-11-20 19:11:33 +00:00
def llama_token_get_score (
model : llama_model_p , token : Union [ llama_token , int ]
) - > float :
2023-10-24 07:13:32 +00:00
return _lib . llama_token_get_score ( model , token )
2023-03-25 20:26:03 +00:00
2023-10-24 07:13:32 +00:00
_lib . llama_token_get_score . argtypes = [ llama_model_p , llama_token ]
2023-08-24 04:17:00 +00:00
_lib . llama_token_get_score . restype = c_float
2023-03-25 20:26:03 +00:00
2023-10-24 07:13:32 +00:00
# LLAMA_API enum llama_token_type llama_token_get_type(const struct llama_model * model, llama_token token);
2023-11-05 21:57:10 +00:00
def llama_token_get_type ( model : llama_model_p , token : Union [ llama_token , int ] ) - > int :
2023-10-24 07:13:32 +00:00
return _lib . llama_token_get_type ( model , token )
2023-07-15 19:11:01 +00:00
2023-10-24 07:13:32 +00:00
_lib . llama_token_get_type . argtypes = [ llama_model_p , llama_token ]
2023-08-24 04:17:00 +00:00
_lib . llama_token_get_type . restype = ctypes . c_int
2023-07-15 19:11:01 +00:00
2023-08-24 04:17:00 +00:00
# // Special tokens
2023-07-15 19:11:01 +00:00
2023-10-24 07:13:32 +00:00
# LLAMA_API llama_token llama_token_bos(const struct llama_model * model); // beginning-of-sentence
def llama_token_bos ( model : llama_model_p ) - > int :
2023-11-23 05:26:26 +00:00
""" beginning-of-sentence """
2023-10-24 07:13:32 +00:00
return _lib . llama_token_bos ( model )
2023-07-15 19:11:01 +00:00
2023-10-24 07:13:32 +00:00
_lib . llama_token_bos . argtypes = [ llama_model_p ]
2023-08-24 04:17:00 +00:00
_lib . llama_token_bos . restype = llama_token
2023-07-15 19:11:01 +00:00
2023-10-24 07:13:32 +00:00
# LLAMA_API llama_token llama_token_eos(const struct llama_model * model); // end-of-sentence
def llama_token_eos ( model : llama_model_p ) - > int :
2023-11-23 05:26:26 +00:00
""" end-of-sentence """
2023-10-24 07:13:32 +00:00
return _lib . llama_token_eos ( model )
2023-07-15 19:11:01 +00:00
2023-10-24 07:13:32 +00:00
_lib . llama_token_eos . argtypes = [ llama_model_p ]
2023-08-24 04:17:00 +00:00
_lib . llama_token_eos . restype = llama_token
2023-10-24 07:13:32 +00:00
# LLAMA_API llama_token llama_token_nl (const struct llama_model * model); // next-line
def llama_token_nl ( model : llama_model_p ) - > int :
2023-11-23 05:26:26 +00:00
""" next-line """
2023-10-24 07:13:32 +00:00
return _lib . llama_token_nl ( model )
2023-08-24 04:17:00 +00:00
2023-10-24 07:13:32 +00:00
_lib . llama_token_nl . argtypes = [ llama_model_p ]
2023-08-24 04:17:00 +00:00
_lib . llama_token_nl . restype = llama_token
2023-11-20 19:11:33 +00:00
# // Returns -1 if unknown, 1 for true or 0 for false.
2024-01-04 03:04:04 +00:00
# LLAMA_API int32_t llama_add_bos_token(const struct llama_model * model);
2023-11-20 19:11:33 +00:00
def llama_add_bos_token ( model : llama_model_p ) - > int :
2023-11-23 05:26:26 +00:00
""" Returns -1 if unknown, 1 for true or 0 for false. """
2023-11-20 19:11:33 +00:00
return _lib . llama_add_bos_token ( model )
_lib . llama_add_bos_token . argtypes = [ llama_model_p ]
2024-01-04 03:04:04 +00:00
_lib . llama_add_bos_token . restype = c_int32
2023-11-20 19:11:33 +00:00
# // Returns -1 if unknown, 1 for true or 0 for false.
2024-01-04 03:04:04 +00:00
# LLAMA_API int32_t llama_add_eos_token(const struct llama_model * model);
2023-11-20 19:11:33 +00:00
def llama_add_eos_token ( model : llama_model_p ) - > int :
2023-11-23 05:26:26 +00:00
""" Returns -1 if unknown, 1 for true or 0 for false. """
2023-11-20 19:11:33 +00:00
return _lib . llama_add_eos_token ( model )
_lib . llama_add_eos_token . argtypes = [ llama_model_p ]
2024-01-04 03:04:04 +00:00
_lib . llama_add_eos_token . restype = c_int32
2023-11-20 19:11:33 +00:00
2023-10-03 19:23:35 +00:00
# // codellama infill tokens
2023-10-24 07:13:32 +00:00
# LLAMA_API llama_token llama_token_prefix(const struct llama_model * model); // Beginning of infill prefix
def llama_token_prefix ( model : llama_model_p ) - > int :
2023-11-23 05:26:26 +00:00
""" codellama infill tokens """
2023-10-24 07:13:32 +00:00
return _lib . llama_token_prefix ( model )
2023-10-03 19:23:35 +00:00
2023-10-24 07:13:32 +00:00
_lib . llama_token_prefix . argtypes = [ llama_model_p ]
2023-10-03 19:23:35 +00:00
_lib . llama_token_prefix . restype = llama_token
2023-10-24 07:13:32 +00:00
# LLAMA_API llama_token llama_token_middle(const struct llama_model * model); // Beginning of infill middle
def llama_token_middle ( model : llama_model_p ) - > int :
return _lib . llama_token_middle ( model )
2023-10-03 19:23:35 +00:00
2023-10-24 07:13:32 +00:00
_lib . llama_token_middle . argtypes = [ llama_model_p ]
2023-10-03 19:23:35 +00:00
_lib . llama_token_middle . restype = llama_token
2023-10-24 07:13:32 +00:00
# LLAMA_API llama_token llama_token_suffix(const struct llama_model * model); // Beginning of infill suffix
def llama_token_suffix ( model : llama_model_p ) - > int :
return _lib . llama_token_suffix ( model )
2023-10-03 19:23:35 +00:00
2023-10-24 07:13:32 +00:00
_lib . llama_token_suffix . argtypes = [ llama_model_p ]
2023-10-03 19:23:35 +00:00
_lib . llama_token_suffix . restype = llama_token
2023-10-24 07:13:32 +00:00
# LLAMA_API llama_token llama_token_eot (const struct llama_model * model); // End of infill middle
def llama_token_eot ( model : llama_model_p ) - > int :
return _lib . llama_token_eot ( model )
2023-10-03 19:23:35 +00:00
2023-10-24 07:13:32 +00:00
_lib . llama_token_eot . argtypes = [ llama_model_p ]
2023-10-03 19:23:35 +00:00
_lib . llama_token_eot . restype = llama_token
2023-08-24 04:17:00 +00:00
# //
# // Tokenization
# //
2023-10-19 06:55:08 +00:00
# /// @details Convert the provided text into tokens.
# /// @param tokens The tokens pointer must be large enough to hold the resulting tokens.
# /// @return Returns the number of tokens on success, no more than n_max_tokens
# /// @return Returns a negative number on failure - the number of tokens that would have been returned
# /// @param special Allow tokenizing special and/or control tokens which otherwise are not exposed and treated as plaintext.
# /// Does not insert a leading space.
2024-01-04 03:04:04 +00:00
# LLAMA_API int32_t llama_tokenize(
2023-10-19 06:55:08 +00:00
# const struct llama_model * model,
# const char * text,
2024-01-04 03:04:04 +00:00
# int32_t text_len,
2023-10-19 06:55:08 +00:00
# llama_token * tokens,
2024-01-04 03:04:04 +00:00
# int32_t n_max_tokens,
2023-10-19 06:55:08 +00:00
# bool add_bos,
# bool special);
def llama_tokenize (
model : llama_model_p ,
text : bytes ,
text_len : Union [ c_int , int ] ,
tokens , # type: Array[llama_token]
n_max_tokens : Union [ c_int , int ] ,
add_bos : Union [ c_bool , bool ] ,
special : Union [ c_bool , bool ] ,
) - > int :
2023-11-23 05:26:26 +00:00
""" Convert the provided text into tokens. """
2023-10-19 06:55:08 +00:00
return _lib . llama_tokenize (
model , text , text_len , tokens , n_max_tokens , add_bos , special
)
_lib . llama_tokenize . argtypes = [
llama_model_p ,
c_char_p ,
2024-01-04 03:04:04 +00:00
c_int32 ,
2023-10-19 06:55:08 +00:00
llama_token_p ,
2024-01-04 03:04:04 +00:00
c_int32 ,
2023-10-19 06:55:08 +00:00
c_bool ,
c_bool ,
]
2024-01-04 03:04:04 +00:00
_lib . llama_tokenize . restype = c_int32
2023-10-19 06:55:08 +00:00
2023-08-27 16:59:20 +00:00
# // Token Id -> Piece.
# // Uses the vocabulary in the provided context.
# // Does not write null terminator to the buffer.
# // User code is responsible to remove the leading whitespace of the first non-BOS token when decoding multiple tokens.
2024-01-04 03:04:04 +00:00
# LLAMA_API int32_t llama_token_to_piece(
2023-09-29 02:42:03 +00:00
# const struct llama_model * model,
# llama_token token,
# char * buf,
2024-01-04 03:04:04 +00:00
# int32_t length);
2023-08-27 16:59:20 +00:00
def llama_token_to_piece (
2023-09-29 02:42:03 +00:00
model : llama_model_p ,
2023-11-05 21:57:10 +00:00
token : Union [ llama_token , int ] ,
2023-09-29 02:42:03 +00:00
buf : Union [ c_char_p , bytes ] ,
length : Union [ c_int , int ] ,
2023-08-24 04:17:00 +00:00
) - > int :
2023-11-23 05:26:26 +00:00
""" Token Id -> Piece.
Uses the vocabulary in the provided context .
Does not write null terminator to the buffer .
2023-11-23 21:26:00 +00:00
User code is responsible to remove the leading whitespace of the first non - BOS token when decoding multiple tokens .
"""
2023-09-29 02:42:03 +00:00
return _lib . llama_token_to_piece ( model , token , buf , length )
2023-03-24 18:58:42 +00:00
2023-03-24 18:59:29 +00:00
2024-01-04 03:04:04 +00:00
_lib . llama_token_to_piece . argtypes = [ llama_model_p , llama_token , c_char_p , c_int32 ]
_lib . llama_token_to_piece . restype = c_int32
2023-03-24 18:58:42 +00:00
2023-03-24 18:59:29 +00:00
2023-08-24 04:17:00 +00:00
# //
2023-07-24 17:04:34 +00:00
# // Grammar
# //
2023-08-24 04:17:00 +00:00
2023-07-24 17:04:34 +00:00
# LLAMA_API struct llama_grammar * llama_grammar_init(
# const llama_grammar_element ** rules,
# size_t n_rules,
# size_t start_rule_index);
def llama_grammar_init (
rules , # type: Array[llama_grammar_element_p] # type: ignore
2023-07-24 19:42:07 +00:00
n_rules : Union [ c_size_t , int ] ,
start_rule_index : Union [ c_size_t , int ] ,
2023-07-24 17:04:34 +00:00
) - > llama_grammar_p :
2023-11-28 00:03:02 +00:00
""" Initialize a grammar from a set of rules. """
2023-07-24 17:04:34 +00:00
return _lib . llama_grammar_init ( rules , n_rules , start_rule_index )
_lib . llama_grammar_init . argtypes = [
POINTER ( llama_grammar_element_p ) ,
c_size_t ,
c_size_t ,
]
_lib . llama_grammar_init . restype = llama_grammar_p
# LLAMA_API void llama_grammar_free(struct llama_grammar * grammar);
def llama_grammar_free ( grammar : llama_grammar_p ) :
2023-11-28 00:03:02 +00:00
""" Free a grammar. """
2023-07-24 17:04:34 +00:00
return _lib . llama_grammar_free ( grammar )
_lib . llama_grammar_free . argtypes = [ llama_grammar_p ]
_lib . llama_grammar_free . restype = None
2023-08-25 18:35:53 +00:00
2023-09-09 16:12:32 +00:00
# LLAMA_API struct llama_grammar * llama_grammar_copy(const struct llama_grammar * grammar);
def llama_grammar_copy ( grammar : llama_grammar_p ) - > llama_grammar_p :
2023-11-28 00:03:02 +00:00
""" Copy a grammar. """
2023-09-09 16:12:32 +00:00
return _lib . llama_grammar_copy ( grammar )
2023-08-25 18:35:53 +00:00
2023-09-09 16:12:32 +00:00
_lib . llama_grammar_copy . argtypes = [ llama_grammar_p ]
_lib . llama_grammar_copy . restype = llama_grammar_p
2023-07-24 17:04:34 +00:00
2023-08-24 04:17:00 +00:00
# //
# // Sampling functions
# //
2023-05-01 18:02:06 +00:00
2023-09-29 02:42:03 +00:00
# // Sets the current rng seed.
# LLAMA_API void llama_set_rng_seed(struct llama_context * ctx, uint32_t seed);
def llama_set_rng_seed ( ctx : llama_context_p , seed : Union [ c_uint32 , int ] ) :
2023-11-23 05:26:26 +00:00
""" Sets the current rng seed. """
2023-09-29 02:42:03 +00:00
return _lib . llama_set_rng_seed ( ctx , seed )
_lib . llama_set_rng_seed . argtypes = [ llama_context_p , c_uint32 ]
_lib . llama_set_rng_seed . restype = None
2023-10-24 07:13:32 +00:00
# /// @details Repetition penalty described in CTRL academic paper https://arxiv.org/abs/1909.05858, with negative logit fix.
# /// @details Frequency and presence penalties described in OpenAI API https://platform.openai.com/docs/api-reference/parameter-details.
# LLAMA_API void llama_sample_repetition_penalties(
2023-09-29 02:42:03 +00:00
# struct llama_context * ctx,
# llama_token_data_array * candidates,
# const llama_token * last_tokens,
2023-10-24 07:13:32 +00:00
# size_t penalty_last_n,
# float penalty_repeat,
# float penalty_freq,
# float penalty_present);
def llama_sample_repetition_penalties (
2023-05-01 14:44:28 +00:00
ctx : llama_context_p ,
2023-05-05 18:12:26 +00:00
candidates , # type: _Pointer[llama_token_data_array]
last_tokens_data , # type: Array[llama_token]
2023-10-24 07:13:32 +00:00
penalty_last_n : Union [ c_size_t , int ] ,
penalty_repeat : Union [ c_float , float ] ,
penalty_freq : Union [ c_float , float ] ,
penalty_present : Union [ c_float , float ] ,
2023-05-01 18:02:06 +00:00
) :
2023-11-23 05:26:26 +00:00
""" Repetition penalty described in CTRL academic paper https://arxiv.org/abs/1909.05858, with negative logit fix.
2023-11-23 21:26:00 +00:00
Frequency and presence penalties described in OpenAI API https : / / platform . openai . com / docs / api - reference / parameter - details .
"""
2023-10-24 07:13:32 +00:00
return _lib . llama_sample_repetition_penalties (
2023-05-01 14:44:28 +00:00
ctx ,
candidates ,
last_tokens_data ,
2023-10-24 07:13:32 +00:00
penalty_last_n ,
penalty_repeat ,
penalty_freq ,
penalty_present ,
2023-03-24 18:35:41 +00:00
)
2023-03-23 09:33:06 +00:00
2023-10-24 07:13:32 +00:00
_lib . llama_sample_repetition_penalties . argtypes = [
2023-03-24 18:58:42 +00:00
llama_context_p ,
2023-05-01 14:44:28 +00:00
llama_token_data_array_p ,
2023-03-24 18:58:42 +00:00
llama_token_p ,
2023-10-24 07:13:32 +00:00
c_size_t ,
c_float ,
2023-05-01 14:44:28 +00:00
c_float ,
c_float ,
]
2023-10-24 07:13:32 +00:00
_lib . llama_sample_repetition_penalties . restype = None
2023-05-01 14:44:28 +00:00
2023-07-15 19:11:01 +00:00
# /// @details Apply classifier-free guidance to the logits as described in academic paper "Stay on topic with Classifier-Free Guidance" https://arxiv.org/abs/2306.17806
2024-01-15 15:12:10 +00:00
# /// @param logits Logits extracted from the original generation context.
# /// @param logits_guidance Logits extracted from a separate context from the same model. Other than a negative prompt at the beginning, it should have all generated and user input tokens copied from the main context.
# /// @param scale Guidance strength. 1.0f means no guidance. Higher values mean stronger guidance.
# LLAMA_API void llama_sample_apply_guidance(
# struct llama_context * ctx,
# float * logits,
# float * logits_guidance,
# float scale);
def llama_sample_apply_guidance (
ctx : llama_context_p ,
logits , # type: _Pointer[c_float]
logits_guidance , # type: _Pointer[c_float]
scale : Union [ c_float , float ] ,
) :
""" Apply classifier-free guidance to the logits as described in academic paper " Stay on topic with Classifier-Free Guidance " https://arxiv.org/abs/2306.17806 """
return _lib . llama_sample_apply_guidance ( ctx , logits , logits_guidance , scale )
_lib . llama_sample_apply_guidance . argtypes = [
llama_context_p ,
c_float_p ,
c_float_p ,
c_float ,
]
_lib . llama_sample_apply_guidance . restype = None
# LLAMA_API DEPRECATED(void llama_sample_classifier_free_guidance(
# struct llama_context * ctx,
2023-07-15 19:11:01 +00:00
# llama_token_data_array * candidates,
2024-01-15 15:12:10 +00:00
# struct llama_context * guidance_ctx,
# float scale),
# "use llama_sample_apply_guidance() instead");
2023-07-15 19:11:01 +00:00
def llama_sample_classifier_free_guidance (
ctx : llama_context_p ,
candidates , # type: _Pointer[llama_token_data_array]
guidance_ctx : llama_context_p ,
2023-07-24 19:42:07 +00:00
scale : Union [ c_float , float ] ,
2023-07-15 19:11:01 +00:00
) :
2023-11-23 05:26:26 +00:00
""" Apply classifier-free guidance to the logits as described in academic paper " Stay on topic with Classifier-Free Guidance " https://arxiv.org/abs/2306.17806 """
2023-07-15 19:11:01 +00:00
return _lib . llama_sample_classifier_free_guidance (
2023-07-21 16:33:27 +00:00
ctx , candidates , guidance_ctx , scale
2023-07-15 19:11:01 +00:00
)
_lib . llama_sample_classifier_free_guidance . argtypes = [
llama_context_p ,
llama_token_data_array_p ,
llama_context_p ,
c_float ,
]
_lib . llama_sample_classifier_free_guidance . restype = None
2023-11-01 01:29:35 +00:00
# /// @details Sorts candidate tokens by their logits in descending order and calculate probabilities based on logits.
2023-09-29 02:42:03 +00:00
# LLAMA_API void llama_sample_softmax(
# struct llama_context * ctx,
# llama_token_data_array * candidates);
2023-05-05 18:12:26 +00:00
def llama_sample_softmax (
ctx : llama_context_p , candidates # type: _Pointer[llama_token_data]
) :
2023-11-23 05:26:26 +00:00
""" Sorts candidate tokens by their logits in descending order and calculate probabilities based on logits. """
2023-05-01 14:44:28 +00:00
return _lib . llama_sample_softmax ( ctx , candidates )
_lib . llama_sample_softmax . argtypes = [
llama_context_p ,
llama_token_data_array_p ,
]
2023-05-01 18:02:06 +00:00
_lib . llama_sample_softmax . restype = None
2023-05-01 14:44:28 +00:00
2023-11-01 01:29:35 +00:00
# /// @details Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751
2023-09-29 02:42:03 +00:00
# LLAMA_API void llama_sample_top_k(
# struct llama_context * ctx,
# llama_token_data_array * candidates,
2024-01-04 03:04:04 +00:00
# int32_t k,
2023-09-29 02:42:03 +00:00
# size_t min_keep);
2023-05-01 18:47:55 +00:00
def llama_sample_top_k (
2023-05-05 17:54:22 +00:00
ctx : llama_context_p ,
2023-05-05 18:12:26 +00:00
candidates , # type: _Pointer[llama_token_data_array]
2023-07-24 19:42:07 +00:00
k : Union [ c_int , int ] ,
min_keep : Union [ c_size_t , int ] ,
2023-05-01 18:47:55 +00:00
) :
2023-11-23 05:26:26 +00:00
""" Top-K sampling described in academic paper " The Curious Case of Neural Text Degeneration " https://arxiv.org/abs/1904.09751 """
2023-05-01 14:44:28 +00:00
return _lib . llama_sample_top_k ( ctx , candidates , k , min_keep )
_lib . llama_sample_top_k . argtypes = [
llama_context_p ,
llama_token_data_array_p ,
2024-01-04 03:04:04 +00:00
c_int32 ,
2023-05-01 18:47:55 +00:00
c_size_t ,
2023-05-01 14:44:28 +00:00
]
2023-05-01 18:02:06 +00:00
_lib . llama_sample_top_k . restype = None
2023-05-01 14:44:28 +00:00
2023-11-01 01:29:35 +00:00
# /// @details Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751
2023-09-29 02:42:03 +00:00
# LLAMA_API void llama_sample_top_p(
# struct llama_context * ctx,
# llama_token_data_array * candidates,
# float p,
# size_t min_keep);
2023-05-01 18:47:55 +00:00
def llama_sample_top_p (
2023-05-05 17:54:22 +00:00
ctx : llama_context_p ,
2023-05-05 18:12:26 +00:00
candidates , # type: _Pointer[llama_token_data_array]
2023-07-24 19:42:07 +00:00
p : Union [ c_float , float ] ,
min_keep : Union [ c_size_t , int ] ,
2023-05-01 18:47:55 +00:00
) :
2023-11-23 05:26:26 +00:00
""" Nucleus sampling described in academic paper " The Curious Case of Neural Text Degeneration " https://arxiv.org/abs/1904.09751 """
2023-05-01 14:44:28 +00:00
return _lib . llama_sample_top_p ( ctx , candidates , p , min_keep )
_lib . llama_sample_top_p . argtypes = [
llama_context_p ,
llama_token_data_array_p ,
c_float ,
2023-05-01 18:47:55 +00:00
c_size_t ,
2023-05-01 14:44:28 +00:00
]
2023-05-01 18:02:06 +00:00
_lib . llama_sample_top_p . restype = None
2023-05-01 14:44:28 +00:00
2023-11-01 01:29:35 +00:00
# /// @details Minimum P sampling as described in https://github.com/ggerganov/llama.cpp/pull/3841
# LLAMA_API void llama_sample_min_p(
# struct llama_context * ctx,
# llama_token_data_array * candidates,
# float p,
# size_t min_keep);
def llama_sample_min_p (
ctx : llama_context_p ,
candidates , # type: _Pointer[llama_token_data_array]
p : Union [ c_float , float ] ,
min_keep : Union [ c_size_t , int ] ,
) :
2023-11-23 05:26:26 +00:00
""" Minimum P sampling as described in https://github.com/ggerganov/llama.cpp/pull/3841 """
2023-11-01 01:29:35 +00:00
return _lib . llama_sample_min_p ( ctx , candidates , p , min_keep )
_lib . llama_sample_min_p . argtypes = [
llama_context_p ,
llama_token_data_array_p ,
c_float ,
c_size_t ,
]
_lib . llama_sample_min_p . restype = None
# /// @details Tail Free Sampling described in https://www.trentonbricken.com/Tail-Free-Sampling/.
2023-09-29 02:42:03 +00:00
# LLAMA_API void llama_sample_tail_free(
# struct llama_context * ctx,
# llama_token_data_array * candidates,
# float z,
# size_t min_keep);
2023-05-01 14:44:28 +00:00
def llama_sample_tail_free (
2023-05-05 16:22:27 +00:00
ctx : llama_context_p ,
2023-05-05 18:12:26 +00:00
candidates , # type: _Pointer[llama_token_data_array]
2023-07-24 19:42:07 +00:00
z : Union [ c_float , float ] ,
min_keep : Union [ c_size_t , int ] ,
2023-05-01 18:02:06 +00:00
) :
2023-11-23 05:26:26 +00:00
""" Tail Free Sampling described in https://www.trentonbricken.com/Tail-Free-Sampling/. """
2023-05-01 14:44:28 +00:00
return _lib . llama_sample_tail_free ( ctx , candidates , z , min_keep )
_lib . llama_sample_tail_free . argtypes = [
llama_context_p ,
llama_token_data_array_p ,
c_float ,
2023-05-01 18:47:55 +00:00
c_size_t ,
2023-05-01 14:44:28 +00:00
]
2023-05-01 18:02:06 +00:00
_lib . llama_sample_tail_free . restype = None
2023-05-01 14:44:28 +00:00
2023-11-01 01:29:35 +00:00
# /// @details Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666.
2023-09-29 02:42:03 +00:00
# LLAMA_API void llama_sample_typical(
# struct llama_context * ctx,
# llama_token_data_array * candidates,
# float p,
# size_t min_keep);
2023-05-01 18:47:55 +00:00
def llama_sample_typical (
2023-05-05 16:22:27 +00:00
ctx : llama_context_p ,
2023-05-05 18:12:26 +00:00
candidates , # type: _Pointer[llama_token_data_array]
2023-07-24 19:42:07 +00:00
p : Union [ c_float , float ] ,
min_keep : Union [ c_size_t , int ] ,
2023-05-01 18:47:55 +00:00
) :
2023-11-23 05:26:26 +00:00
""" Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666. """
2023-05-01 14:44:28 +00:00
return _lib . llama_sample_typical ( ctx , candidates , p , min_keep )
_lib . llama_sample_typical . argtypes = [
llama_context_p ,
llama_token_data_array_p ,
c_float ,
2023-05-01 18:47:55 +00:00
c_size_t ,
2023-05-01 14:44:28 +00:00
]
2023-05-01 18:02:06 +00:00
_lib . llama_sample_typical . restype = None
2023-05-01 14:44:28 +00:00
2024-01-26 16:45:48 +00:00
# /// @details Dynamic temperature implementation described in the paper https://arxiv.org/abs/2309.02772.
# LLAMA_API void llama_sample_entropy(
# struct llama_context * ctx,
# llama_token_data_array * candidates_p,
# float min_temp,
# float max_temp,
# float exponent_val);
def llama_sample_entropy (
ctx : llama_context_p ,
candidates , # type: _Pointer[llama_token_data_array]
min_temp : Union [ c_float , float ] ,
max_temp : Union [ c_float , float ] ,
exponent_val : Union [ c_float , float ] ,
) :
""" Dynamic temperature implementation described in the paper https://arxiv.org/abs/2309.02772. """
return _lib . llama_sample_entropy ( ctx , candidates , min_temp , max_temp , exponent_val )
_lib . llama_sample_entropy . argtypes = [
llama_context_p ,
llama_token_data_array_p ,
c_float ,
c_float ,
c_float ,
]
_lib . llama_sample_entropy . restype = None
2023-09-29 02:42:03 +00:00
# LLAMA_API void llama_sample_temp(
# struct llama_context * ctx,
# llama_token_data_array * candidates,
# float temp);
def llama_sample_temp (
ctx : llama_context_p ,
candidates , # type: _Pointer[llama_token_data_array]
temp : Union [ c_float , float ] ,
) :
2023-11-28 00:03:02 +00:00
""" Temperature sampling described in academic paper " Generating Long Sequences with Sparse Transformers " https://arxiv.org/abs/1904.10509
2024-01-08 19:51:29 +00:00
2023-11-28 00:03:02 +00:00
Parameters :
candidates : A vector of ` llama_token_data ` containing the candidate tokens , their probabilities ( p ) , and log - odds ( logit ) for the current position in the generated text .
2024-01-08 19:51:29 +00:00
temp : The temperature value to use for the sampling . A higher value corresponds to more surprising or less predictable text , while a lower value corresponds to less surprising or more predictable text .
"""
2023-09-29 02:42:03 +00:00
return _lib . llama_sample_temp ( ctx , candidates , temp )
_lib . llama_sample_temp . argtypes = [
llama_context_p ,
llama_token_data_array_p ,
c_float ,
]
_lib . llama_sample_temp . restype = None
# LLAMA_API DEPRECATED(void llama_sample_temperature(
# struct llama_context * ctx,
# llama_token_data_array * candidates,
# float temp),
# "use llama_sample_temp instead");
2023-05-05 16:22:27 +00:00
def llama_sample_temperature (
2023-05-05 18:12:26 +00:00
ctx : llama_context_p ,
candidates , # type: _Pointer[llama_token_data_array]
2023-07-24 19:42:07 +00:00
temp : Union [ c_float , float ] ,
2023-05-05 16:22:27 +00:00
) :
2023-11-28 00:03:02 +00:00
""" use llama_sample_temp instead """
2023-05-01 14:44:28 +00:00
return _lib . llama_sample_temperature ( ctx , candidates , temp )
_lib . llama_sample_temperature . argtypes = [
llama_context_p ,
llama_token_data_array_p ,
2023-03-29 01:10:23 +00:00
c_float ,
2023-05-01 14:44:28 +00:00
]
2023-05-01 18:02:06 +00:00
_lib . llama_sample_temperature . restype = None
2023-05-01 14:44:28 +00:00
2023-11-01 01:29:35 +00:00
# /// @details Apply constraints from grammar
# LLAMA_API void llama_sample_grammar(
# struct llama_context * ctx,
# llama_token_data_array * candidates,
# const struct llama_grammar * grammar);
2023-07-24 19:42:31 +00:00
def llama_sample_grammar (
ctx : llama_context_p ,
candidates , # type: _Pointer[llama_token_data_array]
2023-08-05 05:43:35 +00:00
grammar , # type: llama_grammar_p
2023-07-24 19:42:31 +00:00
) :
2023-11-28 00:03:02 +00:00
""" Apply constraints from grammar
2024-01-08 19:51:29 +00:00
2023-11-28 00:03:02 +00:00
Parameters :
candidates : A vector of ` llama_token_data ` containing the candidate tokens , their probabilities ( p ) , and log - odds ( logit ) for the current position in the generated text .
2024-01-08 19:51:29 +00:00
grammar : A grammar object containing the rules and constraints to apply to the generated text .
"""
2023-07-24 19:42:31 +00:00
return _lib . llama_sample_grammar ( ctx , candidates , grammar )
_lib . llama_sample_grammar . argtypes = [
llama_context_p ,
llama_token_data_array_p ,
llama_grammar_p ,
]
_lib . llama_sample_grammar . restype = None
2023-11-01 01:29:35 +00:00
# /// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words.
# /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text.
# /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text.
# /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates.
# /// @param m The number of tokens considered in the estimation of `s_hat`. This is an arbitrary value that is used to calculate `s_hat`, which in turn helps to calculate the value of `k`. In the paper, they use `m = 100`, but you can experiment with different values to see how it affects the performance of the algorithm.
# /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal.
2023-09-29 02:42:03 +00:00
# LLAMA_API llama_token llama_sample_token_mirostat(
# struct llama_context * ctx,
# llama_token_data_array * candidates,
# float tau,
# float eta,
2024-01-04 03:04:04 +00:00
# int32_t m,
2023-09-29 02:42:03 +00:00
# float * mu);
2023-05-01 14:44:28 +00:00
def llama_sample_token_mirostat (
2023-05-05 16:22:27 +00:00
ctx : llama_context_p ,
2023-05-05 18:12:26 +00:00
candidates , # type: _Pointer[llama_token_data_array]
2023-07-24 19:42:07 +00:00
tau : Union [ c_float , float ] ,
eta : Union [ c_float , float ] ,
m : Union [ c_int , int ] ,
2023-05-05 18:12:26 +00:00
mu , # type: _Pointer[c_float]
2023-05-19 15:59:33 +00:00
) - > int :
2023-11-28 00:03:02 +00:00
""" Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words.
2024-01-08 19:51:29 +00:00
2023-11-28 00:03:02 +00:00
Parameters :
candidates : A vector of ` llama_token_data ` containing the candidate tokens , their probabilities ( p ) , and log - odds ( logit ) for the current position in the generated text .
tau : The target cross - entropy ( or surprise ) value you want to achieve for the generated text . A higher value corresponds to more surprising or less predictable text , while a lower value corresponds to less surprising or more predictable text .
eta : The learning rate used to update ` mu ` based on the error between the target and observed surprisal of the sampled word . A larger learning rate will cause ` mu ` to be updated more quickly , while a smaller learning rate will result in slower updates .
m : The number of tokens considered in the estimation of ` s_hat ` . This is an arbitrary value that is used to calculate ` s_hat ` , which in turn helps to calculate the value of ` k ` . In the paper , they use ` m = 100 ` , but you can experiment with different values to see how it affects the performance of the algorithm .
2024-01-08 19:51:29 +00:00
mu : Maximum cross - entropy . This value is initialized to be twice the target cross - entropy ( ` 2 * tau ` ) and is updated in the algorithm based on the error between the target and observed surprisal .
"""
2023-05-01 14:44:28 +00:00
return _lib . llama_sample_token_mirostat ( ctx , candidates , tau , eta , m , mu )
_lib . llama_sample_token_mirostat . argtypes = [
llama_context_p ,
llama_token_data_array_p ,
c_float ,
c_float ,
2024-01-04 03:04:04 +00:00
c_int32 ,
2023-05-05 18:04:12 +00:00
c_float_p ,
2023-05-01 14:44:28 +00:00
]
_lib . llama_sample_token_mirostat . restype = llama_token
2023-11-01 01:29:35 +00:00
# /// @details Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words.
# /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text.
# /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text.
# /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates.
# /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal.
2023-09-29 02:42:03 +00:00
# LLAMA_API llama_token llama_sample_token_mirostat_v2(
# struct llama_context * ctx,
# llama_token_data_array * candidates,
# float tau,
# float eta,
# float * mu);
2023-05-01 14:44:28 +00:00
def llama_sample_token_mirostat_v2 (
2023-05-05 16:22:27 +00:00
ctx : llama_context_p ,
2023-05-05 18:12:26 +00:00
candidates , # type: _Pointer[llama_token_data_array]
2023-07-24 19:42:07 +00:00
tau : Union [ c_float , float ] ,
eta : Union [ c_float , float ] ,
2023-05-05 18:12:26 +00:00
mu , # type: _Pointer[c_float]
2023-05-19 15:59:33 +00:00
) - > int :
2023-11-28 00:03:02 +00:00
""" Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words.
2024-01-08 19:51:29 +00:00
2023-11-28 00:03:02 +00:00
Parameters :
candidates : A vector of ` llama_token_data ` containing the candidate tokens , their probabilities ( p ) , and log - odds ( logit ) for the current position in the generated text .
tau : The target cross - entropy ( or surprise ) value you want to achieve for the generated text . A higher value corresponds to more surprising or less predictable text , while a lower value corresponds to less surprising or more predictable text .
eta : The learning rate used to update ` mu ` based on the error between the target and observed surprisal of the sampled word . A larger learning rate will cause ` mu ` to be updated more quickly , while a smaller learning rate will result in slower updates .
2024-01-08 19:51:29 +00:00
mu : Maximum cross - entropy . This value is initialized to be twice the target cross - entropy ( ` 2 * tau ` ) and is updated in the algorithm based on the error between the target and observed surprisal .
"""
2023-05-01 14:44:28 +00:00
return _lib . llama_sample_token_mirostat_v2 ( ctx , candidates , tau , eta , mu )
_lib . llama_sample_token_mirostat_v2 . argtypes = [
llama_context_p ,
llama_token_data_array_p ,
2023-03-29 01:10:23 +00:00
c_float ,
c_float ,
2023-05-05 18:04:12 +00:00
c_float_p ,
2023-05-01 14:44:28 +00:00
]
_lib . llama_sample_token_mirostat_v2 . restype = llama_token
2023-11-01 01:29:35 +00:00
# /// @details Selects the token with the highest probability.
# /// Does not compute the token probabilities. Use llama_sample_softmax() instead.
2023-09-29 02:42:03 +00:00
# LLAMA_API llama_token llama_sample_token_greedy(
# struct llama_context * ctx,
# llama_token_data_array * candidates);
2023-05-05 16:22:27 +00:00
def llama_sample_token_greedy (
2023-05-05 18:12:26 +00:00
ctx : llama_context_p ,
candidates , # type: _Pointer[llama_token_data_array]
2023-05-19 15:59:33 +00:00
) - > int :
2023-11-28 00:03:02 +00:00
""" Selects the token with the highest probability. """
2023-05-01 14:44:28 +00:00
return _lib . llama_sample_token_greedy ( ctx , candidates )
_lib . llama_sample_token_greedy . argtypes = [
llama_context_p ,
llama_token_data_array_p ,
]
_lib . llama_sample_token_greedy . restype = llama_token
2023-11-01 01:29:35 +00:00
# /// @details Randomly selects a token from the candidates based on their probabilities.
2023-09-29 02:42:03 +00:00
# LLAMA_API llama_token llama_sample_token(
# struct llama_context * ctx,
# llama_token_data_array * candidates);
2023-05-05 16:22:27 +00:00
def llama_sample_token (
2023-05-05 18:12:26 +00:00
ctx : llama_context_p ,
candidates , # type: _Pointer[llama_token_data_array]
2023-05-19 15:59:33 +00:00
) - > int :
2023-11-28 00:03:02 +00:00
""" Randomly selects a token from the candidates based on their probabilities. """
2023-05-01 14:44:28 +00:00
return _lib . llama_sample_token ( ctx , candidates )
_lib . llama_sample_token . argtypes = [
llama_context_p ,
llama_token_data_array_p ,
2023-03-24 18:58:42 +00:00
]
2023-05-01 14:44:28 +00:00
_lib . llama_sample_token . restype = llama_token
2023-03-24 18:58:42 +00:00
2023-07-24 19:55:26 +00:00
# /// @details Accepts the sampled token into the grammar
2023-09-29 02:42:03 +00:00
# LLAMA_API void llama_grammar_accept_token(
# struct llama_context * ctx,
# struct llama_grammar * grammar,
# llama_token token);
2023-07-24 19:55:26 +00:00
def llama_grammar_accept_token (
2023-08-05 05:43:35 +00:00
ctx : llama_context_p ,
grammar : llama_grammar_p ,
2023-11-05 21:57:10 +00:00
token : Union [ llama_token , int ] ,
2023-08-05 05:43:35 +00:00
) - > None :
2023-11-28 00:03:02 +00:00
""" Accepts the sampled token into the grammar """
2023-08-05 05:43:35 +00:00
_lib . llama_grammar_accept_token ( ctx , grammar , token )
2023-07-24 19:55:26 +00:00
_lib . llama_grammar_accept_token . argtypes = [
llama_context_p ,
llama_grammar_p ,
llama_token ,
]
_lib . llama_grammar_accept_token . restype = None
2023-09-29 02:42:03 +00:00
2023-09-09 16:12:32 +00:00
# //
# // Beam search
# //
# struct llama_beam_view {
# const llama_token * tokens;
2023-09-29 02:42:03 +00:00
2023-10-03 19:23:35 +00:00
2023-09-09 16:12:32 +00:00
# size_t n_tokens;
2023-09-29 02:42:03 +00:00
# float p; // Cumulative beam probability (renormalized relative to all beams)
# bool eob; // Callback should set this to true when a beam is at end-of-beam.
2023-09-09 16:12:32 +00:00
# };
class llama_beam_view ( ctypes . Structure ) :
_fields_ = [
( " tokens " , llama_token_p ) ,
( " n_tokens " , c_size_t ) ,
( " p " , c_float ) ,
( " eob " , c_bool ) ,
]
# // Passed to beam_search_callback function.
# // Whenever 0 < common_prefix_length, this number of tokens should be copied from any of the beams
# // (e.g. beams[0]) as they will be removed (shifted) from all beams in all subsequent callbacks.
# // These pointers are valid only during the synchronous callback, so should not be saved.
# struct llama_beams_state {
# struct llama_beam_view * beam_views;
# size_t n_beams; // Number of elements in beam_views[].
# size_t common_prefix_length; // Current max length of prefix tokens shared by all beams.
2023-09-29 02:42:03 +00:00
# bool last_call; // True iff this is the last callback invocation.
2023-09-09 16:12:32 +00:00
# };
class llama_beams_state ( ctypes . Structure ) :
_fields_ = [
( " beam_views " , POINTER ( llama_beam_view ) ) ,
( " n_beams " , c_size_t ) ,
( " common_prefix_length " , c_size_t ) ,
( " last_call " , c_bool ) ,
]
# // Type of pointer to the beam_search_callback function.
# // void* callback_data is any custom data passed to llama_beam_search, that is subsequently
# // passed back to beam_search_callback. This avoids having to use global variables in the callback.
# typedef void (*llama_beam_search_callback_fn_t)(void * callback_data, struct llama_beams_state);
llama_beam_search_callback_fn_t = ctypes . CFUNCTYPE ( None , c_void_p , llama_beams_state )
# /// @details Deterministically returns entire sentence constructed by a beam search.
# /// @param ctx Pointer to the llama_context.
# /// @param callback Invoked for each iteration of the beam_search loop, passing in beams_state.
# /// @param callback_data A pointer that is simply passed back to callback.
# /// @param n_beams Number of beams to use.
# /// @param n_past Number of tokens already evaluated.
# /// @param n_predict Maximum number of tokens to predict. EOS may occur earlier.
# /// @param n_threads Number of threads as passed to llama_eval().
2023-09-29 02:42:03 +00:00
# LLAMA_API void llama_beam_search(
# struct llama_context * ctx,
# llama_beam_search_callback_fn_t callback,
# void * callback_data,
# size_t n_beams,
2024-01-04 03:04:04 +00:00
# int32_t n_past,
# int32_t n_predict);
2023-09-09 16:12:32 +00:00
def llama_beam_search (
ctx : llama_context_p ,
callback : " ctypes._CFuncPtr[None, c_void_p, llama_beams_state] " , # type: ignore
callback_data : c_void_p ,
2023-09-14 01:11:52 +00:00
n_beams : Union [ c_size_t , int ] ,
n_past : Union [ c_int , int ] ,
n_predict : Union [ c_int , int ] ,
2023-09-09 16:12:32 +00:00
) :
return _lib . llama_beam_search (
2023-09-29 02:42:03 +00:00
ctx , callback , callback_data , n_beams , n_past , n_predict
2023-09-09 16:12:32 +00:00
)
2023-10-03 19:23:35 +00:00
2023-09-29 02:42:03 +00:00
_lib . llama_beam_search . argtypes = [
llama_context_p ,
llama_beam_search_callback_fn_t ,
c_void_p ,
c_size_t ,
2024-01-04 03:04:04 +00:00
c_int32 ,
c_int32 ,
2023-09-29 02:42:03 +00:00
]
_lib . llama_beam_search . restype = None
2023-07-24 19:55:26 +00:00
2023-03-24 18:58:42 +00:00
# Performance information
2023-03-24 18:59:29 +00:00
2023-07-06 21:57:56 +00:00
# LLAMA_API struct llama_timings llama_get_timings(struct llama_context * ctx);
def llama_get_timings ( ctx : llama_context_p ) - > llama_timings :
2023-11-28 00:03:02 +00:00
""" Get performance information """
2023-07-06 21:57:56 +00:00
return _lib . llama_get_timings ( ctx )
_lib . llama_get_timings . argtypes = [ llama_context_p ]
_lib . llama_get_timings . restype = llama_timings
2023-05-21 21:47:21 +00:00
# LLAMA_API void llama_print_timings(struct llama_context * ctx);
2023-03-23 09:33:06 +00:00
def llama_print_timings ( ctx : llama_context_p ) :
2023-11-28 00:03:02 +00:00
""" Print performance information """
2023-03-24 22:43:29 +00:00
_lib . llama_print_timings ( ctx )
2023-03-23 09:33:06 +00:00
2023-03-24 18:35:41 +00:00
2023-03-24 22:43:29 +00:00
_lib . llama_print_timings . argtypes = [ llama_context_p ]
_lib . llama_print_timings . restype = None
2023-03-24 18:58:42 +00:00
2023-05-21 21:47:21 +00:00
# LLAMA_API void llama_reset_timings(struct llama_context * ctx);
2023-03-23 09:33:06 +00:00
def llama_reset_timings ( ctx : llama_context_p ) :
2023-11-28 00:03:02 +00:00
""" Reset performance information """
2023-03-24 22:43:29 +00:00
_lib . llama_reset_timings ( ctx )
2023-03-23 09:33:06 +00:00
2023-03-24 18:35:41 +00:00
2023-03-24 22:43:29 +00:00
_lib . llama_reset_timings . argtypes = [ llama_context_p ]
_lib . llama_reset_timings . restype = None
2023-03-24 18:58:42 +00:00
# Print system information
2023-05-21 21:47:21 +00:00
# LLAMA_API const char * llama_print_system_info(void);
2023-03-23 09:33:06 +00:00
def llama_print_system_info ( ) - > bytes :
2023-11-28 00:03:02 +00:00
""" Print system information """
2023-03-24 22:43:29 +00:00
return _lib . llama_print_system_info ( )
2023-03-24 18:58:42 +00:00
2023-03-24 22:43:29 +00:00
_lib . llama_print_system_info . argtypes = [ ]
_lib . llama_print_system_info . restype = c_char_p
2023-05-21 21:47:21 +00:00
2023-08-24 04:17:00 +00:00
2023-09-29 02:42:03 +00:00
# NOTE: THIS IS CURRENTLY BROKEN AS ggml_log_callback IS NOT EXPOSED IN LLAMA.H
2023-08-24 04:17:00 +00:00
# // Set callback for all future logging events.
# // If this is not called, or NULL is supplied, everything is output on stderr.
2023-09-29 02:42:03 +00:00
# LLAMA_API void llama_log_set(ggml_log_callback log_callback, void * user_data);
2023-08-24 04:17:00 +00:00
def llama_log_set (
2024-01-24 15:38:30 +00:00
log_callback : Union [ " ctypes._FuncPointer " , c_void_p ] , user_data : c_void_p # type: ignore
2023-08-24 04:17:00 +00:00
) :
2023-11-28 00:03:02 +00:00
""" Set callback for all future logging events.
If this is not called , or NULL is supplied , everything is output on stderr . """
2023-08-24 04:17:00 +00:00
return _lib . llama_log_set ( log_callback , user_data )
2024-01-24 15:38:30 +00:00
_lib . llama_log_set . argtypes = [ ctypes . c_void_p , c_void_p ]
2023-08-24 04:17:00 +00:00
_lib . llama_log_set . restype = None
2023-08-29 11:36:20 +00:00
# LLAMA_API void llama_dump_timing_info_yaml(FILE * stream, const struct llama_context * ctx);
def llama_dump_timing_info_yaml ( stream : ctypes . c_void_p , ctx : llama_context_p ) :
return _lib . llama_dump_timing_info_yaml ( stream , ctx )
2023-09-09 16:12:32 +00:00
2023-08-29 11:36:20 +00:00
_lib . llama_dump_timing_info_yaml . argtypes = [ ctypes . c_void_p , llama_context_p ]
_lib . llama_dump_timing_info_yaml . restype = None