baalajimaestro/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Fix mlock_supported and mmap_supported return type

Browse source
This commit is contained in:
Andrei Betlen 2023-05-07 03:04:22 -04:00
parent 5a3413eee3
commit 3fbda71790
2 changed files with 10 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
llama_cpp/llama_cpp.py
View file

@ -157,7 +157,7 @@ _lib.llama_context_default_params.argtypes = []
_lib.llama_context_default_params.restype = llama_context_params _lib.llama_context_default_params.restype = llama_context_params
def llama_mmap_supported() -> c_bool: def llama_mmap_supported() -> bool:
return _lib.llama_mmap_supported() return _lib.llama_mmap_supported()
@ -165,7 +165,7 @@ _lib.llama_mmap_supported.argtypes = []
_lib.llama_mmap_supported.restype = c_bool _lib.llama_mmap_supported.restype = c_bool
def llama_mlock_supported() -> c_bool: def llama_mlock_supported() -> bool:
return _lib.llama_mlock_supported() return _lib.llama_mlock_supported()
@ -387,7 +387,9 @@ _lib.llama_n_embd.restype = c_int
# Can be mutated in order to change the probabilities of the next token # Can be mutated in order to change the probabilities of the next token
# Rows: n_tokens # Rows: n_tokens
# Cols: n_vocab # Cols: n_vocab
def llama_get_logits(ctx: llama_context_p): # type: (...) -> Array[float] # type: ignore def llama_get_logits(
ctx: llama_context_p,
): # type: (...) -> Array[float] # type: ignore
return _lib.llama_get_logits(ctx) return _lib.llama_get_logits(ctx)
@ -397,7 +399,9 @@ _lib.llama_get_logits.restype = c_float_p
# Get the embeddings for the input # Get the embeddings for the input
# shape: [n_embd] (1-dimensional) # shape: [n_embd] (1-dimensional)
def llama_get_embeddings(ctx: llama_context_p): # type: (...) -> Array[float] # type: ignore def llama_get_embeddings(
ctx: llama_context_p,
): # type: (...) -> Array[float] # type: ignore
return _lib.llama_get_embeddings(ctx) return _lib.llama_get_embeddings(ctx)

4
llama_cpp/server/app.py
View file

@ -27,11 +27,11 @@ class Settings(BaseSettings):
) )
f16_kv: bool = Field(default=True, description="Whether to use f16 key/value.") f16_kv: bool = Field(default=True, description="Whether to use f16 key/value.")
use_mlock: bool = Field( use_mlock: bool = Field(
default=bool(llama_cpp.llama_mlock_supported().value), default=llama_cpp.llama_mlock_supported(),
description="Use mlock.", description="Use mlock.",
) )
use_mmap: bool = Field( use_mmap: bool = Field(
default=bool(llama_cpp.llama_mmap_supported().value), default=llama_cpp.llama_mmap_supported(),
description="Use mmap.", description="Use mmap.",
) )
embedding: bool = Field(default=True, description="Whether to use embeddings.") embedding: bool = Field(default=True, description="Whether to use embeddings.")