baalajimaestro/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge branch 'main' of github.com:abetlen/llama_cpp_python into main

Browse source
This commit is contained in:
Andrei Betlen 2023-08-14 22:41:47 -04:00
commit 620cd2fd69
3 changed files with 3 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
llama_cpp/llama.py
View file

@ -238,6 +238,7 @@ class Llama:
n_ctx: Maximum context size. n_ctx: Maximum context size.
n_parts: Number of parts to split the model into. If -1, the number of parts is automatically determined. n_parts: Number of parts to split the model into. If -1, the number of parts is automatically determined.
seed: Random seed. -1 for random. seed: Random seed. -1 for random.
n_gpu_layers: Number of layers to offload to GPU (-ngl). If -1, all layers are offloaded.
f16_kv: Use half-precision for key/value cache. f16_kv: Use half-precision for key/value cache.
logits_all: Return logits for all tokens, not just the last token. logits_all: Return logits for all tokens, not just the last token.
vocab_only: Only load the vocabulary no weights. vocab_only: Only load the vocabulary no weights.
@ -266,7 +267,7 @@ class Llama:
self.params = llama_cpp.llama_context_default_params() self.params = llama_cpp.llama_context_default_params()
self.params.n_ctx = n_ctx self.params.n_ctx = n_ctx
self.params.n_gpu_layers = n_gpu_layers self.params.n_gpu_layers = 0x7FFFFFFF if n_gpu_layers == -1 else n_gpu_layers # 0x7FFFFFFF is INT32 max, will be auto set to all layers
self.params.seed = seed self.params.seed = seed
self.params.f16_kv = f16_kv self.params.f16_kv = f16_kv
self.params.logits_all = logits_all self.params.logits_all = logits_all

0
llama_cpp/py.typed Normal file
View file

1
setup.py
View file

@ -15,6 +15,7 @@ setup(
author_email="abetlen@gmail.com", author_email="abetlen@gmail.com",
license="MIT", license="MIT",
package_dir={"llama_cpp": "llama_cpp", "llama_cpp.server": "llama_cpp/server"}, package_dir={"llama_cpp": "llama_cpp", "llama_cpp.server": "llama_cpp/server"},
package_data={"llama_cpp": ["py.typed"]},
packages=["llama_cpp", "llama_cpp.server"], packages=["llama_cpp", "llama_cpp.server"],
install_requires=["typing-extensions>=4.5.0", "numpy>=1.20.0", "diskcache>=5.6.1"], install_requires=["typing-extensions>=4.5.0", "numpy>=1.20.0", "diskcache>=5.6.1"],
extras_require={ extras_require={