From c14201dc0f7c544232e5f894726f8ca4d92424f7 Mon Sep 17 00:00:00 2001 From: Niek van der Maas Date: Wed, 12 Apr 2023 11:53:39 +0200 Subject: [PATCH 01/21] Add Dockerfile + build workflow --- .github/workflows/publish.yaml | 31 ++++++++++++++++++++++++++++++- Dockerfile | 10 ++++++++++ README.md | 8 ++++++++ 3 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 Dockerfile diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 92b6e5b..16a6012 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -28,4 +28,33 @@ jobs: # if: startsWith(github.ref, 'refs/tags') uses: pypa/gh-action-pypi-publish@release/v1 with: - password: ${{ secrets.PYPI_API_TOKEN }} \ No newline at end of file + password: ${{ secrets.PYPI_API_TOKEN }} + + docker: + name: Build and push Docker image + runs-on: ubuntu-latest + needs: build-n-publish + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v4 + with: + push: true # push to registry + pull: true # always fetch the latest base images + platforms: linux/amd64,linux/arm64 # build for both amd64 and arm64 + tags: ghcr.io/abetlen/llama-cpp-python:latest \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b500a0b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3-buster + +# We need to set the host to 0.0.0.0 to allow outside access +ENV HOST 0.0.0.0 + +# Install the package +RUN pip install llama-cpp-python[server] + +# Run the server +CMD python3 -m llama_cpp.server \ No newline at end of file diff --git a/README.md b/README.md index 2c8c0a5..81ad723 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,14 @@ python3 -m llama_cpp.server Navigate to [http://localhost:8000/docs](http://localhost:8000/docs) to see the OpenAPI documentation. +## Docker image + +A Docker image is available on [GHCR](https://ghcr.io/abetlen/llama-cpp-python). To run the server: + +```bash +docker run --rm -it -p8000:8000 -v /path/to/models:/models -eMODEL=/models/ggml-vicuna-7b-4bit.bin ghcr.io/abetlen/llama-cpp-python:latest +``` + ## Low-level API The low-level API is a direct `ctypes` binding to the C API provided by `llama.cpp`. From 9ce8146231d77e9aceb8a0f2c0f2721755640eed Mon Sep 17 00:00:00 2001 From: Niek van der Maas Date: Wed, 12 Apr 2023 11:56:16 +0200 Subject: [PATCH 02/21] More generic model name --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 81ad723..bcb25e3 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ Navigate to [http://localhost:8000/docs](http://localhost:8000/docs) to see the A Docker image is available on [GHCR](https://ghcr.io/abetlen/llama-cpp-python). To run the server: ```bash -docker run --rm -it -p8000:8000 -v /path/to/models:/models -eMODEL=/models/ggml-vicuna-7b-4bit.bin ghcr.io/abetlen/llama-cpp-python:latest +docker run --rm -it -p8000:8000 -v /path/to/models:/models -eMODEL=/models/ggml-model-name.bin ghcr.io/abetlen/llama-cpp-python:latest ``` ## Low-level API From 59b37bbbd2fb2a69788c5de6bd103439befbc845 Mon Sep 17 00:00:00 2001 From: Niek van der Maas Date: Sat, 15 Apr 2023 20:24:46 +0200 Subject: [PATCH 03/21] Support openblas --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b500a0b..5bd28b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ FROM python:3-buster ENV HOST 0.0.0.0 # Install the package -RUN pip install llama-cpp-python[server] +RUN apt update && apt install -y libopenblas-dev && LLAMA_OPENBLAS=1 pip install llama-cpp-python[server] # Run the server CMD python3 -m llama_cpp.server \ No newline at end of file From 8476b325f127d66477424f6767c3330fa520728e Mon Sep 17 00:00:00 2001 From: Niek van der Maas Date: Mon, 24 Apr 2023 09:54:38 +0200 Subject: [PATCH 04/21] Change to bullseye --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5bd28b7..ade4ac9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3-buster +FROM python:3-bullseye # We need to set the host to 0.0.0.0 to allow outside access ENV HOST 0.0.0.0 From e9e0654aed80cd5bf8f9f23b16bb6dac76e2845a Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 1 May 2023 22:52:25 -0400 Subject: [PATCH 05/21] Bump version --- pyproject.toml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 54088b6..64f7a0d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "llama_cpp_python" -version = "0.1.40" +version = "0.1.41" description = "Python bindings for the llama.cpp library" authors = ["Andrei Betlen "] license = "MIT" diff --git a/setup.py b/setup.py index fbd22c6..f7f0fa4 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( description="A Python wrapper for llama.cpp", long_description=long_description, long_description_content_type="text/markdown", - version="0.1.40", + version="0.1.41", author="Andrei Betlen", author_email="abetlen@gmail.com", license="MIT", From f97ff3c5bbc2ac4ca5e96c793faf7f8a1d0bbc31 Mon Sep 17 00:00:00 2001 From: Matt Hoffner Date: Mon, 1 May 2023 20:40:06 -0700 Subject: [PATCH 06/21] Update llama_cpp.py --- llama_cpp/llama_cpp.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index b4717bf..e0424c6 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -136,9 +136,9 @@ LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = ctypes.c_int( ) # tok_embeddings.weight and output.weight are F16 LLAMA_FTYPE_MOSTLY_Q4_2 = ctypes.c_int(5) # except 1d tensors # LLAMA_FTYPE_MOSTYL_Q4_3 = ctypes.c_int(6) # except 1d tensors -LLAMA_FTYPE_MOSTYL_Q8_0 = ctypes.c_int(7) # except 1d tensors -LLAMA_FTYPE_MOSTYL_Q5_0 = ctypes.c_int(8) # except 1d tensors -LLAMA_FTYPE_MOSTYL_Q5_1 = ctypes.c_int(9) # except 1d tensors +LLAMA_FTYPE_MOSTLY_Q8_0 = ctypes.c_int(7) # except 1d tensors +LLAMA_FTYPE_MOSTLY_Q5_0 = ctypes.c_int(8) # except 1d tensors +LLAMA_FTYPE_MOSTLY_Q5_1 = ctypes.c_int(9) # except 1d tensors # Functions From d605408f9917943bc0c969b502335ab56b5b2d59 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 2 May 2023 00:55:34 -0400 Subject: [PATCH 07/21] Add dockerignore --- .dockerignore | 166 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..fd64c09 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,166 @@ +_skbuild/ + +.envrc + +models/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +.idea/ From 81631afc48990135c20ece1d52872a7de3033715 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 2 May 2023 00:55:51 -0400 Subject: [PATCH 08/21] Install from local directory --- Dockerfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ade4ac9..14fb3be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,8 +3,13 @@ FROM python:3-bullseye # We need to set the host to 0.0.0.0 to allow outside access ENV HOST 0.0.0.0 +COPY . . + # Install the package -RUN apt update && apt install -y libopenblas-dev && LLAMA_OPENBLAS=1 pip install llama-cpp-python[server] +RUN apt update && apt install -y libopenblas-dev +RUN python -m pip install --upgrade pip pytest cmake scikit-build setuptools fastapi uvicorn sse-starlette + +RUN LLAMA_OPENBLAS=1 python3 setup.py develop # Run the server CMD python3 -m llama_cpp.server \ No newline at end of file From 5d5421b29ddd45ea693d0ce36552c7ff40d83187 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 2 May 2023 01:04:02 -0400 Subject: [PATCH 09/21] Add build docker --- .github/workflows/build-docker.yaml | 33 +++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/build-docker.yaml diff --git a/.github/workflows/build-docker.yaml b/.github/workflows/build-docker.yaml new file mode 100644 index 0000000..9a06da5 --- /dev/null +++ b/.github/workflows/build-docker.yaml @@ -0,0 +1,33 @@ +name: Build Docker + +on: workflow_dispatch + +jobs: + docker: + name: Build and push Docker image + runs-on: ubuntu-latest + needs: build-n-publish + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v4 + with: + push: true # push to registry + pull: true # always fetch the latest base images + platforms: linux/amd64,linux/arm64 # build for both amd64 and arm64 + tags: ghcr.io/abetlen/llama-cpp-python:latest \ No newline at end of file From 36c81489e7dfb3af9c78b2f07da62f39775cd23e Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 2 May 2023 01:04:36 -0400 Subject: [PATCH 10/21] Remove docker section of publish --- .github/workflows/publish.yaml | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 16a6012..ddefd68 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -29,32 +29,3 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.PYPI_API_TOKEN }} - - docker: - name: Build and push Docker image - runs-on: ubuntu-latest - needs: build-n-publish - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push - uses: docker/build-push-action@v4 - with: - push: true # push to registry - pull: true # always fetch the latest base images - platforms: linux/amd64,linux/arm64 # build for both amd64 and arm64 - tags: ghcr.io/abetlen/llama-cpp-python:latest \ No newline at end of file From 62de4692f227a6dc468ce2054bef09caa805b44a Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 2 May 2023 01:09:27 -0400 Subject: [PATCH 11/21] Fix missing dependency --- .github/workflows/build-docker.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-docker.yaml b/.github/workflows/build-docker.yaml index 9a06da5..f4290cd 100644 --- a/.github/workflows/build-docker.yaml +++ b/.github/workflows/build-docker.yaml @@ -6,7 +6,6 @@ jobs: docker: name: Build and push Docker image runs-on: ubuntu-latest - needs: build-n-publish steps: - name: Checkout uses: actions/checkout@v3 From 872b2ec33f97c69ac601df9f5c73b8ad110f6aa0 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 2 May 2023 01:11:34 -0400 Subject: [PATCH 12/21] Clone submodules --- .github/workflows/build-docker.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-docker.yaml b/.github/workflows/build-docker.yaml index f4290cd..2a0c52d 100644 --- a/.github/workflows/build-docker.yaml +++ b/.github/workflows/build-docker.yaml @@ -9,6 +9,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 + with: + submodules: "true" - name: Set up QEMU uses: docker/setup-qemu-action@v2 From c21a34506ebca1eda2a57ef5b6e13e2151049d37 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 2 May 2023 01:13:43 -0400 Subject: [PATCH 13/21] Update permsissions --- .github/workflows/build-docker.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-docker.yaml b/.github/workflows/build-docker.yaml index 2a0c52d..7265b32 100644 --- a/.github/workflows/build-docker.yaml +++ b/.github/workflows/build-docker.yaml @@ -2,6 +2,9 @@ name: Build Docker on: workflow_dispatch +permissions: + contents: write + jobs: docker: name: Build and push Docker image From 63f8d3a6fb97a1973f3a0529050b71f1d884511d Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 2 May 2023 01:16:44 -0400 Subject: [PATCH 14/21] Update context --- .github/workflows/build-docker.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-docker.yaml b/.github/workflows/build-docker.yaml index 7265b32..dda2afe 100644 --- a/.github/workflows/build-docker.yaml +++ b/.github/workflows/build-docker.yaml @@ -31,6 +31,7 @@ jobs: - name: Build and push uses: docker/build-push-action@v4 with: + context: . push: true # push to registry pull: true # always fetch the latest base images platforms: linux/amd64,linux/arm64 # build for both amd64 and arm64 From c2e31eeceee5380e3c6357b73105585265de68bf Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 2 May 2023 01:23:17 -0400 Subject: [PATCH 15/21] Update permissions --- .github/workflows/build-docker.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-docker.yaml b/.github/workflows/build-docker.yaml index dda2afe..16b00a2 100644 --- a/.github/workflows/build-docker.yaml +++ b/.github/workflows/build-docker.yaml @@ -4,6 +4,7 @@ on: workflow_dispatch permissions: contents: write + packages: write jobs: docker: From 1d47cce22269f4422d0d302e9d3bab2583dfdb2a Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 3 May 2023 09:33:30 -0400 Subject: [PATCH 16/21] Update llama.cpp --- llama_cpp/llama_cpp.py | 5 +++-- vendor/llama.cpp | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index e0424c6..30414f5 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -71,7 +71,7 @@ LLAMA_FILE_VERSION = ctypes.c_int(1) LLAMA_FILE_MAGIC = b"ggjt" LLAMA_FILE_MAGIC_UNVERSIONED = b"ggml" LLAMA_SESSION_MAGIC = b"ggsn" -LLAMA_SESSION_VERSION = ctypes.c_int(0) +LLAMA_SESSION_VERSION = ctypes.c_int(1) llama_context_p = c_void_p @@ -239,7 +239,8 @@ _lib.llama_set_rng_seed.argtypes = [llama_context_p, c_int] _lib.llama_set_rng_seed.restype = None -# Returns the size in bytes of the state (rng, logits, embedding and kv_cache) +# Returns the maximum size in bytes of the state (rng, logits, embedding +# and kv_cache) - will often be smaller after compacting tokens def llama_get_state_size(ctx: llama_context_p) -> c_size_t: return _lib.llama_get_state_size(ctx) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 58b367c..e216aa0 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 58b367c2d757c0ea12aec672382462b42204c724 +Subproject commit e216aa04633892b972d013719e38b59fd4917341 From 43f2907e3aecc81d26cc7c73b08b7a973a67aabb Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 3 May 2023 09:33:50 -0400 Subject: [PATCH 17/21] Support smaller state sizes --- llama_cpp/llama.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index d201013..1b9f9e9 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -53,12 +53,14 @@ class LlamaState: def __init__( self, eval_tokens: Deque[llama_cpp.llama_token], - eval_logits: Deque[List[float]], + eval_logits: Deque[List[llama_cpp.c_float]], llama_state, + llama_state_size: llama_cpp.c_size_t, ): self.eval_tokens = eval_tokens self.eval_logits = eval_logits self.llama_state = llama_state + self.llama_state_size = llama_state_size class Llama: @@ -950,19 +952,23 @@ class Llama: assert self.ctx is not None state_size = llama_cpp.llama_get_state_size(self.ctx) llama_state = (llama_cpp.c_uint8 * int(state_size))() - if llama_cpp.llama_copy_state_data(self.ctx, llama_state) != state_size: + n_bytes = llama_cpp.llama_copy_state_data(self.ctx, llama_state) + if int(n_bytes) > int(state_size): raise RuntimeError("Failed to copy llama state data") + llama_state_compact = (llama_cpp.c_uint8 * int(n_bytes))() + llama_cpp.ctypes.memmove(llama_state_compact, llama_state, int(n_bytes)) return LlamaState( eval_tokens=self.eval_tokens.copy(), eval_logits=self.eval_logits.copy(), - llama_state=llama_state, + llama_state=llama_state_compact, + llama_state_size=n_bytes, ) def load_state(self, state: LlamaState) -> None: assert self.ctx is not None self.eval_tokens = state.eval_tokens.copy() self.eval_logits = state.eval_logits.copy() - state_size = llama_cpp.llama_get_state_size(self.ctx) + state_size = state.llama_state_size if llama_cpp.llama_set_state_data(self.ctx, state.llama_state) != state_size: raise RuntimeError("Failed to set llama state data") From 9e5b6d675a49a4466b1fab841baf570e5efeb549 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 3 May 2023 10:28:10 -0400 Subject: [PATCH 18/21] Improve logging messages --- llama_cpp/llama.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 1b9f9e9..fef7b3e 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -396,7 +396,7 @@ class Llama: and tuple(self.eval_tokens) == tuple(tokens[: len(self.eval_tokens)]) ): if self.verbose: - print("generate cache hit", file=sys.stderr) + print("Llama.generate: cache hit", file=sys.stderr) reset = False tokens = tokens[len(self.eval_tokens) :] @@ -518,7 +518,7 @@ class Llama: if self.cache and prompt_tokens in self.cache: if self.verbose: - print("cache hit", file=sys.stderr) + print("Llama._create_completion: cache hit", file=sys.stderr) self.load_state(self.cache[prompt_tokens]) finish_reason = "length" @@ -538,7 +538,7 @@ class Llama: if self.cache and len(completion_tokens) == 0: if prompt_tokens not in self.cache: if self.verbose: - print("cache miss", file=sys.stderr) + print("Llama._create_completion: cache miss", file=sys.stderr) self.cache[prompt_tokens] = self.save_state() completion_tokens.append(token) @@ -957,6 +957,8 @@ class Llama: raise RuntimeError("Failed to copy llama state data") llama_state_compact = (llama_cpp.c_uint8 * int(n_bytes))() llama_cpp.ctypes.memmove(llama_state_compact, llama_state, int(n_bytes)) + if self.verbose: + print(f"Llama.save_state: saving {n_bytes} bytes of llama state", file=sys.stderr) return LlamaState( eval_tokens=self.eval_tokens.copy(), eval_logits=self.eval_logits.copy(), From 7839eb14d3c75a589a3665e447e995745eee7f30 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 3 May 2023 10:29:05 -0400 Subject: [PATCH 19/21] Add docker cuda image. Closes #143 --- .github/workflows/build-docker.yaml | 34 ++++++++++++++++++++++++++++- Dockerfile.cuda | 15 +++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 Dockerfile.cuda diff --git a/.github/workflows/build-docker.yaml b/.github/workflows/build-docker.yaml index 16b00a2..2b340d8 100644 --- a/.github/workflows/build-docker.yaml +++ b/.github/workflows/build-docker.yaml @@ -36,4 +36,36 @@ jobs: push: true # push to registry pull: true # always fetch the latest base images platforms: linux/amd64,linux/arm64 # build for both amd64 and arm64 - tags: ghcr.io/abetlen/llama-cpp-python:latest \ No newline at end of file + tags: ghcr.io/abetlen/llama-cpp-python:latest + + docker-cuda: + name: Build and push Docker image + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: "true" + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v4 + with: + file: Dockerfile.cuda + context: . + push: true # push to registry + pull: true # always fetch the latest base images + platforms: linux/amd64,linux/arm64 # build for both amd64 and arm64 + tags: ghcr.io/abetlen/llama-cpp-python-cuda:latest \ No newline at end of file diff --git a/Dockerfile.cuda b/Dockerfile.cuda new file mode 100644 index 0000000..a852f3c --- /dev/null +++ b/Dockerfile.cuda @@ -0,0 +1,15 @@ +FROM nvidia/cuda:12.1.1-devel-ubuntu20.04 + +# We need to set the host to 0.0.0.0 to allow outside access +ENV HOST 0.0.0.0 + +COPY . . + +# Install the package +RUN apt update && apt install -y python3 python3-pip +RUN python3 -m pip install --upgrade pip pytest cmake scikit-build setuptools fastapi uvicorn sse-starlette + +RUN LLAMA_CUBLAS=1 python3 setup.py develop + +# Run the server +CMD python3 -m llama_cpp.server \ No newline at end of file From 07a56dd9c2e16ba51313831d41016e13d3ce9a3f Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 3 May 2023 10:39:39 -0400 Subject: [PATCH 20/21] Update job name --- .github/workflows/build-docker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-docker.yaml b/.github/workflows/build-docker.yaml index 2b340d8..878660e 100644 --- a/.github/workflows/build-docker.yaml +++ b/.github/workflows/build-docker.yaml @@ -39,7 +39,7 @@ jobs: tags: ghcr.io/abetlen/llama-cpp-python:latest docker-cuda: - name: Build and push Docker image + name: Build and push Docker CUDA image runs-on: ubuntu-latest steps: - name: Checkout From a02aa121da0639e93c58403d1d70d37bfc63c5f0 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 3 May 2023 10:50:48 -0400 Subject: [PATCH 21/21] Remove cuda build job --- .github/workflows/build-docker.yaml | 32 ----------------------------- 1 file changed, 32 deletions(-) diff --git a/.github/workflows/build-docker.yaml b/.github/workflows/build-docker.yaml index 878660e..44196f1 100644 --- a/.github/workflows/build-docker.yaml +++ b/.github/workflows/build-docker.yaml @@ -37,35 +37,3 @@ jobs: pull: true # always fetch the latest base images platforms: linux/amd64,linux/arm64 # build for both amd64 and arm64 tags: ghcr.io/abetlen/llama-cpp-python:latest - - docker-cuda: - name: Build and push Docker CUDA image - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: "true" - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push - uses: docker/build-push-action@v4 - with: - file: Dockerfile.cuda - context: . - push: true # push to registry - pull: true # always fetch the latest base images - platforms: linux/amd64,linux/arm64 # build for both amd64 and arm64 - tags: ghcr.io/abetlen/llama-cpp-python-cuda:latest \ No newline at end of file