From eb7645b3ba84e182a903663d68c0b4864b670f9b Mon Sep 17 00:00:00 2001 From: Tanner Hobson Date: Fri, 9 Jun 2023 13:13:08 -0400 Subject: [PATCH 1/4] Add support for logit_bias and logit_bias_type parameters --- llama_cpp/llama.py | 2 ++ llama_cpp/server/app.py | 53 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 02fe774..197511c 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -1380,6 +1380,7 @@ class Llama: mirostat_tau: float = 5.0, mirostat_eta: float = 0.1, model: Optional[str] = None, + logits_processor: Optional[LogitsProcessorList] = None, ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]: """Generate a chat completion from a list of messages. @@ -1421,6 +1422,7 @@ class Llama: mirostat_tau=mirostat_tau, mirostat_eta=mirostat_eta, model=model, + logits_processor=logits_processor, ) if stream: chunks: Iterator[CompletionChunk] = completion_or_chunks # type: ignore diff --git a/llama_cpp/server/app.py b/llama_cpp/server/app.py index f70d8f0..a6194f5 100644 --- a/llama_cpp/server/app.py +++ b/llama_cpp/server/app.py @@ -249,13 +249,14 @@ class CreateCompletionRequest(BaseModel): ) presence_penalty: Optional[float] = presence_penalty_field frequency_penalty: Optional[float] = frequency_penalty_field + logit_bias: Optional[Dict[str, float]] = Field(None) + logit_bias_type: Optional[Literal["input_ids", "tokens"]] = Field(None) # ignored or currently unsupported model: Optional[str] = model_field n: Optional[int] = 1 logprobs: Optional[int] = Field(None) best_of: Optional[int] = 1 - logit_bias: Optional[Dict[str, float]] = Field(None) user: Optional[str] = Field(None) # llama.cpp specific parameters @@ -274,6 +275,39 @@ class CreateCompletionRequest(BaseModel): CreateCompletionResponse = create_model_from_typeddict(llama_cpp.Completion) +def make_logit_bias_processor( + llama: llama_cpp.Llama, + logit_bias: Dict[str, float], + logit_bias_type: Optional[Literal["input_ids", "tokens"]], +): + if logit_bias_type is None: + logit_bias_type = "input_ids" + + to_bias: Dict[int, float] = {} + if logit_bias_type == "input_ids": + for input_id, score in logit_bias.items(): + input_id = int(input_id) + to_bias[input_id] = score + + elif logit_bias_type == "tokens": + for token, score in logit_bias.items(): + token = token.encode('utf-8') + for input_id in llama.tokenize(token, add_bos=False): + to_bias[input_id] = score + + def logit_bias_processor( + input_ids: List[int], + scores: List[float], + ) -> List[float]: + new_scores = [None] * len(scores) + for input_id, score in enumerate(scores): + new_scores[input_id] = score + to_bias.get(input_id, 0.0) + + return new_scores + + return logit_bias_processor + + @router.post( "/v1/completions", response_model=CreateCompletionResponse, @@ -291,9 +325,16 @@ async def create_completion( "n", "best_of", "logit_bias", + "logit_bias_type", "user", } kwargs = body.dict(exclude=exclude) + + if body.logit_bias is not None: + kwargs['logits_processor'] = llama_cpp.LogitsProcessorList([ + make_logit_bias_processor(llama, body.logit_bias, body.logit_bias_type), + ]) + if body.stream: send_chan, recv_chan = anyio.create_memory_object_stream(10) @@ -372,11 +413,12 @@ class CreateChatCompletionRequest(BaseModel): stream: bool = stream_field presence_penalty: Optional[float] = presence_penalty_field frequency_penalty: Optional[float] = frequency_penalty_field + logit_bias: Optional[Dict[str, float]] = Field(None) + logit_bias_type: Optional[Literal["input_ids", "tokens"]] = Field(None) # ignored or currently unsupported model: Optional[str] = model_field n: Optional[int] = 1 - logit_bias: Optional[Dict[str, float]] = Field(None) user: Optional[str] = Field(None) # llama.cpp specific parameters @@ -413,9 +455,16 @@ async def create_chat_completion( exclude = { "n", "logit_bias", + "logit_bias_type", "user", } kwargs = body.dict(exclude=exclude) + + if body.logit_bias is not None: + kwargs['logits_processor'] = llama_cpp.LogitsProcessorList([ + make_logit_bias_processor(llama, body.logit_bias, body.logit_bias_type), + ]) + if body.stream: send_chan, recv_chan = anyio.create_memory_object_stream(10) From 94f63a66b9f6f24f3e0079efa0f98c5872ef3a82 Mon Sep 17 00:00:00 2001 From: Ian Scrivener Date: Tue, 13 Jun 2023 09:49:19 +1000 Subject: [PATCH 2/4] Create macos_install.md add MacOS Metal markdown install instructions --- docs/macos_install.md | 62 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 docs/macos_install.md diff --git a/docs/macos_install.md b/docs/macos_install.md new file mode 100644 index 0000000..7d46bc4 --- /dev/null +++ b/docs/macos_install.md @@ -0,0 +1,62 @@ + +# llama-cpp-python - MacOS Install with Metal GPU + + +**(1) Make sure you have xcode installed... at least the command line parts** +``` +# check the path of your xcode install +xcode-select -p + +# xcode installed returns +# /Applications/Xcode-beta.app/Contents/Developer + +# if xcode is missing then install it... it takes ages; +xcode-select --install +``` + +**(2) Install the conda version for MacOS that supports Metal GPU** +``` +wget https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-MacOSX-arm64.sh +bash Miniforge3-MacOSX-arm64.sh +``` + +**(3) Make a conda environment** +``` +conda create -n llama python=3.9.16 +conda activate llama +``` + +**(4) Install the LATEST llama-cpp-python.. which, as of just today, happily supports MacOS Metal GPU** + *(you needed xcode installed in order pip to build/compile the C++ code)* +``` +pip uninstall llama-cpp-python -y +CMAKE_ARGS="-DLLAMA_METAL=on" FORCE_CMAKE=1 pip install -U llama-cpp-python --no-cache-dir +pip install 'llama-cpp-python[server]' + +# you should now have llama-cpp-python v0.1.62 installed +llama-cpp-python         0.1.62      + +``` + +**(4) Download a v3 ggml llama/vicuna/alpaca model** + - **ggmlv3** + - file name ends with **q4_0.bin** - indicating it is 4bit quantized, with quantisation method 0 + +https://huggingface.co/vicuna/ggml-vicuna-13b-1.1/blob/main/ggml-vic13b-q4_0.bin +https://huggingface.co/vicuna/ggml-vicuna-13b-1.1/blob/main/ggml-vic13b-uncensored-q4_0.bin +https://huggingface.co/TheBloke/LLaMa-7B-GGML/blob/main/llama-7b.ggmlv3.q4_0.bin +https://huggingface.co/TheBloke/LLaMa-13B-GGML/blob/main/llama-13b.ggmlv3.q4_0.bin + + +**(6) run the llama-cpp-python API server with MacOS Metal GPU support** +``` +# config your ggml model path +# make sure it is ggml v3 +# make sure it is q4_0 +export MODEL=[path to your llama.cpp ggml models]]/[ggml-model-name]]q4_0.bin +python3 -m llama_cpp.server --model $MODEL --n_gpu_layers 1 +``` + +***Note:** If you omit the `--n_gpu_layers 1` then CPU will be used* + + From 7ca50a3e45a89fda886a3f8179b7a70fc2bda197 Mon Sep 17 00:00:00 2001 From: Ian Scrivener Date: Tue, 13 Jun 2023 09:52:22 +1000 Subject: [PATCH 3/4] Update README.md add link to main README>md --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index ee6e540..a4ca04d 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,9 @@ This package provides: Documentation is available at [https://abetlen.github.io/llama-cpp-python](https://abetlen.github.io/llama-cpp-python). +Detailed MacOS Metal GPU install documentation is available at [docs/macos_install.md](docs/macos_install.md) + + ## Installation from PyPI (recommended) Install from PyPI (requires a c compiler): From 613dd70c8a9e54c373428055102283fdd468f09b Mon Sep 17 00:00:00 2001 From: Matt Dennewitz Date: Tue, 13 Jun 2023 00:56:05 -0500 Subject: [PATCH 4/4] Update README.md Fixes typo in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ee6e540..c099cbf 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Install from PyPI (requires a c compiler): pip install llama-cpp-python ``` -The above command will attempt to install the package and build build `llama.cpp` from source. +The above command will attempt to install the package and build `llama.cpp` from source. This is the recommended installation method as it ensures that `llama.cpp` is built with the available optimizations for your system. If you have previously installed `llama-cpp-python` through pip and want to upgrade your version or rebuild the package with different compiler options, please add the following flags to ensure that the package is rebuilt correctly: