From 85ead98a3e385191bc2782773f697d19eb14f267 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 10 Nov 2023 12:49:14 -0500 Subject: [PATCH 1/5] Update Functions notebook example --- examples/notebooks/Functions.ipynb | 96 ++++++++++++++++++++++++------ 1 file changed, 78 insertions(+), 18 deletions(-) diff --git a/examples/notebooks/Functions.ipynb b/examples/notebooks/Functions.ipynb index 81d58f6..7825cc8 100644 --- a/examples/notebooks/Functions.ipynb +++ b/examples/notebooks/Functions.ipynb @@ -1,15 +1,22 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Function Calling with OpenAI Python Client" + ] + }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "ChatCompletion(id='chatcmpl-b6dcbb47-1120-4761-8cd9-83542c97647b', choices=[Choice(finish_reason='stop', index=0, message=ChatCompletionMessage(content=\"The current temperature in San Francisco is 72 degrees Fahrenheit. It's a sunny day with clear skies, making it perfect for outdoor activities.\\n \", role='assistant', function_call=None, tool_calls=None))], created=1699602158, model='gpt-3.5-turbo-1106', object='chat.completion', system_fingerprint=None, usage=CompletionUsage(completion_tokens=38, prompt_tokens=135, total_tokens=173))\n" + "ChatCompletion(id='chatcmpl-a2d9eb9f-7354-472f-b6ad-4d7a807729a3', choices=[Choice(finish_reason='stop', index=0, message=ChatCompletionMessage(content='The current weather in San Francisco is **72°F** (22°C).\\n ', role='assistant', function_call=None, tool_calls=None))], created=1699638365, model='gpt-3.5-turbo-1106', object='chat.completion', system_fingerprint=None, usage=CompletionUsage(completion_tokens=22, prompt_tokens=136, total_tokens=158))\n" ] } ], @@ -100,9 +107,26 @@ "print(run_conversation())" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Function Calling with Instructor\n", + "\n", + "You'll need to install the [`instructor`](https://github.com/jxnl/instructor/) package to run this notebook. You can do so by running the following command in your terminal:\n", + "\n", + "```bash\n", + "pip install instructor\n", + "```\n", + "\n", + "We'll highlight a few basic examples taken from the [instructor cookbook](https://jxnl.github.io/instructor/)\n", + "\n", + "## Basic Usage" + ] + }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -140,10 +164,27 @@ ] }, { - "cell_type": "code", - "execution_count": 31, + "cell_type": "markdown", "metadata": {}, - "outputs": [], + "source": [ + "## Text Classification\n", + "\n", + "### Single-Label Classification" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "class_label=\n" + ] + } + ], "source": [ "import enum\n", "\n", @@ -172,19 +213,27 @@ " ) # type: ignore\n", "\n", "prediction = classify(\"Hello there I'm a Nigerian prince and I want to give you money\")\n", - "assert prediction.class_label == Labels.SPAM" + "assert prediction.class_label == Labels.SPAM\n", + "print(prediction)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Multi-Label Classification" ] }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "class_labels=[, ]\n" + "class_labels=[, ]\n" ] } ], @@ -223,16 +272,27 @@ "print(prediction)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Self-Critique" + ] + }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "question='What is the meaning of life?' answer='The meaning of life, according to the Devil, is to live a life of sin and debauchery.'\n" + "question='What is the meaning of life?' answer='According to the Devil, the meaning of life is to live a life of sin and debauchery.'\n", + "1 validation error for QuestionAnswerNoEvil\n", + "answer\n", + " Assertion failed, The statement promotes sin and debauchery, which can be considered objectionable. [type=assertion_error, input_value='According to the Devil, ... of sin and debauchery.', input_type=str]\n", + " For further information visit https://errors.pydantic.dev/2.3/v/assertion_error\n" ] } ], @@ -294,6 +354,13 @@ " print(e)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Answering Questions with Validated Citations" + ] + }, { "cell_type": "code", "execution_count": 42, @@ -366,13 +433,6 @@ "qa = ask_ai(question, context)\n", "print(qa)" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { From 74167bdfb23daee74104bdfced4a4e1b01561069 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 10 Nov 2023 13:02:30 -0500 Subject: [PATCH 2/5] Update Functions notebook --- examples/notebooks/Functions.ipynb | 33 ++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/examples/notebooks/Functions.ipynb b/examples/notebooks/Functions.ipynb index 7825cc8..7a7b899 100644 --- a/examples/notebooks/Functions.ipynb +++ b/examples/notebooks/Functions.ipynb @@ -4,7 +4,26 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Function Calling with OpenAI Python Client" + "# Functions\n", + "\n", + "The OpenAI compatbile web server in `llama-cpp-python` supports function calling.\n", + "\n", + "Function calling allows API clients to specify a schema that gives the model a format it should respond in.\n", + "Function calling in `llama-cpp-python` works by combining models pretrained for function calling such as [`functionary`](https://huggingface.co/abetlen/functionary-7b-v1-GGUF) with constrained sampling to produce a response that is compatible with the schema.\n", + "\n", + "Note however that this improves but does not guarantee that the response will be compatible with the schema.\n", + "\n", + "## Requirements\n", + "\n", + "Before we begin you will need the following:\n", + "\n", + "- A running `llama-cpp-python` server with a function calling compatible model. [See here](https://llama-cpp-python.readthedocs.io/en/latest/server/#function-calling)\n", + "- The OpenAI Python Client `pip install openai`\n", + "- (Optional) The Instructor Python Library `pip install instructor`\n", + "\n", + "## Function Calling with OpenAI Python Client\n", + "\n", + "We'll start with a basic demo that only uses the OpenAI Python Client." ] }, { @@ -27,7 +46,7 @@ "\n", "client = openai.OpenAI(\n", " api_key = \"sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\", # can be anything\n", - " base_url = \"http://100.64.159.73:8000/v1\"\n", + " base_url = \"http://100.64.159.73:8000/v1\" # NOTE: Replace with IP address and port of your llama-cpp-python server\n", ")\n", "\n", "# Example dummy function hard coded to return the same weather\n", @@ -113,13 +132,19 @@ "source": [ "# Function Calling with Instructor\n", "\n", - "You'll need to install the [`instructor`](https://github.com/jxnl/instructor/) package to run this notebook. You can do so by running the following command in your terminal:\n", + "The above example is a bit verbose and requires you to manually verify the schema.\n", + "\n", + "For our next examples we'll use the `instructor` library to simplify the process and accomplish a number of different tasks with function calling.\n", + "\n", + "You'll first need to install the [`instructor`](https://github.com/jxnl/instructor/).\n", + "\n", + "You can do so by running the following command in your terminal:\n", "\n", "```bash\n", "pip install instructor\n", "```\n", "\n", - "We'll highlight a few basic examples taken from the [instructor cookbook](https://jxnl.github.io/instructor/)\n", + "Below we'll go through a few basic examples taken directly from the [instructor cookbook](https://jxnl.github.io/instructor/)\n", "\n", "## Basic Usage" ] From 4388f3341413110217b98c4f097ac5c590bdf40b Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 10 Nov 2023 16:32:36 -0500 Subject: [PATCH 3/5] Set CUDA_ARCHITECTURES=OFF for windows --- CMakeLists.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8d06370..658b71d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,6 +6,8 @@ option(LLAMA_BUILD "Build llama.cpp shared library and install alongside python if (LLAMA_BUILD) set(BUILD_SHARED_LIBS "On") + + # Building llama if (APPLE AND NOT CMAKE_SYSTEM_PROCESSOR MATCHES "arm64") # Need to disable these llama.cpp flags on Apple x86_64, # otherwise users may encounter invalid instruction errors @@ -41,8 +43,14 @@ if (LLAMA_BUILD) FILES $ DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp ) + + # Building llava add_subdirectory(vendor/llama.cpp/examples/llava) set_target_properties(llava_shared PROPERTIES OUTPUT_NAME "llava") + # Set CUDA_ARCHITECTURES to OFF on windows + if (WIN32) + set_target_properties(llava_shared PROPERTIES CUDA_ARCHITECTURES OFF) + endif() install( TARGETS llava_shared LIBRARY DESTINATION ${SKBUILD_PLATLIB_DIR}/llama_cpp From 7e3e85b53de475185c28f2019cc8ef25bc5b00d0 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 10 Nov 2023 16:33:55 -0500 Subject: [PATCH 4/5] Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index a75fa57..df9d129 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit a75fa576abba9d37f463580c379e4bbf1e1ad03c +Subproject commit df9d1293defe783f42bc83af732d3c670552c541 From 01846a76b980224609ac8b69c8eb3f5785816752 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 10 Nov 2023 16:36:12 -0500 Subject: [PATCH 5/5] Bump version --- CHANGELOG.md | 5 +++++ llama_cpp/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e6f0241..c9ee7b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.17] + +- Update llama.cpp to ggerganov/llama.cpp@df9d1293defe783f42bc83af732d3c670552c541 +- Hotfix: Set `CUDA_ARCHITECTURES=OFF` for `llava_shared` target on Windows by @abetlen in 4388f3341413110217b98c4f097ac5c590bdf40b + ## [0.2.16] - Update llama.cpp to ggerganov/llama.cp@a75fa576abba9d37f463580c379e4bbf1e1ad03c diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index a24e550..8b2eed7 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.16" \ No newline at end of file +__version__ = "0.2.17" \ No newline at end of file