diff --git a/.golangci.yaml b/.golangci.yaml index c9c9f620..2e0ed3c7 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -32,6 +32,10 @@ linters: linters-settings: gci: sections: [standard, default, localmodule] + staticcheck: + checks: + - all + - -SA1019 # omit Deprecated check severity: default-severity: error rules: diff --git a/api/types.go b/api/types.go index 2f5a9424..df7bab21 100644 --- a/api/types.go +++ b/api/types.go @@ -296,15 +296,17 @@ type EmbeddingResponse struct { // CreateRequest is the request passed to [Client.Create]. type CreateRequest struct { Model string `json:"model"` - Path string `json:"path"` Modelfile string `json:"modelfile"` Stream *bool `json:"stream,omitempty"` Quantize string `json:"quantize,omitempty"` - // Name is deprecated, see Model + // Deprecated: set the model name with Model instead Name string `json:"name"` - // Quantization is deprecated, see Quantize + // Deprecated: set the file content with Modelfile instead + Path string `json:"path"` + + // Deprecated: use Quantize instead Quantization string `json:"quantization,omitempty"` } @@ -312,7 +314,7 @@ type CreateRequest struct { type DeleteRequest struct { Model string `json:"model"` - // Name is deprecated, see Model + // Deprecated: set the model name with Model instead Name string `json:"name"` } @@ -327,7 +329,7 @@ type ShowRequest struct { Options map[string]interface{} `json:"options"` - // Name is deprecated, see Model + // Deprecated: set the model name with Model instead Name string `json:"name"` } @@ -359,7 +361,7 @@ type PullRequest struct { Password string `json:"password"` Stream *bool `json:"stream,omitempty"` - // Name is deprecated, see Model + // Deprecated: set the model name with Model instead Name string `json:"name"` } @@ -380,7 +382,7 @@ type PushRequest struct { Password string `json:"password"` Stream *bool `json:"stream,omitempty"` - // Name is deprecated, see Model + // Deprecated: set the model name with Model instead Name string `json:"name"` } diff --git a/app/ollama.iss b/app/ollama.iss index bce0a337..34cc5c4c 100644 --- a/app/ollama.iss +++ b/app/ollama.iss @@ -87,7 +87,7 @@ DialogFontSize=12 [Files] Source: ".\app.exe"; DestDir: "{app}"; DestName: "{#MyAppExeName}" ; Flags: ignoreversion 64bit -Source: "..\ollama.exe"; DestDir: "{app}\bin"; Flags: ignoreversion 64bit +Source: "..\ollama.exe"; DestDir: "{app}"; Flags: ignoreversion 64bit Source: "..\dist\windows-{#ARCH}\lib\ollama\runners\*"; DestDir: "{app}\lib\ollama\runners"; Flags: ignoreversion 64bit recursesubdirs Source: "..\dist\ollama_welcome.ps1"; DestDir: "{app}"; Flags: ignoreversion Source: ".\assets\app.ico"; DestDir: "{app}"; Flags: ignoreversion @@ -99,7 +99,7 @@ Name: "{userstartup}\{#MyAppName}"; Filename: "{app}\{#MyAppExeName}"; IconFilen Name: "{userprograms}\{#MyAppName}"; Filename: "{app}\{#MyAppExeName}"; IconFilename: "{app}\app.ico" [Run] -Filename: "{cmd}"; Parameters: "/C set PATH={app}\bin;%PATH% & ""{app}\{#MyAppExeName}"""; Flags: postinstall nowait runhidden +Filename: "{cmd}"; Parameters: "/C set PATH={app};%PATH% & ""{app}\{#MyAppExeName}"""; Flags: postinstall nowait runhidden [UninstallRun] ; Filename: "{cmd}"; Parameters: "/C ""taskkill /im ''{#MyAppExeName}'' /f /t"; Flags: runhidden @@ -134,8 +134,8 @@ SetupAppRunningError=Another Ollama installer is running.%n%nPlease cancel or fi [Registry] Root: HKCU; Subkey: "Environment"; \ - ValueType: expandsz; ValueName: "Path"; ValueData: "{olddata};{app}\bin"; \ - Check: NeedsAddPath('{app}\bin') + ValueType: expandsz; ValueName: "Path"; ValueData: "{olddata};{app}"; \ + Check: NeedsAddPath('{app}') [Code] diff --git a/convert/convert_test.go b/convert/convert_test.go index 56b34f22..9eb1632f 100644 --- a/convert/convert_test.go +++ b/convert/convert_test.go @@ -89,7 +89,7 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } -func TestConvertFull(t *testing.T) { +func TestConvertModel(t *testing.T) { cases := []string{ "Meta-Llama-3-8B-Instruct", "Meta-Llama-3.1-8B-Instruct", @@ -140,6 +140,107 @@ func TestConvertFull(t *testing.T) { } } +func TestConvertInvalidDatatype(t *testing.T) { + f, err := os.CreateTemp(t.TempDir(), "testmodel") + if err != nil { + t.Fatal(err) + } + defer f.Close() + + tempDir := t.TempDir() + generateSafetensorTestData(t, tempDir) + + err = ConvertModel(os.DirFS(tempDir), f) + if err == nil || err.Error() != "unsupported safetensors model" { + t.Errorf("expected error but didn't get one") + } +} + +func generateSafetensorTestData(t *testing.T, tempDir string) { + type tensorData struct { + Offsets []int `json:"data_offsets"` + Type string `json:"dtype"` + Shape []int `json:"shape"` + } + offset := 4096 * 14336 + + td := map[string]*tensorData{} + td["model.layers.0.mlp.down_proj.weight"] = &tensorData{ + Offsets: []int{0, offset}, + Type: "I8", + Shape: []int{4096, 14336}, + } + td["model.layers.0.mlp.down_proj.weight_format"] = &tensorData{ + Offsets: []int{offset, offset}, + Type: "U8", + Shape: []int{}, + } + + data, err := json.Marshal(td) + if err != nil { + t.Fatal(err) + } + + var buf bytes.Buffer + + l := int64(len(data)) + err = binary.Write(&buf, binary.LittleEndian, l) + if err != nil { + t.Fatal(err) + } + + _, err = buf.Write(data) + if err != nil { + t.Fatal(err) + } + + fdata, err := os.Create(filepath.Join(tempDir, "model-00001-of-00001.safetensors")) + if err != nil { + t.Fatal(err) + } + defer fdata.Close() + + _, err = fdata.Write(buf.Bytes()) + if err != nil { + t.Fatal(err) + } + + configData := ` +{ + "architectures": [ + "LlamaForCausalLM" + ] +} +` + + f, err := os.Create(filepath.Join(tempDir, "config.json")) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + _, err = f.WriteString(configData) + if err != nil { + t.Fatal(err) + } + + tokenizerData := ` +{ +} +` + + f, err = os.Create(filepath.Join(tempDir, "tokenizer.json")) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + _, err = f.WriteString(tokenizerData) + if err != nil { + t.Fatal(err) + } +} + func TestConvertAdapter(t *testing.T) { type AdapterCase struct { Name string diff --git a/convert/reader_safetensors.go b/convert/reader_safetensors.go index 32a362cd..e1dde8fa 100644 --- a/convert/reader_safetensors.go +++ b/convert/reader_safetensors.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/binary" "encoding/json" + "errors" "fmt" "io" "io/fs" @@ -50,6 +51,10 @@ func parseSafetensors(fsys fs.FS, replacer *strings.Replacer, ps ...string) ([]T for _, key := range keys { if value := headers[key]; value.Type != "" { + // bitsandbytes quantized models are unsupported + if len(value.Shape) == 0 { + return nil, errors.New("unsupported safetensors model") + } ts = append(ts, safetensor{ fs: fsys, path: p, diff --git a/convert/tokenizer.go b/convert/tokenizer.go index 653df6d2..14d6ba66 100644 --- a/convert/tokenizer.go +++ b/convert/tokenizer.go @@ -100,8 +100,21 @@ func parseTokenizer(fsys fs.FS, specialTokenTypes []string) (*Tokenizer, error) } if template, ok := p["chat_template"]; ok { - if err := json.Unmarshal(template, &t.Template); err != nil { - return nil, err + var s []struct { + Name string `json:"name"` + Template string `json:"template"` + } + if err := json.Unmarshal(template, &t.Template); err == nil { + // noop + } else if err := json.Unmarshal(template, &s); err == nil { + for _, e := range s { + if e.Name == "default" { + t.Template = e.Template + break + } + } + } else { + return nil, fmt.Errorf("invalid chat_template: %w", err) } } @@ -141,7 +154,6 @@ func parseTokenizer(fsys fs.FS, specialTokenTypes []string) (*Tokenizer, error) } type tokenizer struct { - Version string `json:"version"` AddedTokens []token `json:"added_tokens"` Model struct { Type string `json:"type"` @@ -239,7 +251,7 @@ func parseVocabulary(fsys fs.FS) (*Vocabulary, error) { return pattern.Func(fsys) } - return nil, errors.New("unknown tensor format") + return nil, errors.New("unknown tokenizer format") } type SpecialVocabulary struct { diff --git a/convert/tokenizer_test.go b/convert/tokenizer_test.go new file mode 100644 index 00000000..d9550e09 --- /dev/null +++ b/convert/tokenizer_test.go @@ -0,0 +1,208 @@ +package convert + +import ( + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func createTokenizerFS(t *testing.T, dir string, files map[string]io.Reader) fs.FS { + t.Helper() + + for k, v := range files { + if err := func() error { + f, err := os.Create(filepath.Join(dir, k)) + if err != nil { + return err + } + defer f.Close() + + if _, err := io.Copy(f, v); err != nil { + return err + } + + return nil + }(); err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + + return os.DirFS(dir) +} + +func TestParseTokenizer(t *testing.T) { + cases := []struct { + name string + fsys fs.FS + specialTokenTypes []string + want *Tokenizer + }{ + { + name: "string chat template", + fsys: createTokenizerFS(t, t.TempDir(), map[string]io.Reader{ + "tokenizer.json": strings.NewReader(`{}`), + "tokenizer_config.json": strings.NewReader(`{ + "chat_template": "" + }`), + }), + want: &Tokenizer{ + Vocabulary: &Vocabulary{Model: "gpt2"}, + Pre: "default", + Template: "", + }, + }, + { + name: "list chat template", + fsys: createTokenizerFS(t, t.TempDir(), map[string]io.Reader{ + "tokenizer.json": strings.NewReader(`{}`), + "tokenizer_config.json": strings.NewReader(`{ + "chat_template": [ + { + "name": "default", + "template": "" + }, + { + "name": "tools", + "template": "" + } + ] + }`), + }), + want: &Tokenizer{ + Vocabulary: &Vocabulary{Model: "gpt2"}, + Pre: "default", + Template: "", + }, + }, + { + name: "added tokens", + fsys: createTokenizerFS(t, t.TempDir(), map[string]io.Reader{ + "tokenizer.json": strings.NewReader(`{ + "added_tokens": [ + { + "id": 999, + "content": "", + "special": false + } + ] + }`), + }), + want: &Tokenizer{ + Vocabulary: &Vocabulary{ + Model: "gpt2", + Tokens: []string{""}, + Scores: []float32{999}, + Types: []int32{4}, + }, + Pre: "default", + }, + }, + { + name: "added tokens overlap vocab", + fsys: createTokenizerFS(t, t.TempDir(), map[string]io.Reader{ + "tokenizer.json": strings.NewReader(`{ + "added_tokens": [ + { + "id": 0, + "content": "", + "special": true + } + ], + "model": { + "vocab": { + "": 0 + } + } + }`), + }), + want: &Tokenizer{ + Vocabulary: &Vocabulary{ + Model: "gpt2", + Tokens: []string{""}, + Scores: []float32{0}, + Types: []int32{3}, + }, + Pre: "default", + }, + }, + { + name: "special token types", + fsys: createTokenizerFS(t, t.TempDir(), map[string]io.Reader{ + "tokenizer.json": strings.NewReader(`{ + "added_tokens": [ + { + "id": 0, + "content": "", + "special": true + }, + { + "id": 1, + "content": "", + "special": true + }, + { + "id": 2, + "content": "", + "special": true + }, + { + "id": 3, + "content": "", + "special": true + } + ], + "model": { + "vocab": { + "": 0, + "": 1, + "": 2, + "": 3 + } + } + }`), + "tokenizer_config.json": strings.NewReader(`{ + "add_bos_token": true, + "add_eos_token": false, + "bos_token": "", + "eos_token": "", + "pad_token": "", + "unk_token": "" + }`), + }), + specialTokenTypes: []string{"pad", "eos", "bos", "unk"}, + want: &Tokenizer{ + Vocabulary: &Vocabulary{ + Model: "gpt2", + Tokens: []string{"", "", "", ""}, + Scores: []float32{0, 1, 2, 3}, + Types: []int32{3, 3, 3, 3}, + }, + SpecialVocabulary: []*SpecialVocabulary{ + {Type: "pad", Content: "", ID: 0, AddToken: false}, + {Type: "eos", Content: "", ID: 1, AddToken: false}, + {Type: "bos", Content: "", ID: 2, AddToken: true}, + {Type: "unk", Content: "", ID: 3, AddToken: false}, + }, + Pre: "default", + }, + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + tokenizer, err := parseTokenizer(tt.fsys, tt.specialTokenTypes) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if diff := cmp.Diff(tt.want, tokenizer); diff != "" { + t.Errorf("unexpected tokenizer (-want +got):\n%s", diff) + } + }) + } +} diff --git a/docs/images/ollama-keys.png b/docs/images/ollama-keys.png new file mode 100644 index 00000000..942079a8 Binary files /dev/null and b/docs/images/ollama-keys.png differ diff --git a/docs/images/signup.png b/docs/images/signup.png new file mode 100644 index 00000000..e80bb4e7 Binary files /dev/null and b/docs/images/signup.png differ diff --git a/docs/import.md b/docs/import.md index 82ea9ba5..1a90bc48 100644 --- a/docs/import.md +++ b/docs/import.md @@ -1,44 +1,129 @@ -# Import +# Importing a model -GGUF models and select Safetensors models can be imported directly into Ollama. +## Table of Contents -## Import GGUF + * [Importing a Safetensors adapter](#Importing-a-fine-tuned-adapter-from-Safetensors-weights) + * [Importing a Safetensors model](#Importing-a-model-from-Safetensors-weights) + * [Importing a GGUF file](#Importing-a-GGUF-based-model-or-adapter) + * [Sharing models on ollama.com](#Sharing-your-model-on-ollamacom) -A binary GGUF file can be imported directly into Ollama through a Modelfile. +## Importing a fine tuned adapter from Safetensors weights + +First, create a `Modelfile` with a `FROM` command pointing at the base model you used for fine tuning, and an `ADAPTER` command which points to the directory with your Safetensors adapter: ```dockerfile -FROM /path/to/file.gguf +FROM +ADAPTER /path/to/safetensors/adapter/directory ``` -## Import Safetensors +Make sure that you use the same base model in the `FROM` command as you used to create the adapter otherwise you will get erratic results. Most frameworks use different quantization methods, so it's best to use non-quantized (i.e. non-QLoRA) adapters. If your adapter is in the same directory as your `Modelfile`, use `ADAPTER .` to specify the adapter path. -If the model being imported is one of these architectures, it can be imported directly into Ollama through a Modelfile: +Now run `ollama create` from the directory where the `Modelfile` was created: - - LlamaForCausalLM - - MistralForCausalLM - - MixtralForCausalLM - - GemmaForCausalLM - - Phi3ForCausalLM +```bash +ollama create my-model +``` + +Lastly, test the model: + +```bash +ollama run my-model +``` + +Ollama supports importing adapters based on several different model architectures including: + + * Llama (including Llama 2, Llama 3, and Llama 3.1); + * Mistral (including Mistral 1, Mistral 2, and Mixtral); and + * Gemma (including Gemma 1 and Gemma 2) + +You can create the adapter using a fine tuning framework or tool which can output adapters in the Safetensors format, such as: + + * Hugging Face [fine tuning framework] (https://huggingface.co/docs/transformers/en/training) + * [Unsloth](https://github.com/unslothai/unsloth) + * [MLX](https://github.com/ml-explore/mlx) + + +## Importing a model from Safetensors weights + +First, create a `Modelfile` with a `FROM` command which points to the directory containing your Safetensors weights: ```dockerfile FROM /path/to/safetensors/directory ``` -For architectures not directly convertable by Ollama, see llama.cpp's [guide](https://github.com/ggerganov/llama.cpp/blob/master/README.md#prepare-and-quantize) on conversion. After conversion, see [Import GGUF](#import-gguf). +If you create the Modelfile in the same directory as the weights, you can use the command `FROM .`. -## Automatic Quantization +Now run the `ollama create` command from the directory where you created the `Modelfile`: -> [!NOTE] -> Automatic quantization requires v0.1.35 or higher. +```shell +ollama create my-model +``` -Ollama is capable of quantizing FP16 or FP32 models to any of the supported quantizations with the `-q/--quantize` flag in `ollama create`. +Lastly, test the model: + +```shell +ollama run my-model +``` + +Ollama supports importing models for several different architectures including: + + * Llama (including Llama 2, Llama 3, and Llama 3.1); + * Mistral (including Mistral 1, Mistral 2, and Mixtral); + * Gemma (including Gemma 1 and Gemma 2); and + * Phi3 + +This includes importing foundation models as well as any fine tuned models which which have been _fused_ with a foundation model. + + +## Importing a GGUF based model or adapter + +If you have a GGUF based model or adapter it is possible to import it into Ollama. You can obtain a GGUF model or adapter by: + + * converting a Safetensors model with the `convert_hf_to_gguf.py` from Llama.cpp; + * converting a Safetensors adapter with the `convert_lora_to_gguf.py` from Llama.cpp; or + * downloading a model or adapter from a place such as HuggingFace + +To import a GGUF model, create a `Modelfile` containg: + +```dockerfile +FROM /path/to/file.gguf +``` + +For a GGUF adapter, create the `Modelfile` with: + +```dockerfile +FROM +ADAPTER /path/to/file.gguf +``` + +When importing a GGUF adapter, it's important to use the same base model as the base model that the adapter was created with. You can use: + + * a model from Ollama + * a GGUF file + * a Safetensors based model + +Once you have created your `Modelfile`, use the `ollama create` command to build the model. + +```shell +ollama create my-model +``` + +## Quantizing a Model + +Quantizing a model allows you to run models faster and with less memory consumption but at reduced accuracy. This allows you to run a model on more modest hardware. + +Ollama can quantize FP16 and FP32 based models into different quantization levels using the `-q/--quantize` flag with the `ollama create` command. + +First, create a Modelfile with the FP16 or FP32 based model you wish to quantize. ```dockerfile FROM /path/to/my/gemma/f16/model ``` +Use `ollama create` to then create the quantized model. + ```shell -$ ollama create -q Q4_K_M mymodel +$ ollama create --quantize q4_K_M mymodel transferring model data quantizing F16 model to Q4_K_M creating new layer sha256:735e246cc1abfd06e9cdcf95504d6789a6cd1ad7577108a70d9902fef503c1bd @@ -49,42 +134,53 @@ success ### Supported Quantizations -- `Q4_0` -- `Q4_1` -- `Q5_0` -- `Q5_1` -- `Q8_0` +- `q4_0` +- `q4_1` +- `q5_0` +- `q5_1` +- `q8_0` #### K-means Quantizations -- `Q3_K_S` -- `Q3_K_M` -- `Q3_K_L` -- `Q4_K_S` -- `Q4_K_M` -- `Q5_K_S` -- `Q5_K_M` -- `Q6_K` +- `q3_K_S` +- `q3_K_M` +- `q3_K_L` +- `q4_K_S` +- `q4_K_M` +- `q5_K_S` +- `q5_K_M` +- `q6_K` -## Template Detection -> [!NOTE] -> Template detection requires v0.1.42 or higher. +## Sharing your model on ollama.com -Ollama uses model metadata, specifically `tokenizer.chat_template`, to automatically create a template appropriate for the model you're importing. +You can share any model you have created by pushing it to [ollama.com](https://ollama.com) so that other users can try it out. -```dockerfile -FROM /path/to/my/gemma/model -``` +First, use your browser to go to the [Ollama Sign-Up](https://ollama.com/signup) page. If you already have an account, you can skip this step. + +Sign-Up + +The `Username` field will be used as part of your model's name (e.g. `jmorganca/mymodel`), so make sure you are comfortable with the username that you have selected. + +Now that you have created an account and are signed-in, go to the [Ollama Keys Settings](https://ollama.com/settings/keys) page. + +Follow the directions on the page to determine where your Ollama Public Key is located. + +Ollama Keys + +Click on the `Add Ollama Public Key` button, and copy and paste the contents of your Ollama Public Key into the text field. + +To push a model to [ollama.com](https://ollama.com), first make sure that it is named correctly with your username. You may have to use the `ollama cp` command to copy +your model to give it the correct name. Once you're happy with your model's name, use the `ollama push` command to push it to [ollama.com](https://ollama.com). ```shell -$ ollama create mymodel -transferring model data -using autodetected template gemma-instruct -creating new layer sha256:baa2a0edc27d19cc6b7537578a9a7ba1a4e3214dc185ed5ae43692b319af7b84 -creating new layer sha256:ba66c3309914dbef07e5149a648fd1877f030d337a4f240d444ea335008943cb -writing manifest -success +ollama cp mymodel myuser/mymodel +ollama push myuser/mymodel +``` + +Once your model has been pushed, other users can pull and run it by using the command: + +```shell +ollama run myuser/mymodel ``` -Defining a template in the Modelfile will disable this feature which may be useful if you want to use a different template than the autodetected one. diff --git a/docs/linux.md b/docs/linux.md index d1d5892c..fbaf4845 100644 --- a/docs/linux.md +++ b/docs/linux.md @@ -28,6 +28,11 @@ Download and extract the Linux package: curl -fsSL https://ollama.com/download/ollama-linux-amd64.tgz | sudo tar zx -C /usr ``` +If you have an AMD GPU, also download and extract the ROCm package into the same location +```bash +curl -fsSL https://ollama.com/download/ollama-linux-amd64-rocm.tgz | sudo tar zx -C /usr +``` + ### Adding Ollama as a startup service (recommended) Create a user for Ollama: diff --git a/docs/modelfile.md b/docs/modelfile.md index 852bf96c..51827e74 100644 --- a/docs/modelfile.md +++ b/docs/modelfile.md @@ -11,8 +11,9 @@ A model file is the blueprint to create and share models with Ollama. - [Examples](#examples) - [Instructions](#instructions) - [FROM (Required)](#from-required) - - [Build from llama3](#build-from-llama3) - - [Build from a bin file](#build-from-a-bin-file) + - [Build from llama3.1](#build-from-llama31) + - [Build from a Safetensors model](#build-from-a-safetensors-model) + - [Build from a GGUF file](#build-from-a-gguf-file) - [PARAMETER](#parameter) - [Valid Parameters and Values](#valid-parameters-and-values) - [TEMPLATE](#template) @@ -99,22 +100,39 @@ The `FROM` instruction defines the base model to use when creating a model. FROM : ``` -#### Build from llama3 +#### Build from llama3.1 ```modelfile -FROM llama3 +FROM llama3.1 ``` A list of available base models: +Additional models can be found at: + -#### Build from a `bin` file +#### Build from a Safetensors model + +```modelfile +FROM +``` + +The model directory should contain the Safetensors weights for a supported architecture. + +Currently supported model architectures: + * Llama (including Llama 2, Llama 3, and Llama 3.1) + * Mistral (including Mistral 1, Mistral 2, and Mixtral) + * Gemma (including Gemma 1 and Gemma 2) + * Phi3 + +#### Build from a GGUF file ```modelfile FROM ./ollama-model.bin ``` -This bin file location should be specified as an absolute path or relative to the `Modelfile` location. +The GGUF bin file location should be specified as an absolute path or relative to the `Modelfile` location. + ### PARAMETER @@ -174,7 +192,20 @@ SYSTEM """""" ### ADAPTER -The `ADAPTER` instruction is an optional instruction that specifies any LoRA adapter that should apply to the base model. The value of this instruction should be an absolute path or a path relative to the Modelfile and the file must be in a GGML file format. The adapter should be tuned from the base model otherwise the behaviour is undefined. +The `ADAPTER` instruction specifies a fine tuned LoRA adapter that should apply to the base model. The value of the adapter should be an absolute path or a path relative to the Modelfile. The base model should be specified with a `FROM` instruction. If the base model is not the same as the base model that the adapter was tuned from the behaviour will be erratic. + +#### Safetensor adapter + +```modelfile +ADAPTER +``` + +Currently supported Safetensor adapters: + * Llama (including Llama 2, Llama 3, and Llama 3.1) + * Mistral (including Mistral 1, Mistral 2, and Mixtral) + * Gemma (including Gemma 1 and Gemma 2) + +#### GGUF adapter ```modelfile ADAPTER ./ollama-lora.bin diff --git a/docs/openai.md b/docs/openai.md index 75d2c595..0cbea6cc 100644 --- a/docs/openai.md +++ b/docs/openai.md @@ -300,3 +300,28 @@ curl http://localhost:11434/v1/chat/completions \ ] }' ``` + +### Setting the context size + +The OpenAI API does not have a way of setting the context size for a model. If you need to change the context size, create a `Modelfile` which looks like: + +```modelfile +FROM +PARAMETER num_ctx +``` + +Use the `ollama create mymodel` command to create a new model with the updated context size. Call the API with the updated model name: + +```shell +curl http://localhost:11434/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "mymodel", + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }' +``` diff --git a/envconfig/config.go b/envconfig/config.go index 7e45a4f5..806a2d08 100644 --- a/envconfig/config.go +++ b/envconfig/config.go @@ -190,7 +190,7 @@ func RunnersDir() (p string) { } var paths []string - for _, root := range []string{filepath.Dir(exe), filepath.Join(filepath.Dir(exe), ".."), cwd} { + for _, root := range []string{filepath.Dir(exe), filepath.Join(filepath.Dir(exe), LibRelativeToExe()), cwd} { paths = append(paths, root, filepath.Join(root, runtime.GOOS+"-"+runtime.GOARCH), @@ -282,3 +282,12 @@ func Values() map[string]string { func Var(key string) string { return strings.Trim(strings.TrimSpace(os.Getenv(key)), "\"'") } + +// On windows, we keep the binary at the top directory, but +// other platforms use a "bin" directory, so this returns ".." +func LibRelativeToExe() string { + if runtime.GOOS == "windows" { + return "." + } + return ".." +} diff --git a/gpu/amd_common.go b/gpu/amd_common.go index 72d204f7..2894ac2c 100644 --- a/gpu/amd_common.go +++ b/gpu/amd_common.go @@ -9,6 +9,8 @@ import ( "path/filepath" "runtime" "strings" + + "github.com/ollama/ollama/envconfig" ) // Determine if the given ROCm lib directory is usable by checking for existence of some glob patterns @@ -54,7 +56,7 @@ func commonAMDValidateLibDir() (string, error) { // Installer payload location if we're running the installed binary exe, err := os.Executable() if err == nil { - rocmTargetDir := filepath.Join(filepath.Dir(exe), "..", "lib", "ollama") + rocmTargetDir := filepath.Join(filepath.Dir(exe), envconfig.LibRelativeToExe(), "lib", "ollama") if rocmLibUsable(rocmTargetDir) { slog.Debug("detected ROCM next to ollama executable " + rocmTargetDir) return rocmTargetDir, nil diff --git a/gpu/amd_windows.go b/gpu/amd_windows.go index a0ae7c96..ef6bf830 100644 --- a/gpu/amd_windows.go +++ b/gpu/amd_windows.go @@ -153,7 +153,7 @@ func AMDValidateLibDir() (string, error) { // Installer payload (if we're running from some other location) localAppData := os.Getenv("LOCALAPPDATA") appDir := filepath.Join(localAppData, "Programs", "Ollama") - rocmTargetDir := filepath.Join(appDir, "..", "lib", "ollama") + rocmTargetDir := filepath.Join(appDir, envconfig.LibRelativeToExe(), "lib", "ollama") if rocmLibUsable(rocmTargetDir) { slog.Debug("detected ollama installed ROCm at " + rocmTargetDir) return rocmTargetDir, nil diff --git a/gpu/gpu.go b/gpu/gpu.go index 10afb1e3..3de93f7f 100644 --- a/gpu/gpu.go +++ b/gpu/gpu.go @@ -653,7 +653,7 @@ func LibraryDir() string { slog.Warn("failed to lookup working directory", "error", err) } // Scan for any of our dependeices, and pick first match - for _, root := range []string{filepath.Dir(appExe), filepath.Join(filepath.Dir(appExe), ".."), cwd} { + for _, root := range []string{filepath.Dir(appExe), filepath.Join(filepath.Dir(appExe), envconfig.LibRelativeToExe()), cwd} { libDep := filepath.Join("lib", "ollama") if _, err := os.Stat(filepath.Join(root, libDep)); err == nil { return filepath.Join(root, libDep) diff --git a/llm/server.go b/llm/server.go index 4e5dac28..c38bc6bb 100644 --- a/llm/server.go +++ b/llm/server.go @@ -409,7 +409,7 @@ func NewLlamaServer(gpus gpu.GpuInfoList, model string, ggml *GGML, adapters, pr } if err = s.cmd.Start(); err != nil { - // Detect permission denied and augment them essage about noexec + // Detect permission denied and augment the message about noexec if errors.Is(err, os.ErrPermission) { finalErr = fmt.Errorf("unable to start server %w. %s may have noexec set. Set OLLAMA_TMPDIR for server to a writable executable directory", err, dir) continue diff --git a/scripts/build_windows.ps1 b/scripts/build_windows.ps1 index 9cebf1f4..eb8570c8 100644 --- a/scripts/build_windows.ps1 +++ b/scripts/build_windows.ps1 @@ -122,8 +122,8 @@ function buildOllama() { /csp "Google Cloud KMS Provider" /kc ${env:KEY_CONTAINER} ollama.exe if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)} } - New-Item -ItemType Directory -Path .\dist\windows-${script:TARGET_ARCH}\bin\ -Force - cp .\ollama.exe .\dist\windows-${script:TARGET_ARCH}\bin\ + New-Item -ItemType Directory -Path .\dist\windows-${script:TARGET_ARCH}\ -Force + cp .\ollama.exe .\dist\windows-${script:TARGET_ARCH}\ } function buildApp() { diff --git a/server/model_test.go b/server/model_test.go index 7753c549..e1737a5b 100644 --- a/server/model_test.go +++ b/server/model_test.go @@ -139,6 +139,7 @@ The temperature in San Francisco, CA is 70°F and in Toronto, Canada is 20°C.`, func TestParseFromFileFromLayer(t *testing.T) { tempModels := t.TempDir() + t.Setenv("OLLAMA_MODELS", tempModels) file, err := os.CreateTemp(tempModels, "") if err != nil { @@ -189,6 +190,7 @@ func TestParseFromFileFromLayer(t *testing.T) { func TestParseLayerFromCopy(t *testing.T) { tempModels := t.TempDir() + t.Setenv("OLLAMA_MODELS", tempModels) file2, err := os.CreateTemp(tempModels, "") if err != nil { diff --git a/server/modelpath.go b/server/modelpath.go index 354eeed7..d498c467 100644 --- a/server/modelpath.go +++ b/server/modelpath.go @@ -73,18 +73,6 @@ func ParseModelPath(name string) ModelPath { var errModelPathInvalid = errors.New("invalid model path") -func (mp ModelPath) Validate() error { - if mp.Repository == "" { - return fmt.Errorf("%w: model repository name is required", errModelPathInvalid) - } - - if strings.Contains(mp.Tag, ":") { - return fmt.Errorf("%w: ':' (colon) is not allowed in tag names", errModelPathInvalid) - } - - return nil -} - func (mp ModelPath) GetNamespaceRepository() string { return fmt.Sprintf("%s/%s", mp.Namespace, mp.Repository) } @@ -105,7 +93,11 @@ func (mp ModelPath) GetShortTagname() string { // GetManifestPath returns the path to the manifest file for the given model path, it is up to the caller to create the directory if it does not exist. func (mp ModelPath) GetManifestPath() (string, error) { - return filepath.Join(envconfig.Models(), "manifests", mp.Registry, mp.Namespace, mp.Repository, mp.Tag), nil + if p := filepath.Join(mp.Registry, mp.Namespace, mp.Repository, mp.Tag); filepath.IsLocal(p) { + return filepath.Join(envconfig.Models(), "manifests", p), nil + } + + return "", errModelPathInvalid } func (mp ModelPath) BaseURL() *url.URL { diff --git a/server/modelpath_test.go b/server/modelpath_test.go index 849e0fa7..ef26266b 100644 --- a/server/modelpath_test.go +++ b/server/modelpath_test.go @@ -1,6 +1,7 @@ package server import ( + "errors" "os" "path/filepath" "testing" @@ -154,3 +155,10 @@ func TestParseModelPath(t *testing.T) { }) } } + +func TestInsecureModelpath(t *testing.T) { + mp := ParseModelPath("../../..:something") + if _, err := mp.GetManifestPath(); !errors.Is(err, errModelPathInvalid) { + t.Errorf("expected error: %v", err) + } +} diff --git a/server/routes.go b/server/routes.go index 6c470c17..5e9f51e1 100644 --- a/server/routes.go +++ b/server/routes.go @@ -463,7 +463,7 @@ func (s *Server) EmbeddingsHandler(c *gin.Context) { c.JSON(http.StatusOK, resp) } -func (s *Server) PullModelHandler(c *gin.Context) { +func (s *Server) PullHandler(c *gin.Context) { var req api.PullRequest err := c.ShouldBindJSON(&req) switch { @@ -513,7 +513,7 @@ func (s *Server) PullModelHandler(c *gin.Context) { streamResponse(c, ch) } -func (s *Server) PushModelHandler(c *gin.Context) { +func (s *Server) PushHandler(c *gin.Context) { var req api.PushRequest err := c.ShouldBindJSON(&req) switch { @@ -577,7 +577,7 @@ func checkNameExists(name model.Name) error { return nil } -func (s *Server) CreateModelHandler(c *gin.Context) { +func (s *Server) CreateHandler(c *gin.Context) { var r api.CreateRequest if err := c.ShouldBindJSON(&r); errors.Is(err, io.EOF) { c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"}) @@ -647,7 +647,7 @@ func (s *Server) CreateModelHandler(c *gin.Context) { streamResponse(c, ch) } -func (s *Server) DeleteModelHandler(c *gin.Context) { +func (s *Server) DeleteHandler(c *gin.Context) { var r api.DeleteRequest if err := c.ShouldBindJSON(&r); errors.Is(err, io.EOF) { c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"}) @@ -680,7 +680,7 @@ func (s *Server) DeleteModelHandler(c *gin.Context) { } } -func (s *Server) ShowModelHandler(c *gin.Context) { +func (s *Server) ShowHandler(c *gin.Context) { var req api.ShowRequest err := c.ShouldBindJSON(&req) switch { @@ -829,7 +829,7 @@ func getKVData(digest string, verbose bool) (llm.KV, error) { return kv, nil } -func (s *Server) ListModelsHandler(c *gin.Context) { +func (s *Server) ListHandler(c *gin.Context) { ms, err := Manifests() if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) @@ -879,7 +879,7 @@ func (s *Server) ListModelsHandler(c *gin.Context) { c.JSON(http.StatusOK, api.ListResponse{Models: models}) } -func (s *Server) CopyModelHandler(c *gin.Context) { +func (s *Server) CopyHandler(c *gin.Context) { var r api.CopyRequest if err := c.ShouldBindJSON(&r); errors.Is(err, io.EOF) { c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"}) @@ -1081,33 +1081,33 @@ func (s *Server) GenerateRoutes() http.Handler { allowedHostsMiddleware(s.addr), ) - r.POST("/api/pull", s.PullModelHandler) + r.POST("/api/pull", s.PullHandler) r.POST("/api/generate", s.GenerateHandler) r.POST("/api/chat", s.ChatHandler) r.POST("/api/embed", s.EmbedHandler) r.POST("/api/embeddings", s.EmbeddingsHandler) - r.POST("/api/create", s.CreateModelHandler) - r.POST("/api/push", s.PushModelHandler) - r.POST("/api/copy", s.CopyModelHandler) - r.DELETE("/api/delete", s.DeleteModelHandler) - r.POST("/api/show", s.ShowModelHandler) + r.POST("/api/create", s.CreateHandler) + r.POST("/api/push", s.PushHandler) + r.POST("/api/copy", s.CopyHandler) + r.DELETE("/api/delete", s.DeleteHandler) + r.POST("/api/show", s.ShowHandler) r.POST("/api/blobs/:digest", s.CreateBlobHandler) r.HEAD("/api/blobs/:digest", s.HeadBlobHandler) - r.GET("/api/ps", s.ProcessHandler) + r.GET("/api/ps", s.PsHandler) // Compatibility endpoints r.POST("/v1/chat/completions", openai.ChatMiddleware(), s.ChatHandler) r.POST("/v1/completions", openai.CompletionsMiddleware(), s.GenerateHandler) r.POST("/v1/embeddings", openai.EmbeddingsMiddleware(), s.EmbedHandler) - r.GET("/v1/models", openai.ListMiddleware(), s.ListModelsHandler) - r.GET("/v1/models/:model", openai.RetrieveMiddleware(), s.ShowModelHandler) + r.GET("/v1/models", openai.ListMiddleware(), s.ListHandler) + r.GET("/v1/models/:model", openai.RetrieveMiddleware(), s.ShowHandler) for _, method := range []string{http.MethodGet, http.MethodHead} { r.Handle(method, "/", func(c *gin.Context) { c.String(http.StatusOK, "Ollama is running") }) - r.Handle(method, "/api/tags", s.ListModelsHandler) + r.Handle(method, "/api/tags", s.ListHandler) r.Handle(method, "/api/version", func(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"version": version.Version}) }) @@ -1269,7 +1269,7 @@ func streamResponse(c *gin.Context, ch chan any) { }) } -func (s *Server) ProcessHandler(c *gin.Context) { +func (s *Server) PsHandler(c *gin.Context) { models := []api.ProcessModelResponse{} for _, v := range s.sched.loaded { diff --git a/server/routes_create_test.go b/server/routes_create_test.go index 4de07b25..d436f26c 100644 --- a/server/routes_create_test.go +++ b/server/routes_create_test.go @@ -93,7 +93,7 @@ func TestCreateFromBin(t *testing.T) { t.Setenv("OLLAMA_MODELS", p) var s Server - w := createRequest(t, s.CreateModelHandler, api.CreateRequest{ + w := createRequest(t, s.CreateHandler, api.CreateRequest{ Name: "test", Modelfile: fmt.Sprintf("FROM %s", createBinFile(t, nil, nil)), Stream: &stream, @@ -120,7 +120,7 @@ func TestCreateFromModel(t *testing.T) { t.Setenv("OLLAMA_MODELS", p) var s Server - w := createRequest(t, s.CreateModelHandler, api.CreateRequest{ + w := createRequest(t, s.CreateHandler, api.CreateRequest{ Name: "test", Modelfile: fmt.Sprintf("FROM %s", createBinFile(t, nil, nil)), Stream: &stream, @@ -134,7 +134,7 @@ func TestCreateFromModel(t *testing.T) { filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"), }) - w = createRequest(t, s.CreateModelHandler, api.CreateRequest{ + w = createRequest(t, s.CreateHandler, api.CreateRequest{ Name: "test2", Modelfile: "FROM test", Stream: &stream, @@ -162,7 +162,7 @@ func TestCreateRemovesLayers(t *testing.T) { t.Setenv("OLLAMA_MODELS", p) var s Server - w := createRequest(t, s.CreateModelHandler, api.CreateRequest{ + w := createRequest(t, s.CreateHandler, api.CreateRequest{ Name: "test", Modelfile: fmt.Sprintf("FROM %s\nTEMPLATE {{ .Prompt }}", createBinFile(t, nil, nil)), Stream: &stream, @@ -182,7 +182,7 @@ func TestCreateRemovesLayers(t *testing.T) { filepath.Join(p, "blobs", "sha256-bc80b03733773e0728011b2f4adf34c458b400e1aad48cb28d61170f3a2ad2d6"), }) - w = createRequest(t, s.CreateModelHandler, api.CreateRequest{ + w = createRequest(t, s.CreateHandler, api.CreateRequest{ Name: "test", Modelfile: fmt.Sprintf("FROM %s\nTEMPLATE {{ .System }} {{ .Prompt }}", createBinFile(t, nil, nil)), Stream: &stream, @@ -210,7 +210,7 @@ func TestCreateUnsetsSystem(t *testing.T) { t.Setenv("OLLAMA_MODELS", p) var s Server - w := createRequest(t, s.CreateModelHandler, api.CreateRequest{ + w := createRequest(t, s.CreateHandler, api.CreateRequest{ Name: "test", Modelfile: fmt.Sprintf("FROM %s\nSYSTEM Say hi!", createBinFile(t, nil, nil)), Stream: &stream, @@ -230,7 +230,7 @@ func TestCreateUnsetsSystem(t *testing.T) { filepath.Join(p, "blobs", "sha256-f29e82a8284dbdf5910b1555580ff60b04238b8da9d5e51159ada67a4d0d5851"), }) - w = createRequest(t, s.CreateModelHandler, api.CreateRequest{ + w = createRequest(t, s.CreateHandler, api.CreateRequest{ Name: "test", Modelfile: fmt.Sprintf("FROM %s\nSYSTEM \"\"", createBinFile(t, nil, nil)), Stream: &stream, @@ -267,7 +267,7 @@ func TestCreateMergeParameters(t *testing.T) { t.Setenv("OLLAMA_MODELS", p) var s Server - w := createRequest(t, s.CreateModelHandler, api.CreateRequest{ + w := createRequest(t, s.CreateHandler, api.CreateRequest{ Name: "test", Modelfile: fmt.Sprintf("FROM %s\nPARAMETER temperature 1\nPARAMETER top_k 10\nPARAMETER stop USER:\nPARAMETER stop ASSISTANT:", createBinFile(t, nil, nil)), Stream: &stream, @@ -288,7 +288,7 @@ func TestCreateMergeParameters(t *testing.T) { }) // in order to merge parameters, the second model must be created FROM the first - w = createRequest(t, s.CreateModelHandler, api.CreateRequest{ + w = createRequest(t, s.CreateHandler, api.CreateRequest{ Name: "test2", Modelfile: "FROM test\nPARAMETER temperature 0.6\nPARAMETER top_p 0.7", Stream: &stream, @@ -326,7 +326,7 @@ func TestCreateMergeParameters(t *testing.T) { } // slices are replaced - w = createRequest(t, s.CreateModelHandler, api.CreateRequest{ + w = createRequest(t, s.CreateHandler, api.CreateRequest{ Name: "test2", Modelfile: "FROM test\nPARAMETER temperature 0.6\nPARAMETER top_p 0.7\nPARAMETER stop <|endoftext|>", Stream: &stream, @@ -371,7 +371,7 @@ func TestCreateReplacesMessages(t *testing.T) { t.Setenv("OLLAMA_MODELS", p) var s Server - w := createRequest(t, s.CreateModelHandler, api.CreateRequest{ + w := createRequest(t, s.CreateHandler, api.CreateRequest{ Name: "test", Modelfile: fmt.Sprintf("FROM %s\nMESSAGE assistant \"What is my purpose?\"\nMESSAGE user \"You run tests.\"\nMESSAGE assistant \"Oh, my god.\"", createBinFile(t, nil, nil)), Stream: &stream, @@ -391,7 +391,7 @@ func TestCreateReplacesMessages(t *testing.T) { filepath.Join(p, "blobs", "sha256-e0e27d47045063ccb167ae852c51d49a98eab33fabaee4633fdddf97213e40b5"), }) - w = createRequest(t, s.CreateModelHandler, api.CreateRequest{ + w = createRequest(t, s.CreateHandler, api.CreateRequest{ Name: "test2", Modelfile: "FROM test\nMESSAGE assistant \"You're a test, Harry.\"\nMESSAGE user \"I-I'm a what?\"\nMESSAGE assistant \"A test. And a thumping good one at that, I'd wager.\"", Stream: &stream, @@ -448,7 +448,7 @@ func TestCreateTemplateSystem(t *testing.T) { t.Setenv("OLLAMA_MODELS", p) var s Server - w := createRequest(t, s.CreateModelHandler, api.CreateRequest{ + w := createRequest(t, s.CreateHandler, api.CreateRequest{ Name: "test", Modelfile: fmt.Sprintf("FROM %s\nTEMPLATE {{ .Prompt }}\nSYSTEM Say hello!\nTEMPLATE {{ .System }} {{ .Prompt }}\nSYSTEM Say bye!", createBinFile(t, nil, nil)), Stream: &stream, @@ -488,7 +488,7 @@ func TestCreateTemplateSystem(t *testing.T) { } t.Run("incomplete template", func(t *testing.T) { - w := createRequest(t, s.CreateModelHandler, api.CreateRequest{ + w := createRequest(t, s.CreateHandler, api.CreateRequest{ Name: "test", Modelfile: fmt.Sprintf("FROM %s\nTEMPLATE {{ .Prompt", createBinFile(t, nil, nil)), Stream: &stream, @@ -500,7 +500,7 @@ func TestCreateTemplateSystem(t *testing.T) { }) t.Run("template with unclosed if", func(t *testing.T) { - w := createRequest(t, s.CreateModelHandler, api.CreateRequest{ + w := createRequest(t, s.CreateHandler, api.CreateRequest{ Name: "test", Modelfile: fmt.Sprintf("FROM %s\nTEMPLATE {{ if .Prompt }}", createBinFile(t, nil, nil)), Stream: &stream, @@ -512,7 +512,7 @@ func TestCreateTemplateSystem(t *testing.T) { }) t.Run("template with undefined function", func(t *testing.T) { - w := createRequest(t, s.CreateModelHandler, api.CreateRequest{ + w := createRequest(t, s.CreateHandler, api.CreateRequest{ Name: "test", Modelfile: fmt.Sprintf("FROM %s\nTEMPLATE {{ Prompt }}", createBinFile(t, nil, nil)), Stream: &stream, @@ -531,7 +531,7 @@ func TestCreateLicenses(t *testing.T) { t.Setenv("OLLAMA_MODELS", p) var s Server - w := createRequest(t, s.CreateModelHandler, api.CreateRequest{ + w := createRequest(t, s.CreateHandler, api.CreateRequest{ Name: "test", Modelfile: fmt.Sprintf("FROM %s\nLICENSE MIT\nLICENSE Apache-2.0", createBinFile(t, nil, nil)), Stream: &stream, @@ -579,7 +579,7 @@ func TestCreateDetectTemplate(t *testing.T) { var s Server t.Run("matched", func(t *testing.T) { - w := createRequest(t, s.CreateModelHandler, api.CreateRequest{ + w := createRequest(t, s.CreateHandler, api.CreateRequest{ Name: "test", Modelfile: fmt.Sprintf("FROM %s", createBinFile(t, llm.KV{ "tokenizer.chat_template": "{{ bos_token }}{% for message in messages %}{{'<|' + message['role'] + '|>' + '\n' + message['content'] + '<|end|>\n' }}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% else %}{{ eos_token }}{% endif %}", @@ -600,7 +600,7 @@ func TestCreateDetectTemplate(t *testing.T) { }) t.Run("unmatched", func(t *testing.T) { - w := createRequest(t, s.CreateModelHandler, api.CreateRequest{ + w := createRequest(t, s.CreateHandler, api.CreateRequest{ Name: "test", Modelfile: fmt.Sprintf("FROM %s", createBinFile(t, nil, nil)), Stream: &stream, diff --git a/server/routes_delete_test.go b/server/routes_delete_test.go index 82fac9f5..5a337e79 100644 --- a/server/routes_delete_test.go +++ b/server/routes_delete_test.go @@ -22,7 +22,7 @@ func TestDelete(t *testing.T) { var s Server - w := createRequest(t, s.CreateModelHandler, api.CreateRequest{ + w := createRequest(t, s.CreateHandler, api.CreateRequest{ Name: "test", Modelfile: fmt.Sprintf("FROM %s", createBinFile(t, nil, nil)), }) @@ -31,7 +31,7 @@ func TestDelete(t *testing.T) { t.Fatalf("expected status code 200, actual %d", w.Code) } - w = createRequest(t, s.CreateModelHandler, api.CreateRequest{ + w = createRequest(t, s.CreateHandler, api.CreateRequest{ Name: "test2", Modelfile: fmt.Sprintf("FROM %s\nTEMPLATE {{ .System }} {{ .Prompt }}", createBinFile(t, nil, nil)), }) @@ -52,7 +52,7 @@ func TestDelete(t *testing.T) { filepath.Join(p, "blobs", "sha256-fe7ac77b725cda2ccad03f88a880ecdfd7a33192d6cae08fce2c0ee1455991ed"), }) - w = createRequest(t, s.DeleteModelHandler, api.DeleteRequest{Name: "test"}) + w = createRequest(t, s.DeleteHandler, api.DeleteRequest{Name: "test"}) if w.Code != http.StatusOK { t.Fatalf("expected status code 200, actual %d", w.Code) @@ -68,7 +68,7 @@ func TestDelete(t *testing.T) { filepath.Join(p, "blobs", "sha256-fe7ac77b725cda2ccad03f88a880ecdfd7a33192d6cae08fce2c0ee1455991ed"), }) - w = createRequest(t, s.DeleteModelHandler, api.DeleteRequest{Name: "test2"}) + w = createRequest(t, s.DeleteHandler, api.DeleteRequest{Name: "test2"}) if w.Code != http.StatusOK { t.Fatalf("expected status code 200, actual %d", w.Code) @@ -102,7 +102,7 @@ func TestDeleteDuplicateLayers(t *testing.T) { t.Fatal(err) } - w := createRequest(t, s.DeleteModelHandler, api.DeleteRequest{Name: "test"}) + w := createRequest(t, s.DeleteHandler, api.DeleteRequest{Name: "test"}) if w.Code != http.StatusOK { t.Errorf("expected status code 200, actual %d", w.Code) } diff --git a/server/routes_generate_test.go b/server/routes_generate_test.go index 5c0caff1..480b9672 100644 --- a/server/routes_generate_test.go +++ b/server/routes_generate_test.go @@ -84,7 +84,7 @@ func TestGenerateChat(t *testing.T) { go s.sched.Run(context.TODO()) - w := createRequest(t, s.CreateModelHandler, api.CreateRequest{ + w := createRequest(t, s.CreateHandler, api.CreateRequest{ Model: "test", Modelfile: fmt.Sprintf(`FROM %s TEMPLATE """ @@ -144,7 +144,7 @@ func TestGenerateChat(t *testing.T) { }) t.Run("missing capabilities chat", func(t *testing.T) { - w := createRequest(t, s.CreateModelHandler, api.CreateRequest{ + w := createRequest(t, s.CreateHandler, api.CreateRequest{ Model: "bert", Modelfile: fmt.Sprintf("FROM %s", createBinFile(t, llm.KV{ "general.architecture": "bert", @@ -270,7 +270,7 @@ func TestGenerateChat(t *testing.T) { checkChatResponse(t, w.Body, "test", "Hi!") }) - w = createRequest(t, s.CreateModelHandler, api.CreateRequest{ + w = createRequest(t, s.CreateHandler, api.CreateRequest{ Model: "test-system", Modelfile: "FROM test\nSYSTEM You are a helpful assistant.", }) @@ -382,7 +382,7 @@ func TestGenerate(t *testing.T) { go s.sched.Run(context.TODO()) - w := createRequest(t, s.CreateModelHandler, api.CreateRequest{ + w := createRequest(t, s.CreateHandler, api.CreateRequest{ Model: "test", Modelfile: fmt.Sprintf(`FROM %s TEMPLATE """ @@ -442,7 +442,7 @@ func TestGenerate(t *testing.T) { }) t.Run("missing capabilities generate", func(t *testing.T) { - w := createRequest(t, s.CreateModelHandler, api.CreateRequest{ + w := createRequest(t, s.CreateHandler, api.CreateRequest{ Model: "bert", Modelfile: fmt.Sprintf("FROM %s", createBinFile(t, llm.KV{ "general.architecture": "bert", @@ -583,7 +583,7 @@ func TestGenerate(t *testing.T) { checkGenerateResponse(t, w.Body, "test", "Hi!") }) - w = createRequest(t, s.CreateModelHandler, api.CreateRequest{ + w = createRequest(t, s.CreateHandler, api.CreateRequest{ Model: "test-system", Modelfile: "FROM test\nSYSTEM You are a helpful assistant.", }) @@ -652,7 +652,7 @@ func TestGenerate(t *testing.T) { checkGenerateResponse(t, w.Body, "test-system", "Abra kadabra!") }) - w = createRequest(t, s.CreateModelHandler, api.CreateRequest{ + w = createRequest(t, s.CreateHandler, api.CreateRequest{ Model: "test-suffix", Modelfile: `FROM test TEMPLATE """{{- if .Suffix }}
 {{ .Prompt }} {{ .Suffix }} 
diff --git a/server/routes_list_test.go b/server/routes_list_test.go
index 6e92b7a1..56b40830 100644
--- a/server/routes_list_test.go
+++ b/server/routes_list_test.go
@@ -31,13 +31,13 @@ func TestList(t *testing.T) {
 
 	var s Server
 	for _, n := range expectNames {
-		createRequest(t, s.CreateModelHandler, api.CreateRequest{
+		createRequest(t, s.CreateHandler, api.CreateRequest{
 			Name:      n,
 			Modelfile: fmt.Sprintf("FROM %s", createBinFile(t, nil, nil)),
 		})
 	}
 
-	w := createRequest(t, s.ListModelsHandler, nil)
+	w := createRequest(t, s.ListHandler, nil)
 	if w.Code != http.StatusOK {
 		t.Fatalf("expected status code 200, actual %d", w.Code)
 	}
diff --git a/server/routes_test.go b/server/routes_test.go
index 242875d6..bffcea20 100644
--- a/server/routes_test.go
+++ b/server/routes_test.go
@@ -318,7 +318,7 @@ func TestCase(t *testing.T) {
 	var s Server
 	for _, tt := range cases {
 		t.Run(tt, func(t *testing.T) {
-			w := createRequest(t, s.CreateModelHandler, api.CreateRequest{
+			w := createRequest(t, s.CreateHandler, api.CreateRequest{
 				Name:      tt,
 				Modelfile: fmt.Sprintf("FROM %s", createBinFile(t, nil, nil)),
 				Stream:    &stream,
@@ -334,7 +334,7 @@ func TestCase(t *testing.T) {
 			}
 
 			t.Run("create", func(t *testing.T) {
-				w = createRequest(t, s.CreateModelHandler, api.CreateRequest{
+				w = createRequest(t, s.CreateHandler, api.CreateRequest{
 					Name:      strings.ToUpper(tt),
 					Modelfile: fmt.Sprintf("FROM %s", createBinFile(t, nil, nil)),
 					Stream:    &stream,
@@ -350,7 +350,7 @@ func TestCase(t *testing.T) {
 			})
 
 			t.Run("pull", func(t *testing.T) {
-				w := createRequest(t, s.PullModelHandler, api.PullRequest{
+				w := createRequest(t, s.PullHandler, api.PullRequest{
 					Name:   strings.ToUpper(tt),
 					Stream: &stream,
 				})
@@ -365,7 +365,7 @@ func TestCase(t *testing.T) {
 			})
 
 			t.Run("copy", func(t *testing.T) {
-				w := createRequest(t, s.CopyModelHandler, api.CopyRequest{
+				w := createRequest(t, s.CopyHandler, api.CopyRequest{
 					Source:      tt,
 					Destination: strings.ToUpper(tt),
 				})
@@ -387,7 +387,7 @@ func TestShow(t *testing.T) {
 
 	var s Server
 
-	createRequest(t, s.CreateModelHandler, api.CreateRequest{
+	createRequest(t, s.CreateHandler, api.CreateRequest{
 		Name: "show-model",
 		Modelfile: fmt.Sprintf(
 			"FROM %s\nFROM %s",
@@ -396,7 +396,7 @@ func TestShow(t *testing.T) {
 		),
 	})
 
-	w := createRequest(t, s.ShowModelHandler, api.ShowRequest{
+	w := createRequest(t, s.ShowHandler, api.ShowRequest{
 		Name: "show-model",
 	})
 
diff --git a/template/index.json b/template/index.json
index e2d41893..0ce6ac0f 100644
--- a/template/index.json
+++ b/template/index.json
@@ -91,6 +91,10 @@
     "template": "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}",
     "name": "llama3-instruct"
   },
+  {
+    "template": "{{- bos_token }}\n{%- if custom_tools is defined %}\n    {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n    {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n    {%- set date_string = \"26 Jul 2024\" %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content']|trim %}\n    {%- set messages = messages[1:] %}\n{%- else %}\n    {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message + builtin tools #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if builtin_tools is defined or tools is not none %}\n    {{- \"Environment: ipython\\n\" }}\n{%- endif %}\n{%- if builtin_tools is defined %}\n    {{- \"Tools: \" + builtin_tools | reject('equalto', 'code_interpreter') | join(\", \") + \"\\n\\n\"}}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\" }}\n{{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n    {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n    {#- Extract the first user message so we can plug it in here #}\n    {%- if messages | length != 0 %}\n        {%- set first_user_message = messages[0]['content']|trim %}\n        {%- set messages = messages[1:] %}\n    {%- else %}\n        {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n    {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n    {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n    {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n    {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n    {%- elif 'tool_calls' in message %}\n        {%- if not message.tool_calls|length == 1 %}\n            {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n        {%- endif %}\n        {%- set tool_call = message.tool_calls[0].function %}\n        {%- if builtin_tools is defined and tool_call.name in builtin_tools %}\n            {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n            {{- \"<|python_tag|>\" + tool_call.name + \".call(\" }}\n            {%- for arg_name, arg_val in tool_call.arguments | items %}\n                {{- arg_name + '=\"' + arg_val + '\"' }}\n                {%- if not loop.last %}\n                    {{- \", \" }}\n                {%- endif %}\n                {%- endfor %}\n            {{- \")\" }}\n        {%- else  %}\n            {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n            {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n            {{- '\"parameters\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- \"}\" }}\n        {%- endif %}\n        {%- if builtin_tools is defined %}\n            {#- This means we're in ipython mode #}\n            {{- \"<|eom_id|>\" }}\n        {%- else %}\n            {{- \"<|eot_id|>\" }}\n        {%- endif %}\n    {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n        {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n        {%- if message.content is mapping or message.content is iterable %}\n            {{- message.content | tojson }}\n        {%- else %}\n            {{- message.content }}\n        {%- endif %}\n        {{- \"<|eot_id|>\" }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n",
+    "name": "llama3-instruct"
+  },
   {
     "template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ 'Question:\n' + message['content'] + '\n\n' }}{% elif message['role'] == 'system' %}\n{{ 'System:\n' + message['content'] + '\n\n' }}{% elif message['role'] == 'assistant' %}{{ 'Answer:\n'  + message['content'] + '\n\n' }}{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ 'Answer:\n' }}{% endif %}{% endfor %}",
     "name": "granite-instruct"