From d5eec16d2311b244e93f89027c3b716adfabee70 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Thu, 9 May 2024 09:06:13 -0700 Subject: [PATCH] use model defaults for `num_gqa`, `rope_frequency_base ` and `rope_frequency_scale` (#1983) --- api/types.go | 132 ++++++++++++++++++++++------------------------- docs/api.md | 3 -- server/routes.go | 12 ----- 3 files changed, 61 insertions(+), 86 deletions(-) diff --git a/api/types.go b/api/types.go index 5d0212e5..380c179f 100644 --- a/api/types.go +++ b/api/types.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "log/slog" "math" "os" "reflect" @@ -161,7 +162,6 @@ type Runner struct { UseNUMA bool `json:"numa,omitempty"` NumCtx int `json:"num_ctx,omitempty"` NumBatch int `json:"num_batch,omitempty"` - NumGQA int `json:"num_gqa,omitempty"` NumGPU int `json:"num_gpu,omitempty"` MainGPU int `json:"main_gpu,omitempty"` LowVRAM bool `json:"low_vram,omitempty"` @@ -171,11 +171,6 @@ type Runner struct { UseMMap bool `json:"use_mmap,omitempty"` UseMLock bool `json:"use_mlock,omitempty"` NumThread int `json:"num_thread,omitempty"` - - // Unused: RopeFrequencyBase is ignored. Instead the value in the model will be used - RopeFrequencyBase float32 `json:"rope_frequency_base,omitempty"` - // Unused: RopeFrequencyScale is ignored. Instead the value in the model will be used - RopeFrequencyScale float32 `json:"rope_frequency_scale,omitempty"` } // EmbeddingRequest is the request passed to [Client.Embeddings]. @@ -359,8 +354,6 @@ func (m *Metrics) Summary() { } } -// ErrInvalidOpts is returned when invalid options are passed to the client. -var ErrInvalidOpts = errors.New("invalid options") var ErrInvalidHostPort = errors.New("invalid port specified in OLLAMA_HOST") func (opts *Options) FromMap(m map[string]interface{}) error { @@ -376,73 +369,71 @@ func (opts *Options) FromMap(m map[string]interface{}) error { } } - invalidOpts := []string{} for key, val := range m { - if opt, ok := jsonOpts[key]; ok { - field := valueOpts.FieldByName(opt.Name) - if field.IsValid() && field.CanSet() { - if val == nil { - continue - } + opt, ok := jsonOpts[key] + if !ok { + slog.Warn("invalid option provided", "option", opt.Name) + continue + } - switch field.Kind() { - case reflect.Int: - switch t := val.(type) { - case int64: - field.SetInt(t) - case float64: - // when JSON unmarshals numbers, it uses float64, not int - field.SetInt(int64(t)) - default: - return fmt.Errorf("option %q must be of type integer", key) - } - case reflect.Bool: - val, ok := val.(bool) - if !ok { - return fmt.Errorf("option %q must be of type boolean", key) - } - field.SetBool(val) - case reflect.Float32: - // JSON unmarshals to float64 - val, ok := val.(float64) - if !ok { - return fmt.Errorf("option %q must be of type float32", key) - } - field.SetFloat(val) - case reflect.String: - val, ok := val.(string) - if !ok { - return fmt.Errorf("option %q must be of type string", key) - } - field.SetString(val) - case reflect.Slice: - // JSON unmarshals to []interface{}, not []string - val, ok := val.([]interface{}) - if !ok { - return fmt.Errorf("option %q must be of type array", key) - } - // convert []interface{} to []string - slice := make([]string, len(val)) - for i, item := range val { - str, ok := item.(string) - if !ok { - return fmt.Errorf("option %q must be of an array of strings", key) - } - slice[i] = str - } - field.Set(reflect.ValueOf(slice)) - default: - return fmt.Errorf("unknown type loading config params: %v", field.Kind()) - } + field := valueOpts.FieldByName(opt.Name) + if field.IsValid() && field.CanSet() { + if val == nil { + continue + } + + switch field.Kind() { + case reflect.Int: + switch t := val.(type) { + case int64: + field.SetInt(t) + case float64: + // when JSON unmarshals numbers, it uses float64, not int + field.SetInt(int64(t)) + default: + return fmt.Errorf("option %q must be of type integer", key) + } + case reflect.Bool: + val, ok := val.(bool) + if !ok { + return fmt.Errorf("option %q must be of type boolean", key) + } + field.SetBool(val) + case reflect.Float32: + // JSON unmarshals to float64 + val, ok := val.(float64) + if !ok { + return fmt.Errorf("option %q must be of type float32", key) + } + field.SetFloat(val) + case reflect.String: + val, ok := val.(string) + if !ok { + return fmt.Errorf("option %q must be of type string", key) + } + field.SetString(val) + case reflect.Slice: + // JSON unmarshals to []interface{}, not []string + val, ok := val.([]interface{}) + if !ok { + return fmt.Errorf("option %q must be of type array", key) + } + // convert []interface{} to []string + slice := make([]string, len(val)) + for i, item := range val { + str, ok := item.(string) + if !ok { + return fmt.Errorf("option %q must be of an array of strings", key) + } + slice[i] = str + } + field.Set(reflect.ValueOf(slice)) + default: + return fmt.Errorf("unknown type loading config params: %v", field.Kind()) } - } else { - invalidOpts = append(invalidOpts, key) } } - if len(invalidOpts) > 0 { - return fmt.Errorf("%w: %v", ErrInvalidOpts, strings.Join(invalidOpts, ", ")) - } return nil } @@ -475,8 +466,7 @@ func DefaultOptions() Options { NumCtx: 2048, NumBatch: 512, NumGPU: -1, // -1 here indicates that NumGPU should be set dynamically - NumGQA: 1, - NumThread: 0, // let the runtime decide + NumThread: 0, // let the runtime decide LowVRAM: false, F16KV: true, UseMLock: false, diff --git a/docs/api.md b/docs/api.md index 2f52c55a..94cd9c90 100644 --- a/docs/api.md +++ b/docs/api.md @@ -313,7 +313,6 @@ curl http://localhost:11434/api/generate -d '{ "numa": false, "num_ctx": 1024, "num_batch": 2, - "num_gqa": 1, "num_gpu": 1, "main_gpu": 0, "low_vram": false, @@ -321,8 +320,6 @@ curl http://localhost:11434/api/generate -d '{ "vocab_only": false, "use_mmap": true, "use_mlock": false, - "rope_frequency_base": 1.1, - "rope_frequency_scale": 0.8, "num_thread": 8 } }' diff --git a/server/routes.go b/server/routes.go index 7dfeb513..4121483e 100644 --- a/server/routes.go +++ b/server/routes.go @@ -127,10 +127,6 @@ func (s *Server) GenerateHandler(c *gin.Context) { opts, err := modelOptions(model, req.Options) if err != nil { - if errors.Is(err, api.ErrInvalidOpts) { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) - return - } c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } @@ -370,10 +366,6 @@ func (s *Server) EmbeddingsHandler(c *gin.Context) { opts, err := modelOptions(model, req.Options) if err != nil { - if errors.Is(err, api.ErrInvalidOpts) { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) - return - } c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } @@ -1177,10 +1169,6 @@ func (s *Server) ChatHandler(c *gin.Context) { opts, err := modelOptions(model, req.Options) if err != nil { - if errors.Is(err, api.ErrInvalidOpts) { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) - return - } c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return }