ollama/server/routes.go

623 lines
14 KiB
Go
Raw Normal View History

package server
import (
"context"
2023-07-06 17:40:11 +00:00
"encoding/json"
"fmt"
"io"
"log"
"net"
"net/http"
"os"
"os/signal"
2023-07-15 00:27:14 +00:00
"path/filepath"
2023-08-01 01:35:18 +00:00
"reflect"
"runtime"
2023-09-06 18:04:17 +00:00
"strconv"
2023-07-06 17:40:11 +00:00
"strings"
2023-07-18 18:59:42 +00:00
"sync"
"syscall"
2023-07-13 01:18:06 +00:00
"time"
2023-07-22 01:01:24 +00:00
"github.com/gin-contrib/cors"
"github.com/gin-gonic/gin"
"gonum.org/v1/gonum/mat"
2023-07-03 20:32:48 +00:00
"github.com/jmorganca/ollama/api"
2023-07-21 20:33:56 +00:00
"github.com/jmorganca/ollama/llm"
2023-08-04 22:56:40 +00:00
"github.com/jmorganca/ollama/vector"
)
2023-08-22 16:48:35 +00:00
var mode string = gin.DebugMode
func init() {
switch mode {
case gin.DebugMode:
case gin.ReleaseMode:
case gin.TestMode:
default:
mode = gin.DebugMode
}
gin.SetMode(mode)
}
2023-08-01 01:35:18 +00:00
var loaded struct {
2023-07-19 22:00:28 +00:00
mu sync.Mutex
2023-07-21 20:33:56 +00:00
llm llm.LLM
2023-08-04 22:56:40 +00:00
Embeddings []vector.Embedding
2023-07-19 22:00:28 +00:00
expireAt time.Time
expireTimer *time.Timer
2023-08-01 01:35:18 +00:00
digest string
options api.Options
2023-07-18 18:59:42 +00:00
}
2023-08-15 13:35:39 +00:00
var defaultSessionDuration = 5 * time.Minute
// load a model into memory if it is not already loaded, it is up to the caller to lock loaded.mu before calling this function
func load(ctx context.Context, workDir string, model *Model, reqOpts map[string]interface{}, sessionDuration time.Duration) error {
2023-08-03 19:55:35 +00:00
opts := api.DefaultOptions()
if err := opts.FromMap(model.Options); err != nil {
log.Printf("could not load model options: %v", err)
return err
2023-08-03 19:55:35 +00:00
}
if err := opts.FromMap(reqOpts); err != nil {
2023-08-03 19:55:35 +00:00
log.Printf("could not merge model options: %v", err)
return err
2023-08-03 19:55:35 +00:00
}
// check if the loaded model is still running in a subprocess, in case something unexpected happened
if loaded.llm != nil {
if err := loaded.llm.Ping(ctx); err != nil {
log.Print("loaded llm process not responding, closing now")
// the subprocess is no longer running, so close it
loaded.llm.Close()
loaded.llm = nil
loaded.digest = ""
}
}
2023-08-03 19:55:35 +00:00
if model.Digest != loaded.digest || !reflect.DeepEqual(loaded.options, opts) {
2023-08-01 01:35:18 +00:00
if loaded.llm != nil {
log.Println("changing loaded model")
2023-08-01 01:35:18 +00:00
loaded.llm.Close()
loaded.llm = nil
loaded.digest = ""
2023-07-18 18:59:42 +00:00
}
2023-07-17 19:08:10 +00:00
2023-08-04 22:56:40 +00:00
if model.Embeddings != nil && len(model.Embeddings) > 0 {
opts.EmbeddingOnly = true // this is requried to generate embeddings, completions will still work
loaded.Embeddings = model.Embeddings
}
llmModel, err := llm.New(workDir, model.ModelPath, model.AdapterPaths, opts)
2023-07-18 18:59:42 +00:00
if err != nil {
return err
2023-07-18 18:59:42 +00:00
}
2023-07-21 20:33:56 +00:00
// set cache values before modifying opts
loaded.llm = llmModel
loaded.digest = model.Digest
loaded.options = opts
if opts.NumKeep < 0 {
2023-08-09 14:45:57 +00:00
promptWithSystem, err := model.Prompt(api.GenerateRequest{}, "")
if err != nil {
return err
}
2023-08-09 14:45:57 +00:00
promptNoSystem, err := model.Prompt(api.GenerateRequest{Context: []int{0}}, "")
if err != nil {
return err
}
tokensWithSystem, err := llmModel.Encode(ctx, promptWithSystem)
if err != nil {
return err
}
2023-09-03 21:36:14 +00:00
tokensNoSystem, err := llmModel.Encode(ctx, promptNoSystem)
if err != nil {
return err
}
2023-09-03 21:36:14 +00:00
opts.NumKeep = len(tokensWithSystem) - len(tokensNoSystem)
2023-07-21 20:33:56 +00:00
llmModel.SetOptions(opts)
}
2023-07-19 22:00:28 +00:00
}
2023-08-01 01:35:18 +00:00
loaded.expireAt = time.Now().Add(sessionDuration)
2023-08-01 01:35:18 +00:00
if loaded.expireTimer == nil {
loaded.expireTimer = time.AfterFunc(sessionDuration, func() {
loaded.mu.Lock()
defer loaded.mu.Unlock()
2023-07-19 22:00:28 +00:00
2023-08-01 01:35:18 +00:00
if time.Now().Before(loaded.expireAt) {
2023-07-19 22:00:28 +00:00
return
}
2023-08-01 01:35:18 +00:00
if loaded.llm == nil {
2023-07-19 22:00:28 +00:00
return
}
2023-08-01 01:35:18 +00:00
loaded.llm.Close()
loaded.llm = nil
loaded.digest = ""
2023-07-19 22:00:28 +00:00
})
2023-07-06 17:40:11 +00:00
}
2023-08-01 01:35:18 +00:00
loaded.expireTimer.Reset(sessionDuration)
return nil
}
func GenerateHandler(c *gin.Context) {
loaded.mu.Lock()
defer loaded.mu.Unlock()
checkpointStart := time.Now()
var req api.GenerateRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
model, err := GetModel(req.Model)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
workDir := c.GetString("workDir")
// TODO: set this duration from the request if specified
sessionDuration := defaultSessionDuration
if err := load(c.Request.Context(), workDir, model, req.Options, sessionDuration); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
2023-07-06 17:40:11 +00:00
2023-07-18 19:02:02 +00:00
checkpointLoaded := time.Now()
embedding := ""
if model.Embeddings != nil && len(model.Embeddings) > 0 {
promptEmbed, err := loaded.llm.Embedding(c.Request.Context(), req.Prompt)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
// TODO: set embed_top from specified parameters in modelfile
embed_top := 3
topK := vector.TopK(embed_top, mat.NewVecDense(len(promptEmbed), promptEmbed), loaded.Embeddings)
for _, e := range topK {
embedding = fmt.Sprintf("%s %s", embedding, e.Embedding.Data)
}
}
prompt, err := model.Prompt(req, embedding)
2023-07-11 21:57:17 +00:00
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
2023-07-04 04:47:00 +00:00
ch := make(chan any)
go func() {
defer close(ch)
2023-07-20 19:12:08 +00:00
fn := func(r api.GenerateResponse) {
2023-08-01 01:35:18 +00:00
loaded.expireAt = time.Now().Add(sessionDuration)
loaded.expireTimer.Reset(sessionDuration)
2023-07-19 22:00:28 +00:00
r.Model = req.Model
r.CreatedAt = time.Now().UTC()
if r.Done {
2023-07-18 19:02:02 +00:00
r.TotalDuration = time.Since(checkpointStart)
r.LoadDuration = checkpointLoaded.Sub(checkpointStart)
}
ch <- r
2023-07-20 19:12:08 +00:00
}
// an empty request loads the model
if req.Prompt == "" && req.Template == "" && req.System == "" {
2023-09-18 19:26:56 +00:00
ch <- api.GenerateResponse{Model: req.Model, Done: true}
} else {
if err := loaded.llm.Predict(c.Request.Context(), req.Context, prompt, fn); err != nil {
ch <- gin.H{"error": err.Error()}
}
2023-07-20 19:12:08 +00:00
}
}()
2023-07-11 21:57:17 +00:00
streamResponse(c, ch)
2023-07-11 18:54:22 +00:00
}
2023-07-06 17:40:11 +00:00
func EmbeddingHandler(c *gin.Context) {
loaded.mu.Lock()
defer loaded.mu.Unlock()
var req api.EmbeddingRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
model, err := GetModel(req.Model)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
workDir := c.GetString("workDir")
if err := load(c.Request.Context(), workDir, model, req.Options, 5*time.Minute); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
if !loaded.options.EmbeddingOnly {
c.JSON(http.StatusBadRequest, gin.H{"error": "embedding option must be set to true"})
return
}
embedding, err := loaded.llm.Embedding(c.Request.Context(), req.Prompt)
if err != nil {
log.Printf("embedding generation failed: %v", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to generate embedding"})
return
}
resp := api.EmbeddingResponse{
Embedding: embedding,
}
c.JSON(http.StatusOK, resp)
}
2023-07-20 23:09:23 +00:00
func PullModelHandler(c *gin.Context) {
2023-07-11 18:54:22 +00:00
var req api.PullRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
ch := make(chan any)
go func() {
defer close(ch)
2023-07-19 01:51:30 +00:00
fn := func(r api.ProgressResponse) {
ch <- r
}
2023-07-19 01:51:30 +00:00
regOpts := &RegistryOptions{
Insecure: req.Insecure,
}
ctx, cancel := context.WithCancel(c.Request.Context())
defer cancel()
if err := PullModel(ctx, req.Name, regOpts, fn); err != nil {
2023-07-20 19:12:08 +00:00
ch <- gin.H{"error": err.Error()}
}
}()
streamResponse(c, ch)
}
2023-07-20 23:09:23 +00:00
func PushModelHandler(c *gin.Context) {
var req api.PushRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
2023-07-11 18:54:22 +00:00
return
}
2023-07-06 17:40:11 +00:00
ch := make(chan any)
go func() {
defer close(ch)
2023-07-19 01:51:30 +00:00
fn := func(r api.ProgressResponse) {
ch <- r
}
2023-07-19 01:51:30 +00:00
regOpts := &RegistryOptions{
Insecure: req.Insecure,
}
ctx := context.Background()
if err := PushModel(ctx, req.Name, regOpts, fn); err != nil {
2023-07-20 19:12:08 +00:00
ch <- gin.H{"error": err.Error()}
}
}()
streamResponse(c, ch)
}
2023-07-20 23:09:23 +00:00
func CreateModelHandler(c *gin.Context) {
var req api.CreateRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"message": err.Error()})
2023-07-13 02:07:15 +00:00
return
}
workDir := c.GetString("workDir")
2023-07-11 18:54:22 +00:00
ch := make(chan any)
go func() {
defer close(ch)
fn := func(resp api.ProgressResponse) {
ch <- resp
}
ctx, cancel := context.WithCancel(c.Request.Context())
defer cancel()
if err := CreateModel(ctx, workDir, req.Name, req.Path, fn); err != nil {
2023-07-20 19:12:08 +00:00
ch <- gin.H{"error": err.Error()}
}
}()
2023-07-07 22:29:17 +00:00
streamResponse(c, ch)
2023-07-05 19:37:33 +00:00
}
2023-07-20 23:09:23 +00:00
func DeleteModelHandler(c *gin.Context) {
var req api.DeleteRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
if err := DeleteModel(req.Name); err != nil {
if os.IsNotExist(err) {
c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Name)})
} else {
2023-07-20 23:09:23 +00:00
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
}
return
}
2023-09-27 00:28:14 +00:00
manifestsPath, err := GetManifestPath()
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
if err := PruneDirectory(manifestsPath); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, nil)
2023-07-20 23:09:23 +00:00
}
2023-09-06 18:04:17 +00:00
func ShowModelHandler(c *gin.Context) {
var req api.ShowRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
resp, err := GetModelInfo(req.Name)
if err != nil {
if os.IsNotExist(err) {
c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Name)})
} else {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
}
return
}
c.JSON(http.StatusOK, resp)
}
func GetModelInfo(name string) (*api.ShowResponse, error) {
model, err := GetModel(name)
if err != nil {
return nil, err
}
resp := &api.ShowResponse{
License: strings.Join(model.License, "\n"),
System: model.System,
Template: model.Template,
}
mf, err := ShowModelfile(model)
if err != nil {
return nil, err
}
resp.Modelfile = mf
var params []string
cs := 30
for k, v := range model.Options {
switch val := v.(type) {
case string:
params = append(params, fmt.Sprintf("%-*s %s", cs, k, val))
case int:
params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.Itoa(val)))
case float64:
params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatFloat(val, 'f', 0, 64)))
case bool:
params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatBool(val)))
case []interface{}:
for _, nv := range val {
switch nval := nv.(type) {
case string:
params = append(params, fmt.Sprintf("%-*s %s", cs, k, nval))
case int:
params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.Itoa(nval)))
case float64:
params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatFloat(nval, 'f', 0, 64)))
case bool:
params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatBool(nval)))
}
}
}
}
resp.Parameters = strings.Join(params, "\n")
return resp, nil
}
2023-07-20 23:09:23 +00:00
func ListModelsHandler(c *gin.Context) {
2023-08-30 15:10:27 +00:00
var models []api.ModelResponse
2023-07-18 16:09:45 +00:00
fp, err := GetManifestPath()
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
2023-08-30 18:14:12 +00:00
walkFunc := func(path string, info os.FileInfo, _ error) error {
2023-07-18 16:09:45 +00:00
if !info.IsDir() {
2023-08-30 18:14:12 +00:00
dir, file := filepath.Split(path)
dir = strings.Trim(strings.TrimPrefix(dir, fp), string(os.PathSeparator))
tag := strings.Join([]string{dir, file}, ":")
2023-08-22 04:56:56 +00:00
mp := ParseModelPath(tag)
2023-08-29 03:50:24 +00:00
manifest, digest, err := GetManifest(mp)
2023-07-18 16:09:45 +00:00
if err != nil {
log.Printf("skipping file: %s", fp)
return nil
2023-07-18 16:09:45 +00:00
}
2023-08-30 18:14:12 +00:00
models = append(models, api.ModelResponse{
2023-07-18 16:09:45 +00:00
Name: mp.GetShortTagname(),
Size: manifest.GetTotalSize(),
2023-08-29 03:50:24 +00:00
Digest: digest,
2023-08-30 18:14:12 +00:00
ModifiedAt: info.ModTime(),
})
2023-07-18 16:09:45 +00:00
}
2023-08-30 18:14:12 +00:00
2023-07-18 16:09:45 +00:00
return nil
2023-08-30 18:14:12 +00:00
}
if err := filepath.Walk(fp, walkFunc); err != nil {
2023-07-18 16:09:45 +00:00
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
2023-07-19 22:00:28 +00:00
c.JSON(http.StatusOK, api.ListResponse{Models: models})
2023-07-18 16:09:45 +00:00
}
2023-07-24 15:27:28 +00:00
func CopyModelHandler(c *gin.Context) {
var req api.CopyRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
if err := CopyModel(req.Source, req.Destination); err != nil {
if os.IsNotExist(err) {
c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Source)})
} else {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
}
return
}
}
var defaultAllowOrigins = []string{
"localhost",
"127.0.0.1",
"0.0.0.0",
}
func Serve(ln net.Listener, allowOrigins []string) error {
2023-07-22 01:01:24 +00:00
config := cors.DefaultConfig()
config.AllowWildcard = true
config.AllowOrigins = allowOrigins
for _, allowOrigin := range defaultAllowOrigins {
config.AllowOrigins = append(config.AllowOrigins,
fmt.Sprintf("http://%s", allowOrigin),
fmt.Sprintf("https://%s", allowOrigin),
fmt.Sprintf("http://%s:*", allowOrigin),
fmt.Sprintf("https://%s:*", allowOrigin),
)
}
2023-07-22 01:01:24 +00:00
workDir, err := os.MkdirTemp("", "ollama")
if err != nil {
return err
}
defer os.RemoveAll(workDir)
2023-07-05 19:37:33 +00:00
r := gin.Default()
r.Use(
cors.New(config),
func(c *gin.Context) {
c.Set("workDir", workDir)
c.Next()
},
)
2023-07-05 19:37:33 +00:00
2023-07-20 23:09:23 +00:00
r.POST("/api/pull", PullModelHandler)
r.POST("/api/generate", GenerateHandler)
r.POST("/api/embeddings", EmbeddingHandler)
2023-07-20 23:09:23 +00:00
r.POST("/api/create", CreateModelHandler)
r.POST("/api/push", PushModelHandler)
2023-07-24 15:27:28 +00:00
r.POST("/api/copy", CopyModelHandler)
2023-07-20 23:09:23 +00:00
r.DELETE("/api/delete", DeleteModelHandler)
2023-09-06 18:04:17 +00:00
r.POST("/api/show", ShowModelHandler)
2023-09-21 23:38:03 +00:00
for _, method := range []string{http.MethodGet, http.MethodHead} {
r.Handle(method, "/", func(c *gin.Context) {
c.String(http.StatusOK, "Ollama is running")
})
r.Handle(method, "/api/tags", ListModelsHandler)
}
log.Printf("Listening on %s", ln.Addr())
s := &http.Server{
Handler: r,
}
// listen for a ctrl+c and stop any loaded llm
signals := make(chan os.Signal, 1)
signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
go func() {
<-signals
2023-09-22 18:41:52 +00:00
if loaded.llm != nil {
loaded.llm.Close()
}
os.RemoveAll(workDir)
os.Exit(0)
}()
if runtime.GOOS == "linux" {
// check compatibility to log warnings
if _, err := llm.CheckVRAM(); err != nil {
2023-09-20 19:00:41 +00:00
log.Printf("Warning: GPU support may not enabled, check you have installed install GPU drivers: %v", err)
}
}
return s.Serve(ln)
}
2023-07-06 17:40:11 +00:00
func streamResponse(c *gin.Context, ch chan any) {
c.Header("Content-Type", "application/x-ndjson")
2023-07-11 18:54:22 +00:00
c.Stream(func(w io.Writer) bool {
val, ok := <-ch
if !ok {
return false
}
bts, err := json.Marshal(val)
if err != nil {
2023-07-31 20:46:37 +00:00
log.Printf("streamResponse: json.Marshal failed with %s", err)
2023-07-11 18:54:22 +00:00
return false
}
// Delineate chunks with new-line delimiter
2023-07-11 18:54:22 +00:00
bts = append(bts, '\n')
if _, err := w.Write(bts); err != nil {
2023-07-31 20:46:37 +00:00
log.Printf("streamResponse: w.Write failed with %s", err)
2023-07-11 18:54:22 +00:00
return false
}
return true
})
}