2023-07-03 19:22:44 +00:00
|
|
|
package server
|
|
|
|
|
|
|
|
import (
|
2023-07-25 21:08:51 +00:00
|
|
|
"context"
|
2023-07-06 17:40:11 +00:00
|
|
|
"encoding/json"
|
2023-07-22 06:02:12 +00:00
|
|
|
"fmt"
|
2023-07-03 19:22:44 +00:00
|
|
|
"io"
|
|
|
|
"log"
|
|
|
|
"net"
|
|
|
|
"net/http"
|
2023-07-07 19:27:43 +00:00
|
|
|
"os"
|
2023-08-30 20:35:03 +00:00
|
|
|
"os/signal"
|
2023-07-15 00:27:14 +00:00
|
|
|
"path/filepath"
|
2023-08-01 01:35:18 +00:00
|
|
|
"reflect"
|
2023-09-12 15:04:35 +00:00
|
|
|
"runtime"
|
2023-09-06 18:04:17 +00:00
|
|
|
"strconv"
|
2023-07-06 17:40:11 +00:00
|
|
|
"strings"
|
2023-07-18 18:59:42 +00:00
|
|
|
"sync"
|
2023-08-30 20:35:03 +00:00
|
|
|
"syscall"
|
2023-07-13 01:18:06 +00:00
|
|
|
"time"
|
2023-07-03 19:22:44 +00:00
|
|
|
|
2023-07-22 01:01:24 +00:00
|
|
|
"github.com/gin-contrib/cors"
|
2023-07-03 19:22:44 +00:00
|
|
|
"github.com/gin-gonic/gin"
|
2023-08-08 17:49:37 +00:00
|
|
|
"gonum.org/v1/gonum/mat"
|
2023-07-03 19:22:44 +00:00
|
|
|
|
2023-07-03 20:32:48 +00:00
|
|
|
"github.com/jmorganca/ollama/api"
|
2023-07-21 20:33:56 +00:00
|
|
|
"github.com/jmorganca/ollama/llm"
|
2023-08-04 22:56:40 +00:00
|
|
|
"github.com/jmorganca/ollama/vector"
|
2023-07-03 19:22:44 +00:00
|
|
|
)
|
|
|
|
|
2023-08-22 16:48:35 +00:00
|
|
|
var mode string = gin.DebugMode
|
|
|
|
|
|
|
|
func init() {
|
|
|
|
switch mode {
|
|
|
|
case gin.DebugMode:
|
|
|
|
case gin.ReleaseMode:
|
|
|
|
case gin.TestMode:
|
|
|
|
default:
|
|
|
|
mode = gin.DebugMode
|
|
|
|
}
|
|
|
|
|
|
|
|
gin.SetMode(mode)
|
|
|
|
}
|
|
|
|
|
2023-08-01 01:35:18 +00:00
|
|
|
var loaded struct {
|
2023-07-19 22:00:28 +00:00
|
|
|
mu sync.Mutex
|
|
|
|
|
2023-07-21 20:33:56 +00:00
|
|
|
llm llm.LLM
|
2023-08-04 22:56:40 +00:00
|
|
|
Embeddings []vector.Embedding
|
2023-07-19 22:00:28 +00:00
|
|
|
|
|
|
|
expireAt time.Time
|
|
|
|
expireTimer *time.Timer
|
2023-08-01 01:35:18 +00:00
|
|
|
|
2023-08-02 20:15:40 +00:00
|
|
|
digest string
|
|
|
|
options api.Options
|
2023-07-18 18:59:42 +00:00
|
|
|
}
|
|
|
|
|
2023-08-15 13:35:39 +00:00
|
|
|
var defaultSessionDuration = 5 * time.Minute
|
|
|
|
|
2023-08-08 19:13:22 +00:00
|
|
|
// load a model into memory if it is not already loaded, it is up to the caller to lock loaded.mu before calling this function
|
2023-08-30 20:35:03 +00:00
|
|
|
func load(ctx context.Context, model *Model, reqOpts map[string]interface{}, sessionDuration time.Duration) error {
|
2023-08-03 19:55:35 +00:00
|
|
|
opts := api.DefaultOptions()
|
|
|
|
if err := opts.FromMap(model.Options); err != nil {
|
|
|
|
log.Printf("could not load model options: %v", err)
|
2023-08-08 19:13:22 +00:00
|
|
|
return err
|
2023-08-03 19:55:35 +00:00
|
|
|
}
|
|
|
|
|
2023-08-08 19:13:22 +00:00
|
|
|
if err := opts.FromMap(reqOpts); err != nil {
|
2023-08-03 19:55:35 +00:00
|
|
|
log.Printf("could not merge model options: %v", err)
|
2023-08-08 19:13:22 +00:00
|
|
|
return err
|
2023-08-03 19:55:35 +00:00
|
|
|
}
|
|
|
|
|
2023-08-30 20:35:03 +00:00
|
|
|
// check if the loaded model is still running in a subprocess, in case something unexpected happened
|
|
|
|
if loaded.llm != nil {
|
|
|
|
if err := loaded.llm.Ping(ctx); err != nil {
|
|
|
|
log.Print("loaded llm process not responding, closing now")
|
|
|
|
// the subprocess is no longer running, so close it
|
|
|
|
loaded.llm.Close()
|
|
|
|
loaded.llm = nil
|
|
|
|
loaded.digest = ""
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-08-03 19:55:35 +00:00
|
|
|
if model.Digest != loaded.digest || !reflect.DeepEqual(loaded.options, opts) {
|
2023-08-01 01:35:18 +00:00
|
|
|
if loaded.llm != nil {
|
2023-08-30 20:35:03 +00:00
|
|
|
log.Println("changing loaded model")
|
2023-08-01 01:35:18 +00:00
|
|
|
loaded.llm.Close()
|
|
|
|
loaded.llm = nil
|
2023-08-02 20:15:40 +00:00
|
|
|
loaded.digest = ""
|
2023-07-18 18:59:42 +00:00
|
|
|
}
|
2023-07-17 19:08:10 +00:00
|
|
|
|
2023-08-04 22:56:40 +00:00
|
|
|
if model.Embeddings != nil && len(model.Embeddings) > 0 {
|
|
|
|
opts.EmbeddingOnly = true // this is requried to generate embeddings, completions will still work
|
|
|
|
loaded.Embeddings = model.Embeddings
|
|
|
|
}
|
|
|
|
|
2023-08-04 00:16:05 +00:00
|
|
|
llmModel, err := llm.New(model.ModelPath, model.AdapterPaths, opts)
|
2023-07-18 18:59:42 +00:00
|
|
|
if err != nil {
|
2023-08-08 19:13:22 +00:00
|
|
|
return err
|
2023-07-18 18:59:42 +00:00
|
|
|
}
|
|
|
|
|
2023-07-21 20:33:56 +00:00
|
|
|
// set cache values before modifying opts
|
|
|
|
loaded.llm = llmModel
|
|
|
|
loaded.digest = model.Digest
|
|
|
|
loaded.options = opts
|
|
|
|
|
2023-08-07 21:51:31 +00:00
|
|
|
if opts.NumKeep < 0 {
|
2023-08-09 14:45:57 +00:00
|
|
|
promptWithSystem, err := model.Prompt(api.GenerateRequest{}, "")
|
2023-08-07 21:51:31 +00:00
|
|
|
if err != nil {
|
2023-08-08 19:13:22 +00:00
|
|
|
return err
|
2023-08-07 21:51:31 +00:00
|
|
|
}
|
|
|
|
|
2023-08-09 14:45:57 +00:00
|
|
|
promptNoSystem, err := model.Prompt(api.GenerateRequest{Context: []int{0}}, "")
|
2023-08-07 21:51:31 +00:00
|
|
|
if err != nil {
|
2023-08-08 19:13:22 +00:00
|
|
|
return err
|
2023-08-07 21:51:31 +00:00
|
|
|
}
|
|
|
|
|
2023-08-30 20:35:03 +00:00
|
|
|
tokensWithSystem, err := llmModel.Encode(ctx, promptWithSystem)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2023-09-03 21:36:14 +00:00
|
|
|
|
2023-08-30 20:35:03 +00:00
|
|
|
tokensNoSystem, err := llmModel.Encode(ctx, promptNoSystem)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2023-08-07 21:51:31 +00:00
|
|
|
|
2023-09-03 21:36:14 +00:00
|
|
|
opts.NumKeep = len(tokensWithSystem) - len(tokensNoSystem)
|
2023-08-07 21:51:31 +00:00
|
|
|
|
2023-07-21 20:33:56 +00:00
|
|
|
llmModel.SetOptions(opts)
|
|
|
|
}
|
2023-07-19 22:00:28 +00:00
|
|
|
}
|
2023-08-01 01:35:18 +00:00
|
|
|
loaded.expireAt = time.Now().Add(sessionDuration)
|
2023-08-08 19:13:22 +00:00
|
|
|
|
2023-08-01 01:35:18 +00:00
|
|
|
if loaded.expireTimer == nil {
|
|
|
|
loaded.expireTimer = time.AfterFunc(sessionDuration, func() {
|
|
|
|
loaded.mu.Lock()
|
|
|
|
defer loaded.mu.Unlock()
|
2023-07-19 22:00:28 +00:00
|
|
|
|
2023-08-01 01:35:18 +00:00
|
|
|
if time.Now().Before(loaded.expireAt) {
|
2023-07-19 22:00:28 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2023-08-01 01:35:18 +00:00
|
|
|
if loaded.llm == nil {
|
2023-07-19 22:00:28 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2023-08-01 01:35:18 +00:00
|
|
|
loaded.llm.Close()
|
|
|
|
loaded.llm = nil
|
2023-08-02 20:15:40 +00:00
|
|
|
loaded.digest = ""
|
2023-07-19 22:00:28 +00:00
|
|
|
})
|
2023-07-06 17:40:11 +00:00
|
|
|
}
|
2023-08-01 01:35:18 +00:00
|
|
|
loaded.expireTimer.Reset(sessionDuration)
|
2023-08-08 19:13:22 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func GenerateHandler(c *gin.Context) {
|
|
|
|
loaded.mu.Lock()
|
|
|
|
defer loaded.mu.Unlock()
|
|
|
|
|
|
|
|
checkpointStart := time.Now()
|
|
|
|
|
|
|
|
var req api.GenerateRequest
|
|
|
|
if err := c.ShouldBindJSON(&req); err != nil {
|
|
|
|
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
model, err := GetModel(req.Model)
|
|
|
|
if err != nil {
|
|
|
|
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2023-08-15 13:35:39 +00:00
|
|
|
sessionDuration := defaultSessionDuration // TODO: set this duration from the request if specified
|
2023-08-30 20:35:03 +00:00
|
|
|
if err := load(c.Request.Context(), model, req.Options, sessionDuration); err != nil {
|
2023-08-08 19:13:22 +00:00
|
|
|
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
|
|
|
return
|
|
|
|
}
|
2023-07-06 17:40:11 +00:00
|
|
|
|
2023-07-18 19:02:02 +00:00
|
|
|
checkpointLoaded := time.Now()
|
|
|
|
|
2023-08-08 17:49:37 +00:00
|
|
|
embedding := ""
|
|
|
|
if model.Embeddings != nil && len(model.Embeddings) > 0 {
|
2023-08-30 20:35:03 +00:00
|
|
|
promptEmbed, err := loaded.llm.Embedding(c.Request.Context(), req.Prompt)
|
2023-08-08 17:49:37 +00:00
|
|
|
if err != nil {
|
|
|
|
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
|
|
|
return
|
|
|
|
}
|
|
|
|
// TODO: set embed_top from specified parameters in modelfile
|
|
|
|
embed_top := 3
|
|
|
|
topK := vector.TopK(embed_top, mat.NewVecDense(len(promptEmbed), promptEmbed), loaded.Embeddings)
|
|
|
|
for _, e := range topK {
|
|
|
|
embedding = fmt.Sprintf("%s %s", embedding, e.Embedding.Data)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
prompt, err := model.Prompt(req, embedding)
|
2023-07-11 21:57:17 +00:00
|
|
|
if err != nil {
|
|
|
|
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
|
|
|
return
|
|
|
|
}
|
2023-07-04 04:47:00 +00:00
|
|
|
|
2023-07-14 21:15:53 +00:00
|
|
|
ch := make(chan any)
|
|
|
|
go func() {
|
|
|
|
defer close(ch)
|
2023-07-20 19:12:08 +00:00
|
|
|
fn := func(r api.GenerateResponse) {
|
2023-08-01 01:35:18 +00:00
|
|
|
loaded.expireAt = time.Now().Add(sessionDuration)
|
|
|
|
loaded.expireTimer.Reset(sessionDuration)
|
2023-07-19 22:00:28 +00:00
|
|
|
|
2023-07-14 21:15:53 +00:00
|
|
|
r.Model = req.Model
|
|
|
|
r.CreatedAt = time.Now().UTC()
|
|
|
|
if r.Done {
|
2023-07-18 19:02:02 +00:00
|
|
|
r.TotalDuration = time.Since(checkpointStart)
|
|
|
|
r.LoadDuration = checkpointLoaded.Sub(checkpointStart)
|
2023-07-14 21:15:53 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
ch <- r
|
2023-07-20 19:12:08 +00:00
|
|
|
}
|
|
|
|
|
2023-08-30 20:35:03 +00:00
|
|
|
if err := loaded.llm.Predict(c.Request.Context(), req.Context, prompt, fn); err != nil {
|
2023-07-20 19:12:08 +00:00
|
|
|
ch <- gin.H{"error": err.Error()}
|
|
|
|
}
|
2023-07-14 21:15:53 +00:00
|
|
|
}()
|
2023-07-11 21:57:17 +00:00
|
|
|
|
2023-07-14 21:15:53 +00:00
|
|
|
streamResponse(c, ch)
|
2023-07-11 18:54:22 +00:00
|
|
|
}
|
2023-07-06 17:40:11 +00:00
|
|
|
|
2023-08-08 19:13:22 +00:00
|
|
|
func EmbeddingHandler(c *gin.Context) {
|
|
|
|
loaded.mu.Lock()
|
|
|
|
defer loaded.mu.Unlock()
|
|
|
|
|
|
|
|
var req api.EmbeddingRequest
|
|
|
|
if err := c.ShouldBindJSON(&req); err != nil {
|
|
|
|
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
model, err := GetModel(req.Model)
|
|
|
|
if err != nil {
|
|
|
|
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
|
|
|
return
|
|
|
|
}
|
2023-08-30 20:35:03 +00:00
|
|
|
if err := load(c.Request.Context(), model, req.Options, 5*time.Minute); err != nil {
|
2023-08-08 19:13:22 +00:00
|
|
|
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
if !loaded.options.EmbeddingOnly {
|
|
|
|
c.JSON(http.StatusBadRequest, gin.H{"error": "embedding option must be set to true"})
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2023-08-30 20:35:03 +00:00
|
|
|
embedding, err := loaded.llm.Embedding(c.Request.Context(), req.Prompt)
|
2023-08-08 19:13:22 +00:00
|
|
|
if err != nil {
|
|
|
|
log.Printf("embedding generation failed: %v", err)
|
|
|
|
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to generate embedding"})
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
resp := api.EmbeddingResponse{
|
|
|
|
Embedding: embedding,
|
|
|
|
}
|
|
|
|
c.JSON(http.StatusOK, resp)
|
|
|
|
}
|
|
|
|
|
2023-07-20 23:09:23 +00:00
|
|
|
func PullModelHandler(c *gin.Context) {
|
2023-07-11 18:54:22 +00:00
|
|
|
var req api.PullRequest
|
|
|
|
if err := c.ShouldBindJSON(&req); err != nil {
|
|
|
|
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2023-07-17 00:02:22 +00:00
|
|
|
ch := make(chan any)
|
|
|
|
go func() {
|
|
|
|
defer close(ch)
|
2023-07-19 01:51:30 +00:00
|
|
|
fn := func(r api.ProgressResponse) {
|
|
|
|
ch <- r
|
2023-07-17 00:02:22 +00:00
|
|
|
}
|
2023-07-19 01:51:30 +00:00
|
|
|
|
2023-07-21 22:42:19 +00:00
|
|
|
regOpts := &RegistryOptions{
|
|
|
|
Insecure: req.Insecure,
|
|
|
|
Username: req.Username,
|
|
|
|
Password: req.Password,
|
|
|
|
}
|
|
|
|
|
2023-07-25 21:08:51 +00:00
|
|
|
ctx, cancel := context.WithCancel(c.Request.Context())
|
|
|
|
defer cancel()
|
|
|
|
|
|
|
|
if err := PullModel(ctx, req.Name, regOpts, fn); err != nil {
|
2023-07-20 19:12:08 +00:00
|
|
|
ch <- gin.H{"error": err.Error()}
|
2023-07-17 00:02:22 +00:00
|
|
|
}
|
|
|
|
}()
|
|
|
|
|
|
|
|
streamResponse(c, ch)
|
|
|
|
}
|
|
|
|
|
2023-07-20 23:09:23 +00:00
|
|
|
func PushModelHandler(c *gin.Context) {
|
2023-07-17 00:02:22 +00:00
|
|
|
var req api.PushRequest
|
|
|
|
if err := c.ShouldBindJSON(&req); err != nil {
|
|
|
|
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
2023-07-11 18:54:22 +00:00
|
|
|
return
|
|
|
|
}
|
2023-07-06 17:40:11 +00:00
|
|
|
|
2023-07-17 00:02:22 +00:00
|
|
|
ch := make(chan any)
|
|
|
|
go func() {
|
|
|
|
defer close(ch)
|
2023-07-19 01:51:30 +00:00
|
|
|
fn := func(r api.ProgressResponse) {
|
|
|
|
ch <- r
|
2023-07-17 00:02:22 +00:00
|
|
|
}
|
2023-07-19 01:51:30 +00:00
|
|
|
|
2023-07-21 22:42:19 +00:00
|
|
|
regOpts := &RegistryOptions{
|
|
|
|
Insecure: req.Insecure,
|
|
|
|
Username: req.Username,
|
|
|
|
Password: req.Password,
|
|
|
|
}
|
|
|
|
|
2023-08-11 22:41:55 +00:00
|
|
|
ctx := context.Background()
|
|
|
|
if err := PushModel(ctx, req.Name, regOpts, fn); err != nil {
|
2023-07-20 19:12:08 +00:00
|
|
|
ch <- gin.H{"error": err.Error()}
|
2023-07-17 00:02:22 +00:00
|
|
|
}
|
|
|
|
}()
|
|
|
|
|
|
|
|
streamResponse(c, ch)
|
|
|
|
}
|
|
|
|
|
2023-07-20 23:09:23 +00:00
|
|
|
func CreateModelHandler(c *gin.Context) {
|
2023-07-17 00:02:22 +00:00
|
|
|
var req api.CreateRequest
|
|
|
|
if err := c.ShouldBindJSON(&req); err != nil {
|
|
|
|
c.JSON(http.StatusBadRequest, gin.H{"message": err.Error()})
|
2023-07-13 02:07:15 +00:00
|
|
|
return
|
2023-07-17 00:02:22 +00:00
|
|
|
}
|
|
|
|
|
2023-07-11 18:54:22 +00:00
|
|
|
ch := make(chan any)
|
2023-07-14 21:15:53 +00:00
|
|
|
go func() {
|
|
|
|
defer close(ch)
|
2023-07-25 18:25:13 +00:00
|
|
|
fn := func(resp api.ProgressResponse) {
|
|
|
|
ch <- resp
|
2023-07-17 00:02:22 +00:00
|
|
|
}
|
|
|
|
|
2023-07-25 21:08:51 +00:00
|
|
|
ctx, cancel := context.WithCancel(c.Request.Context())
|
|
|
|
defer cancel()
|
|
|
|
|
|
|
|
if err := CreateModel(ctx, req.Name, req.Path, fn); err != nil {
|
2023-07-20 19:12:08 +00:00
|
|
|
ch <- gin.H{"error": err.Error()}
|
2023-07-17 00:02:22 +00:00
|
|
|
}
|
2023-07-14 21:15:53 +00:00
|
|
|
}()
|
2023-07-07 22:29:17 +00:00
|
|
|
|
2023-07-14 21:15:53 +00:00
|
|
|
streamResponse(c, ch)
|
2023-07-05 19:37:33 +00:00
|
|
|
}
|
|
|
|
|
2023-07-20 23:09:23 +00:00
|
|
|
func DeleteModelHandler(c *gin.Context) {
|
|
|
|
var req api.DeleteRequest
|
|
|
|
if err := c.ShouldBindJSON(&req); err != nil {
|
|
|
|
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2023-07-22 06:02:12 +00:00
|
|
|
if err := DeleteModel(req.Name); err != nil {
|
|
|
|
if os.IsNotExist(err) {
|
|
|
|
c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Name)})
|
|
|
|
} else {
|
2023-07-20 23:09:23 +00:00
|
|
|
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
|
|
|
}
|
2023-07-22 06:02:12 +00:00
|
|
|
return
|
|
|
|
}
|
2023-09-11 18:46:35 +00:00
|
|
|
c.JSON(http.StatusOK, nil)
|
2023-07-20 23:09:23 +00:00
|
|
|
}
|
|
|
|
|
2023-09-06 18:04:17 +00:00
|
|
|
func ShowModelHandler(c *gin.Context) {
|
|
|
|
var req api.ShowRequest
|
|
|
|
if err := c.ShouldBindJSON(&req); err != nil {
|
|
|
|
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
resp, err := GetModelInfo(req.Name)
|
|
|
|
if err != nil {
|
|
|
|
if os.IsNotExist(err) {
|
|
|
|
c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Name)})
|
|
|
|
} else {
|
|
|
|
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
|
|
|
}
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
c.JSON(http.StatusOK, resp)
|
|
|
|
}
|
|
|
|
|
|
|
|
func GetModelInfo(name string) (*api.ShowResponse, error) {
|
|
|
|
model, err := GetModel(name)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
resp := &api.ShowResponse{
|
|
|
|
License: strings.Join(model.License, "\n"),
|
|
|
|
System: model.System,
|
|
|
|
Template: model.Template,
|
|
|
|
}
|
|
|
|
|
|
|
|
mf, err := ShowModelfile(model)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
resp.Modelfile = mf
|
|
|
|
|
|
|
|
var params []string
|
|
|
|
cs := 30
|
|
|
|
for k, v := range model.Options {
|
|
|
|
switch val := v.(type) {
|
|
|
|
case string:
|
|
|
|
params = append(params, fmt.Sprintf("%-*s %s", cs, k, val))
|
|
|
|
case int:
|
|
|
|
params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.Itoa(val)))
|
|
|
|
case float64:
|
|
|
|
params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatFloat(val, 'f', 0, 64)))
|
|
|
|
case bool:
|
|
|
|
params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatBool(val)))
|
|
|
|
case []interface{}:
|
|
|
|
for _, nv := range val {
|
|
|
|
switch nval := nv.(type) {
|
|
|
|
case string:
|
|
|
|
params = append(params, fmt.Sprintf("%-*s %s", cs, k, nval))
|
|
|
|
case int:
|
|
|
|
params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.Itoa(nval)))
|
|
|
|
case float64:
|
|
|
|
params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatFloat(nval, 'f', 0, 64)))
|
|
|
|
case bool:
|
|
|
|
params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatBool(nval)))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
resp.Parameters = strings.Join(params, "\n")
|
|
|
|
|
|
|
|
return resp, nil
|
|
|
|
}
|
|
|
|
|
2023-07-20 23:09:23 +00:00
|
|
|
func ListModelsHandler(c *gin.Context) {
|
2023-08-30 15:10:27 +00:00
|
|
|
var models []api.ModelResponse
|
2023-07-18 16:09:45 +00:00
|
|
|
fp, err := GetManifestPath()
|
|
|
|
if err != nil {
|
|
|
|
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
|
|
|
return
|
|
|
|
}
|
2023-08-30 18:14:12 +00:00
|
|
|
|
|
|
|
walkFunc := func(path string, info os.FileInfo, _ error) error {
|
2023-07-18 16:09:45 +00:00
|
|
|
if !info.IsDir() {
|
2023-08-30 18:14:12 +00:00
|
|
|
dir, file := filepath.Split(path)
|
|
|
|
dir = strings.Trim(strings.TrimPrefix(dir, fp), string(os.PathSeparator))
|
|
|
|
tag := strings.Join([]string{dir, file}, ":")
|
2023-08-22 04:56:56 +00:00
|
|
|
|
2023-08-22 16:39:42 +00:00
|
|
|
mp := ParseModelPath(tag)
|
2023-08-29 03:50:24 +00:00
|
|
|
manifest, digest, err := GetManifest(mp)
|
2023-07-18 16:09:45 +00:00
|
|
|
if err != nil {
|
2023-07-18 19:39:08 +00:00
|
|
|
log.Printf("skipping file: %s", fp)
|
|
|
|
return nil
|
2023-07-18 16:09:45 +00:00
|
|
|
}
|
2023-08-30 18:14:12 +00:00
|
|
|
|
|
|
|
models = append(models, api.ModelResponse{
|
2023-07-18 16:09:45 +00:00
|
|
|
Name: mp.GetShortTagname(),
|
|
|
|
Size: manifest.GetTotalSize(),
|
2023-08-29 03:50:24 +00:00
|
|
|
Digest: digest,
|
2023-08-30 18:14:12 +00:00
|
|
|
ModifiedAt: info.ModTime(),
|
|
|
|
})
|
2023-07-18 16:09:45 +00:00
|
|
|
}
|
2023-08-30 18:14:12 +00:00
|
|
|
|
2023-07-18 16:09:45 +00:00
|
|
|
return nil
|
2023-08-30 18:14:12 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if err := filepath.Walk(fp, walkFunc); err != nil {
|
2023-07-18 16:09:45 +00:00
|
|
|
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2023-07-19 22:00:28 +00:00
|
|
|
c.JSON(http.StatusOK, api.ListResponse{Models: models})
|
2023-07-18 16:09:45 +00:00
|
|
|
}
|
|
|
|
|
2023-07-24 15:27:28 +00:00
|
|
|
func CopyModelHandler(c *gin.Context) {
|
|
|
|
var req api.CopyRequest
|
|
|
|
if err := c.ShouldBindJSON(&req); err != nil {
|
|
|
|
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := CopyModel(req.Source, req.Destination); err != nil {
|
|
|
|
if os.IsNotExist(err) {
|
|
|
|
c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Source)})
|
|
|
|
} else {
|
|
|
|
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
|
|
|
}
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-08-10 16:27:03 +00:00
|
|
|
func Serve(ln net.Listener, origins []string) error {
|
2023-07-22 01:01:24 +00:00
|
|
|
config := cors.DefaultConfig()
|
|
|
|
config.AllowWildcard = true
|
2023-08-10 16:27:03 +00:00
|
|
|
config.AllowOrigins = append(origins, []string{
|
2023-07-22 01:01:24 +00:00
|
|
|
"http://localhost",
|
|
|
|
"http://localhost:*",
|
|
|
|
"https://localhost",
|
|
|
|
"https://localhost:*",
|
|
|
|
"http://127.0.0.1",
|
|
|
|
"http://127.0.0.1:*",
|
|
|
|
"https://127.0.0.1",
|
|
|
|
"https://127.0.0.1:*",
|
2023-08-08 20:39:50 +00:00
|
|
|
"http://0.0.0.0",
|
|
|
|
"http://0.0.0.0:*",
|
|
|
|
"https://0.0.0.0",
|
|
|
|
"https://0.0.0.0:*",
|
2023-08-10 16:27:03 +00:00
|
|
|
}...)
|
2023-07-22 01:01:24 +00:00
|
|
|
|
2023-07-05 19:37:33 +00:00
|
|
|
r := gin.Default()
|
2023-07-22 01:01:24 +00:00
|
|
|
r.Use(cors.New(config))
|
2023-07-05 19:37:33 +00:00
|
|
|
|
2023-07-08 03:46:15 +00:00
|
|
|
r.GET("/", func(c *gin.Context) {
|
|
|
|
c.String(http.StatusOK, "Ollama is running")
|
|
|
|
})
|
2023-08-01 18:50:38 +00:00
|
|
|
r.HEAD("/", func(c *gin.Context) {
|
|
|
|
c.Status(http.StatusOK)
|
|
|
|
})
|
2023-07-08 03:46:15 +00:00
|
|
|
|
2023-07-20 23:09:23 +00:00
|
|
|
r.POST("/api/pull", PullModelHandler)
|
|
|
|
r.POST("/api/generate", GenerateHandler)
|
2023-08-08 19:13:22 +00:00
|
|
|
r.POST("/api/embeddings", EmbeddingHandler)
|
2023-07-20 23:09:23 +00:00
|
|
|
r.POST("/api/create", CreateModelHandler)
|
|
|
|
r.POST("/api/push", PushModelHandler)
|
2023-07-24 15:27:28 +00:00
|
|
|
r.POST("/api/copy", CopyModelHandler)
|
2023-07-20 23:09:23 +00:00
|
|
|
r.GET("/api/tags", ListModelsHandler)
|
|
|
|
r.DELETE("/api/delete", DeleteModelHandler)
|
2023-09-06 18:04:17 +00:00
|
|
|
r.POST("/api/show", ShowModelHandler)
|
2023-07-03 19:22:44 +00:00
|
|
|
|
|
|
|
log.Printf("Listening on %s", ln.Addr())
|
|
|
|
s := &http.Server{
|
|
|
|
Handler: r,
|
|
|
|
}
|
|
|
|
|
2023-08-30 20:35:03 +00:00
|
|
|
// listen for a ctrl+c and stop any loaded llm
|
|
|
|
signals := make(chan os.Signal, 1)
|
|
|
|
signal.Notify(signals, syscall.SIGINT)
|
|
|
|
go func() {
|
|
|
|
<-signals
|
|
|
|
if loaded.llm != nil {
|
|
|
|
loaded.llm.Close()
|
|
|
|
}
|
|
|
|
os.Exit(0)
|
|
|
|
}()
|
|
|
|
|
2023-09-12 15:04:35 +00:00
|
|
|
if runtime.GOOS == "linux" {
|
|
|
|
// check compatibility to log warnings
|
|
|
|
if _, err := llm.CheckVRAM(); err != nil {
|
|
|
|
log.Printf("Warning: GPU support not enabled, you may need to install GPU drivers: %v", err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-07-03 19:22:44 +00:00
|
|
|
return s.Serve(ln)
|
|
|
|
}
|
2023-07-06 17:40:11 +00:00
|
|
|
|
2023-07-14 21:15:53 +00:00
|
|
|
func streamResponse(c *gin.Context, ch chan any) {
|
2023-08-09 04:38:10 +00:00
|
|
|
c.Header("Content-Type", "application/x-ndjson")
|
2023-07-11 18:54:22 +00:00
|
|
|
c.Stream(func(w io.Writer) bool {
|
|
|
|
val, ok := <-ch
|
|
|
|
if !ok {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
bts, err := json.Marshal(val)
|
|
|
|
if err != nil {
|
2023-07-31 20:46:37 +00:00
|
|
|
log.Printf("streamResponse: json.Marshal failed with %s", err)
|
2023-07-11 18:54:22 +00:00
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
bts = append(bts, '\n')
|
|
|
|
if _, err := w.Write(bts); err != nil {
|
2023-07-31 20:46:37 +00:00
|
|
|
log.Printf("streamResponse: w.Write failed with %s", err)
|
2023-07-11 18:54:22 +00:00
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
return true
|
|
|
|
})
|
|
|
|
}
|