2023-07-17 00:02:22 +00:00
|
|
|
package server
|
|
|
|
|
|
|
|
import (
|
2023-08-04 22:56:40 +00:00
|
|
|
"bufio"
|
2023-07-17 00:02:22 +00:00
|
|
|
"bytes"
|
2023-07-25 21:08:51 +00:00
|
|
|
"context"
|
2023-07-17 00:02:22 +00:00
|
|
|
"crypto/sha256"
|
2023-08-29 03:50:24 +00:00
|
|
|
"encoding/hex"
|
2023-07-17 00:02:22 +00:00
|
|
|
"encoding/json"
|
|
|
|
"errors"
|
|
|
|
"fmt"
|
2023-08-08 04:55:34 +00:00
|
|
|
"html/template"
|
2023-07-17 00:02:22 +00:00
|
|
|
"io"
|
|
|
|
"log"
|
|
|
|
"net/http"
|
2023-08-22 01:38:31 +00:00
|
|
|
"net/url"
|
2023-07-17 00:02:22 +00:00
|
|
|
"os"
|
2023-08-14 22:07:00 +00:00
|
|
|
"path"
|
2023-07-17 00:02:22 +00:00
|
|
|
"path/filepath"
|
2023-07-17 19:08:10 +00:00
|
|
|
"reflect"
|
2023-08-22 01:24:42 +00:00
|
|
|
"runtime"
|
2023-07-17 00:02:22 +00:00
|
|
|
"strconv"
|
|
|
|
"strings"
|
|
|
|
|
|
|
|
"github.com/jmorganca/ollama/api"
|
2023-07-21 20:33:56 +00:00
|
|
|
"github.com/jmorganca/ollama/llm"
|
2023-07-17 00:02:22 +00:00
|
|
|
"github.com/jmorganca/ollama/parser"
|
2023-08-04 22:56:40 +00:00
|
|
|
"github.com/jmorganca/ollama/vector"
|
2023-08-22 01:24:42 +00:00
|
|
|
"github.com/jmorganca/ollama/version"
|
2023-07-17 00:02:22 +00:00
|
|
|
)
|
|
|
|
|
2023-08-11 22:41:55 +00:00
|
|
|
const MaxRetries = 3
|
|
|
|
|
2023-07-21 22:42:19 +00:00
|
|
|
type RegistryOptions struct {
|
|
|
|
Insecure bool
|
|
|
|
Username string
|
|
|
|
Password string
|
2023-08-10 18:34:25 +00:00
|
|
|
Token string
|
2023-07-21 22:42:19 +00:00
|
|
|
}
|
|
|
|
|
2023-07-17 00:02:22 +00:00
|
|
|
type Model struct {
|
2023-08-04 00:16:05 +00:00
|
|
|
Name string `json:"name"`
|
|
|
|
ModelPath string
|
|
|
|
AdapterPaths []string
|
|
|
|
Template string
|
|
|
|
System string
|
|
|
|
Digest string
|
2023-08-29 03:50:24 +00:00
|
|
|
ConfigDigest string
|
2023-08-04 00:16:05 +00:00
|
|
|
Options map[string]interface{}
|
|
|
|
Embeddings []vector.Embedding
|
2023-07-17 00:02:22 +00:00
|
|
|
}
|
|
|
|
|
2023-08-08 17:49:37 +00:00
|
|
|
func (m *Model) Prompt(request api.GenerateRequest, embedding string) (string, error) {
|
2023-08-08 04:55:34 +00:00
|
|
|
t := m.Template
|
|
|
|
if request.Template != "" {
|
|
|
|
t = request.Template
|
|
|
|
}
|
|
|
|
|
|
|
|
tmpl, err := template.New("").Parse(t)
|
2023-07-17 21:21:27 +00:00
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
|
|
|
|
var vars struct {
|
2023-07-20 06:22:19 +00:00
|
|
|
First bool
|
2023-07-17 21:21:27 +00:00
|
|
|
System string
|
|
|
|
Prompt string
|
2023-08-04 22:56:40 +00:00
|
|
|
Embed string
|
2023-07-20 02:43:00 +00:00
|
|
|
|
|
|
|
// deprecated: versions <= 0.0.7 used this to omit the system prompt
|
|
|
|
Context []int
|
2023-07-17 21:21:27 +00:00
|
|
|
}
|
|
|
|
|
2023-07-22 03:45:32 +00:00
|
|
|
vars.First = len(request.Context) == 0
|
2023-07-17 21:21:27 +00:00
|
|
|
vars.System = m.System
|
|
|
|
vars.Prompt = request.Prompt
|
2023-07-20 06:22:19 +00:00
|
|
|
vars.Context = request.Context
|
2023-08-08 17:49:37 +00:00
|
|
|
vars.Embed = embedding
|
2023-07-17 21:21:27 +00:00
|
|
|
|
2023-08-08 04:55:34 +00:00
|
|
|
if request.System != "" {
|
|
|
|
vars.System = request.System
|
|
|
|
}
|
|
|
|
|
2023-07-17 21:21:27 +00:00
|
|
|
var sb strings.Builder
|
|
|
|
if err := tmpl.Execute(&sb, vars); err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
|
|
|
|
return sb.String(), nil
|
|
|
|
}
|
|
|
|
|
2023-07-17 00:02:22 +00:00
|
|
|
type ManifestV2 struct {
|
|
|
|
SchemaVersion int `json:"schemaVersion"`
|
|
|
|
MediaType string `json:"mediaType"`
|
|
|
|
Config Layer `json:"config"`
|
|
|
|
Layers []*Layer `json:"layers"`
|
|
|
|
}
|
|
|
|
|
|
|
|
type Layer struct {
|
|
|
|
MediaType string `json:"mediaType"`
|
|
|
|
Digest string `json:"digest"`
|
|
|
|
Size int `json:"size"`
|
2023-08-14 22:07:00 +00:00
|
|
|
From string `json:"from,omitempty"`
|
2023-07-17 00:02:22 +00:00
|
|
|
}
|
|
|
|
|
2023-07-19 00:14:12 +00:00
|
|
|
type LayerReader struct {
|
2023-07-17 00:02:22 +00:00
|
|
|
Layer
|
2023-07-19 00:14:12 +00:00
|
|
|
io.Reader
|
2023-07-17 00:02:22 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
type ConfigV2 struct {
|
2023-07-21 20:33:56 +00:00
|
|
|
ModelFamily llm.ModelFamily `json:"model_family"`
|
2023-08-17 22:32:31 +00:00
|
|
|
ModelType string `json:"model_type"`
|
|
|
|
FileType string `json:"file_type"`
|
|
|
|
RootFS RootFS `json:"rootfs"`
|
2023-07-21 20:33:56 +00:00
|
|
|
|
|
|
|
// required by spec
|
2023-07-17 00:02:22 +00:00
|
|
|
Architecture string `json:"architecture"`
|
|
|
|
OS string `json:"os"`
|
|
|
|
}
|
|
|
|
|
|
|
|
type RootFS struct {
|
|
|
|
Type string `json:"type"`
|
|
|
|
DiffIDs []string `json:"diff_ids"`
|
|
|
|
}
|
|
|
|
|
2023-07-18 16:09:45 +00:00
|
|
|
func (m *ManifestV2) GetTotalSize() int {
|
|
|
|
var total int
|
|
|
|
for _, layer := range m.Layers {
|
|
|
|
total += layer.Size
|
|
|
|
}
|
|
|
|
total += m.Config.Size
|
|
|
|
return total
|
|
|
|
}
|
|
|
|
|
2023-08-29 03:50:24 +00:00
|
|
|
func GetManifest(mp ModelPath) (*ManifestV2, string, error) {
|
2023-07-18 05:44:21 +00:00
|
|
|
fp, err := mp.GetManifestPath(false)
|
2023-07-17 18:03:55 +00:00
|
|
|
if err != nil {
|
2023-08-29 03:50:24 +00:00
|
|
|
return nil, "", err
|
2023-07-17 18:03:55 +00:00
|
|
|
}
|
2023-07-17 21:21:27 +00:00
|
|
|
|
2023-07-22 06:02:12 +00:00
|
|
|
if _, err = os.Stat(fp); err != nil {
|
2023-08-29 03:50:24 +00:00
|
|
|
return nil, "", err
|
2023-07-17 00:02:22 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
var manifest *ManifestV2
|
|
|
|
|
2023-07-17 21:21:27 +00:00
|
|
|
bts, err := os.ReadFile(fp)
|
2023-07-17 00:02:22 +00:00
|
|
|
if err != nil {
|
2023-08-29 03:50:24 +00:00
|
|
|
return nil, "", fmt.Errorf("couldn't open file '%s'", fp)
|
2023-07-17 00:02:22 +00:00
|
|
|
}
|
|
|
|
|
2023-08-29 03:50:24 +00:00
|
|
|
shaSum := sha256.Sum256(bts)
|
|
|
|
shaStr := hex.EncodeToString(shaSum[:])
|
|
|
|
|
2023-07-17 21:21:27 +00:00
|
|
|
if err := json.Unmarshal(bts, &manifest); err != nil {
|
2023-08-29 03:50:24 +00:00
|
|
|
return nil, "", err
|
2023-07-17 00:02:22 +00:00
|
|
|
}
|
|
|
|
|
2023-08-29 03:50:24 +00:00
|
|
|
return manifest, shaStr, nil
|
2023-07-17 00:02:22 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func GetModel(name string) (*Model, error) {
|
2023-08-22 16:39:42 +00:00
|
|
|
mp := ParseModelPath(name)
|
2023-08-29 03:50:24 +00:00
|
|
|
manifest, digest, err := GetManifest(mp)
|
2023-07-17 00:02:22 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
model := &Model{
|
2023-08-29 03:50:24 +00:00
|
|
|
Name: mp.GetFullTagname(),
|
|
|
|
Digest: digest,
|
|
|
|
ConfigDigest: manifest.Config.Digest,
|
|
|
|
Template: "{{ .Prompt }}",
|
2023-07-17 00:02:22 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
for _, layer := range manifest.Layers {
|
2023-07-18 05:44:21 +00:00
|
|
|
filename, err := GetBlobsPath(layer.Digest)
|
2023-07-17 18:03:55 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2023-07-17 00:02:22 +00:00
|
|
|
switch layer.MediaType {
|
|
|
|
case "application/vnd.ollama.image.model":
|
|
|
|
model.ModelPath = filename
|
2023-08-04 22:56:40 +00:00
|
|
|
case "application/vnd.ollama.image.embed":
|
|
|
|
file, err := os.Open(filename)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("failed to open file: %s", filename)
|
|
|
|
}
|
|
|
|
defer file.Close()
|
|
|
|
|
|
|
|
if err = json.NewDecoder(file).Decode(&model.Embeddings); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2023-08-04 00:16:05 +00:00
|
|
|
case "application/vnd.ollama.image.adapter":
|
|
|
|
model.AdapterPaths = append(model.AdapterPaths, filename)
|
2023-07-17 21:21:27 +00:00
|
|
|
case "application/vnd.ollama.image.template":
|
|
|
|
bts, err := os.ReadFile(filename)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
model.Template = string(bts)
|
|
|
|
case "application/vnd.ollama.image.system":
|
|
|
|
bts, err := os.ReadFile(filename)
|
2023-07-17 00:02:22 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2023-07-17 21:21:27 +00:00
|
|
|
|
|
|
|
model.System = string(bts)
|
2023-07-20 02:43:00 +00:00
|
|
|
case "application/vnd.ollama.image.prompt":
|
|
|
|
bts, err := os.ReadFile(filename)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
model.Template = string(bts)
|
2023-07-17 00:02:22 +00:00
|
|
|
case "application/vnd.ollama.image.params":
|
2023-07-17 19:08:10 +00:00
|
|
|
params, err := os.Open(filename)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
defer params.Close()
|
2023-07-17 00:02:22 +00:00
|
|
|
|
2023-07-31 19:07:04 +00:00
|
|
|
// parse model options parameters into a map so that we can see which fields have been specified explicitly
|
2023-08-01 17:36:31 +00:00
|
|
|
if err = json.NewDecoder(params).Decode(&model.Options); err != nil {
|
2023-07-31 19:07:04 +00:00
|
|
|
return nil, err
|
|
|
|
}
|
2023-07-17 00:02:22 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return model, nil
|
|
|
|
}
|
|
|
|
|
2023-08-04 22:56:40 +00:00
|
|
|
func filenameWithPath(path, f string) (string, error) {
|
|
|
|
// if filePath starts with ~/, replace it with the user's home directory.
|
|
|
|
if strings.HasPrefix(f, "~/") {
|
|
|
|
parts := strings.Split(f, "/")
|
|
|
|
home, err := os.UserHomeDir()
|
|
|
|
if err != nil {
|
|
|
|
return "", fmt.Errorf("failed to open file: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
f = filepath.Join(home, filepath.Join(parts[1:]...))
|
|
|
|
}
|
|
|
|
|
|
|
|
// if filePath is not an absolute path, make it relative to the modelfile path
|
|
|
|
if !filepath.IsAbs(f) {
|
|
|
|
f = filepath.Join(filepath.Dir(path), f)
|
|
|
|
}
|
|
|
|
|
|
|
|
return f, nil
|
|
|
|
}
|
|
|
|
|
2023-07-25 21:08:51 +00:00
|
|
|
func CreateModel(ctx context.Context, name string, path string, fn func(resp api.ProgressResponse)) error {
|
2023-07-20 04:55:15 +00:00
|
|
|
mf, err := os.Open(path)
|
|
|
|
if err != nil {
|
2023-07-25 18:25:13 +00:00
|
|
|
fn(api.ProgressResponse{Status: fmt.Sprintf("couldn't open modelfile '%s'", path)})
|
2023-07-20 04:55:15 +00:00
|
|
|
return fmt.Errorf("failed to open file: %w", err)
|
2023-07-17 00:02:22 +00:00
|
|
|
}
|
2023-07-20 04:55:15 +00:00
|
|
|
defer mf.Close()
|
2023-07-17 00:02:22 +00:00
|
|
|
|
2023-07-25 18:25:13 +00:00
|
|
|
fn(api.ProgressResponse{Status: "parsing modelfile"})
|
2023-07-17 00:02:22 +00:00
|
|
|
commands, err := parser.Parse(mf)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2023-07-21 20:33:56 +00:00
|
|
|
config := ConfigV2{
|
|
|
|
Architecture: "amd64",
|
|
|
|
OS: "linux",
|
|
|
|
}
|
|
|
|
|
2023-07-19 00:14:12 +00:00
|
|
|
var layers []*LayerReader
|
2023-07-28 15:29:00 +00:00
|
|
|
params := make(map[string][]string)
|
2023-08-15 13:35:39 +00:00
|
|
|
embed := EmbeddingParams{fn: fn}
|
2023-07-17 00:02:22 +00:00
|
|
|
for _, c := range commands {
|
2023-07-17 21:21:27 +00:00
|
|
|
log.Printf("[%s] - %s\n", c.Name, c.Args)
|
2023-07-17 00:02:22 +00:00
|
|
|
switch c.Name {
|
|
|
|
case "model":
|
2023-07-25 18:25:13 +00:00
|
|
|
fn(api.ProgressResponse{Status: "looking for model"})
|
2023-08-04 22:56:40 +00:00
|
|
|
embed.model = c.Args
|
2023-08-22 04:56:56 +00:00
|
|
|
|
2023-08-22 16:39:42 +00:00
|
|
|
mp := ParseModelPath(c.Args)
|
2023-08-29 03:50:24 +00:00
|
|
|
mf, _, err := GetManifest(mp)
|
2023-07-17 00:02:22 +00:00
|
|
|
if err != nil {
|
2023-08-04 22:56:40 +00:00
|
|
|
modelFile, err := filenameWithPath(path, c.Args)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
2023-07-17 00:02:22 +00:00
|
|
|
}
|
2023-08-04 22:56:40 +00:00
|
|
|
if _, err := os.Stat(modelFile); err != nil {
|
2023-07-25 18:25:13 +00:00
|
|
|
// the model file does not exist, try pulling it
|
|
|
|
if errors.Is(err, os.ErrNotExist) {
|
|
|
|
fn(api.ProgressResponse{Status: "pulling model file"})
|
2023-07-25 21:08:51 +00:00
|
|
|
if err := PullModel(ctx, c.Args, &RegistryOptions{}, fn); err != nil {
|
2023-07-25 18:25:13 +00:00
|
|
|
return err
|
|
|
|
}
|
2023-08-29 03:50:24 +00:00
|
|
|
mf, _, err = GetManifest(mp)
|
2023-07-25 18:25:13 +00:00
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("failed to open file after pull: %v", err)
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
} else {
|
2023-08-15 13:35:39 +00:00
|
|
|
embed.model = modelFile
|
2023-07-25 18:25:13 +00:00
|
|
|
// create a model from this specified file
|
|
|
|
fn(api.ProgressResponse{Status: "creating model layer"})
|
2023-08-04 22:56:40 +00:00
|
|
|
file, err := os.Open(modelFile)
|
2023-07-25 18:25:13 +00:00
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("failed to open file: %v", err)
|
|
|
|
}
|
|
|
|
defer file.Close()
|
|
|
|
|
2023-07-21 20:33:56 +00:00
|
|
|
ggml, err := llm.DecodeGGML(file, llm.ModelFamilyLlama)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2023-08-17 18:37:27 +00:00
|
|
|
config.ModelFamily = ggml.ModelFamily()
|
|
|
|
config.ModelType = ggml.ModelType().String()
|
|
|
|
config.FileType = ggml.FileType().String()
|
2023-07-21 20:33:56 +00:00
|
|
|
|
|
|
|
// reset the file
|
|
|
|
file.Seek(0, io.SeekStart)
|
|
|
|
|
2023-07-25 18:25:13 +00:00
|
|
|
l, err := CreateLayer(file)
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("failed to create layer: %v", err)
|
|
|
|
}
|
|
|
|
l.MediaType = "application/vnd.ollama.image.model"
|
|
|
|
layers = append(layers, l)
|
2023-07-17 00:02:22 +00:00
|
|
|
}
|
2023-07-25 18:25:13 +00:00
|
|
|
}
|
2023-07-21 20:33:56 +00:00
|
|
|
|
2023-07-25 18:25:13 +00:00
|
|
|
if mf != nil {
|
2023-08-18 04:52:11 +00:00
|
|
|
sourceBlobPath, err := GetBlobsPath(mf.Config.Digest)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
sourceBlob, err := os.Open(sourceBlobPath)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
defer sourceBlob.Close()
|
|
|
|
|
|
|
|
var source ConfigV2
|
|
|
|
if err := json.NewDecoder(sourceBlob).Decode(&source); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// copie the model metadata
|
|
|
|
config.ModelFamily = source.ModelFamily
|
|
|
|
config.ModelType = source.ModelType
|
|
|
|
config.FileType = source.FileType
|
|
|
|
|
2023-07-17 00:02:22 +00:00
|
|
|
for _, l := range mf.Layers {
|
|
|
|
newLayer, err := GetLayerWithBufferFromLayer(l)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2023-08-14 22:07:00 +00:00
|
|
|
newLayer.From = mp.GetNamespaceRepository()
|
2023-07-17 00:02:22 +00:00
|
|
|
layers = append(layers, newLayer)
|
|
|
|
}
|
|
|
|
}
|
2023-08-04 22:56:40 +00:00
|
|
|
case "embed":
|
|
|
|
embedFilePath, err := filenameWithPath(path, c.Args)
|
2023-08-01 04:34:52 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2023-08-04 22:56:40 +00:00
|
|
|
embed.files = append(embed.files, embedFilePath)
|
2023-08-04 00:16:05 +00:00
|
|
|
case "adapter":
|
|
|
|
fn(api.ProgressResponse{Status: fmt.Sprintf("creating model %s layer", c.Name)})
|
|
|
|
|
|
|
|
fp := c.Args
|
|
|
|
if strings.HasPrefix(fp, "~/") {
|
|
|
|
parts := strings.Split(fp, "/")
|
|
|
|
home, err := os.UserHomeDir()
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("failed to open file: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
fp = filepath.Join(home, filepath.Join(parts[1:]...))
|
|
|
|
}
|
|
|
|
|
|
|
|
// If filePath is not an absolute path, make it relative to the modelfile path
|
|
|
|
if !filepath.IsAbs(fp) {
|
|
|
|
fp = filepath.Join(filepath.Dir(path), fp)
|
|
|
|
}
|
|
|
|
|
|
|
|
// create a model from this specified file
|
|
|
|
fn(api.ProgressResponse{Status: "creating model layer"})
|
|
|
|
|
|
|
|
file, err := os.Open(fp)
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("failed to open file: %v", err)
|
|
|
|
}
|
|
|
|
defer file.Close()
|
|
|
|
|
|
|
|
l, err := CreateLayer(file)
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("failed to create layer: %v", err)
|
|
|
|
}
|
|
|
|
l.MediaType = "application/vnd.ollama.image.adapter"
|
|
|
|
layers = append(layers, l)
|
2023-08-08 20:56:48 +00:00
|
|
|
case "license":
|
|
|
|
fn(api.ProgressResponse{Status: fmt.Sprintf("creating model %s layer", c.Name)})
|
|
|
|
mediaType := fmt.Sprintf("application/vnd.ollama.image.%s", c.Name)
|
|
|
|
|
|
|
|
layer, err := CreateLayer(strings.NewReader(c.Args))
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
layer.MediaType = mediaType
|
|
|
|
layers = append(layers, layer)
|
|
|
|
case "template", "system", "prompt":
|
2023-07-25 18:25:13 +00:00
|
|
|
fn(api.ProgressResponse{Status: fmt.Sprintf("creating model %s layer", c.Name)})
|
2023-07-21 20:33:56 +00:00
|
|
|
// remove the layer if one exists
|
2023-07-17 21:21:27 +00:00
|
|
|
mediaType := fmt.Sprintf("application/vnd.ollama.image.%s", c.Name)
|
|
|
|
layers = removeLayerFromLayers(layers, mediaType)
|
2023-07-17 00:02:22 +00:00
|
|
|
|
2023-07-17 21:21:27 +00:00
|
|
|
layer, err := CreateLayer(strings.NewReader(c.Args))
|
2023-07-17 00:02:22 +00:00
|
|
|
if err != nil {
|
2023-07-17 21:21:27 +00:00
|
|
|
return err
|
2023-07-17 00:02:22 +00:00
|
|
|
}
|
2023-07-17 21:21:27 +00:00
|
|
|
|
|
|
|
layer.MediaType = mediaType
|
|
|
|
layers = append(layers, layer)
|
2023-07-17 00:02:22 +00:00
|
|
|
default:
|
2023-08-30 15:53:42 +00:00
|
|
|
// runtime parameters, build a list of args for each parameter to allow multiple values to be specified (ex: multiple stop sequences)
|
2023-07-28 15:29:00 +00:00
|
|
|
params[c.Name] = append(params[c.Name], c.Args)
|
2023-07-17 00:02:22 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Create a single layer for the parameters
|
2023-07-17 19:08:10 +00:00
|
|
|
if len(params) > 0 {
|
2023-07-25 18:25:13 +00:00
|
|
|
fn(api.ProgressResponse{Status: "creating parameter layer"})
|
2023-07-17 00:02:22 +00:00
|
|
|
layers = removeLayerFromLayers(layers, "application/vnd.ollama.image.params")
|
2023-08-04 22:56:40 +00:00
|
|
|
formattedParams, err := formatParams(params)
|
2023-07-17 00:02:22 +00:00
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("couldn't create params json: %v", err)
|
|
|
|
}
|
2023-08-04 22:56:40 +00:00
|
|
|
|
|
|
|
bts, err := json.Marshal(formattedParams)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
l, err := CreateLayer(bytes.NewReader(bts))
|
2023-07-17 00:02:22 +00:00
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("failed to create layer: %v", err)
|
|
|
|
}
|
|
|
|
l.MediaType = "application/vnd.ollama.image.params"
|
|
|
|
layers = append(layers, l)
|
2023-08-04 22:56:40 +00:00
|
|
|
|
|
|
|
// apply these parameters to the embedding options, in case embeddings need to be generated using this model
|
2023-08-15 13:35:39 +00:00
|
|
|
embed.opts = formattedParams
|
2023-07-17 00:02:22 +00:00
|
|
|
}
|
|
|
|
|
2023-08-04 22:56:40 +00:00
|
|
|
// generate the embedding layers
|
|
|
|
embeddingLayers, err := embeddingLayers(embed)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
layers = append(layers, embeddingLayers...)
|
|
|
|
|
2023-07-17 00:02:22 +00:00
|
|
|
digests, err := getLayerDigests(layers)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
var manifestLayers []*Layer
|
|
|
|
for _, l := range layers {
|
|
|
|
manifestLayers = append(manifestLayers, &l.Layer)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Create a layer for the config object
|
2023-07-25 18:25:13 +00:00
|
|
|
fn(api.ProgressResponse{Status: "creating config layer"})
|
2023-07-21 20:33:56 +00:00
|
|
|
cfg, err := createConfigLayer(config, digests)
|
2023-07-17 00:02:22 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
layers = append(layers, cfg)
|
|
|
|
|
2023-08-14 22:07:00 +00:00
|
|
|
if err := SaveLayers(layers, fn, false); err != nil {
|
2023-07-17 00:02:22 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Create the manifest
|
2023-07-25 18:25:13 +00:00
|
|
|
fn(api.ProgressResponse{Status: "writing manifest"})
|
2023-07-17 00:02:22 +00:00
|
|
|
err = CreateManifest(name, cfg, manifestLayers)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2023-07-25 18:25:13 +00:00
|
|
|
fn(api.ProgressResponse{Status: "success"})
|
2023-07-17 00:02:22 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2023-08-04 22:56:40 +00:00
|
|
|
type EmbeddingParams struct {
|
|
|
|
model string
|
2023-08-15 13:35:39 +00:00
|
|
|
opts map[string]interface{}
|
2023-08-04 22:56:40 +00:00
|
|
|
files []string // paths to files to embed
|
|
|
|
fn func(resp api.ProgressResponse)
|
|
|
|
}
|
|
|
|
|
|
|
|
// embeddingLayers loads the associated LLM and generates the embeddings to be stored from an input file
|
|
|
|
func embeddingLayers(e EmbeddingParams) ([]*LayerReader, error) {
|
|
|
|
layers := []*LayerReader{}
|
|
|
|
if len(e.files) > 0 {
|
2023-08-15 13:35:39 +00:00
|
|
|
// check if the model is a file path or a model name
|
|
|
|
model, err := GetModel(e.model)
|
|
|
|
if err != nil {
|
|
|
|
if !strings.Contains(err.Error(), "couldn't open file") {
|
|
|
|
return nil, fmt.Errorf("unexpected error opening model to generate embeddings: %v", err)
|
2023-08-08 18:38:57 +00:00
|
|
|
}
|
2023-08-15 13:35:39 +00:00
|
|
|
// the model may be a file path, create a model from this file
|
|
|
|
model = &Model{ModelPath: e.model}
|
2023-08-04 22:56:40 +00:00
|
|
|
}
|
|
|
|
|
2023-08-30 20:35:03 +00:00
|
|
|
if err := load(context.Background(), model, e.opts, defaultSessionDuration); err != nil {
|
2023-08-04 22:56:40 +00:00
|
|
|
return nil, fmt.Errorf("load model to generate embeddings: %v", err)
|
|
|
|
}
|
|
|
|
|
2023-08-14 14:57:12 +00:00
|
|
|
// this will be used to check if we already have embeddings for a file
|
2023-08-15 13:35:39 +00:00
|
|
|
modelInfo, err := os.Stat(model.ModelPath)
|
2023-08-14 14:57:12 +00:00
|
|
|
if err != nil {
|
2023-08-15 13:39:59 +00:00
|
|
|
return nil, fmt.Errorf("failed to get model file info: %v", err)
|
2023-08-14 14:57:12 +00:00
|
|
|
}
|
|
|
|
|
2023-08-08 17:49:37 +00:00
|
|
|
addedFiles := make(map[string]bool) // keep track of files that have already been added
|
|
|
|
for _, filePattern := range e.files {
|
|
|
|
matchingFiles, err := filepath.Glob(filePattern)
|
2023-08-04 22:56:40 +00:00
|
|
|
if err != nil {
|
2023-08-08 17:49:37 +00:00
|
|
|
return nil, fmt.Errorf("could not find files with pattern %s: %w", filePattern, err)
|
2023-08-04 22:56:40 +00:00
|
|
|
}
|
|
|
|
|
2023-08-08 17:49:37 +00:00
|
|
|
for _, filePath := range matchingFiles {
|
|
|
|
if addedFiles[filePath] {
|
2023-08-04 22:56:40 +00:00
|
|
|
continue
|
|
|
|
}
|
2023-08-08 17:49:37 +00:00
|
|
|
addedFiles[filePath] = true
|
2023-08-14 13:34:17 +00:00
|
|
|
// check if we already have embeddings for this file path
|
2023-08-14 15:11:04 +00:00
|
|
|
layerIdentifier := fmt.Sprintf("%s:%s:%s:%d", filePath, e.model, modelInfo.ModTime().Format("2006-01-02 15:04:05"), modelInfo.Size())
|
2023-08-14 13:34:17 +00:00
|
|
|
digest, _ := GetSHA256Digest(strings.NewReader(layerIdentifier))
|
|
|
|
existing, err := existingFileEmbeddings(digest)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("failed to check existing embeddings for file %s: %v", filePath, err)
|
|
|
|
}
|
|
|
|
|
2023-08-08 17:49:37 +00:00
|
|
|
// TODO: check file type
|
|
|
|
f, err := os.Open(filePath)
|
2023-08-04 22:56:40 +00:00
|
|
|
if err != nil {
|
2023-08-08 17:49:37 +00:00
|
|
|
return nil, fmt.Errorf("could not open embed file: %w", err)
|
2023-08-04 22:56:40 +00:00
|
|
|
}
|
2023-08-08 17:49:37 +00:00
|
|
|
scanner := bufio.NewScanner(f)
|
|
|
|
scanner.Split(bufio.ScanLines)
|
|
|
|
|
|
|
|
data := []string{}
|
|
|
|
for scanner.Scan() {
|
|
|
|
data = append(data, scanner.Text())
|
|
|
|
}
|
|
|
|
f.Close()
|
|
|
|
|
|
|
|
// the digest of the file is set here so that the client knows a new operation is in progress
|
|
|
|
fileDigest, _ := GetSHA256Digest(bytes.NewReader([]byte(filePath)))
|
|
|
|
|
|
|
|
embeddings := []vector.Embedding{}
|
|
|
|
for i, d := range data {
|
|
|
|
if strings.TrimSpace(d) == "" {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
e.fn(api.ProgressResponse{
|
|
|
|
Status: fmt.Sprintf("creating embeddings for file %s", filePath),
|
|
|
|
Digest: fileDigest,
|
|
|
|
Total: len(data) - 1,
|
|
|
|
Completed: i,
|
|
|
|
})
|
2023-08-14 13:34:17 +00:00
|
|
|
if len(existing[d]) > 0 {
|
|
|
|
// already have an embedding for this line
|
|
|
|
embeddings = append(embeddings, vector.Embedding{Data: d, Vector: existing[d]})
|
|
|
|
continue
|
|
|
|
}
|
2023-08-30 20:35:03 +00:00
|
|
|
embed, err := loaded.llm.Embedding(context.Background(), d)
|
2023-08-08 17:49:37 +00:00
|
|
|
if err != nil {
|
2023-08-09 20:13:24 +00:00
|
|
|
log.Printf("failed to generate embedding for '%s' line %d: %v", filePath, i+1, err)
|
|
|
|
continue
|
2023-08-08 17:49:37 +00:00
|
|
|
}
|
|
|
|
embeddings = append(embeddings, vector.Embedding{Data: d, Vector: embed})
|
2023-08-04 22:56:40 +00:00
|
|
|
}
|
|
|
|
|
2023-08-08 17:49:37 +00:00
|
|
|
b, err := json.Marshal(embeddings)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("failed to encode embeddings: %w", err)
|
|
|
|
}
|
|
|
|
r := bytes.NewReader(b)
|
2023-08-04 22:56:40 +00:00
|
|
|
|
2023-08-08 17:49:37 +00:00
|
|
|
layer := &LayerReader{
|
|
|
|
Layer: Layer{
|
|
|
|
MediaType: "application/vnd.ollama.image.embed",
|
|
|
|
Digest: digest,
|
2023-08-14 13:34:17 +00:00
|
|
|
Size: r.Len(),
|
2023-08-08 17:49:37 +00:00
|
|
|
},
|
|
|
|
Reader: r,
|
|
|
|
}
|
2023-08-04 22:56:40 +00:00
|
|
|
|
2023-08-08 17:49:37 +00:00
|
|
|
layers = append(layers, layer)
|
|
|
|
}
|
2023-08-04 22:56:40 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return layers, nil
|
|
|
|
}
|
|
|
|
|
2023-08-14 13:34:17 +00:00
|
|
|
// existingFileEmbeddings checks if we already have embeddings for a file and loads them into a look-up map
|
|
|
|
func existingFileEmbeddings(digest string) (map[string][]float64, error) {
|
|
|
|
path, err := GetBlobsPath(digest)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("embeddings blobs path: %w", err)
|
|
|
|
}
|
|
|
|
existingFileEmbeddings := make(map[string][]float64)
|
|
|
|
if _, err := os.Stat(path); err == nil {
|
|
|
|
// already have some embeddings for this file, load embeddings previously generated
|
|
|
|
file, err := os.Open(path)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("failed to open existing embedding file: %s", err)
|
|
|
|
}
|
|
|
|
defer file.Close()
|
|
|
|
|
|
|
|
existing := []vector.Embedding{}
|
|
|
|
if err = json.NewDecoder(file).Decode(&existing); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
for _, e := range existing {
|
|
|
|
existingFileEmbeddings[e.Data] = e.Vector
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return existingFileEmbeddings, nil
|
|
|
|
}
|
|
|
|
|
2023-07-19 00:14:12 +00:00
|
|
|
func removeLayerFromLayers(layers []*LayerReader, mediaType string) []*LayerReader {
|
2023-07-17 00:02:22 +00:00
|
|
|
j := 0
|
|
|
|
for _, l := range layers {
|
|
|
|
if l.MediaType != mediaType {
|
|
|
|
layers[j] = l
|
|
|
|
j++
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return layers[:j]
|
|
|
|
}
|
|
|
|
|
2023-07-25 18:25:13 +00:00
|
|
|
func SaveLayers(layers []*LayerReader, fn func(resp api.ProgressResponse), force bool) error {
|
2023-07-17 00:02:22 +00:00
|
|
|
// Write each of the layers to disk
|
|
|
|
for _, layer := range layers {
|
2023-07-18 05:44:21 +00:00
|
|
|
fp, err := GetBlobsPath(layer.Digest)
|
2023-07-17 18:03:55 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2023-07-17 00:02:22 +00:00
|
|
|
|
|
|
|
_, err = os.Stat(fp)
|
2023-08-14 13:34:17 +00:00
|
|
|
// note: embed layers are always written since their digest doesnt indicate anything about the contents
|
|
|
|
if os.IsNotExist(err) || force || layer.MediaType == "application/vnd.ollama.image.embed" {
|
2023-07-25 18:25:13 +00:00
|
|
|
fn(api.ProgressResponse{Status: fmt.Sprintf("writing layer %s", layer.Digest)})
|
|
|
|
|
2023-07-17 00:02:22 +00:00
|
|
|
out, err := os.Create(fp)
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("couldn't create %s", fp)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
defer out.Close()
|
|
|
|
|
2023-07-19 00:14:12 +00:00
|
|
|
if _, err = io.Copy(out, layer.Reader); err != nil {
|
2023-07-17 00:02:22 +00:00
|
|
|
return err
|
|
|
|
}
|
2023-07-19 00:14:12 +00:00
|
|
|
|
2023-07-17 00:02:22 +00:00
|
|
|
} else {
|
2023-07-25 18:25:13 +00:00
|
|
|
fn(api.ProgressResponse{Status: fmt.Sprintf("using already created layer %s", layer.Digest)})
|
2023-07-17 00:02:22 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2023-07-19 00:14:12 +00:00
|
|
|
func CreateManifest(name string, cfg *LayerReader, layers []*Layer) error {
|
2023-08-22 16:39:42 +00:00
|
|
|
mp := ParseModelPath(name)
|
2023-07-17 00:02:22 +00:00
|
|
|
manifest := ManifestV2{
|
|
|
|
SchemaVersion: 2,
|
|
|
|
MediaType: "application/vnd.docker.distribution.manifest.v2+json",
|
|
|
|
Config: Layer{
|
|
|
|
MediaType: cfg.MediaType,
|
|
|
|
Size: cfg.Size,
|
|
|
|
Digest: cfg.Digest,
|
|
|
|
},
|
|
|
|
Layers: layers,
|
|
|
|
}
|
|
|
|
|
|
|
|
manifestJSON, err := json.Marshal(manifest)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2023-07-18 05:44:21 +00:00
|
|
|
fp, err := mp.GetManifestPath(true)
|
2023-07-17 00:02:22 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2023-07-17 18:03:55 +00:00
|
|
|
return os.WriteFile(fp, manifestJSON, 0o644)
|
2023-07-17 00:02:22 +00:00
|
|
|
}
|
|
|
|
|
2023-07-19 00:14:12 +00:00
|
|
|
func GetLayerWithBufferFromLayer(layer *Layer) (*LayerReader, error) {
|
2023-07-18 05:44:21 +00:00
|
|
|
fp, err := GetBlobsPath(layer.Digest)
|
2023-07-17 00:02:22 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
file, err := os.Open(fp)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("could not open blob: %w", err)
|
|
|
|
}
|
|
|
|
defer file.Close()
|
|
|
|
|
|
|
|
newLayer, err := CreateLayer(file)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
newLayer.MediaType = layer.MediaType
|
|
|
|
return newLayer, nil
|
|
|
|
}
|
|
|
|
|
2023-08-04 22:56:40 +00:00
|
|
|
// formatParams converts specified parameter options to their correct types
|
|
|
|
func formatParams(params map[string][]string) (map[string]interface{}, error) {
|
2023-08-01 17:36:31 +00:00
|
|
|
opts := api.Options{}
|
|
|
|
valueOpts := reflect.ValueOf(&opts).Elem() // names of the fields in the options struct
|
|
|
|
typeOpts := reflect.TypeOf(opts) // types of the fields in the options struct
|
2023-07-17 19:08:10 +00:00
|
|
|
|
2023-08-01 17:36:31 +00:00
|
|
|
// build map of json struct tags to their types
|
2023-07-17 19:08:10 +00:00
|
|
|
jsonOpts := make(map[string]reflect.StructField)
|
|
|
|
for _, field := range reflect.VisibleFields(typeOpts) {
|
|
|
|
jsonTag := strings.Split(field.Tag.Get("json"), ",")[0]
|
|
|
|
if jsonTag != "" {
|
|
|
|
jsonOpts[jsonTag] = field
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-08-01 17:36:31 +00:00
|
|
|
out := make(map[string]interface{})
|
2023-07-17 19:08:10 +00:00
|
|
|
// iterate params and set values based on json struct tags
|
2023-07-28 15:29:00 +00:00
|
|
|
for key, vals := range params {
|
2023-07-17 19:08:10 +00:00
|
|
|
if opt, ok := jsonOpts[key]; ok {
|
|
|
|
field := valueOpts.FieldByName(opt.Name)
|
|
|
|
if field.IsValid() && field.CanSet() {
|
|
|
|
switch field.Kind() {
|
|
|
|
case reflect.Float32:
|
2023-07-28 15:29:00 +00:00
|
|
|
floatVal, err := strconv.ParseFloat(vals[0], 32)
|
2023-07-17 19:08:10 +00:00
|
|
|
if err != nil {
|
2023-07-28 15:29:00 +00:00
|
|
|
return nil, fmt.Errorf("invalid float value %s", vals)
|
2023-07-17 19:08:10 +00:00
|
|
|
}
|
|
|
|
|
2023-08-01 17:36:31 +00:00
|
|
|
out[key] = floatVal
|
2023-07-17 19:08:10 +00:00
|
|
|
case reflect.Int:
|
2023-07-28 15:29:00 +00:00
|
|
|
intVal, err := strconv.ParseInt(vals[0], 10, 0)
|
2023-07-17 19:08:10 +00:00
|
|
|
if err != nil {
|
2023-07-28 15:29:00 +00:00
|
|
|
return nil, fmt.Errorf("invalid int value %s", vals)
|
2023-07-17 19:08:10 +00:00
|
|
|
}
|
|
|
|
|
2023-08-01 17:36:31 +00:00
|
|
|
out[key] = intVal
|
2023-07-17 19:08:10 +00:00
|
|
|
case reflect.Bool:
|
2023-07-28 15:29:00 +00:00
|
|
|
boolVal, err := strconv.ParseBool(vals[0])
|
2023-07-17 19:08:10 +00:00
|
|
|
if err != nil {
|
2023-07-28 15:29:00 +00:00
|
|
|
return nil, fmt.Errorf("invalid bool value %s", vals)
|
2023-07-17 19:08:10 +00:00
|
|
|
}
|
|
|
|
|
2023-08-01 17:36:31 +00:00
|
|
|
out[key] = boolVal
|
2023-07-17 19:08:10 +00:00
|
|
|
case reflect.String:
|
2023-08-01 17:36:31 +00:00
|
|
|
out[key] = vals[0]
|
2023-07-27 21:02:14 +00:00
|
|
|
case reflect.Slice:
|
2023-08-01 17:36:31 +00:00
|
|
|
// TODO: only string slices are supported right now
|
|
|
|
out[key] = vals
|
2023-07-17 19:08:10 +00:00
|
|
|
default:
|
|
|
|
return nil, fmt.Errorf("unknown type %s for %s", field.Kind(), key)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-08-04 22:56:40 +00:00
|
|
|
return out, nil
|
2023-07-17 00:02:22 +00:00
|
|
|
}
|
|
|
|
|
2023-07-19 00:14:12 +00:00
|
|
|
func getLayerDigests(layers []*LayerReader) ([]string, error) {
|
2023-07-17 00:02:22 +00:00
|
|
|
var digests []string
|
|
|
|
for _, l := range layers {
|
|
|
|
if l.Digest == "" {
|
|
|
|
return nil, fmt.Errorf("layer is missing a digest")
|
|
|
|
}
|
|
|
|
digests = append(digests, l.Digest)
|
|
|
|
}
|
|
|
|
return digests, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// CreateLayer creates a Layer object from a given file
|
2023-07-19 00:14:12 +00:00
|
|
|
func CreateLayer(f io.ReadSeeker) (*LayerReader, error) {
|
|
|
|
digest, size := GetSHA256Digest(f)
|
2023-07-21 20:33:56 +00:00
|
|
|
f.Seek(0, io.SeekStart)
|
2023-07-17 00:02:22 +00:00
|
|
|
|
2023-07-19 00:14:12 +00:00
|
|
|
layer := &LayerReader{
|
2023-07-17 00:02:22 +00:00
|
|
|
Layer: Layer{
|
|
|
|
MediaType: "application/vnd.docker.image.rootfs.diff.tar",
|
|
|
|
Digest: digest,
|
|
|
|
Size: size,
|
|
|
|
},
|
2023-07-19 00:14:12 +00:00
|
|
|
Reader: f,
|
2023-07-17 00:02:22 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return layer, nil
|
|
|
|
}
|
|
|
|
|
2023-07-24 15:27:28 +00:00
|
|
|
func CopyModel(src, dest string) error {
|
2023-08-22 16:39:42 +00:00
|
|
|
srcModelPath := ParseModelPath(src)
|
2023-08-22 04:56:56 +00:00
|
|
|
srcPath, err := srcModelPath.GetManifestPath(false)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2023-08-22 16:39:42 +00:00
|
|
|
destModelPath := ParseModelPath(dest)
|
2023-08-22 04:56:56 +00:00
|
|
|
destPath, err := destModelPath.GetManifestPath(true)
|
2023-07-24 15:27:28 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// copy the file
|
2023-07-28 17:38:15 +00:00
|
|
|
input, err := os.ReadFile(srcPath)
|
2023-07-24 15:27:28 +00:00
|
|
|
if err != nil {
|
|
|
|
fmt.Println("Error reading file:", err)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2023-07-28 17:38:15 +00:00
|
|
|
err = os.WriteFile(destPath, input, 0o644)
|
2023-07-24 15:27:28 +00:00
|
|
|
if err != nil {
|
|
|
|
fmt.Println("Error reading file:", err)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2023-07-22 06:02:12 +00:00
|
|
|
func DeleteModel(name string) error {
|
2023-08-22 16:39:42 +00:00
|
|
|
mp := ParseModelPath(name)
|
2023-08-29 03:50:24 +00:00
|
|
|
manifest, _, err := GetManifest(mp)
|
2023-07-20 23:09:23 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
deleteMap := make(map[string]bool)
|
|
|
|
for _, layer := range manifest.Layers {
|
|
|
|
deleteMap[layer.Digest] = true
|
|
|
|
}
|
|
|
|
deleteMap[manifest.Config.Digest] = true
|
|
|
|
|
|
|
|
fp, err := GetManifestPath()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
err = filepath.Walk(fp, func(path string, info os.FileInfo, err error) error {
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if !info.IsDir() {
|
|
|
|
path := path[len(fp)+1:]
|
|
|
|
slashIndex := strings.LastIndex(path, "/")
|
|
|
|
if slashIndex == -1 {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
tag := path[:slashIndex] + ":" + path[slashIndex+1:]
|
2023-08-22 16:39:42 +00:00
|
|
|
fmp := ParseModelPath(tag)
|
2023-07-20 23:09:23 +00:00
|
|
|
|
|
|
|
// skip the manifest we're trying to delete
|
|
|
|
if mp.GetFullTagname() == fmp.GetFullTagname() {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// save (i.e. delete from the deleteMap) any files used in other manifests
|
2023-08-29 03:50:24 +00:00
|
|
|
manifest, _, err := GetManifest(fmp)
|
2023-07-20 23:09:23 +00:00
|
|
|
if err != nil {
|
|
|
|
log.Printf("skipping file: %s", fp)
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
for _, layer := range manifest.Layers {
|
|
|
|
delete(deleteMap, layer.Digest)
|
|
|
|
}
|
|
|
|
delete(deleteMap, manifest.Config.Digest)
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
})
|
2023-07-31 22:26:18 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2023-07-20 23:09:23 +00:00
|
|
|
|
|
|
|
// only delete the files which are still in the deleteMap
|
|
|
|
for k, v := range deleteMap {
|
|
|
|
if v {
|
2023-07-22 00:30:40 +00:00
|
|
|
fp, err := GetBlobsPath(k)
|
2023-07-20 23:09:23 +00:00
|
|
|
if err != nil {
|
2023-07-22 00:30:40 +00:00
|
|
|
log.Printf("couldn't get file path for '%s': %v", k, err)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if err := os.Remove(fp); err != nil {
|
|
|
|
log.Printf("couldn't remove file '%s': %v", fp, err)
|
2023-07-20 23:09:23 +00:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
fp, err = mp.GetManifestPath(false)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
err = os.Remove(fp)
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("couldn't remove manifest file '%s': %v", fp, err)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2023-08-11 22:41:55 +00:00
|
|
|
func PushModel(ctx context.Context, name string, regOpts *RegistryOptions, fn func(api.ProgressResponse)) error {
|
2023-08-22 16:39:42 +00:00
|
|
|
mp := ParseModelPath(name)
|
2023-07-19 01:51:30 +00:00
|
|
|
fn(api.ProgressResponse{Status: "retrieving manifest"})
|
|
|
|
|
2023-08-22 16:39:42 +00:00
|
|
|
if mp.ProtocolScheme == "http" && !regOpts.Insecure {
|
|
|
|
return fmt.Errorf("insecure protocol http")
|
|
|
|
}
|
|
|
|
|
2023-08-29 03:50:24 +00:00
|
|
|
manifest, _, err := GetManifest(mp)
|
2023-07-17 00:02:22 +00:00
|
|
|
if err != nil {
|
2023-07-19 01:51:30 +00:00
|
|
|
fn(api.ProgressResponse{Status: "couldn't retrieve manifest"})
|
2023-07-17 00:02:22 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
var layers []*Layer
|
2023-08-01 01:37:40 +00:00
|
|
|
layers = append(layers, manifest.Layers...)
|
2023-07-17 00:02:22 +00:00
|
|
|
layers = append(layers, &manifest.Config)
|
|
|
|
|
|
|
|
for _, layer := range layers {
|
2023-08-11 22:41:55 +00:00
|
|
|
exists, err := checkBlobExistence(ctx, mp, layer.Digest, regOpts)
|
2023-07-17 00:02:22 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
if exists {
|
2023-07-19 01:51:30 +00:00
|
|
|
fn(api.ProgressResponse{
|
|
|
|
Status: "using existing layer",
|
|
|
|
Digest: layer.Digest,
|
2023-07-23 00:31:26 +00:00
|
|
|
Total: layer.Size,
|
|
|
|
Completed: layer.Size,
|
2023-07-19 01:51:30 +00:00
|
|
|
})
|
2023-07-23 00:31:26 +00:00
|
|
|
log.Printf("Layer %s already exists", layer.Digest)
|
2023-07-17 00:02:22 +00:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2023-07-19 01:51:30 +00:00
|
|
|
fn(api.ProgressResponse{
|
2023-07-23 00:31:26 +00:00
|
|
|
Status: "starting upload",
|
|
|
|
Digest: layer.Digest,
|
|
|
|
Total: layer.Size,
|
2023-07-19 01:51:30 +00:00
|
|
|
})
|
2023-07-17 00:02:22 +00:00
|
|
|
|
2023-08-14 22:07:00 +00:00
|
|
|
location, err := startUpload(ctx, mp, layer, regOpts)
|
2023-07-17 00:02:22 +00:00
|
|
|
if err != nil {
|
|
|
|
log.Printf("couldn't start upload: %v", err)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2023-08-22 01:38:31 +00:00
|
|
|
if strings.HasPrefix(path.Base(location.Path), "sha256:") {
|
|
|
|
layer.Digest = path.Base(location.Path)
|
2023-08-14 22:07:00 +00:00
|
|
|
fn(api.ProgressResponse{
|
|
|
|
Status: "using existing layer",
|
|
|
|
Digest: layer.Digest,
|
|
|
|
Total: layer.Size,
|
|
|
|
Completed: layer.Size,
|
|
|
|
})
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2023-08-28 22:35:18 +00:00
|
|
|
if err := uploadBlobChunked(ctx, location, layer, regOpts, fn); err != nil {
|
2023-07-17 00:02:22 +00:00
|
|
|
log.Printf("error uploading blob: %v", err)
|
|
|
|
return err
|
|
|
|
}
|
2023-07-19 01:51:30 +00:00
|
|
|
}
|
|
|
|
|
2023-07-23 00:31:26 +00:00
|
|
|
fn(api.ProgressResponse{Status: "pushing manifest"})
|
2023-08-22 01:38:31 +00:00
|
|
|
requestURL := mp.BaseURL()
|
|
|
|
requestURL = requestURL.JoinPath("v2", mp.GetNamespaceRepository(), "manifests", mp.Tag)
|
2023-07-17 00:02:22 +00:00
|
|
|
|
|
|
|
manifestJSON, err := json.Marshal(manifest)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2023-08-22 01:24:42 +00:00
|
|
|
headers := make(http.Header)
|
|
|
|
headers.Set("Content-Type", "application/vnd.docker.distribution.manifest.v2+json")
|
2023-08-22 01:38:31 +00:00
|
|
|
resp, err := makeRequestWithRetry(ctx, "PUT", requestURL, headers, bytes.NewReader(manifestJSON), regOpts)
|
2023-07-17 00:02:22 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
defer resp.Body.Close()
|
|
|
|
|
2023-07-23 00:31:26 +00:00
|
|
|
fn(api.ProgressResponse{Status: "success"})
|
2023-07-17 00:02:22 +00:00
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2023-07-25 21:08:51 +00:00
|
|
|
func PullModel(ctx context.Context, name string, regOpts *RegistryOptions, fn func(api.ProgressResponse)) error {
|
2023-08-22 16:39:42 +00:00
|
|
|
mp := ParseModelPath(name)
|
|
|
|
|
|
|
|
if mp.ProtocolScheme == "http" && !regOpts.Insecure {
|
|
|
|
return fmt.Errorf("insecure protocol http")
|
2023-08-22 04:56:56 +00:00
|
|
|
}
|
2023-07-17 00:02:22 +00:00
|
|
|
|
2023-07-19 01:51:30 +00:00
|
|
|
fn(api.ProgressResponse{Status: "pulling manifest"})
|
2023-07-17 00:02:22 +00:00
|
|
|
|
2023-08-11 22:41:55 +00:00
|
|
|
manifest, err := pullModelManifest(ctx, mp, regOpts)
|
2023-07-17 00:02:22 +00:00
|
|
|
if err != nil {
|
2023-07-24 21:48:17 +00:00
|
|
|
return fmt.Errorf("pull model manifest: %s", err)
|
2023-07-17 00:02:22 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
var layers []*Layer
|
2023-07-20 18:18:00 +00:00
|
|
|
layers = append(layers, manifest.Layers...)
|
2023-07-17 00:02:22 +00:00
|
|
|
layers = append(layers, &manifest.Config)
|
|
|
|
|
|
|
|
for _, layer := range layers {
|
2023-08-15 18:07:19 +00:00
|
|
|
if err := downloadBlob(
|
|
|
|
ctx,
|
|
|
|
downloadOpts{
|
|
|
|
mp: mp,
|
|
|
|
digest: layer.Digest,
|
|
|
|
regOpts: regOpts,
|
|
|
|
fn: fn,
|
|
|
|
}); err != nil {
|
2023-07-17 00:02:22 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-07-20 18:44:05 +00:00
|
|
|
fn(api.ProgressResponse{Status: "verifying sha256 digest"})
|
|
|
|
for _, layer := range layers {
|
|
|
|
if err := verifyBlob(layer.Digest); err != nil {
|
2023-07-24 18:53:01 +00:00
|
|
|
if errors.Is(err, errDigestMismatch) {
|
|
|
|
// something went wrong, delete the blob
|
|
|
|
fp, err := GetBlobsPath(layer.Digest)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if err := os.Remove(fp); err != nil {
|
|
|
|
// log this, but return the original error
|
|
|
|
log.Printf("couldn't remove file with digest mismatch '%s': %v", fp, err)
|
|
|
|
}
|
|
|
|
}
|
2023-07-20 18:44:05 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-07-19 01:51:30 +00:00
|
|
|
fn(api.ProgressResponse{Status: "writing manifest"})
|
2023-07-17 00:02:22 +00:00
|
|
|
|
2023-07-17 18:03:55 +00:00
|
|
|
manifestJSON, err := json.Marshal(manifest)
|
2023-07-17 00:02:22 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2023-07-18 05:44:21 +00:00
|
|
|
fp, err := mp.GetManifestPath(true)
|
2023-07-17 00:02:22 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2023-07-20 18:18:00 +00:00
|
|
|
err = os.WriteFile(fp, manifestJSON, 0o644)
|
2023-07-17 00:02:22 +00:00
|
|
|
if err != nil {
|
|
|
|
log.Printf("couldn't write to %s", fp)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2023-07-19 01:51:30 +00:00
|
|
|
fn(api.ProgressResponse{Status: "success"})
|
2023-07-17 00:02:22 +00:00
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2023-08-11 22:41:55 +00:00
|
|
|
func pullModelManifest(ctx context.Context, mp ModelPath, regOpts *RegistryOptions) (*ManifestV2, error) {
|
2023-08-22 01:38:31 +00:00
|
|
|
requestURL := mp.BaseURL().JoinPath("v2", mp.GetNamespaceRepository(), "manifests", mp.Tag)
|
2023-07-17 00:02:22 +00:00
|
|
|
|
2023-08-22 01:24:42 +00:00
|
|
|
headers := make(http.Header)
|
|
|
|
headers.Set("Accept", "application/vnd.docker.distribution.manifest.v2+json")
|
2023-08-22 01:38:31 +00:00
|
|
|
resp, err := makeRequest(ctx, "GET", requestURL, headers, nil, regOpts)
|
2023-07-17 00:02:22 +00:00
|
|
|
if err != nil {
|
|
|
|
log.Printf("couldn't get manifest: %v", err)
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
defer resp.Body.Close()
|
|
|
|
|
2023-08-27 04:55:21 +00:00
|
|
|
if resp.StatusCode >= http.StatusBadRequest {
|
2023-07-24 21:48:17 +00:00
|
|
|
if resp.StatusCode == http.StatusNotFound {
|
2023-07-25 14:30:14 +00:00
|
|
|
return nil, fmt.Errorf("model not found")
|
2023-07-24 21:48:17 +00:00
|
|
|
}
|
2023-08-27 04:55:21 +00:00
|
|
|
|
2023-07-17 00:02:22 +00:00
|
|
|
body, _ := io.ReadAll(resp.Body)
|
2023-07-24 19:04:21 +00:00
|
|
|
return nil, fmt.Errorf("on pull registry responded with code %d: %s", resp.StatusCode, body)
|
2023-07-17 00:02:22 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
var m *ManifestV2
|
|
|
|
if err := json.NewDecoder(resp.Body).Decode(&m); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return m, err
|
|
|
|
}
|
|
|
|
|
2023-07-21 20:33:56 +00:00
|
|
|
func createConfigLayer(config ConfigV2, layers []string) (*LayerReader, error) {
|
|
|
|
config.RootFS = RootFS{
|
|
|
|
Type: "layers",
|
|
|
|
DiffIDs: layers,
|
2023-07-17 00:02:22 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
configJSON, err := json.Marshal(config)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2023-07-19 07:47:55 +00:00
|
|
|
digest, size := GetSHA256Digest(bytes.NewBuffer(configJSON))
|
2023-07-17 00:02:22 +00:00
|
|
|
|
2023-07-19 00:14:12 +00:00
|
|
|
layer := &LayerReader{
|
2023-07-17 00:02:22 +00:00
|
|
|
Layer: Layer{
|
|
|
|
MediaType: "application/vnd.docker.container.image.v1+json",
|
|
|
|
Digest: digest,
|
|
|
|
Size: size,
|
|
|
|
},
|
2023-07-19 07:47:55 +00:00
|
|
|
Reader: bytes.NewBuffer(configJSON),
|
2023-07-17 00:02:22 +00:00
|
|
|
}
|
|
|
|
return layer, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// GetSHA256Digest returns the SHA256 hash of a given buffer and returns it, and the size of buffer
|
2023-07-19 00:14:12 +00:00
|
|
|
func GetSHA256Digest(r io.Reader) (string, int) {
|
|
|
|
h := sha256.New()
|
|
|
|
n, err := io.Copy(h, r)
|
|
|
|
if err != nil {
|
|
|
|
log.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
return fmt.Sprintf("sha256:%x", h.Sum(nil)), int(n)
|
2023-07-17 00:02:22 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Function to check if a blob already exists in the Docker registry
|
2023-08-11 22:41:55 +00:00
|
|
|
func checkBlobExistence(ctx context.Context, mp ModelPath, digest string, regOpts *RegistryOptions) (bool, error) {
|
2023-08-22 01:38:31 +00:00
|
|
|
requestURL := mp.BaseURL()
|
|
|
|
requestURL = requestURL.JoinPath("v2", mp.GetNamespaceRepository(), "blobs", digest)
|
2023-07-17 00:02:22 +00:00
|
|
|
|
2023-08-22 01:38:31 +00:00
|
|
|
resp, err := makeRequest(ctx, "HEAD", requestURL, nil, nil, regOpts)
|
2023-07-17 00:02:22 +00:00
|
|
|
if err != nil {
|
|
|
|
log.Printf("couldn't check for blob: %v", err)
|
|
|
|
return false, err
|
|
|
|
}
|
|
|
|
defer resp.Body.Close()
|
|
|
|
|
|
|
|
// Check for success: If the blob exists, the Docker registry will respond with a 200 OK
|
2023-08-27 04:55:21 +00:00
|
|
|
return resp.StatusCode < http.StatusBadRequest, nil
|
2023-07-17 00:02:22 +00:00
|
|
|
}
|
|
|
|
|
2023-08-22 01:38:31 +00:00
|
|
|
func makeRequestWithRetry(ctx context.Context, method string, requestURL *url.URL, headers http.Header, body io.ReadSeeker, regOpts *RegistryOptions) (*http.Response, error) {
|
2023-08-17 19:35:29 +00:00
|
|
|
var status string
|
|
|
|
for try := 0; try < MaxRetries; try++ {
|
2023-08-22 01:38:31 +00:00
|
|
|
resp, err := makeRequest(ctx, method, requestURL, headers, body, regOpts)
|
2023-08-17 19:35:29 +00:00
|
|
|
if err != nil {
|
|
|
|
log.Printf("couldn't start upload: %v", err)
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
status = resp.Status
|
|
|
|
|
2023-08-27 04:55:21 +00:00
|
|
|
switch {
|
|
|
|
case resp.StatusCode == http.StatusUnauthorized:
|
2023-08-17 19:35:29 +00:00
|
|
|
auth := resp.Header.Get("www-authenticate")
|
|
|
|
authRedir := ParseAuthRedirectString(auth)
|
|
|
|
token, err := getAuthToken(ctx, authRedir, regOpts)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
regOpts.Token = token
|
|
|
|
if body != nil {
|
|
|
|
if _, err := body.Seek(0, io.SeekStart); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
continue
|
2023-08-27 04:55:21 +00:00
|
|
|
case resp.StatusCode >= http.StatusBadRequest:
|
2023-08-17 19:35:29 +00:00
|
|
|
body, _ := io.ReadAll(resp.Body)
|
|
|
|
return nil, fmt.Errorf("on upload registry responded with code %d: %s", resp.StatusCode, body)
|
2023-08-27 04:55:21 +00:00
|
|
|
default:
|
|
|
|
return resp, nil
|
2023-08-17 19:35:29 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil, fmt.Errorf("max retry exceeded: %v", status)
|
|
|
|
}
|
|
|
|
|
2023-08-22 01:38:31 +00:00
|
|
|
func makeRequest(ctx context.Context, method string, requestURL *url.URL, headers http.Header, body io.Reader, regOpts *RegistryOptions) (*http.Response, error) {
|
|
|
|
if requestURL.Scheme != "http" && regOpts.Insecure {
|
|
|
|
requestURL.Scheme = "http"
|
2023-07-21 22:42:19 +00:00
|
|
|
}
|
|
|
|
|
2023-08-22 01:38:31 +00:00
|
|
|
req, err := http.NewRequestWithContext(ctx, method, requestURL.String(), body)
|
2023-07-17 00:02:22 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2023-08-22 01:24:42 +00:00
|
|
|
if headers != nil {
|
|
|
|
req.Header = headers
|
|
|
|
}
|
|
|
|
|
2023-08-10 18:34:25 +00:00
|
|
|
if regOpts.Token != "" {
|
|
|
|
req.Header.Set("Authorization", "Bearer "+regOpts.Token)
|
|
|
|
} else if regOpts.Username != "" && regOpts.Password != "" {
|
|
|
|
req.SetBasicAuth(regOpts.Username, regOpts.Password)
|
2023-07-17 00:02:22 +00:00
|
|
|
}
|
|
|
|
|
2023-08-22 01:24:42 +00:00
|
|
|
req.Header.Set("User-Agent", fmt.Sprintf("ollama/%s (%s %s) Go/%s", version.Version, runtime.GOARCH, runtime.GOOS, runtime.Version()))
|
2023-07-17 00:02:22 +00:00
|
|
|
|
|
|
|
client := &http.Client{
|
|
|
|
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
|
|
|
if len(via) >= 10 {
|
|
|
|
return fmt.Errorf("too many redirects")
|
|
|
|
}
|
|
|
|
log.Printf("redirected to: %s\n", req.URL)
|
|
|
|
return nil
|
|
|
|
},
|
|
|
|
}
|
2023-08-16 17:30:41 +00:00
|
|
|
|
2023-07-17 00:02:22 +00:00
|
|
|
resp, err := client.Do(req)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return resp, nil
|
|
|
|
}
|
2023-07-20 18:44:05 +00:00
|
|
|
|
2023-08-10 18:34:25 +00:00
|
|
|
func getValue(header, key string) string {
|
|
|
|
startIdx := strings.Index(header, key+"=")
|
|
|
|
if startIdx == -1 {
|
|
|
|
return ""
|
|
|
|
}
|
|
|
|
|
|
|
|
// Move the index to the starting quote after the key.
|
|
|
|
startIdx += len(key) + 2
|
|
|
|
endIdx := startIdx
|
|
|
|
|
|
|
|
for endIdx < len(header) {
|
|
|
|
if header[endIdx] == '"' {
|
|
|
|
if endIdx+1 < len(header) && header[endIdx+1] != ',' { // If the next character isn't a comma, continue
|
|
|
|
endIdx++
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
break
|
|
|
|
}
|
|
|
|
endIdx++
|
|
|
|
}
|
|
|
|
return header[startIdx:endIdx]
|
|
|
|
}
|
|
|
|
|
|
|
|
func ParseAuthRedirectString(authStr string) AuthRedirect {
|
|
|
|
authStr = strings.TrimPrefix(authStr, "Bearer ")
|
|
|
|
|
|
|
|
return AuthRedirect{
|
|
|
|
Realm: getValue(authStr, "realm"),
|
|
|
|
Service: getValue(authStr, "service"),
|
|
|
|
Scope: getValue(authStr, "scope"),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-07-24 18:53:01 +00:00
|
|
|
var errDigestMismatch = fmt.Errorf("digest mismatch, file must be downloaded again")
|
|
|
|
|
2023-07-20 18:44:05 +00:00
|
|
|
func verifyBlob(digest string) error {
|
|
|
|
fp, err := GetBlobsPath(digest)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
f, err := os.Open(fp)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
defer f.Close()
|
|
|
|
|
|
|
|
fileDigest, _ := GetSHA256Digest(f)
|
|
|
|
if digest != fileDigest {
|
2023-07-24 18:53:01 +00:00
|
|
|
return fmt.Errorf("%w: want %s, got %s", errDigestMismatch, digest, fileDigest)
|
2023-07-20 18:44:05 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|