70a93057cd
previous layer creation was not ideal because: 1. it required reading the input file multiple times, once to calculate the sha256 checksum, another to write it to disk, and potentially one more to decode the underlying gguf 2. used io.ReadSeeker which is prone to user error. if the file isn't reset correctly or in the right place, it could end up reading an empty file there are also some brittleness when reading existing layers else writing the inherited layers will error reading an already closed file this commit aims to fix these issues by restructuring layer creation. 1. it will now write the layer to a temporary file as well as the hash function and move it to the final location on Commit 2. layers are read once once when copied to the destination. exception is raw model files which still requires a second read to decode the model metadata
109 lines
1.9 KiB
Go
109 lines
1.9 KiB
Go
package server
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"runtime"
|
|
"strings"
|
|
|
|
"golang.org/x/exp/slices"
|
|
)
|
|
|
|
type Layers struct {
|
|
items []*Layer
|
|
}
|
|
|
|
func (ls *Layers) Add(layer *Layer) {
|
|
if layer.Size > 0 {
|
|
ls.items = append(ls.items, layer)
|
|
}
|
|
}
|
|
|
|
func (ls *Layers) Replace(layer *Layer) {
|
|
if layer.Size > 0 {
|
|
mediatype := layer.MediaType
|
|
layers := slices.DeleteFunc(ls.items, func(l *Layer) bool {
|
|
return l.MediaType == mediatype
|
|
})
|
|
|
|
ls.items = append(layers, layer)
|
|
}
|
|
}
|
|
|
|
type Layer struct {
|
|
MediaType string `json:"mediaType"`
|
|
Digest string `json:"digest"`
|
|
Size int64 `json:"size"`
|
|
From string `json:"from,omitempty"`
|
|
|
|
tempFileName string
|
|
}
|
|
|
|
func NewLayer(r io.Reader, mediatype string) (*Layer, error) {
|
|
blobs, err := GetBlobsPath("")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
delimiter := ":"
|
|
if runtime.GOOS == "windows" {
|
|
delimiter = "-"
|
|
}
|
|
|
|
pattern := strings.Join([]string{"sha256", "*-partial"}, delimiter)
|
|
temp, err := os.CreateTemp(blobs, pattern)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer temp.Close()
|
|
|
|
sha256sum := sha256.New()
|
|
n, err := io.Copy(io.MultiWriter(temp, sha256sum), r)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return &Layer{
|
|
MediaType: mediatype,
|
|
Digest: fmt.Sprintf("sha256:%x", sha256sum.Sum(nil)),
|
|
Size: n,
|
|
tempFileName: temp.Name(),
|
|
}, nil
|
|
}
|
|
|
|
func NewLayerFromLayer(digest, mediatype, from string) (*Layer, error) {
|
|
blob, err := GetBlobsPath(digest)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
fi, err := os.Stat(blob)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return &Layer{
|
|
MediaType: mediatype,
|
|
Digest: digest,
|
|
Size: fi.Size(),
|
|
From: from,
|
|
}, nil
|
|
}
|
|
|
|
func (l *Layer) Commit() (bool, error) {
|
|
// always remove temp
|
|
defer os.Remove(l.tempFileName)
|
|
|
|
blob, err := GetBlobsPath(l.Digest)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
if _, err := os.Stat(blob); err != nil {
|
|
return true, os.Rename(l.tempFileName, blob)
|
|
}
|
|
|
|
return false, nil
|
|
}
|