baalajimaestro/ollama
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

fix resumable downloads

Browse source 
glob returns files in lexical order which is not appropriate when
rebuilding the parts list
This commit is contained in:
Michael Yang 2023-10-02 15:26:27 -07:00
parent 090d08422b
commit 711e891f0f
1 changed files with 23 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
server/download.go
View file

@ -12,6 +12,7 @@ import (
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
@ -39,9 +40,18 @@ type blobDownload struct {
}
type blobDownloadPart struct {
N int
Offset int64
Size int64
Completed int64
*blobDownload `json:"-"`
}
func (p *blobDownloadPart) Name() string {
return strings.Join([]string{
p.blobDownload.Name, "partial", strconv.Itoa(p.N),
}, "-")
}
func (b *blobDownload) Prepare(ctx context.Context, requestURL *url.URL, opts *RegistryOptions) error {
@ -78,14 +88,10 @@ func (b *blobDownload) Prepare(ctx context.Context, requestURL *url.URL, opts *R
size = b.Total - offset
}
partName := b.Name + "-partial-" + strconv.Itoa(len(b.Parts))
part := blobDownloadPart{Offset: offset, Size: size}
if err := b.writePart(partName, &part); err != nil {
if err := b.newPart(offset, size); err != nil {
return err
}
b.Parts = append(b.Parts, &part)
offset += size
}
}
@ -161,7 +167,6 @@ func (b *blobDownload) Run(ctx context.Context, requestURL *url.URL, opts *Regis
func (b *blobDownload) downloadChunk(ctx context.Context, requestURL *url.URL, i int, opts *RegistryOptions) error {
part := b.Parts[i]
partName := b.File.Name() + "-" + strconv.Itoa(i)
offset := part.Offset + part.Completed
w := io.NewOffsetWriter(b.File, offset)
@ -181,7 +186,7 @@ func (b *blobDownload) downloadChunk(ctx context.Context, requestURL *url.URL, i
}
part.Completed += n
if err := b.writePart(partName, part); err != nil {
if err := b.writePart(part.Name(), part); err != nil {
return err
}
@ -189,6 +194,16 @@ func (b *blobDownload) downloadChunk(ctx context.Context, requestURL *url.URL, i
return err
}
func (b *blobDownload) newPart(offset, size int64) error {
part := blobDownloadPart{blobDownload: b, Offset: offset, Size: size, N: len(b.Parts)}
if err := b.writePart(part.Name(), &part); err != nil {
return err
}
b.Parts = append(b.Parts, &part)
return nil
}
func (b *blobDownload) readPart(partName string) (*blobDownloadPart, error) {
var part blobDownloadPart
partFile, err := os.Open(partName)
@ -201,6 +216,7 @@ func (b *blobDownload) readPart(partName string) (*blobDownloadPart, error) {
return nil, err
}
part.blobDownload = b
return &part, nil
}