From 997253143f36f8988c04f2bb184b0e8bb6e154d4 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Fri, 15 Dec 2023 11:33:52 -0800 Subject: [PATCH 1/7] add lint and test on pull_request --- .github/workflows/test.yaml | 78 +++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 .github/workflows/test.yaml diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 00000000..4ffab937 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,78 @@ +name: test + +on: + pull_request: + +jobs: + generate: + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v4 + with: + go-version: '1.20' + cache: true + - if: ${{ startsWith(matrix.os, 'windows-') }} + shell: pwsh + run: | + $path = vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath + if ($path) { + $path = join-path $path 'Common7\Tools\vsdevcmd.bat' + if (test-path $path) { + cmd /s /c """$path"" $args && set" | where { $_ -match '(\w+)=(.*)' } | foreach { + echo "$($Matches[1])=$($Matches[2])" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append + } + } + } + + echo "C:\Program Files\Git\usr\bin" | Out-File -FilePath $Env:GITHUB_PATH -Encoding utf8 -Append + - run: go get ./... + - run: go generate -x ./... + - uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.os }}-libraries + path: | + llm/llama.cpp/build/**/lib/* + lint: + needs: generate + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - uses: actions/setup-go@v4 + with: + go-version: '1.20' + cache: false + - uses: actions/download-artifact@v4 + with: + name: ${{ matrix.os }}-libraries + path: llm/llama.cpp/build + - uses: golangci/golangci-lint-action@v3 + test: + needs: generate + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - uses: actions/setup-go@v4 + with: + go-version: '1.20' + cache: true + - run: go get + - uses: actions/download-artifact@v4 + with: + name: ${{ matrix.os }}-libraries + path: llm/llama.cpp/build + - run: go build + - run: go test -v ./... From acfc376efda39919baeca9a8b68992b7fb74bcc0 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Fri, 15 Dec 2023 14:25:12 -0800 Subject: [PATCH 2/7] add .golangci.yaml --- .golangci.yaml | 27 +++++++++++++++++++++++++++ readline/history.go | 4 ++-- server/download.go | 6 +++--- server/manifests.go | 4 ++-- server/routes.go | 3 ++- server/routes_test.go | 1 - server/upload.go | 2 +- 7 files changed, 37 insertions(+), 10 deletions(-) create mode 100644 .golangci.yaml diff --git a/.golangci.yaml b/.golangci.yaml new file mode 100644 index 00000000..85558f03 --- /dev/null +++ b/.golangci.yaml @@ -0,0 +1,27 @@ +run: + timeout: 5m +linters: + enable: + - asasalint + - bidichk + - bodyclose + - containedctx + - contextcheck + - exportloopref + - gocheckcompilerdirectives + # FIXME: for some reason this errors on windows + # - gofmt + # - goimports + - misspell + - nilerr + - unused +linters-settings: + errcheck: + # exclude the following functions since we don't generally + # need to be concerned with the returned errors + exclude-functions: + - encoding/binary.Read + - (*os.File).Seek + - (*bufio.Writer).WriteString + - (*github.com/spf13/pflag.FlagSet).Set + - (*github.com/jmorganca/ollama/llm.readSeekOffset).Seek diff --git a/readline/history.go b/readline/history.go index 57fccb7c..52c7d342 100644 --- a/readline/history.go +++ b/readline/history.go @@ -49,7 +49,7 @@ func (h *History) Init() error { h.Filename = path - f, err := os.OpenFile(path, os.O_CREATE|os.O_RDONLY, 0600) + f, err := os.OpenFile(path, os.O_CREATE|os.O_RDONLY, 0o600) if err != nil { if errors.Is(err, os.ErrNotExist) { return nil @@ -132,7 +132,7 @@ func (h *History) Save() error { tmpFile := h.Filename + ".tmp" - f, err := os.OpenFile(tmpFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC|os.O_APPEND, 0666) + f, err := os.OpenFile(tmpFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC|os.O_APPEND, 0o666) if err != nil { return err } diff --git a/server/download.go b/server/download.go index 3c823f32..d586c9cd 100644 --- a/server/download.go +++ b/server/download.go @@ -98,7 +98,7 @@ func (b *blobDownload) Prepare(ctx context.Context, requestURL *url.URL, opts *R b.Total, _ = strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64) - var size = b.Total / numDownloadParts + size := b.Total / numDownloadParts switch { case size < minDownloadPartSize: size = minDownloadPartSize @@ -132,7 +132,7 @@ func (b *blobDownload) run(ctx context.Context, requestURL *url.URL, opts *Regis defer blobDownloadManager.Delete(b.Digest) ctx, b.CancelFunc = context.WithCancel(ctx) - file, err := os.OpenFile(b.Name+"-partial", os.O_CREATE|os.O_RDWR, 0644) + file, err := os.OpenFile(b.Name+"-partial", os.O_CREATE|os.O_RDWR, 0o644) if err != nil { return err } @@ -246,7 +246,7 @@ func (b *blobDownload) readPart(partName string) (*blobDownloadPart, error) { } func (b *blobDownload) writePart(partName string, part *blobDownloadPart) error { - partFile, err := os.OpenFile(partName, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644) + partFile, err := os.OpenFile(partName, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0o644) if err != nil { return err } diff --git a/server/manifests.go b/server/manifests.go index 4e42b2ca..2b39db65 100644 --- a/server/manifests.go +++ b/server/manifests.go @@ -26,9 +26,9 @@ func WriteManifest(name string, config *Layer, layers []*Layer) error { return err } - if err := os.MkdirAll(filepath.Dir(manifestPath), 0755); err != nil { + if err := os.MkdirAll(filepath.Dir(manifestPath), 0o755); err != nil { return err } - return os.WriteFile(manifestPath, b.Bytes(), 0644) + return os.WriteFile(manifestPath, b.Bytes(), 0o644) } diff --git a/server/routes.go b/server/routes.go index 648a408c..fe4102d2 100644 --- a/server/routes.go +++ b/server/routes.go @@ -198,7 +198,8 @@ func GenerateHandler(c *gin.Context) { c.JSON(http.StatusOK, api.GenerateResponse{ CreatedAt: time.Now().UTC(), Model: req.Model, - Done: true}) + Done: true, + }) return } diff --git a/server/routes_test.go b/server/routes_test.go index 07bb699d..e09648f9 100644 --- a/server/routes_test.go +++ b/server/routes_test.go @@ -201,5 +201,4 @@ func Test_Routes(t *testing.T) { } } - } diff --git a/server/upload.go b/server/upload.go index 721c5e7c..7e8f55ea 100644 --- a/server/upload.go +++ b/server/upload.go @@ -88,7 +88,7 @@ func (b *blobUpload) Prepare(ctx context.Context, requestURL *url.URL, opts *Reg return nil } - var size = b.Total / numUploadParts + size := b.Total / numUploadParts switch { case size < minUploadPartSize: size = minUploadPartSize From 2bb2bdd5d41af52a19d34cf3ee5d4148839562e5 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Fri, 15 Dec 2023 14:07:34 -0800 Subject: [PATCH 3/7] fix lint --- cmd/interactive.go | 5 +---- llm/ggml.go | 6 +++--- progress/progress.go | 4 +--- readline/history.go | 4 ++-- readline/readline.go | 1 + readline/readline_unix.go | 2 +- server/download.go | 3 ++- server/images.go | 1 + server/routes.go | 1 + server/routes_test.go | 2 +- server/upload.go | 1 + 11 files changed, 15 insertions(+), 15 deletions(-) diff --git a/cmd/interactive.go b/cmd/interactive.go index 0b264a69..40cbd60d 100644 --- a/cmd/interactive.go +++ b/cmd/interactive.go @@ -238,10 +238,7 @@ func generateInteractive(cmd *cobra.Command, opts generateOptions) error { usageParameters() continue } - var params []string - for _, p := range args[3:] { - params = append(params, p) - } + params := args[3:] fp, err := api.FormatParams(map[string][]string{args[2]: params}) if err != nil { fmt.Printf("Couldn't set parameter: %q\n\n", err) diff --git a/llm/ggml.go b/llm/ggml.go index 7ba42962..ac3a39d8 100644 --- a/llm/ggml.go +++ b/llm/ggml.go @@ -98,9 +98,9 @@ func (c *containerLORA) Name() string { return "ggla" } -func (c *containerLORA) Decode(ro *readSeekOffset) (model, error) { +func (c *containerLORA) Decode(rso *readSeekOffset) (model, error) { var version uint32 - binary.Read(ro, binary.LittleEndian, &version) + binary.Read(rso, binary.LittleEndian, &version) switch version { case 1: @@ -111,7 +111,7 @@ func (c *containerLORA) Decode(ro *readSeekOffset) (model, error) { c.version = version // remaining file contents aren't decoded - ro.Seek(0, io.SeekEnd) + rso.Seek(0, io.SeekEnd) return nil, nil } diff --git a/progress/progress.go b/progress/progress.go index 78917e9c..556ba00f 100644 --- a/progress/progress.go +++ b/progress/progress.go @@ -77,7 +77,7 @@ func (p *Progress) Add(key string, state State) { p.states = append(p.states, state) } -func (p *Progress) render() error { +func (p *Progress) render() { p.mu.Lock() defer p.mu.Unlock() @@ -101,8 +101,6 @@ func (p *Progress) render() error { } p.pos = len(p.states) - - return nil } func (p *Progress) start() { diff --git a/readline/history.go b/readline/history.go index 52c7d342..db99fad0 100644 --- a/readline/history.go +++ b/readline/history.go @@ -23,7 +23,7 @@ type History struct { func NewHistory() (*History, error) { h := &History{ Buf: arraylist.New(), - Limit: 100, //resizeme + Limit: 100, // resizeme Autosave: true, Enabled: true, } @@ -84,7 +84,7 @@ func (h *History) Add(l []rune) { h.Compact() h.Pos = h.Size() if h.Autosave { - h.Save() + _ = h.Save() } } diff --git a/readline/readline.go b/readline/readline.go index ba9cf4bf..202d9fa7 100644 --- a/readline/readline.go +++ b/readline/readline.go @@ -72,6 +72,7 @@ func (i *Instance) Readline() (string, error) { if err != nil { return "", err } + // nolint: errcheck defer UnsetRawMode(fd, termios) buf, _ := NewBuffer(i.Prompt) diff --git a/readline/readline_unix.go b/readline/readline_unix.go index a1e34d27..73930c3d 100644 --- a/readline/readline_unix.go +++ b/readline/readline_unix.go @@ -11,7 +11,7 @@ func handleCharCtrlZ(fd int, termios *Termios) (string, error) { return "", err } - syscall.Kill(0, syscall.SIGSTOP) + _ = syscall.Kill(0, syscall.SIGSTOP) // on resume... return "", nil diff --git a/server/download.go b/server/download.go index d586c9cd..cf86f994 100644 --- a/server/download.go +++ b/server/download.go @@ -138,7 +138,7 @@ func (b *blobDownload) run(ctx context.Context, requestURL *url.URL, opts *Regis } defer file.Close() - file.Truncate(b.Total) + _ = file.Truncate(b.Total) g, inner := errgroup.WithContext(ctx) g.SetLimit(numDownloadParts) @@ -340,6 +340,7 @@ func downloadBlob(ctx context.Context, opts downloadOpts) error { return err } + // nolint: contextcheck go download.Run(context.Background(), requestURL, opts.regOpts) } diff --git a/server/images.go b/server/images.go index 346ff592..4742a363 100644 --- a/server/images.go +++ b/server/images.go @@ -747,6 +747,7 @@ func deleteUnusedLayers(skipModelPath *ModelPath, deleteMap map[string]struct{}, // save (i.e. delete from the deleteMap) any files used in other manifests manifest, _, err := GetManifest(fmp) if err != nil { + // nolint: nilerr return nil } diff --git a/server/routes.go b/server/routes.go index fe4102d2..794c4206 100644 --- a/server/routes.go +++ b/server/routes.go @@ -748,6 +748,7 @@ func ListModelsHandler(c *gin.Context) { resp, err := modelResponse(tag) if err != nil { log.Printf("skipping file: %s", fp) + // nolint: nilerr return nil } diff --git a/server/routes_test.go b/server/routes_test.go index e09648f9..aa561d98 100644 --- a/server/routes_test.go +++ b/server/routes_test.go @@ -193,8 +193,8 @@ func Test_Routes(t *testing.T) { } resp, err := httpSrv.Client().Do(req) - defer resp.Body.Close() assert.Nil(t, err) + defer resp.Body.Close() if tc.Expected != nil { tc.Expected(t, resp) diff --git a/server/upload.go b/server/upload.go index 7e8f55ea..f0852f85 100644 --- a/server/upload.go +++ b/server/upload.go @@ -395,6 +395,7 @@ func uploadBlob(ctx context.Context, mp ModelPath, layer *Layer, opts *RegistryO return err } + // nolint: contextcheck go upload.Run(context.Background(), opts) } From 2b9892a808a56f9d2634f85907fdc5e0498000c6 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Fri, 15 Dec 2023 15:50:51 -0800 Subject: [PATCH 4/7] fix(windows): modelpath and list --- server/modelpath.go | 2 +- server/routes.go | 15 ++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/server/modelpath.go b/server/modelpath.go index 651ee1be..f09ff8e9 100644 --- a/server/modelpath.go +++ b/server/modelpath.go @@ -46,7 +46,7 @@ func ParseModelPath(name string) ModelPath { name = after } - parts := strings.Split(name, string(os.PathSeparator)) + parts := strings.Split(name, "/") switch len(parts) { case 3: mp.Registry = parts[0] diff --git a/server/routes.go b/server/routes.go index 794c4206..72c0d051 100644 --- a/server/routes.go +++ b/server/routes.go @@ -711,7 +711,7 @@ func GetModelInfo(req api.ShowRequest) (*api.ShowResponse, error) { func ListModelsHandler(c *gin.Context) { models := make([]api.ModelResponse, 0) - fp, err := GetManifestPath() + manifestsPath, err := GetManifestPath() if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return @@ -741,13 +741,14 @@ func ListModelsHandler(c *gin.Context) { walkFunc := func(path string, info os.FileInfo, _ error) error { if !info.IsDir() { - dir, file := filepath.Split(path) - dir = strings.Trim(strings.TrimPrefix(dir, fp), string(os.PathSeparator)) - tag := strings.Join([]string{dir, file}, ":") + path, tag := filepath.Split(path) + model := strings.Trim(strings.TrimPrefix(path, manifestsPath), string(os.PathSeparator)) + modelPath := strings.Join([]string{model, tag}, ":") + canonicalModelPath := strings.ReplaceAll(modelPath, string(os.PathSeparator), "/") - resp, err := modelResponse(tag) + resp, err := modelResponse(canonicalModelPath) if err != nil { - log.Printf("skipping file: %s", fp) + log.Printf("skipping file: %s", canonicalModelPath) // nolint: nilerr return nil } @@ -759,7 +760,7 @@ func ListModelsHandler(c *gin.Context) { return nil } - if err := filepath.Walk(fp, walkFunc); err != nil { + if err := filepath.Walk(manifestsPath, walkFunc); err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } From f95d2f25f35c5140acbec44aaf8078c3c0526a7b Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Mon, 18 Dec 2023 10:53:51 -0800 Subject: [PATCH 5/7] fix temporary history file permissions --- readline/history.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readline/history.go b/readline/history.go index db99fad0..d0138cd9 100644 --- a/readline/history.go +++ b/readline/history.go @@ -132,7 +132,7 @@ func (h *History) Save() error { tmpFile := h.Filename + ".tmp" - f, err := os.OpenFile(tmpFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC|os.O_APPEND, 0o666) + f, err := os.OpenFile(tmpFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC|os.O_APPEND, 0o600) if err != nil { return err } From 4a33cede207fef9f23ea6fa1580fd749b3b60940 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Fri, 22 Dec 2023 09:55:18 -0800 Subject: [PATCH 6/7] remove unused fields and functions --- llm/llama.go | 49 +------------------------------------------------ 1 file changed, 1 insertion(+), 48 deletions(-) diff --git a/llm/llama.go b/llm/llama.go index 0856dca5..80b4f75b 100644 --- a/llm/llama.go +++ b/llm/llama.go @@ -1,17 +1,11 @@ package llm import ( - "bytes" - "context" _ "embed" - "errors" "fmt" - "os" - "os/exec" "time" "github.com/jmorganca/ollama/api" - "github.com/jmorganca/ollama/format" ) const jsonGrammar = ` @@ -42,51 +36,12 @@ number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? ws ws ::= ([ \t\n] ws)? ` -type Running struct { - Port int - Cmd *exec.Cmd - Cancel context.CancelFunc - *StatusWriter // captures error messages from the llama runner process -} - type ImageData struct { Data []byte `json:"data"` ID int `json:"id"` } -var ( - errNvidiaSMI = errors.New("warning: gpu support may not be enabled, check that you have installed GPU drivers: nvidia-smi command failed") - errAvailableVRAM = errors.New("not enough VRAM available, falling back to CPU only") - payloadMissing = fmt.Errorf("expected dynamic library payloads not included in this build of ollama") -) - -// StatusWriter is a writer that captures error messages from the llama runner process -type StatusWriter struct { - ErrCh chan error - LastErrMsg string -} - -func NewStatusWriter() *StatusWriter { - return &StatusWriter{ - ErrCh: make(chan error, 1), - } -} - -func (w *StatusWriter) Write(b []byte) (int, error) { - var errMsg string - if _, after, ok := bytes.Cut(b, []byte("error:")); ok { - errMsg = string(bytes.TrimSpace(after)) - } else if _, after, ok := bytes.Cut(b, []byte("CUDA error")); ok { - errMsg = string(bytes.TrimSpace(after)) - } - - if errMsg != "" { - w.LastErrMsg = errMsg - w.ErrCh <- fmt.Errorf("llama runner: %s", errMsg) - } - - return os.Stderr.Write(b) -} +var payloadMissing = fmt.Errorf("expected dynamic library payloads not included in this build of ollama") type prediction struct { Content string `json:"content"` @@ -102,9 +57,7 @@ type prediction struct { } } -const maxBufferSize = 512 * format.KiloByte const maxRetries = 3 -const retryDelay = 1 * time.Second type PredictOpts struct { Prompt string From f921e2696ed21e1f169ad26f69c21d0f2629840f Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Tue, 9 Jan 2024 09:45:42 -0800 Subject: [PATCH 7/7] typo --- llm/llm.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llm/llm.go b/llm/llm.go index 023077aa..0ef834b4 100644 --- a/llm/llm.go +++ b/llm/llm.go @@ -61,7 +61,7 @@ func New(workDir, model string, adapters, projectors []string, opts api.Options) requiredKv := 2 * 2 * int64(opts.NumCtx) * int64(ggml.NumLayers()) * int64(ggml.NumEmbed()) * int64(ggml.NumHeadKv()) / int64(ggml.NumHead()) // this amount is the overhead + tensors in memory - // TODO: get this from the llama.cpp's graph calcluations instead of + // TODO: get this from the llama.cpp's graph calculations instead of // estimating it's 1/6 * kv_cache_size * num_gqa requiredAlloc := int64(ggml.NumGQA()) * requiredKv / 6