ollama/cmd/cmd.go

1089 lines
26 KiB
Go
Raw Normal View History

package cmd
import (
2023-11-14 22:07:40 +00:00
"bytes"
"context"
"crypto/ed25519"
"crypto/rand"
2023-11-14 22:07:40 +00:00
"crypto/sha256"
"encoding/pem"
2023-07-06 22:43:04 +00:00
"errors"
2023-07-06 16:24:49 +00:00
"fmt"
2023-07-18 21:01:19 +00:00
"io"
"log"
"net"
"net/http"
"os"
"os/exec"
"os/signal"
"path/filepath"
"runtime"
2023-07-07 17:12:58 +00:00
"strings"
"syscall"
2023-07-06 22:43:04 +00:00
"time"
2023-07-18 16:09:45 +00:00
"github.com/olekukonko/tablewriter"
2023-07-06 20:49:31 +00:00
"github.com/spf13/cobra"
"golang.org/x/crypto/ssh"
"golang.org/x/term"
2023-07-06 20:49:31 +00:00
2023-07-03 20:32:48 +00:00
"github.com/jmorganca/ollama/api"
2023-07-18 16:09:45 +00:00
"github.com/jmorganca/ollama/format"
2023-11-14 22:07:40 +00:00
"github.com/jmorganca/ollama/parser"
2023-11-15 00:33:24 +00:00
"github.com/jmorganca/ollama/progress"
2023-10-25 23:41:18 +00:00
"github.com/jmorganca/ollama/readline"
2023-07-03 20:32:48 +00:00
"github.com/jmorganca/ollama/server"
2023-08-22 01:24:42 +00:00
"github.com/jmorganca/ollama/version"
)
2023-07-20 23:09:23 +00:00
func CreateHandler(cmd *cobra.Command, args []string) error {
filename, _ := cmd.Flags().GetString("file")
filename, err := filepath.Abs(filename)
if err != nil {
return err
}
2023-10-09 19:18:26 +00:00
client, err := api.ClientFromEnvironment()
if err != nil {
return err
}
2023-11-15 00:33:24 +00:00
p := progress.NewProgress(os.Stderr)
defer p.Stop()
bars := make(map[string]*progress.Bar)
2023-11-14 22:07:40 +00:00
modelfile, err := os.ReadFile(filename)
if err != nil {
return err
}
commands, err := parser.Parse(bytes.NewReader(modelfile))
if err != nil {
return err
}
home, err := os.UserHomeDir()
if err != nil {
return err
}
status := "transferring model data"
spinner := progress.NewSpinner(status)
2023-11-16 00:59:49 +00:00
p.Add(status, spinner)
2023-11-14 22:07:40 +00:00
for _, c := range commands {
switch c.Name {
case "model", "adapter":
path := c.Args
if path == "~" {
path = home
} else if strings.HasPrefix(path, "~/") {
path = filepath.Join(home, path[2:])
}
if !filepath.IsAbs(path) {
path = filepath.Join(filepath.Dir(filename), path)
}
2023-11-14 22:07:40 +00:00
bin, err := os.Open(path)
if errors.Is(err, os.ErrNotExist) && c.Name == "model" {
2023-11-15 18:57:09 +00:00
continue
2023-11-14 22:07:40 +00:00
} else if err != nil {
return err
}
defer bin.Close()
hash := sha256.New()
if _, err := io.Copy(hash, bin); err != nil {
return err
}
bin.Seek(0, io.SeekStart)
digest := fmt.Sprintf("sha256:%x", hash.Sum(nil))
2023-11-15 18:59:38 +00:00
if err = client.CreateBlob(cmd.Context(), digest, bin); err != nil {
2023-11-14 22:07:40 +00:00
return err
}
2023-11-15 18:59:38 +00:00
modelfile = bytes.ReplaceAll(modelfile, []byte(c.Args), []byte("@"+digest))
2023-11-14 22:07:40 +00:00
}
}
2023-07-17 21:14:41 +00:00
fn := func(resp api.ProgressResponse) error {
2023-11-15 00:33:24 +00:00
if resp.Digest != "" {
spinner.Stop()
bar, ok := bars[resp.Digest]
if !ok {
bar = progress.NewBar(fmt.Sprintf("pulling %s...", resp.Digest[7:19]), resp.Total, resp.Completed)
2023-11-15 00:33:24 +00:00
bars[resp.Digest] = bar
p.Add(resp.Digest, bar)
}
bar.Set(resp.Completed)
} else if status != resp.Status {
spinner.Stop()
status = resp.Status
spinner = progress.NewSpinner(status)
p.Add(status, spinner)
}
return nil
}
request := api.CreateRequest{Name: args[0], Modelfile: string(modelfile)}
if err := client.Create(context.Background(), &request, fn); err != nil {
return err
}
return nil
}
func RunHandler(cmd *cobra.Command, args []string) error {
2023-10-09 19:18:26 +00:00
client, err := api.ClientFromEnvironment()
2023-08-22 04:56:56 +00:00
if err != nil {
return err
}
name := args[0]
// check if the model exists on the server
_, err = client.Show(context.Background(), &api.ShowRequest{Name: name})
2023-11-01 03:49:48 +00:00
var statusError api.StatusError
switch {
case errors.As(err, &statusError) && statusError.StatusCode == http.StatusNotFound:
if err := PullHandler(cmd, args); err != nil {
return err
2023-07-06 22:43:04 +00:00
}
2023-11-01 03:49:48 +00:00
case err != nil:
return err
}
2023-07-06 21:05:55 +00:00
return RunGenerate(cmd, args)
2023-07-06 18:18:40 +00:00
}
2023-07-20 23:09:23 +00:00
func PushHandler(cmd *cobra.Command, args []string) error {
2023-10-09 19:18:26 +00:00
client, err := api.ClientFromEnvironment()
if err != nil {
return err
}
insecure, err := cmd.Flags().GetBool("insecure")
if err != nil {
return err
}
2023-11-15 00:33:24 +00:00
p := progress.NewProgress(os.Stderr)
defer p.Stop()
bars := make(map[string]*progress.Bar)
var status string
var spinner *progress.Spinner
2023-11-15 00:33:24 +00:00
2023-07-19 01:51:30 +00:00
fn := func(resp api.ProgressResponse) error {
2023-11-15 00:33:24 +00:00
if resp.Digest != "" {
if spinner != nil {
spinner.Stop()
}
2023-11-15 00:33:24 +00:00
bar, ok := bars[resp.Digest]
if !ok {
bar = progress.NewBar(fmt.Sprintf("pushing %s...", resp.Digest[7:19]), resp.Total, resp.Completed)
2023-11-15 00:33:24 +00:00
bars[resp.Digest] = bar
p.Add(resp.Digest, bar)
}
bar.Set(resp.Completed)
} else if status != resp.Status {
if spinner != nil {
spinner.Stop()
}
2023-11-15 00:33:24 +00:00
status = resp.Status
spinner = progress.NewSpinner(status)
p.Add(status, spinner)
}
return nil
}
2023-11-15 00:33:24 +00:00
request := api.PushRequest{Name: args[0], Insecure: insecure}
if err := client.Push(context.Background(), &request, fn); err != nil {
return err
}
2023-11-16 00:59:49 +00:00
spinner.Stop()
2023-11-15 00:33:24 +00:00
return nil
}
2023-07-20 23:09:23 +00:00
func ListHandler(cmd *cobra.Command, args []string) error {
2023-10-09 19:18:26 +00:00
client, err := api.ClientFromEnvironment()
if err != nil {
return err
}
2023-07-18 16:09:45 +00:00
models, err := client.List(context.Background())
if err != nil {
return err
}
var data [][]string
for _, m := range models.Models {
2023-07-18 21:01:19 +00:00
if len(args) == 0 || strings.HasPrefix(m.Name, args[0]) {
data = append(data, []string{m.Name, m.Digest[:12], format.HumanBytes(m.Size), format.HumanTime(m.ModifiedAt, "Never")})
2023-07-18 21:01:19 +00:00
}
2023-07-18 16:09:45 +00:00
}
table := tablewriter.NewWriter(os.Stdout)
2023-08-29 03:50:24 +00:00
table.SetHeader([]string{"NAME", "ID", "SIZE", "MODIFIED"})
2023-07-18 16:09:45 +00:00
table.SetHeaderAlignment(tablewriter.ALIGN_LEFT)
table.SetAlignment(tablewriter.ALIGN_LEFT)
table.SetHeaderLine(false)
table.SetBorder(false)
table.SetNoWhiteSpace(true)
table.SetTablePadding("\t")
table.AppendBulk(data)
table.Render()
return nil
}
2023-07-20 23:09:23 +00:00
func DeleteHandler(cmd *cobra.Command, args []string) error {
2023-10-09 19:18:26 +00:00
client, err := api.ClientFromEnvironment()
if err != nil {
return err
}
2023-07-20 23:09:23 +00:00
for _, name := range args {
req := api.DeleteRequest{Name: name}
if err := client.Delete(context.Background(), &req); err != nil {
return err
}
fmt.Printf("deleted '%s'\n", name)
2023-07-20 23:09:23 +00:00
}
return nil
}
2023-09-06 18:04:17 +00:00
func ShowHandler(cmd *cobra.Command, args []string) error {
2023-10-09 19:18:26 +00:00
client, err := api.ClientFromEnvironment()
2023-09-06 18:04:17 +00:00
if err != nil {
return err
}
if len(args) != 1 {
return errors.New("missing model name")
}
license, errLicense := cmd.Flags().GetBool("license")
modelfile, errModelfile := cmd.Flags().GetBool("modelfile")
parameters, errParams := cmd.Flags().GetBool("parameters")
system, errSystem := cmd.Flags().GetBool("system")
template, errTemplate := cmd.Flags().GetBool("template")
for _, boolErr := range []error{errLicense, errModelfile, errParams, errSystem, errTemplate} {
if boolErr != nil {
return errors.New("error retrieving flags")
}
}
flagsSet := 0
showType := ""
if license {
flagsSet++
showType = "license"
}
if modelfile {
flagsSet++
showType = "modelfile"
}
if parameters {
flagsSet++
showType = "parameters"
}
if system {
flagsSet++
showType = "system"
}
if template {
flagsSet++
showType = "template"
}
if flagsSet > 1 {
return errors.New("only one of '--license', '--modelfile', '--parameters', '--system', or '--template' can be specified")
2023-09-06 18:04:17 +00:00
} else if flagsSet == 0 {
return errors.New("one of '--license', '--modelfile', '--parameters', '--system', or '--template' must be specified")
2023-09-06 18:04:17 +00:00
}
req := api.ShowRequest{Name: args[0]}
resp, err := client.Show(context.Background(), &req)
if err != nil {
return err
}
switch showType {
case "license":
fmt.Println(resp.License)
case "modelfile":
fmt.Println(resp.Modelfile)
case "parameters":
fmt.Println(resp.Parameters)
case "system":
fmt.Println(resp.System)
case "template":
fmt.Println(resp.Template)
}
return nil
}
2023-07-24 15:27:28 +00:00
func CopyHandler(cmd *cobra.Command, args []string) error {
2023-10-09 19:18:26 +00:00
client, err := api.ClientFromEnvironment()
if err != nil {
return err
}
2023-07-24 15:27:28 +00:00
req := api.CopyRequest{Source: args[0], Destination: args[1]}
if err := client.Copy(context.Background(), &req); err != nil {
return err
}
fmt.Printf("copied '%s' to '%s'\n", args[0], args[1])
return nil
}
2023-07-20 23:09:23 +00:00
func PullHandler(cmd *cobra.Command, args []string) error {
insecure, err := cmd.Flags().GetBool("insecure")
if err != nil {
return err
}
2023-10-09 19:18:26 +00:00
client, err := api.ClientFromEnvironment()
if err != nil {
return err
}
2023-11-15 00:33:24 +00:00
p := progress.NewProgress(os.Stderr)
defer p.Stop()
bars := make(map[string]*progress.Bar)
var status string
var spinner *progress.Spinner
2023-11-15 00:33:24 +00:00
2023-07-19 01:51:30 +00:00
fn := func(resp api.ProgressResponse) error {
2023-11-15 00:33:24 +00:00
if resp.Digest != "" {
if spinner != nil {
spinner.Stop()
}
2023-11-15 00:33:24 +00:00
bar, ok := bars[resp.Digest]
if !ok {
bar = progress.NewBar(fmt.Sprintf("pulling %s...", resp.Digest[7:19]), resp.Total, resp.Completed)
2023-11-15 00:33:24 +00:00
bars[resp.Digest] = bar
p.Add(resp.Digest, bar)
}
bar.Set(resp.Completed)
} else if status != resp.Status {
if spinner != nil {
spinner.Stop()
}
2023-11-15 00:33:24 +00:00
status = resp.Status
spinner = progress.NewSpinner(status)
p.Add(status, spinner)
}
return nil
}
2023-07-07 14:22:37 +00:00
2023-11-15 00:33:24 +00:00
request := api.PullRequest{Name: args[0], Insecure: insecure}
if err := client.Pull(context.Background(), &request, fn); err != nil {
return err
}
return nil
2023-07-06 20:49:31 +00:00
}
2023-07-13 01:18:06 +00:00
func RunGenerate(cmd *cobra.Command, args []string) error {
interactive := true
opts := generateOptions{
Model: args[0],
WordWrap: os.Getenv("TERM") == "xterm-256color",
Options: map[string]interface{}{},
}
format, err := cmd.Flags().GetString("format")
if err != nil {
return err
}
opts.Format = format
prompts := args[1:]
// prepend stdin to the prompt if provided
if !term.IsTerminal(int(os.Stdin.Fd())) {
in, err := io.ReadAll(os.Stdin)
2023-11-14 02:54:02 +00:00
if err != nil {
return err
}
prompts = append([]string{string(in)}, prompts...)
opts.WordWrap = false
interactive = false
2023-07-06 20:49:31 +00:00
}
opts.Prompt = strings.Join(prompts, " ")
if len(prompts) > 0 {
interactive = false
2023-07-06 20:49:31 +00:00
}
nowrap, err := cmd.Flags().GetBool("nowordwrap")
if err != nil {
return err
}
opts.WordWrap = !nowrap
if !interactive {
return generate(cmd, opts)
}
return generateInteractive(cmd, opts)
2023-07-06 20:49:31 +00:00
}
2023-07-18 18:59:42 +00:00
type generateContextKey string
type generateOptions struct {
Model string
Prompt string
WordWrap bool
Format string
Options map[string]interface{}
}
func generate(cmd *cobra.Command, opts generateOptions) error {
2023-10-09 19:18:26 +00:00
client, err := api.ClientFromEnvironment()
2023-09-18 19:26:56 +00:00
if err != nil {
return err
}
2023-07-07 17:12:58 +00:00
2023-11-15 00:58:51 +00:00
p := progress.NewProgress(os.Stderr)
defer p.StopAndClear()
2023-11-15 00:58:51 +00:00
spinner := progress.NewSpinner("")
p.Add("", spinner)
2023-09-18 19:26:56 +00:00
var latest api.GenerateResponse
2023-07-13 01:18:06 +00:00
2023-09-18 19:26:56 +00:00
generateContext, ok := cmd.Context().Value(generateContextKey("context")).([]int)
if !ok {
generateContext = []int{}
}
termWidth, _, err := term.GetSize(int(os.Stdout.Fd()))
if err != nil {
opts.WordWrap = false
}
cancelCtx, cancel := context.WithCancel(context.Background())
defer cancel()
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT)
var abort bool
go func() {
<-sigChan
cancel()
abort = true
}()
var currentLineLength int
var wordBuffer string
request := api.GenerateRequest{
Model: opts.Model,
Prompt: opts.Prompt,
Context: generateContext,
Format: opts.Format,
Options: opts.Options,
}
2023-09-18 19:26:56 +00:00
fn := func(response api.GenerateResponse) error {
2023-11-15 00:58:51 +00:00
p.StopAndClear()
2023-09-18 19:26:56 +00:00
latest = response
2023-07-13 01:18:06 +00:00
if opts.WordWrap {
for _, ch := range response.Response {
if currentLineLength+1 > termWidth-5 {
// backtrack the length of the last word and clear to the end of the line
fmt.Printf("\x1b[%dD\x1b[K\n", len(wordBuffer))
fmt.Printf("%s%c", wordBuffer, ch)
currentLineLength = len(wordBuffer) + 1
} else {
fmt.Print(string(ch))
currentLineLength += 1
switch ch {
case ' ':
wordBuffer = ""
case '\n':
currentLineLength = 0
default:
wordBuffer += string(ch)
}
}
}
} else {
fmt.Print(response.Response)
}
2023-09-18 19:26:56 +00:00
return nil
}
2023-07-07 21:04:43 +00:00
if err := client.Generate(cancelCtx, &request, fn); err != nil {
if strings.Contains(err.Error(), "context canceled") && abort {
return nil
2023-07-07 21:04:43 +00:00
}
2023-09-18 19:26:56 +00:00
return err
}
if opts.Prompt != "" {
2023-07-07 17:12:58 +00:00
fmt.Println()
fmt.Println()
2023-09-18 19:26:56 +00:00
}
2023-07-13 01:18:06 +00:00
2023-09-18 19:26:56 +00:00
if !latest.Done {
if abort {
return nil
}
2023-09-18 19:26:56 +00:00
return errors.New("unexpected end of response")
}
2023-07-13 01:18:06 +00:00
2023-09-18 19:26:56 +00:00
verbose, err := cmd.Flags().GetBool("verbose")
if err != nil {
return err
}
2023-07-18 18:59:42 +00:00
2023-09-18 19:26:56 +00:00
if verbose {
latest.Summary()
2023-07-07 17:12:58 +00:00
}
2023-07-06 20:49:31 +00:00
2023-09-18 19:26:56 +00:00
ctx := cmd.Context()
ctx = context.WithValue(ctx, generateContextKey("context"), latest.Context)
cmd.SetContext(ctx)
2023-07-06 20:49:31 +00:00
return nil
}
func generateInteractive(cmd *cobra.Command, opts generateOptions) error {
2023-09-18 19:26:56 +00:00
// load the model
loadOpts := generateOptions{
Model: opts.Model,
Prompt: "",
}
if err := generate(cmd, loadOpts); err != nil {
2023-09-18 19:26:56 +00:00
return err
}
2023-07-18 21:01:19 +00:00
usage := func() {
2023-10-05 18:36:07 +00:00
fmt.Fprintln(os.Stderr, "Available Commands:")
fmt.Fprintln(os.Stderr, " /set Set session variables")
fmt.Fprintln(os.Stderr, " /show Show model information")
fmt.Fprintln(os.Stderr, " /bye Exit")
fmt.Fprintln(os.Stderr, " /?, /help Help for a command")
fmt.Fprintln(os.Stderr, "")
fmt.Fprintln(os.Stderr, "Use \"\"\" to begin a multi-line message.")
fmt.Fprintln(os.Stderr, "")
}
usageSet := func() {
fmt.Fprintln(os.Stderr, "Available Commands:")
fmt.Fprintln(os.Stderr, " /set parameter Set a parameter")
2023-10-05 18:36:07 +00:00
fmt.Fprintln(os.Stderr, " /set history Enable history")
fmt.Fprintln(os.Stderr, " /set nohistory Disable history")
fmt.Fprintln(os.Stderr, " /set wordwrap Enable wordwrap")
fmt.Fprintln(os.Stderr, " /set nowordwrap Disable wordwrap")
2023-11-14 02:54:02 +00:00
fmt.Fprintln(os.Stderr, " /set format json Enable JSON mode")
fmt.Fprintln(os.Stderr, " /set noformat Disable formatting")
2023-10-05 18:36:07 +00:00
fmt.Fprintln(os.Stderr, " /set verbose Show LLM stats")
fmt.Fprintln(os.Stderr, " /set quiet Disable LLM stats")
fmt.Fprintln(os.Stderr, "")
}
usageShow := func() {
fmt.Fprintln(os.Stderr, "Available Commands:")
fmt.Fprintln(os.Stderr, " /show license Show model license")
fmt.Fprintln(os.Stderr, " /show modelfile Show Modelfile for this model")
fmt.Fprintln(os.Stderr, " /show parameters Show parameters for this model")
fmt.Fprintln(os.Stderr, " /show system Show system prompt")
fmt.Fprintln(os.Stderr, " /show template Show prompt template")
fmt.Fprintln(os.Stderr, "")
2023-07-18 21:01:19 +00:00
}
// only list out the most common parameters
usageParameters := func() {
fmt.Fprintln(os.Stderr, "Available Parameters:")
fmt.Fprintln(os.Stderr, " /set parameter seed <int> Random number seed")
fmt.Fprintln(os.Stderr, " /set parameter num_predict <int> Max number of tokens to predict")
fmt.Fprintln(os.Stderr, " /set parameter top_k <int> Pick from top k num of tokens")
fmt.Fprintln(os.Stderr, " /set parameter top_p <float> Pick token based on sum of probabilities")
fmt.Fprintln(os.Stderr, " /set parameter num_ctx <int> Set the context size")
fmt.Fprintln(os.Stderr, " /set parameter temperature <float> Set creativity level")
fmt.Fprintln(os.Stderr, " /set parameter repeat_penalty <float> How strongly to penalize repetitions")
fmt.Fprintln(os.Stderr, " /set parameter repeat_last_n <int> Set how far back to look for repetitions")
fmt.Fprintln(os.Stderr, " /set parameter num_gpu <int> The number of layers to send to the GPU")
fmt.Fprintln(os.Stderr, " /set parameter stop \"<string>\", ... Set the stop parameters")
fmt.Fprintln(os.Stderr, "")
}
scanner, err := readline.New(readline.Prompt{
2023-10-25 23:41:18 +00:00
Prompt: ">>> ",
AltPrompt: "... ",
Placeholder: "Send a message (/? for help)",
AltPlaceholder: `Use """ to end multi-line input`,
})
2023-07-18 21:01:19 +00:00
if err != nil {
return err
}
2023-10-26 22:57:00 +00:00
fmt.Print(readline.StartBracketedPaste)
defer fmt.Printf(readline.EndBracketedPaste)
var prompt string
2023-07-18 21:01:19 +00:00
for {
line, err := scanner.Readline()
switch {
case errors.Is(err, io.EOF):
fmt.Println()
2023-07-18 21:01:19 +00:00
return nil
case errors.Is(err, readline.ErrInterrupt):
2023-07-20 07:53:08 +00:00
if line == "" {
2023-10-25 23:41:18 +00:00
fmt.Println("\nUse Ctrl-D or /bye to exit.")
2023-07-20 07:53:08 +00:00
}
scanner.Prompt.UseAlt = false
prompt = ""
2023-07-18 21:01:19 +00:00
continue
case err != nil:
2023-07-06 20:49:31 +00:00
return err
}
2023-07-18 21:01:19 +00:00
switch {
case strings.HasPrefix(prompt, `"""`):
// if the prompt so far starts with """ then we're in multiline mode
// and we need to keep reading until we find a line that ends with """
cut, found := strings.CutSuffix(line, `"""`)
prompt += cut + "\n"
if !found {
continue
}
prompt = strings.TrimPrefix(prompt, `"""`)
scanner.Prompt.UseAlt = false
case strings.HasPrefix(line, `"""`) && len(prompt) == 0:
2023-10-25 23:41:18 +00:00
scanner.Prompt.UseAlt = true
prompt += line + "\n"
continue
case scanner.Pasting:
prompt += line + "\n"
continue
2023-07-18 21:01:19 +00:00
case strings.HasPrefix(line, "/list"):
args := strings.Fields(line)
2023-07-20 23:09:23 +00:00
if err := ListHandler(cmd, args[1:]); err != nil {
2023-07-18 21:01:19 +00:00
return err
}
case strings.HasPrefix(line, "/set"):
args := strings.Fields(line)
if len(args) > 1 {
switch args[1] {
case "history":
scanner.HistoryEnable()
case "nohistory":
scanner.HistoryDisable()
case "wordwrap":
opts.WordWrap = true
fmt.Println("Set 'wordwrap' mode.")
case "nowordwrap":
opts.WordWrap = false
fmt.Println("Set 'nowordwrap' mode.")
2023-07-19 16:25:23 +00:00
case "verbose":
cmd.Flags().Set("verbose", "true")
fmt.Println("Set 'verbose' mode.")
2023-07-19 16:25:23 +00:00
case "quiet":
cmd.Flags().Set("verbose", "false")
fmt.Println("Set 'quiet' mode.")
2023-11-14 02:54:02 +00:00
case "format":
if len(args) < 3 || args[2] != "json" {
fmt.Println("Invalid or missing format. For 'json' mode use '/set format json'")
} else {
opts.Format = args[2]
2023-11-14 02:54:02 +00:00
fmt.Printf("Set format to '%s' mode.\n", args[2])
}
case "noformat":
opts.Format = ""
2023-11-14 02:54:02 +00:00
fmt.Println("Disabled format.")
case "parameter":
if len(args) < 4 {
usageParameters()
continue
}
var params []string
for _, p := range args[3:] {
params = append(params, p)
}
fp, err := api.FormatParams(map[string][]string{args[2]: params})
if err != nil {
fmt.Printf("Couldn't set parameter: %q\n\n", err)
continue
}
fmt.Printf("Set parameter '%s' to '%s'\n\n", args[2], strings.Join(params, ", "))
opts.Options[args[2]] = fp[args[2]]
default:
fmt.Printf("Unknown command '/set %s'. Type /? for help\n", args[1])
}
} else {
2023-10-05 18:36:07 +00:00
usageSet()
}
case strings.HasPrefix(line, "/show"):
args := strings.Fields(line)
if len(args) > 1 {
client, err := api.ClientFromEnvironment()
if err != nil {
fmt.Println("error: couldn't connect to ollama server")
return err
}
resp, err := client.Show(cmd.Context(), &api.ShowRequest{Name: opts.Model})
2023-08-22 04:56:56 +00:00
if err != nil {
2023-09-06 18:04:17 +00:00
fmt.Println("error: couldn't get model")
return err
}
2023-09-06 18:04:17 +00:00
switch args[1] {
case "license":
if resp.License == "" {
fmt.Print("No license was specified for this model.\n\n")
} else {
fmt.Println(resp.License)
}
2023-09-06 18:04:17 +00:00
case "modelfile":
fmt.Println(resp.Modelfile)
case "parameters":
if resp.Parameters == "" {
fmt.Print("No parameters were specified for this model.\n\n")
} else {
if len(opts.Options) > 0 {
fmt.Println("User defined parameters:")
for k, v := range opts.Options {
fmt.Printf("%-*s %v\n", 30, k, v)
}
fmt.Println()
}
fmt.Println("Model defined parameters:")
fmt.Println(resp.Parameters)
}
case "system":
if resp.System == "" {
fmt.Print("No system prompt was specified for this model.\n\n")
} else {
fmt.Println(resp.System)
}
case "template":
if resp.Template == "" {
fmt.Print("No prompt template was specified for this model.\n\n")
} else {
fmt.Println(resp.Template)
}
default:
fmt.Printf("Unknown command '/show %s'. Type /? for help\n", args[1])
2023-07-18 21:01:19 +00:00
}
2023-10-05 18:36:07 +00:00
} else {
usageShow()
}
case strings.HasPrefix(line, "/help"), strings.HasPrefix(line, "/?"):
args := strings.Fields(line)
if len(args) > 1 {
switch args[1] {
case "set", "/set":
usageSet()
case "show", "/show":
usageShow()
}
} else {
usage()
2023-07-18 21:01:19 +00:00
}
case line == "/exit", line == "/bye":
return nil
case strings.HasPrefix(line, "/"):
args := strings.Fields(line)
fmt.Printf("Unknown command '%s'. Type /? for help\n", args[0])
continue
default:
prompt += line
2023-07-18 21:01:19 +00:00
}
if len(prompt) > 0 && prompt[0] != '/' {
opts.Prompt = prompt
if err := generate(cmd, opts); err != nil {
2023-09-18 19:26:56 +00:00
return err
}
prompt = ""
2023-07-18 21:01:19 +00:00
}
}
2023-07-06 16:24:49 +00:00
}
func RunServer(cmd *cobra.Command, _ []string) error {
2023-09-21 00:49:48 +00:00
host, port, err := net.SplitHostPort(os.Getenv("OLLAMA_HOST"))
if err != nil {
host, port = "127.0.0.1", "11434"
2023-09-22 17:40:54 +00:00
if ip := net.ParseIP(strings.Trim(os.Getenv("OLLAMA_HOST"), "[]")); ip != nil {
2023-09-21 00:49:48 +00:00
host = ip.String()
}
2023-07-04 04:47:00 +00:00
}
2023-09-21 00:49:48 +00:00
if err := initializeKeypair(); err != nil {
return err
}
2023-09-21 00:49:48 +00:00
ln, err := net.Listen("tcp", net.JoinHostPort(host, port))
if err != nil {
return err
}
2023-07-04 04:47:00 +00:00
var origins []string
if o := os.Getenv("OLLAMA_ORIGINS"); o != "" {
origins = strings.Split(o, ",")
}
2023-08-10 16:27:03 +00:00
return server.Serve(ln, origins)
2023-07-04 04:47:00 +00:00
}
func initializeKeypair() error {
home, err := os.UserHomeDir()
if err != nil {
return err
}
privKeyPath := filepath.Join(home, ".ollama", "id_ed25519")
pubKeyPath := filepath.Join(home, ".ollama", "id_ed25519.pub")
_, err = os.Stat(privKeyPath)
if os.IsNotExist(err) {
fmt.Printf("Couldn't find '%s'. Generating new private key.\n", privKeyPath)
_, privKey, err := ed25519.GenerateKey(rand.Reader)
if err != nil {
return err
}
privKeyBytes, err := format.OpenSSHPrivateKey(privKey, "")
if err != nil {
return err
}
2023-09-19 16:36:30 +00:00
err = os.MkdirAll(filepath.Dir(privKeyPath), 0o755)
if err != nil {
return fmt.Errorf("could not create directory %w", err)
}
2023-08-22 04:56:56 +00:00
err = os.WriteFile(privKeyPath, pem.EncodeToMemory(privKeyBytes), 0o600)
if err != nil {
return err
}
sshPrivateKey, err := ssh.NewSignerFromKey(privKey)
if err != nil {
return err
}
pubKeyData := ssh.MarshalAuthorizedKey(sshPrivateKey.PublicKey())
2023-08-22 04:56:56 +00:00
err = os.WriteFile(pubKeyPath, pubKeyData, 0o644)
if err != nil {
return err
}
fmt.Printf("Your new public key is: \n\n%s\n", string(pubKeyData))
}
return nil
}
2023-07-31 21:38:10 +00:00
func startMacApp(client *api.Client) error {
2023-08-01 20:01:55 +00:00
exe, err := os.Executable()
if err != nil {
return err
}
link, err := os.Readlink(exe)
2023-07-31 21:38:10 +00:00
if err != nil {
return err
}
2023-08-01 17:48:48 +00:00
if !strings.Contains(link, "Ollama.app") {
return fmt.Errorf("could not find ollama app")
}
2023-07-31 21:38:10 +00:00
path := strings.Split(link, "Ollama.app")
if err := exec.Command("/usr/bin/open", "-a", path[0]+"Ollama.app").Run(); err != nil {
return err
}
// wait for the server to start
timeout := time.After(5 * time.Second)
tick := time.Tick(500 * time.Millisecond)
for {
select {
case <-timeout:
return errors.New("timed out waiting for server to start")
case <-tick:
if err := client.Heartbeat(context.Background()); err == nil {
return nil // server has started
}
}
}
}
func checkServerHeartbeat(_ *cobra.Command, _ []string) error {
2023-10-09 19:18:26 +00:00
client, err := api.ClientFromEnvironment()
if err != nil {
return err
}
if err := client.Heartbeat(context.Background()); err != nil {
2023-07-31 21:38:10 +00:00
if !strings.Contains(err.Error(), "connection refused") {
return err
}
if runtime.GOOS == "darwin" {
if err := startMacApp(client); err != nil {
2023-08-01 17:48:48 +00:00
return fmt.Errorf("could not connect to ollama app, is it running?")
}
2023-07-31 21:38:10 +00:00
} else {
return fmt.Errorf("could not connect to ollama server, run 'ollama serve' to start it")
}
}
return nil
}
func NewCLI() *cobra.Command {
log.SetFlags(log.LstdFlags | log.Lshortfile)
rootCmd := &cobra.Command{
2023-08-14 18:15:53 +00:00
Use: "ollama",
Short: "Large language model runner",
SilenceUsage: true,
SilenceErrors: true,
CompletionOptions: cobra.CompletionOptions{
DisableDefaultCmd: true,
},
2023-08-22 01:24:42 +00:00
Version: version.Version,
}
cobra.EnableCommandSorting = false
createCmd := &cobra.Command{
Use: "create MODEL",
Short: "Create a model from a Modelfile",
2023-10-18 18:57:22 +00:00
Args: cobra.ExactArgs(1),
PreRunE: checkServerHeartbeat,
RunE: CreateHandler,
}
createCmd.Flags().StringP("file", "f", "Modelfile", "Name of the Modelfile (default \"Modelfile\")")
2023-09-06 18:04:17 +00:00
showCmd := &cobra.Command{
Use: "show MODEL",
Short: "Show information for a model",
2023-10-18 18:57:22 +00:00
Args: cobra.ExactArgs(1),
2023-09-06 18:04:17 +00:00
PreRunE: checkServerHeartbeat,
RunE: ShowHandler,
}
showCmd.Flags().Bool("license", false, "Show license of a model")
showCmd.Flags().Bool("modelfile", false, "Show Modelfile of a model")
showCmd.Flags().Bool("parameters", false, "Show parameters of a model")
showCmd.Flags().Bool("template", false, "Show template of a model")
showCmd.Flags().Bool("system", false, "Show system prompt of a model")
runCmd := &cobra.Command{
Use: "run MODEL [PROMPT]",
Short: "Run a model",
Args: cobra.MinimumNArgs(1),
PreRunE: checkServerHeartbeat,
RunE: RunHandler,
}
2023-07-13 01:18:06 +00:00
runCmd.Flags().Bool("verbose", false, "Show timings for response")
2023-08-22 04:56:56 +00:00
runCmd.Flags().Bool("insecure", false, "Use an insecure registry")
runCmd.Flags().Bool("nowordwrap", false, "Don't wrap words to the next line automatically")
2023-11-14 02:54:02 +00:00
runCmd.Flags().String("format", "", "Response format (e.g. json)")
2023-07-13 01:18:06 +00:00
serveCmd := &cobra.Command{
Use: "serve",
Aliases: []string{"start"},
Short: "Start ollama",
2023-10-18 18:57:22 +00:00
Args: cobra.ExactArgs(0),
2023-07-06 20:49:31 +00:00
RunE: RunServer,
}
pullCmd := &cobra.Command{
Use: "pull MODEL",
Short: "Pull a model from a registry",
2023-10-18 18:57:22 +00:00
Args: cobra.ExactArgs(1),
PreRunE: checkServerHeartbeat,
RunE: PullHandler,
}
pullCmd.Flags().Bool("insecure", false, "Use an insecure registry")
pushCmd := &cobra.Command{
Use: "push MODEL",
Short: "Push a model to a registry",
2023-10-18 18:57:22 +00:00
Args: cobra.ExactArgs(1),
PreRunE: checkServerHeartbeat,
RunE: PushHandler,
}
pushCmd.Flags().Bool("insecure", false, "Use an insecure registry")
2023-07-18 16:09:45 +00:00
listCmd := &cobra.Command{
Use: "list",
2023-07-20 22:28:27 +00:00
Aliases: []string{"ls"},
Short: "List models",
PreRunE: checkServerHeartbeat,
RunE: ListHandler,
2023-07-20 23:09:23 +00:00
}
2023-07-24 15:27:28 +00:00
copyCmd := &cobra.Command{
2023-10-18 18:57:22 +00:00
Use: "cp SOURCE TARGET",
Short: "Copy a model",
2023-10-18 18:57:22 +00:00
Args: cobra.ExactArgs(2),
PreRunE: checkServerHeartbeat,
RunE: CopyHandler,
2023-07-24 15:27:28 +00:00
}
2023-07-20 23:09:23 +00:00
deleteCmd := &cobra.Command{
2023-10-18 18:57:22 +00:00
Use: "rm MODEL [MODEL...]",
Short: "Remove a model",
Args: cobra.MinimumNArgs(1),
PreRunE: checkServerHeartbeat,
RunE: DeleteHandler,
2023-07-18 16:09:45 +00:00
}
rootCmd.AddCommand(
serveCmd,
createCmd,
2023-09-06 18:04:17 +00:00
showCmd,
runCmd,
pullCmd,
pushCmd,
2023-07-18 16:09:45 +00:00
listCmd,
2023-07-24 15:27:28 +00:00
copyCmd,
2023-07-20 23:09:23 +00:00
deleteCmd,
)
return rootCmd
}