ollama/llm/llm.go

104 lines
3.2 KiB
Go
Raw Normal View History

2023-07-21 20:33:56 +00:00
package llm
import (
"context"
2023-07-21 20:33:56 +00:00
"fmt"
"log"
2023-07-21 20:33:56 +00:00
"os"
"runtime"
2023-07-21 20:33:56 +00:00
"github.com/pbnjay/memory"
2023-07-21 20:33:56 +00:00
"github.com/jmorganca/ollama/api"
)
type LLM interface {
Predict(context.Context, []int, string, func(api.GenerateResponse)) error
Embedding(context.Context, string) ([]float64, error)
Encode(context.Context, string) ([]int, error)
Decode(context.Context, []int) (string, error)
2023-07-21 20:33:56 +00:00
SetOptions(api.Options)
Close()
Ping(context.Context) error
2023-07-21 20:33:56 +00:00
}
func New(workDir, model string, adapters []string, opts api.Options) (LLM, error) {
2023-07-21 20:33:56 +00:00
if _, err := os.Stat(model); err != nil {
return nil, err
}
f, err := os.Open(model)
if err != nil {
return nil, err
}
2023-08-14 23:08:02 +00:00
defer f.Close()
2023-07-21 20:33:56 +00:00
2023-09-07 17:55:37 +00:00
ggml, err := DecodeGGML(f)
2023-07-21 20:33:56 +00:00
if err != nil {
return nil, err
}
if runtime.GOOS == "darwin" {
switch ggml.FileType() {
case "Q8_0":
if ggml.Name() != "gguf" && opts.NumGPU != 0 {
// GGML Q8_0 do not support Metal API and will
// cause the runner to segmentation fault so disable GPU
log.Printf("WARNING: GPU disabled for F32, Q5_0, Q5_1, and Q8_0")
opts.NumGPU = 0
}
case "F32", "Q5_0", "Q5_1":
if opts.NumGPU != 0 {
// F32, Q5_0, Q5_1, and Q8_0 do not support Metal API and will
// cause the runner to segmentation fault so disable GPU
log.Printf("WARNING: GPU disabled for F32, Q5_0, Q5_1, and Q8_0")
opts.NumGPU = 0
}
}
}
totalResidentMemory := memory.TotalMemory()
2023-08-17 18:37:27 +00:00
switch ggml.ModelType() {
case "3B", "7B":
2023-10-11 17:55:07 +00:00
if ggml.FileType() == "F16" && totalResidentMemory < 16*1000*1000 {
return nil, fmt.Errorf("F16 model requires at least 16 GB of memory")
} else if totalResidentMemory < 8*1000*1000 {
return nil, fmt.Errorf("model requires at least 8 GB of memory")
}
case "13B":
2023-10-11 17:55:07 +00:00
if ggml.FileType() == "F16" && totalResidentMemory < 32*1000*1000 {
return nil, fmt.Errorf("F16 model requires at least 32 GB of memory")
} else if totalResidentMemory < 16*1000*1000 {
return nil, fmt.Errorf("model requires at least 16 GB of memory")
}
case "30B", "34B", "40B":
2023-10-11 17:55:07 +00:00
if ggml.FileType() == "F16" && totalResidentMemory < 64*1000*1000 {
return nil, fmt.Errorf("F16 model requires at least 64 GB of memory")
} else if totalResidentMemory < 32*1000*1000 {
return nil, fmt.Errorf("model requires at least 32 GB of memory")
}
case "65B", "70B":
2023-10-11 17:55:07 +00:00
if ggml.FileType() == "F16" && totalResidentMemory < 128*1000*1000 {
return nil, fmt.Errorf("F16 model requires at least 128 GB of memory")
} else if totalResidentMemory < 64*1000*1000 {
return nil, fmt.Errorf("model requires at least 64 GB of memory")
}
case "180B":
2023-10-11 17:55:07 +00:00
if ggml.FileType() == "F16" && totalResidentMemory < 512*1000*1000 {
return nil, fmt.Errorf("F16 model requires at least 512GB of memory")
2023-10-11 17:55:07 +00:00
} else if totalResidentMemory < 128*1000*1000 {
return nil, fmt.Errorf("model requires at least 128GB of memory")
}
}
2023-09-07 17:55:37 +00:00
switch ggml.Name() {
case "gguf":
opts.NumGQA = 0 // TODO: remove this when llama.cpp runners differ enough to need separate newLlama functions
return newLlama(model, adapters, chooseRunners(workDir, "gguf"), ggml.NumLayers(), opts)
2023-09-07 17:55:37 +00:00
case "ggml", "ggmf", "ggjt", "ggla":
return newLlama(model, adapters, chooseRunners(workDir, "ggml"), ggml.NumLayers(), opts)
2023-07-21 20:33:56 +00:00
default:
2023-08-17 18:37:27 +00:00
return nil, fmt.Errorf("unknown ggml type: %s", ggml.ModelFamily())
2023-07-21 20:33:56 +00:00
}
}