baalajimaestro/ollama
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

comments

Browse source
This commit is contained in:
Michael Yang 2024-04-23 15:18:45 -07:00
parent a7248f6ea8
commit 01811c176a
5 changed files with 86 additions and 80 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

146
llm/filetype.go
View file

@ -2,137 +2,139 @@ package llm
import "fmt"
type filetype uint32
type fileType uint32
const (
filetypeF32 filetype = iota
filetypeF16
filetypeQ4_0
filetypeQ4_1
filetypeQ4_1_F16
filetypeQ8_0 filetype = iota + 2
filetypeQ5_0
filetypeQ5_1
filetypeQ2_K
filetypeQ3_K_S
filetypeQ3_K_M
filetypeQ3_K_L
filetypeQ4_K_S
filetypeQ4_K_M
filetypeQ5_K_S
filetypeQ5_K_M
filetypeQ6_K
filetypeIQ2_XXS
filetypeIQ2_XS
filetypeQ2_K_S
filetypeQ3_K_XS
filetypeIQ3_XXS
fileTypeF32 fileType = iota
fileTypeF16
fileTypeQ4_0
fileTypeQ4_1
fileTypeQ4_1_F16
fileTypeQ4_2 // unused
fileTypeQ4_3 // unused
fileTypeQ8_0
fileTypeQ5_0
fileTypeQ5_1
fileTypeQ2_K
fileTypeQ3_K_S
fileTypeQ3_K_M
fileTypeQ3_K_L
fileTypeQ4_K_S
fileTypeQ4_K_M
fileTypeQ5_K_S
fileTypeQ5_K_M
fileTypeQ6_K
fileTypeIQ2_XXS
fileTypeIQ2_XS
fileTypeQ2_K_S
fileTypeQ3_K_XS
fileTypeIQ3_XXS
filetypeUnknown
fileTypeUnknown
)
func ParseFileType(s string) (filetype, error) {
func ParseFileType(s string) (fileType, error) {
switch s {
case "F32":
return filetypeF32, nil
return fileTypeF32, nil
case "F16":
return filetypeF16, nil
return fileTypeF16, nil
case "Q4_0":
return filetypeQ4_0, nil
return fileTypeQ4_0, nil
case "Q4_1":
return filetypeQ4_1, nil
return fileTypeQ4_1, nil
case "Q4_1_F16":
return filetypeQ4_1_F16, nil
return fileTypeQ4_1_F16, nil
case "Q8_0":
return filetypeQ8_0, nil
return fileTypeQ8_0, nil
case "Q5_0":
return filetypeQ5_0, nil
return fileTypeQ5_0, nil
case "Q5_1":
return filetypeQ5_1, nil
return fileTypeQ5_1, nil
case "Q2_K":
return filetypeQ2_K, nil
return fileTypeQ2_K, nil
case "Q3_K_S":
return filetypeQ3_K_S, nil
return fileTypeQ3_K_S, nil
case "Q3_K_M":
return filetypeQ3_K_M, nil
return fileTypeQ3_K_M, nil
case "Q3_K_L":
return filetypeQ3_K_L, nil
return fileTypeQ3_K_L, nil
case "Q4_K_S":
return filetypeQ4_K_S, nil
return fileTypeQ4_K_S, nil
case "Q4_K_M":
return filetypeQ4_K_M, nil
return fileTypeQ4_K_M, nil
case "Q5_K_S":
return filetypeQ5_K_S, nil
return fileTypeQ5_K_S, nil
case "Q5_K_M":
return filetypeQ5_K_M, nil
return fileTypeQ5_K_M, nil
case "Q6_K":
return filetypeQ6_K, nil
return fileTypeQ6_K, nil
case "IQ2_XXS":
return filetypeIQ2_XXS, nil
return fileTypeIQ2_XXS, nil
case "IQ2_XS":
return filetypeIQ2_XS, nil
return fileTypeIQ2_XS, nil
case "Q2_K_S":
return filetypeQ2_K_S, nil
return fileTypeQ2_K_S, nil
case "Q3_K_XS":
return filetypeQ3_K_XS, nil
return fileTypeQ3_K_XS, nil
case "IQ3_XXS":
return filetypeIQ3_XXS, nil
return fileTypeIQ3_XXS, nil
default:
return filetypeUnknown, fmt.Errorf("unknown filetype: %s", s)
return fileTypeUnknown, fmt.Errorf("unknown fileType: %s", s)
}
}
func (t filetype) String() string {
func (t fileType) String() string {
switch t {
case filetypeF32:
case fileTypeF32:
return "F32"
case filetypeF16:
case fileTypeF16:
return "F16"
case filetypeQ4_0:
case fileTypeQ4_0:
return "Q4_0"
case filetypeQ4_1:
case fileTypeQ4_1:
return "Q4_1"
case filetypeQ4_1_F16:
case fileTypeQ4_1_F16:
return "Q4_1_F16"
case filetypeQ8_0:
case fileTypeQ8_0:
return "Q8_0"
case filetypeQ5_0:
case fileTypeQ5_0:
return "Q5_0"
case filetypeQ5_1:
case fileTypeQ5_1:
return "Q5_1"
case filetypeQ2_K:
case fileTypeQ2_K:
return "Q2_K"
case filetypeQ3_K_S:
case fileTypeQ3_K_S:
return "Q3_K_S"
case filetypeQ3_K_M:
case fileTypeQ3_K_M:
return "Q3_K_M"
case filetypeQ3_K_L:
case fileTypeQ3_K_L:
return "Q3_K_L"
case filetypeQ4_K_S:
case fileTypeQ4_K_S:
return "Q4_K_S"
case filetypeQ4_K_M:
case fileTypeQ4_K_M:
return "Q4_K_M"
case filetypeQ5_K_S:
case fileTypeQ5_K_S:
return "Q5_K_S"
case filetypeQ5_K_M:
case fileTypeQ5_K_M:
return "Q5_K_M"
case filetypeQ6_K:
case fileTypeQ6_K:
return "Q6_K"
case filetypeIQ2_XXS:
case fileTypeIQ2_XXS:
return "IQ2_XXS"
case filetypeIQ2_XS:
case fileTypeIQ2_XS:
return "IQ2_XS"
case filetypeQ2_K_S:
case fileTypeQ2_K_S:
return "Q2_K_S"
case filetypeQ3_K_XS:
case fileTypeQ3_K_XS:
return "Q3_K_XS"
case filetypeIQ3_XXS:
case fileTypeIQ3_XXS:
return "IQ3_XXS"
default:
return "unknown"
}
}
func (t filetype) Value() uint32 {
func (t fileType) Value() uint32 {
return uint32(t)
}

2
llm/ggml.go
View file

@ -47,7 +47,7 @@ func (kv KV) ParameterCount() uint64 {
func (kv KV) FileType() string {
if u64 := kv.u64("general.file_type"); u64 > 0 {
return filetype(uint32(u64)).String()
return fileType(uint32(u64)).String()
}
return "unknown"

2
llm/llm.go
View file

@ -20,7 +20,7 @@ func SystemInfo() string {
return C.GoString(C.llama_print_system_info())
}
func Quantize(infile, outfile string, ftype filetype) error {
func Quantize(infile, outfile string, ftype fileType) error {
cinfile := C.CString(infile)
defer C.free(unsafe.Pointer(cinfile))

10
server/images.go
View file

@ -344,9 +344,9 @@ func CreateModel(ctx context.Context, name, modelFileDir, quantization string, m
switch c.Name {
case "model", "adapter":
var layers2 *ordered.Map[*Layer, *llm.GGML]
var baseLayers *ordered.Map[*Layer, *llm.GGML]
if name := model.ParseName(c.Args, ""); name.IsValid() {
layers2, err = parseFromModel(ctx, name, fn)
baseLayers, err = parseFromModel(ctx, name, fn)
if err != nil {
return err
}
@ -362,14 +362,14 @@ func CreateModel(ctx context.Context, name, modelFileDir, quantization string, m
}
defer blob.Close()
layers2, err = parseFromFile(ctx, blob, fn)
baseLayers, err = parseFromFile(ctx, blob, fn)
if err != nil {
return err
}
} else if file, err := os.Open(realpath(modelFileDir, c.Args)); err == nil {
defer file.Close()
layers2, err = parseFromFile(ctx, file, fn)
baseLayers, err = parseFromFile(ctx, file, fn)
if err != nil {
return err
}
@ -381,7 +381,7 @@ func CreateModel(ctx context.Context, name, modelFileDir, quantization string, m
var tempfiles []*os.File
// TODO(mxyng): replace with rangefunc
layers2.Items()(func(layer *Layer, ggml *llm.GGML) bool {
baseLayers.Items()(func(layer *Layer, ggml *llm.GGML) bool {
if quantization != "" && ggml != nil && ggml.Name() == "gguf" {
ftype, err := llm.ParseFileType(quantization)
if err != nil {

6
server/model.go
View file

@ -27,7 +27,11 @@ func parseFromModel(ctx context.Context, name model.Name, fn func(api.ProgressRe
return nil, err
}
return parseFromModel(ctx, name, fn)
modelpath = ParseModelPath(name.DisplayLongest())
manifest, _, err = GetManifest(modelpath)
if err != nil {
return nil, err
}
case err != nil:
return nil, err
}