From 01811c176a43e2aa5bc288188f94949b8a0299b5 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Tue, 23 Apr 2024 15:18:45 -0700 Subject: [PATCH] comments --- llm/filetype.go | 146 ++++++++++++++++++++++++----------------------- llm/ggml.go | 2 +- llm/llm.go | 2 +- server/images.go | 10 ++-- server/model.go | 6 +- 5 files changed, 86 insertions(+), 80 deletions(-) diff --git a/llm/filetype.go b/llm/filetype.go index bae2f5d1..e5e9410d 100644 --- a/llm/filetype.go +++ b/llm/filetype.go @@ -2,137 +2,139 @@ package llm import "fmt" -type filetype uint32 +type fileType uint32 const ( - filetypeF32 filetype = iota - filetypeF16 - filetypeQ4_0 - filetypeQ4_1 - filetypeQ4_1_F16 - filetypeQ8_0 filetype = iota + 2 - filetypeQ5_0 - filetypeQ5_1 - filetypeQ2_K - filetypeQ3_K_S - filetypeQ3_K_M - filetypeQ3_K_L - filetypeQ4_K_S - filetypeQ4_K_M - filetypeQ5_K_S - filetypeQ5_K_M - filetypeQ6_K - filetypeIQ2_XXS - filetypeIQ2_XS - filetypeQ2_K_S - filetypeQ3_K_XS - filetypeIQ3_XXS + fileTypeF32 fileType = iota + fileTypeF16 + fileTypeQ4_0 + fileTypeQ4_1 + fileTypeQ4_1_F16 + fileTypeQ4_2 // unused + fileTypeQ4_3 // unused + fileTypeQ8_0 + fileTypeQ5_0 + fileTypeQ5_1 + fileTypeQ2_K + fileTypeQ3_K_S + fileTypeQ3_K_M + fileTypeQ3_K_L + fileTypeQ4_K_S + fileTypeQ4_K_M + fileTypeQ5_K_S + fileTypeQ5_K_M + fileTypeQ6_K + fileTypeIQ2_XXS + fileTypeIQ2_XS + fileTypeQ2_K_S + fileTypeQ3_K_XS + fileTypeIQ3_XXS - filetypeUnknown + fileTypeUnknown ) -func ParseFileType(s string) (filetype, error) { +func ParseFileType(s string) (fileType, error) { switch s { case "F32": - return filetypeF32, nil + return fileTypeF32, nil case "F16": - return filetypeF16, nil + return fileTypeF16, nil case "Q4_0": - return filetypeQ4_0, nil + return fileTypeQ4_0, nil case "Q4_1": - return filetypeQ4_1, nil + return fileTypeQ4_1, nil case "Q4_1_F16": - return filetypeQ4_1_F16, nil + return fileTypeQ4_1_F16, nil case "Q8_0": - return filetypeQ8_0, nil + return fileTypeQ8_0, nil case "Q5_0": - return filetypeQ5_0, nil + return fileTypeQ5_0, nil case "Q5_1": - return filetypeQ5_1, nil + return fileTypeQ5_1, nil case "Q2_K": - return filetypeQ2_K, nil + return fileTypeQ2_K, nil case "Q3_K_S": - return filetypeQ3_K_S, nil + return fileTypeQ3_K_S, nil case "Q3_K_M": - return filetypeQ3_K_M, nil + return fileTypeQ3_K_M, nil case "Q3_K_L": - return filetypeQ3_K_L, nil + return fileTypeQ3_K_L, nil case "Q4_K_S": - return filetypeQ4_K_S, nil + return fileTypeQ4_K_S, nil case "Q4_K_M": - return filetypeQ4_K_M, nil + return fileTypeQ4_K_M, nil case "Q5_K_S": - return filetypeQ5_K_S, nil + return fileTypeQ5_K_S, nil case "Q5_K_M": - return filetypeQ5_K_M, nil + return fileTypeQ5_K_M, nil case "Q6_K": - return filetypeQ6_K, nil + return fileTypeQ6_K, nil case "IQ2_XXS": - return filetypeIQ2_XXS, nil + return fileTypeIQ2_XXS, nil case "IQ2_XS": - return filetypeIQ2_XS, nil + return fileTypeIQ2_XS, nil case "Q2_K_S": - return filetypeQ2_K_S, nil + return fileTypeQ2_K_S, nil case "Q3_K_XS": - return filetypeQ3_K_XS, nil + return fileTypeQ3_K_XS, nil case "IQ3_XXS": - return filetypeIQ3_XXS, nil + return fileTypeIQ3_XXS, nil default: - return filetypeUnknown, fmt.Errorf("unknown filetype: %s", s) + return fileTypeUnknown, fmt.Errorf("unknown fileType: %s", s) } } -func (t filetype) String() string { +func (t fileType) String() string { switch t { - case filetypeF32: + case fileTypeF32: return "F32" - case filetypeF16: + case fileTypeF16: return "F16" - case filetypeQ4_0: + case fileTypeQ4_0: return "Q4_0" - case filetypeQ4_1: + case fileTypeQ4_1: return "Q4_1" - case filetypeQ4_1_F16: + case fileTypeQ4_1_F16: return "Q4_1_F16" - case filetypeQ8_0: + case fileTypeQ8_0: return "Q8_0" - case filetypeQ5_0: + case fileTypeQ5_0: return "Q5_0" - case filetypeQ5_1: + case fileTypeQ5_1: return "Q5_1" - case filetypeQ2_K: + case fileTypeQ2_K: return "Q2_K" - case filetypeQ3_K_S: + case fileTypeQ3_K_S: return "Q3_K_S" - case filetypeQ3_K_M: + case fileTypeQ3_K_M: return "Q3_K_M" - case filetypeQ3_K_L: + case fileTypeQ3_K_L: return "Q3_K_L" - case filetypeQ4_K_S: + case fileTypeQ4_K_S: return "Q4_K_S" - case filetypeQ4_K_M: + case fileTypeQ4_K_M: return "Q4_K_M" - case filetypeQ5_K_S: + case fileTypeQ5_K_S: return "Q5_K_S" - case filetypeQ5_K_M: + case fileTypeQ5_K_M: return "Q5_K_M" - case filetypeQ6_K: + case fileTypeQ6_K: return "Q6_K" - case filetypeIQ2_XXS: + case fileTypeIQ2_XXS: return "IQ2_XXS" - case filetypeIQ2_XS: + case fileTypeIQ2_XS: return "IQ2_XS" - case filetypeQ2_K_S: + case fileTypeQ2_K_S: return "Q2_K_S" - case filetypeQ3_K_XS: + case fileTypeQ3_K_XS: return "Q3_K_XS" - case filetypeIQ3_XXS: + case fileTypeIQ3_XXS: return "IQ3_XXS" default: return "unknown" } } -func (t filetype) Value() uint32 { +func (t fileType) Value() uint32 { return uint32(t) } diff --git a/llm/ggml.go b/llm/ggml.go index a7dbfeb3..a83bba8f 100644 --- a/llm/ggml.go +++ b/llm/ggml.go @@ -47,7 +47,7 @@ func (kv KV) ParameterCount() uint64 { func (kv KV) FileType() string { if u64 := kv.u64("general.file_type"); u64 > 0 { - return filetype(uint32(u64)).String() + return fileType(uint32(u64)).String() } return "unknown" diff --git a/llm/llm.go b/llm/llm.go index 0e96511f..2a0c4b91 100644 --- a/llm/llm.go +++ b/llm/llm.go @@ -20,7 +20,7 @@ func SystemInfo() string { return C.GoString(C.llama_print_system_info()) } -func Quantize(infile, outfile string, ftype filetype) error { +func Quantize(infile, outfile string, ftype fileType) error { cinfile := C.CString(infile) defer C.free(unsafe.Pointer(cinfile)) diff --git a/server/images.go b/server/images.go index 1de10929..998d1335 100644 --- a/server/images.go +++ b/server/images.go @@ -344,9 +344,9 @@ func CreateModel(ctx context.Context, name, modelFileDir, quantization string, m switch c.Name { case "model", "adapter": - var layers2 *ordered.Map[*Layer, *llm.GGML] + var baseLayers *ordered.Map[*Layer, *llm.GGML] if name := model.ParseName(c.Args, ""); name.IsValid() { - layers2, err = parseFromModel(ctx, name, fn) + baseLayers, err = parseFromModel(ctx, name, fn) if err != nil { return err } @@ -362,14 +362,14 @@ func CreateModel(ctx context.Context, name, modelFileDir, quantization string, m } defer blob.Close() - layers2, err = parseFromFile(ctx, blob, fn) + baseLayers, err = parseFromFile(ctx, blob, fn) if err != nil { return err } } else if file, err := os.Open(realpath(modelFileDir, c.Args)); err == nil { defer file.Close() - layers2, err = parseFromFile(ctx, file, fn) + baseLayers, err = parseFromFile(ctx, file, fn) if err != nil { return err } @@ -381,7 +381,7 @@ func CreateModel(ctx context.Context, name, modelFileDir, quantization string, m var tempfiles []*os.File // TODO(mxyng): replace with rangefunc - layers2.Items()(func(layer *Layer, ggml *llm.GGML) bool { + baseLayers.Items()(func(layer *Layer, ggml *llm.GGML) bool { if quantization != "" && ggml != nil && ggml.Name() == "gguf" { ftype, err := llm.ParseFileType(quantization) if err != nil { diff --git a/server/model.go b/server/model.go index 2d7797f0..cf036052 100644 --- a/server/model.go +++ b/server/model.go @@ -27,7 +27,11 @@ func parseFromModel(ctx context.Context, name model.Name, fn func(api.ProgressRe return nil, err } - return parseFromModel(ctx, name, fn) + modelpath = ParseModelPath(name.DisplayLongest()) + manifest, _, err = GetManifest(modelpath) + if err != nil { + return nil, err + } case err != nil: return nil, err }