From e835ef183691db1cc7da30cfc61fb4b96b321e80 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Fri, 21 Jun 2024 13:30:43 -0700 Subject: [PATCH] fix: quantization with template --- server/images.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/server/images.go b/server/images.go index 53a95771..98794149 100644 --- a/server/images.go +++ b/server/images.go @@ -414,17 +414,22 @@ func CreateModel(ctx context.Context, name model.Name, modelFileDir, quantizatio return err } - layers, err := parseFromFile(ctx, temp, "", fn) + layer, err := NewLayer(temp, baseLayer.MediaType) if err != nil { return err } - if len(layers) != 1 { - return errors.New("quantization failed") + if _, err := temp.Seek(0, io.SeekStart); err != nil { + return err } - baseLayer.Layer = layers[0].Layer - baseLayer.GGML = layers[0].GGML + ggml, _, err := llm.DecodeGGML(temp) + if err != nil { + return err + } + + baseLayer.Layer = layer + baseLayer.GGML = ggml } }