fix: quantization with template

This commit is contained in:
Michael Yang 2024-06-21 13:30:43 -07:00
parent c7c2f3bc22
commit e835ef1836

View file

@ -414,17 +414,22 @@ func CreateModel(ctx context.Context, name model.Name, modelFileDir, quantizatio
return err return err
} }
layers, err := parseFromFile(ctx, temp, "", fn) layer, err := NewLayer(temp, baseLayer.MediaType)
if err != nil { if err != nil {
return err return err
} }
if len(layers) != 1 { if _, err := temp.Seek(0, io.SeekStart); err != nil {
return errors.New("quantization failed") return err
} }
baseLayer.Layer = layers[0].Layer ggml, _, err := llm.DecodeGGML(temp)
baseLayer.GGML = layers[0].GGML if err != nil {
return err
}
baseLayer.Layer = layer
baseLayer.GGML = ggml
} }
} }