fix: quantization with template
This commit is contained in:
parent
c7c2f3bc22
commit
e835ef1836
1 changed files with 10 additions and 5 deletions
|
@ -414,17 +414,22 @@ func CreateModel(ctx context.Context, name model.Name, modelFileDir, quantizatio
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
layers, err := parseFromFile(ctx, temp, "", fn)
|
layer, err := NewLayer(temp, baseLayer.MediaType)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(layers) != 1 {
|
if _, err := temp.Seek(0, io.SeekStart); err != nil {
|
||||||
return errors.New("quantization failed")
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
baseLayer.Layer = layers[0].Layer
|
ggml, _, err := llm.DecodeGGML(temp)
|
||||||
baseLayer.GGML = layers[0].GGML
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
baseLayer.Layer = layer
|
||||||
|
baseLayer.GGML = ggml
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue