Merge pull request #1212 from jmorganca/mxyng/metal
enable metal for fp32, q5_0, q5_1
This commit is contained in:
commit
0b7ade0d4c
1 changed files with 1 additions and 8 deletions
|
@ -41,20 +41,13 @@ func New(workDir, model string, adapters []string, opts api.Options) (LLM, error
|
||||||
|
|
||||||
if runtime.GOOS == "darwin" {
|
if runtime.GOOS == "darwin" {
|
||||||
switch ggml.FileType() {
|
switch ggml.FileType() {
|
||||||
case "Q8_0":
|
case "F32", "Q5_0", "Q5_1", "Q8_0":
|
||||||
if ggml.Name() != "gguf" && opts.NumGPU != 0 {
|
if ggml.Name() != "gguf" && opts.NumGPU != 0 {
|
||||||
// GGML Q8_0 do not support Metal API and will
|
// GGML Q8_0 do not support Metal API and will
|
||||||
// cause the runner to segmentation fault so disable GPU
|
// cause the runner to segmentation fault so disable GPU
|
||||||
log.Printf("WARNING: GPU disabled for F32, Q5_0, Q5_1, and Q8_0")
|
log.Printf("WARNING: GPU disabled for F32, Q5_0, Q5_1, and Q8_0")
|
||||||
opts.NumGPU = 0
|
opts.NumGPU = 0
|
||||||
}
|
}
|
||||||
case "F32", "Q5_0", "Q5_1":
|
|
||||||
if opts.NumGPU != 0 {
|
|
||||||
// F32, Q5_0, Q5_1, and Q8_0 do not support Metal API and will
|
|
||||||
// cause the runner to segmentation fault so disable GPU
|
|
||||||
log.Printf("WARNING: GPU disabled for F32, Q5_0, Q5_1, and Q8_0")
|
|
||||||
opts.NumGPU = 0
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var requiredMemory int64
|
var requiredMemory int64
|
||||||
|
|
Loading…
Reference in a new issue