f457d63400
If the system has multiple numa nodes, enable numa support in llama.cpp If we detect numactl in the path, use that, else use the basic "distribute" mode.
37 lines
694 B
Go
37 lines
694 B
Go
package gpu
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"strings"
|
|
|
|
"golang.org/x/sys/cpu"
|
|
)
|
|
|
|
func GetCPUCapability() CPUCapability {
|
|
if cpu.X86.HasAVX2 {
|
|
return CPUCapabilityAVX2
|
|
}
|
|
if cpu.X86.HasAVX {
|
|
return CPUCapabilityAVX
|
|
}
|
|
// else LCD
|
|
return CPUCapabilityNone
|
|
}
|
|
|
|
func IsNUMA() bool {
|
|
if runtime.GOOS != "linux" {
|
|
// numa support in llama.cpp is linux only
|
|
return false
|
|
}
|
|
ids := map[string]interface{}{}
|
|
packageIds, _ := filepath.Glob("/sys/devices/system/cpu/cpu*/topology/physical_package_id")
|
|
for _, packageId := range packageIds {
|
|
id, err := os.ReadFile(packageId)
|
|
if err == nil {
|
|
ids[strings.TrimSpace(string(id))] = struct{}{}
|
|
}
|
|
}
|
|
return len(ids) > 1
|
|
}
|