From 667a2ba18add1031cc4b208eba7cedf8b33548e6 Mon Sep 17 00:00:00 2001 From: Daniel Hiltgen Date: Fri, 26 Jan 2024 11:11:09 -0800 Subject: [PATCH] Detect lack of AVX and fallback to CPU mode We build the GPU libraries with AVX enabled to ensure that if not all layers fit on the GPU we get better performance in a mixed mode. If the user is using a virtualization/emulation system that lacks AVX this used to result in an illegal instruction error and crash before this fix. Now we will report a warning in the server log, and just use CPU mode to ensure we don't crash. --- gpu/gpu.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/gpu/gpu.go b/gpu/gpu.go index 743b27d1..e234ec39 100644 --- a/gpu/gpu.go +++ b/gpu/gpu.go @@ -122,9 +122,15 @@ func GetGPUInfo() GpuInfo { initGPUHandles() } + // All our GPU builds have AVX enabled, so fallback to CPU if we don't detect at least AVX + cpuVariant := GetCPUVariant() + if cpuVariant == "" { + slog.Warn("CPU does not have AVX or AVX2, disabling GPU support.") + } + var memInfo C.mem_info_t resp := GpuInfo{} - if gpuHandles.cuda != nil { + if gpuHandles.cuda != nil && cpuVariant != "" { C.cuda_check_vram(*gpuHandles.cuda, &memInfo) if memInfo.err != nil { slog.Info(fmt.Sprintf("error looking up CUDA GPU memory: %s", C.GoString(memInfo.err))) @@ -143,7 +149,7 @@ func GetGPUInfo() GpuInfo { slog.Info(fmt.Sprintf("CUDA GPU is too old. Falling back to CPU mode. Compute Capability detected: %d.%d", cc.major, cc.minor)) } } - } else if gpuHandles.rocm != nil { + } else if gpuHandles.rocm != nil && cpuVariant != "" { C.rocm_check_vram(*gpuHandles.rocm, &memInfo) if memInfo.err != nil { slog.Info(fmt.Sprintf("error looking up ROCm GPU memory: %s", C.GoString(memInfo.err))) @@ -185,7 +191,7 @@ func GetGPUInfo() GpuInfo { if resp.Library == "" { C.cpu_check_ram(&memInfo) resp.Library = "cpu" - resp.Variant = GetCPUVariant() + resp.Variant = cpuVariant } if memInfo.err != nil { slog.Info(fmt.Sprintf("error looking up CPU memory: %s", C.GoString(memInfo.err)))