2023-11-29 19:00:37 +00:00
|
|
|
package gpu
|
|
|
|
|
2024-05-07 21:54:26 +00:00
|
|
|
import (
|
|
|
|
"fmt"
|
|
|
|
"log/slog"
|
|
|
|
|
|
|
|
"github.com/ollama/ollama/format"
|
|
|
|
)
|
|
|
|
|
2023-12-22 23:43:31 +00:00
|
|
|
type memInfo struct {
|
2023-11-29 19:00:37 +00:00
|
|
|
TotalMemory uint64 `json:"total_memory,omitempty"`
|
|
|
|
FreeMemory uint64 `json:"free_memory,omitempty"`
|
2023-12-22 23:43:31 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Beginning of an `ollama info` command
|
|
|
|
type GpuInfo struct {
|
|
|
|
memInfo
|
|
|
|
Library string `json:"library,omitempty"`
|
2023-11-29 19:00:37 +00:00
|
|
|
|
2024-01-05 20:13:08 +00:00
|
|
|
// Optional variant to select (e.g. versions, cpu feature flags)
|
2024-06-05 19:07:20 +00:00
|
|
|
Variant CPUCapability `json:"variant"`
|
2024-01-05 20:13:08 +00:00
|
|
|
|
2024-03-18 09:45:22 +00:00
|
|
|
// MinimumMemory represents the minimum memory required to use the GPU
|
2024-04-05 21:50:38 +00:00
|
|
|
MinimumMemory uint64 `json:"-"`
|
2024-03-18 09:45:22 +00:00
|
|
|
|
2024-03-30 16:50:05 +00:00
|
|
|
// Any extra PATH/LD_LIBRARY_PATH dependencies required for the Library to operate properly
|
|
|
|
DependencyPath string `json:"lib_path,omitempty"`
|
|
|
|
|
|
|
|
// GPU information
|
2024-05-07 21:54:26 +00:00
|
|
|
ID string `json:"gpu_id"` // string to use for selection of this specific GPU
|
|
|
|
Name string `json:"name"` // user friendly name if available
|
|
|
|
Compute string `json:"compute"` // Compute Capability or gfx
|
|
|
|
|
|
|
|
// Driver Information - TODO no need to put this on each GPU
|
|
|
|
DriverMajor int `json:"driver_major,omitempty"`
|
|
|
|
DriverMinor int `json:"driver_minor,omitempty"`
|
2024-03-30 16:50:05 +00:00
|
|
|
|
|
|
|
// TODO other performance capability info to help in scheduling decisions
|
2023-11-29 19:00:37 +00:00
|
|
|
}
|
2024-02-11 22:50:06 +00:00
|
|
|
|
2024-05-15 22:13:16 +00:00
|
|
|
type CPUInfo struct {
|
|
|
|
GpuInfo
|
|
|
|
}
|
|
|
|
|
|
|
|
type CudaGPUInfo struct {
|
|
|
|
GpuInfo
|
2024-06-05 19:07:20 +00:00
|
|
|
index int //nolint:unused,nolintlint
|
2024-05-15 22:13:16 +00:00
|
|
|
}
|
|
|
|
type CudaGPUInfoList []CudaGPUInfo
|
|
|
|
|
|
|
|
type RocmGPUInfo struct {
|
|
|
|
GpuInfo
|
2024-06-05 19:07:20 +00:00
|
|
|
usedFilepath string //nolint:unused,nolintlint
|
|
|
|
index int //nolint:unused,nolintlint
|
2024-05-15 22:13:16 +00:00
|
|
|
}
|
|
|
|
type RocmGPUInfoList []RocmGPUInfo
|
|
|
|
|
|
|
|
type OneapiGPUInfo struct {
|
|
|
|
GpuInfo
|
2024-06-05 19:07:20 +00:00
|
|
|
driverIndex int //nolint:unused,nolintlint
|
|
|
|
gpuIndex int //nolint:unused,nolintlint
|
2024-05-15 22:13:16 +00:00
|
|
|
}
|
|
|
|
type OneapiGPUInfoList []OneapiGPUInfo
|
|
|
|
|
2024-03-30 16:50:05 +00:00
|
|
|
type GpuInfoList []GpuInfo
|
|
|
|
|
|
|
|
// Split up the set of gpu info's by Library and variant
|
|
|
|
func (l GpuInfoList) ByLibrary() []GpuInfoList {
|
|
|
|
resp := []GpuInfoList{}
|
|
|
|
libs := []string{}
|
|
|
|
for _, info := range l {
|
|
|
|
found := false
|
|
|
|
requested := info.Library
|
2024-06-05 19:07:20 +00:00
|
|
|
if info.Variant != CPUCapabilityNone {
|
|
|
|
requested += "_" + info.Variant.String()
|
2024-03-30 16:50:05 +00:00
|
|
|
}
|
|
|
|
for i, lib := range libs {
|
|
|
|
if lib == requested {
|
|
|
|
resp[i] = append(resp[i], info)
|
|
|
|
found = true
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if !found {
|
|
|
|
libs = append(libs, info.Library)
|
|
|
|
resp = append(resp, []GpuInfo{info})
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return resp
|
2024-02-11 22:50:06 +00:00
|
|
|
}
|
2024-03-30 16:50:05 +00:00
|
|
|
|
2024-05-07 21:54:26 +00:00
|
|
|
// Report the GPU information into the log an Info level
|
|
|
|
func (l GpuInfoList) LogDetails() {
|
|
|
|
for _, g := range l {
|
|
|
|
slog.Info("inference compute",
|
|
|
|
"id", g.ID,
|
|
|
|
"library", g.Library,
|
|
|
|
"compute", g.Compute,
|
|
|
|
"driver", fmt.Sprintf("%d.%d", g.DriverMajor, g.DriverMinor),
|
|
|
|
"name", g.Name,
|
|
|
|
"total", format.HumanBytes2(g.TotalMemory),
|
|
|
|
"available", format.HumanBytes2(g.FreeMemory),
|
|
|
|
)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2024-03-30 16:50:05 +00:00
|
|
|
// Sort by Free Space
|
|
|
|
type ByFreeMemory []GpuInfo
|
|
|
|
|
|
|
|
func (a ByFreeMemory) Len() int { return len(a) }
|
|
|
|
func (a ByFreeMemory) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
|
|
|
|
func (a ByFreeMemory) Less(i, j int) bool { return a[i].FreeMemory < a[j].FreeMemory }
|
2024-05-15 22:13:16 +00:00
|
|
|
|
|
|
|
type CPUCapability uint32
|
|
|
|
|
|
|
|
// Override at build time when building base GPU runners
|
|
|
|
var GPURunnerCPUCapability = CPUCapabilityAVX
|
|
|
|
|
|
|
|
const (
|
2024-06-05 19:07:20 +00:00
|
|
|
CPUCapabilityNone CPUCapability = iota
|
2024-05-15 22:13:16 +00:00
|
|
|
CPUCapabilityAVX
|
|
|
|
CPUCapabilityAVX2
|
|
|
|
// TODO AVX512
|
|
|
|
)
|
|
|
|
|
2024-06-05 19:07:20 +00:00
|
|
|
func (c CPUCapability) String() string {
|
2024-05-15 22:13:16 +00:00
|
|
|
switch c {
|
|
|
|
case CPUCapabilityAVX:
|
|
|
|
return "avx"
|
|
|
|
case CPUCapabilityAVX2:
|
|
|
|
return "avx2"
|
|
|
|
default:
|
2024-06-05 19:07:20 +00:00
|
|
|
return "no vector extensions"
|
2024-05-15 22:13:16 +00:00
|
|
|
}
|
|
|
|
}
|