baalajimaestro/ollama
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

gpu: add 512MiB to darwin minimum, metal doesn't have partial offloading overhead (#4068)

Browse source
This commit is contained in:
Jeffrey Morgan 2024-05-01 11:46:03 -04:00 committed by GitHub
parent b9f74ff3d6
commit f0c454ab57
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 12 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
gpu/gpu_darwin.go
View file

@ -10,6 +10,12 @@ package gpu
import "C" import "C"
import ( import (
"runtime" "runtime"
"github.com/ollama/ollama/format"
)
const (
metalMinimumMemory = 512 * format.MebiByte
) )
func GetGPUInfo() GpuInfoList { func GetGPUInfo() GpuInfoList {
@ -32,7 +38,7 @@ func GetGPUInfo() GpuInfoList {
// TODO is there a way to gather actual allocated video memory? (currentAllocatedSize doesn't work) // TODO is there a way to gather actual allocated video memory? (currentAllocatedSize doesn't work)
info.FreeMemory = info.TotalMemory info.FreeMemory = info.TotalMemory
info.MinimumMemory = 0 info.MinimumMemory = metalMinimumMemory
return []GpuInfo{info} return []GpuInfo{info}
} }

5
llm/memory.go
View file

@ -88,6 +88,11 @@ func EstimateGPULayers(gpus []gpu.GpuInfo, ggml *GGML, projectors []string, opts
graphFullOffload *= uint64(len(gpus)) graphFullOffload *= uint64(len(gpus))
graphPartialOffload *= uint64(len(gpus)) graphPartialOffload *= uint64(len(gpus))
// on metal there's no partial offload overhead
if gpus[0].Library == "metal" {
graphPartialOffload = graphFullOffload
}
// memoryRequiredTotal represents the memory required for full GPU offloading (all layers) // memoryRequiredTotal represents the memory required for full GPU offloading (all layers)
memoryRequiredTotal := memoryMinimum + graphFullOffload memoryRequiredTotal := memoryMinimum + graphFullOffload