baalajimaestro/ollama
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

fix the cpu estimatedTotal memory + get the expiry time for loading models (#4461)

Browse source
This commit is contained in:
Patrick Devine 2024-05-15 15:43:16 -07:00 committed by GitHub
parent 5fa36a0833
commit d1692fd3e0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 9 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
llm/server.go
View file

@ -89,6 +89,7 @@ func NewLlamaServer(gpus gpu.GpuInfoList, model string, ggml *GGML, adapters, pr
cpuRunner = serverForCpu() cpuRunner = serverForCpu()
gpuCount = 0 gpuCount = 0
_, _, estimatedTotal = EstimateGPULayers(gpus, ggml, projectors, opts)
} else { } else {
if gpus[0].Library == "metal" { if gpus[0].Library == "metal" {
memInfo, err := gpu.GetCPUMem() memInfo, err := gpu.GetCPUMem()

8
server/routes.go
View file

@ -1161,6 +1161,14 @@ func (s *Server) ProcessHandler(c *gin.Context) {
Details: modelDetails, Details: modelDetails,
ExpiresAt: v.expiresAt, ExpiresAt: v.expiresAt,
} }
// The scheduler waits to set expiresAt, so if a model is loading it's
// possible that it will be set to the unix epoch. For those cases, just
// calculate the time w/ the sessionDuration instead.
var epoch time.Time
if v.expiresAt == epoch {
mr.ExpiresAt = time.Now().Add(v.sessionDuration)
}
models = append(models, mr) models = append(models, mr)
} }