baalajimaestro/ollama
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #3566 from dhiltgen/more_time

Browse source 
Handle very slow model loads
This commit is contained in:
Daniel Hiltgen 2024-04-09 16:53:49 -07:00 committed by GitHub
commit 4c7db6b7e9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 2 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
llm/server.go
View file

@ -383,7 +383,8 @@ func (s *LlamaServer) Ping(ctx context.Context) error {
func (s *LlamaServer) waitUntilRunning() error {
start := time.Now()
expiresAt := time.Now().Add(3 * time.Minute) // be generous with timeout, large models can take a while to load
// TODO we need to wire up a better way to detect hangs during model load and startup of the server
expiresAt := time.Now().Add(10 * time.Minute) // be generous with timeout, large models can take a while to load
ticker := time.NewTicker(50 * time.Millisecond)
defer ticker.Stop()