baalajimaestro/ollama
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

logs: explain client aborts better (#7783)

Browse source 
Users get confused by "Failed to acquire semaphore" error="context canceled"
messages in the logs, which are actually clients giving up.  While there could be
a legitimate hang bug in the system, sometimes this is just short client timeouts
with an overloaded system, so this should help users understand what's going on
better.
This commit is contained in:
Daniel Hiltgen 2024-11-22 08:05:32 -08:00 committed by GitHub
parent d88972ea48
commit b85520bfb9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 20 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
llama/runner/runner.go
View file

@ -651,7 +651,11 @@ func (s *Server) completion(w http.ResponseWriter, r *http.Request) {
// Ensure that a place to put the sequence is available // Ensure that a place to put the sequence is available
if err := s.seqsSem.Acquire(r.Context(), 1); err != nil { if err := s.seqsSem.Acquire(r.Context(), 1); err != nil {
if errors.Is(err, context.Canceled) {
slog.Info("aborting completion request due to client closing the connection")
} else {
slog.Error("Failed to acquire semaphore", "error", err) slog.Error("Failed to acquire semaphore", "error", err)
}
return return
} }
defer s.seqsSem.Release(1) defer s.seqsSem.Release(1)
@ -740,7 +744,11 @@ func (s *Server) embeddings(w http.ResponseWriter, r *http.Request) {
// Ensure that a place to put the sequence is available // Ensure that a place to put the sequence is available
if err := s.seqsSem.Acquire(r.Context(), 1); err != nil { if err := s.seqsSem.Acquire(r.Context(), 1); err != nil {
if errors.Is(err, context.Canceled) {
slog.Info("aborting embeddings request due to client closing the connection")
} else {
slog.Error("Failed to acquire semaphore", "error", err) slog.Error("Failed to acquire semaphore", "error", err)
}
return return
} }
defer s.seqsSem.Release(1) defer s.seqsSem.Release(1)

8
llm/server.go
View file

@ -687,7 +687,11 @@ type CompletionResponse struct {
func (s *llmServer) Completion(ctx context.Context, req CompletionRequest, fn func(CompletionResponse)) error { func (s *llmServer) Completion(ctx context.Context, req CompletionRequest, fn func(CompletionResponse)) error {
if err := s.sem.Acquire(ctx, 1); err != nil { if err := s.sem.Acquire(ctx, 1); err != nil {
if errors.Is(err, context.Canceled) {
slog.Info("aborting completion request due to client closing the connection")
} else {
slog.Error("Failed to acquire semaphore", "error", err) slog.Error("Failed to acquire semaphore", "error", err)
}
return err return err
} }
defer s.sem.Release(1) defer s.sem.Release(1)
@ -865,7 +869,11 @@ type EmbeddingResponse struct {
func (s *llmServer) Embedding(ctx context.Context, input string) ([]float32, error) { func (s *llmServer) Embedding(ctx context.Context, input string) ([]float32, error) {
if err := s.sem.Acquire(ctx, 1); err != nil { if err := s.sem.Acquire(ctx, 1); err != nil {
if errors.Is(err, context.Canceled) {
slog.Info("aborting embedding request due to client closing the connection")
} else {
slog.Error("Failed to acquire semaphore", "error", err) slog.Error("Failed to acquire semaphore", "error", err)
}
return nil, err return nil, err
} }
defer s.sem.Release(1) defer s.sem.Release(1)