From b85520bfb9c9dd92056ecded55bf8c4cfd28088f Mon Sep 17 00:00:00 2001 From: Daniel Hiltgen Date: Fri, 22 Nov 2024 08:05:32 -0800 Subject: [PATCH] logs: explain client aborts better (#7783) Users get confused by "Failed to acquire semaphore" error="context canceled" messages in the logs, which are actually clients giving up. While there could be a legitimate hang bug in the system, sometimes this is just short client timeouts with an overloaded system, so this should help users understand what's going on better. --- llama/runner/runner.go | 12 ++++++++++-- llm/server.go | 12 ++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/llama/runner/runner.go b/llama/runner/runner.go index c7662b33..c3d0353f 100644 --- a/llama/runner/runner.go +++ b/llama/runner/runner.go @@ -651,7 +651,11 @@ func (s *Server) completion(w http.ResponseWriter, r *http.Request) { // Ensure that a place to put the sequence is available if err := s.seqsSem.Acquire(r.Context(), 1); err != nil { - slog.Error("Failed to acquire semaphore", "error", err) + if errors.Is(err, context.Canceled) { + slog.Info("aborting completion request due to client closing the connection") + } else { + slog.Error("Failed to acquire semaphore", "error", err) + } return } defer s.seqsSem.Release(1) @@ -740,7 +744,11 @@ func (s *Server) embeddings(w http.ResponseWriter, r *http.Request) { // Ensure that a place to put the sequence is available if err := s.seqsSem.Acquire(r.Context(), 1); err != nil { - slog.Error("Failed to acquire semaphore", "error", err) + if errors.Is(err, context.Canceled) { + slog.Info("aborting embeddings request due to client closing the connection") + } else { + slog.Error("Failed to acquire semaphore", "error", err) + } return } defer s.seqsSem.Release(1) diff --git a/llm/server.go b/llm/server.go index d7c5198d..b2405905 100644 --- a/llm/server.go +++ b/llm/server.go @@ -687,7 +687,11 @@ type CompletionResponse struct { func (s *llmServer) Completion(ctx context.Context, req CompletionRequest, fn func(CompletionResponse)) error { if err := s.sem.Acquire(ctx, 1); err != nil { - slog.Error("Failed to acquire semaphore", "error", err) + if errors.Is(err, context.Canceled) { + slog.Info("aborting completion request due to client closing the connection") + } else { + slog.Error("Failed to acquire semaphore", "error", err) + } return err } defer s.sem.Release(1) @@ -865,7 +869,11 @@ type EmbeddingResponse struct { func (s *llmServer) Embedding(ctx context.Context, input string) ([]float32, error) { if err := s.sem.Acquire(ctx, 1); err != nil { - slog.Error("Failed to acquire semaphore", "error", err) + if errors.Is(err, context.Canceled) { + slog.Info("aborting embedding request due to client closing the connection") + } else { + slog.Error("Failed to acquire semaphore", "error", err) + } return nil, err } defer s.sem.Release(1)