diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index ff1955dc9..80d12cbe2 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -19,6 +19,7 @@ package observabilityproxy import ( "context" "encoding/json" + "errors" "io" "log/slog" "net/http" @@ -116,8 +117,10 @@ func (s *Server) applicationScope(ctx context.Context, r *http.Request, store st names, err := s.lookupNames(ctx, r, store, l.label, match, ns, start, req.last, l.max) if err != nil { slog.Warn("application lookup failed", "label", l.label, "namespace", ns, "kind", kind, "name", name, "error", err) - if herr, ok := err.(*HTTPError); ok && herr.Code == http.StatusGatewayTimeout { - return nil, herr + // The deadline can also expire while the answer is read, after + // post has handed it over. + if errors.Is(context.Cause(ctx), context.DeadlineExceeded) { + return nil, errorf(http.StatusGatewayTimeout, "the application's %s lookup did not answer in time", l.label) } return nil, errorf(http.StatusBadGateway, "the application's %s lookup failed", l.label) } diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 395deb18b..53216e575 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -380,3 +380,31 @@ func TestServerApplicationLookupCoversAlignedEnd(t *testing.T) { t.Fatalf("lookup %s end = %s, want %s", r.path, r.form.Get("end"), want) } } + +// TestServerApplicationLookupDeadline: a lookup that misses the request +// deadline answers 504, whether the store sent nothing or stalled after +// its headers, and the query itself is not sent. +func TestServerApplicationLookupDeadline(t *testing.T) { + for _, headers := range []bool{false, true} { + h := newPolicyHarness(t, Config{RequestTimeout: 50 * time.Millisecond}, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v1/label/pod/values" { + appMetricsUpstream(nil, nil, http.StatusOK)(w, r) + return + } + if headers { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"status":"success","data":[`) + w.(http.Flusher).Flush() + } + <-r.Context().Done() + }, appPolicy, nil) + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?query=up", "good"); got != http.StatusGatewayTimeout { + t.Fatalf("headers sent %v: status = %d, want 504", headers, got) + } + for _, r := range h.requests() { + if !strings.HasPrefix(r.path, "/api/v1/label/") { + t.Fatalf("headers sent %v: the query reached the store after a failed lookup", headers) + } + } + } +}