fix(observability-proxy): answer 504 to a lookup that misses the deadline

Only a deadline hit before the lookup's response headers was reported
as 504. When the store sent its headers and then stalled, the deadline
expired while the body was decoded, and the decode error came back as
a generic 502, so a slow store looked like a broken one.

The request deadline is now checked whichever step of the lookup it
cut, and both cases are covered by a test.

Assisted-by: LLM
Signed-off-by: ZverGuy <maximbel2003@gmail.com>
This commit is contained in:
ZverGuy committed 2026-11-10 04:03:24 +07:00
1 parent 66e2b99f09
commit bc823b704e
2 files changed
+33 -2

No files matched your search

+5 -2
View File
@@ -19,6 +19,7 @@ package observabilityproxy
import (
"context"
"encoding/json"
"errors"
"io"
"log/slog"
"net/http"
@@ -116,8 +117,10 @@ func (s *Server) applicationScope(ctx context.Context, r *http.Request, store st
names, err := s.lookupNames(ctx, r, store, l.label, match, ns, start, req.last, l.max)
if err != nil {
slog.Warn("application lookup failed", "label", l.label, "namespace", ns, "kind", kind, "name", name, "error", err)
if herr, ok := err.(*HTTPError); ok && herr.Code == http.StatusGatewayTimeout {
return nil, herr
// The deadline can also expire while the answer is read, after
// post has handed it over.
if errors.Is(context.Cause(ctx), context.DeadlineExceeded) {
return nil, errorf(http.StatusGatewayTimeout, "the application's %s lookup did not answer in time", l.label)
}
return nil, errorf(http.StatusBadGateway, "the application's %s lookup failed", l.label)
}
@@ -380,3 +380,31 @@ func TestServerApplicationLookupCoversAlignedEnd(t *testing.T) {
t.Fatalf("lookup %s end = %s, want %s", r.path, r.form.Get("end"), want)
}
}
// TestServerApplicationLookupDeadline: a lookup that misses the request
// deadline answers 504, whether the store sent nothing or stalled after
// its headers, and the query itself is not sent.
func TestServerApplicationLookupDeadline(t *testing.T) {
for _, headers := range []bool{false, true} {
h := newPolicyHarness(t, Config{RequestTimeout: 50 * time.Millisecond}, func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api/v1/label/pod/values" {
appMetricsUpstream(nil, nil, http.StatusOK)(w, r)
return
}
if headers {
w.Header().Set("Content-Type", "application/json")
_, _ = io.WriteString(w, `{"status":"success","data":[`)
w.(http.Flusher).Flush()
}
<-r.Context().Done()
}, appPolicy, nil)
if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?query=up", "good"); got != http.StatusGatewayTimeout {
t.Fatalf("headers sent %v: status = %d, want 504", headers, got)
}
for _, r := range h.requests() {
if !strings.HasPrefix(r.path, "/api/v1/label/") {
t.Fatalf("headers sent %v: the query reached the store after a failed lookup", headers)
}
}
}
}