From 1a3517fa81d6ca87f4387ce4e50e831d981d60e7 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 00:55:50 +0700 Subject: [PATCH] fix(observability-proxy): fail an application query whose lookup failed When the pod or PVC lookup failed, its branch was dropped and the query still answered 200, without the cAdvisor and volume series of the application, so a store error showed up as an empty graph. That is the same silently incomplete answer that a lookup over its cap is refused for. A failed lookup now fails the request: 504 when the store missed the deadline, 502 otherwise. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 13 ++++++++----- internal/observabilityproxy/server_app_test.go | 9 +++++---- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index 88acb4ba4..ff1955dc9 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -95,9 +95,9 @@ func applicationSelectors(ns, kind, name string, pods, pvcs []string) []string { // applicationScope looks the application's pods and PVCs up in store over // the request window, reaching back by the query's own lookback up to the // configured limit and forward to the latest sample a negative offset -// reads. A failed lookup drops its branch, which only narrows the answer; -// a list over its cap, or a query whose latest sample is unknown, is -// refused, because the answer would silently miss series. +// reads. A failed lookup, a list over its cap, or a query whose latest +// sample is unknown fails the request, because a successful answer would +// silently miss the application's series. func (s *Server) applicationScope(ctx context.Context, r *http.Request, store string, req metricsRequest, ns, kind, name string) ([]string, *HTTPError) { if req.last.IsZero() { return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ other than start() or end(), or an offset that is not a plain duration); use the namespace root") @@ -115,8 +115,11 @@ func (s *Server) applicationScope(ctx context.Context, r *http.Request, store st match := l.metric + identitySelector(ns, ksmKindLabel, ksmNameLabel, kind, name) names, err := s.lookupNames(ctx, r, store, l.label, match, ns, start, req.last, l.max) if err != nil { - slog.Warn("application lookup failed; its branch is left out", "label", l.label, "namespace", ns, "kind", kind, "name", name, "error", err) - continue + slog.Warn("application lookup failed", "label", l.label, "namespace", ns, "kind", kind, "name", name, "error", err) + if herr, ok := err.(*HTTPError); ok && herr.Code == http.StatusGatewayTimeout { + return nil, herr + } + return nil, errorf(http.StatusBadGateway, "the application's %s lookup failed", l.label) } if len(names) > l.max { return nil, errorf(http.StatusUnprocessableEntity, "the application has more than %d %ss in this window; use a shorter range", l.max, l.label) diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index c7e7a67ef..268ea7b8f 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -218,8 +218,9 @@ func TestServerApplicationMetricsBranches(t *testing.T) { want []string }{ {name: "no pods or pvcs", path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch}}, - {name: "pod lookup fails", pods: []string{"p"}, pvcs: []string{"v"}, podStatus: http.StatusServiceUnavailable, path: "api/v1/query?query=up", wantCode: http.StatusOK, - want: []string{wantIdentityBranch, `{namespace="tenant-a",persistentvolumeclaim=~"^(v)$"}`}}, + // A failed lookup would leave the application's own series out of + // a successful answer, so it fails the request instead. + {name: "pod lookup fails", pods: []string{"p"}, pvcs: []string{"v"}, podStatus: http.StatusServiceUnavailable, path: "api/v1/query?query=up", wantCode: http.StatusBadGateway}, {name: "pods over the cap", pods: []string{"p", "q"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity}, {name: "over the cap with an empty name", pods: []string{"", "p"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, @@ -232,8 +233,8 @@ func TestServerApplicationMetricsBranches(t *testing.T) { // series without a pod label. {name: "empty pod name dropped", pods: []string{"", "p"}, path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch, podBranch}}, {name: "only empty pod names", pods: []string{""}, path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch}}, - {name: "lookup status error", podBody: `{"status":"error","data":["p"]}`, path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch}}, - {name: "truncated lookup answer", podBody: `{"status":"success","data":["p"`, path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch}}, + {name: "lookup status error", podBody: `{"status":"error","data":["p"]}`, path: "api/v1/query?query=up", wantCode: http.StatusBadGateway}, + {name: "truncated lookup answer", podBody: `{"status":"success","data":["p"`, path: "api/v1/query?query=up", wantCode: http.StatusBadGateway}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) {