fix(observability-proxy): fail an application query whose lookup failed

When the pod or PVC lookup failed, its branch was dropped and the
query still answered 200, without the cAdvisor and volume series of
the application, so a store error showed up as an empty graph. That
is the same silently incomplete answer that a lookup over its cap is
refused for.

A failed lookup now fails the request: 504 when the store missed the
deadline, 502 otherwise.

Assisted-by: LLM
Signed-off-by: ZverGuy <maximbel2003@gmail.com>
This commit is contained in:
ZverGuy committed 2026-11-10 04:03:24 +07:00
1 parent 97a55a167b
commit 1a3517fa81
2 files changed
+13 -9

No files matched your search

+8 -5
View File
@@ -95,9 +95,9 @@ func applicationSelectors(ns, kind, name string, pods, pvcs []string) []string {
// applicationScope looks the application's pods and PVCs up in store over
// the request window, reaching back by the query's own lookback up to the
// configured limit and forward to the latest sample a negative offset
// reads. A failed lookup drops its branch, which only narrows the answer;
// a list over its cap, or a query whose latest sample is unknown, is
// refused, because the answer would silently miss series.
// reads. A failed lookup, a list over its cap, or a query whose latest
// sample is unknown fails the request, because a successful answer would
// silently miss the application's series.
func (s *Server) applicationScope(ctx context.Context, r *http.Request, store string, req metricsRequest, ns, kind, name string) ([]string, *HTTPError) {
if req.last.IsZero() {
return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ other than start() or end(), or an offset that is not a plain duration); use the namespace root")
@@ -115,8 +115,11 @@ func (s *Server) applicationScope(ctx context.Context, r *http.Request, store st
match := l.metric + identitySelector(ns, ksmKindLabel, ksmNameLabel, kind, name)
names, err := s.lookupNames(ctx, r, store, l.label, match, ns, start, req.last, l.max)
if err != nil {
slog.Warn("application lookup failed; its branch is left out", "label", l.label, "namespace", ns, "kind", kind, "name", name, "error", err)
continue
slog.Warn("application lookup failed", "label", l.label, "namespace", ns, "kind", kind, "name", name, "error", err)
if herr, ok := err.(*HTTPError); ok && herr.Code == http.StatusGatewayTimeout {
return nil, herr
}
return nil, errorf(http.StatusBadGateway, "the application's %s lookup failed", l.label)
}
if len(names) > l.max {
return nil, errorf(http.StatusUnprocessableEntity, "the application has more than %d %ss in this window; use a shorter range", l.max, l.label)
@@ -218,8 +218,9 @@ func TestServerApplicationMetricsBranches(t *testing.T) {
want []string
}{
{name: "no pods or pvcs", path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch}},
{name: "pod lookup fails", pods: []string{"p"}, pvcs: []string{"v"}, podStatus: http.StatusServiceUnavailable, path: "api/v1/query?query=up", wantCode: http.StatusOK,
want: []string{wantIdentityBranch, `{namespace="tenant-a",persistentvolumeclaim=~"^(v)$"}`}},
// A failed lookup would leave the application's own series out of
// a successful answer, so it fails the request instead.
{name: "pod lookup fails", pods: []string{"p"}, pvcs: []string{"v"}, podStatus: http.StatusServiceUnavailable, path: "api/v1/query?query=up", wantCode: http.StatusBadGateway},
{name: "pods over the cap", pods: []string{"p", "q"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}},
path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity},
{name: "over the cap with an empty name", pods: []string{"", "p"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}},
@@ -232,8 +233,8 @@ func TestServerApplicationMetricsBranches(t *testing.T) {
// series without a pod label.
{name: "empty pod name dropped", pods: []string{"", "p"}, path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch, podBranch}},
{name: "only empty pod names", pods: []string{""}, path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch}},
{name: "lookup status error", podBody: `{"status":"error","data":["p"]}`, path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch}},
{name: "truncated lookup answer", podBody: `{"status":"success","data":["p"`, path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch}},
{name: "lookup status error", podBody: `{"status":"error","data":["p"]}`, path: "api/v1/query?query=up", wantCode: http.StatusBadGateway},
{name: "truncated lookup answer", podBody: `{"status":"success","data":["p"`, path: "api/v1/query?query=up", wantCode: http.StatusBadGateway},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {