From 1a3e1827ef2104e7f0ae881bd03f284442e4a5d5 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 01:29:13 +0700 Subject: [PATCH] fix(observability-proxy): pass a store 422 from the app lookup through vmselect answers 422 when a label lookup exceeds its own series limits. The application root turned that into a 502, which reads as a broken store, while the client could have fixed it by narrowing the window. The 422 and vmselect's message now reach the client. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 9 +++++++++ internal/observabilityproxy/server_app_test.go | 3 +++ 2 files changed, 12 insertions(+) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index 835d29b84..eec1f0eb6 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -120,6 +120,9 @@ func (s *Server) applicationScope(ctx context.Context, r *http.Request, store st slog.Warn("application lookup failed", "label", l.label, "namespace", ns, "kind", kind, "name", name, "error", err) // The deadline can also expire while the answer is read, after // post has handed it over. + if herr, ok := err.(*HTTPError); ok && herr.Code == http.StatusUnprocessableEntity { + return nil, herr + } if errors.Is(context.Cause(ctx), context.DeadlineExceeded) { return nil, errorf(http.StatusGatewayTimeout, "the application's %s lookup did not answer in time", l.label) } @@ -147,6 +150,12 @@ func (s *Server) lookupNames(ctx context.Context, r *http.Request, store, label, return nil, herr } defer func() { _ = resp.Body.Close() }() + if resp.StatusCode == http.StatusUnprocessableEntity { + // vmselect's refusal of a lookup over its own limits; the client + // can act on it by narrowing the window. + msg, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<10)) + return nil, errorf(http.StatusUnprocessableEntity, "the application's %s lookup: %s", label, strings.TrimSpace(string(msg))) + } if resp.StatusCode != http.StatusOK { return nil, errorf(resp.StatusCode, "metrics store answered %s", resp.Status) } diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index d3dcb683a..d1d691b00 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -222,6 +222,9 @@ func TestServerApplicationMetricsBranches(t *testing.T) { // A failed lookup would leave the application's own series out of // a successful answer, so it fails the request instead. {name: "pod lookup fails", pods: []string{"p"}, pvcs: []string{"v"}, podStatus: http.StatusServiceUnavailable, path: "api/v1/query?query=up", wantCode: http.StatusBadGateway}, + // vmselect refuses a lookup over its own series limits with 422; + // the client can act on that by narrowing the window. + {name: "pod lookup over the store's limits", pods: []string{"p"}, podStatus: http.StatusUnprocessableEntity, path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity}, {name: "pods over the cap", pods: []string{"p", "q"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity}, {name: "over the cap with an empty name", pods: []string{"", "p"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}},