diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index eec1f0eb6..796280eb6 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -120,12 +120,12 @@ func (s *Server) applicationScope(ctx context.Context, r *http.Request, store st slog.Warn("application lookup failed", "label", l.label, "namespace", ns, "kind", kind, "name", name, "error", err) // The deadline can also expire while the answer is read, after // post has handed it over. - if herr, ok := err.(*HTTPError); ok && herr.Code == http.StatusUnprocessableEntity { - return nil, herr - } if errors.Is(context.Cause(ctx), context.DeadlineExceeded) { return nil, errorf(http.StatusGatewayTimeout, "the application's %s lookup did not answer in time", l.label) } + if herr, ok := err.(*HTTPError); ok && (herr.Code == http.StatusUnprocessableEntity || herr.Code == http.StatusGatewayTimeout) { + return nil, herr + } return nil, errorf(http.StatusBadGateway, "the application's %s lookup failed", l.label) } if len(names) > l.max { @@ -151,10 +151,7 @@ func (s *Server) lookupNames(ctx context.Context, r *http.Request, store, label, } defer func() { _ = resp.Body.Close() }() if resp.StatusCode == http.StatusUnprocessableEntity { - // vmselect's refusal of a lookup over its own limits; the client - // can act on it by narrowing the window. - msg, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<10)) - return nil, errorf(http.StatusUnprocessableEntity, "the application's %s lookup: %s", label, strings.TrimSpace(string(msg))) + return nil, lookupRefusal(label, resp.Body) } if resp.StatusCode != http.StatusOK { return nil, errorf(resp.StatusCode, "metrics store answered %s", resp.Status) @@ -182,3 +179,27 @@ func (s *Server) lookupNames(ctx context.Context, r *http.Request, store, label, } return names, nil } + +// seriesLimitError is the message vmstorage gives when a search matches +// more series than the -search.max* flags allow (lib/storage/index_db.go); +// cluster vmselect wraps it in its own context, hence a substring match. +const seriesLimitError = "the number of matching timeseries exceeds" + +// lookupRefusal reads a vmselect 422. vmselect answers 422 to any error +// without a status of its own (httpserver.SendPrometheusError), storage +// failures and its own timeout included, so only the series limit, which +// a narrower window fixes, stays a 422 for the client. +func lookupRefusal(label string, body io.Reader) *HTTPError { + var answer struct { + Error string `json:"error"` + } + _ = json.NewDecoder(io.LimitReader(body, 64<<10)).Decode(&answer) + switch { + case strings.Contains(answer.Error, seriesLimitError): + return errorf(http.StatusUnprocessableEntity, "the application's %s lookup: %s", label, answer.Error) + case strings.Contains(answer.Error, "timeout exceeded"): + return errorf(http.StatusGatewayTimeout, "the application's %s lookup did not answer in time", label) + default: + return errorf(http.StatusBadGateway, "the application's %s lookup failed: %s", label, answer.Error) + } +} diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 0b894b8fc..2cd2a0eda 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -222,9 +222,6 @@ func TestServerApplicationMetricsBranches(t *testing.T) { // A failed lookup would leave the application's own series out of // a successful answer, so it fails the request instead. {name: "pod lookup fails", pods: []string{"p"}, pvcs: []string{"v"}, podStatus: http.StatusServiceUnavailable, path: "api/v1/query?query=up", wantCode: http.StatusBadGateway}, - // vmselect refuses a lookup over its own series limits with 422; - // the client can act on that by narrowing the window. - {name: "pod lookup over the store's limits", pods: []string{"p"}, podStatus: http.StatusUnprocessableEntity, path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity}, {name: "pods over the cap", pods: []string{"p", "q"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity}, {name: "over the cap with an empty name", pods: []string{"", "p"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, @@ -449,3 +446,41 @@ func TestServerApplicationDefaultLookbackServesLongPanels(t *testing.T) { } } } + +// TestServerApplicationLookupStoreErrors: vmselect answers 422 to any error +// without a status of its own (SendPrometheusError), so only its series +// limit, which a narrower window fixes, reaches the client as 422. +func TestServerApplicationLookupStoreErrors(t *testing.T) { + for _, tc := range []struct { + label, message string + want int + }{ + {"pod", "the number of matching timeseries exceeds 300000; either narrow down the search or increase -search.max* command-line flag values", http.StatusUnprocessableEntity}, + {"persistentvolumeclaim", "the number of matching timeseries exceeds 300000; either narrow down the search", http.StatusUnprocessableEntity}, + {"pod", "cannot fetch label values from vmstorage nodes: the number of matching timeseries exceeds 300000", http.StatusUnprocessableEntity}, + {"pod", "timeout exceeded before starting the query processing: 30s", http.StatusGatewayTimeout}, + {"pod", "error when searching for label values: cannot obtain data from vmstorage", http.StatusBadGateway}, + } { + h := newPolicyHarness(t, Config{}, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/v1/label/"+tc.label+"/values" { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusUnprocessableEntity) + _ = json.NewEncoder(w).Encode(map[string]string{"status": "error", "errorType": "422", "error": tc.message}) + return + } + appMetricsUpstream([]string{"p"}, nil, http.StatusOK)(w, r) + }, appPolicy, nil) + resp := h.do(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?query=up", "good", nil, nil) + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != tc.want { + t.Errorf("%s lookup %q answered %d, want %d", tc.label, tc.message, resp.StatusCode, tc.want) + } + if tc.want == http.StatusUnprocessableEntity && !strings.Contains(string(body), "the number of matching timeseries exceeds 300000") { + t.Errorf("the store's limit message did not reach the client: %q", body) + } + if strings.Contains(string(body), `"errorType"`) { + t.Errorf("the store's raw JSON reached the client: %q", body) + } + } +}