From 84665a42a798d935a1b078d6fbdd3c720c62bffc Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 01:28:12 +0700 Subject: [PATCH] fix(observability-proxy): refuse app queries that read past the lookup The pod and PVC lookup never started earlier than --metrics-app-lookback before the window. A query that read further back, such as rate(x[7d]) with the 24h default, or whose lookback is unknown (an @ modifier, WITH, a step-multiple window), answered 200 without the series of pods that existed only before the lookup start. Such queries now answer 422 on the application root, matching how a query whose latest sample cannot be covered is handled; the namespace root has no lookup and still serves them. Assisted-by: LLM Signed-off-by: ZverGuy --- cmd/observability-proxy/main.go | 2 +- internal/observabilityproxy/appscope.go | 17 ++++++----- .../observabilityproxy/server_app_test.go | 29 ++++++++++++++----- 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/cmd/observability-proxy/main.go b/cmd/observability-proxy/main.go index 4a99aa4cc..93f4c4553 100644 --- a/cmd/observability-proxy/main.go +++ b/cmd/observability-proxy/main.go @@ -94,7 +94,7 @@ func parseFlags(args []string) (options, error) { fs.DurationVar(&o.metrics.Limits.MaxRange, "metrics-max-range", 30*24*time.Hour, "longest [start, end] and step a metrics request may use") fs.IntVar(&o.metrics.Limits.MaxPoints, "metrics-max-points", 11000, "most points per series a range query may return") fs.DurationVar(&o.metrics.Limits.MaxTimeout, "metrics-max-timeout", 30*time.Second, "timeout forwarded to vmselect; a longer client timeout is clamped; must be below --request-timeout when that is set") - fs.DurationVar(&o.metrics.App.Lookback, "metrics-app-lookback", 24*time.Hour, "how far before the window the application root looks up the application's pods and PVCs at most") + fs.DurationVar(&o.metrics.App.Lookback, "metrics-app-lookback", 24*time.Hour, "how far before the window the application root looks up the application's pods and PVCs; a query reading further back answers 422") fs.IntVar(&o.metrics.App.MaxPods, "metrics-app-max-pods", 500, "most pods of one application in the lookup window; more answer 422") fs.IntVar(&o.metrics.App.MaxPVCs, "metrics-app-max-pvcs", 500, "most PVCs of one application in the lookup window; more answer 422") fs.IntVar(&o.limiter.MaxConcurrentPerUser, "max-concurrent-per-user", 10, "concurrent non-tail requests per user; 0 disables") diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index 80d12cbe2..835d29b84 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -94,20 +94,21 @@ func applicationSelectors(ns, kind, name string, pods, pvcs []string) []string { } // applicationScope looks the application's pods and PVCs up in store over -// the request window, reaching back by the query's own lookback up to the -// configured limit and forward to the latest sample a negative offset -// reads. A failed lookup, a list over its cap, or a query whose latest -// sample is unknown fails the request, because a successful answer would -// silently miss the application's series. +// the samples the request reads: back by the query's own lookback, and +// forward to the latest sample a negative offset reads. A query that reads +// further back than the configured lookback, or whose earliest or latest +// sample is unknown, a failed lookup, and a list over its cap all fail the +// request, because a successful answer would silently miss the +// application's series. func (s *Server) applicationScope(ctx context.Context, r *http.Request, store string, req metricsRequest, ns, kind, name string) ([]string, *HTTPError) { if req.last.IsZero() { return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ other than start() or end(), or an offset that is not a plain duration); use the namespace root") } lim := s.cfg.Metrics.App - start := req.from.Add(-lim.Lookback) - if req.start.After(start) { - start = req.start + if req.start.IsZero() || req.start.Before(req.from.Add(-lim.Lookback)) { + return nil, errorf(http.StatusUnprocessableEntity, "the query reads further back than the application scope looks up pods (%s before the window, or an unknown lookback); use a shorter window or the namespace root", lim.Lookback) } + start := req.start var lists [2][]string for i, l := range []struct { label, metric string diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 53216e575..d3dcb683a 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -316,17 +316,30 @@ func TestServerApplicationNarrowedGrant(t *testing.T) { } } -// An expression whose lookback cannot be bounded looks pods up from the -// configured lookback before the window. -func TestServerApplicationLookupUnknownLookback(t *testing.T) { - h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) - q := url.Values{"query": {"up @ end()"}, "time": {unix(testNow)}} +// TestServerApplicationLookbackBeyondCap: the lookup reaches back at most +// --metrics-app-lookback, so a query that reads further back, or whose +// lookback is unknown, would miss pods that existed only earlier; it is +// refused rather than answered without their series. +func TestServerApplicationLookbackBeyondCap(t *testing.T) { + cfg := Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 100, MaxPVCs: 100}}} + h := newPolicyHarness(t, cfg, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + for _, query := range []string{"rate(x[7d])", "x[30d] @ end()", "rate(x[5i])"} { + q := url.Values{"query": {query}, "time": {unix(testNow)}} + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusUnprocessableEntity { + t.Errorf("%s answered %d, want 422", query, got) + } + } + if n := len(h.requests()); n != 0 { + t.Fatalf("upstream saw %d requests for refused queries", n) + } + q := url.Values{"query": {"rate(x[5m])"}, "time": {unix(testNow)}} if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusOK { - t.Fatalf("status = %d", got) + t.Fatalf("rate(x[5m]) answered %d, want 200", got) } r := h.requests()[0] - if want := formatTime(testNow.Add(-24 * time.Hour)); r.path != "/api/v1/label/pod/values" || r.form.Get("start") != want { - t.Fatalf("lookup %s start = %s, want %s", r.path, r.form.Get("start"), want) + want := formatTime(buildMetrics(t, "api/v1/query", q).start) + if r.path != "/api/v1/label/pod/values" || r.form.Get("start") != want { + t.Fatalf("lookup %s start = %s, want the query's earliest read %s", r.path, r.form.Get("start"), want) } }