fix(observability-proxy): refuse app queries that read past the lookup

The pod and PVC lookup never started earlier than --metrics-app-lookback
before the window. A query that read further back, such as rate(x[7d])
with the 24h default, or whose lookback is unknown (an @ modifier, WITH,
a step-multiple window), answered 200 without the series of pods that
existed only before the lookup start.

Such queries now answer 422 on the application root, matching how a
query whose latest sample cannot be covered is handled; the namespace
root has no lookup and still serves them.

Assisted-by: LLM
Signed-off-by: ZverGuy <maximbel2003@gmail.com>
This commit is contained in:
ZverGuy committed 2026-11-10 04:03:24 +07:00
1 parent bc823b704e
commit 84665a42a7
3 files changed
+31 -17

No files matched your search

+1 -1
View File
@@ -94,7 +94,7 @@ func parseFlags(args []string) (options, error) {
fs.DurationVar(&o.metrics.Limits.MaxRange, "metrics-max-range", 30*24*time.Hour, "longest [start, end] and step a metrics request may use")
fs.IntVar(&o.metrics.Limits.MaxPoints, "metrics-max-points", 11000, "most points per series a range query may return")
fs.DurationVar(&o.metrics.Limits.MaxTimeout, "metrics-max-timeout", 30*time.Second, "timeout forwarded to vmselect; a longer client timeout is clamped; must be below --request-timeout when that is set")
fs.DurationVar(&o.metrics.App.Lookback, "metrics-app-lookback", 24*time.Hour, "how far before the window the application root looks up the application's pods and PVCs at most")
fs.DurationVar(&o.metrics.App.Lookback, "metrics-app-lookback", 24*time.Hour, "how far before the window the application root looks up the application's pods and PVCs; a query reading further back answers 422")
fs.IntVar(&o.metrics.App.MaxPods, "metrics-app-max-pods", 500, "most pods of one application in the lookup window; more answer 422")
fs.IntVar(&o.metrics.App.MaxPVCs, "metrics-app-max-pvcs", 500, "most PVCs of one application in the lookup window; more answer 422")
fs.IntVar(&o.limiter.MaxConcurrentPerUser, "max-concurrent-per-user", 10, "concurrent non-tail requests per user; 0 disables")
+9 -8
View File
@@ -94,20 +94,21 @@ func applicationSelectors(ns, kind, name string, pods, pvcs []string) []string {
}
// applicationScope looks the application's pods and PVCs up in store over
// the request window, reaching back by the query's own lookback up to the
// configured limit and forward to the latest sample a negative offset
// reads. A failed lookup, a list over its cap, or a query whose latest
// sample is unknown fails the request, because a successful answer would
// silently miss the application's series.
// the samples the request reads: back by the query's own lookback, and
// forward to the latest sample a negative offset reads. A query that reads
// further back than the configured lookback, or whose earliest or latest
// sample is unknown, a failed lookup, and a list over its cap all fail the
// request, because a successful answer would silently miss the
// application's series.
func (s *Server) applicationScope(ctx context.Context, r *http.Request, store string, req metricsRequest, ns, kind, name string) ([]string, *HTTPError) {
if req.last.IsZero() {
return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ other than start() or end(), or an offset that is not a plain duration); use the namespace root")
}
lim := s.cfg.Metrics.App
start := req.from.Add(-lim.Lookback)
if req.start.After(start) {
start = req.start
if req.start.IsZero() || req.start.Before(req.from.Add(-lim.Lookback)) {
return nil, errorf(http.StatusUnprocessableEntity, "the query reads further back than the application scope looks up pods (%s before the window, or an unknown lookback); use a shorter window or the namespace root", lim.Lookback)
}
start := req.start
var lists [2][]string
for i, l := range []struct {
label, metric string
+21 -8
View File
@@ -316,17 +316,30 @@ func TestServerApplicationNarrowedGrant(t *testing.T) {
}
}
// An expression whose lookback cannot be bounded looks pods up from the
// configured lookback before the window.
func TestServerApplicationLookupUnknownLookback(t *testing.T) {
h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil)
q := url.Values{"query": {"up @ end()"}, "time": {unix(testNow)}}
// TestServerApplicationLookbackBeyondCap: the lookup reaches back at most
// --metrics-app-lookback, so a query that reads further back, or whose
// lookback is unknown, would miss pods that existed only earlier; it is
// refused rather than answered without their series.
func TestServerApplicationLookbackBeyondCap(t *testing.T) {
cfg := Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 100, MaxPVCs: 100}}}
h := newPolicyHarness(t, cfg, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil)
for _, query := range []string{"rate(x[7d])", "x[30d] @ end()", "rate(x[5i])"} {
q := url.Values{"query": {query}, "time": {unix(testNow)}}
if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusUnprocessableEntity {
t.Errorf("%s answered %d, want 422", query, got)
}
}
if n := len(h.requests()); n != 0 {
t.Fatalf("upstream saw %d requests for refused queries", n)
}
q := url.Values{"query": {"rate(x[5m])"}, "time": {unix(testNow)}}
if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusOK {
t.Fatalf("status = %d", got)
t.Fatalf("rate(x[5m]) answered %d, want 200", got)
}
r := h.requests()[0]
if want := formatTime(testNow.Add(-24 * time.Hour)); r.path != "/api/v1/label/pod/values" || r.form.Get("start") != want {
t.Fatalf("lookup %s start = %s, want %s", r.path, r.form.Get("start"), want)
want := formatTime(buildMetrics(t, "api/v1/query", q).start)
if r.path != "/api/v1/label/pod/values" || r.form.Get("start") != want {
t.Fatalf("lookup %s start = %s, want the query's earliest read %s", r.path, r.form.Get("start"), want)
}
}