From cb35399ee70f49399c467bb2a4be9117b923255d Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 06:06:04 +0700 Subject: [PATCH] fix(observability-proxy): refuse rollup_candlestick on the app root vmselect evaluates rollup_candlestick one step ahead of the evaluation time, with no offset in the query: evalRollupFuncWithoutAt adds the step in effect to the window's start and end (app/vmselect/promql/ eval.go in v1.136.0-cluster, unchanged in v1.139.0). The lookahead scan returned zero for it, so the lookup stopped at the evaluation time and pods created during that step were left out of a 200. The step in effect is the subquery's when the call is nested, so rather than size it the application root answers 422. No other function reads ahead: the subquery evaluation past its end in evalRollupFuncWithSubquery is outside the outer rollup's window. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 2 +- internal/observabilityproxy/metricslookback.go | 8 +++++++- internal/observabilityproxy/metricsparams_test.go | 5 +++++ internal/observabilityproxy/server_app_test.go | 1 + 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index 6c3a9fb4f..9911abeeb 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -102,7 +102,7 @@ func applicationSelectors(ns, kind, name string, pods, pvcs []string) []string { // application's series. func (s *Server) applicationScope(ctx context.Context, r *http.Request, store string, req metricsRequest, ns, kind, name string) ([]string, *HTTPError) { if req.last.IsZero() { - return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ anywhere in the query, WITH with a negative offset, or an offset that is not a plain duration); use the namespace root") + return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ anywhere in the query, rollup_candlestick, WITH with a negative offset, or an offset that is not a plain duration); use the namespace root") } lim := s.cfg.Metrics.App if req.start.IsZero() || req.start.Before(req.from.Add(-lim.Lookback)) { diff --git a/internal/observabilityproxy/metricslookback.go b/internal/observabilityproxy/metricslookback.go index 886d9e43b..379a42be0 100644 --- a/internal/observabilityproxy/metricslookback.go +++ b/internal/observabilityproxy/metricslookback.go @@ -133,12 +133,18 @@ func queryLookback(query string, step time.Duration) (time.Duration, bool) { // a query reads, from the same lexical scan. Only a negative offset reads // forward; the negative offsets are summed, which covers nesting. Any @ // makes the bound unknown, as it does for queryLookback, and so does WITH, -// which can apply one offset several times. +// which can apply one offset several times. vmselect also evaluates +// rollup_candlestick one step ahead of the evaluation time +// (evalRollupFuncWithoutAt), with the step of whatever subquery it sits +// in, so it makes the bound unknown too. func queryLookahead(query string) (time.Duration, bool) { if strings.Contains(query, "@") { return 0, false } lower := strings.ToLower(query) + if hasKeyword(lower, "rollup_candlestick") { + return 0, false + } _, ahead, ok := scanOffsets(lower) if !ok || ahead > 0 && hasKeyword(lower, "with") || ahead > float64(maxMetricsDuration) { return 0, false diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go index 29802a220..d320333fd 100644 --- a/internal/observabilityproxy/metricsparams_test.go +++ b/internal/observabilityproxy/metricsparams_test.go @@ -473,6 +473,11 @@ func TestQueryLookahead(t *testing.T) { `x offset 1h-2h`, `x offset 1h-30m`, `rate(x[5m] offset 1h-30m)`, + // vmselect evaluates rollup_candlestick one step ahead + // (evalRollupFuncWithoutAt), whatever the offset. + `rollup_candlestick(x)`, + `ROLLUP_CANDLESTICK(x[5m])`, + `sum(rollup_candlestick(x[1h:5m]))`, `x offset -$__interval`, `with (f(a) = a offset -1h) f(f(x))`, } { diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index e4f0ca34f..3ede9f239 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -522,6 +522,7 @@ func TestServerApplicationRefusesForwardReads(t *testing.T) { for _, query := range []string{ "container_cpu_usage_seconds_total offset -1h-30m", "container_cpu_usage_seconds_total offset 1h-2h", + "rollup_candlestick(container_cpu_usage_seconds_total)", } { q := url.Values{"query": {query}, "time": {unix(testNow.Add(-2 * time.Hour))}} resp := h.do(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good", nil, nil)