fix(observability-proxy): reach the subquery steps in the app lookup

vmselect evaluates a subquery up to its end plus one subquery step,
rounded up to the step (evalRollupFuncWithSubquery in
app/vmselect/promql/eval.go), and range_* transforms (setLastValues)
and keep_next_value carry those later values back onto earlier points.
A series that starts after the evaluation time, such as a pod created
within that reach, can therefore appear in the answer, which the
earlier claim that only rollup_candlestick reads ahead missed. The
lookup stopped at the evaluation time and left such a pod out of a 200.

The lookahead now adds two steps per subquery level, counting every
"(" as a possible implicit subquery and using the largest step seen,
as the lookback already does. Overreaching only widens the lookup over
the application's own pods.

Assisted-by: LLM
Signed-off-by: ZverGuy <maximbel2003@gmail.com>
This commit is contained in:
ZverGuy committed 2026-11-10 04:03:24 +07:00
1 parent cb35399ee7
commit 84f19e14d8
4 files changed
+84 -16

No files matched your search

+43 -9
View File
@@ -130,14 +130,19 @@ func queryLookback(query string, step time.Duration) (time.Duration, bool) {
}
// queryLookahead is an upper bound on how far after its evaluation window
// a query reads, from the same lexical scan. Only a negative offset reads
// forward; the negative offsets are summed, which covers nesting. Any @
// makes the bound unknown, as it does for queryLookback, and so does WITH,
// which can apply one offset several times. vmselect also evaluates
// rollup_candlestick one step ahead of the evaluation time
// (evalRollupFuncWithoutAt), with the step of whatever subquery it sits
// in, so it makes the bound unknown too.
func queryLookahead(query string) (time.Duration, bool) {
// a query reads, from the same lexical scan, with step the query's step.
// A negative offset reads forward; the negative offsets are summed, which
// covers nesting. vmselect also evaluates a subquery up to its end plus
// its step, rounded up to the step (evalRollupFuncWithSubquery), and
// range_* transforms and keep_next_value carry those later values back
// onto earlier points, so each subquery level adds two steps; as for the
// lookback, every "(" counts as a possible implicit subquery and each
// term uses the largest step seen. Any @ makes the bound unknown, as it
// does for queryLookback, and so does WITH, which can apply one offset
// several times. vmselect also evaluates rollup_candlestick one step ahead
// of the evaluation time (evalRollupFuncWithoutAt), so it makes the bound
// unknown too.
func queryLookahead(query string, step time.Duration) (time.Duration, bool) {
if strings.Contains(query, "@") {
return 0, false
}
@@ -146,7 +151,36 @@ func queryLookahead(query string) (time.Duration, bool) {
return 0, false
}
_, ahead, ok := scanOffsets(lower)
if !ok || ahead > 0 && hasKeyword(lower, "with") || ahead > float64(maxMetricsDuration) {
if !ok || ahead > 0 && hasKeyword(lower, "with") {
return 0, false
}
subqueries := 0
maxStep := float64(step)
for rest := query; ; {
open := strings.IndexByte(rest, '[')
if open < 0 {
break
}
closing := strings.IndexByte(rest[open:], ']')
if closing < 0 {
return 0, false
}
_, subStep, isSubquery := strings.Cut(rest[open+1:open+closing], ":")
rest = rest[open+closing+1:]
if !isSubquery {
continue
}
subqueries++
if strings.TrimSpace(subStep) != "" {
d, ok := parseQueryDuration(subStep)
if !ok {
return 0, false
}
maxStep = max(maxStep, d)
}
}
ahead += 2 * float64(subqueries+strings.Count(query, "(")) * maxStep
if ahead > float64(maxMetricsDuration) {
return 0, false
}
return time.Duration(ahead), true
+2 -2
View File
@@ -146,7 +146,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti
}
req.start = queryStart(out.Get("query"), at.Add(-latencyOffset), step)
req.from, req.to = at, at
if ahead, ok := queryLookahead(out.Get("query")); ok {
if ahead, ok := queryLookahead(out.Get("query"), step); ok {
req.last = at.Add(ahead)
}
return req, nil
@@ -174,7 +174,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti
req.start = queryStart(out.Get("query"), start, step)
req.from, req.to = start, end
// vmselect rounds end up to a whole step (AdjustStartEnd).
if ahead, ok := queryLookahead(out.Get("query")); ok {
if ahead, ok := queryLookahead(out.Get("query"), step); ok {
req.last = end.Add(step + ahead)
}
out.Set("start", formatTime(start))
@@ -439,7 +439,9 @@ func TestQueryLookahead(t *testing.T) {
{`x offset 1h`, 0},
{`x offset -1h`, time.Hour},
{`x OFFSET - 90m`, 90 * time.Minute},
{`rate(x[5m] offset -30m) offset -1h`, 90 * time.Minute},
// Each "(" may be an implicit subquery, which vmselect evaluates
// up to two steps past its end; here the step is 1m.
{`rate(x[5m] offset -30m) offset -1h`, 92 * time.Minute},
{`x offset -1h + y offset 2h`, time.Hour},
{`node_offset_seconds - 1`, 0},
// metricsql lexes a duration as one token, so a sign with spaces,
@@ -451,7 +453,7 @@ func TestQueryLookahead(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.query, func(t *testing.T) {
got, ok := queryLookahead(tt.query)
got, ok := queryLookahead(tt.query, time.Minute)
if !ok || got != tt.want {
t.Fatalf("queryLookahead = %s, %v; want %s", got, ok, tt.want)
}
@@ -481,8 +483,22 @@ func TestQueryLookahead(t *testing.T) {
`x offset -$__interval`,
`with (f(a) = a offset -1h) f(f(x))`,
} {
if got, ok := queryLookahead(q); ok {
if got, ok := queryLookahead(q, time.Minute); ok {
t.Errorf("queryLookahead(%q) = %s, want unknown", q, got)
}
}
}
// TestQueryLookaheadSubquery: vmselect evaluates a subquery up to its end
// plus one subquery step, rounded up to the step (evalRollupFuncWithSubquery),
// and range_last and the other range_* transforms copy the last of those
// values back onto earlier points, so a series that starts after the
// evaluation time can reach the answer.
func TestQueryLookaheadSubquery(t *testing.T) {
for _, q := range []string{`max_over_time(range_last(x)[1h:1h])`, `max_over_time(keep_next_value(x)[1h:1h])`} {
got, ok := queryLookahead(q, time.Minute)
if !ok || got < 2*time.Hour {
t.Errorf("queryLookahead(%q) = %s, %v; want at least two subquery steps", q, got, ok)
}
}
}
+20 -2
View File
@@ -184,8 +184,9 @@ func TestServerApplicationMetricsScope(t *testing.T) {
if want := formatTime(buildMetrics(t, "api/v1/query_range", hostile).start); r.form.Get("start") != want {
t.Errorf("%s start = %s, want the query's earliest read %s", r.path, r.form.Get("start"), want)
}
// One step past end, which vmselect rounds up to a whole step.
if r.form.Get("end") != formatTime(testNow.Add(time.Minute)) || r.form.Get("limit") != "101" {
// One step past end, which vmselect rounds up to a whole step, and
// two steps for each "(" that may be an implicit subquery.
if r.form.Get("end") != formatTime(testNow.Add(5*time.Minute)) || r.form.Get("limit") != "101" {
t.Errorf("%s end/limit = %s/%s", r.path, r.form.Get("end"), r.form.Get("limit"))
}
}
@@ -535,3 +536,20 @@ func TestServerApplicationRefusesForwardReads(t *testing.T) {
t.Fatalf("upstream saw %d requests for refused queries", n)
}
}
// TestServerApplicationLookupCoversSubquery: a pod created during the two
// subquery steps past the evaluation time can show up through range_last,
// so the lookup reaches that far.
func TestServerApplicationLookupCoversSubquery(t *testing.T) {
h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil)
at := testNow.Add(-6 * time.Hour)
q := url.Values{"query": {"max_over_time(range_last(container_cpu_usage_seconds_total)[1h:1h])"}, "time": {unix(at)}}
if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusOK {
t.Fatalf("status = %d", got)
}
r := h.requests()[0]
end, err := time.Parse(time.RFC3339, r.form.Get("end"))
if err != nil || r.path != "/api/v1/label/pod/values" || end.Before(at.Add(2*time.Hour)) {
t.Fatalf("lookup %s end = %s, want at least %s", r.path, r.form.Get("end"), formatTime(at.Add(2*time.Hour)))
}
}