diff --git a/internal/observabilityproxy/metricslookback.go b/internal/observabilityproxy/metricslookback.go index a1de3ca32..ed7e9d166 100644 --- a/internal/observabilityproxy/metricslookback.go +++ b/internal/observabilityproxy/metricslookback.go @@ -66,12 +66,15 @@ func queryStart(query string, start time.Time, step time.Duration) time.Time { // queryLookback is an upper bound on how far before its evaluation window // a query reads, from a lexical scan that never parses the query. Every -// [...] and offset counts, including ones inside quoted label values: an -// extra hit only makes the bound larger or unknown, never smaller. Windows -// and offsets are summed rather than maxed, which also covers nesting. +// [...], offset and "(" counts, including ones inside quoted label values: +// an extra hit only makes the bound larger or unknown, never smaller. +// Windows and offsets are summed rather than maxed, which covers nesting; +// WITH is refused because a template applies one window several times. // vmselect starts a rollup max(window, step) plus the silence interval // before the first point, a subquery window+step+silence before it and -// aligns both down to a step; each step term uses the largest step seen. +// aligns both down to a step. A rollup function or offset over anything +// but a selector is an implicit subquery (getRollupExprArg), so every "(" +// is counted as one. Each step term uses the largest step seen. func queryLookback(query string, step time.Duration) (time.Duration, bool) { if strings.Contains(query, "@") { return 0, false @@ -117,7 +120,7 @@ func queryLookback(query string, step time.Duration) (time.Duration, bool) { } k += i i = k + len("offset") - if k > 0 && continuesIdent(lower[k-1], false) || i < len(lower) && continuesIdent(lower[i], true) { + if !isKeyword(lower, k, i) { continue } arg := strings.TrimLeft(lower[i:], " \t\n\r") @@ -126,7 +129,7 @@ func queryLookback(query string, step time.Duration) (time.Duration, bool) { arg = strings.TrimLeft(arg[1:], " \t\n\r") } end := strings.IndexFunc(arg, func(r rune) bool { - return !(r >= '0' && r <= '9' || r >= 'a' && r <= 'z' || r == '.') + return !(r >= '0' && r <= '9' || r >= 'a' && r <= 'z' || r == '.' || r == '_') }) if end < 0 { end = len(arg) @@ -139,7 +142,10 @@ func queryLookback(query string, step time.Duration) (time.Duration, bool) { sum += d } } - n := float64(subqueries) + if hasKeyword(lower, "with") { + return 0, false + } + n := float64(subqueries + strings.Count(query, "(")) sum += (2+3*n)*maxStep + (1+n)*float64(silenceInterval) if sum > float64(maxMetricsDuration) { return 0, false @@ -147,6 +153,26 @@ func queryLookback(query string, step time.Duration) (time.Duration, bool) { return time.Duration(sum), true } +// isKeyword reports whether lower[start:end] stands on its own rather than +// inside an identifier. +func isKeyword(lower string, start, end int) bool { + return !(start > 0 && continuesIdent(lower[start-1], false) || end < len(lower) && continuesIdent(lower[end], true)) +} + +func hasKeyword(lower, word string) bool { + for i := 0; ; { + k := strings.Index(lower[i:], word) + if k < 0 { + return false + } + k += i + i = k + len(word) + if isKeyword(lower, k, i) { + return true + } + } +} + // continuesIdent reports whether c, next to "offset", makes it part of an // identifier rather than the keyword. Digits and dots only continue an // identifier after it, so before it they still count as a keyword hit. diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go index 4b034c661..4a8565564 100644 --- a/internal/observabilityproxy/metricsparams_test.go +++ b/internal/observabilityproxy/metricsparams_test.go @@ -265,18 +265,19 @@ func TestQueryLookback(t *testing.T) { want time.Duration }{ {`up`, 5 * time.Minute, 15 * time.Minute}, - {`increase(x[7d])`, 5 * time.Minute, 7*24*time.Hour + 15*time.Minute}, - {`rate(x[300])`, time.Minute, 12 * time.Minute}, - {`rate(x[ 1h30m ])`, time.Minute, 97 * time.Minute}, - {`max_over_time(x[1h:5m])`, time.Minute, 95 * time.Minute}, - {`max_over_time(x[1h:])`, time.Minute, 75 * time.Minute}, - {`max_over_time(max_over_time(rate(x[5m])[1h:1m])[1d:1h])`, time.Minute, 33*time.Hour + 20*time.Minute}, + {`increase(x[7d])`, 5 * time.Minute, 7*24*time.Hour + 35*time.Minute}, + {`rate(x[300])`, time.Minute, 20 * time.Minute}, + {`rate(x[ 1h30m ])`, time.Minute, 105 * time.Minute}, + {`max_over_time(x[1h:5m])`, time.Minute, 115 * time.Minute}, + {`max_over_time(x[1h:])`, time.Minute, 83 * time.Minute}, + {`max_over_time(max_over_time(rate(x[5m])[1h:1m])[1d:1h])`, time.Minute, 42*time.Hour + 35*time.Minute}, + {`rate(rate(x))`, time.Hour, 2*(3*time.Hour+5*time.Minute) + 2*time.Hour + 5*time.Minute}, {`x offset 10d`, 5 * time.Minute, 10*24*time.Hour + 15*time.Minute}, - {`sum(x OFFSET 2h)`, 5 * time.Minute, 135 * time.Minute}, - {`rate(x[5m] offset 1h)`, time.Minute, 72 * time.Minute}, + {`sum(x OFFSET 2h)`, 5 * time.Minute, 155 * time.Minute}, + {`rate(x[5m] offset 1h)`, time.Minute, 80 * time.Minute}, {`x offset -1h`, 5 * time.Minute, 15 * time.Minute}, {`x offset - 1h`, 5 * time.Minute, 15 * time.Minute}, - {`node_offset_seconds + offset_total`, 5 * time.Minute, 15 * time.Minute}, + {`node_offset_seconds + offset_total + with_x`, 5 * time.Minute, 15 * time.Minute}, {`x{a="[1h]"}`, 5 * time.Minute, 75 * time.Minute}, {`x{a="]"}[7d]`, 5 * time.Minute, 7*24*time.Hour + 15*time.Minute}, } @@ -310,6 +311,9 @@ func TestQueryLookbackUnknown(t *testing.T) { `x{path="/offset/1"}`, `x[99999999999y]`, `x offset 99999999999y`, + `x offset 3_000_000`, + `WITH (f(q) = max_over_time(q[2d:1h])) f(f(x))`, + `with (s(m) = m offset 2d) s(s(x))`, } { t.Run(q, func(t *testing.T) { if got, ok := queryLookback(q, 5*time.Minute); ok { @@ -338,8 +342,11 @@ func TestMetricsStoreByLookback(t *testing.T) { {"instant at start", "api/v1/query", url.Values{"query": {"x @ start()"}}, "longterm"}, {"instant step window", "api/v1/query", url.Values{"query": {"rate(x[5i])"}}, "longterm"}, {"instant step past retention", "api/v1/query", url.Values{"query": {"up"}, "step": {"259200"}}, "longterm"}, - {"instant latency offset at the edge", "api/v1/query", url.Values{"query": {"default_rollup(x[71h54m59s])"}, "step": {"0.001"}}, "longterm"}, + {"instant latency offset at the edge", "api/v1/query", url.Values{"query": {"x[71h54m59s]"}, "step": {"0.001"}}, "longterm"}, {"instant subquery past retention", "api/v1/query", url.Values{"query": {"max_over_time(rate(x[5m])[4d:1h])"}}, "longterm"}, + {"instant offset with underscores", "api/v1/query", url.Values{"query": {"x offset 3_000_000"}}, "longterm"}, + {"instant WITH template", "api/v1/query", url.Values{"query": {"WITH (f(q) = max_over_time(q[2d:1h])) f(f(x))"}}, "longterm"}, + {"instant implicit subqueries", "api/v1/query", url.Values{"query": {"rate(rate(rate(rate(rate(rate(x))))))"}, "step": {"43200"}}, "longterm"}, {"quoted at sign", "api/v1/query", url.Values{"query": {`x{a="a@b"}`}}, "longterm"}, {"quoted open bracket hides window", "api/v1/query", url.Values{"query": {`rate(x{a="[5m"}[7d])`}}, "longterm"}, {"quoted close bracket before window", "api/v1/query", url.Values{"query": {`rate(x{a="]"}[7d])`}}, "longterm"},