mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
fix(observability-proxy): bound implicit subqueries and WITH templates
The lookback scan assumed every window and offset applies once and
explicitly. A rollup function or offset over anything but a selector is
an implicit subquery in vmselect, so every "(" now counts as one level;
a WITH template can apply one window several times, so WITH makes the
bound unknown; and an offset written with digit separators was read as
its first digits only, so "_" now ends in an unknown bound instead.
Assisted-by: LLM
Signed-off-by: ZverGuy <maximbel2003@gmail.com>
This commit is contained in:
1 parent
2fad05b01f
commit
1ead35a37a
2 files changed
+50
-17
No files matched your search
@@ -66,12 +66,15 @@ func queryStart(query string, start time.Time, step time.Duration) time.Time {
|
||||
|
||||
// queryLookback is an upper bound on how far before its evaluation window
|
||||
// a query reads, from a lexical scan that never parses the query. Every
|
||||
// [...] and offset counts, including ones inside quoted label values: an
|
||||
// extra hit only makes the bound larger or unknown, never smaller. Windows
|
||||
// and offsets are summed rather than maxed, which also covers nesting.
|
||||
// [...], offset and "(" counts, including ones inside quoted label values:
|
||||
// an extra hit only makes the bound larger or unknown, never smaller.
|
||||
// Windows and offsets are summed rather than maxed, which covers nesting;
|
||||
// WITH is refused because a template applies one window several times.
|
||||
// vmselect starts a rollup max(window, step) plus the silence interval
|
||||
// before the first point, a subquery window+step+silence before it and
|
||||
// aligns both down to a step; each step term uses the largest step seen.
|
||||
// aligns both down to a step. A rollup function or offset over anything
|
||||
// but a selector is an implicit subquery (getRollupExprArg), so every "("
|
||||
// is counted as one. Each step term uses the largest step seen.
|
||||
func queryLookback(query string, step time.Duration) (time.Duration, bool) {
|
||||
if strings.Contains(query, "@") {
|
||||
return 0, false
|
||||
@@ -117,7 +120,7 @@ func queryLookback(query string, step time.Duration) (time.Duration, bool) {
|
||||
}
|
||||
k += i
|
||||
i = k + len("offset")
|
||||
if k > 0 && continuesIdent(lower[k-1], false) || i < len(lower) && continuesIdent(lower[i], true) {
|
||||
if !isKeyword(lower, k, i) {
|
||||
continue
|
||||
}
|
||||
arg := strings.TrimLeft(lower[i:], " \t\n\r")
|
||||
@@ -126,7 +129,7 @@ func queryLookback(query string, step time.Duration) (time.Duration, bool) {
|
||||
arg = strings.TrimLeft(arg[1:], " \t\n\r")
|
||||
}
|
||||
end := strings.IndexFunc(arg, func(r rune) bool {
|
||||
return !(r >= '0' && r <= '9' || r >= 'a' && r <= 'z' || r == '.')
|
||||
return !(r >= '0' && r <= '9' || r >= 'a' && r <= 'z' || r == '.' || r == '_')
|
||||
})
|
||||
if end < 0 {
|
||||
end = len(arg)
|
||||
@@ -139,7 +142,10 @@ func queryLookback(query string, step time.Duration) (time.Duration, bool) {
|
||||
sum += d
|
||||
}
|
||||
}
|
||||
n := float64(subqueries)
|
||||
if hasKeyword(lower, "with") {
|
||||
return 0, false
|
||||
}
|
||||
n := float64(subqueries + strings.Count(query, "("))
|
||||
sum += (2+3*n)*maxStep + (1+n)*float64(silenceInterval)
|
||||
if sum > float64(maxMetricsDuration) {
|
||||
return 0, false
|
||||
@@ -147,6 +153,26 @@ func queryLookback(query string, step time.Duration) (time.Duration, bool) {
|
||||
return time.Duration(sum), true
|
||||
}
|
||||
|
||||
// isKeyword reports whether lower[start:end] stands on its own rather than
|
||||
// inside an identifier.
|
||||
func isKeyword(lower string, start, end int) bool {
|
||||
return !(start > 0 && continuesIdent(lower[start-1], false) || end < len(lower) && continuesIdent(lower[end], true))
|
||||
}
|
||||
|
||||
func hasKeyword(lower, word string) bool {
|
||||
for i := 0; ; {
|
||||
k := strings.Index(lower[i:], word)
|
||||
if k < 0 {
|
||||
return false
|
||||
}
|
||||
k += i
|
||||
i = k + len(word)
|
||||
if isKeyword(lower, k, i) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// continuesIdent reports whether c, next to "offset", makes it part of an
|
||||
// identifier rather than the keyword. Digits and dots only continue an
|
||||
// identifier after it, so before it they still count as a keyword hit.
|
||||
|
||||
@@ -265,18 +265,19 @@ func TestQueryLookback(t *testing.T) {
|
||||
want time.Duration
|
||||
}{
|
||||
{`up`, 5 * time.Minute, 15 * time.Minute},
|
||||
{`increase(x[7d])`, 5 * time.Minute, 7*24*time.Hour + 15*time.Minute},
|
||||
{`rate(x[300])`, time.Minute, 12 * time.Minute},
|
||||
{`rate(x[ 1h30m ])`, time.Minute, 97 * time.Minute},
|
||||
{`max_over_time(x[1h:5m])`, time.Minute, 95 * time.Minute},
|
||||
{`max_over_time(x[1h:])`, time.Minute, 75 * time.Minute},
|
||||
{`max_over_time(max_over_time(rate(x[5m])[1h:1m])[1d:1h])`, time.Minute, 33*time.Hour + 20*time.Minute},
|
||||
{`increase(x[7d])`, 5 * time.Minute, 7*24*time.Hour + 35*time.Minute},
|
||||
{`rate(x[300])`, time.Minute, 20 * time.Minute},
|
||||
{`rate(x[ 1h30m ])`, time.Minute, 105 * time.Minute},
|
||||
{`max_over_time(x[1h:5m])`, time.Minute, 115 * time.Minute},
|
||||
{`max_over_time(x[1h:])`, time.Minute, 83 * time.Minute},
|
||||
{`max_over_time(max_over_time(rate(x[5m])[1h:1m])[1d:1h])`, time.Minute, 42*time.Hour + 35*time.Minute},
|
||||
{`rate(rate(x))`, time.Hour, 2*(3*time.Hour+5*time.Minute) + 2*time.Hour + 5*time.Minute},
|
||||
{`x offset 10d`, 5 * time.Minute, 10*24*time.Hour + 15*time.Minute},
|
||||
{`sum(x OFFSET 2h)`, 5 * time.Minute, 135 * time.Minute},
|
||||
{`rate(x[5m] offset 1h)`, time.Minute, 72 * time.Minute},
|
||||
{`sum(x OFFSET 2h)`, 5 * time.Minute, 155 * time.Minute},
|
||||
{`rate(x[5m] offset 1h)`, time.Minute, 80 * time.Minute},
|
||||
{`x offset -1h`, 5 * time.Minute, 15 * time.Minute},
|
||||
{`x offset - 1h`, 5 * time.Minute, 15 * time.Minute},
|
||||
{`node_offset_seconds + offset_total`, 5 * time.Minute, 15 * time.Minute},
|
||||
{`node_offset_seconds + offset_total + with_x`, 5 * time.Minute, 15 * time.Minute},
|
||||
{`x{a="[1h]"}`, 5 * time.Minute, 75 * time.Minute},
|
||||
{`x{a="]"}[7d]`, 5 * time.Minute, 7*24*time.Hour + 15*time.Minute},
|
||||
}
|
||||
@@ -310,6 +311,9 @@ func TestQueryLookbackUnknown(t *testing.T) {
|
||||
`x{path="/offset/1"}`,
|
||||
`x[99999999999y]`,
|
||||
`x offset 99999999999y`,
|
||||
`x offset 3_000_000`,
|
||||
`WITH (f(q) = max_over_time(q[2d:1h])) f(f(x))`,
|
||||
`with (s(m) = m offset 2d) s(s(x))`,
|
||||
} {
|
||||
t.Run(q, func(t *testing.T) {
|
||||
if got, ok := queryLookback(q, 5*time.Minute); ok {
|
||||
@@ -338,8 +342,11 @@ func TestMetricsStoreByLookback(t *testing.T) {
|
||||
{"instant at start", "api/v1/query", url.Values{"query": {"x @ start()"}}, "longterm"},
|
||||
{"instant step window", "api/v1/query", url.Values{"query": {"rate(x[5i])"}}, "longterm"},
|
||||
{"instant step past retention", "api/v1/query", url.Values{"query": {"up"}, "step": {"259200"}}, "longterm"},
|
||||
{"instant latency offset at the edge", "api/v1/query", url.Values{"query": {"default_rollup(x[71h54m59s])"}, "step": {"0.001"}}, "longterm"},
|
||||
{"instant latency offset at the edge", "api/v1/query", url.Values{"query": {"x[71h54m59s]"}, "step": {"0.001"}}, "longterm"},
|
||||
{"instant subquery past retention", "api/v1/query", url.Values{"query": {"max_over_time(rate(x[5m])[4d:1h])"}}, "longterm"},
|
||||
{"instant offset with underscores", "api/v1/query", url.Values{"query": {"x offset 3_000_000"}}, "longterm"},
|
||||
{"instant WITH template", "api/v1/query", url.Values{"query": {"WITH (f(q) = max_over_time(q[2d:1h])) f(f(x))"}}, "longterm"},
|
||||
{"instant implicit subqueries", "api/v1/query", url.Values{"query": {"rate(rate(rate(rate(rate(rate(x))))))"}, "step": {"43200"}}, "longterm"},
|
||||
{"quoted at sign", "api/v1/query", url.Values{"query": {`x{a="a@b"}`}}, "longterm"},
|
||||
{"quoted open bracket hides window", "api/v1/query", url.Values{"query": {`rate(x{a="[5m"}[7d])`}}, "longterm"},
|
||||
{"quoted close bracket before window", "api/v1/query", url.Values{"query": {`rate(x{a="]"}[7d])`}}, "longterm"},
|
||||
|
||||
Reference in new issue
Block a user