mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-22 12:43:32 +09:00
fix(observability-proxy): reach the subquery steps in the app lookup
vmselect evaluates a subquery up to its end plus one subquery step,
rounded up to the step (evalRollupFuncWithSubquery in
app/vmselect/promql/eval.go), and range_* transforms (setLastValues)
and keep_next_value carry those later values back onto earlier points.
A series that starts after the evaluation time, such as a pod created
within that reach, can therefore appear in the answer, which the
earlier claim that only rollup_candlestick reads ahead missed. The
lookup stopped at the evaluation time and left such a pod out of a 200.
The lookahead now adds two steps per subquery level, counting every
"(" as a possible implicit subquery and using the largest step seen,
as the lookback already does. Overreaching only widens the lookup over
the application's own pods.
Assisted-by: LLM
Signed-off-by: ZverGuy <maximbel2003@gmail.com>
This commit is contained in:
1 parent
cb35399ee7
commit
84f19e14d8
4 files changed
+84
-16
No files matched your search
@@ -130,14 +130,19 @@ func queryLookback(query string, step time.Duration) (time.Duration, bool) {
|
||||
}
|
||||
|
||||
// queryLookahead is an upper bound on how far after its evaluation window
|
||||
// a query reads, from the same lexical scan. Only a negative offset reads
|
||||
// forward; the negative offsets are summed, which covers nesting. Any @
|
||||
// makes the bound unknown, as it does for queryLookback, and so does WITH,
|
||||
// which can apply one offset several times. vmselect also evaluates
|
||||
// rollup_candlestick one step ahead of the evaluation time
|
||||
// (evalRollupFuncWithoutAt), with the step of whatever subquery it sits
|
||||
// in, so it makes the bound unknown too.
|
||||
func queryLookahead(query string) (time.Duration, bool) {
|
||||
// a query reads, from the same lexical scan, with step the query's step.
|
||||
// A negative offset reads forward; the negative offsets are summed, which
|
||||
// covers nesting. vmselect also evaluates a subquery up to its end plus
|
||||
// its step, rounded up to the step (evalRollupFuncWithSubquery), and
|
||||
// range_* transforms and keep_next_value carry those later values back
|
||||
// onto earlier points, so each subquery level adds two steps; as for the
|
||||
// lookback, every "(" counts as a possible implicit subquery and each
|
||||
// term uses the largest step seen. Any @ makes the bound unknown, as it
|
||||
// does for queryLookback, and so does WITH, which can apply one offset
|
||||
// several times. vmselect also evaluates rollup_candlestick one step ahead
|
||||
// of the evaluation time (evalRollupFuncWithoutAt), so it makes the bound
|
||||
// unknown too.
|
||||
func queryLookahead(query string, step time.Duration) (time.Duration, bool) {
|
||||
if strings.Contains(query, "@") {
|
||||
return 0, false
|
||||
}
|
||||
@@ -146,7 +151,36 @@ func queryLookahead(query string) (time.Duration, bool) {
|
||||
return 0, false
|
||||
}
|
||||
_, ahead, ok := scanOffsets(lower)
|
||||
if !ok || ahead > 0 && hasKeyword(lower, "with") || ahead > float64(maxMetricsDuration) {
|
||||
if !ok || ahead > 0 && hasKeyword(lower, "with") {
|
||||
return 0, false
|
||||
}
|
||||
subqueries := 0
|
||||
maxStep := float64(step)
|
||||
for rest := query; ; {
|
||||
open := strings.IndexByte(rest, '[')
|
||||
if open < 0 {
|
||||
break
|
||||
}
|
||||
closing := strings.IndexByte(rest[open:], ']')
|
||||
if closing < 0 {
|
||||
return 0, false
|
||||
}
|
||||
_, subStep, isSubquery := strings.Cut(rest[open+1:open+closing], ":")
|
||||
rest = rest[open+closing+1:]
|
||||
if !isSubquery {
|
||||
continue
|
||||
}
|
||||
subqueries++
|
||||
if strings.TrimSpace(subStep) != "" {
|
||||
d, ok := parseQueryDuration(subStep)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
maxStep = max(maxStep, d)
|
||||
}
|
||||
}
|
||||
ahead += 2 * float64(subqueries+strings.Count(query, "(")) * maxStep
|
||||
if ahead > float64(maxMetricsDuration) {
|
||||
return 0, false
|
||||
}
|
||||
return time.Duration(ahead), true
|
||||
|
||||
@@ -146,7 +146,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti
|
||||
}
|
||||
req.start = queryStart(out.Get("query"), at.Add(-latencyOffset), step)
|
||||
req.from, req.to = at, at
|
||||
if ahead, ok := queryLookahead(out.Get("query")); ok {
|
||||
if ahead, ok := queryLookahead(out.Get("query"), step); ok {
|
||||
req.last = at.Add(ahead)
|
||||
}
|
||||
return req, nil
|
||||
@@ -174,7 +174,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti
|
||||
req.start = queryStart(out.Get("query"), start, step)
|
||||
req.from, req.to = start, end
|
||||
// vmselect rounds end up to a whole step (AdjustStartEnd).
|
||||
if ahead, ok := queryLookahead(out.Get("query")); ok {
|
||||
if ahead, ok := queryLookahead(out.Get("query"), step); ok {
|
||||
req.last = end.Add(step + ahead)
|
||||
}
|
||||
out.Set("start", formatTime(start))
|
||||
|
||||
@@ -439,7 +439,9 @@ func TestQueryLookahead(t *testing.T) {
|
||||
{`x offset 1h`, 0},
|
||||
{`x offset -1h`, time.Hour},
|
||||
{`x OFFSET - 90m`, 90 * time.Minute},
|
||||
{`rate(x[5m] offset -30m) offset -1h`, 90 * time.Minute},
|
||||
// Each "(" may be an implicit subquery, which vmselect evaluates
|
||||
// up to two steps past its end; here the step is 1m.
|
||||
{`rate(x[5m] offset -30m) offset -1h`, 92 * time.Minute},
|
||||
{`x offset -1h + y offset 2h`, time.Hour},
|
||||
{`node_offset_seconds - 1`, 0},
|
||||
// metricsql lexes a duration as one token, so a sign with spaces,
|
||||
@@ -451,7 +453,7 @@ func TestQueryLookahead(t *testing.T) {
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.query, func(t *testing.T) {
|
||||
got, ok := queryLookahead(tt.query)
|
||||
got, ok := queryLookahead(tt.query, time.Minute)
|
||||
if !ok || got != tt.want {
|
||||
t.Fatalf("queryLookahead = %s, %v; want %s", got, ok, tt.want)
|
||||
}
|
||||
@@ -481,8 +483,22 @@ func TestQueryLookahead(t *testing.T) {
|
||||
`x offset -$__interval`,
|
||||
`with (f(a) = a offset -1h) f(f(x))`,
|
||||
} {
|
||||
if got, ok := queryLookahead(q); ok {
|
||||
if got, ok := queryLookahead(q, time.Minute); ok {
|
||||
t.Errorf("queryLookahead(%q) = %s, want unknown", q, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestQueryLookaheadSubquery: vmselect evaluates a subquery up to its end
|
||||
// plus one subquery step, rounded up to the step (evalRollupFuncWithSubquery),
|
||||
// and range_last and the other range_* transforms copy the last of those
|
||||
// values back onto earlier points, so a series that starts after the
|
||||
// evaluation time can reach the answer.
|
||||
func TestQueryLookaheadSubquery(t *testing.T) {
|
||||
for _, q := range []string{`max_over_time(range_last(x)[1h:1h])`, `max_over_time(keep_next_value(x)[1h:1h])`} {
|
||||
got, ok := queryLookahead(q, time.Minute)
|
||||
if !ok || got < 2*time.Hour {
|
||||
t.Errorf("queryLookahead(%q) = %s, %v; want at least two subquery steps", q, got, ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -184,8 +184,9 @@ func TestServerApplicationMetricsScope(t *testing.T) {
|
||||
if want := formatTime(buildMetrics(t, "api/v1/query_range", hostile).start); r.form.Get("start") != want {
|
||||
t.Errorf("%s start = %s, want the query's earliest read %s", r.path, r.form.Get("start"), want)
|
||||
}
|
||||
// One step past end, which vmselect rounds up to a whole step.
|
||||
if r.form.Get("end") != formatTime(testNow.Add(time.Minute)) || r.form.Get("limit") != "101" {
|
||||
// One step past end, which vmselect rounds up to a whole step, and
|
||||
// two steps for each "(" that may be an implicit subquery.
|
||||
if r.form.Get("end") != formatTime(testNow.Add(5*time.Minute)) || r.form.Get("limit") != "101" {
|
||||
t.Errorf("%s end/limit = %s/%s", r.path, r.form.Get("end"), r.form.Get("limit"))
|
||||
}
|
||||
}
|
||||
@@ -535,3 +536,20 @@ func TestServerApplicationRefusesForwardReads(t *testing.T) {
|
||||
t.Fatalf("upstream saw %d requests for refused queries", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestServerApplicationLookupCoversSubquery: a pod created during the two
|
||||
// subquery steps past the evaluation time can show up through range_last,
|
||||
// so the lookup reaches that far.
|
||||
func TestServerApplicationLookupCoversSubquery(t *testing.T) {
|
||||
h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil)
|
||||
at := testNow.Add(-6 * time.Hour)
|
||||
q := url.Values{"query": {"max_over_time(range_last(container_cpu_usage_seconds_total)[1h:1h])"}, "time": {unix(at)}}
|
||||
if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusOK {
|
||||
t.Fatalf("status = %d", got)
|
||||
}
|
||||
r := h.requests()[0]
|
||||
end, err := time.Parse(time.RFC3339, r.form.Get("end"))
|
||||
if err != nil || r.path != "/api/v1/label/pod/values" || end.Before(at.Add(2*time.Hour)) {
|
||||
t.Fatalf("lookup %s end = %s, want at least %s", r.path, r.form.Get("end"), formatTime(at.Add(2*time.Hour)))
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user