From 560e4d7ba5c2bb7123fd1e1e618b1995fdaf3578 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 05:40:06 +0700 Subject: [PATCH] fix(observability-proxy): count the query lookback in the store choice The short-term store was picked from the request window alone, so an instant increase(x[7d]) or x offset 10d went to the store that keeps 72h and silently returned a wrong number. A lexical scan of the query now bounds how far back it reads: the sum of every [...] window and subquery step, positive offsets, and the step and silence interval vmselect reads before each rollup. An @ modifier, step-multiple units or any duration the scan cannot read make the bound unknown and pick the long-term store, which receives the same writes. The scan sees quoted label values too, so a false hit only moves a query to the long-term store. The query is still forwarded unchanged. Assisted-by: LLM Signed-off-by: ZverGuy --- .../observabilityproxy/metricslookback.go | 172 ++++++++++++++++++ internal/observabilityproxy/metricsparams.go | 15 +- .../observabilityproxy/metricsparams_test.go | 109 ++++++++++- 3 files changed, 288 insertions(+), 8 deletions(-) create mode 100644 internal/observabilityproxy/metricslookback.go diff --git a/internal/observabilityproxy/metricslookback.go b/internal/observabilityproxy/metricslookback.go new file mode 100644 index 000000000..17a586bb9 --- /dev/null +++ b/internal/observabilityproxy/metricslookback.go @@ -0,0 +1,172 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "regexp" + "strconv" + "strings" + "time" +) + +// instantDefaultStep is the step vmselect uses for an instant query without +// one (defaultStep in app/vmselect/prometheus); it is also the implicit +// rollup window of a bare selector. +const instantDefaultStep = 5 * time.Minute + +// silenceInterval is how far before a rollup or subquery window vmselect +// starts reading (maxSilenceInterval in app/vmselect/promql: the value of +// -search.minStalenessInterval, 5m when unset; cozystack does not set it). +const silenceInterval = 5 * time.Minute + +// queryDurationPart is the subset of the MetricsQL duration grammar +// (scanSingleDuration in metricsql) the scan understands. Step multiples +// (5i, $__interval), negative parts and upper-case units are left out on +// purpose: anything else makes the bound unknown. +var ( + queryDurationPart = regexp.MustCompile(`(\d+(?:\.\d+)?)(ms|s|m|h|d|w|y)`) + queryDuration = regexp.MustCompile(`^(?:\d+(?:\.\d+)?(?:ms|s|m|h|d|w|y))+$`) + querySeconds = regexp.MustCompile(`^\d+(?:\.\d+)?$`) +) + +var queryDurationUnits = map[string]time.Duration{ + "ms": time.Millisecond, "s": time.Second, "m": time.Minute, "h": time.Hour, + "d": 24 * time.Hour, "w": 7 * 24 * time.Hour, "y": 365 * 24 * time.Hour, +} + +// queryStart is the earliest instant a query evaluated from start may read; +// zero when the bound is unknown. +func queryStart(query string, start time.Time, step time.Duration) time.Time { + lookback, ok := queryLookback(query, step) + if !ok { + return time.Time{} + } + return start.Add(-lookback) +} + +// queryLookback is an upper bound on how far before its evaluation window +// a query reads, from a lexical scan that never parses the query. Every +// [...] and offset counts, including ones inside quoted label values: an +// extra hit only makes the bound larger or unknown, never smaller. Windows +// and offsets are summed rather than maxed, which also covers nesting. +// vmselect starts a rollup max(window, step) plus the silence interval +// before the first point, a subquery window+step+silence before it and +// aligns both down to a step; each step term uses the largest step seen. +func queryLookback(query string, step time.Duration) (time.Duration, bool) { + if strings.Contains(query, "@") { + return 0, false + } + var sum float64 + subqueries := 0 + maxStep := float64(step) + for rest := query; ; { + open := strings.IndexByte(rest, '[') + if open < 0 { + break + } + closing := strings.IndexByte(rest[open:], ']') + if closing < 0 { + return 0, false + } + window, subStep, isSubquery := strings.Cut(rest[open+1:open+closing], ":") + rest = rest[open+closing+1:] + if isSubquery { + subqueries++ + if strings.TrimSpace(window) == "" { + window = "0s" + } + if strings.TrimSpace(subStep) != "" { + d, ok := parseQueryDuration(subStep) + if !ok { + return 0, false + } + maxStep = max(maxStep, d) + } + } + d, ok := parseQueryDuration(window) + if !ok { + return 0, false + } + sum += d + } + lower := strings.ToLower(query) + for i := 0; ; { + k := strings.Index(lower[i:], "offset") + if k < 0 { + break + } + k += i + i = k + len("offset") + if k > 0 && continuesIdent(lower[k-1], false) || i < len(lower) && continuesIdent(lower[i], true) { + continue + } + arg := strings.TrimLeft(lower[i:], " \t\n\r") + negative := strings.HasPrefix(arg, "-") + if negative { + arg = strings.TrimLeft(arg[1:], " \t\n\r") + } + end := strings.IndexFunc(arg, func(r rune) bool { + return !(r >= '0' && r <= '9' || r >= 'a' && r <= 'z' || r == '.') + }) + if end < 0 { + end = len(arg) + } + d, ok := parseQueryDuration(arg[:end]) + if !ok { + return 0, false + } + if !negative { + sum += d + } + } + n := float64(subqueries) + sum += (2+3*n)*maxStep + (1+n)*float64(silenceInterval) + if sum > float64(maxMetricsDuration) { + return 0, false + } + return time.Duration(sum), true +} + +// continuesIdent reports whether c, next to "offset", makes it part of an +// identifier rather than the keyword. Digits and dots only continue an +// identifier after it, so before it they still count as a keyword hit. +func continuesIdent(c byte, after bool) bool { + return c >= 'a' && c <= 'z' || c == '_' || c == ':' || c >= 0x80 || + after && (c >= '0' && c <= '9' || c == '.' || c == '\\') +} + +// parseQueryDuration returns nanoseconds as a float so that a sum of huge +// durations cannot wrap; the caller compares it against the bound. +func parseQueryDuration(s string) (float64, bool) { + s = strings.TrimSpace(s) + if querySeconds.MatchString(s) { + f, err := strconv.ParseFloat(s, 64) + return f * float64(time.Second), err == nil + } + if !queryDuration.MatchString(s) { + return 0, false + } + var d float64 + for _, m := range queryDurationPart.FindAllStringSubmatch(s, -1) { + f, err := strconv.ParseFloat(m[1], 64) + if err != nil { + return 0, false + } + d += f * float64(queryDurationUnits[m[2]]) + } + return d, true +} diff --git a/internal/observabilityproxy/metricsparams.go b/internal/observabilityproxy/metricsparams.go index fe9c8aafd..117f1b573 100644 --- a/internal/observabilityproxy/metricsparams.go +++ b/internal/observabilityproxy/metricsparams.go @@ -69,7 +69,8 @@ var labelName = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`) type metricsRequest struct { path string params url.Values - // start is the earliest instant the request reads, for the store choice. + // start is the earliest instant the request reads, for the store choice; + // zero when that is unknown, which picks the long-term store. start time.Time } @@ -103,22 +104,24 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti if herr := copyQuery(out, in); herr != nil { return req, herr } - req.start = now + at := now if v := in.Get("time"); v != "" { t, herr := parseMetricsTime("time", v) if herr != nil { return req, herr } - req.start = t + at = t out.Set("time", formatTime(t)) } + step := instantDefaultStep if v := in.Get("step"); v != "" { - step, herr := parseStep(v, lim) - if herr != nil { + var herr *HTTPError + if step, herr = parseStep(v, lim); herr != nil { return req, herr } out.Set("step", formatSeconds(step)) } + req.start = queryStart(out.Get("query"), at, step) return req, nil case "api/v1/query_range": @@ -141,7 +144,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti if points := end.Sub(start)/step + 1; points > time.Duration(lim.MaxPoints) { return req, errorf(http.StatusUnprocessableEntity, "%d points per series exceed the limit of %d; raise step", points, lim.MaxPoints) } - req.start = start + req.start = queryStart(out.Get("query"), start, step) out.Set("start", formatTime(start)) out.Set("end", formatTime(end)) out.Set("step", formatSeconds(step)) diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go index fc403191b..54e5acd04 100644 --- a/internal/observabilityproxy/metricsparams_test.go +++ b/internal/observabilityproxy/metricsparams_test.go @@ -224,7 +224,7 @@ func TestBuildMetricsParamsCanonicalForms(t *testing.T) { if req.params.Get("end") != "1793793600.000" || req.params.Get("start") != "1793793300.000" { t.Errorf("labels window = %s..%s, want the last 5m", req.params.Get("start"), req.params.Get("end")) } - if q := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}}); q.params.Has("time") || !q.start.Equal(testNow) { + if q := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}}); q.params.Has("time") || !q.start.Equal(testNow.Add(-15*time.Minute)) { t.Errorf("instant query without time: params %v, start %v", q.params, q.start) } } @@ -241,7 +241,8 @@ func TestMetricsStoreByRange(t *testing.T) { {"instant within retention", "api/v1/query", url.Values{"query": {"up"}, "time": {unix(testNow.Add(-71 * time.Hour))}}, "shortterm"}, {"instant past retention", "api/v1/query", url.Values{"query": {"up"}, "time": {unix(testNow.Add(-73 * time.Hour))}}, "longterm"}, {"live graph", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-6 * time.Hour))}, "end": {unix(testNow)}, "step": {"60"}}, "shortterm"}, - {"range fits exactly", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-72 * time.Hour))}, "end": {unix(testNow)}, "step": {"600"}}, "shortterm"}, + {"range fits with lookback", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-71 * time.Hour))}, "end": {unix(testNow)}, "step": {"600"}}, "shortterm"}, + {"range lookback past retention", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-72 * time.Hour))}, "end": {unix(testNow)}, "step": {"600"}}, "longterm"}, {"range past retention", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-7 * 24 * time.Hour))}, "end": {unix(testNow)}, "step": {"600"}}, "longterm"}, {"old window ending long ago", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-5 * 24 * time.Hour))}, "end": {unix(testNow.Add(-4 * 24 * time.Hour))}, "step": {"600"}}, "longterm"}, {"labels default window", "api/v1/labels", url.Values{}, "shortterm"}, @@ -256,3 +257,107 @@ func TestMetricsStoreByRange(t *testing.T) { }) } } + +func TestQueryLookback(t *testing.T) { + tests := []struct { + query string + step time.Duration + want time.Duration + }{ + {`up`, 5 * time.Minute, 15 * time.Minute}, + {`increase(x[7d])`, 5 * time.Minute, 7*24*time.Hour + 15*time.Minute}, + {`rate(x[300])`, time.Minute, 12 * time.Minute}, + {`rate(x[ 1h30m ])`, time.Minute, 97 * time.Minute}, + {`max_over_time(x[1h:5m])`, time.Minute, 95 * time.Minute}, + {`max_over_time(x[1h:])`, time.Minute, 75 * time.Minute}, + {`max_over_time(max_over_time(rate(x[5m])[1h:1m])[1d:1h])`, time.Minute, 33*time.Hour + 20*time.Minute}, + {`x offset 10d`, 5 * time.Minute, 10*24*time.Hour + 15*time.Minute}, + {`sum(x OFFSET 2h)`, 5 * time.Minute, 135 * time.Minute}, + {`rate(x[5m] offset 1h)`, time.Minute, 72 * time.Minute}, + {`x offset -1h`, 5 * time.Minute, 15 * time.Minute}, + {`x offset - 1h`, 5 * time.Minute, 15 * time.Minute}, + {`node_offset_seconds + offset_total`, 5 * time.Minute, 15 * time.Minute}, + {`x{a="[1h]"}`, 5 * time.Minute, 75 * time.Minute}, + {`x{a="]"}[7d]`, 5 * time.Minute, 7*24*time.Hour + 15*time.Minute}, + } + for _, tt := range tests { + t.Run(tt.query, func(t *testing.T) { + got, ok := queryLookback(tt.query, tt.step) + if !ok || got != tt.want { + t.Fatalf("queryLookback = %s, %t; want %s, true", got, ok, tt.want) + } + }) + } +} + +func TestQueryLookbackUnknown(t *testing.T) { + for _, q := range []string{ + `x @ 123`, + `x @ start()`, + `x{a="a@b"}`, + `rate(x[5i])`, + `rate(x[$__interval])`, + `x[1h-5m]`, + `x[7D]`, + `x[1.d]`, + `x[]`, + `rate(x[7d)`, + `x offset`, + `x offset w`, + `WITH (w = 7d) rate(x[w])`, + `x{a="["}`, + `x{a="[5m"}[7d]`, + `x{path="/offset/1"}`, + `x[99999999999y]`, + `x offset 99999999999y`, + } { + t.Run(q, func(t *testing.T) { + if got, ok := queryLookback(q, 5*time.Minute); ok { + t.Fatalf("queryLookback = %s, true; want unknown", got) + } + }) + } +} + +func TestMetricsStoreByLookback(t *testing.T) { + cfg := MetricsConfig{ShortTermStore: "shortterm", LongTermStore: "longterm", ShortTermRetention: 72 * time.Hour} + rangeFrom := func(q string, ago time.Duration, step string) url.Values { + return url.Values{"query": {q}, "start": {unix(testNow.Add(-ago))}, "end": {unix(testNow)}, "step": {step}} + } + tests := []struct { + name string + endpoint string + in url.Values + want string + }{ + {"instant short window", "api/v1/query", url.Values{"query": {"rate(x[5m])"}}, "shortterm"}, + {"instant 7d window", "api/v1/query", url.Values{"query": {"increase(x[7d])"}}, "longterm"}, + {"instant offset 10d", "api/v1/query", url.Values{"query": {"x offset 10d"}}, "longterm"}, + {"instant negative offset", "api/v1/query", url.Values{"query": {"x offset -1h"}}, "shortterm"}, + {"instant at timestamp", "api/v1/query", url.Values{"query": {"x @ 123"}}, "longterm"}, + {"instant at start", "api/v1/query", url.Values{"query": {"x @ start()"}}, "longterm"}, + {"instant step window", "api/v1/query", url.Values{"query": {"rate(x[5i])"}}, "longterm"}, + {"instant step past retention", "api/v1/query", url.Values{"query": {"up"}, "step": {"259200"}}, "longterm"}, + {"instant subquery past retention", "api/v1/query", url.Values{"query": {"max_over_time(rate(x[5m])[4d:1h])"}}, "longterm"}, + {"quoted at sign", "api/v1/query", url.Values{"query": {`x{a="a@b"}`}}, "longterm"}, + {"quoted open bracket hides window", "api/v1/query", url.Values{"query": {`rate(x{a="[5m"}[7d])`}}, "longterm"}, + {"quoted close bracket before window", "api/v1/query", url.Values{"query": {`rate(x{a="]"}[7d])`}}, "longterm"}, + {"quoted offset before offset", "api/v1/query", url.Values{"query": {`x{a="offset 1m"} offset 10d`}}, "longterm"}, + {"range short window", "api/v1/query_range", rangeFrom("rate(x[1h])", 6*time.Hour, "60"), "shortterm"}, + {"range 7d window", "api/v1/query_range", rangeFrom("increase(x[7d])", 6*time.Hour, "60"), "longterm"}, + {"range offset past retention", "api/v1/query_range", rangeFrom("x offset 3d", time.Hour, "60"), "longterm"}, + {"range lookback fits", "api/v1/query_range", rangeFrom("rate(x[5m])", 70*time.Hour, "600"), "shortterm"}, + {"range lookback crosses retention", "api/v1/query_range", rangeFrom("rate(x[5m])", 71*time.Hour+50*time.Minute, "600"), "longterm"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := buildMetrics(t, tt.endpoint, tt.in) + if got := cfg.store(req.start, testNow); got != tt.want { + t.Fatalf("store = %s, want %s", got, tt.want) + } + if q := req.params.Get("query"); q != tt.in.Get("query") { + t.Fatalf("query forwarded as %q, want it unchanged", q) + } + }) + } +}