fix(observability-proxy): count the query lookback in the store choice

The short-term store was picked from the request window alone, so an
instant increase(x[7d]) or x offset 10d went to the store that keeps
72h and silently returned a wrong number.

A lexical scan of the query now bounds how far back it reads: the sum
of every [...] window and subquery step, positive offsets, and the
step and silence interval vmselect reads before each rollup. An @
modifier, step-multiple units or any duration the scan cannot read
make the bound unknown and pick the long-term store, which receives
the same writes. The scan sees quoted label values too, so a false
hit only moves a query to the long-term store. The query is still
forwarded unchanged.

Assisted-by: LLM
Signed-off-by: ZverGuy <maximbel2003@gmail.com>
This commit is contained in:
ZverGuy committed 2026-11-10 03:56:18 +07:00
1 parent 19030c7a1a
commit 560e4d7ba5
3 files changed
+288 -8

No files matched your search

@@ -0,0 +1,172 @@
/*
Copyright 2026 The Cozystack Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package observabilityproxy
import (
"regexp"
"strconv"
"strings"
"time"
)
// instantDefaultStep is the step vmselect uses for an instant query without
// one (defaultStep in app/vmselect/prometheus); it is also the implicit
// rollup window of a bare selector.
const instantDefaultStep = 5 * time.Minute
// silenceInterval is how far before a rollup or subquery window vmselect
// starts reading (maxSilenceInterval in app/vmselect/promql: the value of
// -search.minStalenessInterval, 5m when unset; cozystack does not set it).
const silenceInterval = 5 * time.Minute
// queryDurationPart is the subset of the MetricsQL duration grammar
// (scanSingleDuration in metricsql) the scan understands. Step multiples
// (5i, $__interval), negative parts and upper-case units are left out on
// purpose: anything else makes the bound unknown.
var (
queryDurationPart = regexp.MustCompile(`(\d+(?:\.\d+)?)(ms|s|m|h|d|w|y)`)
queryDuration = regexp.MustCompile(`^(?:\d+(?:\.\d+)?(?:ms|s|m|h|d|w|y))+$`)
querySeconds = regexp.MustCompile(`^\d+(?:\.\d+)?$`)
)
var queryDurationUnits = map[string]time.Duration{
"ms": time.Millisecond, "s": time.Second, "m": time.Minute, "h": time.Hour,
"d": 24 * time.Hour, "w": 7 * 24 * time.Hour, "y": 365 * 24 * time.Hour,
}
// queryStart is the earliest instant a query evaluated from start may read;
// zero when the bound is unknown.
func queryStart(query string, start time.Time, step time.Duration) time.Time {
lookback, ok := queryLookback(query, step)
if !ok {
return time.Time{}
}
return start.Add(-lookback)
}
// queryLookback is an upper bound on how far before its evaluation window
// a query reads, from a lexical scan that never parses the query. Every
// [...] and offset counts, including ones inside quoted label values: an
// extra hit only makes the bound larger or unknown, never smaller. Windows
// and offsets are summed rather than maxed, which also covers nesting.
// vmselect starts a rollup max(window, step) plus the silence interval
// before the first point, a subquery window+step+silence before it and
// aligns both down to a step; each step term uses the largest step seen.
func queryLookback(query string, step time.Duration) (time.Duration, bool) {
if strings.Contains(query, "@") {
return 0, false
}
var sum float64
subqueries := 0
maxStep := float64(step)
for rest := query; ; {
open := strings.IndexByte(rest, '[')
if open < 0 {
break
}
closing := strings.IndexByte(rest[open:], ']')
if closing < 0 {
return 0, false
}
window, subStep, isSubquery := strings.Cut(rest[open+1:open+closing], ":")
rest = rest[open+closing+1:]
if isSubquery {
subqueries++
if strings.TrimSpace(window) == "" {
window = "0s"
}
if strings.TrimSpace(subStep) != "" {
d, ok := parseQueryDuration(subStep)
if !ok {
return 0, false
}
maxStep = max(maxStep, d)
}
}
d, ok := parseQueryDuration(window)
if !ok {
return 0, false
}
sum += d
}
lower := strings.ToLower(query)
for i := 0; ; {
k := strings.Index(lower[i:], "offset")
if k < 0 {
break
}
k += i
i = k + len("offset")
if k > 0 && continuesIdent(lower[k-1], false) || i < len(lower) && continuesIdent(lower[i], true) {
continue
}
arg := strings.TrimLeft(lower[i:], " \t\n\r")
negative := strings.HasPrefix(arg, "-")
if negative {
arg = strings.TrimLeft(arg[1:], " \t\n\r")
}
end := strings.IndexFunc(arg, func(r rune) bool {
return !(r >= '0' && r <= '9' || r >= 'a' && r <= 'z' || r == '.')
})
if end < 0 {
end = len(arg)
}
d, ok := parseQueryDuration(arg[:end])
if !ok {
return 0, false
}
if !negative {
sum += d
}
}
n := float64(subqueries)
sum += (2+3*n)*maxStep + (1+n)*float64(silenceInterval)
if sum > float64(maxMetricsDuration) {
return 0, false
}
return time.Duration(sum), true
}
// continuesIdent reports whether c, next to "offset", makes it part of an
// identifier rather than the keyword. Digits and dots only continue an
// identifier after it, so before it they still count as a keyword hit.
func continuesIdent(c byte, after bool) bool {
return c >= 'a' && c <= 'z' || c == '_' || c == ':' || c >= 0x80 ||
after && (c >= '0' && c <= '9' || c == '.' || c == '\\')
}
// parseQueryDuration returns nanoseconds as a float so that a sum of huge
// durations cannot wrap; the caller compares it against the bound.
func parseQueryDuration(s string) (float64, bool) {
s = strings.TrimSpace(s)
if querySeconds.MatchString(s) {
f, err := strconv.ParseFloat(s, 64)
return f * float64(time.Second), err == nil
}
if !queryDuration.MatchString(s) {
return 0, false
}
var d float64
for _, m := range queryDurationPart.FindAllStringSubmatch(s, -1) {
f, err := strconv.ParseFloat(m[1], 64)
if err != nil {
return 0, false
}
d += f * float64(queryDurationUnits[m[2]])
}
return d, true
}
+9 -6
View File
@@ -69,7 +69,8 @@ var labelName = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`)
type metricsRequest struct {
path string
params url.Values
// start is the earliest instant the request reads, for the store choice.
// start is the earliest instant the request reads, for the store choice;
// zero when that is unknown, which picks the long-term store.
start time.Time
}
@@ -103,22 +104,24 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti
if herr := copyQuery(out, in); herr != nil {
return req, herr
}
req.start = now
at := now
if v := in.Get("time"); v != "" {
t, herr := parseMetricsTime("time", v)
if herr != nil {
return req, herr
}
req.start = t
at = t
out.Set("time", formatTime(t))
}
step := instantDefaultStep
if v := in.Get("step"); v != "" {
step, herr := parseStep(v, lim)
if herr != nil {
var herr *HTTPError
if step, herr = parseStep(v, lim); herr != nil {
return req, herr
}
out.Set("step", formatSeconds(step))
}
req.start = queryStart(out.Get("query"), at, step)
return req, nil
case "api/v1/query_range":
@@ -141,7 +144,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti
if points := end.Sub(start)/step + 1; points > time.Duration(lim.MaxPoints) {
return req, errorf(http.StatusUnprocessableEntity, "%d points per series exceed the limit of %d; raise step", points, lim.MaxPoints)
}
req.start = start
req.start = queryStart(out.Get("query"), start, step)
out.Set("start", formatTime(start))
out.Set("end", formatTime(end))
out.Set("step", formatSeconds(step))
@@ -224,7 +224,7 @@ func TestBuildMetricsParamsCanonicalForms(t *testing.T) {
if req.params.Get("end") != "1793793600.000" || req.params.Get("start") != "1793793300.000" {
t.Errorf("labels window = %s..%s, want the last 5m", req.params.Get("start"), req.params.Get("end"))
}
if q := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}}); q.params.Has("time") || !q.start.Equal(testNow) {
if q := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}}); q.params.Has("time") || !q.start.Equal(testNow.Add(-15*time.Minute)) {
t.Errorf("instant query without time: params %v, start %v", q.params, q.start)
}
}
@@ -241,7 +241,8 @@ func TestMetricsStoreByRange(t *testing.T) {
{"instant within retention", "api/v1/query", url.Values{"query": {"up"}, "time": {unix(testNow.Add(-71 * time.Hour))}}, "shortterm"},
{"instant past retention", "api/v1/query", url.Values{"query": {"up"}, "time": {unix(testNow.Add(-73 * time.Hour))}}, "longterm"},
{"live graph", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-6 * time.Hour))}, "end": {unix(testNow)}, "step": {"60"}}, "shortterm"},
{"range fits exactly", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-72 * time.Hour))}, "end": {unix(testNow)}, "step": {"600"}}, "shortterm"},
{"range fits with lookback", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-71 * time.Hour))}, "end": {unix(testNow)}, "step": {"600"}}, "shortterm"},
{"range lookback past retention", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-72 * time.Hour))}, "end": {unix(testNow)}, "step": {"600"}}, "longterm"},
{"range past retention", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-7 * 24 * time.Hour))}, "end": {unix(testNow)}, "step": {"600"}}, "longterm"},
{"old window ending long ago", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-5 * 24 * time.Hour))}, "end": {unix(testNow.Add(-4 * 24 * time.Hour))}, "step": {"600"}}, "longterm"},
{"labels default window", "api/v1/labels", url.Values{}, "shortterm"},
@@ -256,3 +257,107 @@ func TestMetricsStoreByRange(t *testing.T) {
})
}
}
func TestQueryLookback(t *testing.T) {
tests := []struct {
query string
step time.Duration
want time.Duration
}{
{`up`, 5 * time.Minute, 15 * time.Minute},
{`increase(x[7d])`, 5 * time.Minute, 7*24*time.Hour + 15*time.Minute},
{`rate(x[300])`, time.Minute, 12 * time.Minute},
{`rate(x[ 1h30m ])`, time.Minute, 97 * time.Minute},
{`max_over_time(x[1h:5m])`, time.Minute, 95 * time.Minute},
{`max_over_time(x[1h:])`, time.Minute, 75 * time.Minute},
{`max_over_time(max_over_time(rate(x[5m])[1h:1m])[1d:1h])`, time.Minute, 33*time.Hour + 20*time.Minute},
{`x offset 10d`, 5 * time.Minute, 10*24*time.Hour + 15*time.Minute},
{`sum(x OFFSET 2h)`, 5 * time.Minute, 135 * time.Minute},
{`rate(x[5m] offset 1h)`, time.Minute, 72 * time.Minute},
{`x offset -1h`, 5 * time.Minute, 15 * time.Minute},
{`x offset - 1h`, 5 * time.Minute, 15 * time.Minute},
{`node_offset_seconds + offset_total`, 5 * time.Minute, 15 * time.Minute},
{`x{a="[1h]"}`, 5 * time.Minute, 75 * time.Minute},
{`x{a="]"}[7d]`, 5 * time.Minute, 7*24*time.Hour + 15*time.Minute},
}
for _, tt := range tests {
t.Run(tt.query, func(t *testing.T) {
got, ok := queryLookback(tt.query, tt.step)
if !ok || got != tt.want {
t.Fatalf("queryLookback = %s, %t; want %s, true", got, ok, tt.want)
}
})
}
}
func TestQueryLookbackUnknown(t *testing.T) {
for _, q := range []string{
`x @ 123`,
`x @ start()`,
`x{a="a@b"}`,
`rate(x[5i])`,
`rate(x[$__interval])`,
`x[1h-5m]`,
`x[7D]`,
`x[1.d]`,
`x[]`,
`rate(x[7d)`,
`x offset`,
`x offset w`,
`WITH (w = 7d) rate(x[w])`,
`x{a="["}`,
`x{a="[5m"}[7d]`,
`x{path="/offset/1"}`,
`x[99999999999y]`,
`x offset 99999999999y`,
} {
t.Run(q, func(t *testing.T) {
if got, ok := queryLookback(q, 5*time.Minute); ok {
t.Fatalf("queryLookback = %s, true; want unknown", got)
}
})
}
}
func TestMetricsStoreByLookback(t *testing.T) {
cfg := MetricsConfig{ShortTermStore: "shortterm", LongTermStore: "longterm", ShortTermRetention: 72 * time.Hour}
rangeFrom := func(q string, ago time.Duration, step string) url.Values {
return url.Values{"query": {q}, "start": {unix(testNow.Add(-ago))}, "end": {unix(testNow)}, "step": {step}}
}
tests := []struct {
name string
endpoint string
in url.Values
want string
}{
{"instant short window", "api/v1/query", url.Values{"query": {"rate(x[5m])"}}, "shortterm"},
{"instant 7d window", "api/v1/query", url.Values{"query": {"increase(x[7d])"}}, "longterm"},
{"instant offset 10d", "api/v1/query", url.Values{"query": {"x offset 10d"}}, "longterm"},
{"instant negative offset", "api/v1/query", url.Values{"query": {"x offset -1h"}}, "shortterm"},
{"instant at timestamp", "api/v1/query", url.Values{"query": {"x @ 123"}}, "longterm"},
{"instant at start", "api/v1/query", url.Values{"query": {"x @ start()"}}, "longterm"},
{"instant step window", "api/v1/query", url.Values{"query": {"rate(x[5i])"}}, "longterm"},
{"instant step past retention", "api/v1/query", url.Values{"query": {"up"}, "step": {"259200"}}, "longterm"},
{"instant subquery past retention", "api/v1/query", url.Values{"query": {"max_over_time(rate(x[5m])[4d:1h])"}}, "longterm"},
{"quoted at sign", "api/v1/query", url.Values{"query": {`x{a="a@b"}`}}, "longterm"},
{"quoted open bracket hides window", "api/v1/query", url.Values{"query": {`rate(x{a="[5m"}[7d])`}}, "longterm"},
{"quoted close bracket before window", "api/v1/query", url.Values{"query": {`rate(x{a="]"}[7d])`}}, "longterm"},
{"quoted offset before offset", "api/v1/query", url.Values{"query": {`x{a="offset 1m"} offset 10d`}}, "longterm"},
{"range short window", "api/v1/query_range", rangeFrom("rate(x[1h])", 6*time.Hour, "60"), "shortterm"},
{"range 7d window", "api/v1/query_range", rangeFrom("increase(x[7d])", 6*time.Hour, "60"), "longterm"},
{"range offset past retention", "api/v1/query_range", rangeFrom("x offset 3d", time.Hour, "60"), "longterm"},
{"range lookback fits", "api/v1/query_range", rangeFrom("rate(x[5m])", 70*time.Hour, "600"), "shortterm"},
{"range lookback crosses retention", "api/v1/query_range", rangeFrom("rate(x[5m])", 71*time.Hour+50*time.Minute, "600"), "longterm"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
req := buildMetrics(t, tt.endpoint, tt.in)
if got := cfg.store(req.start, testNow); got != tt.want {
t.Fatalf("store = %s, want %s", got, tt.want)
}
if q := req.params.Get("query"); q != tt.in.Get("query") {
t.Fatalf("query forwarded as %q, want it unchanged", q)
}
})
}
}