mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
fix(observability-proxy): count the query lookback in the store choice
The short-term store was picked from the request window alone, so an instant increase(x[7d]) or x offset 10d went to the store that keeps 72h and silently returned a wrong number. A lexical scan of the query now bounds how far back it reads: the sum of every [...] window and subquery step, positive offsets, and the step and silence interval vmselect reads before each rollup. An @ modifier, step-multiple units or any duration the scan cannot read make the bound unknown and pick the long-term store, which receives the same writes. The scan sees quoted label values too, so a false hit only moves a query to the long-term store. The query is still forwarded unchanged. Assisted-by: LLM Signed-off-by: ZverGuy <maximbel2003@gmail.com>
This commit is contained in:
1 parent
19030c7a1a
commit
560e4d7ba5
3 files changed
+288
-8
No files matched your search
@@ -0,0 +1,172 @@
|
||||
/*
|
||||
Copyright 2026 The Cozystack Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package observabilityproxy
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// instantDefaultStep is the step vmselect uses for an instant query without
|
||||
// one (defaultStep in app/vmselect/prometheus); it is also the implicit
|
||||
// rollup window of a bare selector.
|
||||
const instantDefaultStep = 5 * time.Minute
|
||||
|
||||
// silenceInterval is how far before a rollup or subquery window vmselect
|
||||
// starts reading (maxSilenceInterval in app/vmselect/promql: the value of
|
||||
// -search.minStalenessInterval, 5m when unset; cozystack does not set it).
|
||||
const silenceInterval = 5 * time.Minute
|
||||
|
||||
// queryDurationPart is the subset of the MetricsQL duration grammar
|
||||
// (scanSingleDuration in metricsql) the scan understands. Step multiples
|
||||
// (5i, $__interval), negative parts and upper-case units are left out on
|
||||
// purpose: anything else makes the bound unknown.
|
||||
var (
|
||||
queryDurationPart = regexp.MustCompile(`(\d+(?:\.\d+)?)(ms|s|m|h|d|w|y)`)
|
||||
queryDuration = regexp.MustCompile(`^(?:\d+(?:\.\d+)?(?:ms|s|m|h|d|w|y))+$`)
|
||||
querySeconds = regexp.MustCompile(`^\d+(?:\.\d+)?$`)
|
||||
)
|
||||
|
||||
var queryDurationUnits = map[string]time.Duration{
|
||||
"ms": time.Millisecond, "s": time.Second, "m": time.Minute, "h": time.Hour,
|
||||
"d": 24 * time.Hour, "w": 7 * 24 * time.Hour, "y": 365 * 24 * time.Hour,
|
||||
}
|
||||
|
||||
// queryStart is the earliest instant a query evaluated from start may read;
|
||||
// zero when the bound is unknown.
|
||||
func queryStart(query string, start time.Time, step time.Duration) time.Time {
|
||||
lookback, ok := queryLookback(query, step)
|
||||
if !ok {
|
||||
return time.Time{}
|
||||
}
|
||||
return start.Add(-lookback)
|
||||
}
|
||||
|
||||
// queryLookback is an upper bound on how far before its evaluation window
|
||||
// a query reads, from a lexical scan that never parses the query. Every
|
||||
// [...] and offset counts, including ones inside quoted label values: an
|
||||
// extra hit only makes the bound larger or unknown, never smaller. Windows
|
||||
// and offsets are summed rather than maxed, which also covers nesting.
|
||||
// vmselect starts a rollup max(window, step) plus the silence interval
|
||||
// before the first point, a subquery window+step+silence before it and
|
||||
// aligns both down to a step; each step term uses the largest step seen.
|
||||
func queryLookback(query string, step time.Duration) (time.Duration, bool) {
|
||||
if strings.Contains(query, "@") {
|
||||
return 0, false
|
||||
}
|
||||
var sum float64
|
||||
subqueries := 0
|
||||
maxStep := float64(step)
|
||||
for rest := query; ; {
|
||||
open := strings.IndexByte(rest, '[')
|
||||
if open < 0 {
|
||||
break
|
||||
}
|
||||
closing := strings.IndexByte(rest[open:], ']')
|
||||
if closing < 0 {
|
||||
return 0, false
|
||||
}
|
||||
window, subStep, isSubquery := strings.Cut(rest[open+1:open+closing], ":")
|
||||
rest = rest[open+closing+1:]
|
||||
if isSubquery {
|
||||
subqueries++
|
||||
if strings.TrimSpace(window) == "" {
|
||||
window = "0s"
|
||||
}
|
||||
if strings.TrimSpace(subStep) != "" {
|
||||
d, ok := parseQueryDuration(subStep)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
maxStep = max(maxStep, d)
|
||||
}
|
||||
}
|
||||
d, ok := parseQueryDuration(window)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
sum += d
|
||||
}
|
||||
lower := strings.ToLower(query)
|
||||
for i := 0; ; {
|
||||
k := strings.Index(lower[i:], "offset")
|
||||
if k < 0 {
|
||||
break
|
||||
}
|
||||
k += i
|
||||
i = k + len("offset")
|
||||
if k > 0 && continuesIdent(lower[k-1], false) || i < len(lower) && continuesIdent(lower[i], true) {
|
||||
continue
|
||||
}
|
||||
arg := strings.TrimLeft(lower[i:], " \t\n\r")
|
||||
negative := strings.HasPrefix(arg, "-")
|
||||
if negative {
|
||||
arg = strings.TrimLeft(arg[1:], " \t\n\r")
|
||||
}
|
||||
end := strings.IndexFunc(arg, func(r rune) bool {
|
||||
return !(r >= '0' && r <= '9' || r >= 'a' && r <= 'z' || r == '.')
|
||||
})
|
||||
if end < 0 {
|
||||
end = len(arg)
|
||||
}
|
||||
d, ok := parseQueryDuration(arg[:end])
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
if !negative {
|
||||
sum += d
|
||||
}
|
||||
}
|
||||
n := float64(subqueries)
|
||||
sum += (2+3*n)*maxStep + (1+n)*float64(silenceInterval)
|
||||
if sum > float64(maxMetricsDuration) {
|
||||
return 0, false
|
||||
}
|
||||
return time.Duration(sum), true
|
||||
}
|
||||
|
||||
// continuesIdent reports whether c, next to "offset", makes it part of an
|
||||
// identifier rather than the keyword. Digits and dots only continue an
|
||||
// identifier after it, so before it they still count as a keyword hit.
|
||||
func continuesIdent(c byte, after bool) bool {
|
||||
return c >= 'a' && c <= 'z' || c == '_' || c == ':' || c >= 0x80 ||
|
||||
after && (c >= '0' && c <= '9' || c == '.' || c == '\\')
|
||||
}
|
||||
|
||||
// parseQueryDuration returns nanoseconds as a float so that a sum of huge
|
||||
// durations cannot wrap; the caller compares it against the bound.
|
||||
func parseQueryDuration(s string) (float64, bool) {
|
||||
s = strings.TrimSpace(s)
|
||||
if querySeconds.MatchString(s) {
|
||||
f, err := strconv.ParseFloat(s, 64)
|
||||
return f * float64(time.Second), err == nil
|
||||
}
|
||||
if !queryDuration.MatchString(s) {
|
||||
return 0, false
|
||||
}
|
||||
var d float64
|
||||
for _, m := range queryDurationPart.FindAllStringSubmatch(s, -1) {
|
||||
f, err := strconv.ParseFloat(m[1], 64)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
d += f * float64(queryDurationUnits[m[2]])
|
||||
}
|
||||
return d, true
|
||||
}
|
||||
@@ -69,7 +69,8 @@ var labelName = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`)
|
||||
type metricsRequest struct {
|
||||
path string
|
||||
params url.Values
|
||||
// start is the earliest instant the request reads, for the store choice.
|
||||
// start is the earliest instant the request reads, for the store choice;
|
||||
// zero when that is unknown, which picks the long-term store.
|
||||
start time.Time
|
||||
}
|
||||
|
||||
@@ -103,22 +104,24 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti
|
||||
if herr := copyQuery(out, in); herr != nil {
|
||||
return req, herr
|
||||
}
|
||||
req.start = now
|
||||
at := now
|
||||
if v := in.Get("time"); v != "" {
|
||||
t, herr := parseMetricsTime("time", v)
|
||||
if herr != nil {
|
||||
return req, herr
|
||||
}
|
||||
req.start = t
|
||||
at = t
|
||||
out.Set("time", formatTime(t))
|
||||
}
|
||||
step := instantDefaultStep
|
||||
if v := in.Get("step"); v != "" {
|
||||
step, herr := parseStep(v, lim)
|
||||
if herr != nil {
|
||||
var herr *HTTPError
|
||||
if step, herr = parseStep(v, lim); herr != nil {
|
||||
return req, herr
|
||||
}
|
||||
out.Set("step", formatSeconds(step))
|
||||
}
|
||||
req.start = queryStart(out.Get("query"), at, step)
|
||||
return req, nil
|
||||
|
||||
case "api/v1/query_range":
|
||||
@@ -141,7 +144,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti
|
||||
if points := end.Sub(start)/step + 1; points > time.Duration(lim.MaxPoints) {
|
||||
return req, errorf(http.StatusUnprocessableEntity, "%d points per series exceed the limit of %d; raise step", points, lim.MaxPoints)
|
||||
}
|
||||
req.start = start
|
||||
req.start = queryStart(out.Get("query"), start, step)
|
||||
out.Set("start", formatTime(start))
|
||||
out.Set("end", formatTime(end))
|
||||
out.Set("step", formatSeconds(step))
|
||||
|
||||
@@ -224,7 +224,7 @@ func TestBuildMetricsParamsCanonicalForms(t *testing.T) {
|
||||
if req.params.Get("end") != "1793793600.000" || req.params.Get("start") != "1793793300.000" {
|
||||
t.Errorf("labels window = %s..%s, want the last 5m", req.params.Get("start"), req.params.Get("end"))
|
||||
}
|
||||
if q := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}}); q.params.Has("time") || !q.start.Equal(testNow) {
|
||||
if q := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}}); q.params.Has("time") || !q.start.Equal(testNow.Add(-15*time.Minute)) {
|
||||
t.Errorf("instant query without time: params %v, start %v", q.params, q.start)
|
||||
}
|
||||
}
|
||||
@@ -241,7 +241,8 @@ func TestMetricsStoreByRange(t *testing.T) {
|
||||
{"instant within retention", "api/v1/query", url.Values{"query": {"up"}, "time": {unix(testNow.Add(-71 * time.Hour))}}, "shortterm"},
|
||||
{"instant past retention", "api/v1/query", url.Values{"query": {"up"}, "time": {unix(testNow.Add(-73 * time.Hour))}}, "longterm"},
|
||||
{"live graph", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-6 * time.Hour))}, "end": {unix(testNow)}, "step": {"60"}}, "shortterm"},
|
||||
{"range fits exactly", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-72 * time.Hour))}, "end": {unix(testNow)}, "step": {"600"}}, "shortterm"},
|
||||
{"range fits with lookback", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-71 * time.Hour))}, "end": {unix(testNow)}, "step": {"600"}}, "shortterm"},
|
||||
{"range lookback past retention", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-72 * time.Hour))}, "end": {unix(testNow)}, "step": {"600"}}, "longterm"},
|
||||
{"range past retention", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-7 * 24 * time.Hour))}, "end": {unix(testNow)}, "step": {"600"}}, "longterm"},
|
||||
{"old window ending long ago", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-5 * 24 * time.Hour))}, "end": {unix(testNow.Add(-4 * 24 * time.Hour))}, "step": {"600"}}, "longterm"},
|
||||
{"labels default window", "api/v1/labels", url.Values{}, "shortterm"},
|
||||
@@ -256,3 +257,107 @@ func TestMetricsStoreByRange(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryLookback(t *testing.T) {
|
||||
tests := []struct {
|
||||
query string
|
||||
step time.Duration
|
||||
want time.Duration
|
||||
}{
|
||||
{`up`, 5 * time.Minute, 15 * time.Minute},
|
||||
{`increase(x[7d])`, 5 * time.Minute, 7*24*time.Hour + 15*time.Minute},
|
||||
{`rate(x[300])`, time.Minute, 12 * time.Minute},
|
||||
{`rate(x[ 1h30m ])`, time.Minute, 97 * time.Minute},
|
||||
{`max_over_time(x[1h:5m])`, time.Minute, 95 * time.Minute},
|
||||
{`max_over_time(x[1h:])`, time.Minute, 75 * time.Minute},
|
||||
{`max_over_time(max_over_time(rate(x[5m])[1h:1m])[1d:1h])`, time.Minute, 33*time.Hour + 20*time.Minute},
|
||||
{`x offset 10d`, 5 * time.Minute, 10*24*time.Hour + 15*time.Minute},
|
||||
{`sum(x OFFSET 2h)`, 5 * time.Minute, 135 * time.Minute},
|
||||
{`rate(x[5m] offset 1h)`, time.Minute, 72 * time.Minute},
|
||||
{`x offset -1h`, 5 * time.Minute, 15 * time.Minute},
|
||||
{`x offset - 1h`, 5 * time.Minute, 15 * time.Minute},
|
||||
{`node_offset_seconds + offset_total`, 5 * time.Minute, 15 * time.Minute},
|
||||
{`x{a="[1h]"}`, 5 * time.Minute, 75 * time.Minute},
|
||||
{`x{a="]"}[7d]`, 5 * time.Minute, 7*24*time.Hour + 15*time.Minute},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.query, func(t *testing.T) {
|
||||
got, ok := queryLookback(tt.query, tt.step)
|
||||
if !ok || got != tt.want {
|
||||
t.Fatalf("queryLookback = %s, %t; want %s, true", got, ok, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryLookbackUnknown(t *testing.T) {
|
||||
for _, q := range []string{
|
||||
`x @ 123`,
|
||||
`x @ start()`,
|
||||
`x{a="a@b"}`,
|
||||
`rate(x[5i])`,
|
||||
`rate(x[$__interval])`,
|
||||
`x[1h-5m]`,
|
||||
`x[7D]`,
|
||||
`x[1.d]`,
|
||||
`x[]`,
|
||||
`rate(x[7d)`,
|
||||
`x offset`,
|
||||
`x offset w`,
|
||||
`WITH (w = 7d) rate(x[w])`,
|
||||
`x{a="["}`,
|
||||
`x{a="[5m"}[7d]`,
|
||||
`x{path="/offset/1"}`,
|
||||
`x[99999999999y]`,
|
||||
`x offset 99999999999y`,
|
||||
} {
|
||||
t.Run(q, func(t *testing.T) {
|
||||
if got, ok := queryLookback(q, 5*time.Minute); ok {
|
||||
t.Fatalf("queryLookback = %s, true; want unknown", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricsStoreByLookback(t *testing.T) {
|
||||
cfg := MetricsConfig{ShortTermStore: "shortterm", LongTermStore: "longterm", ShortTermRetention: 72 * time.Hour}
|
||||
rangeFrom := func(q string, ago time.Duration, step string) url.Values {
|
||||
return url.Values{"query": {q}, "start": {unix(testNow.Add(-ago))}, "end": {unix(testNow)}, "step": {step}}
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
endpoint string
|
||||
in url.Values
|
||||
want string
|
||||
}{
|
||||
{"instant short window", "api/v1/query", url.Values{"query": {"rate(x[5m])"}}, "shortterm"},
|
||||
{"instant 7d window", "api/v1/query", url.Values{"query": {"increase(x[7d])"}}, "longterm"},
|
||||
{"instant offset 10d", "api/v1/query", url.Values{"query": {"x offset 10d"}}, "longterm"},
|
||||
{"instant negative offset", "api/v1/query", url.Values{"query": {"x offset -1h"}}, "shortterm"},
|
||||
{"instant at timestamp", "api/v1/query", url.Values{"query": {"x @ 123"}}, "longterm"},
|
||||
{"instant at start", "api/v1/query", url.Values{"query": {"x @ start()"}}, "longterm"},
|
||||
{"instant step window", "api/v1/query", url.Values{"query": {"rate(x[5i])"}}, "longterm"},
|
||||
{"instant step past retention", "api/v1/query", url.Values{"query": {"up"}, "step": {"259200"}}, "longterm"},
|
||||
{"instant subquery past retention", "api/v1/query", url.Values{"query": {"max_over_time(rate(x[5m])[4d:1h])"}}, "longterm"},
|
||||
{"quoted at sign", "api/v1/query", url.Values{"query": {`x{a="a@b"}`}}, "longterm"},
|
||||
{"quoted open bracket hides window", "api/v1/query", url.Values{"query": {`rate(x{a="[5m"}[7d])`}}, "longterm"},
|
||||
{"quoted close bracket before window", "api/v1/query", url.Values{"query": {`rate(x{a="]"}[7d])`}}, "longterm"},
|
||||
{"quoted offset before offset", "api/v1/query", url.Values{"query": {`x{a="offset 1m"} offset 10d`}}, "longterm"},
|
||||
{"range short window", "api/v1/query_range", rangeFrom("rate(x[1h])", 6*time.Hour, "60"), "shortterm"},
|
||||
{"range 7d window", "api/v1/query_range", rangeFrom("increase(x[7d])", 6*time.Hour, "60"), "longterm"},
|
||||
{"range offset past retention", "api/v1/query_range", rangeFrom("x offset 3d", time.Hour, "60"), "longterm"},
|
||||
{"range lookback fits", "api/v1/query_range", rangeFrom("rate(x[5m])", 70*time.Hour, "600"), "shortterm"},
|
||||
{"range lookback crosses retention", "api/v1/query_range", rangeFrom("rate(x[5m])", 71*time.Hour+50*time.Minute, "600"), "longterm"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
req := buildMetrics(t, tt.endpoint, tt.in)
|
||||
if got := cfg.store(req.start, testNow); got != tt.want {
|
||||
t.Fatalf("store = %s, want %s", got, tt.want)
|
||||
}
|
||||
if q := req.params.Get("query"); q != tt.in.Get("query") {
|
||||
t.Fatalf("query forwarded as %q, want it unchanged", q)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user