fix(observability-proxy): cover forward-shifted samples in the app lookup

The pod and PVC lookup of the application metrics root ended at the
evaluation time, but a negative offset reads samples after it. An
instant query evaluated two hours ago with "offset -1h" reads samples
from one hour ago, so a pod created in between was missing from the
scope and its series were dropped while the request succeeded.

The lookup now ends at the latest instant the query reads, its window
end plus the sum of its negative offsets. A query whose latest sample
cannot be bounded, an @ modifier other than start() or end() or an
offset that is not a plain duration, is refused with 422 on the
application root; the namespace root has no lookup and still serves
it.

Assisted-by: LLM
Signed-off-by: ZverGuy <maximbel2003@gmail.com>
This commit is contained in:
ZverGuy committed 2026-11-10 04:03:24 +07:00
1 parent c4593d4c3e
commit 07f4f187ce
5 files changed
+142 -18

No files matched your search

+8 -4
View File
@@ -94,10 +94,14 @@ func applicationSelectors(ns, kind, name string, pods, pvcs []string) []string {
// applicationScope looks the application's pods and PVCs up in store over
// the request window, reaching back by the query's own lookback up to the
// configured limit. A failed lookup drops its branch, which only narrows
// the answer; a list over its cap is refused, because truncating it would
// silently drop series.
// configured limit and forward to the latest sample a negative offset
// reads. A failed lookup drops its branch, which only narrows the answer;
// a list over its cap, or a query whose latest sample is unknown, is
// refused, because the answer would silently miss series.
func (s *Server) applicationScope(ctx context.Context, r *http.Request, store string, req metricsRequest, ns, kind, name string) ([]string, *HTTPError) {
if req.last.IsZero() {
return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ other than start() or end(), or an offset that is not a plain duration); use the namespace root")
}
lim := s.cfg.Metrics.App
start := req.from.Add(-lim.Lookback)
if req.start.After(start) {
@@ -109,7 +113,7 @@ func (s *Server) applicationScope(ctx context.Context, r *http.Request, store st
max int
}{{"pod", "kube_pod_labels", lim.MaxPods}, {"persistentvolumeclaim", "kube_persistentvolumeclaim_labels", lim.MaxPVCs}} {
match := l.metric + identitySelector(ns, ksmKindLabel, ksmNameLabel, kind, name)
names, err := s.lookupNames(ctx, r, store, l.label, match, ns, start, req.to, l.max)
names, err := s.lookupNames(ctx, r, store, l.label, match, ns, start, req.last, l.max)
if err != nil {
slog.Warn("application lookup failed; its branch is left out", "label", l.label, "namespace", ns, "kind", kind, "name", name, "error", err)
continue
+51 -13
View File
@@ -113,10 +113,55 @@ func queryLookback(query string, step time.Duration) (time.Duration, bool) {
sum += d
}
lower := strings.ToLower(query)
back, _, ok := scanOffsets(lower)
if !ok {
return 0, false
}
sum += back
if hasKeyword(lower, "with") {
return 0, false
}
n := float64(subqueries + strings.Count(query, "("))
sum += (2+3*n)*maxStep + (1+n)*float64(silenceInterval)
if sum > float64(maxMetricsDuration) {
return 0, false
}
return time.Duration(sum), true
}
// queryLookahead is an upper bound on how far after its evaluation window
// a query reads, from the same lexical scan. Only a negative offset reads
// forward; the negative offsets are summed, which covers nesting. An @
// modifier pins samples to any instant unless it names start() or end(),
// which lie in the window, and WITH can apply one offset several times;
// either makes the bound unknown.
func queryLookahead(query string) (time.Duration, bool) {
lower := strings.ToLower(query)
for rest := lower; ; {
at := strings.IndexByte(rest, '@')
if at < 0 {
break
}
rest = strings.TrimLeft(rest[at+1:], " \t\n\r")
if !strings.HasPrefix(rest, "start()") && !strings.HasPrefix(rest, "end()") {
return 0, false
}
}
_, ahead, ok := scanOffsets(lower)
if !ok || ahead > 0 && hasKeyword(lower, "with") || ahead > float64(maxMetricsDuration) {
return 0, false
}
return time.Duration(ahead), true
}
// scanOffsets sums every offset of the lowercased query, the positive ones
// into back and the negative ones into ahead, in nanoseconds; ok is false
// when one is not a plain duration.
func scanOffsets(lower string) (back, ahead float64, ok bool) {
for i := 0; ; {
k := strings.Index(lower[i:], "offset")
if k < 0 {
break
return back, ahead, true
}
k += i
i = k + len("offset")
@@ -134,21 +179,14 @@ func queryLookback(query string, step time.Duration) (time.Duration, bool) {
}
d, ok := parseQueryDuration(arg[:end])
if !ok {
return 0, false
return 0, 0, false
}
if !negative {
sum += d
if negative {
ahead += d
} else {
back += d
}
}
if hasKeyword(lower, "with") {
return 0, false
}
n := float64(subqueries + strings.Count(query, "("))
sum += (2+3*n)*maxStep + (1+n)*float64(silenceInterval)
if sum > float64(maxMetricsDuration) {
return 0, false
}
return time.Duration(sum), true
}
// isKeyword reports whether lower[start:end] stands on its own rather than
+10 -1
View File
@@ -81,6 +81,9 @@ type metricsRequest struct {
// from and to are the window the query is evaluated over, without the
// lookback of its expression.
from, to time.Time
// last is the latest instant the request reads, past to by a negative
// offset; zero when that is unknown.
last time.Time
// scopedMatch is set when match[] holds the scope, not client matchers.
scopedMatch bool
}
@@ -143,6 +146,9 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti
}
req.start = queryStart(out.Get("query"), at.Add(-latencyOffset), step)
req.from, req.to = at, at
if ahead, ok := queryLookahead(out.Get("query")); ok {
req.last = at.Add(ahead)
}
return req, nil
case "api/v1/query_range":
@@ -167,6 +173,9 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti
}
req.start = queryStart(out.Get("query"), start, step)
req.from, req.to = start, end
if ahead, ok := queryLookahead(out.Get("query")); ok {
req.last = end.Add(ahead)
}
out.Set("start", formatTime(start))
out.Set("end", formatTime(end))
out.Set("step", formatSeconds(step))
@@ -189,7 +198,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti
if herr != nil {
return req, herr
}
req.start, req.from, req.to = start, start, end
req.start, req.from, req.to, req.last = start, start, end, end
out.Set("start", formatTime(start))
out.Set("end", formatTime(end))
@@ -429,3 +429,40 @@ func TestBuildMetricsParamsPointsAsVmselectCounts(t *testing.T) {
t.Fatalf("11001 points answered %d, want 422", got)
}
}
func TestQueryLookahead(t *testing.T) {
tests := []struct {
query string
want time.Duration
}{
{`up`, 0},
{`x offset 1h`, 0},
{`x offset -1h`, time.Hour},
{`x OFFSET - 90m`, 90 * time.Minute},
{`rate(x[5m] offset -30m) offset -1h`, 90 * time.Minute},
{`x offset -1h + y offset 2h`, time.Hour},
{`x @ end()`, 0},
{`x @ start() offset -1h`, time.Hour},
{`node_offset_seconds - 1`, 0},
}
for _, tt := range tests {
t.Run(tt.query, func(t *testing.T) {
got, ok := queryLookahead(tt.query)
if !ok || got != tt.want {
t.Fatalf("queryLookahead = %s, %v; want %s", got, ok, tt.want)
}
})
}
for _, q := range []string{
`x @ 1700000000`,
`x @ (end() - 1h)`,
`x{a="a@b"}`,
`x offset -5i`,
`x offset -$__interval`,
`with (f(a) = a offset -1h) f(f(x))`,
} {
if got, ok := queryLookahead(q); ok {
t.Errorf("queryLookahead(%q) = %s, want unknown", q, got)
}
}
}
@@ -327,3 +327,39 @@ func TestServerApplicationLookupUnknownLookback(t *testing.T) {
t.Fatalf("lookup %s start = %s, want %s", r.path, r.form.Get("start"), want)
}
}
// TestServerApplicationLookupCoversNegativeOffset evaluates two hours ago
// a selector shifted an hour forward: the pod lookup must reach the time
// of the samples, or a pod created in between is left out.
func TestServerApplicationLookupCoversNegativeOffset(t *testing.T) {
h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil)
at := testNow.Add(-2 * time.Hour)
q := url.Values{"query": {"container_cpu_usage_seconds_total offset -1h"}, "time": {unix(at)}}
if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusOK {
t.Fatalf("status = %d", got)
}
r := h.requests()[0]
if want := formatTime(at.Add(time.Hour)); r.path != "/api/v1/label/pod/values" || r.form.Get("end") != want {
t.Fatalf("lookup %s end = %s, want %s", r.path, r.form.Get("end"), want)
}
}
// TestServerApplicationRefusesUnboundedSampleTime refuses a query whose
// samples may lie at a time the lookup cannot be sized for, rather than
// answer with series silently left out.
func TestServerApplicationRefusesUnboundedSampleTime(t *testing.T) {
h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil)
for _, query := range []string{"up @ 1700000000", "up offset -5i"} {
q := url.Values{"query": {query}}
if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusUnprocessableEntity {
t.Errorf("%s answered %d, want 422", query, got)
}
}
if n := len(h.requests()); n != 0 {
t.Fatalf("upstream saw %d requests for a refused query", n)
}
// The namespace root has no lookup to size and still serves it.
if got := h.status(http.MethodGet, metricsRoot+"api/v1/query?"+url.Values{"query": {"up @ 1700000000"}}.Encode(), "good"); got != http.StatusOK {
t.Fatalf("namespace root answered %d", got)
}
}