From 07f4f187cef1a96e6f39247eeb2bf8ae7ff2fd79 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 00:36:03 +0700 Subject: [PATCH] fix(observability-proxy): cover forward-shifted samples in the app lookup The pod and PVC lookup of the application metrics root ended at the evaluation time, but a negative offset reads samples after it. An instant query evaluated two hours ago with "offset -1h" reads samples from one hour ago, so a pod created in between was missing from the scope and its series were dropped while the request succeeded. The lookup now ends at the latest instant the query reads, its window end plus the sum of its negative offsets. A query whose latest sample cannot be bounded, an @ modifier other than start() or end() or an offset that is not a plain duration, is refused with 422 on the application root; the namespace root has no lookup and still serves it. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 12 ++-- .../observabilityproxy/metricslookback.go | 64 +++++++++++++++---- internal/observabilityproxy/metricsparams.go | 11 +++- .../observabilityproxy/metricsparams_test.go | 37 +++++++++++ .../observabilityproxy/server_app_test.go | 36 +++++++++++ 5 files changed, 142 insertions(+), 18 deletions(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index 55b51afe6..88acb4ba4 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -94,10 +94,14 @@ func applicationSelectors(ns, kind, name string, pods, pvcs []string) []string { // applicationScope looks the application's pods and PVCs up in store over // the request window, reaching back by the query's own lookback up to the -// configured limit. A failed lookup drops its branch, which only narrows -// the answer; a list over its cap is refused, because truncating it would -// silently drop series. +// configured limit and forward to the latest sample a negative offset +// reads. A failed lookup drops its branch, which only narrows the answer; +// a list over its cap, or a query whose latest sample is unknown, is +// refused, because the answer would silently miss series. func (s *Server) applicationScope(ctx context.Context, r *http.Request, store string, req metricsRequest, ns, kind, name string) ([]string, *HTTPError) { + if req.last.IsZero() { + return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ other than start() or end(), or an offset that is not a plain duration); use the namespace root") + } lim := s.cfg.Metrics.App start := req.from.Add(-lim.Lookback) if req.start.After(start) { @@ -109,7 +113,7 @@ func (s *Server) applicationScope(ctx context.Context, r *http.Request, store st max int }{{"pod", "kube_pod_labels", lim.MaxPods}, {"persistentvolumeclaim", "kube_persistentvolumeclaim_labels", lim.MaxPVCs}} { match := l.metric + identitySelector(ns, ksmKindLabel, ksmNameLabel, kind, name) - names, err := s.lookupNames(ctx, r, store, l.label, match, ns, start, req.to, l.max) + names, err := s.lookupNames(ctx, r, store, l.label, match, ns, start, req.last, l.max) if err != nil { slog.Warn("application lookup failed; its branch is left out", "label", l.label, "namespace", ns, "kind", kind, "name", name, "error", err) continue diff --git a/internal/observabilityproxy/metricslookback.go b/internal/observabilityproxy/metricslookback.go index e55d189a5..442e9c8d9 100644 --- a/internal/observabilityproxy/metricslookback.go +++ b/internal/observabilityproxy/metricslookback.go @@ -113,10 +113,55 @@ func queryLookback(query string, step time.Duration) (time.Duration, bool) { sum += d } lower := strings.ToLower(query) + back, _, ok := scanOffsets(lower) + if !ok { + return 0, false + } + sum += back + if hasKeyword(lower, "with") { + return 0, false + } + n := float64(subqueries + strings.Count(query, "(")) + sum += (2+3*n)*maxStep + (1+n)*float64(silenceInterval) + if sum > float64(maxMetricsDuration) { + return 0, false + } + return time.Duration(sum), true +} + +// queryLookahead is an upper bound on how far after its evaluation window +// a query reads, from the same lexical scan. Only a negative offset reads +// forward; the negative offsets are summed, which covers nesting. An @ +// modifier pins samples to any instant unless it names start() or end(), +// which lie in the window, and WITH can apply one offset several times; +// either makes the bound unknown. +func queryLookahead(query string) (time.Duration, bool) { + lower := strings.ToLower(query) + for rest := lower; ; { + at := strings.IndexByte(rest, '@') + if at < 0 { + break + } + rest = strings.TrimLeft(rest[at+1:], " \t\n\r") + if !strings.HasPrefix(rest, "start()") && !strings.HasPrefix(rest, "end()") { + return 0, false + } + } + _, ahead, ok := scanOffsets(lower) + if !ok || ahead > 0 && hasKeyword(lower, "with") || ahead > float64(maxMetricsDuration) { + return 0, false + } + return time.Duration(ahead), true +} + +// scanOffsets sums every offset of the lowercased query, the positive ones +// into back and the negative ones into ahead, in nanoseconds; ok is false +// when one is not a plain duration. +func scanOffsets(lower string) (back, ahead float64, ok bool) { for i := 0; ; { k := strings.Index(lower[i:], "offset") if k < 0 { - break + return back, ahead, true } k += i i = k + len("offset") @@ -134,21 +179,14 @@ func queryLookback(query string, step time.Duration) (time.Duration, bool) { } d, ok := parseQueryDuration(arg[:end]) if !ok { - return 0, false + return 0, 0, false } - if !negative { - sum += d + if negative { + ahead += d + } else { + back += d } } - if hasKeyword(lower, "with") { - return 0, false - } - n := float64(subqueries + strings.Count(query, "(")) - sum += (2+3*n)*maxStep + (1+n)*float64(silenceInterval) - if sum > float64(maxMetricsDuration) { - return 0, false - } - return time.Duration(sum), true } // isKeyword reports whether lower[start:end] stands on its own rather than diff --git a/internal/observabilityproxy/metricsparams.go b/internal/observabilityproxy/metricsparams.go index a0bb42f0d..b55f09318 100644 --- a/internal/observabilityproxy/metricsparams.go +++ b/internal/observabilityproxy/metricsparams.go @@ -81,6 +81,9 @@ type metricsRequest struct { // from and to are the window the query is evaluated over, without the // lookback of its expression. from, to time.Time + // last is the latest instant the request reads, past to by a negative + // offset; zero when that is unknown. + last time.Time // scopedMatch is set when match[] holds the scope, not client matchers. scopedMatch bool } @@ -143,6 +146,9 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti } req.start = queryStart(out.Get("query"), at.Add(-latencyOffset), step) req.from, req.to = at, at + if ahead, ok := queryLookahead(out.Get("query")); ok { + req.last = at.Add(ahead) + } return req, nil case "api/v1/query_range": @@ -167,6 +173,9 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti } req.start = queryStart(out.Get("query"), start, step) req.from, req.to = start, end + if ahead, ok := queryLookahead(out.Get("query")); ok { + req.last = end.Add(ahead) + } out.Set("start", formatTime(start)) out.Set("end", formatTime(end)) out.Set("step", formatSeconds(step)) @@ -189,7 +198,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti if herr != nil { return req, herr } - req.start, req.from, req.to = start, start, end + req.start, req.from, req.to, req.last = start, start, end, end out.Set("start", formatTime(start)) out.Set("end", formatTime(end)) diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go index a4a435567..5c39fdfff 100644 --- a/internal/observabilityproxy/metricsparams_test.go +++ b/internal/observabilityproxy/metricsparams_test.go @@ -429,3 +429,40 @@ func TestBuildMetricsParamsPointsAsVmselectCounts(t *testing.T) { t.Fatalf("11001 points answered %d, want 422", got) } } + +func TestQueryLookahead(t *testing.T) { + tests := []struct { + query string + want time.Duration + }{ + {`up`, 0}, + {`x offset 1h`, 0}, + {`x offset -1h`, time.Hour}, + {`x OFFSET - 90m`, 90 * time.Minute}, + {`rate(x[5m] offset -30m) offset -1h`, 90 * time.Minute}, + {`x offset -1h + y offset 2h`, time.Hour}, + {`x @ end()`, 0}, + {`x @ start() offset -1h`, time.Hour}, + {`node_offset_seconds - 1`, 0}, + } + for _, tt := range tests { + t.Run(tt.query, func(t *testing.T) { + got, ok := queryLookahead(tt.query) + if !ok || got != tt.want { + t.Fatalf("queryLookahead = %s, %v; want %s", got, ok, tt.want) + } + }) + } + for _, q := range []string{ + `x @ 1700000000`, + `x @ (end() - 1h)`, + `x{a="a@b"}`, + `x offset -5i`, + `x offset -$__interval`, + `with (f(a) = a offset -1h) f(f(x))`, + } { + if got, ok := queryLookahead(q); ok { + t.Errorf("queryLookahead(%q) = %s, want unknown", q, got) + } + } +} diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index a767e7f9e..c7e7a67ef 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -327,3 +327,39 @@ func TestServerApplicationLookupUnknownLookback(t *testing.T) { t.Fatalf("lookup %s start = %s, want %s", r.path, r.form.Get("start"), want) } } + +// TestServerApplicationLookupCoversNegativeOffset evaluates two hours ago +// a selector shifted an hour forward: the pod lookup must reach the time +// of the samples, or a pod created in between is left out. +func TestServerApplicationLookupCoversNegativeOffset(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + at := testNow.Add(-2 * time.Hour) + q := url.Values{"query": {"container_cpu_usage_seconds_total offset -1h"}, "time": {unix(at)}} + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusOK { + t.Fatalf("status = %d", got) + } + r := h.requests()[0] + if want := formatTime(at.Add(time.Hour)); r.path != "/api/v1/label/pod/values" || r.form.Get("end") != want { + t.Fatalf("lookup %s end = %s, want %s", r.path, r.form.Get("end"), want) + } +} + +// TestServerApplicationRefusesUnboundedSampleTime refuses a query whose +// samples may lie at a time the lookup cannot be sized for, rather than +// answer with series silently left out. +func TestServerApplicationRefusesUnboundedSampleTime(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + for _, query := range []string{"up @ 1700000000", "up offset -5i"} { + q := url.Values{"query": {query}} + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusUnprocessableEntity { + t.Errorf("%s answered %d, want 422", query, got) + } + } + if n := len(h.requests()); n != 0 { + t.Fatalf("upstream saw %d requests for a refused query", n) + } + // The namespace root has no lookup to size and still serves it. + if got := h.status(http.MethodGet, metricsRoot+"api/v1/query?"+url.Values{"query": {"up @ 1700000000"}}.Encode(), "good"); got != http.StatusOK { + t.Fatalf("namespace root answered %d", got) + } +}