From a54760fd9cc145b79c6c901011fe5cb6857a530a Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 06:37:53 +0700 Subject: [PATCH] fix(observability-proxy): start the instant app lookup a latency early vmselect evaluates an instant query whose time is within 30s of its own clock at that clock minus 30s (queryOffset in prometheus.go), not at time minus 30s. For a time slightly in the future the evaluation starts earlier than the proxy assumed: time=now+29s with a 1s step read from six minutes before, while the lookup started at five minutes 32 seconds, and pods present only in that gap were missing from a 200. The evaluation instant is always within (time-60s, time], so the lookup of an instant query now starts one more latency offset early. That needs no assumption about the proxy's clock matching vmselect's. The store choice on the namespace root is left as it is. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 8 +++++++ .../observabilityproxy/server_app_test.go | 24 ++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index 2f0f23751..0fe8b05e9 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -110,6 +110,14 @@ func (s *Server) applicationScope(ctx context.Context, r *http.Request, store st return nil, errorf(http.StatusUnprocessableEntity, "the query reads further back than the application scope looks up pods (%s before the window, or an unknown lookback); use a shorter window or the namespace root", lim.Lookback) } start := req.start + // vmselect evaluates an instant query whose time is within 30s of its + // own clock at that clock minus 30s (queryOffset in prometheus.go), so + // the evaluation instant lies anywhere in (time-60s, time], while + // req.start assumes time-30s. One more latency offset covers it without + // trusting the proxy's clock to match vmselect's. + if req.path == "api/v1/query" { + start = start.Add(-latencyOffset) + } var lists [2][]string for i, l := range []struct { label, metric string diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 8b3a4e587..39b6ac60f 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -341,7 +341,8 @@ func TestServerApplicationLookbackBeyondCap(t *testing.T) { t.Fatalf("rate(x[5m]) answered %d, want 200", got) } r := h.requests()[0] - want := formatTime(buildMetrics(t, "api/v1/query", q).start) + // An instant query may be evaluated up to one more latency offset early. + want := formatTime(buildMetrics(t, "api/v1/query", q).start.Add(-latencyOffset)) if r.path != "/api/v1/label/pod/values" || r.form.Get("start") != want { t.Fatalf("lookup %s start = %s, want the query's earliest read %s", r.path, r.form.Get("start"), want) } @@ -566,3 +567,24 @@ func TestServerApplicationNamesUnreadableSubqueryStep(t *testing.T) { t.Fatalf("answered %d %q, want 422 naming the subquery step", resp.StatusCode, body) } } + +// TestServerApplicationLookupCoversLatencyOffset: vmselect evaluates an +// instant query whose time is within 30s of its own clock at its clock +// minus 30s (prometheus.go, queryOffset), so for a time slightly in the +// future it reads from earlier than time minus 30s. The lookup starts +// early enough for any evaluation instant in (time-60s, time]. +func TestServerApplicationLookupCoversLatencyOffset(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + at := testNow.Add(29 * time.Second) + q := url.Values{"query": {"container_cpu_usage_seconds_total"}, "time": {at.Format(time.RFC3339)}, "step": {"1s"}} + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusOK { + t.Fatalf("status = %d", got) + } + // The earliest read of the query evaluated at the earliest instant. + want := buildMetrics(t, "api/v1/query", q).start.Add(-latencyOffset) + r := h.requests()[0] + start, err := time.Parse(time.RFC3339, r.form.Get("start")) + if err != nil || start.After(want) { + t.Fatalf("lookup start = %s, want at most %s", r.form.Get("start"), formatTime(want)) + } +}