fix(observability-proxy): start the instant app lookup a latency early

vmselect evaluates an instant query whose time is within 30s of its
own clock at that clock minus 30s (queryOffset in prometheus.go), not
at time minus 30s. For a time slightly in the future the evaluation
starts earlier than the proxy assumed: time=now+29s with a 1s step
read from six minutes before, while the lookup started at five
minutes 32 seconds, and pods present only in that gap were missing
from a 200.

The evaluation instant is always within (time-60s, time], so the
lookup of an instant query now starts one more latency offset early.
That needs no assumption about the proxy's clock matching vmselect's.
The store choice on the namespace root is left as it is.

Assisted-by: LLM
Signed-off-by: ZverGuy <maximbel2003@gmail.com>
This commit is contained in:
ZverGuy committed 2026-11-10 04:03:24 +07:00
1 parent 3198de893a
commit a54760fd9c
2 files changed
+31 -1

No files matched your search

+8
View File
@@ -110,6 +110,14 @@ func (s *Server) applicationScope(ctx context.Context, r *http.Request, store st
return nil, errorf(http.StatusUnprocessableEntity, "the query reads further back than the application scope looks up pods (%s before the window, or an unknown lookback); use a shorter window or the namespace root", lim.Lookback)
}
start := req.start
// vmselect evaluates an instant query whose time is within 30s of its
// own clock at that clock minus 30s (queryOffset in prometheus.go), so
// the evaluation instant lies anywhere in (time-60s, time], while
// req.start assumes time-30s. One more latency offset covers it without
// trusting the proxy's clock to match vmselect's.
if req.path == "api/v1/query" {
start = start.Add(-latencyOffset)
}
var lists [2][]string
for i, l := range []struct {
label, metric string
+23 -1
View File
@@ -341,7 +341,8 @@ func TestServerApplicationLookbackBeyondCap(t *testing.T) {
t.Fatalf("rate(x[5m]) answered %d, want 200", got)
}
r := h.requests()[0]
want := formatTime(buildMetrics(t, "api/v1/query", q).start)
// An instant query may be evaluated up to one more latency offset early.
want := formatTime(buildMetrics(t, "api/v1/query", q).start.Add(-latencyOffset))
if r.path != "/api/v1/label/pod/values" || r.form.Get("start") != want {
t.Fatalf("lookup %s start = %s, want the query's earliest read %s", r.path, r.form.Get("start"), want)
}
@@ -566,3 +567,24 @@ func TestServerApplicationNamesUnreadableSubqueryStep(t *testing.T) {
t.Fatalf("answered %d %q, want 422 naming the subquery step", resp.StatusCode, body)
}
}
// TestServerApplicationLookupCoversLatencyOffset: vmselect evaluates an
// instant query whose time is within 30s of its own clock at its clock
// minus 30s (prometheus.go, queryOffset), so for a time slightly in the
// future it reads from earlier than time minus 30s. The lookup starts
// early enough for any evaluation instant in (time-60s, time].
func TestServerApplicationLookupCoversLatencyOffset(t *testing.T) {
h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil)
at := testNow.Add(29 * time.Second)
q := url.Values{"query": {"container_cpu_usage_seconds_total"}, "time": {at.Format(time.RFC3339)}, "step": {"1s"}}
if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusOK {
t.Fatalf("status = %d", got)
}
// The earliest read of the query evaluated at the earliest instant.
want := buildMetrics(t, "api/v1/query", q).start.Add(-latencyOffset)
r := h.requests()[0]
start, err := time.Parse(time.RFC3339, r.form.Get("start"))
if err != nil || start.After(want) {
t.Fatalf("lookup start = %s, want at most %s", r.form.Get("start"), formatTime(want))
}
}