From ac5a3c88c418b60216b0c6b93441f26b4229a98d Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 06:45:21 +0700 Subject: [PATCH] fix(observability-proxy): put the latency window in the instant start The second latency offset was added to the application lookup only, so the other readers of the instant query's earliest read still assumed time minus 30s: the store choice could send a query to the short-term store up to 30 seconds too late near its retention edge, the lookback cap compared against a start the lookup no longer used, and the field no longer meant the earliest instant read. The earliest read of an instant query now starts from time minus two latency offsets, the earliest instant vmselect can evaluate it at, and the lookup uses it as is. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 8 -------- internal/observabilityproxy/metricsparams.go | 6 +++++- .../observabilityproxy/metricsparams_test.go | 18 +++++++++++++++++- internal/observabilityproxy/server_app_test.go | 6 +++--- 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index 53f8d5582..e6df8ba02 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -110,14 +110,6 @@ func (s *Server) applicationScope(ctx context.Context, r *http.Request, store st return nil, errorf(http.StatusUnprocessableEntity, "the query reads further back than the application scope looks up pods (%s before the window, or an unknown lookback); use a shorter window or the namespace root", lim.Lookback) } start := req.start - // vmselect evaluates an instant query whose time is within 30s of its - // own clock at that clock minus 30s (queryOffset in prometheus.go), so - // the evaluation instant lies anywhere in (time-60s, time], while - // req.start assumes time-30s. One more latency offset covers it without - // trusting the proxy's clock to match vmselect's. - if req.path == "api/v1/query" { - start = start.Add(-latencyOffset) - } var lists [2][]string for i, l := range []struct { label, metric string diff --git a/internal/observabilityproxy/metricsparams.go b/internal/observabilityproxy/metricsparams.go index 63be3c497..f02620b3c 100644 --- a/internal/observabilityproxy/metricsparams.go +++ b/internal/observabilityproxy/metricsparams.go @@ -144,7 +144,11 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti } out.Set("step", formatSeconds(step)) } - req.start = queryStart(out.Get("query"), at.Add(-latencyOffset), step) + // vmselect evaluates an instant query whose time is within the + // latency offset of its own clock at that clock minus the offset + // (queryOffset in prometheus.go), so the evaluation instant lies in + // (time-2*offset, time]; the earlier end needs no clock to agree. + req.start = queryStart(out.Get("query"), at.Add(-2*latencyOffset), step) req.from, req.to = at, at if ahead, ok := queryLookahead(out.Get("query"), step); ok { req.last = at.Add(ahead) diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go index e201a6fa8..34d36f7e5 100644 --- a/internal/observabilityproxy/metricsparams_test.go +++ b/internal/observabilityproxy/metricsparams_test.go @@ -227,7 +227,7 @@ func TestBuildMetricsParamsCanonicalForms(t *testing.T) { if req.params.Get("end") != "2026-11-04T12:00:00.000Z" || req.params.Get("start") != "2026-11-04T11:55:00.000Z" { t.Errorf("labels window = %s..%s, want the last 5m", req.params.Get("start"), req.params.Get("end")) } - if q := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}}); q.params.Has("time") || !q.start.Equal(testNow.Add(-15*time.Minute-30*time.Second)) { + if q := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}}); q.params.Has("time") || !q.start.Equal(testNow.Add(-15*time.Minute-2*latencyOffset)) { t.Errorf("instant query without time: params %v, start %v", q.params, q.start) } } @@ -543,3 +543,19 @@ func TestQueryLookaheadSubquery(t *testing.T) { } } } + +// TestBuildMetricsParamsInstantStartCoversLatencyOffset: vmselect evaluates +// an instant query whose time is within 30s of its clock at that clock +// minus 30s, anywhere in (time-60s, time], so the earliest read the store +// choice and the lookup rely on starts from time-60s. +func TestBuildMetricsParamsInstantStartCoversLatencyOffset(t *testing.T) { + at := testNow.Add(29 * time.Second) + req := buildMetrics(t, "api/v1/query", url.Values{"query": {"x"}, "time": {at.Format(time.RFC3339)}, "step": {"1s"}}) + lookback, ok := queryLookback("x", time.Second) + if !ok { + t.Fatal("lookback of x unknown") + } + if want := at.Add(-2*latencyOffset - lookback); req.start.After(want) { + t.Fatalf("start = %s, want at most %s", req.start, want) + } +} diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 39b6ac60f..e5c0d3710 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -341,8 +341,7 @@ func TestServerApplicationLookbackBeyondCap(t *testing.T) { t.Fatalf("rate(x[5m]) answered %d, want 200", got) } r := h.requests()[0] - // An instant query may be evaluated up to one more latency offset early. - want := formatTime(buildMetrics(t, "api/v1/query", q).start.Add(-latencyOffset)) + want := formatTime(buildMetrics(t, "api/v1/query", q).start) if r.path != "/api/v1/label/pod/values" || r.form.Get("start") != want { t.Fatalf("lookup %s start = %s, want the query's earliest read %s", r.path, r.form.Get("start"), want) } @@ -581,7 +580,8 @@ func TestServerApplicationLookupCoversLatencyOffset(t *testing.T) { t.Fatalf("status = %d", got) } // The earliest read of the query evaluated at the earliest instant. - want := buildMetrics(t, "api/v1/query", q).start.Add(-latencyOffset) + lookback, _ := queryLookback("container_cpu_usage_seconds_total", time.Second) + want := at.Add(-2*latencyOffset - lookback) r := h.requests()[0] start, err := time.Parse(time.RFC3339, r.form.Get("start")) if err != nil || start.After(want) {