mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-22 04:33:36 +09:00
fix(observability-proxy): put the latency window in the instant start
The second latency offset was added to the application lookup only, so the other readers of the instant query's earliest read still assumed time minus 30s: the store choice could send a query to the short-term store up to 30 seconds too late near its retention edge, the lookback cap compared against a start the lookup no longer used, and the field no longer meant the earliest instant read. The earliest read of an instant query now starts from time minus two latency offsets, the earliest instant vmselect can evaluate it at, and the lookup uses it as is. Assisted-by: LLM Signed-off-by: ZverGuy <maximbel2003@gmail.com>
This commit is contained in:
1 parent
cf45a23bf3
commit
ac5a3c88c4
4 files changed
+25
-13
No files matched your search
@@ -110,14 +110,6 @@ func (s *Server) applicationScope(ctx context.Context, r *http.Request, store st
|
||||
return nil, errorf(http.StatusUnprocessableEntity, "the query reads further back than the application scope looks up pods (%s before the window, or an unknown lookback); use a shorter window or the namespace root", lim.Lookback)
|
||||
}
|
||||
start := req.start
|
||||
// vmselect evaluates an instant query whose time is within 30s of its
|
||||
// own clock at that clock minus 30s (queryOffset in prometheus.go), so
|
||||
// the evaluation instant lies anywhere in (time-60s, time], while
|
||||
// req.start assumes time-30s. One more latency offset covers it without
|
||||
// trusting the proxy's clock to match vmselect's.
|
||||
if req.path == "api/v1/query" {
|
||||
start = start.Add(-latencyOffset)
|
||||
}
|
||||
var lists [2][]string
|
||||
for i, l := range []struct {
|
||||
label, metric string
|
||||
|
||||
@@ -144,7 +144,11 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti
|
||||
}
|
||||
out.Set("step", formatSeconds(step))
|
||||
}
|
||||
req.start = queryStart(out.Get("query"), at.Add(-latencyOffset), step)
|
||||
// vmselect evaluates an instant query whose time is within the
|
||||
// latency offset of its own clock at that clock minus the offset
|
||||
// (queryOffset in prometheus.go), so the evaluation instant lies in
|
||||
// (time-2*offset, time]; the earlier end needs no clock to agree.
|
||||
req.start = queryStart(out.Get("query"), at.Add(-2*latencyOffset), step)
|
||||
req.from, req.to = at, at
|
||||
if ahead, ok := queryLookahead(out.Get("query"), step); ok {
|
||||
req.last = at.Add(ahead)
|
||||
|
||||
@@ -227,7 +227,7 @@ func TestBuildMetricsParamsCanonicalForms(t *testing.T) {
|
||||
if req.params.Get("end") != "2026-11-04T12:00:00.000Z" || req.params.Get("start") != "2026-11-04T11:55:00.000Z" {
|
||||
t.Errorf("labels window = %s..%s, want the last 5m", req.params.Get("start"), req.params.Get("end"))
|
||||
}
|
||||
if q := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}}); q.params.Has("time") || !q.start.Equal(testNow.Add(-15*time.Minute-30*time.Second)) {
|
||||
if q := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}}); q.params.Has("time") || !q.start.Equal(testNow.Add(-15*time.Minute-2*latencyOffset)) {
|
||||
t.Errorf("instant query without time: params %v, start %v", q.params, q.start)
|
||||
}
|
||||
}
|
||||
@@ -543,3 +543,19 @@ func TestQueryLookaheadSubquery(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuildMetricsParamsInstantStartCoversLatencyOffset: vmselect evaluates
|
||||
// an instant query whose time is within 30s of its clock at that clock
|
||||
// minus 30s, anywhere in (time-60s, time], so the earliest read the store
|
||||
// choice and the lookup rely on starts from time-60s.
|
||||
func TestBuildMetricsParamsInstantStartCoversLatencyOffset(t *testing.T) {
|
||||
at := testNow.Add(29 * time.Second)
|
||||
req := buildMetrics(t, "api/v1/query", url.Values{"query": {"x"}, "time": {at.Format(time.RFC3339)}, "step": {"1s"}})
|
||||
lookback, ok := queryLookback("x", time.Second)
|
||||
if !ok {
|
||||
t.Fatal("lookback of x unknown")
|
||||
}
|
||||
if want := at.Add(-2*latencyOffset - lookback); req.start.After(want) {
|
||||
t.Fatalf("start = %s, want at most %s", req.start, want)
|
||||
}
|
||||
}
|
||||
@@ -341,8 +341,7 @@ func TestServerApplicationLookbackBeyondCap(t *testing.T) {
|
||||
t.Fatalf("rate(x[5m]) answered %d, want 200", got)
|
||||
}
|
||||
r := h.requests()[0]
|
||||
// An instant query may be evaluated up to one more latency offset early.
|
||||
want := formatTime(buildMetrics(t, "api/v1/query", q).start.Add(-latencyOffset))
|
||||
want := formatTime(buildMetrics(t, "api/v1/query", q).start)
|
||||
if r.path != "/api/v1/label/pod/values" || r.form.Get("start") != want {
|
||||
t.Fatalf("lookup %s start = %s, want the query's earliest read %s", r.path, r.form.Get("start"), want)
|
||||
}
|
||||
@@ -581,7 +580,8 @@ func TestServerApplicationLookupCoversLatencyOffset(t *testing.T) {
|
||||
t.Fatalf("status = %d", got)
|
||||
}
|
||||
// The earliest read of the query evaluated at the earliest instant.
|
||||
want := buildMetrics(t, "api/v1/query", q).start.Add(-latencyOffset)
|
||||
lookback, _ := queryLookback("container_cpu_usage_seconds_total", time.Second)
|
||||
want := at.Add(-2*latencyOffset - lookback)
|
||||
r := h.requests()[0]
|
||||
start, err := time.Parse(time.RFC3339, r.form.Get("start"))
|
||||
if err != nil || start.After(want) {
|
||||
|
||||
Reference in new issue
Block a user