fix(observability-proxy): stop refusing ordinary app queries on lookback

With the lookback refusal, the 24h default turned common application
queries into 422: a day's increase, and month-long panels at the step
a dashboard picks for them. The lookback estimate is a loose upper
bound, made for the store choice, and counts every "(" as an implicit
subquery, so it far exceeds what such queries read.

The lookback is only the refusal threshold, since the lookup itself
spans the window the query reads, so its default now matches
--metrics-max-range. The chart passes it from values, so an operator
can tune it.

Assisted-by: LLM
Signed-off-by: ZverGuy <maximbel2003@gmail.com>
This commit is contained in:
ZverGuy committed 2026-11-10 04:03:24 +07:00
1 parent 86dddb447a
commit 85fb98dd2c
6 files changed
+42 -2

No files matched your search

+5 -1
View File
@@ -94,7 +94,11 @@ func parseFlags(args []string) (options, error) {
fs.DurationVar(&o.metrics.Limits.MaxRange, "metrics-max-range", 30*24*time.Hour, "longest [start, end] and step a metrics request may use")
fs.IntVar(&o.metrics.Limits.MaxPoints, "metrics-max-points", 11000, "most points per series a range query may return")
fs.DurationVar(&o.metrics.Limits.MaxTimeout, "metrics-max-timeout", 30*time.Second, "timeout forwarded to vmselect; a longer client timeout is clamped; must be below --request-timeout when that is set")
fs.DurationVar(&o.metrics.App.Lookback, "metrics-app-lookback", 24*time.Hour, "how far before the window the application root looks up the application's pods and PVCs; a query reading further back answers 422")
// Only the refusal threshold: the lookup itself spans the window the
// query reads, and that estimate is a loose upper bound (every "("
// counts as an implicit subquery), so the default matches
// --metrics-max-range to serve month-long panels at a dashboard step.
fs.DurationVar(&o.metrics.App.Lookback, "metrics-app-lookback", 30*24*time.Hour, "how far before the window an application metrics query may read; one reading further back, or by an unknown amount, answers 422")
fs.IntVar(&o.metrics.App.MaxPods, "metrics-app-max-pods", 500, "most pods of one application in the lookup window; more answer 422")
fs.IntVar(&o.metrics.App.MaxPVCs, "metrics-app-max-pvcs", 500, "most PVCs of one application in the lookup window; more answer 422")
fs.IntVar(&o.limiter.MaxConcurrentPerUser, "max-concurrent-per-user", 10, "concurrent non-tail requests per user; 0 disables")
+1 -1
View File
@@ -40,7 +40,7 @@ func TestParseFlagsDefaults(t *testing.T) {
if m.Limits.MaxRange != 30*24*time.Hour || m.Limits.MaxPoints != 11000 || m.Limits.MaxTimeout != 30*time.Second {
t.Errorf("metrics limit defaults changed: %+v", m.Limits)
}
if m.App.Lookback != 24*time.Hour || m.App.MaxPods != 500 || m.App.MaxPVCs != 500 {
if m.App.Lookback != 30*24*time.Hour || m.App.MaxPods != 500 || m.App.MaxPVCs != 500 {
t.Errorf("application scope defaults changed: %+v", m.App)
}
if m.RootTenant != o.rootTenant {
@@ -424,3 +424,28 @@ func TestServerApplicationLookupDeadline(t *testing.T) {
}
}
}
// TestServerApplicationDefaultLookbackServesLongPanels: the lookback is
// only the refusal threshold, and the lookback estimate is a loose upper
// bound, so the default must serve a day's increase and a month-long
// panel at the step a dashboard picks for it.
func TestServerApplicationDefaultLookbackServesLongPanels(t *testing.T) {
cfg := Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: 30 * 24 * time.Hour, MaxPods: 100, MaxPVCs: 100}}}
h := newPolicyHarness(t, cfg, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil)
month := url.Values{"start": {unix(testNow.Add(-29 * 24 * time.Hour))}, "end": {unix(testNow)}}
for _, tc := range []struct{ path, query, step string }{
{"api/v1/query", "increase(x[24h])", ""},
{"api/v1/query_range", "rate(x[5m])", "21600"},
{"api/v1/query_range", "histogram_quantile(0.99, sum by (le) (rate(x[5m])))", "7200"},
} {
q := url.Values{"query": {tc.query}}
if tc.step != "" {
q.Set("step", tc.step)
q.Set("start", month.Get("start"))
q.Set("end", month.Get("end"))
}
if got := h.status(http.MethodGet, appRoot+"db.main/metrics/"+tc.path+"?"+q.Encode(), "good"); got != http.StatusOK {
t.Errorf("%s step %s answered %d, want 200", tc.query, tc.step, got)
}
}
}
@@ -41,6 +41,7 @@ spec:
- --metrics-shortterm-store={{ .Values.observabilityProxy.metrics.shorttermStore }}
- --metrics-longterm-store={{ .Values.observabilityProxy.metrics.longtermStore }}
- --metrics-shortterm-retention={{ .Values.observabilityProxy.metrics.shorttermRetention }}
- --metrics-app-lookback={{ .Values.observabilityProxy.metrics.appLookback }}
ports:
- name: http
containerPort: 8080
@@ -51,6 +51,9 @@ tests:
- contains:
path: spec.template.spec.containers[0].args
content: --metrics-shortterm-retention=72h
- contains:
path: spec.template.spec.containers[0].args
content: --metrics-app-lookback=720h
- it: follows renamed metrics stores
set:
@@ -58,6 +61,7 @@ tests:
shorttermStore: fast
longtermStore: slow
shorttermRetention: 24h
appLookback: 48h
asserts:
- contains:
path: spec.template.spec.containers[0].args
@@ -68,6 +72,9 @@ tests:
- contains:
path: spec.template.spec.containers[0].args
content: --metrics-shortterm-retention=24h
- contains:
path: spec.template.spec.containers[0].args
content: --metrics-app-lookback=48h
- it: falls back to cozy.local when the platform passes no cluster domain
asserts:
@@ -14,6 +14,9 @@ observabilityProxy:
shorttermStore: shortterm
longtermStore: longterm
shorttermRetention: 72h
# How far before its window an application metrics query may read;
# one reading further back answers 422 on the application root.
appLookback: 720h
resources:
requests:
cpu: 10m