diff --git a/cmd/observability-proxy/main.go b/cmd/observability-proxy/main.go index a3fa9b259..487b6d04d 100644 --- a/cmd/observability-proxy/main.go +++ b/cmd/observability-proxy/main.go @@ -94,7 +94,11 @@ func parseFlags(args []string) (options, error) { fs.DurationVar(&o.metrics.Limits.MaxRange, "metrics-max-range", 30*24*time.Hour, "longest [start, end] and step a metrics request may use") fs.IntVar(&o.metrics.Limits.MaxPoints, "metrics-max-points", 11000, "most points per series a range query may return") fs.DurationVar(&o.metrics.Limits.MaxTimeout, "metrics-max-timeout", 30*time.Second, "timeout forwarded to vmselect; a longer client timeout is clamped; must be below --request-timeout when that is set") - fs.DurationVar(&o.metrics.App.Lookback, "metrics-app-lookback", 24*time.Hour, "how far before the window the application root looks up the application's pods and PVCs; a query reading further back answers 422") + // Only the refusal threshold: the lookup itself spans the window the + // query reads, and that estimate is a loose upper bound (every "(" + // counts as an implicit subquery), so the default matches + // --metrics-max-range to serve month-long panels at a dashboard step. + fs.DurationVar(&o.metrics.App.Lookback, "metrics-app-lookback", 30*24*time.Hour, "how far before the window an application metrics query may read; one reading further back, or by an unknown amount, answers 422") fs.IntVar(&o.metrics.App.MaxPods, "metrics-app-max-pods", 500, "most pods of one application in the lookup window; more answer 422") fs.IntVar(&o.metrics.App.MaxPVCs, "metrics-app-max-pvcs", 500, "most PVCs of one application in the lookup window; more answer 422") fs.IntVar(&o.limiter.MaxConcurrentPerUser, "max-concurrent-per-user", 10, "concurrent non-tail requests per user; 0 disables") diff --git a/cmd/observability-proxy/main_test.go b/cmd/observability-proxy/main_test.go index bf8e032c2..616d25c51 100644 --- a/cmd/observability-proxy/main_test.go +++ b/cmd/observability-proxy/main_test.go @@ -40,7 +40,7 @@ func TestParseFlagsDefaults(t *testing.T) { if m.Limits.MaxRange != 30*24*time.Hour || m.Limits.MaxPoints != 11000 || m.Limits.MaxTimeout != 30*time.Second { t.Errorf("metrics limit defaults changed: %+v", m.Limits) } - if m.App.Lookback != 24*time.Hour || m.App.MaxPods != 500 || m.App.MaxPVCs != 500 { + if m.App.Lookback != 30*24*time.Hour || m.App.MaxPods != 500 || m.App.MaxPVCs != 500 { t.Errorf("application scope defaults changed: %+v", m.App) } if m.RootTenant != o.rootTenant { diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index d1d691b00..0b894b8fc 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -424,3 +424,28 @@ func TestServerApplicationLookupDeadline(t *testing.T) { } } } + +// TestServerApplicationDefaultLookbackServesLongPanels: the lookback is +// only the refusal threshold, and the lookback estimate is a loose upper +// bound, so the default must serve a day's increase and a month-long +// panel at the step a dashboard picks for it. +func TestServerApplicationDefaultLookbackServesLongPanels(t *testing.T) { + cfg := Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: 30 * 24 * time.Hour, MaxPods: 100, MaxPVCs: 100}}} + h := newPolicyHarness(t, cfg, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + month := url.Values{"start": {unix(testNow.Add(-29 * 24 * time.Hour))}, "end": {unix(testNow)}} + for _, tc := range []struct{ path, query, step string }{ + {"api/v1/query", "increase(x[24h])", ""}, + {"api/v1/query_range", "rate(x[5m])", "21600"}, + {"api/v1/query_range", "histogram_quantile(0.99, sum by (le) (rate(x[5m])))", "7200"}, + } { + q := url.Values{"query": {tc.query}} + if tc.step != "" { + q.Set("step", tc.step) + q.Set("start", month.Get("start")) + q.Set("end", month.Get("end")) + } + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/"+tc.path+"?"+q.Encode(), "good"); got != http.StatusOK { + t.Errorf("%s step %s answered %d, want 200", tc.query, tc.step, got) + } + } +} diff --git a/packages/system/observability-proxy/templates/deployment.yaml b/packages/system/observability-proxy/templates/deployment.yaml index d49de140b..5cb32833f 100644 --- a/packages/system/observability-proxy/templates/deployment.yaml +++ b/packages/system/observability-proxy/templates/deployment.yaml @@ -41,6 +41,7 @@ spec: - --metrics-shortterm-store={{ .Values.observabilityProxy.metrics.shorttermStore }} - --metrics-longterm-store={{ .Values.observabilityProxy.metrics.longtermStore }} - --metrics-shortterm-retention={{ .Values.observabilityProxy.metrics.shorttermRetention }} + - --metrics-app-lookback={{ .Values.observabilityProxy.metrics.appLookback }} ports: - name: http containerPort: 8080 diff --git a/packages/system/observability-proxy/tests/deployment_test.yaml b/packages/system/observability-proxy/tests/deployment_test.yaml index a5e948920..0f0104660 100644 --- a/packages/system/observability-proxy/tests/deployment_test.yaml +++ b/packages/system/observability-proxy/tests/deployment_test.yaml @@ -51,6 +51,9 @@ tests: - contains: path: spec.template.spec.containers[0].args content: --metrics-shortterm-retention=72h + - contains: + path: spec.template.spec.containers[0].args + content: --metrics-app-lookback=720h - it: follows renamed metrics stores set: @@ -58,6 +61,7 @@ tests: shorttermStore: fast longtermStore: slow shorttermRetention: 24h + appLookback: 48h asserts: - contains: path: spec.template.spec.containers[0].args @@ -68,6 +72,9 @@ tests: - contains: path: spec.template.spec.containers[0].args content: --metrics-shortterm-retention=24h + - contains: + path: spec.template.spec.containers[0].args + content: --metrics-app-lookback=48h - it: falls back to cozy.local when the platform passes no cluster domain asserts: diff --git a/packages/system/observability-proxy/values.yaml b/packages/system/observability-proxy/values.yaml index 67334a237..3e3ac70fe 100644 --- a/packages/system/observability-proxy/values.yaml +++ b/packages/system/observability-proxy/values.yaml @@ -14,6 +14,9 @@ observabilityProxy: shorttermStore: shortterm longtermStore: longterm shorttermRetention: 72h + # How far before its window an application metrics query may read; + # one reading further back answers 422 on the application root. + appLookback: 720h resources: requests: cpu: 10m