mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
fix(observability-proxy): stop refusing ordinary app queries on lookback
With the lookback refusal, the 24h default turned common application
queries into 422: a day's increase, and month-long panels at the step
a dashboard picks for them. The lookback estimate is a loose upper
bound, made for the store choice, and counts every "(" as an implicit
subquery, so it far exceeds what such queries read.
The lookback is only the refusal threshold, since the lookup itself
spans the window the query reads, so its default now matches
--metrics-max-range. The chart passes it from values, so an operator
can tune it.
Assisted-by: LLM
Signed-off-by: ZverGuy <maximbel2003@gmail.com>
This commit is contained in:
1 parent
86dddb447a
commit
85fb98dd2c
6 files changed
+42
-2
No files matched your search
@@ -94,7 +94,11 @@ func parseFlags(args []string) (options, error) {
|
||||
fs.DurationVar(&o.metrics.Limits.MaxRange, "metrics-max-range", 30*24*time.Hour, "longest [start, end] and step a metrics request may use")
|
||||
fs.IntVar(&o.metrics.Limits.MaxPoints, "metrics-max-points", 11000, "most points per series a range query may return")
|
||||
fs.DurationVar(&o.metrics.Limits.MaxTimeout, "metrics-max-timeout", 30*time.Second, "timeout forwarded to vmselect; a longer client timeout is clamped; must be below --request-timeout when that is set")
|
||||
fs.DurationVar(&o.metrics.App.Lookback, "metrics-app-lookback", 24*time.Hour, "how far before the window the application root looks up the application's pods and PVCs; a query reading further back answers 422")
|
||||
// Only the refusal threshold: the lookup itself spans the window the
|
||||
// query reads, and that estimate is a loose upper bound (every "("
|
||||
// counts as an implicit subquery), so the default matches
|
||||
// --metrics-max-range to serve month-long panels at a dashboard step.
|
||||
fs.DurationVar(&o.metrics.App.Lookback, "metrics-app-lookback", 30*24*time.Hour, "how far before the window an application metrics query may read; one reading further back, or by an unknown amount, answers 422")
|
||||
fs.IntVar(&o.metrics.App.MaxPods, "metrics-app-max-pods", 500, "most pods of one application in the lookup window; more answer 422")
|
||||
fs.IntVar(&o.metrics.App.MaxPVCs, "metrics-app-max-pvcs", 500, "most PVCs of one application in the lookup window; more answer 422")
|
||||
fs.IntVar(&o.limiter.MaxConcurrentPerUser, "max-concurrent-per-user", 10, "concurrent non-tail requests per user; 0 disables")
|
||||
|
||||
@@ -40,7 +40,7 @@ func TestParseFlagsDefaults(t *testing.T) {
|
||||
if m.Limits.MaxRange != 30*24*time.Hour || m.Limits.MaxPoints != 11000 || m.Limits.MaxTimeout != 30*time.Second {
|
||||
t.Errorf("metrics limit defaults changed: %+v", m.Limits)
|
||||
}
|
||||
if m.App.Lookback != 24*time.Hour || m.App.MaxPods != 500 || m.App.MaxPVCs != 500 {
|
||||
if m.App.Lookback != 30*24*time.Hour || m.App.MaxPods != 500 || m.App.MaxPVCs != 500 {
|
||||
t.Errorf("application scope defaults changed: %+v", m.App)
|
||||
}
|
||||
if m.RootTenant != o.rootTenant {
|
||||
|
||||
@@ -424,3 +424,28 @@ func TestServerApplicationLookupDeadline(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestServerApplicationDefaultLookbackServesLongPanels: the lookback is
|
||||
// only the refusal threshold, and the lookback estimate is a loose upper
|
||||
// bound, so the default must serve a day's increase and a month-long
|
||||
// panel at the step a dashboard picks for it.
|
||||
func TestServerApplicationDefaultLookbackServesLongPanels(t *testing.T) {
|
||||
cfg := Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: 30 * 24 * time.Hour, MaxPods: 100, MaxPVCs: 100}}}
|
||||
h := newPolicyHarness(t, cfg, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil)
|
||||
month := url.Values{"start": {unix(testNow.Add(-29 * 24 * time.Hour))}, "end": {unix(testNow)}}
|
||||
for _, tc := range []struct{ path, query, step string }{
|
||||
{"api/v1/query", "increase(x[24h])", ""},
|
||||
{"api/v1/query_range", "rate(x[5m])", "21600"},
|
||||
{"api/v1/query_range", "histogram_quantile(0.99, sum by (le) (rate(x[5m])))", "7200"},
|
||||
} {
|
||||
q := url.Values{"query": {tc.query}}
|
||||
if tc.step != "" {
|
||||
q.Set("step", tc.step)
|
||||
q.Set("start", month.Get("start"))
|
||||
q.Set("end", month.Get("end"))
|
||||
}
|
||||
if got := h.status(http.MethodGet, appRoot+"db.main/metrics/"+tc.path+"?"+q.Encode(), "good"); got != http.StatusOK {
|
||||
t.Errorf("%s step %s answered %d, want 200", tc.query, tc.step, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -41,6 +41,7 @@ spec:
|
||||
- --metrics-shortterm-store={{ .Values.observabilityProxy.metrics.shorttermStore }}
|
||||
- --metrics-longterm-store={{ .Values.observabilityProxy.metrics.longtermStore }}
|
||||
- --metrics-shortterm-retention={{ .Values.observabilityProxy.metrics.shorttermRetention }}
|
||||
- --metrics-app-lookback={{ .Values.observabilityProxy.metrics.appLookback }}
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8080
|
||||
|
||||
@@ -51,6 +51,9 @@ tests:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].args
|
||||
content: --metrics-shortterm-retention=72h
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].args
|
||||
content: --metrics-app-lookback=720h
|
||||
|
||||
- it: follows renamed metrics stores
|
||||
set:
|
||||
@@ -58,6 +61,7 @@ tests:
|
||||
shorttermStore: fast
|
||||
longtermStore: slow
|
||||
shorttermRetention: 24h
|
||||
appLookback: 48h
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].args
|
||||
@@ -68,6 +72,9 @@ tests:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].args
|
||||
content: --metrics-shortterm-retention=24h
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].args
|
||||
content: --metrics-app-lookback=48h
|
||||
|
||||
- it: falls back to cozy.local when the platform passes no cluster domain
|
||||
asserts:
|
||||
|
||||
@@ -14,6 +14,9 @@ observabilityProxy:
|
||||
shorttermStore: shortterm
|
||||
longtermStore: longterm
|
||||
shorttermRetention: 72h
|
||||
# How far before its window an application metrics query may read;
|
||||
# one reading further back answers 422 on the application root.
|
||||
appLookback: 720h
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
|
||||
Reference in new issue
Block a user