mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 20:23:39 +09:00
fix(observability-proxy): refuse app queries that read past the lookup
The pod and PVC lookup never started earlier than --metrics-app-lookback before the window. A query that read further back, such as rate(x[7d]) with the 24h default, or whose lookback is unknown (an @ modifier, WITH, a step-multiple window), answered 200 without the series of pods that existed only before the lookup start. Such queries now answer 422 on the application root, matching how a query whose latest sample cannot be covered is handled; the namespace root has no lookup and still serves them. Assisted-by: LLM Signed-off-by: ZverGuy <maximbel2003@gmail.com>
This commit is contained in:
1 parent
bc823b704e
commit
84665a42a7
3 files changed
+31
-17
No files matched your search
@@ -94,7 +94,7 @@ func parseFlags(args []string) (options, error) {
|
||||
fs.DurationVar(&o.metrics.Limits.MaxRange, "metrics-max-range", 30*24*time.Hour, "longest [start, end] and step a metrics request may use")
|
||||
fs.IntVar(&o.metrics.Limits.MaxPoints, "metrics-max-points", 11000, "most points per series a range query may return")
|
||||
fs.DurationVar(&o.metrics.Limits.MaxTimeout, "metrics-max-timeout", 30*time.Second, "timeout forwarded to vmselect; a longer client timeout is clamped; must be below --request-timeout when that is set")
|
||||
fs.DurationVar(&o.metrics.App.Lookback, "metrics-app-lookback", 24*time.Hour, "how far before the window the application root looks up the application's pods and PVCs at most")
|
||||
fs.DurationVar(&o.metrics.App.Lookback, "metrics-app-lookback", 24*time.Hour, "how far before the window the application root looks up the application's pods and PVCs; a query reading further back answers 422")
|
||||
fs.IntVar(&o.metrics.App.MaxPods, "metrics-app-max-pods", 500, "most pods of one application in the lookup window; more answer 422")
|
||||
fs.IntVar(&o.metrics.App.MaxPVCs, "metrics-app-max-pvcs", 500, "most PVCs of one application in the lookup window; more answer 422")
|
||||
fs.IntVar(&o.limiter.MaxConcurrentPerUser, "max-concurrent-per-user", 10, "concurrent non-tail requests per user; 0 disables")
|
||||
|
||||
@@ -94,20 +94,21 @@ func applicationSelectors(ns, kind, name string, pods, pvcs []string) []string {
|
||||
}
|
||||
|
||||
// applicationScope looks the application's pods and PVCs up in store over
|
||||
// the request window, reaching back by the query's own lookback up to the
|
||||
// configured limit and forward to the latest sample a negative offset
|
||||
// reads. A failed lookup, a list over its cap, or a query whose latest
|
||||
// sample is unknown fails the request, because a successful answer would
|
||||
// silently miss the application's series.
|
||||
// the samples the request reads: back by the query's own lookback, and
|
||||
// forward to the latest sample a negative offset reads. A query that reads
|
||||
// further back than the configured lookback, or whose earliest or latest
|
||||
// sample is unknown, a failed lookup, and a list over its cap all fail the
|
||||
// request, because a successful answer would silently miss the
|
||||
// application's series.
|
||||
func (s *Server) applicationScope(ctx context.Context, r *http.Request, store string, req metricsRequest, ns, kind, name string) ([]string, *HTTPError) {
|
||||
if req.last.IsZero() {
|
||||
return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ other than start() or end(), or an offset that is not a plain duration); use the namespace root")
|
||||
}
|
||||
lim := s.cfg.Metrics.App
|
||||
start := req.from.Add(-lim.Lookback)
|
||||
if req.start.After(start) {
|
||||
start = req.start
|
||||
if req.start.IsZero() || req.start.Before(req.from.Add(-lim.Lookback)) {
|
||||
return nil, errorf(http.StatusUnprocessableEntity, "the query reads further back than the application scope looks up pods (%s before the window, or an unknown lookback); use a shorter window or the namespace root", lim.Lookback)
|
||||
}
|
||||
start := req.start
|
||||
var lists [2][]string
|
||||
for i, l := range []struct {
|
||||
label, metric string
|
||||
|
||||
@@ -316,17 +316,30 @@ func TestServerApplicationNarrowedGrant(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// An expression whose lookback cannot be bounded looks pods up from the
|
||||
// configured lookback before the window.
|
||||
func TestServerApplicationLookupUnknownLookback(t *testing.T) {
|
||||
h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil)
|
||||
q := url.Values{"query": {"up @ end()"}, "time": {unix(testNow)}}
|
||||
// TestServerApplicationLookbackBeyondCap: the lookup reaches back at most
|
||||
// --metrics-app-lookback, so a query that reads further back, or whose
|
||||
// lookback is unknown, would miss pods that existed only earlier; it is
|
||||
// refused rather than answered without their series.
|
||||
func TestServerApplicationLookbackBeyondCap(t *testing.T) {
|
||||
cfg := Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 100, MaxPVCs: 100}}}
|
||||
h := newPolicyHarness(t, cfg, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil)
|
||||
for _, query := range []string{"rate(x[7d])", "x[30d] @ end()", "rate(x[5i])"} {
|
||||
q := url.Values{"query": {query}, "time": {unix(testNow)}}
|
||||
if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusUnprocessableEntity {
|
||||
t.Errorf("%s answered %d, want 422", query, got)
|
||||
}
|
||||
}
|
||||
if n := len(h.requests()); n != 0 {
|
||||
t.Fatalf("upstream saw %d requests for refused queries", n)
|
||||
}
|
||||
q := url.Values{"query": {"rate(x[5m])"}, "time": {unix(testNow)}}
|
||||
if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusOK {
|
||||
t.Fatalf("status = %d", got)
|
||||
t.Fatalf("rate(x[5m]) answered %d, want 200", got)
|
||||
}
|
||||
r := h.requests()[0]
|
||||
if want := formatTime(testNow.Add(-24 * time.Hour)); r.path != "/api/v1/label/pod/values" || r.form.Get("start") != want {
|
||||
t.Fatalf("lookup %s start = %s, want %s", r.path, r.form.Get("start"), want)
|
||||
want := formatTime(buildMetrics(t, "api/v1/query", q).start)
|
||||
if r.path != "/api/v1/label/pod/values" || r.form.Get("start") != want {
|
||||
t.Fatalf("lookup %s start = %s, want the query's earliest read %s", r.path, r.form.Get("start"), want)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user