diff --git a/cmd/observability-proxy/main.go b/cmd/observability-proxy/main.go index 1de6e6e14..4a99aa4cc 100644 --- a/cmd/observability-proxy/main.go +++ b/cmd/observability-proxy/main.go @@ -94,6 +94,9 @@ func parseFlags(args []string) (options, error) { fs.DurationVar(&o.metrics.Limits.MaxRange, "metrics-max-range", 30*24*time.Hour, "longest [start, end] and step a metrics request may use") fs.IntVar(&o.metrics.Limits.MaxPoints, "metrics-max-points", 11000, "most points per series a range query may return") fs.DurationVar(&o.metrics.Limits.MaxTimeout, "metrics-max-timeout", 30*time.Second, "timeout forwarded to vmselect; a longer client timeout is clamped; must be below --request-timeout when that is set") + fs.DurationVar(&o.metrics.App.Lookback, "metrics-app-lookback", 24*time.Hour, "how far before the window the application root looks up the application's pods and PVCs at most") + fs.IntVar(&o.metrics.App.MaxPods, "metrics-app-max-pods", 500, "most pods of one application in the lookup window; more answer 422") + fs.IntVar(&o.metrics.App.MaxPVCs, "metrics-app-max-pvcs", 500, "most PVCs of one application in the lookup window; more answer 422") fs.IntVar(&o.limiter.MaxConcurrentPerUser, "max-concurrent-per-user", 10, "concurrent non-tail requests per user; 0 disables") fs.Float64Var(&o.limiter.RatePerUser, "rate-limit-per-user", 10, "sustained requests per second per user; 0 disables") fs.IntVar(&o.limiter.RateBurst, "rate-limit-burst", 20, "request burst per user") @@ -134,6 +137,8 @@ func parseFlags(args []string) (options, error) { return o, errors.New("--metrics-shortterm-retention, --metrics-max-range and --metrics-max-timeout must be positive") case o.metrics.Limits.MaxPoints < 2: return o, errors.New("--metrics-max-points must be at least 2") + case o.metrics.App.Lookback < 0 || o.metrics.App.MaxPods < 1 || o.metrics.App.MaxPVCs < 1: + return o, errors.New("--metrics-app-lookback must not be negative and --metrics-app-max-pods and --metrics-app-max-pvcs must be positive") case o.logs.MaxTailResume < 0: return o, errors.New("--logs-max-tail-resume must not be negative") // The proxy's deadline starts before the request reaches vmselect, so diff --git a/cmd/observability-proxy/main_test.go b/cmd/observability-proxy/main_test.go index bf7ac396b..6b5c60d7d 100644 --- a/cmd/observability-proxy/main_test.go +++ b/cmd/observability-proxy/main_test.go @@ -40,6 +40,9 @@ func TestParseFlagsDefaults(t *testing.T) { if m.Limits.MaxRange != 30*24*time.Hour || m.Limits.MaxPoints != 11000 || m.Limits.MaxTimeout != 30*time.Second { t.Errorf("metrics limit defaults changed: %+v", m.Limits) } + if m.App.Lookback != 24*time.Hour || m.App.MaxPods != 500 || m.App.MaxPVCs != 500 { + t.Errorf("application scope defaults changed: %+v", m.App) + } if m.RootTenant != o.rootTenant { t.Errorf("metrics root tenant = %q, want --root-tenant", m.RootTenant) } @@ -63,13 +66,15 @@ func TestParseFlagsMetricsOverrides(t *testing.T) { o, err := parseFlags([]string{ "--root-tenant=tenant-x", "--metrics-shortterm-store=fast", "--metrics-longterm-store=slow", "--metrics-shortterm-retention=24h", "--metrics-max-range=168h", "--metrics-max-points=500", "--metrics-max-timeout=10s", + "--metrics-app-lookback=0s", "--metrics-app-max-pods=7", "--metrics-app-max-pvcs=3", }) if err != nil { t.Fatal(err) } m := o.metrics if m.RootTenant != "tenant-x" || m.ShortTermStore != "fast" || m.LongTermStore != "slow" || m.ShortTermRetention != 24*time.Hour || - m.Limits.MaxRange != 168*time.Hour || m.Limits.MaxPoints != 500 || m.Limits.MaxTimeout != 10*time.Second { + m.Limits.MaxRange != 168*time.Hour || m.Limits.MaxPoints != 500 || m.Limits.MaxTimeout != 10*time.Second || + m.App.Lookback != 0 || m.App.MaxPods != 7 || m.App.MaxPVCs != 3 { t.Errorf("overrides not applied: %+v", m) } } @@ -101,6 +106,9 @@ func TestParseFlagsInvalid(t *testing.T) { {"--metrics-max-range=0s"}, {"--metrics-max-points=1"}, {"--metrics-max-timeout=0s"}, + {"--metrics-app-lookback=-1h"}, + {"--metrics-app-max-pods=0"}, + {"--metrics-app-max-pvcs=0"}, {"--unknown"}, // The proxy would cut the request while vmselect is still within // the timeout it was given. diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go new file mode 100644 index 000000000..b969cc502 --- /dev/null +++ b/internal/observabilityproxy/appscope.go @@ -0,0 +1,158 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "context" + "encoding/json" + "io" + "log/slog" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" +) + +// AppScopeLimits bound the pod and PVC lookup of the application metrics +// root. +type AppScopeLimits struct { + // Lookback is how far before the window the lookup reaches at most, so + // pods gone before the window but inside an expression's range count. + Lookback time.Duration + MaxPods int + MaxPVCs int +} + +// maxLookupBytes bounds a lookup answer; the caps keep a valid one far +// below it. +const maxLookupBytes = 8 << 20 + +// The lineage labels as the scrape class writes them on series, and as +// kube-state-metrics writes them on kube_pod_labels and +// kube_persistentvolumeclaim_labels. +const ( + seriesKindLabel = "apps_cozystack_io_application_kind" + seriesNameLabel = "apps_cozystack_io_application_name" + ksmKindLabel = "label_" + seriesKindLabel + ksmNameLabel = "label_" + seriesNameLabel +) + +func identitySelector(ns, kindLabel, nameLabel, kind, name string) string { + return "{namespace=" + strconv.Quote(ns) + "," + kindLabel + "=" + strconv.Quote(kind) + "," + nameLabel + "=" + strconv.Quote(name) + "}" +} + +// nameBranch matches label against any of names within ns. With no names +// it is "": a branch of pod=~"^()$" would match every series without a pod +// label. +func nameBranch(ns, label string, names []string) string { + if len(names) == 0 { + return "" + } + quoted := make([]string, len(names)) + for i, n := range names { + quoted[i] = regexp.QuoteMeta(n) + } + return "{namespace=" + strconv.Quote(ns) + "," + label + "=~" + strconv.Quote("^("+strings.Join(quoted, "|")+")$") + "}" +} + +// applicationSelectors is the application scope: series stamped with the +// application's identity, plus series of its pods and PVCs for those not +// stamped (cAdvisor, kubelet volume stats). Every branch carries the +// namespace, so their OR stays inside it. +// +// Within the namespace, both the identity and the pod and PVC lists come +// from lineage labels on pods and PVCs. They separate applications for a +// grant narrowed by resourceNames only while whoever creates a pod or PVC +// cannot choose those labels; the lineage webhook keeps them as set on an +// object it finds no owner for, and the logs scope rests on the same. +func applicationSelectors(ns, kind, name string, pods, pvcs []string) []string { + out := []string{identitySelector(ns, seriesKindLabel, seriesNameLabel, kind, name)} + for _, b := range []string{nameBranch(ns, "pod", pods), nameBranch(ns, "persistentvolumeclaim", pvcs)} { + if b != "" { + out = append(out, b) + } + } + return out +} + +// applicationScope looks the application's pods and PVCs up in store over +// the request window, reaching back by the query's own lookback up to the +// configured limit. A failed lookup drops its branch, which only narrows +// the answer; a list over its cap is refused, because truncating it would +// silently drop series. +func (s *Server) applicationScope(ctx context.Context, r *http.Request, store string, req metricsRequest, ns, kind, name string) ([]string, *HTTPError) { + lim := s.cfg.Metrics.App + start := req.from.Add(-lim.Lookback) + if req.start.After(start) { + start = req.start + } + var lists [2][]string + for i, l := range []struct { + label, metric string + max int + }{{"pod", "kube_pod_labels", lim.MaxPods}, {"persistentvolumeclaim", "kube_persistentvolumeclaim_labels", lim.MaxPVCs}} { + match := l.metric + identitySelector(ns, ksmKindLabel, ksmNameLabel, kind, name) + names, err := s.lookupNames(ctx, r, store, l.label, match, ns, start, req.to, l.max) + if err != nil { + slog.Warn("application lookup failed; its branch is left out", "label", l.label, "namespace", ns, "kind", kind, "name", name, "error", err) + continue + } + if len(names) > l.max { + return nil, errorf(http.StatusUnprocessableEntity, "the application has more than %d %ss in this window; use a shorter range", l.max, l.label) + } + lists[i] = names + } + return applicationSelectors(ns, kind, name, lists[0], lists[1]), nil +} + +// lookupNames asks for at most max+1 values, enough to tell a list over +// the cap without reading all of it. +func (s *Server) lookupNames(ctx context.Context, r *http.Request, store, label, match, ns string, start, end time.Time, max int) ([]string, error) { + params := url.Values{} + params.Set("match[]", match) + params.Set("extra_filters[]", namespaceSelector(ns)) + params.Set("start", formatTime(start)) + params.Set("end", formatTime(end)) + params.Set("limit", strconv.Itoa(max+1)) + resp, herr := s.post(ctx, r, s.cfg.Metrics.UpstreamURL(store)+"/api/v1/label/"+label+"/values", params, "metrics store", nil) + if herr != nil { + return nil, herr + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return nil, errorf(resp.StatusCode, "metrics store answered %s", resp.Status) + } + var body struct { + Status string `json:"status"` + Data []string `json:"data"` + } + if err := json.NewDecoder(io.LimitReader(resp.Body, maxLookupBytes)).Decode(&body); err != nil { + return nil, err + } + if body.Status != "success" { + return nil, errorf(http.StatusBadGateway, "metrics store answered status %q", body.Status) + } + names := body.Data[:0] + for _, n := range body.Data { + if n != "" { + names = append(names, n) + } + } + return names, nil +} diff --git a/internal/observabilityproxy/appscope_test.go b/internal/observabilityproxy/appscope_test.go new file mode 100644 index 000000000..af028b867 --- /dev/null +++ b/internal/observabilityproxy/appscope_test.go @@ -0,0 +1,116 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "net/url" + "regexp" + "slices" + "strconv" + "testing" + "time" +) + +const wantIdentityBranch = `{namespace="tenant-a",apps_cozystack_io_application_kind="Postgres",apps_cozystack_io_application_name="db.main"}` + +func TestApplicationSelectors(t *testing.T) { + tests := []struct { + name string + pods, pvcs []string + want []string + }{ + {"identity only", nil, nil, []string{wantIdentityBranch}}, + {"pods", []string{"db-main-1", "db.main-2"}, nil, []string{ + wantIdentityBranch, + `{namespace="tenant-a",pod=~"^(db-main-1|db\\.main-2)$"}`, + }}, + {"pvcs", nil, []string{"data-db-main-1"}, []string{ + wantIdentityBranch, + `{namespace="tenant-a",persistentvolumeclaim=~"^(data-db-main-1)$"}`, + }}, + {"all three", []string{"p"}, []string{"v"}, []string{ + wantIdentityBranch, + `{namespace="tenant-a",pod=~"^(p)$"}`, + `{namespace="tenant-a",persistentvolumeclaim=~"^(v)$"}`, + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := applicationSelectors("tenant-a", "Postgres", "db.main", tt.pods, tt.pvcs); !slices.Equal(got, tt.want) { + t.Fatalf("got %q\nwant %q", got, tt.want) + } + }) + } +} + +// DNS-1123 keeps most metacharacters out of names, but the names come +// back from the store, so the encoder must not rely on that. +func TestNameBranchEscapes(t *testing.T) { + names := []string{`a.b`, `x|.*`, `)|(.*`, `q"} or {namespace=~".+`, `back\slash`, `$^+?[]{}`} + sel := nameBranch("tenant-a", "pod", names) + m := regexp.MustCompile(`^\{namespace="tenant-a",pod=~(".*")\}$`).FindStringSubmatch(sel) + if m == nil { + t.Fatalf("selector %s is not one namespace and one pod matcher", sel) + } + pattern, err := strconv.Unquote(m[1]) + if err != nil { + t.Fatalf("pod matcher %s does not unquote: %v", m[1], err) + } + re := regexp.MustCompile(pattern) + for _, n := range names { + if !re.MatchString(n) { + t.Errorf("%q does not match its own branch", n) + } + } + for _, other := range []string{"axb", "x", "anything", "tenant-b", "", `back\\slash`} { + if re.MatchString(other) { + t.Errorf("branch matches %q", other) + } + } + if nameBranch("tenant-a", "pod", nil) != "" { + t.Error("an empty list produced a branch, which would match series without the label") + } +} + +func TestMetricsRequestScope(t *testing.T) { + branches := []string{"{a}", "{b}"} + labels := buildMetrics(t, "api/v1/labels", url.Values{}) + labels.setScope(branches) + if got := labels.params["extra_filters[]"]; !slices.Equal(got, branches) { + t.Errorf("extra_filters[] = %q", got) + } + if got := labels.params["match[]"]; !slices.Equal(got, branches) { + t.Errorf("default match[] = %q, want the branches", got) + } + client := buildMetrics(t, "api/v1/labels", url.Values{"match[]": {"up"}}) + client.setScope(branches) + if got := client.params["match[]"]; !slices.Equal(got, []string{"up"}) { + t.Errorf("client match[] replaced: %q", got) + } + query := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}}) + query.setScope(branches) + if query.params.Has("match[]") || !slices.Equal(query.params["extra_filters[]"], branches) { + t.Errorf("query params = %v", query.params) + } + if !query.from.Equal(testNow) || !query.to.Equal(testNow) { + t.Errorf("instant window = [%s, %s], want now", query.from, query.to) + } + rng := buildMetrics(t, "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-time.Hour))}, "end": {unix(testNow)}, "step": {"60"}}) + if !rng.from.Equal(testNow.Add(-time.Hour)) || !rng.to.Equal(testNow) { + t.Errorf("range window = [%s, %s]", rng.from, rng.to) + } +} diff --git a/internal/observabilityproxy/metricsparams.go b/internal/observabilityproxy/metricsparams.go index 8beb347d0..a0bb42f0d 100644 --- a/internal/observabilityproxy/metricsparams.go +++ b/internal/observabilityproxy/metricsparams.go @@ -46,6 +46,7 @@ type MetricsConfig struct { LongTermStore string ShortTermRetention time.Duration Limits MetricsLimits + App AppScopeLimits // UpstreamURL maps a store name to its Prometheus API base URL, without // a trailing slash; nil means the in-cluster vmselect Service. UpstreamURL func(store string) string @@ -77,6 +78,20 @@ type metricsRequest struct { // start is the earliest instant the request reads, for the store choice; // zero when that is unknown, which picks the long-term store. start time.Time + // from and to are the window the query is evaluated over, without the + // lookback of its expression. + from, to time.Time + // scopedMatch is set when match[] holds the scope, not client matchers. + scopedMatch bool +} + +// setScope replaces the namespace scope with the given selectors, which +// vmselect ORs. +func (req *metricsRequest) setScope(selectors []string) { + req.params["extra_filters[]"] = selectors + if req.scopedMatch { + req.params["match[]"] = selectors + } } // namespaceSelector quotes ns the way MetricsQL unquotes a label value @@ -127,6 +142,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti out.Set("step", formatSeconds(step)) } req.start = queryStart(out.Get("query"), at.Add(-latencyOffset), step) + req.from, req.to = at, at return req, nil case "api/v1/query_range": @@ -150,6 +166,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti return req, errorf(http.StatusUnprocessableEntity, "%d points per series exceed the limit of %d; raise step", points, lim.MaxPoints) } req.start = queryStart(out.Get("query"), start, step) + req.from, req.to = start, end out.Set("start", formatTime(start)) out.Set("end", formatTime(end)) out.Set("step", formatSeconds(step)) @@ -172,7 +189,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti if herr != nil { return req, herr } - req.start = start + req.start, req.from, req.to = start, start, end out.Set("start", formatTime(start)) out.Set("end", formatTime(end)) @@ -194,6 +211,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti // extra_filters[] on the label APIs unless a match[] is present // (getCommonParamsInternal), so the scope is repeated here. out.Set("match[]", namespaceSelector(namespace)) + req.scopedMatch = true } return req, nil } diff --git a/internal/observabilityproxy/server.go b/internal/observabilityproxy/server.go index 617971bf2..fd74491b5 100644 --- a/internal/observabilityproxy/server.go +++ b/internal/observabilityproxy/server.go @@ -193,11 +193,8 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT return herr } } - switch route.Kind { - case NamespaceMetrics: - return s.serveMetrics(w, r, route, user) - case ApplicationMetrics: - return errorf(http.StatusNotFound, "not found") + if route.Kind == NamespaceMetrics || route.Kind == ApplicationMetrics { + return s.serveMetrics(w, r, route, user, kind) } endpoint, ok := strings.CutPrefix(route.Endpoint, "select/logsql/") if !ok { @@ -288,7 +285,9 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT return nil } -func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Route, user string) *HTTPError { +// serveMetrics scopes to the namespace, or with kind set to the application +// route.Name of that kind within it. +func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Route, user, kind string) *HTTPError { release, ok := s.limits.acquire(user) if !ok { return tooManyConcurrent(w) @@ -310,6 +309,13 @@ func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Rout defer cancel() } store := s.cfg.Metrics.store(req.start, now) + if kind != "" { + selectors, herr := s.applicationScope(ctx, r, store, req, route.Namespace, kind, route.Name) + if herr != nil { + return herr + } + req.setScope(selectors) + } resp, herr := s.post(ctx, r, s.cfg.Metrics.UpstreamURL(store)+"/"+req.path, req.params, "metrics store", nil) if herr != nil { return herr diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 0a76ab9a5..14807637e 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -17,11 +17,14 @@ limitations under the License. package observabilityproxy import ( + "encoding/json" "io" "net/http" "net/url" "slices" + "strings" "testing" + "time" authorizationv1 "k8s.io/api/authorization/v1" ) @@ -112,3 +115,170 @@ func TestServerApplicationKinds(t *testing.T) { t.Fatalf("denied caller answered %d, want 403", got) } } + +const wantPodLookup = `kube_pod_labels{namespace="tenant-a",label_apps_cozystack_io_application_kind="Postgres",label_apps_cozystack_io_application_name="db.main"}` + +// appMetricsUpstream answers the pod and PVC lookups with the given names, +// or with status when it is not 200, and every other path like vmselect. +func appMetricsUpstream(pods, pvcs []string, podStatus int) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + names := map[string][]string{"/api/v1/label/pod/values": pods, "/api/v1/label/persistentvolumeclaim/values": pvcs} + list, lookup := names[r.URL.Path] + if !lookup { + promUpstream(w, r) + return + } + if r.URL.Path == "/api/v1/label/pod/values" && podStatus != http.StatusOK { + http.Error(w, "unavailable", podStatus) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"status": "success", "data": list}) + } +} + +func TestServerApplicationMetricsScope(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream([]string{"db-main-0", "db.main-1"}, []string{"data-db-main-0"}, http.StatusOK), appPolicy, nil) + hostile := url.Values{ + "query": {"sum(rate(container_cpu_usage_seconds_total[5m]))"}, + "start": {unix(testNow.Add(-time.Hour))}, + "end": {unix(testNow)}, + "step": {"60"}, + "extra_filters": {`{namespace=~".+"}`}, + "extra_filters[]": {`{namespace="tenant-a"}`}, + "extra_label": {"apps_cozystack_io_application_name=b"}, + } + resp := h.do(http.MethodPost, appRoot+"db.main/metrics/api/v1/query_range", "good", hostile, nil) + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("answered %d %q", resp.StatusCode, body) + } + want := authorizationv1.ResourceAttributes{Verb: "get", Group: "apps.cozystack.io", Resource: "postgreses", Subresource: "metrics", Name: "db.main", Namespace: "tenant-a"} + if got := h.kube.lastSAR.Load().ResourceAttributes; *got != want { + t.Fatalf("SubjectAccessReview = %+v, want %+v", *got, want) + } + reqs := h.requests() + if len(reqs) != 3 { + t.Fatalf("upstream saw %d requests, want two lookups and the query", len(reqs)) + } + lookups := map[string]string{ + "/api/v1/label/pod/values": wantPodLookup, + "/api/v1/label/persistentvolumeclaim/values": `kube_persistentvolumeclaim_labels{namespace="tenant-a",label_apps_cozystack_io_application_kind="Postgres",label_apps_cozystack_io_application_name="db.main"}`, + } + for _, r := range reqs[:2] { + if got := r.form["match[]"]; len(got) != 1 || got[0] != lookups[r.path] { + t.Errorf("%s match[] = %q", r.path, got) + } + if got := r.form["extra_filters[]"]; !slices.Equal(got, []string{wantMetricsScope}) { + t.Errorf("%s extra_filters[] = %q", r.path, got) + } + start, herr := parseMetricsTime("start", r.form.Get("start")) + if herr != nil || start.After(testNow.Add(-time.Hour)) || start.Before(testNow.Add(-25*time.Hour)) { + t.Errorf("%s start = %s, want within the lookback before the window", r.path, r.form.Get("start")) + } + if r.form.Get("end") != formatTime(testNow) || r.form.Get("limit") != "101" { + t.Errorf("%s end/limit = %s/%s", r.path, r.form.Get("end"), r.form.Get("limit")) + } + } + main := reqs[2] + wantBranches := []string{ + wantIdentityBranch, + `{namespace="tenant-a",pod=~"^(db-main-0|db\\.main-1)$"}`, + `{namespace="tenant-a",persistentvolumeclaim=~"^(data-db-main-0)$"}`, + } + if main.path != "/api/v1/query_range" || !slices.Equal(main.form["extra_filters[]"], wantBranches) { + t.Fatalf("query %s extra_filters[] = %q, want %q", main.path, main.form["extra_filters[]"], wantBranches) + } + if main.form.Has("extra_filters") || main.form.Has("extra_label") { + t.Fatalf("client scope reached the store: %v", main.form) + } + if !slices.Equal(h.stores, []string{"shortterm", "shortterm", "shortterm"}) { + t.Fatalf("stores = %q, want the lookups on the query's store", h.stores) + } +} + +func TestServerApplicationMetricsBranches(t *testing.T) { + podBranch := `{namespace="tenant-a",pod=~"^(p)$"}` + tests := []struct { + name string + pods, pvcs []string + podStatus int + cfg Config + path string + wantCode int + want []string + }{ + {name: "no pods or pvcs", path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch}}, + {name: "pod lookup fails", pods: []string{"p"}, pvcs: []string{"v"}, podStatus: http.StatusServiceUnavailable, path: "api/v1/query?query=up", wantCode: http.StatusOK, + want: []string{wantIdentityBranch, `{namespace="tenant-a",persistentvolumeclaim=~"^(v)$"}`}}, + {name: "pods over the cap", pods: []string{"p", "q"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, + path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity}, + {name: "pvcs over the cap", pods: []string{"p"}, pvcs: []string{"v", "w"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, + path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity}, + {name: "labels get the branches as match[]", pods: []string{"p"}, path: "api/v1/labels", wantCode: http.StatusOK, want: []string{wantIdentityBranch, podBranch}}, + {name: "series needs match[]", path: "api/v1/series", wantCode: http.StatusBadRequest}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.podStatus == 0 { + tt.podStatus = http.StatusOK + } + h := newPolicyHarness(t, tt.cfg, appMetricsUpstream(tt.pods, tt.pvcs, tt.podStatus), appPolicy, nil) + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/"+tt.path, "good"); got != tt.wantCode { + t.Fatalf("status = %d, want %d", got, tt.wantCode) + } + var main *upstreamRequest + for _, r := range h.requests() { + if !strings.HasPrefix(r.path, "/api/v1/label/pod/") && !strings.HasPrefix(r.path, "/api/v1/label/persistentvolumeclaim/") { + main = &r + } + } + if tt.want == nil { + if main != nil { + t.Fatalf("query reached the store: %s %v", main.path, main.form) + } + return + } + if main == nil || !slices.Equal(main.form["extra_filters[]"], tt.want) { + t.Fatalf("extra_filters[] = %v, want %q", main, tt.want) + } + if strings.HasSuffix(tt.path, "labels") && !slices.Equal(main.form["match[]"], tt.want) { + t.Fatalf("match[] = %q, want the branches", main.form["match[]"]) + } + }) + } +} + +// A grant narrowed to application "a" by resourceNames must not reach "b" +// through any endpoint of either root. +func TestServerApplicationNarrowedGrant(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream([]string{"a-0"}, nil, http.StatusOK), appPolicy, nil) + paths := []string{ + "metrics/api/v1/query?query=up", + "metrics/api/v1/query_range?query=up&start=" + unix(testNow.Add(-time.Hour)) + "&end=" + unix(testNow) + "&step=60", + "metrics/api/v1/series?match[]=up", + "metrics/api/v1/labels", + "metrics/api/v1/label/pod/values", + "log/select/logsql/query", + "log/select/logsql/hits", + "log/select/logsql/stats_query_range?query=" + url.QueryEscape("* | stats count()"), + "log/select/logsql/tail", + "log/select/logsql/field_values?field=level", + "log/select/logsql/stream_field_values?field=level", + "log/select/logsql/facets", + } + for _, p := range paths { + if got := h.status(http.MethodGet, appRoot+"b/"+p, "bob-token"); got != http.StatusForbidden { + t.Errorf("b/%s answered %d, want 403", p, got) + } + } + if n := len(h.requests()); n != 0 { + t.Fatalf("upstream saw %d requests for a denied application", n) + } + for _, p := range []string{"metrics/api/v1/labels", "log/select/logsql/query"} { + if got := h.status(http.MethodGet, appRoot+"a/"+p, "bob-token"); got != http.StatusOK { + t.Errorf("a/%s answered %d, want 200", p, got) + } + } +} diff --git a/internal/observabilityproxy/server_test.go b/internal/observabilityproxy/server_test.go index b541a18e1..168c402fe 100644 --- a/internal/observabilityproxy/server_test.go +++ b/internal/observabilityproxy/server_test.go @@ -96,6 +96,9 @@ func newPolicyHarness(t *testing.T, cfg Config, handler http.HandlerFunc, allow if cfg.Metrics.Limits == (MetricsLimits{}) { cfg.Metrics.Limits = testMetricsLimits() } + if cfg.Metrics.App == (AppScopeLimits{}) { + cfg.Metrics.App = AppScopeLimits{Lookback: 24 * time.Hour, MaxPods: 100, MaxPVCs: 100} + } if cfg.Logs == (LogsLimits{}) { cfg.Logs = testLimits() }