From dcdf0adab3171c0139766f0999cfb975d0ff97c1 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 06:03:40 +0700 Subject: [PATCH] feat(observability-proxy): serve the application metrics root Graphs of one application need its series without the caller knowing which pods and volumes it ran on. Exporter series carry the identity the scrape class stamps, but cAdvisor and kubelet volume series do not, so the scope ORs three branches: the identity, and the application's pods and PVCs as kube-state-metrics labelled them over the window. Each branch also carries the namespace, so the scope is never wider than the namespace root. The lookup reaches back by the query's own lookback, bounded by a flag, so pods replaced inside an expression's range still count. A failed lookup only drops its branch. A list over its cap answers 422 instead of being truncated, which would hide series without saying so. Assisted-by: LLM Signed-off-by: ZverGuy --- cmd/observability-proxy/main.go | 5 + cmd/observability-proxy/main_test.go | 10 +- internal/observabilityproxy/appscope.go | 158 ++++++++++++++++ internal/observabilityproxy/appscope_test.go | 116 ++++++++++++ internal/observabilityproxy/metricsparams.go | 20 ++- internal/observabilityproxy/server.go | 18 +- .../observabilityproxy/server_app_test.go | 170 ++++++++++++++++++ internal/observabilityproxy/server_test.go | 3 + 8 files changed, 492 insertions(+), 8 deletions(-) create mode 100644 internal/observabilityproxy/appscope.go create mode 100644 internal/observabilityproxy/appscope_test.go diff --git a/cmd/observability-proxy/main.go b/cmd/observability-proxy/main.go index 1de6e6e14..4a99aa4cc 100644 --- a/cmd/observability-proxy/main.go +++ b/cmd/observability-proxy/main.go @@ -94,6 +94,9 @@ func parseFlags(args []string) (options, error) { fs.DurationVar(&o.metrics.Limits.MaxRange, "metrics-max-range", 30*24*time.Hour, "longest [start, end] and step a metrics request may use") fs.IntVar(&o.metrics.Limits.MaxPoints, "metrics-max-points", 11000, "most points per series a range query may return") fs.DurationVar(&o.metrics.Limits.MaxTimeout, "metrics-max-timeout", 30*time.Second, "timeout forwarded to vmselect; a longer client timeout is clamped; must be below --request-timeout when that is set") + fs.DurationVar(&o.metrics.App.Lookback, "metrics-app-lookback", 24*time.Hour, "how far before the window the application root looks up the application's pods and PVCs at most") + fs.IntVar(&o.metrics.App.MaxPods, "metrics-app-max-pods", 500, "most pods of one application in the lookup window; more answer 422") + fs.IntVar(&o.metrics.App.MaxPVCs, "metrics-app-max-pvcs", 500, "most PVCs of one application in the lookup window; more answer 422") fs.IntVar(&o.limiter.MaxConcurrentPerUser, "max-concurrent-per-user", 10, "concurrent non-tail requests per user; 0 disables") fs.Float64Var(&o.limiter.RatePerUser, "rate-limit-per-user", 10, "sustained requests per second per user; 0 disables") fs.IntVar(&o.limiter.RateBurst, "rate-limit-burst", 20, "request burst per user") @@ -134,6 +137,8 @@ func parseFlags(args []string) (options, error) { return o, errors.New("--metrics-shortterm-retention, --metrics-max-range and --metrics-max-timeout must be positive") case o.metrics.Limits.MaxPoints < 2: return o, errors.New("--metrics-max-points must be at least 2") + case o.metrics.App.Lookback < 0 || o.metrics.App.MaxPods < 1 || o.metrics.App.MaxPVCs < 1: + return o, errors.New("--metrics-app-lookback must not be negative and --metrics-app-max-pods and --metrics-app-max-pvcs must be positive") case o.logs.MaxTailResume < 0: return o, errors.New("--logs-max-tail-resume must not be negative") // The proxy's deadline starts before the request reaches vmselect, so diff --git a/cmd/observability-proxy/main_test.go b/cmd/observability-proxy/main_test.go index bf7ac396b..6b5c60d7d 100644 --- a/cmd/observability-proxy/main_test.go +++ b/cmd/observability-proxy/main_test.go @@ -40,6 +40,9 @@ func TestParseFlagsDefaults(t *testing.T) { if m.Limits.MaxRange != 30*24*time.Hour || m.Limits.MaxPoints != 11000 || m.Limits.MaxTimeout != 30*time.Second { t.Errorf("metrics limit defaults changed: %+v", m.Limits) } + if m.App.Lookback != 24*time.Hour || m.App.MaxPods != 500 || m.App.MaxPVCs != 500 { + t.Errorf("application scope defaults changed: %+v", m.App) + } if m.RootTenant != o.rootTenant { t.Errorf("metrics root tenant = %q, want --root-tenant", m.RootTenant) } @@ -63,13 +66,15 @@ func TestParseFlagsMetricsOverrides(t *testing.T) { o, err := parseFlags([]string{ "--root-tenant=tenant-x", "--metrics-shortterm-store=fast", "--metrics-longterm-store=slow", "--metrics-shortterm-retention=24h", "--metrics-max-range=168h", "--metrics-max-points=500", "--metrics-max-timeout=10s", + "--metrics-app-lookback=0s", "--metrics-app-max-pods=7", "--metrics-app-max-pvcs=3", }) if err != nil { t.Fatal(err) } m := o.metrics if m.RootTenant != "tenant-x" || m.ShortTermStore != "fast" || m.LongTermStore != "slow" || m.ShortTermRetention != 24*time.Hour || - m.Limits.MaxRange != 168*time.Hour || m.Limits.MaxPoints != 500 || m.Limits.MaxTimeout != 10*time.Second { + m.Limits.MaxRange != 168*time.Hour || m.Limits.MaxPoints != 500 || m.Limits.MaxTimeout != 10*time.Second || + m.App.Lookback != 0 || m.App.MaxPods != 7 || m.App.MaxPVCs != 3 { t.Errorf("overrides not applied: %+v", m) } } @@ -101,6 +106,9 @@ func TestParseFlagsInvalid(t *testing.T) { {"--metrics-max-range=0s"}, {"--metrics-max-points=1"}, {"--metrics-max-timeout=0s"}, + {"--metrics-app-lookback=-1h"}, + {"--metrics-app-max-pods=0"}, + {"--metrics-app-max-pvcs=0"}, {"--unknown"}, // The proxy would cut the request while vmselect is still within // the timeout it was given. diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go new file mode 100644 index 000000000..b969cc502 --- /dev/null +++ b/internal/observabilityproxy/appscope.go @@ -0,0 +1,158 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "context" + "encoding/json" + "io" + "log/slog" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" +) + +// AppScopeLimits bound the pod and PVC lookup of the application metrics +// root. +type AppScopeLimits struct { + // Lookback is how far before the window the lookup reaches at most, so + // pods gone before the window but inside an expression's range count. + Lookback time.Duration + MaxPods int + MaxPVCs int +} + +// maxLookupBytes bounds a lookup answer; the caps keep a valid one far +// below it. +const maxLookupBytes = 8 << 20 + +// The lineage labels as the scrape class writes them on series, and as +// kube-state-metrics writes them on kube_pod_labels and +// kube_persistentvolumeclaim_labels. +const ( + seriesKindLabel = "apps_cozystack_io_application_kind" + seriesNameLabel = "apps_cozystack_io_application_name" + ksmKindLabel = "label_" + seriesKindLabel + ksmNameLabel = "label_" + seriesNameLabel +) + +func identitySelector(ns, kindLabel, nameLabel, kind, name string) string { + return "{namespace=" + strconv.Quote(ns) + "," + kindLabel + "=" + strconv.Quote(kind) + "," + nameLabel + "=" + strconv.Quote(name) + "}" +} + +// nameBranch matches label against any of names within ns. With no names +// it is "": a branch of pod=~"^()$" would match every series without a pod +// label. +func nameBranch(ns, label string, names []string) string { + if len(names) == 0 { + return "" + } + quoted := make([]string, len(names)) + for i, n := range names { + quoted[i] = regexp.QuoteMeta(n) + } + return "{namespace=" + strconv.Quote(ns) + "," + label + "=~" + strconv.Quote("^("+strings.Join(quoted, "|")+")$") + "}" +} + +// applicationSelectors is the application scope: series stamped with the +// application's identity, plus series of its pods and PVCs for those not +// stamped (cAdvisor, kubelet volume stats). Every branch carries the +// namespace, so their OR stays inside it. +// +// Within the namespace, both the identity and the pod and PVC lists come +// from lineage labels on pods and PVCs. They separate applications for a +// grant narrowed by resourceNames only while whoever creates a pod or PVC +// cannot choose those labels; the lineage webhook keeps them as set on an +// object it finds no owner for, and the logs scope rests on the same. +func applicationSelectors(ns, kind, name string, pods, pvcs []string) []string { + out := []string{identitySelector(ns, seriesKindLabel, seriesNameLabel, kind, name)} + for _, b := range []string{nameBranch(ns, "pod", pods), nameBranch(ns, "persistentvolumeclaim", pvcs)} { + if b != "" { + out = append(out, b) + } + } + return out +} + +// applicationScope looks the application's pods and PVCs up in store over +// the request window, reaching back by the query's own lookback up to the +// configured limit. A failed lookup drops its branch, which only narrows +// the answer; a list over its cap is refused, because truncating it would +// silently drop series. +func (s *Server) applicationScope(ctx context.Context, r *http.Request, store string, req metricsRequest, ns, kind, name string) ([]string, *HTTPError) { + lim := s.cfg.Metrics.App + start := req.from.Add(-lim.Lookback) + if req.start.After(start) { + start = req.start + } + var lists [2][]string + for i, l := range []struct { + label, metric string + max int + }{{"pod", "kube_pod_labels", lim.MaxPods}, {"persistentvolumeclaim", "kube_persistentvolumeclaim_labels", lim.MaxPVCs}} { + match := l.metric + identitySelector(ns, ksmKindLabel, ksmNameLabel, kind, name) + names, err := s.lookupNames(ctx, r, store, l.label, match, ns, start, req.to, l.max) + if err != nil { + slog.Warn("application lookup failed; its branch is left out", "label", l.label, "namespace", ns, "kind", kind, "name", name, "error", err) + continue + } + if len(names) > l.max { + return nil, errorf(http.StatusUnprocessableEntity, "the application has more than %d %ss in this window; use a shorter range", l.max, l.label) + } + lists[i] = names + } + return applicationSelectors(ns, kind, name, lists[0], lists[1]), nil +} + +// lookupNames asks for at most max+1 values, enough to tell a list over +// the cap without reading all of it. +func (s *Server) lookupNames(ctx context.Context, r *http.Request, store, label, match, ns string, start, end time.Time, max int) ([]string, error) { + params := url.Values{} + params.Set("match[]", match) + params.Set("extra_filters[]", namespaceSelector(ns)) + params.Set("start", formatTime(start)) + params.Set("end", formatTime(end)) + params.Set("limit", strconv.Itoa(max+1)) + resp, herr := s.post(ctx, r, s.cfg.Metrics.UpstreamURL(store)+"/api/v1/label/"+label+"/values", params, "metrics store", nil) + if herr != nil { + return nil, herr + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return nil, errorf(resp.StatusCode, "metrics store answered %s", resp.Status) + } + var body struct { + Status string `json:"status"` + Data []string `json:"data"` + } + if err := json.NewDecoder(io.LimitReader(resp.Body, maxLookupBytes)).Decode(&body); err != nil { + return nil, err + } + if body.Status != "success" { + return nil, errorf(http.StatusBadGateway, "metrics store answered status %q", body.Status) + } + names := body.Data[:0] + for _, n := range body.Data { + if n != "" { + names = append(names, n) + } + } + return names, nil +} diff --git a/internal/observabilityproxy/appscope_test.go b/internal/observabilityproxy/appscope_test.go new file mode 100644 index 000000000..af028b867 --- /dev/null +++ b/internal/observabilityproxy/appscope_test.go @@ -0,0 +1,116 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "net/url" + "regexp" + "slices" + "strconv" + "testing" + "time" +) + +const wantIdentityBranch = `{namespace="tenant-a",apps_cozystack_io_application_kind="Postgres",apps_cozystack_io_application_name="db.main"}` + +func TestApplicationSelectors(t *testing.T) { + tests := []struct { + name string + pods, pvcs []string + want []string + }{ + {"identity only", nil, nil, []string{wantIdentityBranch}}, + {"pods", []string{"db-main-1", "db.main-2"}, nil, []string{ + wantIdentityBranch, + `{namespace="tenant-a",pod=~"^(db-main-1|db\\.main-2)$"}`, + }}, + {"pvcs", nil, []string{"data-db-main-1"}, []string{ + wantIdentityBranch, + `{namespace="tenant-a",persistentvolumeclaim=~"^(data-db-main-1)$"}`, + }}, + {"all three", []string{"p"}, []string{"v"}, []string{ + wantIdentityBranch, + `{namespace="tenant-a",pod=~"^(p)$"}`, + `{namespace="tenant-a",persistentvolumeclaim=~"^(v)$"}`, + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := applicationSelectors("tenant-a", "Postgres", "db.main", tt.pods, tt.pvcs); !slices.Equal(got, tt.want) { + t.Fatalf("got %q\nwant %q", got, tt.want) + } + }) + } +} + +// DNS-1123 keeps most metacharacters out of names, but the names come +// back from the store, so the encoder must not rely on that. +func TestNameBranchEscapes(t *testing.T) { + names := []string{`a.b`, `x|.*`, `)|(.*`, `q"} or {namespace=~".+`, `back\slash`, `$^+?[]{}`} + sel := nameBranch("tenant-a", "pod", names) + m := regexp.MustCompile(`^\{namespace="tenant-a",pod=~(".*")\}$`).FindStringSubmatch(sel) + if m == nil { + t.Fatalf("selector %s is not one namespace and one pod matcher", sel) + } + pattern, err := strconv.Unquote(m[1]) + if err != nil { + t.Fatalf("pod matcher %s does not unquote: %v", m[1], err) + } + re := regexp.MustCompile(pattern) + for _, n := range names { + if !re.MatchString(n) { + t.Errorf("%q does not match its own branch", n) + } + } + for _, other := range []string{"axb", "x", "anything", "tenant-b", "", `back\\slash`} { + if re.MatchString(other) { + t.Errorf("branch matches %q", other) + } + } + if nameBranch("tenant-a", "pod", nil) != "" { + t.Error("an empty list produced a branch, which would match series without the label") + } +} + +func TestMetricsRequestScope(t *testing.T) { + branches := []string{"{a}", "{b}"} + labels := buildMetrics(t, "api/v1/labels", url.Values{}) + labels.setScope(branches) + if got := labels.params["extra_filters[]"]; !slices.Equal(got, branches) { + t.Errorf("extra_filters[] = %q", got) + } + if got := labels.params["match[]"]; !slices.Equal(got, branches) { + t.Errorf("default match[] = %q, want the branches", got) + } + client := buildMetrics(t, "api/v1/labels", url.Values{"match[]": {"up"}}) + client.setScope(branches) + if got := client.params["match[]"]; !slices.Equal(got, []string{"up"}) { + t.Errorf("client match[] replaced: %q", got) + } + query := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}}) + query.setScope(branches) + if query.params.Has("match[]") || !slices.Equal(query.params["extra_filters[]"], branches) { + t.Errorf("query params = %v", query.params) + } + if !query.from.Equal(testNow) || !query.to.Equal(testNow) { + t.Errorf("instant window = [%s, %s], want now", query.from, query.to) + } + rng := buildMetrics(t, "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-time.Hour))}, "end": {unix(testNow)}, "step": {"60"}}) + if !rng.from.Equal(testNow.Add(-time.Hour)) || !rng.to.Equal(testNow) { + t.Errorf("range window = [%s, %s]", rng.from, rng.to) + } +} diff --git a/internal/observabilityproxy/metricsparams.go b/internal/observabilityproxy/metricsparams.go index 8beb347d0..a0bb42f0d 100644 --- a/internal/observabilityproxy/metricsparams.go +++ b/internal/observabilityproxy/metricsparams.go @@ -46,6 +46,7 @@ type MetricsConfig struct { LongTermStore string ShortTermRetention time.Duration Limits MetricsLimits + App AppScopeLimits // UpstreamURL maps a store name to its Prometheus API base URL, without // a trailing slash; nil means the in-cluster vmselect Service. UpstreamURL func(store string) string @@ -77,6 +78,20 @@ type metricsRequest struct { // start is the earliest instant the request reads, for the store choice; // zero when that is unknown, which picks the long-term store. start time.Time + // from and to are the window the query is evaluated over, without the + // lookback of its expression. + from, to time.Time + // scopedMatch is set when match[] holds the scope, not client matchers. + scopedMatch bool +} + +// setScope replaces the namespace scope with the given selectors, which +// vmselect ORs. +func (req *metricsRequest) setScope(selectors []string) { + req.params["extra_filters[]"] = selectors + if req.scopedMatch { + req.params["match[]"] = selectors + } } // namespaceSelector quotes ns the way MetricsQL unquotes a label value @@ -127,6 +142,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti out.Set("step", formatSeconds(step)) } req.start = queryStart(out.Get("query"), at.Add(-latencyOffset), step) + req.from, req.to = at, at return req, nil case "api/v1/query_range": @@ -150,6 +166,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti return req, errorf(http.StatusUnprocessableEntity, "%d points per series exceed the limit of %d; raise step", points, lim.MaxPoints) } req.start = queryStart(out.Get("query"), start, step) + req.from, req.to = start, end out.Set("start", formatTime(start)) out.Set("end", formatTime(end)) out.Set("step", formatSeconds(step)) @@ -172,7 +189,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti if herr != nil { return req, herr } - req.start = start + req.start, req.from, req.to = start, start, end out.Set("start", formatTime(start)) out.Set("end", formatTime(end)) @@ -194,6 +211,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti // extra_filters[] on the label APIs unless a match[] is present // (getCommonParamsInternal), so the scope is repeated here. out.Set("match[]", namespaceSelector(namespace)) + req.scopedMatch = true } return req, nil } diff --git a/internal/observabilityproxy/server.go b/internal/observabilityproxy/server.go index 617971bf2..fd74491b5 100644 --- a/internal/observabilityproxy/server.go +++ b/internal/observabilityproxy/server.go @@ -193,11 +193,8 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT return herr } } - switch route.Kind { - case NamespaceMetrics: - return s.serveMetrics(w, r, route, user) - case ApplicationMetrics: - return errorf(http.StatusNotFound, "not found") + if route.Kind == NamespaceMetrics || route.Kind == ApplicationMetrics { + return s.serveMetrics(w, r, route, user, kind) } endpoint, ok := strings.CutPrefix(route.Endpoint, "select/logsql/") if !ok { @@ -288,7 +285,9 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT return nil } -func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Route, user string) *HTTPError { +// serveMetrics scopes to the namespace, or with kind set to the application +// route.Name of that kind within it. +func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Route, user, kind string) *HTTPError { release, ok := s.limits.acquire(user) if !ok { return tooManyConcurrent(w) @@ -310,6 +309,13 @@ func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Rout defer cancel() } store := s.cfg.Metrics.store(req.start, now) + if kind != "" { + selectors, herr := s.applicationScope(ctx, r, store, req, route.Namespace, kind, route.Name) + if herr != nil { + return herr + } + req.setScope(selectors) + } resp, herr := s.post(ctx, r, s.cfg.Metrics.UpstreamURL(store)+"/"+req.path, req.params, "metrics store", nil) if herr != nil { return herr diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 0a76ab9a5..14807637e 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -17,11 +17,14 @@ limitations under the License. package observabilityproxy import ( + "encoding/json" "io" "net/http" "net/url" "slices" + "strings" "testing" + "time" authorizationv1 "k8s.io/api/authorization/v1" ) @@ -112,3 +115,170 @@ func TestServerApplicationKinds(t *testing.T) { t.Fatalf("denied caller answered %d, want 403", got) } } + +const wantPodLookup = `kube_pod_labels{namespace="tenant-a",label_apps_cozystack_io_application_kind="Postgres",label_apps_cozystack_io_application_name="db.main"}` + +// appMetricsUpstream answers the pod and PVC lookups with the given names, +// or with status when it is not 200, and every other path like vmselect. +func appMetricsUpstream(pods, pvcs []string, podStatus int) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + names := map[string][]string{"/api/v1/label/pod/values": pods, "/api/v1/label/persistentvolumeclaim/values": pvcs} + list, lookup := names[r.URL.Path] + if !lookup { + promUpstream(w, r) + return + } + if r.URL.Path == "/api/v1/label/pod/values" && podStatus != http.StatusOK { + http.Error(w, "unavailable", podStatus) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"status": "success", "data": list}) + } +} + +func TestServerApplicationMetricsScope(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream([]string{"db-main-0", "db.main-1"}, []string{"data-db-main-0"}, http.StatusOK), appPolicy, nil) + hostile := url.Values{ + "query": {"sum(rate(container_cpu_usage_seconds_total[5m]))"}, + "start": {unix(testNow.Add(-time.Hour))}, + "end": {unix(testNow)}, + "step": {"60"}, + "extra_filters": {`{namespace=~".+"}`}, + "extra_filters[]": {`{namespace="tenant-a"}`}, + "extra_label": {"apps_cozystack_io_application_name=b"}, + } + resp := h.do(http.MethodPost, appRoot+"db.main/metrics/api/v1/query_range", "good", hostile, nil) + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("answered %d %q", resp.StatusCode, body) + } + want := authorizationv1.ResourceAttributes{Verb: "get", Group: "apps.cozystack.io", Resource: "postgreses", Subresource: "metrics", Name: "db.main", Namespace: "tenant-a"} + if got := h.kube.lastSAR.Load().ResourceAttributes; *got != want { + t.Fatalf("SubjectAccessReview = %+v, want %+v", *got, want) + } + reqs := h.requests() + if len(reqs) != 3 { + t.Fatalf("upstream saw %d requests, want two lookups and the query", len(reqs)) + } + lookups := map[string]string{ + "/api/v1/label/pod/values": wantPodLookup, + "/api/v1/label/persistentvolumeclaim/values": `kube_persistentvolumeclaim_labels{namespace="tenant-a",label_apps_cozystack_io_application_kind="Postgres",label_apps_cozystack_io_application_name="db.main"}`, + } + for _, r := range reqs[:2] { + if got := r.form["match[]"]; len(got) != 1 || got[0] != lookups[r.path] { + t.Errorf("%s match[] = %q", r.path, got) + } + if got := r.form["extra_filters[]"]; !slices.Equal(got, []string{wantMetricsScope}) { + t.Errorf("%s extra_filters[] = %q", r.path, got) + } + start, herr := parseMetricsTime("start", r.form.Get("start")) + if herr != nil || start.After(testNow.Add(-time.Hour)) || start.Before(testNow.Add(-25*time.Hour)) { + t.Errorf("%s start = %s, want within the lookback before the window", r.path, r.form.Get("start")) + } + if r.form.Get("end") != formatTime(testNow) || r.form.Get("limit") != "101" { + t.Errorf("%s end/limit = %s/%s", r.path, r.form.Get("end"), r.form.Get("limit")) + } + } + main := reqs[2] + wantBranches := []string{ + wantIdentityBranch, + `{namespace="tenant-a",pod=~"^(db-main-0|db\\.main-1)$"}`, + `{namespace="tenant-a",persistentvolumeclaim=~"^(data-db-main-0)$"}`, + } + if main.path != "/api/v1/query_range" || !slices.Equal(main.form["extra_filters[]"], wantBranches) { + t.Fatalf("query %s extra_filters[] = %q, want %q", main.path, main.form["extra_filters[]"], wantBranches) + } + if main.form.Has("extra_filters") || main.form.Has("extra_label") { + t.Fatalf("client scope reached the store: %v", main.form) + } + if !slices.Equal(h.stores, []string{"shortterm", "shortterm", "shortterm"}) { + t.Fatalf("stores = %q, want the lookups on the query's store", h.stores) + } +} + +func TestServerApplicationMetricsBranches(t *testing.T) { + podBranch := `{namespace="tenant-a",pod=~"^(p)$"}` + tests := []struct { + name string + pods, pvcs []string + podStatus int + cfg Config + path string + wantCode int + want []string + }{ + {name: "no pods or pvcs", path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch}}, + {name: "pod lookup fails", pods: []string{"p"}, pvcs: []string{"v"}, podStatus: http.StatusServiceUnavailable, path: "api/v1/query?query=up", wantCode: http.StatusOK, + want: []string{wantIdentityBranch, `{namespace="tenant-a",persistentvolumeclaim=~"^(v)$"}`}}, + {name: "pods over the cap", pods: []string{"p", "q"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, + path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity}, + {name: "pvcs over the cap", pods: []string{"p"}, pvcs: []string{"v", "w"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, + path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity}, + {name: "labels get the branches as match[]", pods: []string{"p"}, path: "api/v1/labels", wantCode: http.StatusOK, want: []string{wantIdentityBranch, podBranch}}, + {name: "series needs match[]", path: "api/v1/series", wantCode: http.StatusBadRequest}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.podStatus == 0 { + tt.podStatus = http.StatusOK + } + h := newPolicyHarness(t, tt.cfg, appMetricsUpstream(tt.pods, tt.pvcs, tt.podStatus), appPolicy, nil) + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/"+tt.path, "good"); got != tt.wantCode { + t.Fatalf("status = %d, want %d", got, tt.wantCode) + } + var main *upstreamRequest + for _, r := range h.requests() { + if !strings.HasPrefix(r.path, "/api/v1/label/pod/") && !strings.HasPrefix(r.path, "/api/v1/label/persistentvolumeclaim/") { + main = &r + } + } + if tt.want == nil { + if main != nil { + t.Fatalf("query reached the store: %s %v", main.path, main.form) + } + return + } + if main == nil || !slices.Equal(main.form["extra_filters[]"], tt.want) { + t.Fatalf("extra_filters[] = %v, want %q", main, tt.want) + } + if strings.HasSuffix(tt.path, "labels") && !slices.Equal(main.form["match[]"], tt.want) { + t.Fatalf("match[] = %q, want the branches", main.form["match[]"]) + } + }) + } +} + +// A grant narrowed to application "a" by resourceNames must not reach "b" +// through any endpoint of either root. +func TestServerApplicationNarrowedGrant(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream([]string{"a-0"}, nil, http.StatusOK), appPolicy, nil) + paths := []string{ + "metrics/api/v1/query?query=up", + "metrics/api/v1/query_range?query=up&start=" + unix(testNow.Add(-time.Hour)) + "&end=" + unix(testNow) + "&step=60", + "metrics/api/v1/series?match[]=up", + "metrics/api/v1/labels", + "metrics/api/v1/label/pod/values", + "log/select/logsql/query", + "log/select/logsql/hits", + "log/select/logsql/stats_query_range?query=" + url.QueryEscape("* | stats count()"), + "log/select/logsql/tail", + "log/select/logsql/field_values?field=level", + "log/select/logsql/stream_field_values?field=level", + "log/select/logsql/facets", + } + for _, p := range paths { + if got := h.status(http.MethodGet, appRoot+"b/"+p, "bob-token"); got != http.StatusForbidden { + t.Errorf("b/%s answered %d, want 403", p, got) + } + } + if n := len(h.requests()); n != 0 { + t.Fatalf("upstream saw %d requests for a denied application", n) + } + for _, p := range []string{"metrics/api/v1/labels", "log/select/logsql/query"} { + if got := h.status(http.MethodGet, appRoot+"a/"+p, "bob-token"); got != http.StatusOK { + t.Errorf("a/%s answered %d, want 200", p, got) + } + } +} diff --git a/internal/observabilityproxy/server_test.go b/internal/observabilityproxy/server_test.go index b541a18e1..168c402fe 100644 --- a/internal/observabilityproxy/server_test.go +++ b/internal/observabilityproxy/server_test.go @@ -96,6 +96,9 @@ func newPolicyHarness(t *testing.T, cfg Config, handler http.HandlerFunc, allow if cfg.Metrics.Limits == (MetricsLimits{}) { cfg.Metrics.Limits = testMetricsLimits() } + if cfg.Metrics.App == (AppScopeLimits{}) { + cfg.Metrics.App = AppScopeLimits{Lookback: 24 * time.Hour, MaxPods: 100, MaxPVCs: 100} + } if cfg.Logs == (LogsLimits{}) { cfg.Logs = testLimits() }