diff --git a/internal/observabilityproxy/metricsparams.go b/internal/observabilityproxy/metricsparams.go index 8beb347d0..c300cdc7a 100644 --- a/internal/observabilityproxy/metricsparams.go +++ b/internal/observabilityproxy/metricsparams.go @@ -22,6 +22,7 @@ import ( "net/http" "net/url" "regexp" + "slices" "strconv" "strings" "time" @@ -46,6 +47,9 @@ type MetricsConfig struct { LongTermStore string ShortTermRetention time.Duration Limits MetricsLimits + // PlatformNamespacePrefix names the namespaces whose exporters may + // attribute a series to another namespace; empty trusts none. + PlatformNamespacePrefix string // UpstreamURL maps a store name to its Prometheus API base URL, without // a trailing slash; nil means the in-cluster vmselect Service. UpstreamURL func(store string) string @@ -85,15 +89,34 @@ func namespaceSelector(ns string) string { return "{namespace=" + strconv.Quote(ns) + "}" } +// namespaceScope selects the series of namespace ns. Besides its own, it +// takes the series an exporter in a platform namespace attributes to ns: +// vmagent sets namespace to the scraped target's and, without honorLabels, +// moves the exporter's own namespace label to exported_namespace. Only +// platform namespaces are trusted with that label, since any tenant pod can +// write it; tenants cannot run pods or scrape objects in them. +func (c MetricsConfig) namespaceScope(ns string) []string { + scope := []string{namespaceSelector(ns)} + if c.PlatformNamespacePrefix != "" { + // MetricsQL anchors =~ at both ends. + platform := strconv.Quote(regexp.QuoteMeta(c.PlatformNamespacePrefix) + ".*") + scope = append(scope, "{namespace=~"+platform+",exported_namespace="+strconv.Quote(ns)+"}") + } + return scope +} + // BuildMetricsParams rebuilds the upstream parameters from an allowlist and -// adds the namespace scope as the only extra_filters[] value. vmselect ORs -// repeated extra_filters, so a client value would widen the scope; it ANDs -// the single one into every selector of the query and every match[], which -// is what lets the query pass through unparsed. -func BuildMetricsParams(endpoint string, in url.Values, namespace string, now time.Time, lim MetricsLimits) (metricsRequest, *HTTPError) { +// sends scope as the only extra_filters[] values. vmselect ORs repeated +// extra_filters, so a client value would widen the scope; it ANDs each into +// every selector of the query and every match[], which is what lets the +// query pass through unparsed. +func BuildMetricsParams(endpoint string, in url.Values, scope []string, now time.Time, lim MetricsLimits) (metricsRequest, *HTTPError) { req := metricsRequest{path: endpoint, params: url.Values{}} + if len(scope) == 0 { + return req, errorf(http.StatusInternalServerError, "no metrics scope") + } out := req.params - out.Set("extra_filters[]", namespaceSelector(namespace)) + out["extra_filters[]"] = slices.Clone(scope) timeout := lim.MaxTimeout if v := in.Get("timeout"); v != "" { d, herr := parseMetricsDuration("timeout", v) @@ -192,8 +215,11 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti } // vmselect started with -search.ignoreExtraFiltersAtLabelsAPI drops // extra_filters[] on the label APIs unless a match[] is present - // (getCommonParamsInternal), so the scope is repeated here. - out.Set("match[]", namespaceSelector(namespace)) + // (getCommonParamsInternal), so the scope is repeated here. Where both + // are sent, vmselect ANDs every match[] with every extra_filters[]; + // a cross pair is narrower than one of its halves, so the union is + // still the scope. + out["match[]"] = slices.Clone(scope) } return req, nil } diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go index a4a435567..58551a4f7 100644 --- a/internal/observabilityproxy/metricsparams_test.go +++ b/internal/observabilityproxy/metricsparams_test.go @@ -19,6 +19,7 @@ package observabilityproxy import ( "net/http" "net/url" + "slices" "strconv" "strings" "testing" @@ -29,13 +30,20 @@ func testMetricsLimits() MetricsLimits { return MetricsLimits{MaxRange: 30 * 24 * time.Hour, MaxPoints: 11000, MaxTimeout: 30 * time.Second} } -const wantMetricsScope = `{namespace="tenant-a"}` +// wantMetricsScope is the namespace root's scope under the default +// platform prefix: the namespace's own series, then the series a platform +// exporter attributes to it. +var wantMetricsScope = []string{`{namespace="tenant-a"}`, `{namespace=~"cozy-.*",exported_namespace="tenant-a"}`} + +func testScope() []string { + return MetricsConfig{PlatformNamespacePrefix: "cozy-"}.namespaceScope("tenant-a") +} func unix(t time.Time) string { return strconv.FormatInt(t.Unix(), 10) } func buildMetrics(t *testing.T, endpoint string, in url.Values) metricsRequest { t.Helper() - req, err := BuildMetricsParams(endpoint, in, "tenant-a", testNow, testMetricsLimits()) + req, err := BuildMetricsParams(endpoint, in, testScope(), testNow, testMetricsLimits()) if err != nil { t.Fatalf("BuildMetricsParams(%s, %v): %v", endpoint, in, err) } @@ -43,7 +51,7 @@ func buildMetrics(t *testing.T, endpoint string, in url.Values) metricsRequest { } func metricsStatus(endpoint string, in url.Values) int { - _, err := BuildMetricsParams(endpoint, in, "tenant-a", testNow, testMetricsLimits()) + _, err := BuildMetricsParams(endpoint, in, testScope(), testNow, testMetricsLimits()) if err == nil { return http.StatusOK } @@ -84,8 +92,8 @@ func TestBuildMetricsParamsAllowlist(t *testing.T) { t.Errorf("parameter %q forwarded: %v", k, req.params) } } - if got := req.params["extra_filters[]"]; len(got) != 1 || got[0] != wantMetricsScope { - t.Errorf("extra_filters[] = %q, want exactly [%s]", got, wantMetricsScope) + if got := req.params["extra_filters[]"]; !slices.Equal(got, wantMetricsScope) { + t.Errorf("extra_filters[] = %q, want exactly %q", got, wantMetricsScope) } for _, m := range req.params["match[]"] { if strings.Contains(m, "tenant-b") { @@ -116,8 +124,8 @@ func TestNamespaceSelectorEscapes(t *testing.T) { func TestBuildMetricsParamsLabelEndpointsAlwaysMatch(t *testing.T) { for _, endpoint := range []string{"api/v1/labels", "api/v1/label/__name__/values"} { req := buildMetrics(t, endpoint, url.Values{}) - if got := req.params["match[]"]; len(got) != 1 || got[0] != wantMetricsScope { - t.Errorf("%s without match[] sends %q, want the namespace selector", endpoint, got) + if got := req.params["match[]"]; !slices.Equal(got, wantMetricsScope) { + t.Errorf("%s without match[] sends %q, want the namespace scope %q", endpoint, got, wantMetricsScope) } req = buildMetrics(t, endpoint, url.Values{"match[]": {"up", `{job="x"}`}}) if got := req.params["match[]"]; len(got) != 2 || got[0] != "up" || got[1] != `{job="x"}` { @@ -198,7 +206,7 @@ func TestBuildMetricsParamsCanonicalForms(t *testing.T) { }) want := url.Values{ "query": {"up"}, "start": {"2026-11-04T11:00:00.250Z"}, "end": {"2026-11-04T12:00:00.000Z"}, "step": {"90"}, - "timeout": {"30"}, "extra_filters[]": {wantMetricsScope}, + "timeout": {"30"}, "extra_filters[]": wantMetricsScope, } if req.params.Encode() != want.Encode() { t.Errorf("params = %v\nwant %v", req.params, want) diff --git a/internal/observabilityproxy/metricsscope_test.go b/internal/observabilityproxy/metricsscope_test.go new file mode 100644 index 000000000..ff8d52f41 --- /dev/null +++ b/internal/observabilityproxy/metricsscope_test.go @@ -0,0 +1,237 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "net/http" + "net/url" + "regexp" + "slices" + "strconv" + "strings" + "testing" + "time" +) + +func TestNamespaceScope(t *testing.T) { + tests := []struct { + prefix string + want []string + }{ + {"cozy-", []string{`{namespace="tenant-a"}`, `{namespace=~"cozy-.*",exported_namespace="tenant-a"}`}}, + {"", []string{`{namespace="tenant-a"}`}}, + // A prefix is literal: a dot in it must not match any character. + {"sys.", []string{`{namespace="tenant-a"}`, `{namespace=~"sys\\..*",exported_namespace="tenant-a"}`}}, + } + for _, tt := range tests { + if got := (MetricsConfig{PlatformNamespacePrefix: tt.prefix}).namespaceScope("tenant-a"); !slices.Equal(got, tt.want) { + t.Errorf("prefix %q: scope = %q, want %q", tt.prefix, got, tt.want) + } + } +} + +func TestNamespaceScopeEscapesExportedNamespace(t *testing.T) { + for _, ns := range []string{`a"} or {namespace=~".+`, `back\slash`, "new\nline"} { + scope := MetricsConfig{PlatformNamespacePrefix: "cozy-"}.namespaceScope(ns) + filters, ok := parseSelector(scope[1]) + if !ok || len(filters) != 2 || filters[1].label != "exported_namespace" || filters[1].value != ns { + t.Errorf("scope %q does not decode to exported_namespace=%q: %+v", scope[1], ns, filters) + } + } +} + +// TestBuildMetricsParamsScopeOnEveryEndpoint checks that every endpoint of +// the namespace root sends the whole scope and nothing a client adds. +func TestBuildMetricsParamsScopeOnEveryEndpoint(t *testing.T) { + endpoints := map[string]url.Values{ + "api/v1/query": {"query": {"up"}}, + "api/v1/query_range": {"query": {"up"}, "start": {unix(testNow.Add(-5 * time.Minute))}, "end": {unix(testNow)}, "step": {"60"}}, + "api/v1/series": {"match[]": {"up"}}, + "api/v1/labels": {}, + "api/v1/label/pod/values": {}, + } + for _, prefix := range []string{"cozy-", ""} { + scope := MetricsConfig{PlatformNamespacePrefix: prefix}.namespaceScope("tenant-a") + for endpoint, in := range endpoints { + in = cloneValues(in) + in["extra_filters[]"] = []string{`{namespace=~".+"}`} + req, herr := BuildMetricsParams(endpoint, in, scope, testNow, testMetricsLimits()) + if herr != nil { + t.Fatalf("%s: %v", endpoint, herr) + } + if got := req.params["extra_filters[]"]; !slices.Equal(got, scope) { + t.Errorf("prefix %q, %s: extra_filters[] = %q, want %q", prefix, endpoint, got, scope) + } + } + } +} + +// An empty scope would send no extra_filters[] and read every namespace. +func TestBuildMetricsParamsRefusesEmptyScope(t *testing.T) { + if _, herr := BuildMetricsParams("api/v1/query", url.Values{"query": {"up"}}, nil, testNow, testMetricsLimits()); herr == nil || herr.Code != http.StatusInternalServerError { + t.Errorf("empty scope: %v, want 500", herr) + } +} + +// TestBuildMetricsParamsCopiesScope guards against the forwarded parameters +// sharing the caller's slice. +func TestBuildMetricsParamsCopiesScope(t *testing.T) { + scope := testScope() + req, herr := BuildMetricsParams("api/v1/labels", url.Values{}, scope, testNow, testMetricsLimits()) + if herr != nil { + t.Fatal(herr) + } + scope[1] = `{namespace=~".+"}` + if !slices.Equal(req.params["extra_filters[]"], wantMetricsScope) || !slices.Equal(req.params["match[]"], wantMetricsScope) { + t.Errorf("params follow the caller's slice: %v", req.params) + } +} + +// TestVmselectSeesScopedSeries runs the parameters the proxy builds +// through a model of vmselect v1.136's series selection: repeated +// extra_filters[] are ORed (searchutil.GetExtraTagFilters), and every +// match[] or query selector is ANDed with each of them +// (searchutil.JoinTagFilterss, called from getCommonParamsInternal and +// promql eval). Regex filters match the whole value. +func TestVmselectSeesScopedSeries(t *testing.T) { + series := []struct { + labels map[string]string + visible bool + }{ + {map[string]string{"namespace": "tenant-a"}, true}, + {map[string]string{"namespace": "cozy-clickhouse-operator", "exported_namespace": "tenant-a", "chi": "clickhouse-db"}, true}, + // A tenant pod claiming another namespace is not a platform exporter. + {map[string]string{"namespace": "tenant-y", "exported_namespace": "tenant-a"}, false}, + {map[string]string{"namespace": "xcozy-a", "exported_namespace": "tenant-a"}, false}, + {map[string]string{"namespace": "cozy-clickhouse-operator", "exported_namespace": "tenant-b"}, false}, + {map[string]string{"namespace": "cozy-clickhouse-operator"}, false}, + {map[string]string{"namespace": "tenant-b"}, false}, + } + for _, endpoint := range []string{"api/v1/series", "api/v1/labels"} { + in := url.Values{} + if endpoint == "api/v1/series" { + in.Set("match[]", `{__name__=~".*"}`) + } + req := buildMetrics(t, endpoint, in) + for _, s := range series { + if got := vmselectSelects(t, req.params, s.labels); got != s.visible { + t.Errorf("%s: series %v visible = %v, want %v", endpoint, s.labels, got, s.visible) + } + } + } + + // Without a prefix the exporter's series stay out of the namespace. + req, herr := BuildMetricsParams("api/v1/labels", url.Values{}, MetricsConfig{}.namespaceScope("tenant-a"), testNow, testMetricsLimits()) + if herr != nil { + t.Fatal(herr) + } + if vmselectSelects(t, req.params, map[string]string{"namespace": "cozy-clickhouse-operator", "exported_namespace": "tenant-a"}) { + t.Error("exporter series visible with the platform prefix disabled") + } +} + +func cloneValues(in url.Values) url.Values { + out := url.Values{} + for k, v := range in { + out[k] = slices.Clone(v) + } + return out +} + +type labelFilter struct { + label, value string + regex bool +} + +func (f labelFilter) matches(labels map[string]string) bool { + v := labels[f.label] + if f.regex { + return regexp.MustCompile("^(?:" + f.value + ")$").MatchString(v) + } + return v == f.value +} + +// parseSelector reads the selectors the proxy and these tests write: +// {name="value",name=~"regex",...} with values quoted as MetricsQL reads +// them (strconv.Unquote in metricsql's extractStringValue). +func parseSelector(s string) ([]labelFilter, bool) { + body, ok := strings.CutPrefix(s, "{") + if !ok { + return nil, false + } + var out []labelFilter + for body != "}" { + i := strings.IndexAny(body, "=!") + if i <= 0 { + return nil, false + } + f := labelFilter{label: body[:i]} + body = body[i:] + switch { + case strings.HasPrefix(body, "=~"): + f.regex, body = true, body[2:] + case strings.HasPrefix(body, "="): + body = body[1:] + default: + return nil, false + } + lit, err := strconv.QuotedPrefix(body) + if err != nil { + return nil, false + } + if f.value, err = strconv.Unquote(lit); err != nil { + return nil, false + } + out = append(out, f) + body = strings.TrimPrefix(body[len(lit):], ",") + } + return out, true +} + +func vmselectSelects(t *testing.T, params url.Values, labels map[string]string) bool { + t.Helper() + parse := func(sels []string) [][]labelFilter { + var out [][]labelFilter + for _, s := range sels { + f, ok := parseSelector(s) + if !ok { + t.Fatalf("cannot parse selector %q", s) + } + out = append(out, f) + } + return out + } + matches, extra := parse(params["match[]"]), parse(params["extra_filters[]"]) + if len(matches) == 0 { + matches = [][]labelFilter{nil} + } + for _, m := range matches { + for _, e := range extra { + all := true + for _, f := range append(slices.Clone(m), e...) { + if f.label == "__name__" { + continue + } + all = all && f.matches(labels) + } + if all { + return true + } + } + } + return false +} diff --git a/internal/observabilityproxy/server.go b/internal/observabilityproxy/server.go index eb540a271..2e8d8ebf9 100644 --- a/internal/observabilityproxy/server.go +++ b/internal/observabilityproxy/server.go @@ -289,7 +289,7 @@ func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Rout return herr } now := s.now() - req, herr := BuildMetricsParams(route.Endpoint, r.Form, route.Namespace, now, s.cfg.Metrics.Limits) + req, herr := BuildMetricsParams(route.Endpoint, r.Form, s.cfg.Metrics.namespaceScope(route.Namespace), now, s.cfg.Metrics.Limits) if herr != nil { return herr } diff --git a/internal/observabilityproxy/server_metrics_test.go b/internal/observabilityproxy/server_metrics_test.go index 1ef5e761e..b4cfdeef5 100644 --- a/internal/observabilityproxy/server_metrics_test.go +++ b/internal/observabilityproxy/server_metrics_test.go @@ -20,6 +20,7 @@ import ( "io" "net/http" "net/url" + "slices" "sync" "testing" "time" @@ -75,7 +76,7 @@ func TestServerMetricsRebuildsUpstreamRequest(t *testing.T) { if r.path != "/api/v1/query_range" { t.Errorf("upstream path = %s", r.path) } - if got := r.form["extra_filters[]"]; len(got) != 1 || got[0] != wantMetricsScope { + if got := r.form["extra_filters[]"]; !slices.Equal(got, wantMetricsScope) { t.Errorf("extra_filters[] = %q", got) } if r.form.Has("extra_filters") || r.form.Has("extra_label") || r.form.Has("nocache") { @@ -105,7 +106,7 @@ func TestServerMetricsStoreByRange(t *testing.T) { if want := []string{"shortterm", "longterm", "shortterm"}; len(h.stores) != 3 || h.stores[0] != want[0] || h.stores[1] != want[1] || h.stores[2] != want[2] { t.Fatalf("stores = %v, want %v", h.stores, want) } - if r := h.got[2]; r.path != "/api/v1/label/pod/values" || r.form.Get("match[]") != wantMetricsScope { + if r := h.got[2]; r.path != "/api/v1/label/pod/values" || !slices.Equal(r.form["match[]"], wantMetricsScope) { t.Fatalf("label values upstream = %s %v", r.path, r.form) } } diff --git a/internal/observabilityproxy/server_test.go b/internal/observabilityproxy/server_test.go index 2670d0e3e..d807ae4f3 100644 --- a/internal/observabilityproxy/server_test.go +++ b/internal/observabilityproxy/server_test.go @@ -83,6 +83,7 @@ func newHarness(t *testing.T, cfg Config, handler http.HandlerFunc) *harness { cfg.Metrics.UpstreamURL = cfg.UpstreamURL if cfg.Metrics.ShortTermStore == "" { cfg.Metrics.ShortTermStore, cfg.Metrics.LongTermStore, cfg.Metrics.ShortTermRetention = "shortterm", "longterm", 72*time.Hour + cfg.Metrics.PlatformNamespacePrefix = "cozy-" } if cfg.Metrics.Limits == (MetricsLimits{}) { cfg.Metrics.Limits = testMetricsLimits()