From 19e91d3957e0119d78f6c9e536110b3bdb3546db Mon Sep 17 00:00:00 2001 From: Maxim Belyy Date: Tue, 10 Nov 2026 19:43:21 +0700 Subject: [PATCH 1/5] feat(observability-proxy): include platform exporter series in the namespace root A platform exporter such as the ClickHouse operator's runs in its own cozy-* namespace and reports on objects in tenant namespaces. vmagent labels its series with the exporter's namespace and moves the namespace the exporter reported to exported_namespace, so the namespace root, scoped to namespace="" alone, never returned them. The scope gains a second extra_filters[] value matching exported_namespace="" only on series scraped from a namespace with the platform prefix. A bare exported_namespace condition would let any tenant pod that writes a namespace label inject series into another tenant's namespace root. vmselect ORs the two values, and the label APIs repeat both in match[], where the match[] and extra_filters[] cross join still selects exactly their union. An empty prefix keeps the previous single-value scope. Assisted-by: LLM Signed-off-by: Maxim Belyy --- internal/observabilityproxy/metricsparams.go | 42 +++- .../observabilityproxy/metricsparams_test.go | 24 +- .../observabilityproxy/metricsscope_test.go | 237 ++++++++++++++++++ internal/observabilityproxy/server.go | 2 +- .../observabilityproxy/server_metrics_test.go | 5 +- internal/observabilityproxy/server_test.go | 1 + 6 files changed, 292 insertions(+), 19 deletions(-) create mode 100644 internal/observabilityproxy/metricsscope_test.go diff --git a/internal/observabilityproxy/metricsparams.go b/internal/observabilityproxy/metricsparams.go index 8beb347d0..c300cdc7a 100644 --- a/internal/observabilityproxy/metricsparams.go +++ b/internal/observabilityproxy/metricsparams.go @@ -22,6 +22,7 @@ import ( "net/http" "net/url" "regexp" + "slices" "strconv" "strings" "time" @@ -46,6 +47,9 @@ type MetricsConfig struct { LongTermStore string ShortTermRetention time.Duration Limits MetricsLimits + // PlatformNamespacePrefix names the namespaces whose exporters may + // attribute a series to another namespace; empty trusts none. + PlatformNamespacePrefix string // UpstreamURL maps a store name to its Prometheus API base URL, without // a trailing slash; nil means the in-cluster vmselect Service. UpstreamURL func(store string) string @@ -85,15 +89,34 @@ func namespaceSelector(ns string) string { return "{namespace=" + strconv.Quote(ns) + "}" } +// namespaceScope selects the series of namespace ns. Besides its own, it +// takes the series an exporter in a platform namespace attributes to ns: +// vmagent sets namespace to the scraped target's and, without honorLabels, +// moves the exporter's own namespace label to exported_namespace. Only +// platform namespaces are trusted with that label, since any tenant pod can +// write it; tenants cannot run pods or scrape objects in them. +func (c MetricsConfig) namespaceScope(ns string) []string { + scope := []string{namespaceSelector(ns)} + if c.PlatformNamespacePrefix != "" { + // MetricsQL anchors =~ at both ends. + platform := strconv.Quote(regexp.QuoteMeta(c.PlatformNamespacePrefix) + ".*") + scope = append(scope, "{namespace=~"+platform+",exported_namespace="+strconv.Quote(ns)+"}") + } + return scope +} + // BuildMetricsParams rebuilds the upstream parameters from an allowlist and -// adds the namespace scope as the only extra_filters[] value. vmselect ORs -// repeated extra_filters, so a client value would widen the scope; it ANDs -// the single one into every selector of the query and every match[], which -// is what lets the query pass through unparsed. -func BuildMetricsParams(endpoint string, in url.Values, namespace string, now time.Time, lim MetricsLimits) (metricsRequest, *HTTPError) { +// sends scope as the only extra_filters[] values. vmselect ORs repeated +// extra_filters, so a client value would widen the scope; it ANDs each into +// every selector of the query and every match[], which is what lets the +// query pass through unparsed. +func BuildMetricsParams(endpoint string, in url.Values, scope []string, now time.Time, lim MetricsLimits) (metricsRequest, *HTTPError) { req := metricsRequest{path: endpoint, params: url.Values{}} + if len(scope) == 0 { + return req, errorf(http.StatusInternalServerError, "no metrics scope") + } out := req.params - out.Set("extra_filters[]", namespaceSelector(namespace)) + out["extra_filters[]"] = slices.Clone(scope) timeout := lim.MaxTimeout if v := in.Get("timeout"); v != "" { d, herr := parseMetricsDuration("timeout", v) @@ -192,8 +215,11 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti } // vmselect started with -search.ignoreExtraFiltersAtLabelsAPI drops // extra_filters[] on the label APIs unless a match[] is present - // (getCommonParamsInternal), so the scope is repeated here. - out.Set("match[]", namespaceSelector(namespace)) + // (getCommonParamsInternal), so the scope is repeated here. Where both + // are sent, vmselect ANDs every match[] with every extra_filters[]; + // a cross pair is narrower than one of its halves, so the union is + // still the scope. + out["match[]"] = slices.Clone(scope) } return req, nil } diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go index a4a435567..58551a4f7 100644 --- a/internal/observabilityproxy/metricsparams_test.go +++ b/internal/observabilityproxy/metricsparams_test.go @@ -19,6 +19,7 @@ package observabilityproxy import ( "net/http" "net/url" + "slices" "strconv" "strings" "testing" @@ -29,13 +30,20 @@ func testMetricsLimits() MetricsLimits { return MetricsLimits{MaxRange: 30 * 24 * time.Hour, MaxPoints: 11000, MaxTimeout: 30 * time.Second} } -const wantMetricsScope = `{namespace="tenant-a"}` +// wantMetricsScope is the namespace root's scope under the default +// platform prefix: the namespace's own series, then the series a platform +// exporter attributes to it. +var wantMetricsScope = []string{`{namespace="tenant-a"}`, `{namespace=~"cozy-.*",exported_namespace="tenant-a"}`} + +func testScope() []string { + return MetricsConfig{PlatformNamespacePrefix: "cozy-"}.namespaceScope("tenant-a") +} func unix(t time.Time) string { return strconv.FormatInt(t.Unix(), 10) } func buildMetrics(t *testing.T, endpoint string, in url.Values) metricsRequest { t.Helper() - req, err := BuildMetricsParams(endpoint, in, "tenant-a", testNow, testMetricsLimits()) + req, err := BuildMetricsParams(endpoint, in, testScope(), testNow, testMetricsLimits()) if err != nil { t.Fatalf("BuildMetricsParams(%s, %v): %v", endpoint, in, err) } @@ -43,7 +51,7 @@ func buildMetrics(t *testing.T, endpoint string, in url.Values) metricsRequest { } func metricsStatus(endpoint string, in url.Values) int { - _, err := BuildMetricsParams(endpoint, in, "tenant-a", testNow, testMetricsLimits()) + _, err := BuildMetricsParams(endpoint, in, testScope(), testNow, testMetricsLimits()) if err == nil { return http.StatusOK } @@ -84,8 +92,8 @@ func TestBuildMetricsParamsAllowlist(t *testing.T) { t.Errorf("parameter %q forwarded: %v", k, req.params) } } - if got := req.params["extra_filters[]"]; len(got) != 1 || got[0] != wantMetricsScope { - t.Errorf("extra_filters[] = %q, want exactly [%s]", got, wantMetricsScope) + if got := req.params["extra_filters[]"]; !slices.Equal(got, wantMetricsScope) { + t.Errorf("extra_filters[] = %q, want exactly %q", got, wantMetricsScope) } for _, m := range req.params["match[]"] { if strings.Contains(m, "tenant-b") { @@ -116,8 +124,8 @@ func TestNamespaceSelectorEscapes(t *testing.T) { func TestBuildMetricsParamsLabelEndpointsAlwaysMatch(t *testing.T) { for _, endpoint := range []string{"api/v1/labels", "api/v1/label/__name__/values"} { req := buildMetrics(t, endpoint, url.Values{}) - if got := req.params["match[]"]; len(got) != 1 || got[0] != wantMetricsScope { - t.Errorf("%s without match[] sends %q, want the namespace selector", endpoint, got) + if got := req.params["match[]"]; !slices.Equal(got, wantMetricsScope) { + t.Errorf("%s without match[] sends %q, want the namespace scope %q", endpoint, got, wantMetricsScope) } req = buildMetrics(t, endpoint, url.Values{"match[]": {"up", `{job="x"}`}}) if got := req.params["match[]"]; len(got) != 2 || got[0] != "up" || got[1] != `{job="x"}` { @@ -198,7 +206,7 @@ func TestBuildMetricsParamsCanonicalForms(t *testing.T) { }) want := url.Values{ "query": {"up"}, "start": {"2026-11-04T11:00:00.250Z"}, "end": {"2026-11-04T12:00:00.000Z"}, "step": {"90"}, - "timeout": {"30"}, "extra_filters[]": {wantMetricsScope}, + "timeout": {"30"}, "extra_filters[]": wantMetricsScope, } if req.params.Encode() != want.Encode() { t.Errorf("params = %v\nwant %v", req.params, want) diff --git a/internal/observabilityproxy/metricsscope_test.go b/internal/observabilityproxy/metricsscope_test.go new file mode 100644 index 000000000..ff8d52f41 --- /dev/null +++ b/internal/observabilityproxy/metricsscope_test.go @@ -0,0 +1,237 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "net/http" + "net/url" + "regexp" + "slices" + "strconv" + "strings" + "testing" + "time" +) + +func TestNamespaceScope(t *testing.T) { + tests := []struct { + prefix string + want []string + }{ + {"cozy-", []string{`{namespace="tenant-a"}`, `{namespace=~"cozy-.*",exported_namespace="tenant-a"}`}}, + {"", []string{`{namespace="tenant-a"}`}}, + // A prefix is literal: a dot in it must not match any character. + {"sys.", []string{`{namespace="tenant-a"}`, `{namespace=~"sys\\..*",exported_namespace="tenant-a"}`}}, + } + for _, tt := range tests { + if got := (MetricsConfig{PlatformNamespacePrefix: tt.prefix}).namespaceScope("tenant-a"); !slices.Equal(got, tt.want) { + t.Errorf("prefix %q: scope = %q, want %q", tt.prefix, got, tt.want) + } + } +} + +func TestNamespaceScopeEscapesExportedNamespace(t *testing.T) { + for _, ns := range []string{`a"} or {namespace=~".+`, `back\slash`, "new\nline"} { + scope := MetricsConfig{PlatformNamespacePrefix: "cozy-"}.namespaceScope(ns) + filters, ok := parseSelector(scope[1]) + if !ok || len(filters) != 2 || filters[1].label != "exported_namespace" || filters[1].value != ns { + t.Errorf("scope %q does not decode to exported_namespace=%q: %+v", scope[1], ns, filters) + } + } +} + +// TestBuildMetricsParamsScopeOnEveryEndpoint checks that every endpoint of +// the namespace root sends the whole scope and nothing a client adds. +func TestBuildMetricsParamsScopeOnEveryEndpoint(t *testing.T) { + endpoints := map[string]url.Values{ + "api/v1/query": {"query": {"up"}}, + "api/v1/query_range": {"query": {"up"}, "start": {unix(testNow.Add(-5 * time.Minute))}, "end": {unix(testNow)}, "step": {"60"}}, + "api/v1/series": {"match[]": {"up"}}, + "api/v1/labels": {}, + "api/v1/label/pod/values": {}, + } + for _, prefix := range []string{"cozy-", ""} { + scope := MetricsConfig{PlatformNamespacePrefix: prefix}.namespaceScope("tenant-a") + for endpoint, in := range endpoints { + in = cloneValues(in) + in["extra_filters[]"] = []string{`{namespace=~".+"}`} + req, herr := BuildMetricsParams(endpoint, in, scope, testNow, testMetricsLimits()) + if herr != nil { + t.Fatalf("%s: %v", endpoint, herr) + } + if got := req.params["extra_filters[]"]; !slices.Equal(got, scope) { + t.Errorf("prefix %q, %s: extra_filters[] = %q, want %q", prefix, endpoint, got, scope) + } + } + } +} + +// An empty scope would send no extra_filters[] and read every namespace. +func TestBuildMetricsParamsRefusesEmptyScope(t *testing.T) { + if _, herr := BuildMetricsParams("api/v1/query", url.Values{"query": {"up"}}, nil, testNow, testMetricsLimits()); herr == nil || herr.Code != http.StatusInternalServerError { + t.Errorf("empty scope: %v, want 500", herr) + } +} + +// TestBuildMetricsParamsCopiesScope guards against the forwarded parameters +// sharing the caller's slice. +func TestBuildMetricsParamsCopiesScope(t *testing.T) { + scope := testScope() + req, herr := BuildMetricsParams("api/v1/labels", url.Values{}, scope, testNow, testMetricsLimits()) + if herr != nil { + t.Fatal(herr) + } + scope[1] = `{namespace=~".+"}` + if !slices.Equal(req.params["extra_filters[]"], wantMetricsScope) || !slices.Equal(req.params["match[]"], wantMetricsScope) { + t.Errorf("params follow the caller's slice: %v", req.params) + } +} + +// TestVmselectSeesScopedSeries runs the parameters the proxy builds +// through a model of vmselect v1.136's series selection: repeated +// extra_filters[] are ORed (searchutil.GetExtraTagFilters), and every +// match[] or query selector is ANDed with each of them +// (searchutil.JoinTagFilterss, called from getCommonParamsInternal and +// promql eval). Regex filters match the whole value. +func TestVmselectSeesScopedSeries(t *testing.T) { + series := []struct { + labels map[string]string + visible bool + }{ + {map[string]string{"namespace": "tenant-a"}, true}, + {map[string]string{"namespace": "cozy-clickhouse-operator", "exported_namespace": "tenant-a", "chi": "clickhouse-db"}, true}, + // A tenant pod claiming another namespace is not a platform exporter. + {map[string]string{"namespace": "tenant-y", "exported_namespace": "tenant-a"}, false}, + {map[string]string{"namespace": "xcozy-a", "exported_namespace": "tenant-a"}, false}, + {map[string]string{"namespace": "cozy-clickhouse-operator", "exported_namespace": "tenant-b"}, false}, + {map[string]string{"namespace": "cozy-clickhouse-operator"}, false}, + {map[string]string{"namespace": "tenant-b"}, false}, + } + for _, endpoint := range []string{"api/v1/series", "api/v1/labels"} { + in := url.Values{} + if endpoint == "api/v1/series" { + in.Set("match[]", `{__name__=~".*"}`) + } + req := buildMetrics(t, endpoint, in) + for _, s := range series { + if got := vmselectSelects(t, req.params, s.labels); got != s.visible { + t.Errorf("%s: series %v visible = %v, want %v", endpoint, s.labels, got, s.visible) + } + } + } + + // Without a prefix the exporter's series stay out of the namespace. + req, herr := BuildMetricsParams("api/v1/labels", url.Values{}, MetricsConfig{}.namespaceScope("tenant-a"), testNow, testMetricsLimits()) + if herr != nil { + t.Fatal(herr) + } + if vmselectSelects(t, req.params, map[string]string{"namespace": "cozy-clickhouse-operator", "exported_namespace": "tenant-a"}) { + t.Error("exporter series visible with the platform prefix disabled") + } +} + +func cloneValues(in url.Values) url.Values { + out := url.Values{} + for k, v := range in { + out[k] = slices.Clone(v) + } + return out +} + +type labelFilter struct { + label, value string + regex bool +} + +func (f labelFilter) matches(labels map[string]string) bool { + v := labels[f.label] + if f.regex { + return regexp.MustCompile("^(?:" + f.value + ")$").MatchString(v) + } + return v == f.value +} + +// parseSelector reads the selectors the proxy and these tests write: +// {name="value",name=~"regex",...} with values quoted as MetricsQL reads +// them (strconv.Unquote in metricsql's extractStringValue). +func parseSelector(s string) ([]labelFilter, bool) { + body, ok := strings.CutPrefix(s, "{") + if !ok { + return nil, false + } + var out []labelFilter + for body != "}" { + i := strings.IndexAny(body, "=!") + if i <= 0 { + return nil, false + } + f := labelFilter{label: body[:i]} + body = body[i:] + switch { + case strings.HasPrefix(body, "=~"): + f.regex, body = true, body[2:] + case strings.HasPrefix(body, "="): + body = body[1:] + default: + return nil, false + } + lit, err := strconv.QuotedPrefix(body) + if err != nil { + return nil, false + } + if f.value, err = strconv.Unquote(lit); err != nil { + return nil, false + } + out = append(out, f) + body = strings.TrimPrefix(body[len(lit):], ",") + } + return out, true +} + +func vmselectSelects(t *testing.T, params url.Values, labels map[string]string) bool { + t.Helper() + parse := func(sels []string) [][]labelFilter { + var out [][]labelFilter + for _, s := range sels { + f, ok := parseSelector(s) + if !ok { + t.Fatalf("cannot parse selector %q", s) + } + out = append(out, f) + } + return out + } + matches, extra := parse(params["match[]"]), parse(params["extra_filters[]"]) + if len(matches) == 0 { + matches = [][]labelFilter{nil} + } + for _, m := range matches { + for _, e := range extra { + all := true + for _, f := range append(slices.Clone(m), e...) { + if f.label == "__name__" { + continue + } + all = all && f.matches(labels) + } + if all { + return true + } + } + } + return false +} diff --git a/internal/observabilityproxy/server.go b/internal/observabilityproxy/server.go index eb540a271..2e8d8ebf9 100644 --- a/internal/observabilityproxy/server.go +++ b/internal/observabilityproxy/server.go @@ -289,7 +289,7 @@ func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Rout return herr } now := s.now() - req, herr := BuildMetricsParams(route.Endpoint, r.Form, route.Namespace, now, s.cfg.Metrics.Limits) + req, herr := BuildMetricsParams(route.Endpoint, r.Form, s.cfg.Metrics.namespaceScope(route.Namespace), now, s.cfg.Metrics.Limits) if herr != nil { return herr } diff --git a/internal/observabilityproxy/server_metrics_test.go b/internal/observabilityproxy/server_metrics_test.go index 1ef5e761e..b4cfdeef5 100644 --- a/internal/observabilityproxy/server_metrics_test.go +++ b/internal/observabilityproxy/server_metrics_test.go @@ -20,6 +20,7 @@ import ( "io" "net/http" "net/url" + "slices" "sync" "testing" "time" @@ -75,7 +76,7 @@ func TestServerMetricsRebuildsUpstreamRequest(t *testing.T) { if r.path != "/api/v1/query_range" { t.Errorf("upstream path = %s", r.path) } - if got := r.form["extra_filters[]"]; len(got) != 1 || got[0] != wantMetricsScope { + if got := r.form["extra_filters[]"]; !slices.Equal(got, wantMetricsScope) { t.Errorf("extra_filters[] = %q", got) } if r.form.Has("extra_filters") || r.form.Has("extra_label") || r.form.Has("nocache") { @@ -105,7 +106,7 @@ func TestServerMetricsStoreByRange(t *testing.T) { if want := []string{"shortterm", "longterm", "shortterm"}; len(h.stores) != 3 || h.stores[0] != want[0] || h.stores[1] != want[1] || h.stores[2] != want[2] { t.Fatalf("stores = %v, want %v", h.stores, want) } - if r := h.got[2]; r.path != "/api/v1/label/pod/values" || r.form.Get("match[]") != wantMetricsScope { + if r := h.got[2]; r.path != "/api/v1/label/pod/values" || !slices.Equal(r.form["match[]"], wantMetricsScope) { t.Fatalf("label values upstream = %s %v", r.path, r.form) } } diff --git a/internal/observabilityproxy/server_test.go b/internal/observabilityproxy/server_test.go index 2670d0e3e..d807ae4f3 100644 --- a/internal/observabilityproxy/server_test.go +++ b/internal/observabilityproxy/server_test.go @@ -83,6 +83,7 @@ func newHarness(t *testing.T, cfg Config, handler http.HandlerFunc) *harness { cfg.Metrics.UpstreamURL = cfg.UpstreamURL if cfg.Metrics.ShortTermStore == "" { cfg.Metrics.ShortTermStore, cfg.Metrics.LongTermStore, cfg.Metrics.ShortTermRetention = "shortterm", "longterm", 72*time.Hour + cfg.Metrics.PlatformNamespacePrefix = "cozy-" } if cfg.Metrics.Limits == (MetricsLimits{}) { cfg.Metrics.Limits = testMetricsLimits() From 053026c8dfd942f9cc7120a58e1881114e412f26 Mon Sep 17 00:00:00 2001 From: Maxim Belyy Date: Tue, 10 Nov 2026 19:43:21 +0700 Subject: [PATCH 2/5] feat(observability-proxy): add the --metrics-platform-namespace-prefix flag The prefix decides which namespaces are trusted to attribute series to another namespace, so it defaults to cozy-, the prefix of every platform namespace, and can be emptied to turn that off. It is limited to namespace name characters: a regex metacharacter or an upper case letter would match namespaces other than the ones named. Assisted-by: LLM Signed-off-by: Maxim Belyy --- cmd/observability-proxy/main.go | 6 ++++++ cmd/observability-proxy/main_test.go | 18 +++++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/cmd/observability-proxy/main.go b/cmd/observability-proxy/main.go index a3687fe08..c527f5061 100644 --- a/cmd/observability-proxy/main.go +++ b/cmd/observability-proxy/main.go @@ -92,6 +92,7 @@ func parseFlags(args []string) (options, error) { fs.DurationVar(&o.metrics.Limits.MaxRange, "metrics-max-range", 30*24*time.Hour, "longest [start, end] and step a metrics request may use") fs.IntVar(&o.metrics.Limits.MaxPoints, "metrics-max-points", 11000, "most points per series a range query may return") fs.DurationVar(&o.metrics.Limits.MaxTimeout, "metrics-max-timeout", 30*time.Second, "timeout forwarded to vmselect; a longer client timeout is clamped; must be below --request-timeout when that is set") + fs.StringVar(&o.metrics.PlatformNamespacePrefix, "metrics-platform-namespace-prefix", "cozy-", "namespaces with this prefix run platform exporters, whose exported_namespace label adds series to that namespace's scope; empty disables") fs.IntVar(&o.limiter.MaxConcurrentPerUser, "max-concurrent-per-user", 10, "concurrent non-tail requests per user; 0 disables") fs.Float64Var(&o.limiter.RatePerUser, "rate-limit-per-user", 10, "sustained requests per second per user; 0 disables") fs.IntVar(&o.limiter.RateBurst, "rate-limit-burst", 20, "request burst per user") @@ -112,6 +113,11 @@ func parseFlags(args []string) (options, error) { return o, fmt.Errorf("invalid metrics store name %q", store) } } + // Anything but namespace name characters would make the prefix match + // differently from the namespaces it is meant to name. + if p := o.metrics.PlatformNamespacePrefix; p != "" && len(validation.IsDNS1123Label(p+"x")) > 0 { + return o, fmt.Errorf("invalid metrics platform namespace prefix %q", p) + } switch { case o.rootTenant == "": return o, errors.New("--root-tenant must not be empty") diff --git a/cmd/observability-proxy/main_test.go b/cmd/observability-proxy/main_test.go index bf7ac396b..13e137dd2 100644 --- a/cmd/observability-proxy/main_test.go +++ b/cmd/observability-proxy/main_test.go @@ -43,6 +43,10 @@ func TestParseFlagsDefaults(t *testing.T) { if m.RootTenant != o.rootTenant { t.Errorf("metrics root tenant = %q, want --root-tenant", m.RootTenant) } + // The prefix every platform namespace carries. + if m.PlatformNamespacePrefix != "cozy-" { + t.Errorf("platform namespace prefix = %q, want cozy-", m.PlatformNamespacePrefix) + } } func TestParseFlagsOverrides(t *testing.T) { @@ -63,15 +67,22 @@ func TestParseFlagsMetricsOverrides(t *testing.T) { o, err := parseFlags([]string{ "--root-tenant=tenant-x", "--metrics-shortterm-store=fast", "--metrics-longterm-store=slow", "--metrics-shortterm-retention=24h", "--metrics-max-range=168h", "--metrics-max-points=500", "--metrics-max-timeout=10s", + "--metrics-platform-namespace-prefix=sys-", }) if err != nil { t.Fatal(err) } m := o.metrics if m.RootTenant != "tenant-x" || m.ShortTermStore != "fast" || m.LongTermStore != "slow" || m.ShortTermRetention != 24*time.Hour || - m.Limits.MaxRange != 168*time.Hour || m.Limits.MaxPoints != 500 || m.Limits.MaxTimeout != 10*time.Second { + m.Limits.MaxRange != 168*time.Hour || m.Limits.MaxPoints != 500 || m.Limits.MaxTimeout != 10*time.Second || + m.PlatformNamespacePrefix != "sys-" { t.Errorf("overrides not applied: %+v", m) } + // An empty prefix switches the exporter branch of the scope off. + o, err = parseFlags([]string{"--metrics-platform-namespace-prefix="}) + if err != nil || o.metrics.PlatformNamespacePrefix != "" { + t.Errorf("empty platform namespace prefix: %q, %v", o.metrics.PlatformNamespacePrefix, err) + } } func TestParseFlagsInvalid(t *testing.T) { @@ -101,6 +112,11 @@ func TestParseFlagsInvalid(t *testing.T) { {"--metrics-max-range=0s"}, {"--metrics-max-points=1"}, {"--metrics-max-timeout=0s"}, + // Only namespace name characters: a regex metacharacter or an upper + // case letter would widen or empty the scope. + {"--metrics-platform-namespace-prefix=Cozy-"}, + {"--metrics-platform-namespace-prefix=cozy.*"}, + {"--metrics-platform-namespace-prefix=-"}, {"--unknown"}, // The proxy would cut the request while vmselect is still within // the timeout it was given. From 84d490e1bfc3bf355d8de4194fe1b0be4480b586 Mon Sep 17 00:00:00 2001 From: Maxim Belyy Date: Tue, 10 Nov 2026 19:43:21 +0700 Subject: [PATCH 3/5] feat(observability-proxy): expose the platform namespace prefix in the chart The value is always passed, also when empty, so that clearing it in the values turns the exporter series off instead of falling back to the binary's default. Assisted-by: LLM Signed-off-by: Maxim Belyy --- .../templates/deployment.yaml | 1 + .../tests/deployment_test.yaml | 16 ++++++++++++++++ packages/system/observability-proxy/values.yaml | 4 ++++ 3 files changed, 21 insertions(+) diff --git a/packages/system/observability-proxy/templates/deployment.yaml b/packages/system/observability-proxy/templates/deployment.yaml index d49de140b..c726c4282 100644 --- a/packages/system/observability-proxy/templates/deployment.yaml +++ b/packages/system/observability-proxy/templates/deployment.yaml @@ -41,6 +41,7 @@ spec: - --metrics-shortterm-store={{ .Values.observabilityProxy.metrics.shorttermStore }} - --metrics-longterm-store={{ .Values.observabilityProxy.metrics.longtermStore }} - --metrics-shortterm-retention={{ .Values.observabilityProxy.metrics.shorttermRetention }} + - --metrics-platform-namespace-prefix={{ .Values.observabilityProxy.metrics.platformNamespacePrefix }} ports: - name: http containerPort: 8080 diff --git a/packages/system/observability-proxy/tests/deployment_test.yaml b/packages/system/observability-proxy/tests/deployment_test.yaml index a5e948920..a4c5e760c 100644 --- a/packages/system/observability-proxy/tests/deployment_test.yaml +++ b/packages/system/observability-proxy/tests/deployment_test.yaml @@ -51,6 +51,9 @@ tests: - contains: path: spec.template.spec.containers[0].args content: --metrics-shortterm-retention=72h + - contains: + path: spec.template.spec.containers[0].args + content: --metrics-platform-namespace-prefix=cozy- - it: follows renamed metrics stores set: @@ -69,6 +72,19 @@ tests: path: spec.template.spec.containers[0].args content: --metrics-shortterm-retention=24h + # An empty value must reach the binary as an explicit empty flag; a + # dropped flag would fall back to the cozy- default. + - it: passes an empty platform namespace prefix through + set: + observabilityProxy.metrics.platformNamespacePrefix: "" + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --metrics-platform-namespace-prefix= + - notContains: + path: spec.template.spec.containers[0].args + content: --metrics-platform-namespace-prefix=cozy- + - it: falls back to cozy.local when the platform passes no cluster domain asserts: - contains: diff --git a/packages/system/observability-proxy/values.yaml b/packages/system/observability-proxy/values.yaml index 67334a237..3199d30bc 100644 --- a/packages/system/observability-proxy/values.yaml +++ b/packages/system/observability-proxy/values.yaml @@ -14,6 +14,10 @@ observabilityProxy: shorttermStore: shortterm longtermStore: longterm shorttermRetention: 72h + # Exporters in namespaces with this prefix may attribute a series to a + # tenant namespace through exported_namespace, and the series is then + # part of that namespace's metrics. Empty disables it. + platformNamespacePrefix: cozy- resources: requests: cpu: 10m From 0e4711b17cfb08afd1b40820fb97166cba203ed0 Mon Sep 17 00:00:00 2001 From: Maxim Belyy Date: Wed, 11 Nov 2026 05:41:51 +0700 Subject: [PATCH 4/5] docs(observability-proxy): bound the exporter trust to platform scrapes The cozy-* condition holds only for series the platform's vmagent scraped. A tenant's nested Kubernetes cluster remote-writes into the same store and picks its own namespace labels, cozy-* among them, so the comment must not promise that tenants can never produce such a series. That writer could already pass the namespace="" branch, and the exporter branch gives it nothing more. Assisted-by: LLM Signed-off-by: Maxim Belyy --- internal/observabilityproxy/metricsparams.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/internal/observabilityproxy/metricsparams.go b/internal/observabilityproxy/metricsparams.go index c300cdc7a..b2086e494 100644 --- a/internal/observabilityproxy/metricsparams.go +++ b/internal/observabilityproxy/metricsparams.go @@ -92,9 +92,12 @@ func namespaceSelector(ns string) string { // namespaceScope selects the series of namespace ns. Besides its own, it // takes the series an exporter in a platform namespace attributes to ns: // vmagent sets namespace to the scraped target's and, without honorLabels, -// moves the exporter's own namespace label to exported_namespace. Only -// platform namespaces are trusted with that label, since any tenant pod can -// write it; tenants cannot run pods or scrape objects in them. +// moves the exporter's own namespace label to exported_namespace. Any pod +// can write that label, so it is trusted only where the platform's vmagent +// set namespace to one with the platform prefix, in which tenants cannot run +// pods or scrape objects. Series remote-written from a tenant's nested +// cluster carry whatever namespace that cluster chose, so they are outside +// this guarantee for both values alike. func (c MetricsConfig) namespaceScope(ns string) []string { scope := []string{namespaceSelector(ns)} if c.PlatformNamespacePrefix != "" { From 8351e23d4d653c6460b42b0447bc906e5cb3e7a0 Mon Sep 17 00:00:00 2001 From: Maxim Belyy Date: Wed, 11 Nov 2026 05:42:02 +0700 Subject: [PATCH 5/5] docs(observability-proxy): give the real reason for the prefix check The prefix is quoted before it reaches the regex, so a metacharacter or an upper case letter cannot widen the scope. Such a prefix matches no namespace at all and silently drops the exporter series, which is what the flag check guards against. Assisted-by: LLM Signed-off-by: Maxim Belyy --- cmd/observability-proxy/main.go | 4 ++-- cmd/observability-proxy/main_test.go | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/cmd/observability-proxy/main.go b/cmd/observability-proxy/main.go index c527f5061..e6652ad5f 100644 --- a/cmd/observability-proxy/main.go +++ b/cmd/observability-proxy/main.go @@ -113,8 +113,8 @@ func parseFlags(args []string) (options, error) { return o, fmt.Errorf("invalid metrics store name %q", store) } } - // Anything but namespace name characters would make the prefix match - // differently from the namespaces it is meant to name. + // The prefix is matched literally, so one no namespace name can start + // with would turn the exporter branch off without a word. if p := o.metrics.PlatformNamespacePrefix; p != "" && len(validation.IsDNS1123Label(p+"x")) > 0 { return o, fmt.Errorf("invalid metrics platform namespace prefix %q", p) } diff --git a/cmd/observability-proxy/main_test.go b/cmd/observability-proxy/main_test.go index 13e137dd2..a08f15d9e 100644 --- a/cmd/observability-proxy/main_test.go +++ b/cmd/observability-proxy/main_test.go @@ -112,8 +112,7 @@ func TestParseFlagsInvalid(t *testing.T) { {"--metrics-max-range=0s"}, {"--metrics-max-points=1"}, {"--metrics-max-timeout=0s"}, - // Only namespace name characters: a regex metacharacter or an upper - // case letter would widen or empty the scope. + // No namespace name starts with these, so they would match nothing. {"--metrics-platform-namespace-prefix=Cozy-"}, {"--metrics-platform-namespace-prefix=cozy.*"}, {"--metrics-platform-namespace-prefix=-"},