feat(observability-proxy): include platform exporter series in the namespace root

A platform exporter such as the ClickHouse operator's runs in its own
cozy-* namespace and reports on objects in tenant namespaces. vmagent
labels its series with the exporter's namespace and moves the
namespace the exporter reported to exported_namespace, so the
namespace root, scoped to namespace="<ns>" alone, never returned them.

The scope gains a second extra_filters[] value matching
exported_namespace="<ns>" only on series scraped from a namespace with
the platform prefix. A bare exported_namespace condition would let any
tenant pod that writes a namespace label inject series into another
tenant's namespace root. vmselect ORs the two values, and the label
APIs repeat both in match[], where the match[] and extra_filters[]
cross join still selects exactly their union. An empty prefix keeps
the previous single-value scope.

Assisted-by: LLM
Signed-off-by: Maxim Belyy <maximbel2003@gmail.com>
This commit is contained in:
Maxim Belyy committed 2026-11-10 19:43:21 +07:00
1 parent b3288be903
commit 19e91d3957
6 files changed
+292 -19

No files matched your search

+34 -8
View File
@@ -22,6 +22,7 @@ import (
"net/http"
"net/url"
"regexp"
"slices"
"strconv"
"strings"
"time"
@@ -46,6 +47,9 @@ type MetricsConfig struct {
LongTermStore string
ShortTermRetention time.Duration
Limits MetricsLimits
// PlatformNamespacePrefix names the namespaces whose exporters may
// attribute a series to another namespace; empty trusts none.
PlatformNamespacePrefix string
// UpstreamURL maps a store name to its Prometheus API base URL, without
// a trailing slash; nil means the in-cluster vmselect Service.
UpstreamURL func(store string) string
@@ -85,15 +89,34 @@ func namespaceSelector(ns string) string {
return "{namespace=" + strconv.Quote(ns) + "}"
}
// namespaceScope selects the series of namespace ns. Besides its own, it
// takes the series an exporter in a platform namespace attributes to ns:
// vmagent sets namespace to the scraped target's and, without honorLabels,
// moves the exporter's own namespace label to exported_namespace. Only
// platform namespaces are trusted with that label, since any tenant pod can
// write it; tenants cannot run pods or scrape objects in them.
func (c MetricsConfig) namespaceScope(ns string) []string {
scope := []string{namespaceSelector(ns)}
if c.PlatformNamespacePrefix != "" {
// MetricsQL anchors =~ at both ends.
platform := strconv.Quote(regexp.QuoteMeta(c.PlatformNamespacePrefix) + ".*")
scope = append(scope, "{namespace=~"+platform+",exported_namespace="+strconv.Quote(ns)+"}")
}
return scope
}
// BuildMetricsParams rebuilds the upstream parameters from an allowlist and
// adds the namespace scope as the only extra_filters[] value. vmselect ORs
// repeated extra_filters, so a client value would widen the scope; it ANDs
// the single one into every selector of the query and every match[], which
// is what lets the query pass through unparsed.
func BuildMetricsParams(endpoint string, in url.Values, namespace string, now time.Time, lim MetricsLimits) (metricsRequest, *HTTPError) {
// sends scope as the only extra_filters[] values. vmselect ORs repeated
// extra_filters, so a client value would widen the scope; it ANDs each into
// every selector of the query and every match[], which is what lets the
// query pass through unparsed.
func BuildMetricsParams(endpoint string, in url.Values, scope []string, now time.Time, lim MetricsLimits) (metricsRequest, *HTTPError) {
req := metricsRequest{path: endpoint, params: url.Values{}}
if len(scope) == 0 {
return req, errorf(http.StatusInternalServerError, "no metrics scope")
}
out := req.params
out.Set("extra_filters[]", namespaceSelector(namespace))
out["extra_filters[]"] = slices.Clone(scope)
timeout := lim.MaxTimeout
if v := in.Get("timeout"); v != "" {
d, herr := parseMetricsDuration("timeout", v)
@@ -192,8 +215,11 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti
}
// vmselect started with -search.ignoreExtraFiltersAtLabelsAPI drops
// extra_filters[] on the label APIs unless a match[] is present
// (getCommonParamsInternal), so the scope is repeated here.
out.Set("match[]", namespaceSelector(namespace))
// (getCommonParamsInternal), so the scope is repeated here. Where both
// are sent, vmselect ANDs every match[] with every extra_filters[];
// a cross pair is narrower than one of its halves, so the union is
// still the scope.
out["match[]"] = slices.Clone(scope)
}
return req, nil
}
@@ -19,6 +19,7 @@ package observabilityproxy
import (
"net/http"
"net/url"
"slices"
"strconv"
"strings"
"testing"
@@ -29,13 +30,20 @@ func testMetricsLimits() MetricsLimits {
return MetricsLimits{MaxRange: 30 * 24 * time.Hour, MaxPoints: 11000, MaxTimeout: 30 * time.Second}
}
const wantMetricsScope = `{namespace="tenant-a"}`
// wantMetricsScope is the namespace root's scope under the default
// platform prefix: the namespace's own series, then the series a platform
// exporter attributes to it.
var wantMetricsScope = []string{`{namespace="tenant-a"}`, `{namespace=~"cozy-.*",exported_namespace="tenant-a"}`}
func testScope() []string {
return MetricsConfig{PlatformNamespacePrefix: "cozy-"}.namespaceScope("tenant-a")
}
func unix(t time.Time) string { return strconv.FormatInt(t.Unix(), 10) }
func buildMetrics(t *testing.T, endpoint string, in url.Values) metricsRequest {
t.Helper()
req, err := BuildMetricsParams(endpoint, in, "tenant-a", testNow, testMetricsLimits())
req, err := BuildMetricsParams(endpoint, in, testScope(), testNow, testMetricsLimits())
if err != nil {
t.Fatalf("BuildMetricsParams(%s, %v): %v", endpoint, in, err)
}
@@ -43,7 +51,7 @@ func buildMetrics(t *testing.T, endpoint string, in url.Values) metricsRequest {
}
func metricsStatus(endpoint string, in url.Values) int {
_, err := BuildMetricsParams(endpoint, in, "tenant-a", testNow, testMetricsLimits())
_, err := BuildMetricsParams(endpoint, in, testScope(), testNow, testMetricsLimits())
if err == nil {
return http.StatusOK
}
@@ -84,8 +92,8 @@ func TestBuildMetricsParamsAllowlist(t *testing.T) {
t.Errorf("parameter %q forwarded: %v", k, req.params)
}
}
if got := req.params["extra_filters[]"]; len(got) != 1 || got[0] != wantMetricsScope {
t.Errorf("extra_filters[] = %q, want exactly [%s]", got, wantMetricsScope)
if got := req.params["extra_filters[]"]; !slices.Equal(got, wantMetricsScope) {
t.Errorf("extra_filters[] = %q, want exactly %q", got, wantMetricsScope)
}
for _, m := range req.params["match[]"] {
if strings.Contains(m, "tenant-b") {
@@ -116,8 +124,8 @@ func TestNamespaceSelectorEscapes(t *testing.T) {
func TestBuildMetricsParamsLabelEndpointsAlwaysMatch(t *testing.T) {
for _, endpoint := range []string{"api/v1/labels", "api/v1/label/__name__/values"} {
req := buildMetrics(t, endpoint, url.Values{})
if got := req.params["match[]"]; len(got) != 1 || got[0] != wantMetricsScope {
t.Errorf("%s without match[] sends %q, want the namespace selector", endpoint, got)
if got := req.params["match[]"]; !slices.Equal(got, wantMetricsScope) {
t.Errorf("%s without match[] sends %q, want the namespace scope %q", endpoint, got, wantMetricsScope)
}
req = buildMetrics(t, endpoint, url.Values{"match[]": {"up", `{job="x"}`}})
if got := req.params["match[]"]; len(got) != 2 || got[0] != "up" || got[1] != `{job="x"}` {
@@ -198,7 +206,7 @@ func TestBuildMetricsParamsCanonicalForms(t *testing.T) {
})
want := url.Values{
"query": {"up"}, "start": {"2026-11-04T11:00:00.250Z"}, "end": {"2026-11-04T12:00:00.000Z"}, "step": {"90"},
"timeout": {"30"}, "extra_filters[]": {wantMetricsScope},
"timeout": {"30"}, "extra_filters[]": wantMetricsScope,
}
if req.params.Encode() != want.Encode() {
t.Errorf("params = %v\nwant %v", req.params, want)
@@ -0,0 +1,237 @@
/*
Copyright 2026 The Cozystack Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package observabilityproxy
import (
"net/http"
"net/url"
"regexp"
"slices"
"strconv"
"strings"
"testing"
"time"
)
func TestNamespaceScope(t *testing.T) {
tests := []struct {
prefix string
want []string
}{
{"cozy-", []string{`{namespace="tenant-a"}`, `{namespace=~"cozy-.*",exported_namespace="tenant-a"}`}},
{"", []string{`{namespace="tenant-a"}`}},
// A prefix is literal: a dot in it must not match any character.
{"sys.", []string{`{namespace="tenant-a"}`, `{namespace=~"sys\\..*",exported_namespace="tenant-a"}`}},
}
for _, tt := range tests {
if got := (MetricsConfig{PlatformNamespacePrefix: tt.prefix}).namespaceScope("tenant-a"); !slices.Equal(got, tt.want) {
t.Errorf("prefix %q: scope = %q, want %q", tt.prefix, got, tt.want)
}
}
}
func TestNamespaceScopeEscapesExportedNamespace(t *testing.T) {
for _, ns := range []string{`a"} or {namespace=~".+`, `back\slash`, "new\nline"} {
scope := MetricsConfig{PlatformNamespacePrefix: "cozy-"}.namespaceScope(ns)
filters, ok := parseSelector(scope[1])
if !ok || len(filters) != 2 || filters[1].label != "exported_namespace" || filters[1].value != ns {
t.Errorf("scope %q does not decode to exported_namespace=%q: %+v", scope[1], ns, filters)
}
}
}
// TestBuildMetricsParamsScopeOnEveryEndpoint checks that every endpoint of
// the namespace root sends the whole scope and nothing a client adds.
func TestBuildMetricsParamsScopeOnEveryEndpoint(t *testing.T) {
endpoints := map[string]url.Values{
"api/v1/query": {"query": {"up"}},
"api/v1/query_range": {"query": {"up"}, "start": {unix(testNow.Add(-5 * time.Minute))}, "end": {unix(testNow)}, "step": {"60"}},
"api/v1/series": {"match[]": {"up"}},
"api/v1/labels": {},
"api/v1/label/pod/values": {},
}
for _, prefix := range []string{"cozy-", ""} {
scope := MetricsConfig{PlatformNamespacePrefix: prefix}.namespaceScope("tenant-a")
for endpoint, in := range endpoints {
in = cloneValues(in)
in["extra_filters[]"] = []string{`{namespace=~".+"}`}
req, herr := BuildMetricsParams(endpoint, in, scope, testNow, testMetricsLimits())
if herr != nil {
t.Fatalf("%s: %v", endpoint, herr)
}
if got := req.params["extra_filters[]"]; !slices.Equal(got, scope) {
t.Errorf("prefix %q, %s: extra_filters[] = %q, want %q", prefix, endpoint, got, scope)
}
}
}
}
// An empty scope would send no extra_filters[] and read every namespace.
func TestBuildMetricsParamsRefusesEmptyScope(t *testing.T) {
if _, herr := BuildMetricsParams("api/v1/query", url.Values{"query": {"up"}}, nil, testNow, testMetricsLimits()); herr == nil || herr.Code != http.StatusInternalServerError {
t.Errorf("empty scope: %v, want 500", herr)
}
}
// TestBuildMetricsParamsCopiesScope guards against the forwarded parameters
// sharing the caller's slice.
func TestBuildMetricsParamsCopiesScope(t *testing.T) {
scope := testScope()
req, herr := BuildMetricsParams("api/v1/labels", url.Values{}, scope, testNow, testMetricsLimits())
if herr != nil {
t.Fatal(herr)
}
scope[1] = `{namespace=~".+"}`
if !slices.Equal(req.params["extra_filters[]"], wantMetricsScope) || !slices.Equal(req.params["match[]"], wantMetricsScope) {
t.Errorf("params follow the caller's slice: %v", req.params)
}
}
// TestVmselectSeesScopedSeries runs the parameters the proxy builds
// through a model of vmselect v1.136's series selection: repeated
// extra_filters[] are ORed (searchutil.GetExtraTagFilters), and every
// match[] or query selector is ANDed with each of them
// (searchutil.JoinTagFilterss, called from getCommonParamsInternal and
// promql eval). Regex filters match the whole value.
func TestVmselectSeesScopedSeries(t *testing.T) {
series := []struct {
labels map[string]string
visible bool
}{
{map[string]string{"namespace": "tenant-a"}, true},
{map[string]string{"namespace": "cozy-clickhouse-operator", "exported_namespace": "tenant-a", "chi": "clickhouse-db"}, true},
// A tenant pod claiming another namespace is not a platform exporter.
{map[string]string{"namespace": "tenant-y", "exported_namespace": "tenant-a"}, false},
{map[string]string{"namespace": "xcozy-a", "exported_namespace": "tenant-a"}, false},
{map[string]string{"namespace": "cozy-clickhouse-operator", "exported_namespace": "tenant-b"}, false},
{map[string]string{"namespace": "cozy-clickhouse-operator"}, false},
{map[string]string{"namespace": "tenant-b"}, false},
}
for _, endpoint := range []string{"api/v1/series", "api/v1/labels"} {
in := url.Values{}
if endpoint == "api/v1/series" {
in.Set("match[]", `{__name__=~".*"}`)
}
req := buildMetrics(t, endpoint, in)
for _, s := range series {
if got := vmselectSelects(t, req.params, s.labels); got != s.visible {
t.Errorf("%s: series %v visible = %v, want %v", endpoint, s.labels, got, s.visible)
}
}
}
// Without a prefix the exporter's series stay out of the namespace.
req, herr := BuildMetricsParams("api/v1/labels", url.Values{}, MetricsConfig{}.namespaceScope("tenant-a"), testNow, testMetricsLimits())
if herr != nil {
t.Fatal(herr)
}
if vmselectSelects(t, req.params, map[string]string{"namespace": "cozy-clickhouse-operator", "exported_namespace": "tenant-a"}) {
t.Error("exporter series visible with the platform prefix disabled")
}
}
func cloneValues(in url.Values) url.Values {
out := url.Values{}
for k, v := range in {
out[k] = slices.Clone(v)
}
return out
}
type labelFilter struct {
label, value string
regex bool
}
func (f labelFilter) matches(labels map[string]string) bool {
v := labels[f.label]
if f.regex {
return regexp.MustCompile("^(?:" + f.value + ")$").MatchString(v)
}
return v == f.value
}
// parseSelector reads the selectors the proxy and these tests write:
// {name="value",name=~"regex",...} with values quoted as MetricsQL reads
// them (strconv.Unquote in metricsql's extractStringValue).
func parseSelector(s string) ([]labelFilter, bool) {
body, ok := strings.CutPrefix(s, "{")
if !ok {
return nil, false
}
var out []labelFilter
for body != "}" {
i := strings.IndexAny(body, "=!")
if i <= 0 {
return nil, false
}
f := labelFilter{label: body[:i]}
body = body[i:]
switch {
case strings.HasPrefix(body, "=~"):
f.regex, body = true, body[2:]
case strings.HasPrefix(body, "="):
body = body[1:]
default:
return nil, false
}
lit, err := strconv.QuotedPrefix(body)
if err != nil {
return nil, false
}
if f.value, err = strconv.Unquote(lit); err != nil {
return nil, false
}
out = append(out, f)
body = strings.TrimPrefix(body[len(lit):], ",")
}
return out, true
}
func vmselectSelects(t *testing.T, params url.Values, labels map[string]string) bool {
t.Helper()
parse := func(sels []string) [][]labelFilter {
var out [][]labelFilter
for _, s := range sels {
f, ok := parseSelector(s)
if !ok {
t.Fatalf("cannot parse selector %q", s)
}
out = append(out, f)
}
return out
}
matches, extra := parse(params["match[]"]), parse(params["extra_filters[]"])
if len(matches) == 0 {
matches = [][]labelFilter{nil}
}
for _, m := range matches {
for _, e := range extra {
all := true
for _, f := range append(slices.Clone(m), e...) {
if f.label == "__name__" {
continue
}
all = all && f.matches(labels)
}
if all {
return true
}
}
}
return false
}
+1 -1
View File
@@ -289,7 +289,7 @@ func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Rout
return herr
}
now := s.now()
req, herr := BuildMetricsParams(route.Endpoint, r.Form, route.Namespace, now, s.cfg.Metrics.Limits)
req, herr := BuildMetricsParams(route.Endpoint, r.Form, s.cfg.Metrics.namespaceScope(route.Namespace), now, s.cfg.Metrics.Limits)
if herr != nil {
return herr
}
@@ -20,6 +20,7 @@ import (
"io"
"net/http"
"net/url"
"slices"
"sync"
"testing"
"time"
@@ -75,7 +76,7 @@ func TestServerMetricsRebuildsUpstreamRequest(t *testing.T) {
if r.path != "/api/v1/query_range" {
t.Errorf("upstream path = %s", r.path)
}
if got := r.form["extra_filters[]"]; len(got) != 1 || got[0] != wantMetricsScope {
if got := r.form["extra_filters[]"]; !slices.Equal(got, wantMetricsScope) {
t.Errorf("extra_filters[] = %q", got)
}
if r.form.Has("extra_filters") || r.form.Has("extra_label") || r.form.Has("nocache") {
@@ -105,7 +106,7 @@ func TestServerMetricsStoreByRange(t *testing.T) {
if want := []string{"shortterm", "longterm", "shortterm"}; len(h.stores) != 3 || h.stores[0] != want[0] || h.stores[1] != want[1] || h.stores[2] != want[2] {
t.Fatalf("stores = %v, want %v", h.stores, want)
}
if r := h.got[2]; r.path != "/api/v1/label/pod/values" || r.form.Get("match[]") != wantMetricsScope {
if r := h.got[2]; r.path != "/api/v1/label/pod/values" || !slices.Equal(r.form["match[]"], wantMetricsScope) {
t.Fatalf("label values upstream = %s %v", r.path, r.form)
}
}
@@ -83,6 +83,7 @@ func newHarness(t *testing.T, cfg Config, handler http.HandlerFunc) *harness {
cfg.Metrics.UpstreamURL = cfg.UpstreamURL
if cfg.Metrics.ShortTermStore == "" {
cfg.Metrics.ShortTermStore, cfg.Metrics.LongTermStore, cfg.Metrics.ShortTermRetention = "shortterm", "longterm", 72*time.Hour
cfg.Metrics.PlatformNamespacePrefix = "cozy-"
}
if cfg.Metrics.Limits == (MetricsLimits{}) {
cfg.Metrics.Limits = testMetricsLimits()