mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
feat(observability-proxy): include platform exporter series in the namespace root
A platform exporter such as the ClickHouse operator's runs in its own cozy-* namespace and reports on objects in tenant namespaces. vmagent labels its series with the exporter's namespace and moves the namespace the exporter reported to exported_namespace, so the namespace root, scoped to namespace="<ns>" alone, never returned them. The scope gains a second extra_filters[] value matching exported_namespace="<ns>" only on series scraped from a namespace with the platform prefix. A bare exported_namespace condition would let any tenant pod that writes a namespace label inject series into another tenant's namespace root. vmselect ORs the two values, and the label APIs repeat both in match[], where the match[] and extra_filters[] cross join still selects exactly their union. An empty prefix keeps the previous single-value scope. Assisted-by: LLM Signed-off-by: Maxim Belyy <maximbel2003@gmail.com>
This commit is contained in:
1 parent
b3288be903
commit
19e91d3957
6 files changed
+292
-19
No files matched your search
@@ -22,6 +22,7 @@ import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -46,6 +47,9 @@ type MetricsConfig struct {
|
||||
LongTermStore string
|
||||
ShortTermRetention time.Duration
|
||||
Limits MetricsLimits
|
||||
// PlatformNamespacePrefix names the namespaces whose exporters may
|
||||
// attribute a series to another namespace; empty trusts none.
|
||||
PlatformNamespacePrefix string
|
||||
// UpstreamURL maps a store name to its Prometheus API base URL, without
|
||||
// a trailing slash; nil means the in-cluster vmselect Service.
|
||||
UpstreamURL func(store string) string
|
||||
@@ -85,15 +89,34 @@ func namespaceSelector(ns string) string {
|
||||
return "{namespace=" + strconv.Quote(ns) + "}"
|
||||
}
|
||||
|
||||
// namespaceScope selects the series of namespace ns. Besides its own, it
|
||||
// takes the series an exporter in a platform namespace attributes to ns:
|
||||
// vmagent sets namespace to the scraped target's and, without honorLabels,
|
||||
// moves the exporter's own namespace label to exported_namespace. Only
|
||||
// platform namespaces are trusted with that label, since any tenant pod can
|
||||
// write it; tenants cannot run pods or scrape objects in them.
|
||||
func (c MetricsConfig) namespaceScope(ns string) []string {
|
||||
scope := []string{namespaceSelector(ns)}
|
||||
if c.PlatformNamespacePrefix != "" {
|
||||
// MetricsQL anchors =~ at both ends.
|
||||
platform := strconv.Quote(regexp.QuoteMeta(c.PlatformNamespacePrefix) + ".*")
|
||||
scope = append(scope, "{namespace=~"+platform+",exported_namespace="+strconv.Quote(ns)+"}")
|
||||
}
|
||||
return scope
|
||||
}
|
||||
|
||||
// BuildMetricsParams rebuilds the upstream parameters from an allowlist and
|
||||
// adds the namespace scope as the only extra_filters[] value. vmselect ORs
|
||||
// repeated extra_filters, so a client value would widen the scope; it ANDs
|
||||
// the single one into every selector of the query and every match[], which
|
||||
// is what lets the query pass through unparsed.
|
||||
func BuildMetricsParams(endpoint string, in url.Values, namespace string, now time.Time, lim MetricsLimits) (metricsRequest, *HTTPError) {
|
||||
// sends scope as the only extra_filters[] values. vmselect ORs repeated
|
||||
// extra_filters, so a client value would widen the scope; it ANDs each into
|
||||
// every selector of the query and every match[], which is what lets the
|
||||
// query pass through unparsed.
|
||||
func BuildMetricsParams(endpoint string, in url.Values, scope []string, now time.Time, lim MetricsLimits) (metricsRequest, *HTTPError) {
|
||||
req := metricsRequest{path: endpoint, params: url.Values{}}
|
||||
if len(scope) == 0 {
|
||||
return req, errorf(http.StatusInternalServerError, "no metrics scope")
|
||||
}
|
||||
out := req.params
|
||||
out.Set("extra_filters[]", namespaceSelector(namespace))
|
||||
out["extra_filters[]"] = slices.Clone(scope)
|
||||
timeout := lim.MaxTimeout
|
||||
if v := in.Get("timeout"); v != "" {
|
||||
d, herr := parseMetricsDuration("timeout", v)
|
||||
@@ -192,8 +215,11 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti
|
||||
}
|
||||
// vmselect started with -search.ignoreExtraFiltersAtLabelsAPI drops
|
||||
// extra_filters[] on the label APIs unless a match[] is present
|
||||
// (getCommonParamsInternal), so the scope is repeated here.
|
||||
out.Set("match[]", namespaceSelector(namespace))
|
||||
// (getCommonParamsInternal), so the scope is repeated here. Where both
|
||||
// are sent, vmselect ANDs every match[] with every extra_filters[];
|
||||
// a cross pair is narrower than one of its halves, so the union is
|
||||
// still the scope.
|
||||
out["match[]"] = slices.Clone(scope)
|
||||
}
|
||||
return req, nil
|
||||
}
|
||||
|
||||
@@ -19,6 +19,7 @@ package observabilityproxy
|
||||
import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -29,13 +30,20 @@ func testMetricsLimits() MetricsLimits {
|
||||
return MetricsLimits{MaxRange: 30 * 24 * time.Hour, MaxPoints: 11000, MaxTimeout: 30 * time.Second}
|
||||
}
|
||||
|
||||
const wantMetricsScope = `{namespace="tenant-a"}`
|
||||
// wantMetricsScope is the namespace root's scope under the default
|
||||
// platform prefix: the namespace's own series, then the series a platform
|
||||
// exporter attributes to it.
|
||||
var wantMetricsScope = []string{`{namespace="tenant-a"}`, `{namespace=~"cozy-.*",exported_namespace="tenant-a"}`}
|
||||
|
||||
func testScope() []string {
|
||||
return MetricsConfig{PlatformNamespacePrefix: "cozy-"}.namespaceScope("tenant-a")
|
||||
}
|
||||
|
||||
func unix(t time.Time) string { return strconv.FormatInt(t.Unix(), 10) }
|
||||
|
||||
func buildMetrics(t *testing.T, endpoint string, in url.Values) metricsRequest {
|
||||
t.Helper()
|
||||
req, err := BuildMetricsParams(endpoint, in, "tenant-a", testNow, testMetricsLimits())
|
||||
req, err := BuildMetricsParams(endpoint, in, testScope(), testNow, testMetricsLimits())
|
||||
if err != nil {
|
||||
t.Fatalf("BuildMetricsParams(%s, %v): %v", endpoint, in, err)
|
||||
}
|
||||
@@ -43,7 +51,7 @@ func buildMetrics(t *testing.T, endpoint string, in url.Values) metricsRequest {
|
||||
}
|
||||
|
||||
func metricsStatus(endpoint string, in url.Values) int {
|
||||
_, err := BuildMetricsParams(endpoint, in, "tenant-a", testNow, testMetricsLimits())
|
||||
_, err := BuildMetricsParams(endpoint, in, testScope(), testNow, testMetricsLimits())
|
||||
if err == nil {
|
||||
return http.StatusOK
|
||||
}
|
||||
@@ -84,8 +92,8 @@ func TestBuildMetricsParamsAllowlist(t *testing.T) {
|
||||
t.Errorf("parameter %q forwarded: %v", k, req.params)
|
||||
}
|
||||
}
|
||||
if got := req.params["extra_filters[]"]; len(got) != 1 || got[0] != wantMetricsScope {
|
||||
t.Errorf("extra_filters[] = %q, want exactly [%s]", got, wantMetricsScope)
|
||||
if got := req.params["extra_filters[]"]; !slices.Equal(got, wantMetricsScope) {
|
||||
t.Errorf("extra_filters[] = %q, want exactly %q", got, wantMetricsScope)
|
||||
}
|
||||
for _, m := range req.params["match[]"] {
|
||||
if strings.Contains(m, "tenant-b") {
|
||||
@@ -116,8 +124,8 @@ func TestNamespaceSelectorEscapes(t *testing.T) {
|
||||
func TestBuildMetricsParamsLabelEndpointsAlwaysMatch(t *testing.T) {
|
||||
for _, endpoint := range []string{"api/v1/labels", "api/v1/label/__name__/values"} {
|
||||
req := buildMetrics(t, endpoint, url.Values{})
|
||||
if got := req.params["match[]"]; len(got) != 1 || got[0] != wantMetricsScope {
|
||||
t.Errorf("%s without match[] sends %q, want the namespace selector", endpoint, got)
|
||||
if got := req.params["match[]"]; !slices.Equal(got, wantMetricsScope) {
|
||||
t.Errorf("%s without match[] sends %q, want the namespace scope %q", endpoint, got, wantMetricsScope)
|
||||
}
|
||||
req = buildMetrics(t, endpoint, url.Values{"match[]": {"up", `{job="x"}`}})
|
||||
if got := req.params["match[]"]; len(got) != 2 || got[0] != "up" || got[1] != `{job="x"}` {
|
||||
@@ -198,7 +206,7 @@ func TestBuildMetricsParamsCanonicalForms(t *testing.T) {
|
||||
})
|
||||
want := url.Values{
|
||||
"query": {"up"}, "start": {"2026-11-04T11:00:00.250Z"}, "end": {"2026-11-04T12:00:00.000Z"}, "step": {"90"},
|
||||
"timeout": {"30"}, "extra_filters[]": {wantMetricsScope},
|
||||
"timeout": {"30"}, "extra_filters[]": wantMetricsScope,
|
||||
}
|
||||
if req.params.Encode() != want.Encode() {
|
||||
t.Errorf("params = %v\nwant %v", req.params, want)
|
||||
|
||||
@@ -0,0 +1,237 @@
|
||||
/*
|
||||
Copyright 2026 The Cozystack Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package observabilityproxy
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestNamespaceScope(t *testing.T) {
|
||||
tests := []struct {
|
||||
prefix string
|
||||
want []string
|
||||
}{
|
||||
{"cozy-", []string{`{namespace="tenant-a"}`, `{namespace=~"cozy-.*",exported_namespace="tenant-a"}`}},
|
||||
{"", []string{`{namespace="tenant-a"}`}},
|
||||
// A prefix is literal: a dot in it must not match any character.
|
||||
{"sys.", []string{`{namespace="tenant-a"}`, `{namespace=~"sys\\..*",exported_namespace="tenant-a"}`}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := (MetricsConfig{PlatformNamespacePrefix: tt.prefix}).namespaceScope("tenant-a"); !slices.Equal(got, tt.want) {
|
||||
t.Errorf("prefix %q: scope = %q, want %q", tt.prefix, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNamespaceScopeEscapesExportedNamespace(t *testing.T) {
|
||||
for _, ns := range []string{`a"} or {namespace=~".+`, `back\slash`, "new\nline"} {
|
||||
scope := MetricsConfig{PlatformNamespacePrefix: "cozy-"}.namespaceScope(ns)
|
||||
filters, ok := parseSelector(scope[1])
|
||||
if !ok || len(filters) != 2 || filters[1].label != "exported_namespace" || filters[1].value != ns {
|
||||
t.Errorf("scope %q does not decode to exported_namespace=%q: %+v", scope[1], ns, filters)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuildMetricsParamsScopeOnEveryEndpoint checks that every endpoint of
|
||||
// the namespace root sends the whole scope and nothing a client adds.
|
||||
func TestBuildMetricsParamsScopeOnEveryEndpoint(t *testing.T) {
|
||||
endpoints := map[string]url.Values{
|
||||
"api/v1/query": {"query": {"up"}},
|
||||
"api/v1/query_range": {"query": {"up"}, "start": {unix(testNow.Add(-5 * time.Minute))}, "end": {unix(testNow)}, "step": {"60"}},
|
||||
"api/v1/series": {"match[]": {"up"}},
|
||||
"api/v1/labels": {},
|
||||
"api/v1/label/pod/values": {},
|
||||
}
|
||||
for _, prefix := range []string{"cozy-", ""} {
|
||||
scope := MetricsConfig{PlatformNamespacePrefix: prefix}.namespaceScope("tenant-a")
|
||||
for endpoint, in := range endpoints {
|
||||
in = cloneValues(in)
|
||||
in["extra_filters[]"] = []string{`{namespace=~".+"}`}
|
||||
req, herr := BuildMetricsParams(endpoint, in, scope, testNow, testMetricsLimits())
|
||||
if herr != nil {
|
||||
t.Fatalf("%s: %v", endpoint, herr)
|
||||
}
|
||||
if got := req.params["extra_filters[]"]; !slices.Equal(got, scope) {
|
||||
t.Errorf("prefix %q, %s: extra_filters[] = %q, want %q", prefix, endpoint, got, scope)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// An empty scope would send no extra_filters[] and read every namespace.
|
||||
func TestBuildMetricsParamsRefusesEmptyScope(t *testing.T) {
|
||||
if _, herr := BuildMetricsParams("api/v1/query", url.Values{"query": {"up"}}, nil, testNow, testMetricsLimits()); herr == nil || herr.Code != http.StatusInternalServerError {
|
||||
t.Errorf("empty scope: %v, want 500", herr)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuildMetricsParamsCopiesScope guards against the forwarded parameters
|
||||
// sharing the caller's slice.
|
||||
func TestBuildMetricsParamsCopiesScope(t *testing.T) {
|
||||
scope := testScope()
|
||||
req, herr := BuildMetricsParams("api/v1/labels", url.Values{}, scope, testNow, testMetricsLimits())
|
||||
if herr != nil {
|
||||
t.Fatal(herr)
|
||||
}
|
||||
scope[1] = `{namespace=~".+"}`
|
||||
if !slices.Equal(req.params["extra_filters[]"], wantMetricsScope) || !slices.Equal(req.params["match[]"], wantMetricsScope) {
|
||||
t.Errorf("params follow the caller's slice: %v", req.params)
|
||||
}
|
||||
}
|
||||
|
||||
// TestVmselectSeesScopedSeries runs the parameters the proxy builds
|
||||
// through a model of vmselect v1.136's series selection: repeated
|
||||
// extra_filters[] are ORed (searchutil.GetExtraTagFilters), and every
|
||||
// match[] or query selector is ANDed with each of them
|
||||
// (searchutil.JoinTagFilterss, called from getCommonParamsInternal and
|
||||
// promql eval). Regex filters match the whole value.
|
||||
func TestVmselectSeesScopedSeries(t *testing.T) {
|
||||
series := []struct {
|
||||
labels map[string]string
|
||||
visible bool
|
||||
}{
|
||||
{map[string]string{"namespace": "tenant-a"}, true},
|
||||
{map[string]string{"namespace": "cozy-clickhouse-operator", "exported_namespace": "tenant-a", "chi": "clickhouse-db"}, true},
|
||||
// A tenant pod claiming another namespace is not a platform exporter.
|
||||
{map[string]string{"namespace": "tenant-y", "exported_namespace": "tenant-a"}, false},
|
||||
{map[string]string{"namespace": "xcozy-a", "exported_namespace": "tenant-a"}, false},
|
||||
{map[string]string{"namespace": "cozy-clickhouse-operator", "exported_namespace": "tenant-b"}, false},
|
||||
{map[string]string{"namespace": "cozy-clickhouse-operator"}, false},
|
||||
{map[string]string{"namespace": "tenant-b"}, false},
|
||||
}
|
||||
for _, endpoint := range []string{"api/v1/series", "api/v1/labels"} {
|
||||
in := url.Values{}
|
||||
if endpoint == "api/v1/series" {
|
||||
in.Set("match[]", `{__name__=~".*"}`)
|
||||
}
|
||||
req := buildMetrics(t, endpoint, in)
|
||||
for _, s := range series {
|
||||
if got := vmselectSelects(t, req.params, s.labels); got != s.visible {
|
||||
t.Errorf("%s: series %v visible = %v, want %v", endpoint, s.labels, got, s.visible)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Without a prefix the exporter's series stay out of the namespace.
|
||||
req, herr := BuildMetricsParams("api/v1/labels", url.Values{}, MetricsConfig{}.namespaceScope("tenant-a"), testNow, testMetricsLimits())
|
||||
if herr != nil {
|
||||
t.Fatal(herr)
|
||||
}
|
||||
if vmselectSelects(t, req.params, map[string]string{"namespace": "cozy-clickhouse-operator", "exported_namespace": "tenant-a"}) {
|
||||
t.Error("exporter series visible with the platform prefix disabled")
|
||||
}
|
||||
}
|
||||
|
||||
func cloneValues(in url.Values) url.Values {
|
||||
out := url.Values{}
|
||||
for k, v := range in {
|
||||
out[k] = slices.Clone(v)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type labelFilter struct {
|
||||
label, value string
|
||||
regex bool
|
||||
}
|
||||
|
||||
func (f labelFilter) matches(labels map[string]string) bool {
|
||||
v := labels[f.label]
|
||||
if f.regex {
|
||||
return regexp.MustCompile("^(?:" + f.value + ")$").MatchString(v)
|
||||
}
|
||||
return v == f.value
|
||||
}
|
||||
|
||||
// parseSelector reads the selectors the proxy and these tests write:
|
||||
// {name="value",name=~"regex",...} with values quoted as MetricsQL reads
|
||||
// them (strconv.Unquote in metricsql's extractStringValue).
|
||||
func parseSelector(s string) ([]labelFilter, bool) {
|
||||
body, ok := strings.CutPrefix(s, "{")
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
var out []labelFilter
|
||||
for body != "}" {
|
||||
i := strings.IndexAny(body, "=!")
|
||||
if i <= 0 {
|
||||
return nil, false
|
||||
}
|
||||
f := labelFilter{label: body[:i]}
|
||||
body = body[i:]
|
||||
switch {
|
||||
case strings.HasPrefix(body, "=~"):
|
||||
f.regex, body = true, body[2:]
|
||||
case strings.HasPrefix(body, "="):
|
||||
body = body[1:]
|
||||
default:
|
||||
return nil, false
|
||||
}
|
||||
lit, err := strconv.QuotedPrefix(body)
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
if f.value, err = strconv.Unquote(lit); err != nil {
|
||||
return nil, false
|
||||
}
|
||||
out = append(out, f)
|
||||
body = strings.TrimPrefix(body[len(lit):], ",")
|
||||
}
|
||||
return out, true
|
||||
}
|
||||
|
||||
func vmselectSelects(t *testing.T, params url.Values, labels map[string]string) bool {
|
||||
t.Helper()
|
||||
parse := func(sels []string) [][]labelFilter {
|
||||
var out [][]labelFilter
|
||||
for _, s := range sels {
|
||||
f, ok := parseSelector(s)
|
||||
if !ok {
|
||||
t.Fatalf("cannot parse selector %q", s)
|
||||
}
|
||||
out = append(out, f)
|
||||
}
|
||||
return out
|
||||
}
|
||||
matches, extra := parse(params["match[]"]), parse(params["extra_filters[]"])
|
||||
if len(matches) == 0 {
|
||||
matches = [][]labelFilter{nil}
|
||||
}
|
||||
for _, m := range matches {
|
||||
for _, e := range extra {
|
||||
all := true
|
||||
for _, f := range append(slices.Clone(m), e...) {
|
||||
if f.label == "__name__" {
|
||||
continue
|
||||
}
|
||||
all = all && f.matches(labels)
|
||||
}
|
||||
if all {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -289,7 +289,7 @@ func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Rout
|
||||
return herr
|
||||
}
|
||||
now := s.now()
|
||||
req, herr := BuildMetricsParams(route.Endpoint, r.Form, route.Namespace, now, s.cfg.Metrics.Limits)
|
||||
req, herr := BuildMetricsParams(route.Endpoint, r.Form, s.cfg.Metrics.namespaceScope(route.Namespace), now, s.cfg.Metrics.Limits)
|
||||
if herr != nil {
|
||||
return herr
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"slices"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -75,7 +76,7 @@ func TestServerMetricsRebuildsUpstreamRequest(t *testing.T) {
|
||||
if r.path != "/api/v1/query_range" {
|
||||
t.Errorf("upstream path = %s", r.path)
|
||||
}
|
||||
if got := r.form["extra_filters[]"]; len(got) != 1 || got[0] != wantMetricsScope {
|
||||
if got := r.form["extra_filters[]"]; !slices.Equal(got, wantMetricsScope) {
|
||||
t.Errorf("extra_filters[] = %q", got)
|
||||
}
|
||||
if r.form.Has("extra_filters") || r.form.Has("extra_label") || r.form.Has("nocache") {
|
||||
@@ -105,7 +106,7 @@ func TestServerMetricsStoreByRange(t *testing.T) {
|
||||
if want := []string{"shortterm", "longterm", "shortterm"}; len(h.stores) != 3 || h.stores[0] != want[0] || h.stores[1] != want[1] || h.stores[2] != want[2] {
|
||||
t.Fatalf("stores = %v, want %v", h.stores, want)
|
||||
}
|
||||
if r := h.got[2]; r.path != "/api/v1/label/pod/values" || r.form.Get("match[]") != wantMetricsScope {
|
||||
if r := h.got[2]; r.path != "/api/v1/label/pod/values" || !slices.Equal(r.form["match[]"], wantMetricsScope) {
|
||||
t.Fatalf("label values upstream = %s %v", r.path, r.form)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -83,6 +83,7 @@ func newHarness(t *testing.T, cfg Config, handler http.HandlerFunc) *harness {
|
||||
cfg.Metrics.UpstreamURL = cfg.UpstreamURL
|
||||
if cfg.Metrics.ShortTermStore == "" {
|
||||
cfg.Metrics.ShortTermStore, cfg.Metrics.LongTermStore, cfg.Metrics.ShortTermRetention = "shortterm", "longterm", 72*time.Hour
|
||||
cfg.Metrics.PlatformNamespacePrefix = "cozy-"
|
||||
}
|
||||
if cfg.Metrics.Limits == (MetricsLimits{}) {
|
||||
cfg.Metrics.Limits = testMetricsLimits()
|
||||
|
||||
Reference in new issue
Block a user