diff --git a/internal/observabilityproxy/metricsparams.go b/internal/observabilityproxy/metricsparams.go new file mode 100644 index 000000000..0e24b6597 --- /dev/null +++ b/internal/observabilityproxy/metricsparams.go @@ -0,0 +1,284 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "fmt" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" +) + +// MetricsLimits bound what one metrics request may make vmselect do. +type MetricsLimits struct { + MaxRange time.Duration + MaxPoints int + MaxTimeout time.Duration +} + +// MetricsConfig names the vmselect stores of the root tenant and the +// retention that decides between them. +type MetricsConfig struct { + RootTenant string + ShortTermStore string + LongTermStore string + ShortTermRetention time.Duration + Limits MetricsLimits + // UpstreamURL maps a store name to its Prometheus API base URL, without + // a trailing slash; nil means the in-cluster vmselect Service. + UpstreamURL func(store string) string +} + +// store picks the short-term store when everything from start on is still +// within its retention, so a live graph never waits on the downsampled one. +func (c MetricsConfig) store(start, now time.Time) string { + if now.Sub(start) <= c.ShortTermRetention { + return c.ShortTermStore + } + return c.LongTermStore +} + +// maxMetricsMatchers bounds match[]: vmselect runs one index search per +// matcher and has no cap of its own. +const maxMetricsMatchers = 20 + +// labelsDefaultWindow is vmselect's own default for the label and series +// APIs (defaultStep in app/vmselect/prometheus); the proxy sends it +// explicitly so the store choice and the range check see the same window. +const labelsDefaultWindow = 5 * time.Minute + +var labelName = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`) + +type metricsRequest struct { + path string + params url.Values + // start is the earliest instant the request reads, for the store choice. + start time.Time +} + +// namespaceSelector quotes ns the way MetricsQL unquotes a label value +// (strconv.Unquote in metricsql's extractStringValue). +func namespaceSelector(ns string) string { + return "{namespace=" + strconv.Quote(ns) + "}" +} + +// BuildMetricsParams rebuilds the upstream parameters from an allowlist and +// adds the namespace scope as the only extra_filters[] value. vmselect ORs +// repeated extra_filters, so a client value would widen the scope; it ANDs +// the single one into every selector of the query and every match[], which +// is what lets the query pass through unparsed. +func BuildMetricsParams(endpoint string, in url.Values, namespace string, now time.Time, lim MetricsLimits) (metricsRequest, *HTTPError) { + req := metricsRequest{path: endpoint, params: url.Values{}} + out := req.params + out.Set("extra_filters[]", namespaceSelector(namespace)) + timeout := lim.MaxTimeout + if v := in.Get("timeout"); v != "" { + d, herr := parseMetricsDuration("timeout", v) + if herr != nil { + return req, herr + } + timeout = min(d, lim.MaxTimeout) + } + out.Set("timeout", formatSeconds(timeout)) + + switch endpoint { + case "api/v1/query": + if herr := copyQuery(out, in); herr != nil { + return req, herr + } + req.start = now + if v := in.Get("time"); v != "" { + t, herr := parseMetricsTime("time", v) + if herr != nil { + return req, herr + } + req.start = t + out.Set("time", formatTime(t)) + } + if v := in.Get("step"); v != "" { + step, herr := parseStep(v, lim) + if herr != nil { + return req, herr + } + out.Set("step", formatSeconds(step)) + } + return req, nil + + case "api/v1/query_range": + if herr := copyQuery(out, in); herr != nil { + return req, herr + } + for _, k := range []string{"start", "end", "step"} { + if in.Get(k) == "" { + return req, errorf(http.StatusBadRequest, "%s is required", k) + } + } + start, end, herr := metricsWindow(in, now, lim) + if herr != nil { + return req, herr + } + step, herr := parseStep(in.Get("step"), lim) + if herr != nil { + return req, herr + } + if points := end.Sub(start)/step + 1; points > time.Duration(lim.MaxPoints) { + return req, errorf(http.StatusUnprocessableEntity, "%d points per series exceed the limit of %d; raise step", points, lim.MaxPoints) + } + req.start = start + out.Set("start", formatTime(start)) + out.Set("end", formatTime(end)) + out.Set("step", formatSeconds(step)) + return req, nil + } + + name, isLabelValues := strings.CutPrefix(endpoint, "api/v1/label/") + if isLabelValues { + var ok bool + if name, ok = strings.CutSuffix(name, "/values"); !ok || strings.Contains(name, "/") { + return req, errorf(http.StatusNotFound, "not found") + } + if !labelName.MatchString(name) { + return req, errorf(http.StatusBadRequest, "invalid label name %q", name) + } + } else if endpoint != "api/v1/series" && endpoint != "api/v1/labels" { + return req, errorf(http.StatusNotFound, "not found") + } + start, end, herr := metricsWindow(in, now, lim) + if herr != nil { + return req, herr + } + req.start = start + out.Set("start", formatTime(start)) + out.Set("end", formatTime(end)) + + matchers := in["match[]"] + if len(matchers) > maxMetricsMatchers { + return req, errorf(http.StatusUnprocessableEntity, "at most %d match[] are allowed", maxMetricsMatchers) + } + for _, m := range matchers { + if m == "" { + return req, errorf(http.StatusBadRequest, "empty match[]") + } + out.Add("match[]", m) + } + if len(matchers) == 0 { + if endpoint == "api/v1/series" { + return req, errorf(http.StatusBadRequest, "match[] is required") + } + // vmselect started with -search.ignoreExtraFiltersAtLabelsAPI drops + // extra_filters[] on the label APIs unless a match[] is present + // (getCommonParamsInternal), so the scope is repeated here. + out.Set("match[]", namespaceSelector(namespace)) + } + return req, nil +} + +func copyQuery(out, in url.Values) *HTTPError { + q := in.Get("query") + if q == "" { + return errorf(http.StatusBadRequest, "query is required") + } + out.Set("query", q) + return nil +} + +// metricsWindow reads start and end with vmselect's label API defaults: +// end is now and start is five minutes before end. +func metricsWindow(in url.Values, now time.Time, lim MetricsLimits) (time.Time, time.Time, *HTTPError) { + end := now + if v := in.Get("end"); v != "" { + t, herr := parseMetricsTime("end", v) + if herr != nil { + return time.Time{}, time.Time{}, herr + } + end = t + } + start := end.Add(-labelsDefaultWindow) + if v := in.Get("start"); v != "" { + t, herr := parseMetricsTime("start", v) + if herr != nil { + return time.Time{}, time.Time{}, herr + } + start = t + } + if start.After(end) { + return time.Time{}, time.Time{}, errorf(http.StatusBadRequest, "start is after end") + } + if end.Sub(start) > lim.MaxRange { + return time.Time{}, time.Time{}, errorf(http.StatusUnprocessableEntity, "range %s exceeds the limit of %s", end.Sub(start), lim.MaxRange) + } + return start, end, nil +} + +// parseStep refuses a step above the range limit: vmselect aligns the +// window outward to whole steps, and an implicit rollup window is one step, +// so a huge step reads far more than the range it was sent with. +func parseStep(v string, lim MetricsLimits) (time.Duration, *HTTPError) { + step, herr := parseMetricsDuration("step", v) + if herr != nil { + return 0, herr + } + if step > lim.MaxRange { + return 0, errorf(http.StatusUnprocessableEntity, "step %s exceeds the limit of %s", step, lim.MaxRange) + } + return step, nil +} + +// parseMetricsDuration takes Prometheus float seconds or a duration such as +// 1m30s. vmselect works in milliseconds and refuses anything shorter. +func parseMetricsDuration(key, v string) (time.Duration, *HTTPError) { + d, err := parseDuration(v) + if err != nil || d < time.Millisecond { + return 0, errorf(http.StatusBadRequest, "invalid %s %q", key, v) + } + return d, nil +} + +// parseMetricsTime takes the two forms the Prometheus API defines: float +// unix seconds and RFC3339. vmselect would also guess milliseconds from the +// digit count and accept relative times; a guessed unit silently moves the +// window, so those are refused. +func parseMetricsTime(key, v string) (time.Time, *HTTPError) { + if sec, frac, ok := strings.Cut(v, "."); isDigits(sec) && (!ok || isDigits(frac)) { + if len(sec) > 10 { + return time.Time{}, errorf(http.StatusBadRequest, "invalid %s %q: not unix seconds", key, v) + } + secs, _ := strconv.ParseInt(sec, 10, 64) + var nanos int64 + if ok { + nanos, _ = strconv.ParseInt((frac + "000000000")[:9], 10, 64) + } + return time.Unix(secs, nanos).UTC(), nil + } + t, err := time.Parse(time.RFC3339Nano, v) + if err != nil || t.Before(time.Unix(0, 0)) { + return time.Time{}, errorf(http.StatusBadRequest, "invalid %s %q", key, v) + } + return t.UTC(), nil +} + +func formatTime(t time.Time) string { + ms := t.UnixMilli() + return fmt.Sprintf("%d.%03d", ms/1000, ms%1000) +} + +func formatSeconds(d time.Duration) string { + return strconv.FormatFloat(float64(d.Milliseconds())/1000, 'f', -1, 64) +} diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go new file mode 100644 index 000000000..be1d69c38 --- /dev/null +++ b/internal/observabilityproxy/metricsparams_test.go @@ -0,0 +1,246 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "net/http" + "net/url" + "strconv" + "strings" + "testing" + "time" +) + +func testMetricsLimits() MetricsLimits { + return MetricsLimits{MaxRange: 30 * 24 * time.Hour, MaxPoints: 11000, MaxTimeout: 30 * time.Second} +} + +const wantMetricsScope = `{namespace="tenant-a"}` + +func unix(t time.Time) string { return strconv.FormatInt(t.Unix(), 10) } + +func buildMetrics(t *testing.T, endpoint string, in url.Values) metricsRequest { + t.Helper() + req, err := BuildMetricsParams(endpoint, in, "tenant-a", testNow, testMetricsLimits()) + if err != nil { + t.Fatalf("BuildMetricsParams(%s, %v): %v", endpoint, in, err) + } + return req +} + +func metricsStatus(endpoint string, in url.Values) int { + _, err := BuildMetricsParams(endpoint, in, "tenant-a", testNow, testMetricsLimits()) + if err == nil { + return http.StatusOK + } + return err.Code +} + +func TestBuildMetricsParamsAllowlist(t *testing.T) { + hostile := url.Values{ + "extra_filters": {`{namespace=~".+"}`}, + "extra_filters[]": {`{namespace="tenant-b"}`, `{namespace=~".*"}`}, + "extra_label": {"namespace=tenant-b"}, + "match": {`{namespace="tenant-b"}`}, + "nocache": {"1"}, + "limit": {"1"}, + "AccountID": {"7"}, + "latency_offset": {"1h"}, + "round_digits": {"2"}, + } + cases := map[string]url.Values{ + "api/v1/query": {"query": {"up"}, "time": {unix(testNow)}, "step": {"30s"}}, + "api/v1/query_range": {"query": {"up"}, "start": {unix(testNow.Add(-time.Hour))}, "end": {unix(testNow)}, "step": {"60"}}, + "api/v1/series": {"match[]": {"up"}}, + "api/v1/labels": {"match[]": {"up"}}, + "api/v1/label/pod/values": {"match[]": {"up"}}, + } + allowed := map[string]bool{"query": true, "start": true, "end": true, "step": true, "time": true, "match[]": true, "timeout": true, "extra_filters[]": true} + for endpoint, in := range cases { + t.Run(endpoint, func(t *testing.T) { + for k, v := range hostile { + in[k] = v + } + req := buildMetrics(t, endpoint, in) + if req.path != endpoint { + t.Errorf("path = %q", req.path) + } + for k := range req.params { + if !allowed[k] { + t.Errorf("parameter %q forwarded: %v", k, req.params) + } + } + if got := req.params["extra_filters[]"]; len(got) != 1 || got[0] != wantMetricsScope { + t.Errorf("extra_filters[] = %q, want exactly [%s]", got, wantMetricsScope) + } + for _, m := range req.params["match[]"] { + if strings.Contains(m, "tenant-b") { + t.Errorf("client match without brackets forwarded as match[]: %q", m) + } + } + }) + } +} + +func TestNamespaceSelectorEscapes(t *testing.T) { + for _, ns := range []string{"tenant-a", `a"} or {namespace=~".+`, `back\slash`, "new\nline"} { + sel := namespaceSelector(ns) + lit, ok := strings.CutPrefix(sel, "{namespace=") + lit, ok2 := strings.CutSuffix(lit, "}") + if !ok || !ok2 { + t.Fatalf("selector %q has the wrong shape", sel) + } + // MetricsQL reads a double-quoted label value with strconv.Unquote + // (metricsql extractStringValue). + got, err := strconv.Unquote(lit) + if err != nil || got != ns { + t.Errorf("selector %q decodes to %q, %v; want %q", sel, got, err, ns) + } + } +} + +func TestBuildMetricsParamsLabelEndpointsAlwaysMatch(t *testing.T) { + for _, endpoint := range []string{"api/v1/labels", "api/v1/label/__name__/values"} { + req := buildMetrics(t, endpoint, url.Values{}) + if got := req.params["match[]"]; len(got) != 1 || got[0] != wantMetricsScope { + t.Errorf("%s without match[] sends %q, want the namespace selector", endpoint, got) + } + req = buildMetrics(t, endpoint, url.Values{"match[]": {"up", `{job="x"}`}}) + if got := req.params["match[]"]; len(got) != 2 || got[0] != "up" || got[1] != `{job="x"}` { + t.Errorf("%s client match[] = %q, want them kept", endpoint, got) + } + } + if got := metricsStatus("api/v1/series", url.Values{}); got != http.StatusBadRequest { + t.Errorf("series without match[] = %d, want 400", got) + } + if got := metricsStatus("api/v1/series", url.Values{"match[]": {""}}); got != http.StatusBadRequest { + t.Errorf("empty match[] = %d, want 400", got) + } + many := url.Values{} + for i := range maxMetricsMatchers + 1 { + many.Add("match[]", "up"+strconv.Itoa(i)) + } + if got := metricsStatus("api/v1/labels", many); got != http.StatusUnprocessableEntity { + t.Errorf("%d match[] = %d, want 422", maxMetricsMatchers+1, got) + } +} + +func TestBuildMetricsParamsLimits(t *testing.T) { + ago := func(d time.Duration) string { return unix(testNow.Add(-d)) } + day := 24 * time.Hour + rng := func(start, end, step string) url.Values { + return url.Values{"query": {"up"}, "start": {start}, "end": {end}, "step": {step}} + } + tests := []struct { + name string + endpoint string + in url.Values + want int + }{ + {"range at the limit", "api/v1/query_range", rng(ago(30*day), ago(0), "1h"), http.StatusOK}, + {"range above the limit", "api/v1/query_range", rng(ago(31*day), ago(0), "1h"), http.StatusUnprocessableEntity}, + {"points at the limit", "api/v1/query_range", rng(ago(10999*time.Minute), ago(0), "1m"), http.StatusOK}, + {"points above the limit", "api/v1/query_range", rng(ago(11000*time.Minute), ago(0), "1m"), http.StatusUnprocessableEntity}, + {"step above the range limit", "api/v1/query_range", rng(ago(time.Hour), ago(0), "31d"), http.StatusUnprocessableEntity}, + {"zero step", "api/v1/query_range", rng(ago(time.Hour), ago(0), "0"), http.StatusBadRequest}, + {"sub-millisecond step", "api/v1/query_range", rng(ago(time.Hour), ago(0), "0.0001"), http.StatusBadRequest}, + {"missing step", "api/v1/query_range", url.Values{"query": {"up"}, "start": {ago(time.Hour)}, "end": {ago(0)}}, http.StatusBadRequest}, + {"missing start", "api/v1/query_range", url.Values{"query": {"up"}, "end": {ago(0)}, "step": {"1m"}}, http.StatusBadRequest}, + {"missing query", "api/v1/query_range", url.Values{"start": {ago(time.Hour)}, "end": {ago(0)}, "step": {"1m"}}, http.StatusBadRequest}, + {"inverted range", "api/v1/query_range", rng(ago(0), ago(time.Hour), "1m"), http.StatusBadRequest}, + {"millisecond timestamp", "api/v1/query_range", rng(ago(time.Hour)+"000", ago(0)+"000", "1m"), http.StatusBadRequest}, + {"relative time", "api/v1/query_range", rng("-1h", "now", "1m"), http.StatusBadRequest}, + {"rfc3339", "api/v1/query_range", rng("2026-11-04T11:00:00Z", "2026-11-04T12:00:00.5Z", "15s"), http.StatusOK}, + {"instant without time", "api/v1/query", url.Values{"query": {"up"}}, http.StatusOK}, + {"instant step above the range limit", "api/v1/query", url.Values{"query": {"up"}, "step": {"31d"}}, http.StatusUnprocessableEntity}, + {"instant bad time", "api/v1/query", url.Values{"query": {"up"}, "time": {"yesterday"}}, http.StatusBadRequest}, + {"negative timeout", "api/v1/query", url.Values{"query": {"up"}, "timeout": {"-1s"}}, http.StatusBadRequest}, + {"series range above the limit", "api/v1/series", url.Values{"match[]": {"up"}, "start": {ago(31 * day)}}, http.StatusUnprocessableEntity}, + {"labels inverted range", "api/v1/labels", url.Values{"start": {ago(0)}, "end": {ago(time.Hour)}}, http.StatusBadRequest}, + {"invalid label name", "api/v1/label/a-b/values", url.Values{}, http.StatusBadRequest}, + {"unknown endpoint", "api/v1/status/tsdb", url.Values{}, http.StatusNotFound}, + {"export", "api/v1/export", url.Values{"match[]": {"up"}}, http.StatusNotFound}, + {"label without values", "api/v1/label/pod", url.Values{}, http.StatusNotFound}, + {"root", "", url.Values{}, http.StatusNotFound}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := metricsStatus(tt.endpoint, tt.in); got != tt.want { + t.Fatalf("status = %d, want %d", got, tt.want) + } + }) + } +} + +func TestBuildMetricsParamsCanonicalForms(t *testing.T) { + req := buildMetrics(t, "api/v1/query_range", url.Values{ + "query": {"up"}, "start": {"2026-11-04T11:00:00.25Z"}, "end": {unix(testNow)}, "step": {"1m30s"}, + }) + want := url.Values{ + "query": {"up"}, "start": {"1793790000.250"}, "end": {"1793793600.000"}, "step": {"90"}, + "timeout": {"30"}, "extra_filters[]": {wantMetricsScope}, + } + if req.params.Encode() != want.Encode() { + t.Errorf("params = %v\nwant %v", req.params, want) + } + + for in, want := range map[string]string{"": "30", "10s": "10", "5m": "30", "1.5": "1.5"} { + v := url.Values{"query": {"up"}} + if in != "" { + v.Set("timeout", in) + } + if got := buildMetrics(t, "api/v1/query", v).params.Get("timeout"); got != want { + t.Errorf("timeout %q forwarded as %q, want %q", in, got, want) + } + } + + req = buildMetrics(t, "api/v1/labels", url.Values{}) + if req.params.Get("end") != "1793793600.000" || req.params.Get("start") != "1793793300.000" { + t.Errorf("labels window = %s..%s, want the last 5m", req.params.Get("start"), req.params.Get("end")) + } + if q := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}}); q.params.Has("time") || !q.start.Equal(testNow) { + t.Errorf("instant query without time: params %v, start %v", q.params, q.start) + } +} + +func TestMetricsStoreByRange(t *testing.T) { + cfg := MetricsConfig{ShortTermStore: "shortterm", LongTermStore: "longterm", ShortTermRetention: 72 * time.Hour} + tests := []struct { + name string + endpoint string + in url.Values + want string + }{ + {"instant now", "api/v1/query", url.Values{"query": {"up"}}, "shortterm"}, + {"instant within retention", "api/v1/query", url.Values{"query": {"up"}, "time": {unix(testNow.Add(-71 * time.Hour))}}, "shortterm"}, + {"instant past retention", "api/v1/query", url.Values{"query": {"up"}, "time": {unix(testNow.Add(-73 * time.Hour))}}, "longterm"}, + {"live graph", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-6 * time.Hour))}, "end": {unix(testNow)}, "step": {"60"}}, "shortterm"}, + {"range fits exactly", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-72 * time.Hour))}, "end": {unix(testNow)}, "step": {"600"}}, "shortterm"}, + {"range past retention", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-7 * 24 * time.Hour))}, "end": {unix(testNow)}, "step": {"600"}}, "longterm"}, + {"old window ending long ago", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-5 * 24 * time.Hour))}, "end": {unix(testNow.Add(-4 * 24 * time.Hour))}, "step": {"600"}}, "longterm"}, + {"labels default window", "api/v1/labels", url.Values{}, "shortterm"}, + {"series past retention", "api/v1/series", url.Values{"match[]": {"up"}, "start": {unix(testNow.Add(-4 * 24 * time.Hour))}}, "longterm"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := buildMetrics(t, tt.endpoint, tt.in) + if got := cfg.store(req.start, testNow); got != tt.want { + t.Fatalf("store = %s, want %s", got, tt.want) + } + }) + } +}