diff --git a/cmd/observability-proxy/main.go b/cmd/observability-proxy/main.go index 558908671..a3687fe08 100644 --- a/cmd/observability-proxy/main.go +++ b/cmd/observability-proxy/main.go @@ -14,9 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Command observability-proxy serves tenant logs from the VictoriaLogs -// stores behind a Kubernetes TokenReview and SubjectAccessReview, with the -// tenant's namespace scope forced into every upstream request. +// Command observability-proxy serves tenant logs and metrics from the +// VictoriaLogs and VictoriaMetrics stores behind a Kubernetes TokenReview +// and SubjectAccessReview, with the tenant's namespace scope forced into +// every upstream request. package main import ( @@ -37,6 +38,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" "github.com/prometheus/client_golang/prometheus/promhttp" + "k8s.io/apimachinery/pkg/util/validation" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" @@ -56,6 +58,7 @@ type options struct { tailIdleTimeout time.Duration shutdownTimeout time.Duration logs observabilityproxy.LogsLimits + metrics observabilityproxy.MetricsConfig limiter observabilityproxy.LimiterConfig } @@ -81,6 +84,14 @@ func parseFlags(args []string) (options, error) { fs.IntVar(&o.logs.DefaultLimit, "logs-default-limit", 1000, "entries returned when a request sets no limit") fs.IntVar(&o.logs.MaxLimit, "logs-max-limit", 10000, "largest limit a request may set") fs.DurationVar(&o.logs.MaxTailResume, "logs-max-tail-resume", 15*time.Minute, "how far back a tail may resume with since or start_offset") + // Store names and retention follow metricsStorages in + // packages/system/monitoring/values.yaml. + fs.StringVar(&o.metrics.ShortTermStore, "metrics-shortterm-store", "shortterm", "VMCluster name of the short-term metrics store of the root tenant") + fs.StringVar(&o.metrics.LongTermStore, "metrics-longterm-store", "longterm", "VMCluster name of the long-term metrics store of the root tenant") + fs.DurationVar(&o.metrics.ShortTermRetention, "metrics-shortterm-retention", 72*time.Hour, "retention of the short-term store; older windows are read from the long-term one") + fs.DurationVar(&o.metrics.Limits.MaxRange, "metrics-max-range", 30*24*time.Hour, "longest [start, end] and step a metrics request may use") + fs.IntVar(&o.metrics.Limits.MaxPoints, "metrics-max-points", 11000, "most points per series a range query may return") + fs.DurationVar(&o.metrics.Limits.MaxTimeout, "metrics-max-timeout", 30*time.Second, "timeout forwarded to vmselect; a longer client timeout is clamped; must be below --request-timeout when that is set") fs.IntVar(&o.limiter.MaxConcurrentPerUser, "max-concurrent-per-user", 10, "concurrent non-tail requests per user; 0 disables") fs.Float64Var(&o.limiter.RatePerUser, "rate-limit-per-user", 10, "sustained requests per second per user; 0 disables") fs.IntVar(&o.limiter.RateBurst, "rate-limit-burst", 20, "request burst per user") @@ -94,6 +105,13 @@ func parseFlags(args []string) (options, error) { if fs.NArg() > 0 { return o, fmt.Errorf("unexpected argument %q", fs.Arg(0)) } + o.metrics.RootTenant = o.rootTenant + for _, store := range []string{o.metrics.ShortTermStore, o.metrics.LongTermStore} { + // The name becomes part of a Service host name. + if errs := validation.IsDNS1123Label("vmselect-" + store); store == "" || len(errs) > 0 { + return o, fmt.Errorf("invalid metrics store name %q", store) + } + } switch { case o.rootTenant == "": return o, errors.New("--root-tenant must not be empty") @@ -110,8 +128,17 @@ func parseFlags(args []string) (options, error) { return o, errors.New("--logs-max-range must be at least 3s") case o.logs.DefaultLimit <= 0 || o.logs.MaxLimit < o.logs.DefaultLimit: return o, errors.New("--logs-default-limit must be positive and not above --logs-max-limit") + case o.metrics.ShortTermRetention <= 0 || o.metrics.Limits.MaxRange <= 0 || o.metrics.Limits.MaxTimeout <= 0: + return o, errors.New("--metrics-shortterm-retention, --metrics-max-range and --metrics-max-timeout must be positive") + case o.metrics.Limits.MaxPoints < 2: + return o, errors.New("--metrics-max-points must be at least 2") case o.logs.MaxTailResume < 0: return o, errors.New("--logs-max-tail-resume must not be negative") + // The proxy's deadline starts before the request reaches vmselect, so + // at an equal or longer timeout it cuts a query vmselect is still + // allowed to run and answers 504 for a store that is within its time. + case o.requestTimeout > 0 && o.metrics.Limits.MaxTimeout >= o.requestTimeout: + return o, errors.New("--metrics-max-timeout must be below --request-timeout") case o.requestTimeout < 0 || o.tailIdleTimeout < 0 || o.authCacheTTL < 0 || o.shutdownTimeout < 0: return o, errors.New("timeouts and the auth cache TTL must not be negative") // NaN passes every comparison and +Inf never empties a bucket; either @@ -167,6 +194,7 @@ func run(o options) error { proxy := observabilityproxy.NewServer(observabilityproxy.Config{ ClusterDomain: o.clusterDomain, Logs: o.logs, + Metrics: o.metrics, Limiter: o.limiter, RequestTimeout: o.requestTimeout, TailIdleTimeout: o.tailIdleTimeout, diff --git a/cmd/observability-proxy/main_test.go b/cmd/observability-proxy/main_test.go index ea678461c..bf7ac396b 100644 --- a/cmd/observability-proxy/main_test.go +++ b/cmd/observability-proxy/main_test.go @@ -32,6 +32,17 @@ func TestParseFlagsDefaults(t *testing.T) { if o.authCacheTTL != 10*time.Second || o.logs.MaxTailResume != 15*time.Minute || o.logs.MaxRange != 7*24*time.Hour { t.Errorf("limit defaults changed: %+v", o) } + // Retention and store names of packages/system/monitoring/values.yaml. + m := o.metrics + if m.ShortTermStore != "shortterm" || m.LongTermStore != "longterm" || m.ShortTermRetention != 72*time.Hour { + t.Errorf("metrics store defaults changed: %+v", m) + } + if m.Limits.MaxRange != 30*24*time.Hour || m.Limits.MaxPoints != 11000 || m.Limits.MaxTimeout != 30*time.Second { + t.Errorf("metrics limit defaults changed: %+v", m.Limits) + } + if m.RootTenant != o.rootTenant { + t.Errorf("metrics root tenant = %q, want --root-tenant", m.RootTenant) + } } func TestParseFlagsOverrides(t *testing.T) { @@ -48,6 +59,21 @@ func TestParseFlagsOverrides(t *testing.T) { } } +func TestParseFlagsMetricsOverrides(t *testing.T) { + o, err := parseFlags([]string{ + "--root-tenant=tenant-x", "--metrics-shortterm-store=fast", "--metrics-longterm-store=slow", + "--metrics-shortterm-retention=24h", "--metrics-max-range=168h", "--metrics-max-points=500", "--metrics-max-timeout=10s", + }) + if err != nil { + t.Fatal(err) + } + m := o.metrics + if m.RootTenant != "tenant-x" || m.ShortTermStore != "fast" || m.LongTermStore != "slow" || m.ShortTermRetention != 24*time.Hour || + m.Limits.MaxRange != 168*time.Hour || m.Limits.MaxPoints != 500 || m.Limits.MaxTimeout != 10*time.Second { + t.Errorf("overrides not applied: %+v", m) + } +} + func TestParseFlagsInvalid(t *testing.T) { for _, args := range [][]string{ {"--root-tenant="}, @@ -69,7 +95,17 @@ func TestParseFlagsInvalid(t *testing.T) { {"--max-concurrent-per-user=-1"}, {"--max-tails-per-user=-1"}, {"--max-tails=-1"}, + {"--metrics-shortterm-store="}, + {"--metrics-longterm-store=Long.Term"}, + {"--metrics-shortterm-retention=0s"}, + {"--metrics-max-range=0s"}, + {"--metrics-max-points=1"}, + {"--metrics-max-timeout=0s"}, {"--unknown"}, + // The proxy would cut the request while vmselect is still within + // the timeout it was given. + {"--request-timeout=10s", "--metrics-max-timeout=30s"}, + {"--request-timeout=30s", "--metrics-max-timeout=30s"}, {"unexpected", "--root-tenant=tenant-x"}, } { if _, err := parseFlags(args); err == nil { @@ -77,3 +113,16 @@ func TestParseFlagsInvalid(t *testing.T) { } } } + +// The vmselect timeout must stay strictly below a request timeout, and is +// free when the request has none. +func TestParseFlagsTimeoutPairs(t *testing.T) { + for _, args := range [][]string{ + {"--request-timeout=31s", "--metrics-max-timeout=30s"}, + {"--request-timeout=0", "--metrics-max-timeout=10m"}, + } { + if _, err := parseFlags(args); err != nil { + t.Errorf("parseFlags(%v) refused: %v", args, err) + } + } +} diff --git a/internal/observabilityproxy/metricslookback.go b/internal/observabilityproxy/metricslookback.go new file mode 100644 index 000000000..e55d189a5 --- /dev/null +++ b/internal/observabilityproxy/metricslookback.go @@ -0,0 +1,207 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "regexp" + "strconv" + "strings" + "time" +) + +// instantDefaultStep is the step vmselect uses for an instant query without +// one (defaultStep in app/vmselect/prometheus); it is also the implicit +// rollup window of a bare selector. +const instantDefaultStep = 5 * time.Minute + +// silenceInterval is how far before a rollup or subquery window vmselect +// starts reading (maxSilenceInterval in app/vmselect/promql: the value of +// -search.minStalenessInterval, 5m when unset; cozystack does not set it). +const silenceInterval = 5 * time.Minute + +// latencyOffset is how far vmselect moves an instant query evaluated within +// it of the current time back (-search.latencyOffset, 30s by default and +// not set by cozystack); the proxy forwards neither nocache nor +// latency_offset. +const latencyOffset = 30 * time.Second + +// queryDurationPart is the subset of the MetricsQL duration grammar +// (scanSingleDuration in metricsql) the scan understands. Step multiples +// (5i, $__interval), negative parts and upper-case units are left out on +// purpose: anything else makes the bound unknown. +var ( + queryDurationPart = regexp.MustCompile(`(\d+(?:\.\d+)?)(ms|s|m|h|d|w|y)`) + queryDuration = regexp.MustCompile(`^(?:\d+(?:\.\d+)?(?:ms|s|m|h|d|w|y))+$`) + querySeconds = regexp.MustCompile(`^\d+(?:\.\d+)?$`) +) + +var queryDurationUnits = map[string]time.Duration{ + "ms": time.Millisecond, "s": time.Second, "m": time.Minute, "h": time.Hour, + "d": 24 * time.Hour, "w": 7 * 24 * time.Hour, "y": 365 * 24 * time.Hour, +} + +// queryStart is the earliest instant a query evaluated from start may read; +// zero when the bound is unknown. +func queryStart(query string, start time.Time, step time.Duration) time.Time { + lookback, ok := queryLookback(query, step) + if !ok { + return time.Time{} + } + return start.Add(-lookback) +} + +// queryLookback is an upper bound on how far before its evaluation window +// a query reads, from a lexical scan that never parses the query. Every +// [...], offset and "(" counts, including ones inside quoted label values: +// an extra hit only makes the bound larger or unknown, never smaller. +// Windows and offsets are summed rather than maxed, which covers nesting; +// WITH is refused because a template applies one window several times. +// vmselect starts a rollup max(window, step) plus the silence interval +// before the first point, a subquery window+step+silence before it and +// aligns both down to a step. A rollup function or offset over anything +// but a selector is an implicit subquery (getRollupExprArg), so every "(" +// is counted as one. Each step term uses the largest step seen. +func queryLookback(query string, step time.Duration) (time.Duration, bool) { + if strings.Contains(query, "@") { + return 0, false + } + var sum float64 + subqueries := 0 + maxStep := float64(step) + for rest := query; ; { + open := strings.IndexByte(rest, '[') + if open < 0 { + break + } + closing := strings.IndexByte(rest[open:], ']') + if closing < 0 { + return 0, false + } + window, subStep, isSubquery := strings.Cut(rest[open+1:open+closing], ":") + rest = rest[open+closing+1:] + if isSubquery { + subqueries++ + if strings.TrimSpace(window) == "" { + window = "0s" + } + if strings.TrimSpace(subStep) != "" { + d, ok := parseQueryDuration(subStep) + if !ok { + return 0, false + } + maxStep = max(maxStep, d) + } + } + d, ok := parseQueryDuration(window) + if !ok { + return 0, false + } + sum += d + } + lower := strings.ToLower(query) + for i := 0; ; { + k := strings.Index(lower[i:], "offset") + if k < 0 { + break + } + k += i + i = k + len("offset") + if !isKeyword(lower, k, i) { + continue + } + arg := strings.TrimLeft(lower[i:], " \t\n\r") + negative := strings.HasPrefix(arg, "-") + if negative { + arg = strings.TrimLeft(arg[1:], " \t\n\r") + } + end := strings.IndexFunc(arg, func(r rune) bool { return !inDuration(r) }) + if end < 0 { + end = len(arg) + } + d, ok := parseQueryDuration(arg[:end]) + if !ok { + return 0, false + } + if !negative { + sum += d + } + } + if hasKeyword(lower, "with") { + return 0, false + } + n := float64(subqueries + strings.Count(query, "(")) + sum += (2+3*n)*maxStep + (1+n)*float64(silenceInterval) + if sum > float64(maxMetricsDuration) { + return 0, false + } + return time.Duration(sum), true +} + +// isKeyword reports whether lower[start:end] stands on its own rather than +// inside an identifier. +func isKeyword(lower string, start, end int) bool { + return (start == 0 || !continuesIdent(lower[start-1], false)) && + (end == len(lower) || !continuesIdent(lower[end], true)) +} + +func inDuration(r rune) bool { + return r >= '0' && r <= '9' || r >= 'a' && r <= 'z' || r == '.' || r == '_' +} + +func hasKeyword(lower, word string) bool { + for i := 0; ; { + k := strings.Index(lower[i:], word) + if k < 0 { + return false + } + k += i + i = k + len(word) + if isKeyword(lower, k, i) { + return true + } + } +} + +// continuesIdent reports whether c, next to "offset", makes it part of an +// identifier rather than the keyword. Digits and dots only continue an +// identifier after it, so before it they still count as a keyword hit. +func continuesIdent(c byte, after bool) bool { + return c >= 'a' && c <= 'z' || c == '_' || c == ':' || c >= 0x80 || + after && (c >= '0' && c <= '9' || c == '.' || c == '\\') +} + +// parseQueryDuration returns nanoseconds as a float so that a sum of huge +// durations cannot wrap; the caller compares it against the bound. +func parseQueryDuration(s string) (float64, bool) { + s = strings.TrimSpace(s) + if querySeconds.MatchString(s) { + f, err := strconv.ParseFloat(s, 64) + return f * float64(time.Second), err == nil + } + if !queryDuration.MatchString(s) { + return 0, false + } + var d float64 + for _, m := range queryDurationPart.FindAllStringSubmatch(s, -1) { + f, err := strconv.ParseFloat(m[1], 64) + if err != nil { + return 0, false + } + d += f * float64(queryDurationUnits[m[2]]) + } + return d, true +} diff --git a/internal/observabilityproxy/metricsparams.go b/internal/observabilityproxy/metricsparams.go new file mode 100644 index 000000000..8beb347d0 --- /dev/null +++ b/internal/observabilityproxy/metricsparams.go @@ -0,0 +1,325 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "fmt" + "math" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" +) + +// MetricsLimits bound the window, resolution and timeout a metrics request +// asks vmselect for. Windows and offsets inside the expression are not +// limited here: they cannot read past the store's retention, vmselect caps +// the series and samples one query may touch, and MaxTimeout bounds the +// work of rollups over long overlapping windows. +type MetricsLimits struct { + MaxRange time.Duration + MaxPoints int + MaxTimeout time.Duration +} + +// MetricsConfig names the vmselect stores of the root tenant and the +// retention that decides between them. +type MetricsConfig struct { + RootTenant string + ShortTermStore string + LongTermStore string + ShortTermRetention time.Duration + Limits MetricsLimits + // UpstreamURL maps a store name to its Prometheus API base URL, without + // a trailing slash; nil means the in-cluster vmselect Service. + UpstreamURL func(store string) string +} + +// store picks the short-term store when everything from start on is still +// within its retention, so a live graph never waits on the downsampled one. +func (c MetricsConfig) store(start, now time.Time) string { + if now.Sub(start) <= c.ShortTermRetention { + return c.ShortTermStore + } + return c.LongTermStore +} + +// maxMetricsMatchers bounds match[]: vmselect runs one index search per +// matcher and has no cap of its own. +const maxMetricsMatchers = 20 + +// labelsDefaultWindow is vmselect's own default for the label and series +// APIs (defaultStep in app/vmselect/prometheus); the proxy sends it +// explicitly so the store choice and the range check see the same window. +const labelsDefaultWindow = 5 * time.Minute + +var labelName = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`) + +type metricsRequest struct { + path string + params url.Values + // start is the earliest instant the request reads, for the store choice; + // zero when that is unknown, which picks the long-term store. + start time.Time +} + +// namespaceSelector quotes ns the way MetricsQL unquotes a label value +// (strconv.Unquote in metricsql's extractStringValue). +func namespaceSelector(ns string) string { + return "{namespace=" + strconv.Quote(ns) + "}" +} + +// BuildMetricsParams rebuilds the upstream parameters from an allowlist and +// adds the namespace scope as the only extra_filters[] value. vmselect ORs +// repeated extra_filters, so a client value would widen the scope; it ANDs +// the single one into every selector of the query and every match[], which +// is what lets the query pass through unparsed. +func BuildMetricsParams(endpoint string, in url.Values, namespace string, now time.Time, lim MetricsLimits) (metricsRequest, *HTTPError) { + req := metricsRequest{path: endpoint, params: url.Values{}} + out := req.params + out.Set("extra_filters[]", namespaceSelector(namespace)) + timeout := lim.MaxTimeout + if v := in.Get("timeout"); v != "" { + d, herr := parseMetricsDuration("timeout", v) + if herr != nil { + return req, herr + } + timeout = min(d, lim.MaxTimeout) + } + out.Set("timeout", formatSeconds(timeout)) + + switch endpoint { + case "api/v1/query": + if herr := copyQuery(out, in); herr != nil { + return req, herr + } + at := now + if v := in.Get("time"); v != "" { + t, herr := parseMetricsTime("time", v) + if herr != nil { + return req, herr + } + at = t + out.Set("time", formatTime(t)) + } + step := instantDefaultStep + if v := in.Get("step"); v != "" { + var herr *HTTPError + if step, herr = parseStep(v, lim); herr != nil { + return req, herr + } + out.Set("step", formatSeconds(step)) + } + req.start = queryStart(out.Get("query"), at.Add(-latencyOffset), step) + return req, nil + + case "api/v1/query_range": + if herr := copyQuery(out, in); herr != nil { + return req, herr + } + for _, k := range []string{"start", "end", "step"} { + if in.Get(k) == "" { + return req, errorf(http.StatusBadRequest, "%s is required", k) + } + } + start, end, herr := metricsWindow(in, now, lim) + if herr != nil { + return req, herr + } + step, herr := parseStep(in.Get("step"), lim) + if herr != nil { + return req, herr + } + if points := end.Sub(start)/step + 1; points > time.Duration(lim.MaxPoints) { + return req, errorf(http.StatusUnprocessableEntity, "%d points per series exceed the limit of %d; raise step", points, lim.MaxPoints) + } + req.start = queryStart(out.Get("query"), start, step) + out.Set("start", formatTime(start)) + out.Set("end", formatTime(end)) + out.Set("step", formatSeconds(step)) + return req, nil + } + + name, isLabelValues := strings.CutPrefix(endpoint, "api/v1/label/") + if isLabelValues { + var ok bool + if name, ok = strings.CutSuffix(name, "/values"); !ok || strings.Contains(name, "/") { + return req, errorf(http.StatusNotFound, "not found") + } + if !labelName.MatchString(name) { + return req, errorf(http.StatusBadRequest, "invalid label name %q", name) + } + } else if endpoint != "api/v1/series" && endpoint != "api/v1/labels" { + return req, errorf(http.StatusNotFound, "not found") + } + start, end, herr := metricsWindow(in, now, lim) + if herr != nil { + return req, herr + } + req.start = start + out.Set("start", formatTime(start)) + out.Set("end", formatTime(end)) + + matchers := in["match[]"] + if len(matchers) > maxMetricsMatchers { + return req, errorf(http.StatusUnprocessableEntity, "at most %d match[] are allowed", maxMetricsMatchers) + } + for _, m := range matchers { + if m == "" { + return req, errorf(http.StatusBadRequest, "empty match[]") + } + out.Add("match[]", m) + } + if len(matchers) == 0 { + if endpoint == "api/v1/series" { + return req, errorf(http.StatusBadRequest, "match[] is required") + } + // vmselect started with -search.ignoreExtraFiltersAtLabelsAPI drops + // extra_filters[] on the label APIs unless a match[] is present + // (getCommonParamsInternal), so the scope is repeated here. + out.Set("match[]", namespaceSelector(namespace)) + } + return req, nil +} + +func copyQuery(out, in url.Values) *HTTPError { + q := in.Get("query") + if q == "" { + return errorf(http.StatusBadRequest, "query is required") + } + out.Set("query", q) + return nil +} + +// metricsWindow reads start and end with vmselect's label API defaults: +// end is now and start is five minutes before end. +func metricsWindow(in url.Values, now time.Time, lim MetricsLimits) (time.Time, time.Time, *HTTPError) { + end := now + if v := in.Get("end"); v != "" { + t, herr := parseMetricsTime("end", v) + if herr != nil { + return time.Time{}, time.Time{}, herr + } + end = t + } + start := end.Add(-labelsDefaultWindow) + if v := in.Get("start"); v != "" { + t, herr := parseMetricsTime("start", v) + if herr != nil { + return time.Time{}, time.Time{}, herr + } + start = t + } + if start.After(end) { + return time.Time{}, time.Time{}, errorf(http.StatusBadRequest, "start is after end") + } + if end.Sub(start) > lim.MaxRange { + return time.Time{}, time.Time{}, errorf(http.StatusUnprocessableEntity, "range %s exceeds the limit of %s", end.Sub(start), lim.MaxRange) + } + return start, end, nil +} + +// parseStep refuses a step above the range limit: vmselect aligns the +// window outward to whole steps, and an implicit rollup window is one step, +// so a huge step reads far more than the range it was sent with. +func parseStep(v string, lim MetricsLimits) (time.Duration, *HTTPError) { + step, herr := parseMetricsDuration("step", v) + if herr != nil { + return 0, herr + } + if step > lim.MaxRange { + return 0, errorf(http.StatusUnprocessableEntity, "step %s exceeds the limit of %s", step, lim.MaxRange) + } + return step, nil +} + +// maxMetricsDuration is vmselect's own bound on step and timeout +// (maxDurationMsecs in lib/httputil); it also keeps the rounding below +// from overflowing. +const maxMetricsDuration = 100 * 365 * 24 * time.Hour + +// parseMetricsDuration takes Prometheus float seconds or a duration such as +// 1m30s. vmselect works in whole milliseconds and refuses anything shorter; +// the value is rounded up to the millisecond so the limits are checked +// against what vmselect will use, not against a fraction it drops. +func parseMetricsDuration(key, v string) (time.Duration, *HTTPError) { + d, err := parseDuration(v) + if err != nil || d < time.Millisecond || d > maxMetricsDuration { + return 0, errorf(http.StatusBadRequest, "invalid %s %q", key, v) + } + return (d + time.Millisecond - 1).Truncate(time.Millisecond), nil +} + +// parseMetricsTime takes the two forms the Prometheus API defines: float +// unix seconds and RFC3339. vmselect would also guess milliseconds from the +// digit count and accept relative times; a guessed unit silently moves the +// window, so those are refused. The result is cut to the millisecond, as +// formatTime sends it, so the limits see the window vmselect will use. +func parseMetricsTime(key, v string) (time.Time, *HTTPError) { + if sec, frac, ok := strings.Cut(v, "."); isDigits(sec) && (!ok || isDigits(frac)) { + if len(sec) > 10 { + return time.Time{}, errorf(http.StatusBadRequest, "invalid %s %q: not unix seconds", key, v) + } + secs, _ := strconv.ParseInt(sec, 10, 64) + var nanos int64 + if ok { + nanos, _ = strconv.ParseInt((frac + "000000000")[:9], 10, 64) + } + return metricsTimeInRange(key, v, time.Unix(secs, nanos)) + } + t, err := time.Parse(time.RFC3339Nano, v) + if err != nil { + return time.Time{}, errorf(http.StatusBadRequest, "invalid %s %q", key, v) + } + return metricsTimeInRange(key, v, t) +} + +// maxMetricsTime is the last instant vmselect can read: it takes an +// RFC3339 time through time.UnixNano (timeutil.ParseTimeAt). +var maxMetricsTime = time.Unix(0, math.MaxInt64) + +func metricsTimeInRange(key, v string, t time.Time) (time.Time, *HTTPError) { + if t.Before(time.Unix(0, 0)) || !t.Before(maxMetricsTime) { + return time.Time{}, errorf(http.StatusBadRequest, "invalid %s %q: outside 1970..2262", key, v) + } + return time.UnixMilli(t.UnixMilli()).UTC(), nil +} + +// formatTime writes RFC3339 with milliseconds. vmselect reads a number +// without "-" as a unix time whose unit it guesses from its size +// (timeutil.getUnixTimestampNanoseconds): "1000000.000" is taken as +// 1000000000 seconds, and anything past 2038 in milliseconds as +// microseconds. A dated form never reaches that guess. +func formatTime(t time.Time) string { + return t.UTC().Format("2006-01-02T15:04:05.000Z07:00") +} + +// formatSeconds writes a whole-millisecond duration so that vmselect reads +// back exactly those milliseconds. vmselect multiplies the float seconds +// by 1e3 and truncates (httputil.GetDuration), so "1.001" arrives as +// 1000ms; where the shortest form truncates low, half a millisecond more +// lands the truncation on the intended value without reaching the next. +func formatSeconds(d time.Duration) string { + ms := d.Milliseconds() + s := strconv.FormatFloat(float64(ms)/1000, 'f', -1, 64) + if secs, _ := strconv.ParseFloat(s, 64); int64(secs*1e3) != ms { + s = fmt.Sprintf("%d.%03d5", ms/1000, ms%1000) + } + return s +} diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go new file mode 100644 index 000000000..a4a435567 --- /dev/null +++ b/internal/observabilityproxy/metricsparams_test.go @@ -0,0 +1,431 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "net/http" + "net/url" + "strconv" + "strings" + "testing" + "time" +) + +func testMetricsLimits() MetricsLimits { + return MetricsLimits{MaxRange: 30 * 24 * time.Hour, MaxPoints: 11000, MaxTimeout: 30 * time.Second} +} + +const wantMetricsScope = `{namespace="tenant-a"}` + +func unix(t time.Time) string { return strconv.FormatInt(t.Unix(), 10) } + +func buildMetrics(t *testing.T, endpoint string, in url.Values) metricsRequest { + t.Helper() + req, err := BuildMetricsParams(endpoint, in, "tenant-a", testNow, testMetricsLimits()) + if err != nil { + t.Fatalf("BuildMetricsParams(%s, %v): %v", endpoint, in, err) + } + return req +} + +func metricsStatus(endpoint string, in url.Values) int { + _, err := BuildMetricsParams(endpoint, in, "tenant-a", testNow, testMetricsLimits()) + if err == nil { + return http.StatusOK + } + return err.Code +} + +func TestBuildMetricsParamsAllowlist(t *testing.T) { + hostile := url.Values{ + "extra_filters": {`{namespace=~".+"}`}, + "extra_filters[]": {`{namespace="tenant-b"}`, `{namespace=~".*"}`}, + "extra_label": {"namespace=tenant-b"}, + "match": {`{namespace="tenant-b"}`}, + "nocache": {"1"}, + "limit": {"1"}, + "AccountID": {"7"}, + "latency_offset": {"1h"}, + "round_digits": {"2"}, + } + cases := map[string]url.Values{ + "api/v1/query": {"query": {"up"}, "time": {unix(testNow)}, "step": {"30s"}}, + "api/v1/query_range": {"query": {"up"}, "start": {unix(testNow.Add(-time.Hour))}, "end": {unix(testNow)}, "step": {"60"}}, + "api/v1/series": {"match[]": {"up"}}, + "api/v1/labels": {"match[]": {"up"}}, + "api/v1/label/pod/values": {"match[]": {"up"}}, + } + allowed := map[string]bool{"query": true, "start": true, "end": true, "step": true, "time": true, "match[]": true, "timeout": true, "extra_filters[]": true} + for endpoint, in := range cases { + t.Run(endpoint, func(t *testing.T) { + for k, v := range hostile { + in[k] = v + } + req := buildMetrics(t, endpoint, in) + if req.path != endpoint { + t.Errorf("path = %q", req.path) + } + for k := range req.params { + if !allowed[k] { + t.Errorf("parameter %q forwarded: %v", k, req.params) + } + } + if got := req.params["extra_filters[]"]; len(got) != 1 || got[0] != wantMetricsScope { + t.Errorf("extra_filters[] = %q, want exactly [%s]", got, wantMetricsScope) + } + for _, m := range req.params["match[]"] { + if strings.Contains(m, "tenant-b") { + t.Errorf("client match without brackets forwarded as match[]: %q", m) + } + } + }) + } +} + +func TestNamespaceSelectorEscapes(t *testing.T) { + for _, ns := range []string{"tenant-a", `a"} or {namespace=~".+`, `back\slash`, "new\nline"} { + sel := namespaceSelector(ns) + lit, ok := strings.CutPrefix(sel, "{namespace=") + lit, ok2 := strings.CutSuffix(lit, "}") + if !ok || !ok2 { + t.Fatalf("selector %q has the wrong shape", sel) + } + // MetricsQL reads a double-quoted label value with strconv.Unquote + // (metricsql extractStringValue). + got, err := strconv.Unquote(lit) + if err != nil || got != ns { + t.Errorf("selector %q decodes to %q, %v; want %q", sel, got, err, ns) + } + } +} + +func TestBuildMetricsParamsLabelEndpointsAlwaysMatch(t *testing.T) { + for _, endpoint := range []string{"api/v1/labels", "api/v1/label/__name__/values"} { + req := buildMetrics(t, endpoint, url.Values{}) + if got := req.params["match[]"]; len(got) != 1 || got[0] != wantMetricsScope { + t.Errorf("%s without match[] sends %q, want the namespace selector", endpoint, got) + } + req = buildMetrics(t, endpoint, url.Values{"match[]": {"up", `{job="x"}`}}) + if got := req.params["match[]"]; len(got) != 2 || got[0] != "up" || got[1] != `{job="x"}` { + t.Errorf("%s client match[] = %q, want them kept", endpoint, got) + } + } + if got := metricsStatus("api/v1/series", url.Values{}); got != http.StatusBadRequest { + t.Errorf("series without match[] = %d, want 400", got) + } + if got := metricsStatus("api/v1/series", url.Values{"match[]": {""}}); got != http.StatusBadRequest { + t.Errorf("empty match[] = %d, want 400", got) + } + many := url.Values{} + for i := range maxMetricsMatchers + 1 { + many.Add("match[]", "up"+strconv.Itoa(i)) + } + if got := metricsStatus("api/v1/labels", many); got != http.StatusUnprocessableEntity { + t.Errorf("%d match[] = %d, want 422", maxMetricsMatchers+1, got) + } +} + +func TestBuildMetricsParamsLimits(t *testing.T) { + ago := func(d time.Duration) string { return unix(testNow.Add(-d)) } + day := 24 * time.Hour + rng := func(start, end, step string) url.Values { + return url.Values{"query": {"up"}, "start": {start}, "end": {end}, "step": {step}} + } + tests := []struct { + name string + endpoint string + in url.Values + want int + }{ + {"range at the limit", "api/v1/query_range", rng(ago(30*day), ago(0), "1h"), http.StatusOK}, + {"range above the limit", "api/v1/query_range", rng(ago(31*day), ago(0), "1h"), http.StatusUnprocessableEntity}, + {"points at the limit", "api/v1/query_range", rng(ago(10999*time.Minute), ago(0), "1m"), http.StatusOK}, + {"points above the limit", "api/v1/query_range", rng(ago(11000*time.Minute), ago(0), "1m"), http.StatusUnprocessableEntity}, + {"points above the limit after rounding the step up", "api/v1/query_range", rng(ago(22*time.Second), ago(0), "0.0019"), http.StatusUnprocessableEntity}, + // 10.9999s admits 11000 points, but vmselect gets both bounds cut to + // the millisecond: 11.000s, 11001 points. + {"points above the limit once the bounds are cut to milliseconds", "api/v1/query_range", rng("2026-11-04T11:59:49.0009Z", "2026-11-04T12:00:00.0008Z", "0.001"), http.StatusUnprocessableEntity}, + {"overflowing step", "api/v1/query_range", rng(ago(time.Hour), ago(0), "2562047h47m16.854775807s"), http.StatusBadRequest}, + {"overflowing timeout", "api/v1/query", url.Values{"query": {"up"}, "timeout": {"2562047h47m16.854775807s"}}, http.StatusBadRequest}, + {"step above the range limit", "api/v1/query_range", rng(ago(time.Hour), ago(0), "31d"), http.StatusUnprocessableEntity}, + {"zero step", "api/v1/query_range", rng(ago(time.Hour), ago(0), "0"), http.StatusBadRequest}, + {"sub-millisecond step", "api/v1/query_range", rng(ago(time.Hour), ago(0), "0.0001"), http.StatusBadRequest}, + {"missing step", "api/v1/query_range", url.Values{"query": {"up"}, "start": {ago(time.Hour)}, "end": {ago(0)}}, http.StatusBadRequest}, + {"missing start", "api/v1/query_range", url.Values{"query": {"up"}, "end": {ago(0)}, "step": {"1m"}}, http.StatusBadRequest}, + {"missing query", "api/v1/query_range", url.Values{"start": {ago(time.Hour)}, "end": {ago(0)}, "step": {"1m"}}, http.StatusBadRequest}, + {"inverted range", "api/v1/query_range", rng(ago(0), ago(time.Hour), "1m"), http.StatusBadRequest}, + {"millisecond timestamp", "api/v1/query_range", rng(ago(time.Hour)+"000", ago(0)+"000", "1m"), http.StatusBadRequest}, + {"relative time", "api/v1/query_range", rng("-1h", "now", "1m"), http.StatusBadRequest}, + {"rfc3339", "api/v1/query_range", rng("2026-11-04T11:00:00Z", "2026-11-04T12:00:00.5Z", "15s"), http.StatusOK}, + {"instant without time", "api/v1/query", url.Values{"query": {"up"}}, http.StatusOK}, + {"instant step above the range limit", "api/v1/query", url.Values{"query": {"up"}, "step": {"31d"}}, http.StatusUnprocessableEntity}, + {"instant bad time", "api/v1/query", url.Values{"query": {"up"}, "time": {"yesterday"}}, http.StatusBadRequest}, + {"negative timeout", "api/v1/query", url.Values{"query": {"up"}, "timeout": {"-1s"}}, http.StatusBadRequest}, + {"series range above the limit", "api/v1/series", url.Values{"match[]": {"up"}, "start": {ago(31 * day)}}, http.StatusUnprocessableEntity}, + {"labels inverted range", "api/v1/labels", url.Values{"start": {ago(0)}, "end": {ago(time.Hour)}}, http.StatusBadRequest}, + {"invalid label name", "api/v1/label/a-b/values", url.Values{}, http.StatusBadRequest}, + {"unknown endpoint", "api/v1/status/tsdb", url.Values{}, http.StatusNotFound}, + {"export", "api/v1/export", url.Values{"match[]": {"up"}}, http.StatusNotFound}, + {"label without values", "api/v1/label/pod", url.Values{}, http.StatusNotFound}, + {"root", "", url.Values{}, http.StatusNotFound}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := metricsStatus(tt.endpoint, tt.in); got != tt.want { + t.Fatalf("status = %d, want %d", got, tt.want) + } + }) + } +} + +func TestBuildMetricsParamsCanonicalForms(t *testing.T) { + req := buildMetrics(t, "api/v1/query_range", url.Values{ + "query": {"up"}, "start": {"2026-11-04T11:00:00.25Z"}, "end": {unix(testNow)}, "step": {"1m30s"}, + }) + want := url.Values{ + "query": {"up"}, "start": {"2026-11-04T11:00:00.250Z"}, "end": {"2026-11-04T12:00:00.000Z"}, "step": {"90"}, + "timeout": {"30"}, "extra_filters[]": {wantMetricsScope}, + } + if req.params.Encode() != want.Encode() { + t.Errorf("params = %v\nwant %v", req.params, want) + } + + // vmselect drops a sub-millisecond remainder; the points check has to + // see the step it will use. + for in, want := range map[string]string{"60.0001": "60.001", "0.0019": "0.002"} { + req = buildMetrics(t, "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-20 * time.Second))}, "end": {unix(testNow)}, "step": {in}}) + if got := req.params.Get("step"); got != want { + t.Errorf("step %s forwarded as %q, want %s", in, got, want) + } + } + + for in, want := range map[string]string{"": "30", "10s": "10", "5m": "30", "1.5": "1.5"} { + v := url.Values{"query": {"up"}} + if in != "" { + v.Set("timeout", in) + } + if got := buildMetrics(t, "api/v1/query", v).params.Get("timeout"); got != want { + t.Errorf("timeout %q forwarded as %q, want %q", in, got, want) + } + } + + req = buildMetrics(t, "api/v1/labels", url.Values{}) + if req.params.Get("end") != "2026-11-04T12:00:00.000Z" || req.params.Get("start") != "2026-11-04T11:55:00.000Z" { + t.Errorf("labels window = %s..%s, want the last 5m", req.params.Get("start"), req.params.Get("end")) + } + if q := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}}); q.params.Has("time") || !q.start.Equal(testNow.Add(-15*time.Minute-30*time.Second)) { + t.Errorf("instant query without time: params %v, start %v", q.params, q.start) + } +} + +func TestMetricsStoreByRange(t *testing.T) { + cfg := MetricsConfig{ShortTermStore: "shortterm", LongTermStore: "longterm", ShortTermRetention: 72 * time.Hour} + tests := []struct { + name string + endpoint string + in url.Values + want string + }{ + {"instant now", "api/v1/query", url.Values{"query": {"up"}}, "shortterm"}, + {"instant within retention", "api/v1/query", url.Values{"query": {"up"}, "time": {unix(testNow.Add(-71 * time.Hour))}}, "shortterm"}, + {"instant past retention", "api/v1/query", url.Values{"query": {"up"}, "time": {unix(testNow.Add(-73 * time.Hour))}}, "longterm"}, + {"live graph", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-6 * time.Hour))}, "end": {unix(testNow)}, "step": {"60"}}, "shortterm"}, + {"range fits with lookback", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-71 * time.Hour))}, "end": {unix(testNow)}, "step": {"600"}}, "shortterm"}, + {"range lookback past retention", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-72 * time.Hour))}, "end": {unix(testNow)}, "step": {"600"}}, "longterm"}, + {"range past retention", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-7 * 24 * time.Hour))}, "end": {unix(testNow)}, "step": {"600"}}, "longterm"}, + {"old window ending long ago", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-5 * 24 * time.Hour))}, "end": {unix(testNow.Add(-4 * 24 * time.Hour))}, "step": {"600"}}, "longterm"}, + {"labels default window", "api/v1/labels", url.Values{}, "shortterm"}, + {"series past retention", "api/v1/series", url.Values{"match[]": {"up"}, "start": {unix(testNow.Add(-4 * 24 * time.Hour))}}, "longterm"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := buildMetrics(t, tt.endpoint, tt.in) + if got := cfg.store(req.start, testNow); got != tt.want { + t.Fatalf("store = %s, want %s", got, tt.want) + } + }) + } +} + +func TestQueryLookback(t *testing.T) { + tests := []struct { + query string + step time.Duration + want time.Duration + }{ + {`up`, 5 * time.Minute, 15 * time.Minute}, + {`increase(x[7d])`, 5 * time.Minute, 7*24*time.Hour + 35*time.Minute}, + {`rate(x[300])`, time.Minute, 20 * time.Minute}, + {`rate(x[ 1h30m ])`, time.Minute, 105 * time.Minute}, + {`max_over_time(x[1h:5m])`, time.Minute, 115 * time.Minute}, + {`max_over_time(x[1h:])`, time.Minute, 83 * time.Minute}, + {`max_over_time(max_over_time(rate(x[5m])[1h:1m])[1d:1h])`, time.Minute, 42*time.Hour + 35*time.Minute}, + {`rate(rate(x))`, time.Hour, 2*(3*time.Hour+5*time.Minute) + 2*time.Hour + 5*time.Minute}, + {`x offset 10d`, 5 * time.Minute, 10*24*time.Hour + 15*time.Minute}, + {`sum(x OFFSET 2h)`, 5 * time.Minute, 155 * time.Minute}, + {`rate(x[5m] offset 1h)`, time.Minute, 80 * time.Minute}, + {`x offset -1h`, 5 * time.Minute, 15 * time.Minute}, + {`x offset - 1h`, 5 * time.Minute, 15 * time.Minute}, + {`node_offset_seconds + offset_total + with_x`, 5 * time.Minute, 15 * time.Minute}, + {`x{a="[1h]"}`, 5 * time.Minute, 75 * time.Minute}, + {`x{a="]"}[7d]`, 5 * time.Minute, 7*24*time.Hour + 15*time.Minute}, + } + for _, tt := range tests { + t.Run(tt.query, func(t *testing.T) { + got, ok := queryLookback(tt.query, tt.step) + if !ok || got != tt.want { + t.Fatalf("queryLookback = %s, %t; want %s, true", got, ok, tt.want) + } + }) + } +} + +func TestQueryLookbackUnknown(t *testing.T) { + for _, q := range []string{ + `x @ 123`, + `x @ start()`, + `x{a="a@b"}`, + `rate(x[5i])`, + `rate(x[$__interval])`, + `x[1h-5m]`, + `x[7D]`, + `x[1.d]`, + `x[]`, + `rate(x[7d)`, + `x offset`, + `x offset w`, + `WITH (w = 7d) rate(x[w])`, + `x{a="["}`, + `x{a="[5m"}[7d]`, + `x{path="/offset/1"}`, + `x[99999999999y]`, + `x offset 99999999999y`, + `x offset 3_000_000`, + `WITH (f(q) = max_over_time(q[2d:1h])) f(f(x))`, + `with (s(m) = m offset 2d) s(s(x))`, + } { + t.Run(q, func(t *testing.T) { + if got, ok := queryLookback(q, 5*time.Minute); ok { + t.Fatalf("queryLookback = %s, true; want unknown", got) + } + }) + } +} + +func TestMetricsStoreByLookback(t *testing.T) { + cfg := MetricsConfig{ShortTermStore: "shortterm", LongTermStore: "longterm", ShortTermRetention: 72 * time.Hour} + rangeFrom := func(q string, ago time.Duration, step string) url.Values { + return url.Values{"query": {q}, "start": {unix(testNow.Add(-ago))}, "end": {unix(testNow)}, "step": {step}} + } + tests := []struct { + name string + endpoint string + in url.Values + want string + }{ + {"instant short window", "api/v1/query", url.Values{"query": {"rate(x[5m])"}}, "shortterm"}, + {"instant 7d window", "api/v1/query", url.Values{"query": {"increase(x[7d])"}}, "longterm"}, + {"instant offset 10d", "api/v1/query", url.Values{"query": {"x offset 10d"}}, "longterm"}, + {"instant negative offset", "api/v1/query", url.Values{"query": {"x offset -1h"}}, "shortterm"}, + {"instant at timestamp", "api/v1/query", url.Values{"query": {"x @ 123"}}, "longterm"}, + {"instant at start", "api/v1/query", url.Values{"query": {"x @ start()"}}, "longterm"}, + {"instant step window", "api/v1/query", url.Values{"query": {"rate(x[5i])"}}, "longterm"}, + {"instant step past retention", "api/v1/query", url.Values{"query": {"up"}, "step": {"259200"}}, "longterm"}, + {"instant latency offset at the edge", "api/v1/query", url.Values{"query": {"x[71h54m59s]"}, "step": {"0.001"}}, "longterm"}, + {"instant subquery past retention", "api/v1/query", url.Values{"query": {"max_over_time(rate(x[5m])[4d:1h])"}}, "longterm"}, + {"instant offset with underscores", "api/v1/query", url.Values{"query": {"x offset 3_000_000"}}, "longterm"}, + {"instant WITH template", "api/v1/query", url.Values{"query": {"WITH (f(q) = max_over_time(q[2d:1h])) f(f(x))"}}, "longterm"}, + {"instant implicit subqueries", "api/v1/query", url.Values{"query": {"rate(rate(rate(rate(rate(rate(x))))))"}, "step": {"43200"}}, "longterm"}, + {"quoted at sign", "api/v1/query", url.Values{"query": {`x{a="a@b"}`}}, "longterm"}, + {"quoted open bracket hides window", "api/v1/query", url.Values{"query": {`rate(x{a="[5m"}[7d])`}}, "longterm"}, + {"quoted close bracket before window", "api/v1/query", url.Values{"query": {`rate(x{a="]"}[7d])`}}, "longterm"}, + {"quoted offset before offset", "api/v1/query", url.Values{"query": {`x{a="offset 1m"} offset 10d`}}, "longterm"}, + {"range short window", "api/v1/query_range", rangeFrom("rate(x[1h])", 6*time.Hour, "60"), "shortterm"}, + {"range 7d window", "api/v1/query_range", rangeFrom("increase(x[7d])", 6*time.Hour, "60"), "longterm"}, + {"range offset past retention", "api/v1/query_range", rangeFrom("x offset 3d", time.Hour, "60"), "longterm"}, + {"range lookback fits", "api/v1/query_range", rangeFrom("rate(x[5m])", 70*time.Hour, "600"), "shortterm"}, + {"range lookback crosses retention", "api/v1/query_range", rangeFrom("rate(x[5m])", 71*time.Hour+50*time.Minute, "600"), "longterm"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := buildMetrics(t, tt.endpoint, tt.in) + if got := cfg.store(req.start, testNow); got != tt.want { + t.Fatalf("store = %s, want %s", got, tt.want) + } + if q := req.params.Get("query"); q != tt.in.Get("query") { + t.Fatalf("query forwarded as %q, want it unchanged", q) + } + }) + } +} + +// vmselectMillis repeats how vmselect reads a duration argument +// (httputil.GetDuration in lib/httputil/duration.go, v1.136.22 and +// v1.139.0): float seconds, multiplied by 1e3 and truncated. +func vmselectMillis(t *testing.T, v string) int64 { + t.Helper() + secs, err := strconv.ParseFloat(v, 64) + if err != nil { + t.Fatalf("vmselect cannot parse %q: %v", v, err) + } + return int64(secs * 1e3) +} + +// TestFormatSecondsVmselectExact sweeps every step up to 2000s and the +// large end of the range: vmselect must read back the milliseconds the +// limits were checked with. +func TestFormatSecondsVmselectExact(t *testing.T) { + check := func(ms int64) { + if got := vmselectMillis(t, formatSeconds(time.Duration(ms)*time.Millisecond)); got != ms { + t.Fatalf("%dms formatted as %q reaches vmselect as %dms", ms, formatSeconds(time.Duration(ms)*time.Millisecond), got) + } + } + for ms := int64(1); ms <= 2_000_000; ms++ { + check(ms) + } + for ms := maxMetricsDuration.Milliseconds() - 100_000; ms <= maxMetricsDuration.Milliseconds(); ms++ { + check(ms) + } + // A spread of the range in between, on a stride no power of ten divides. + for ms := int64(2_000_001); ms < maxMetricsDuration.Milliseconds(); ms += 3_153_599_999 { + check(ms) + } + // Whole seconds keep their plain form. + if got := formatSeconds(90 * time.Second); got != "90" { + t.Errorf("90s formatted as %q", got) + } +} + +// TestBuildMetricsParamsPointsAsVmselectCounts: a 1.001s step is 1001ms +// to the proxy; vmselect must count the same 11000 points at the limit. +func TestBuildMetricsParamsPointsAsVmselectCounts(t *testing.T) { + end := testNow + step := 1001 * time.Millisecond + rng := func(points int64) url.Values { + start := end.Add(-time.Duration(points-1) * step) + return url.Values{"query": {"up"}, "start": {start.Format(time.RFC3339Nano)}, "end": {end.Format(time.RFC3339Nano)}, "step": {"1.001"}} + } + req := buildMetrics(t, "api/v1/query_range", rng(11000)) + stepMs := vmselectMillis(t, req.params.Get("step")) + startMs, endMs := vmselectTimeMillis(t, req.params.Get("start")), vmselectTimeMillis(t, req.params.Get("end")) + if points := (endMs-startMs)/stepMs + 1; stepMs != 1001 || points != 11000 { + t.Fatalf("vmselect reads step %dms and %d points, want 1001ms and 11000", stepMs, points) + } + if got := metricsStatus("api/v1/query_range", rng(11001)); got != http.StatusUnprocessableEntity { + t.Fatalf("11001 points answered %d, want 422", got) + } +} diff --git a/internal/observabilityproxy/server.go b/internal/observabilityproxy/server.go index e980818d9..eb540a271 100644 --- a/internal/observabilityproxy/server.go +++ b/internal/observabilityproxy/server.go @@ -25,6 +25,7 @@ import ( "log/slog" "math" "net/http" + "net/url" "os" "strconv" "strings" @@ -56,6 +57,7 @@ type Config struct { // means the in-cluster Service of the "generic" logs storage. UpstreamURL func(store string) string Logs LogsLimits + Metrics MetricsConfig Limiter LimiterConfig RequestTimeout time.Duration TailIdleTimeout time.Duration @@ -84,6 +86,12 @@ func NewServer(cfg Config, auth *Authorizer, stores StoreLookup, reg prometheus. return fmt.Sprintf("http://vlselect-generic.%s.svc.%s:9471", store, domain) } } + if cfg.Metrics.UpstreamURL == nil { + domain, root := cfg.ClusterDomain, cfg.Metrics.RootTenant + cfg.Metrics.UpstreamURL = func(store string) string { + return fmt.Sprintf("http://vmselect-%s.%s.svc.%s:8481/select/0/prometheus", store, root, domain) + } + } s := &Server{ cfg: cfg, auth: auth, @@ -167,7 +175,7 @@ func writeError(w http.ResponseWriter, e *HTTPError) { // serve answers a root it has no handler for before authenticating, so an // unknown path costs no review calls. func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTTPError { - if route.Kind != NamespaceLogs { + if route.Kind != NamespaceLogs && route.Kind != NamespaceMetrics { return errorf(http.StatusNotFound, "not found") } user, herr := s.authorize(r, route) @@ -182,6 +190,9 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT w.Header().Set("Retry-After", strconv.FormatInt(int64(math.Ceil(wait.Seconds())), 10)) return errorf(http.StatusTooManyRequests, "request rate limit exceeded") } + if route.Kind == NamespaceMetrics { + return s.serveMetrics(w, r, route, user) + } endpoint, ok := strings.CutPrefix(route.Endpoint, "select/logsql/") if !ok { return errorf(http.StatusNotFound, "not found") @@ -226,29 +237,17 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT ctx, stop = context.WithTimeout(ctx, limit) defer stop() } - req, err := http.NewRequestWithContext(ctx, http.MethodPost, - s.cfg.UpstreamURL(store)+"/select/logsql/"+endpoint, strings.NewReader(params.Encode())) - if err != nil { - return errorf(http.StatusInternalServerError, "building upstream request: %v", err) - } - req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - req.Header.Set("AccountID", "0") - req.Header.Set("ProjectID", "0") + tenant := http.Header{} + tenant.Set("AccountID", "0") + tenant.Set("ProjectID", "0") if tail { // vlselect holds even the headers of a gzip-encoded tail until the // first line; an explicit value stops the transport from asking for gzip. - req.Header.Set("Accept-Encoding", "identity") + tenant.Set("Accept-Encoding", "identity") } - resp, err := s.client.Do(req) - switch { - case err == nil: - case r.Context().Err() != nil: - return errorf(http.StatusBadGateway, "client went away") - case errors.Is(context.Cause(ctx), context.DeadlineExceeded): - return errorf(http.StatusGatewayTimeout, "log store did not answer within %s", limit) - default: - slog.Warn("log store unreachable", "store", store, "error", err) - return errorf(http.StatusBadGateway, "log store unreachable") + resp, herr := s.post(ctx, r, s.cfg.UpstreamURL(store)+"/select/logsql/"+endpoint, params, "log store", tenant) + if herr != nil { + return herr } defer func() { _ = resp.Body.Close() }() @@ -280,6 +279,75 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT return nil } +func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Route, user string) *HTTPError { + release, ok := s.limits.acquire(user) + if !ok { + return tooManyConcurrent(w) + } + defer release() + if herr := s.readForm(w, r); herr != nil { + return herr + } + now := s.now() + req, herr := BuildMetricsParams(route.Endpoint, r.Form, route.Namespace, now, s.cfg.Metrics.Limits) + if herr != nil { + return herr + } + + ctx := r.Context() + if s.cfg.RequestTimeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, s.cfg.RequestTimeout) + defer cancel() + } + store := s.cfg.Metrics.store(req.start, now) + resp, herr := s.post(ctx, r, s.cfg.Metrics.UpstreamURL(store)+"/"+req.path, req.params, "metrics store", nil) + if herr != nil { + return herr + } + defer func() { _ = resp.Body.Close() }() + if ct := resp.Header.Get("Content-Type"); ct != "" { + w.Header().Set("Content-Type", ct) + } + // As for a bounded logs answer: the client must take it by the request + // deadline, and a break-off cuts the connection. + if deadline, ok := ctx.Deadline(); ok { + _ = http.NewResponseController(w).SetWriteDeadline(deadline) + } + w.WriteHeader(resp.StatusCode) + if _, err := io.Copy(w, resp.Body); err != nil { + slog.Debug("copying upstream response", "error", err) + return errAborted + } + return nil +} + +// post sends params as a form with only the given headers, so nothing the +// client sent reaches the store, and tells an unreachable store (502) from +// one that missed its deadline or, for a tail, its idle timeout (504). +func (s *Server) post(ctx context.Context, r *http.Request, target string, params url.Values, what string, header http.Header) (*http.Response, *HTTPError) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, target, strings.NewReader(params.Encode())) + if err != nil { + return nil, errorf(http.StatusInternalServerError, "building upstream request: %v", err) + } + for k, v := range header { + req.Header[k] = v + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + resp, err := s.client.Do(req) + switch { + case err == nil: + return resp, nil + case r.Context().Err() != nil: + return nil, errorf(http.StatusBadGateway, "client went away") + case errors.Is(context.Cause(ctx), context.DeadlineExceeded): + return nil, errorf(http.StatusGatewayTimeout, "%s did not answer in time", what) + default: + slog.Warn(what+" unreachable", "url", target, "error", err) + return nil, errorf(http.StatusBadGateway, "%s unreachable", what) + } +} + // concurrencyRetryAfter is a hint only: a slot frees when another request // of the user ends, which the proxy cannot predict, and one second keeps // a polling client from retrying in a tight loop. diff --git a/internal/observabilityproxy/server_deadline_test.go b/internal/observabilityproxy/server_deadline_test.go index 7415b46fa..6b367229c 100644 --- a/internal/observabilityproxy/server_deadline_test.go +++ b/internal/observabilityproxy/server_deadline_test.go @@ -421,15 +421,20 @@ func stalledAPIServer(t *testing.T, stallTokens bool) kubernetes.Interface { // a bounded review call ends the request. func TestServerStalledReviewAnswersInTime(t *testing.T) { for _, tc := range []struct { - name string + name, path string stallTokens bool - }{{"token review", true}, {"subject access review", false}} { + }{ + {"logs token review", logsRoot + "query", true}, + {"logs subject access review", logsRoot + "query", false}, + {"metrics token review", metricsRoot + "api/v1/query", true}, + {"metrics subject access review", metricsRoot + "api/v1/query", false}, + } { t.Run(tc.name, func(t *testing.T) { h := newHarness(t, Config{RequestTimeout: 200 * time.Millisecond}, nil) h.server.auth = NewAuthorizer(stalledAPIServer(t, tc.stallTokens), 10*time.Second) got := make(chan int, 1) go func() { - resp := h.do(http.MethodPost, logsRoot+"query", "good", url.Values{"query": {"*"}}, nil) + resp := h.do(http.MethodPost, tc.path, "good", url.Values{"query": {"up"}}, nil) _ = resp.Body.Close() got <- resp.StatusCode }() @@ -444,3 +449,56 @@ func TestServerStalledReviewAnswersInTime(t *testing.T) { }) } } + +// TestServerMetricsRequestLifetime repeats the body, write and break-off +// checks on the metrics root, which has its own handler. +func TestServerMetricsRequestLifetime(t *testing.T) { + query := metricsRoot + "api/v1/query?query=up" + t.Run("slow body", func(t *testing.T) { + h := newHarness(t, Config{ + RequestTimeout: 200 * time.Millisecond, + Limiter: LimiterConfig{MaxConcurrentPerUser: 1}, + }, promUpstream) + pipe := servePipe(t, h.server) + conn := pipe.send(t, "POST "+metricsRoot+"api/v1/query HTTP/1.1\r\nHost: proxy\r\nAuthorization: Bearer good\r\n"+ + "Content-Type: application/x-www-form-urlencoded\r\nContent-Length: 100\r\n\r\nquery=") + eventually(t, time.Second, func() bool { return slotsHeld(h) == 1 }, + "a metrics request with an unfinished body does not hold a concurrency slot") + if resp := readResponse(t, conn); resp.StatusCode != http.StatusRequestTimeout { + t.Fatalf("unfinished body answered %d, want 408", resp.StatusCode) + } + eventually(t, time.Second, func() bool { return slotsHeld(h) == 0 }, "the slot of an unfinished body was not released") + }) + t.Run("stalled client", func(t *testing.T) { + // Larger than the server's write buffer, so the handler itself + // blocks on the client. + big := `{"status":"success","data":"` + strings.Repeat("x", 64<<10) + `"}` + h := newHarness(t, Config{ + RequestTimeout: 200 * time.Millisecond, + Limiter: LimiterConfig{MaxConcurrentPerUser: 1}, + }, func(w http.ResponseWriter, _ *http.Request) { _, _ = io.WriteString(w, big) }) + pipe := servePipe(t, h.server) + pipe.send(t, "GET "+query+" HTTP/1.1\r\nHost: proxy\r\nAuthorization: Bearer good\r\n\r\n") + eventually(t, time.Second, func() bool { return slotsHeld(h) == 1 }, "the stalled metrics request never held its slot") + eventually(t, 2*time.Second, func() bool { return slotsHeld(h) == 0 }, + "a metrics client that stopped reading keeps its concurrency slot") + }) + t.Run("truncated answer", func(t *testing.T) { + h := newHarness(t, Config{}, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Length", "100") + _, _ = io.WriteString(w, `{"status":"success",`) + }) + // Nothing has been flushed yet when the copy fails, so the + // connection may close before the status line. + req, _ := http.NewRequest(http.MethodGet, h.proxy.URL+query, nil) + req.Header.Set("Authorization", "Bearer good") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return + } + defer func() { _ = resp.Body.Close() }() + if _, err := io.ReadAll(resp.Body); err == nil { + t.Fatal("a truncated store answer reached the client as a complete response") + } + }) +} diff --git a/internal/observabilityproxy/server_metrics_test.go b/internal/observabilityproxy/server_metrics_test.go new file mode 100644 index 000000000..1ef5e761e --- /dev/null +++ b/internal/observabilityproxy/server_metrics_test.go @@ -0,0 +1,210 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "io" + "net/http" + "net/url" + "sync" + "testing" + "time" + + authorizationv1 "k8s.io/api/authorization/v1" +) + +const metricsRoot = "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/" + +func promUpstream(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`) +} + +func TestServerMetricsRebuildsUpstreamRequest(t *testing.T) { + h := newHarness(t, Config{}, promUpstream) + hostile := url.Values{ + "query": {"sum(rate(container_cpu_usage_seconds_total[5m]))"}, + "start": {unix(testNow.Add(-time.Hour))}, + "end": {unix(testNow)}, + "step": {"60"}, + "extra_filters": {`{namespace=~".+"}`}, + "extra_filters[]": {`{namespace="tenant-b"}`, `{namespace=~".*"}`}, + "extra_label": {"namespace=tenant-b"}, + "nocache": {"1"}, + } + headers := map[string]string{"AccountID": "9", "X-Forwarded-For": "1.2.3.4", "Cookie": "kc-access=secret"} + + resp := h.do(http.MethodGet, metricsRoot+"api/v1/query_range?"+hostile.Encode(), "good", nil, headers) + _ = resp.Body.Close() + resp = h.do(http.MethodPost, metricsRoot+"api/v1/query_range?extra_filters[]="+url.QueryEscape(`{namespace="x"}`), "good", hostile, headers) + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK || resp.Header.Get("Content-Type") != "application/json" { + t.Fatalf("POST answered %d %s %q", resp.StatusCode, resp.Header.Get("Content-Type"), body) + } + + reqs := h.requests() + if len(reqs) != 2 { + t.Fatalf("upstream saw %d requests", len(reqs)) + } + if reqs[0].form.Encode() != reqs[1].form.Encode() { + t.Fatalf("GET and POST built different upstream requests:\n%v\n%v", reqs[0].form, reqs[1].form) + } + if n := h.kube.sars.Load(); n != 1 { + t.Fatalf("SubjectAccessReviews = %d, want 1: POST must reuse the GET decision", n) + } + want := authorizationv1.ResourceAttributes{Verb: "list", Group: "cozystack.io", Resource: "metrics", Namespace: "tenant-a"} + if got := h.kube.lastSAR.Load().ResourceAttributes; *got != want { + t.Fatalf("SubjectAccessReview = %+v, want %+v", *got, want) + } + for _, r := range reqs { + if r.path != "/api/v1/query_range" { + t.Errorf("upstream path = %s", r.path) + } + if got := r.form["extra_filters[]"]; len(got) != 1 || got[0] != wantMetricsScope { + t.Errorf("extra_filters[] = %q", got) + } + if r.form.Has("extra_filters") || r.form.Has("extra_label") || r.form.Has("nocache") { + t.Errorf("client parameter forwarded: %v", r.form) + } + for _, k := range []string{"Authorization", "AccountID", "X-Forwarded-For", "Cookie"} { + if r.headers.Get(k) != "" { + t.Errorf("client header %s forwarded", k) + } + } + } +} + +func TestServerMetricsStoreByRange(t *testing.T) { + h := newHarness(t, Config{}, promUpstream) + for _, path := range []string{ + "api/v1/query?query=up", + "api/v1/query_range?query=up&step=600&start=" + unix(testNow.Add(-7*24*time.Hour)) + "&end=" + unix(testNow), + "api/v1/label/pod/values", + } { + if got := h.status(http.MethodGet, metricsRoot+path, "good"); got != http.StatusOK { + t.Fatalf("%s answered %d", path, got) + } + } + h.mu.Lock() + defer h.mu.Unlock() + if want := []string{"shortterm", "longterm", "shortterm"}; len(h.stores) != 3 || h.stores[0] != want[0] || h.stores[1] != want[1] || h.stores[2] != want[2] { + t.Fatalf("stores = %v, want %v", h.stores, want) + } + if r := h.got[2]; r.path != "/api/v1/label/pod/values" || r.form.Get("match[]") != wantMetricsScope { + t.Fatalf("label values upstream = %s %v", r.path, r.form) + } +} + +func TestServerMetricsStatusCodes(t *testing.T) { + h := newHarness(t, Config{}, promUpstream) + tooLong := "api/v1/query_range?query=up&step=3600&start=" + unix(testNow.Add(-31*24*time.Hour)) + "&end=" + unix(testNow) + if got := h.status(http.MethodGet, metricsRoot+tooLong, "good"); got != http.StatusUnprocessableEntity { + t.Errorf("range above the limit answered %d, want 422", got) + } + if got := h.status(http.MethodGet, metricsRoot+"api/v1/series", "good"); got != http.StatusBadRequest { + t.Errorf("series without match[] answered %d, want 400", got) + } + if got := h.status(http.MethodPut, metricsRoot+"api/v1/query?query=up", "good"); got != http.StatusMethodNotAllowed { + t.Errorf("PUT answered %d, want 405", got) + } + if n := len(h.requests()); n != 0 { + t.Errorf("upstream saw %d requests for refused ones", n) + } + + h2 := newHarness(t, Config{}, func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, `{"status":"error","errorType":"422"}`, http.StatusUnprocessableEntity) + }) + if got := h2.status(http.MethodGet, metricsRoot+"api/v1/query?query=up", "good"); got != http.StatusUnprocessableEntity { + t.Errorf("upstream 422 answered as %d", got) + } + h2.upstream.Close() + if got := h2.status(http.MethodGet, metricsRoot+"api/v1/query?query=up", "good"); got != http.StatusBadGateway { + t.Errorf("unreachable upstream answered %d, want 502", got) + } + + release := make(chan struct{}) + t.Cleanup(func() { close(release) }) + h3 := newHarness(t, Config{RequestTimeout: 50 * time.Millisecond}, func(_ http.ResponseWriter, r *http.Request) { + select { + case <-release: + case <-r.Context().Done(): + } + }) + if got := h3.status(http.MethodGet, metricsRoot+"api/v1/query?query=up", "good"); got != http.StatusGatewayTimeout { + t.Errorf("slow upstream answered %d, want 504", got) + } + + h4 := newHarness(t, Config{Limiter: LimiterConfig{RatePerUser: 0.001, RateBurst: 1}}, promUpstream) + h4.status(http.MethodGet, metricsRoot+"api/v1/query?query=up", "good") + limited := h4.do(http.MethodGet, metricsRoot+"api/v1/query?query=up", "good", nil, nil) + _ = limited.Body.Close() + if limited.StatusCode != http.StatusTooManyRequests || limited.Header.Get("Retry-After") != "1000" { + t.Errorf("rate-limited request answered %d with Retry-After %q, want 429 with 1000", + limited.StatusCode, limited.Header.Get("Retry-After")) + } +} + +func TestServerMetricsConcurrency(t *testing.T) { + entered, release := make(chan struct{}, 3), make(chan struct{}) + h := newHarness(t, Config{Limiter: LimiterConfig{MaxConcurrentPerUser: 1}}, func(w http.ResponseWriter, _ *http.Request) { + entered <- struct{}{} + <-release + promUpstream(w, nil) + }) + // Registered after the harness so it runs before the servers close. + var once sync.Once + t.Cleanup(func() { once.Do(func() { close(release) }) }) + held := make(chan int, 2) + hold := func(token string) { + go func() { held <- h.status(http.MethodGet, metricsRoot+"api/v1/query?query=up", token) }() + select { + case <-entered: + case <-time.After(2 * time.Second): + t.Fatalf("request of %s did not reach the upstream", token) + } + } + hold("good") + refused := make(chan *http.Response, 1) + go func() { + resp := h.do(http.MethodPost, metricsRoot+"api/v1/query?query=up", "good", nil, nil) + _ = resp.Body.Close() + refused <- resp + }() + select { + case got := <-refused: + if got.StatusCode != http.StatusTooManyRequests || got.Header.Get("Retry-After") != "1" { + t.Fatalf("concurrent request answered %d with Retry-After %q, want 429 with 1", got.StatusCode, got.Header.Get("Retry-After")) + } + case <-time.After(2 * time.Second): + t.Fatal("concurrent request of the same user reached the upstream") + } + hold("bob-token") + once.Do(func() { close(release) }) + for range 2 { + if got := <-held; got != http.StatusOK { + t.Fatalf("held request answered %d", got) + } + } +} + +func TestDefaultMetricsUpstreamURL(t *testing.T) { + s := NewServer(Config{ClusterDomain: "cozy.local", Metrics: MetricsConfig{RootTenant: "tenant-root"}}, nil, nil, nil) + if got, want := s.cfg.Metrics.UpstreamURL("shortterm"), "http://vmselect-shortterm.tenant-root.svc.cozy.local:8481/select/0/prometheus"; got != want { + t.Fatalf("upstream = %s, want %s", got, want) + } +} diff --git a/internal/observabilityproxy/server_test.go b/internal/observabilityproxy/server_test.go index df3e23bc4..2670d0e3e 100644 --- a/internal/observabilityproxy/server_test.go +++ b/internal/observabilityproxy/server_test.go @@ -52,12 +52,12 @@ type mapStores map[string]string func (m mapStores) Resolve(ns string) (string, error) { return m[ns], nil } // newHarness wires the proxy to a fake apiserver that knows tokens "good" -// (alice) and "bob-token" (bob), both allowed list logs in tenant-a only, -// and to an upstream served by handler. +// (alice) and "bob-token" (bob), both allowed list logs and metrics in +// tenant-a only, and to an upstream served by handler. func newHarness(t *testing.T, cfg Config, handler http.HandlerFunc) *harness { h := &harness{t: t} h.kube = newFakeKube(map[string]string{"good": "alice", "bob-token": "bob"}, func(_ string, a *authorizationv1.ResourceAttributes) bool { - return a.Namespace == "tenant-a" && a.Resource == "logs" + return a.Namespace == "tenant-a" && (a.Resource == "logs" || a.Resource == "metrics") }) h.upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if err := r.ParseForm(); err != nil { @@ -80,6 +80,13 @@ func newHarness(t *testing.T, cfg Config, handler http.HandlerFunc) *harness { h.mu.Unlock() return h.upstream.URL } + cfg.Metrics.UpstreamURL = cfg.UpstreamURL + if cfg.Metrics.ShortTermStore == "" { + cfg.Metrics.ShortTermStore, cfg.Metrics.LongTermStore, cfg.Metrics.ShortTermRetention = "shortterm", "longterm", 72*time.Hour + } + if cfg.Metrics.Limits == (MetricsLimits{}) { + cfg.Metrics.Limits = testMetricsLimits() + } if cfg.Logs == (LogsLimits{}) { cfg.Logs = testLimits() } @@ -144,7 +151,9 @@ func TestServerStatusCodes(t *testing.T) { {"other namespace", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-b/logs/select/logsql/query", "good", http.StatusForbidden}, {"invalid namespace", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/Tenant/logs/select/logsql/query", "good", http.StatusBadRequest}, {"unknown root", http.MethodGet, "/api/v1/namespaces/tenant-a/pods", "good", http.StatusNotFound}, - {"namespace metrics not served yet", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/api/v1/query", "good", http.StatusNotFound}, + {"unknown metrics endpoint", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/api/v1/status/tsdb", "good", http.StatusNotFound}, + {"metrics in another namespace", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-b/metrics/api/v1/query?query=up", "good", http.StatusForbidden}, + {"metrics without token", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/api/v1/query?query=up", "", http.StatusUnauthorized}, {"application log not served yet", http.MethodGet, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/log/select/logsql/query", "good", http.StatusNotFound}, {"application metrics not served yet", http.MethodGet, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/metrics/api/v1/query", "good", http.StatusNotFound}, {"tenant_ids not proxied", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/logs/select/tenant_ids", "good", http.StatusNotFound}, diff --git a/internal/observabilityproxy/vmselect_parse_test.go b/internal/observabilityproxy/vmselect_parse_test.go new file mode 100644 index 000000000..1fb23d18f --- /dev/null +++ b/internal/observabilityproxy/vmselect_parse_test.go @@ -0,0 +1,119 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "math" + "net/url" + "strconv" + "strings" + "testing" + "time" +) + +// vmselectTimeMillis repeats how vmselect v1.136.0-cluster reads a time +// argument (httputil.GetTime -> timeutil.ParseTimeMsec -> ParseTimeAt in +// lib/timeutil/time.go) for the forms a client or the proxy may send: a +// value without "-" is a number whose unit vmselect guesses from its size +// (getUnixTimestampNanoseconds), anything else with a zone is RFC3339 read +// through time.UnixNano. Relative and partial dates are left out; the +// proxy refuses them. +func vmselectTimeMillis(t *testing.T, s string) int64 { + t.Helper() + if strings.Contains(s, "-") { + ts, err := time.Parse(time.RFC3339, s) + if err != nil { + t.Fatalf("vmselect cannot parse %q: %v", s, err) + } + return int64(math.Round(float64(ts.UnixNano()) / 1e6)) + } + intPart, frac, _ := strings.Cut(s, ".") + n, err := strconv.ParseInt(intPart+frac, 10, 64) + if err != nil { + t.Fatalf("vmselect cannot parse %q: %v", s, err) + } + for e := len(frac); e%3 != 0; e++ { + n *= 10 + } + var nanos int64 + switch { + case n < 1<<31: + nanos = n * 1e9 + case n < 1e3*(1<<31): + nanos = n * 1e6 + case n < 1e6*(1<<31): + nanos = n * 1e3 + default: + nanos = n + } + return int64(math.Round(float64(nanos) / 1e6)) +} + +// TestVmselectReadsProxyTimes sends times across the range the proxy +// accepts, in both client forms, and checks that vmselect reads the +// forwarded value as the same instant the proxy validated. A number such +// as "1000000.000" is seconds to the proxy but milliseconds-as-seconds +// guesswork to vmselect, so the proxy must not forward one. +func TestVmselectReadsProxyTimes(t *testing.T) { + for _, client := range []string{ + "0", "1000000", "1762400", "2147483", "2147484", "1000000.5", + "1793793600", "1793793600.123", "2147483647", "2147483648", "9223372035", + "1970-01-01T00:00:00Z", "1970-01-12T13:46:40.001Z", "2026-11-04T12:00:00.123Z", "2262-01-01T00:00:00Z", + } { + want, herr := parseMetricsTime("time", client) + if herr != nil { + t.Fatalf("%q refused: %v", client, herr) + } + req := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}, "time": {client}}) + sent := req.params.Get("time") + if got := vmselectTimeMillis(t, sent); got != want.UnixMilli() { + t.Errorf("client time %q forwarded as %q reaches vmselect as %dms, want %dms", client, sent, got, want.UnixMilli()) + } + } +} + +// TestVmselectReadsSecondsWithFraction is the window that passed the proxy as +// 8.8 days and 30 points while vmselect, reading "1000000.000" as a +// thousand times that many seconds, saw 2001..2025 and 30000 points. +func TestVmselectReadsSecondsWithFraction(t *testing.T) { + in := url.Values{"query": {"up"}, "start": {"1000000"}, "end": {"1762400"}, "step": {"25414"}} + req := buildMetrics(t, "api/v1/query_range", in) + start, end := vmselectTimeMillis(t, req.params.Get("start")), vmselectTimeMillis(t, req.params.Get("end")) + step := vmselectMillis(t, req.params.Get("step")) + if start != 1000000*1000 || end != 1762400*1000 { + t.Fatalf("vmselect reads %dms..%dms, want 1000000s..1762400s", start, end) + } + if points := (end-start)/step + 1; points != 30 { + t.Fatalf("vmselect counts %d points, want the 30 the proxy checked", points) + } + labels := buildMetrics(t, "api/v1/labels", url.Values{"start": {"1000000"}, "end": {"1000300"}}) + if got := vmselectTimeMillis(t, labels.params.Get("start")); got != 1000000*1000 { + t.Fatalf("label API start reaches vmselect as %dms, want 1000000s", got) + } +} + +func TestParseMetricsTimeRange(t *testing.T) { + // vmselect reads an RFC3339 time through time.UnixNano, which is + // undefined from 2262-04-11 on. + // Before 1970 vmselect clamps to 0, which would quietly answer for a + // window other than the one asked for. + for _, v := range []string{"9223372037", "9999999999", "2262-04-12T00:00:00Z", "9999-01-01T00:00:00Z", "1969-12-31T23:59:59Z", "1969-12-31T23:59:59.999Z"} { + if _, herr := parseMetricsTime("time", v); herr == nil { + t.Errorf("%q accepted, want refused", v) + } + } +} diff --git a/packages/system/observability-proxy/templates/deployment.yaml b/packages/system/observability-proxy/templates/deployment.yaml index 581c15fde..d49de140b 100644 --- a/packages/system/observability-proxy/templates/deployment.yaml +++ b/packages/system/observability-proxy/templates/deployment.yaml @@ -38,6 +38,9 @@ spec: - --cluster-domain={{ $clusterDomain }} - --root-tenant={{ .Values.observabilityProxy.rootTenant }} - --auth-cache-ttl={{ .Values.observabilityProxy.authCacheTTL }} + - --metrics-shortterm-store={{ .Values.observabilityProxy.metrics.shorttermStore }} + - --metrics-longterm-store={{ .Values.observabilityProxy.metrics.longtermStore }} + - --metrics-shortterm-retention={{ .Values.observabilityProxy.metrics.shorttermRetention }} ports: - name: http containerPort: 8080 diff --git a/packages/system/observability-proxy/templates/tenant-clusterroles.yaml b/packages/system/observability-proxy/templates/tenant-clusterroles.yaml index 23460352f..a2bf94f3c 100644 --- a/packages/system/observability-proxy/templates/tenant-clusterroles.yaml +++ b/packages/system/observability-proxy/templates/tenant-clusterroles.yaml @@ -1,7 +1,7 @@ # Aggregated into cozy-tenant-view (and consequently use, admin, super-admin). -# No API server serves cozystack.io logs: the rule exists only for the -# SubjectAccessReview the observability proxy asks before it answers a read -# under /apis/cozystack.io/v1alpha1/namespaces//logs/. +# No API server serves cozystack.io logs or metrics: the rule exists only for +# the SubjectAccessReview the observability proxy asks before it answers a +# read under /apis/cozystack.io/v1alpha1/namespaces//{logs,metrics}/. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -10,5 +10,5 @@ metadata: rbac.cozystack.io/aggregate-to-tenant-view: "true" rules: - apiGroups: ["cozystack.io"] - resources: ["logs"] + resources: ["logs", "metrics"] verbs: ["list"] diff --git a/packages/system/observability-proxy/tests/deployment_test.yaml b/packages/system/observability-proxy/tests/deployment_test.yaml index ddf9d34b8..a5e948920 100644 --- a/packages/system/observability-proxy/tests/deployment_test.yaml +++ b/packages/system/observability-proxy/tests/deployment_test.yaml @@ -40,6 +40,35 @@ tests: path: spec.template.spec.containers[0].args content: --auth-cache-ttl=10s + - it: passes the metrics store names and short-term retention + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --metrics-shortterm-store=shortterm + - contains: + path: spec.template.spec.containers[0].args + content: --metrics-longterm-store=longterm + - contains: + path: spec.template.spec.containers[0].args + content: --metrics-shortterm-retention=72h + + - it: follows renamed metrics stores + set: + observabilityProxy.metrics: + shorttermStore: fast + longtermStore: slow + shorttermRetention: 24h + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --metrics-shortterm-store=fast + - contains: + path: spec.template.spec.containers[0].args + content: --metrics-longterm-store=slow + - contains: + path: spec.template.spec.containers[0].args + content: --metrics-shortterm-retention=24h + - it: falls back to cozy.local when the platform passes no cluster domain asserts: - contains: diff --git a/packages/system/observability-proxy/tests/rbac_test.yaml b/packages/system/observability-proxy/tests/rbac_test.yaml index af339b5d0..ef96c65d7 100644 --- a/packages/system/observability-proxy/tests/rbac_test.yaml +++ b/packages/system/observability-proxy/tests/rbac_test.yaml @@ -2,7 +2,7 @@ suite: observability-proxy RBAC # The proxy authenticates and authorizes every request through the API server, # so a dropped verb here turns every request into an error at runtime rather # than a render failure. The tenant role is what lets a namespace viewer pass -# the SubjectAccessReview the proxy asks for the namespace logs root. +# the SubjectAccessReview the proxy asks for the namespace logs and metrics roots. templates: - templates/rbac.yaml - templates/tenant-clusterroles.yaml @@ -50,7 +50,7 @@ tests: name: observability-proxy namespace: cozy-observability-proxy - - it: aggregates list on cozystack.io logs into the tenant view role + - it: aggregates list on cozystack.io logs and metrics into the tenant view role template: templates/tenant-clusterroles.yaml asserts: - isKind: @@ -63,5 +63,5 @@ tests: path: rules value: - apiGroups: ["cozystack.io"] - resources: ["logs"] + resources: ["logs", "metrics"] verbs: ["list"] diff --git a/packages/system/observability-proxy/values.yaml b/packages/system/observability-proxy/values.yaml index 919e450a9..67334a237 100644 --- a/packages/system/observability-proxy/values.yaml +++ b/packages/system/observability-proxy/values.yaml @@ -7,6 +7,13 @@ observabilityProxy: replicas: 2 rootTenant: tenant-root authCacheTTL: 10s + # Names and retention of the root tenant's metricsStorages + # (packages/system/monitoring/values.yaml). The retention is a Go + # duration: 3d is written 72h. + metrics: + shorttermStore: shortterm + longtermStore: longterm + shorttermRetention: 72h resources: requests: cpu: 10m