feat(observability-proxy): serve the namespace metrics root

The namespace metrics root shares the logs root's authorization path:
the SubjectAccessReview comes from the path alone, so a POST query,
which Grafana sends by default, is authorized exactly like a GET. Both
roots also share the per-user rate and concurrency limits and the way
an unreachable store (502) is told apart from a slow one (504); the
upstream call moves into one helper so the two stay identical.

Every namespace resolves to the root tenant's vmselect for now,
because the series the graphs need are only scraped there.

Assisted-by: LLM
Signed-off-by: ZverGuy <maximbel2003@gmail.com>
This commit is contained in:
ZverGuy committed 2026-11-10 03:56:18 +07:00
1 parent 4f3260e5d4
commit 47409db70c
3 files changed
+259 -24

No files matched your search

+83 -20
View File
@@ -25,6 +25,7 @@ import (
"log/slog"
"math"
"net/http"
"net/url"
"os"
"strconv"
"strings"
@@ -56,6 +57,7 @@ type Config struct {
// means the in-cluster Service of the "generic" logs storage.
UpstreamURL func(store string) string
Logs LogsLimits
Metrics MetricsConfig
Limiter LimiterConfig
RequestTimeout time.Duration
TailIdleTimeout time.Duration
@@ -84,6 +86,12 @@ func NewServer(cfg Config, auth *Authorizer, stores StoreLookup, reg prometheus.
return fmt.Sprintf("http://vlselect-generic.%s.svc.%s:9471", store, domain)
}
}
if cfg.Metrics.UpstreamURL == nil {
domain, root := cfg.ClusterDomain, cfg.Metrics.RootTenant
cfg.Metrics.UpstreamURL = func(store string) string {
return fmt.Sprintf("http://vmselect-%s.%s.svc.%s:8481/select/0/prometheus", store, root, domain)
}
}
s := &Server{
cfg: cfg,
auth: auth,
@@ -167,7 +175,7 @@ func writeError(w http.ResponseWriter, e *HTTPError) {
// serve answers a root it has no handler for before authenticating, so an
// unknown path costs no review calls.
func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTTPError {
if route.Kind != NamespaceLogs {
if route.Kind != NamespaceLogs && route.Kind != NamespaceMetrics {
return errorf(http.StatusNotFound, "not found")
}
user, herr := s.authorize(r, route)
@@ -182,6 +190,9 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT
w.Header().Set("Retry-After", strconv.FormatInt(int64(math.Ceil(wait.Seconds())), 10))
return errorf(http.StatusTooManyRequests, "request rate limit exceeded")
}
if route.Kind == NamespaceMetrics {
return s.serveMetrics(w, r, route, user)
}
endpoint, ok := strings.CutPrefix(route.Endpoint, "select/logsql/")
if !ok {
return errorf(http.StatusNotFound, "not found")
@@ -226,29 +237,17 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT
ctx, stop = context.WithTimeout(ctx, limit)
defer stop()
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
s.cfg.UpstreamURL(store)+"/select/logsql/"+endpoint, strings.NewReader(params.Encode()))
if err != nil {
return errorf(http.StatusInternalServerError, "building upstream request: %v", err)
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
req.Header.Set("AccountID", "0")
req.Header.Set("ProjectID", "0")
tenant := http.Header{}
tenant.Set("AccountID", "0")
tenant.Set("ProjectID", "0")
if tail {
// vlselect holds even the headers of a gzip-encoded tail until the
// first line; an explicit value stops the transport from asking for gzip.
req.Header.Set("Accept-Encoding", "identity")
tenant.Set("Accept-Encoding", "identity")
}
resp, err := s.client.Do(req)
switch {
case err == nil:
case r.Context().Err() != nil:
return errorf(http.StatusBadGateway, "client went away")
case errors.Is(context.Cause(ctx), context.DeadlineExceeded):
return errorf(http.StatusGatewayTimeout, "log store did not answer within %s", limit)
default:
slog.Warn("log store unreachable", "store", store, "error", err)
return errorf(http.StatusBadGateway, "log store unreachable")
resp, herr := s.post(ctx, r, s.cfg.UpstreamURL(store)+"/select/logsql/"+endpoint, params, "log store", tenant)
if herr != nil {
return herr
}
defer func() { _ = resp.Body.Close() }()
@@ -280,6 +279,70 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT
return nil
}
func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Route, user string) *HTTPError {
r.Body = http.MaxBytesReader(w, r.Body, maxFormBytes)
if err := r.ParseForm(); err != nil {
return errorf(http.StatusBadRequest, "invalid form: %v", err)
}
now := s.now()
req, herr := BuildMetricsParams(route.Endpoint, r.Form, route.Namespace, now, s.cfg.Metrics.Limits)
if herr != nil {
return herr
}
release, ok := s.limits.acquire(user)
if !ok {
return errorf(http.StatusUnprocessableEntity, "too many concurrent requests")
}
defer release()
ctx := r.Context()
if s.cfg.RequestTimeout > 0 {
var cancel context.CancelFunc
ctx, cancel = context.WithTimeout(ctx, s.cfg.RequestTimeout)
defer cancel()
}
store := s.cfg.Metrics.store(req.start, now)
resp, herr := s.post(ctx, r, s.cfg.Metrics.UpstreamURL(store)+"/"+req.path, req.params, "metrics store", nil)
if herr != nil {
return herr
}
defer func() { _ = resp.Body.Close() }()
if ct := resp.Header.Get("Content-Type"); ct != "" {
w.Header().Set("Content-Type", ct)
}
w.WriteHeader(resp.StatusCode)
if _, err := io.Copy(w, resp.Body); err != nil && !errors.Is(err, context.Canceled) {
slog.Debug("copying upstream response", "error", err)
}
return nil
}
// post sends params as a form with only the given headers, so nothing the
// client sent reaches the store, and tells an unreachable store (502) from
// one that missed its deadline or, for a tail, its idle timeout (504).
func (s *Server) post(ctx context.Context, r *http.Request, target string, params url.Values, what string, header http.Header) (*http.Response, *HTTPError) {
req, err := http.NewRequestWithContext(ctx, http.MethodPost, target, strings.NewReader(params.Encode()))
if err != nil {
return nil, errorf(http.StatusInternalServerError, "building upstream request: %v", err)
}
for k, v := range header {
req.Header[k] = v
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
resp, err := s.client.Do(req)
switch {
case err == nil:
return resp, nil
case r.Context().Err() != nil:
return nil, errorf(http.StatusBadGateway, "client went away")
case errors.Is(context.Cause(ctx), context.DeadlineExceeded):
return nil, errorf(http.StatusGatewayTimeout, "%s did not answer in time", what)
default:
slog.Warn(what+" unreachable", "url", target, "error", err)
return nil, errorf(http.StatusBadGateway, "%s unreachable", what)
}
}
// concurrencyRetryAfter is a hint only: a slot frees when another request
// of the user ends, which the proxy cannot predict, and one second keeps
// a polling client from retrying in a tight loop.
@@ -0,0 +1,163 @@
/*
Copyright 2026 The Cozystack Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package observabilityproxy
import (
"io"
"net/http"
"net/url"
"testing"
"time"
authorizationv1 "k8s.io/api/authorization/v1"
)
const metricsRoot = "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/"
func promUpstream(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = io.WriteString(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`)
}
func TestServerMetricsRebuildsUpstreamRequest(t *testing.T) {
h := newHarness(t, Config{}, promUpstream)
hostile := url.Values{
"query": {"sum(rate(container_cpu_usage_seconds_total[5m]))"},
"start": {unix(testNow.Add(-time.Hour))},
"end": {unix(testNow)},
"step": {"60"},
"extra_filters": {`{namespace=~".+"}`},
"extra_filters[]": {`{namespace="tenant-b"}`, `{namespace=~".*"}`},
"extra_label": {"namespace=tenant-b"},
"nocache": {"1"},
}
headers := map[string]string{"AccountID": "9", "X-Forwarded-For": "1.2.3.4", "Cookie": "kc-access=secret"}
resp := h.do(http.MethodGet, metricsRoot+"api/v1/query_range?"+hostile.Encode(), "good", nil, headers)
_ = resp.Body.Close()
resp = h.do(http.MethodPost, metricsRoot+"api/v1/query_range?extra_filters[]="+url.QueryEscape(`{namespace="x"}`), "good", hostile, headers)
body, _ := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if resp.StatusCode != http.StatusOK || resp.Header.Get("Content-Type") != "application/json" {
t.Fatalf("POST answered %d %s %q", resp.StatusCode, resp.Header.Get("Content-Type"), body)
}
reqs := h.requests()
if len(reqs) != 2 {
t.Fatalf("upstream saw %d requests", len(reqs))
}
if reqs[0].form.Encode() != reqs[1].form.Encode() {
t.Fatalf("GET and POST built different upstream requests:\n%v\n%v", reqs[0].form, reqs[1].form)
}
if n := h.kube.sars.Load(); n != 1 {
t.Fatalf("SubjectAccessReviews = %d, want 1: POST must reuse the GET decision", n)
}
want := authorizationv1.ResourceAttributes{Verb: "list", Group: "cozystack.io", Resource: "metrics", Namespace: "tenant-a"}
if got := h.kube.lastSAR.Load().ResourceAttributes; *got != want {
t.Fatalf("SubjectAccessReview = %+v, want %+v", *got, want)
}
for _, r := range reqs {
if r.path != "/api/v1/query_range" {
t.Errorf("upstream path = %s", r.path)
}
if got := r.form["extra_filters[]"]; len(got) != 1 || got[0] != wantMetricsScope {
t.Errorf("extra_filters[] = %q", got)
}
if r.form.Has("extra_filters") || r.form.Has("extra_label") || r.form.Has("nocache") {
t.Errorf("client parameter forwarded: %v", r.form)
}
for _, k := range []string{"Authorization", "AccountID", "X-Forwarded-For", "Cookie"} {
if r.headers.Get(k) != "" {
t.Errorf("client header %s forwarded", k)
}
}
}
}
func TestServerMetricsStoreByRange(t *testing.T) {
h := newHarness(t, Config{}, promUpstream)
for _, path := range []string{
"api/v1/query?query=up",
"api/v1/query_range?query=up&step=600&start=" + unix(testNow.Add(-7*24*time.Hour)) + "&end=" + unix(testNow),
"api/v1/label/pod/values",
} {
if got := h.status(http.MethodGet, metricsRoot+path, "good"); got != http.StatusOK {
t.Fatalf("%s answered %d", path, got)
}
}
h.mu.Lock()
defer h.mu.Unlock()
if want := []string{"shortterm", "longterm", "shortterm"}; len(h.stores) != 3 || h.stores[0] != want[0] || h.stores[1] != want[1] || h.stores[2] != want[2] {
t.Fatalf("stores = %v, want %v", h.stores, want)
}
if r := h.got[2]; r.path != "/api/v1/label/pod/values" || r.form.Get("match[]") != wantMetricsScope {
t.Fatalf("label values upstream = %s %v", r.path, r.form)
}
}
func TestServerMetricsStatusCodes(t *testing.T) {
h := newHarness(t, Config{}, promUpstream)
tooLong := "api/v1/query_range?query=up&step=3600&start=" + unix(testNow.Add(-31*24*time.Hour)) + "&end=" + unix(testNow)
if got := h.status(http.MethodGet, metricsRoot+tooLong, "good"); got != http.StatusUnprocessableEntity {
t.Errorf("range above the limit answered %d, want 422", got)
}
if got := h.status(http.MethodGet, metricsRoot+"api/v1/series", "good"); got != http.StatusBadRequest {
t.Errorf("series without match[] answered %d, want 400", got)
}
if got := h.status(http.MethodPut, metricsRoot+"api/v1/query?query=up", "good"); got != http.StatusMethodNotAllowed {
t.Errorf("PUT answered %d, want 405", got)
}
if n := len(h.requests()); n != 0 {
t.Errorf("upstream saw %d requests for refused ones", n)
}
h2 := newHarness(t, Config{}, func(w http.ResponseWriter, _ *http.Request) {
http.Error(w, `{"status":"error","errorType":"422"}`, http.StatusUnprocessableEntity)
})
if got := h2.status(http.MethodGet, metricsRoot+"api/v1/query?query=up", "good"); got != http.StatusUnprocessableEntity {
t.Errorf("upstream 422 answered as %d", got)
}
h2.upstream.Close()
if got := h2.status(http.MethodGet, metricsRoot+"api/v1/query?query=up", "good"); got != http.StatusBadGateway {
t.Errorf("unreachable upstream answered %d, want 502", got)
}
release := make(chan struct{})
t.Cleanup(func() { close(release) })
h3 := newHarness(t, Config{RequestTimeout: 50 * time.Millisecond}, func(_ http.ResponseWriter, r *http.Request) {
select {
case <-release:
case <-r.Context().Done():
}
})
if got := h3.status(http.MethodGet, metricsRoot+"api/v1/query?query=up", "good"); got != http.StatusGatewayTimeout {
t.Errorf("slow upstream answered %d, want 504", got)
}
h4 := newHarness(t, Config{Limiter: LimiterConfig{RatePerUser: 0.001, RateBurst: 1}}, promUpstream)
h4.status(http.MethodGet, metricsRoot+"api/v1/query?query=up", "good")
if got := h4.status(http.MethodGet, metricsRoot+"api/v1/query?query=up", "good"); got != http.StatusUnprocessableEntity {
t.Errorf("rate-limited request answered %d, want 422", got)
}
}
func TestDefaultMetricsUpstreamURL(t *testing.T) {
s := NewServer(Config{ClusterDomain: "cozy.local", Metrics: MetricsConfig{RootTenant: "tenant-root"}}, nil, nil, nil)
if got, want := s.cfg.Metrics.UpstreamURL("shortterm"), "http://vmselect-shortterm.tenant-root.svc.cozy.local:8481/select/0/prometheus"; got != want {
t.Fatalf("upstream = %s, want %s", got, want)
}
}
+13 -4
View File
@@ -52,12 +52,12 @@ type mapStores map[string]string
func (m mapStores) Resolve(ns string) (string, error) { return m[ns], nil }
// newHarness wires the proxy to a fake apiserver that knows tokens "good"
// (alice) and "bob-token" (bob), both allowed list logs in tenant-a only,
// and to an upstream served by handler.
// (alice) and "bob-token" (bob), both allowed list logs and metrics in
// tenant-a only, and to an upstream served by handler.
func newHarness(t *testing.T, cfg Config, handler http.HandlerFunc) *harness {
h := &harness{t: t}
h.kube = newFakeKube(map[string]string{"good": "alice", "bob-token": "bob"}, func(_ string, a *authorizationv1.ResourceAttributes) bool {
return a.Namespace == "tenant-a" && a.Resource == "logs"
return a.Namespace == "tenant-a" && (a.Resource == "logs" || a.Resource == "metrics")
})
h.upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseForm(); err != nil {
@@ -80,6 +80,13 @@ func newHarness(t *testing.T, cfg Config, handler http.HandlerFunc) *harness {
h.mu.Unlock()
return h.upstream.URL
}
cfg.Metrics.UpstreamURL = cfg.UpstreamURL
if cfg.Metrics.ShortTermStore == "" {
cfg.Metrics.ShortTermStore, cfg.Metrics.LongTermStore, cfg.Metrics.ShortTermRetention = "shortterm", "longterm", 72*time.Hour
}
if cfg.Metrics.Limits == (MetricsLimits{}) {
cfg.Metrics.Limits = testMetricsLimits()
}
if cfg.Logs == (LogsLimits{}) {
cfg.Logs = testLimits()
}
@@ -144,7 +151,9 @@ func TestServerStatusCodes(t *testing.T) {
{"other namespace", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-b/logs/select/logsql/query", "good", http.StatusForbidden},
{"invalid namespace", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/Tenant/logs/select/logsql/query", "good", http.StatusBadRequest},
{"unknown root", http.MethodGet, "/api/v1/namespaces/tenant-a/pods", "good", http.StatusNotFound},
{"namespace metrics not served yet", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/api/v1/query", "good", http.StatusNotFound},
{"unknown metrics endpoint", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/api/v1/status/tsdb", "good", http.StatusNotFound},
{"metrics in another namespace", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-b/metrics/api/v1/query?query=up", "good", http.StatusForbidden},
{"metrics without token", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/api/v1/query?query=up", "", http.StatusUnauthorized},
{"application log not served yet", http.MethodGet, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/log/select/logsql/query", "good", http.StatusNotFound},
{"application metrics not served yet", http.MethodGet, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/metrics/api/v1/query", "good", http.StatusNotFound},
{"tenant_ids not proxied", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/logs/select/tenant_ids", "good", http.StatusNotFound},