From 2f600d5354f672b5e53d382c248d867624298dc2 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 05:59:44 +0700 Subject: [PATCH] feat(observability-proxy): serve the application logs root A viewer of one application gets that application's lines without writing a namespace query and filtering by hand, and a grant narrowed to one application by resourceNames is honoured on every logs endpoint, tail and the pickers included. The namespace filter stays and the application's identity is ANDed next to it as a second stream filter, so the result can only be narrower than the namespace root. The plural is mapped to its kind only after the caller is authorized, so an anonymous client cannot probe which applications exist, and the proxy is not ready until the ApplicationDefinitions cache is. Assisted-by: LLM Signed-off-by: ZverGuy --- cmd/observability-proxy/main.go | 20 ++- internal/observabilityproxy/server.go | 43 +++++-- .../observabilityproxy/server_app_test.go | 114 ++++++++++++++++++ .../observabilityproxy/server_metrics_test.go | 2 +- internal/observabilityproxy/server_test.go | 21 +++- .../observability-proxy/templates/rbac.yaml | 5 + .../observability-proxy/tests/rbac_test.yaml | 5 +- 7 files changed, 192 insertions(+), 18 deletions(-) create mode 100644 internal/observabilityproxy/server_app_test.go diff --git a/cmd/observability-proxy/main.go b/cmd/observability-proxy/main.go index a3687fe08..1de6e6e14 100644 --- a/cmd/observability-proxy/main.go +++ b/cmd/observability-proxy/main.go @@ -39,6 +39,8 @@ import ( "github.com/prometheus/client_golang/prometheus/collectors" "github.com/prometheus/client_golang/prometheus/promhttp" "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/dynamic/dynamicinformer" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" @@ -178,13 +180,24 @@ func run(o options) error { return fmt.Errorf("building client: %w", err) } + dynClient, err := dynamic.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("building dynamic client: %w", err) + } + factory := informers.NewSharedInformerFactory(client, 0) namespaces := factory.Core().V1().Namespaces() lister := namespaces.Lister() + appDefs := dynamicinformer.NewDynamicSharedInformerFactory(dynClient, 0). + ForResource(observabilityproxy.ApplicationDefinitionsResource).Informer() + if err := appDefs.AddIndexers(observabilityproxy.ApplicationPluralIndexers()); err != nil { + return fmt.Errorf("indexing application definitions: %w", err) + } factory.Start(ctx.Done()) + go appDefs.Run(ctx.Done()) var ready atomic.Bool go func() { - if cache.WaitForCacheSync(ctx.Done(), namespaces.Informer().HasSynced) { + if cache.WaitForCacheSync(ctx.Done(), namespaces.Informer().HasSynced, appDefs.HasSynced) { ready.Store(true) } }() @@ -198,13 +211,14 @@ func run(o options) error { Limiter: o.limiter, RequestTimeout: o.requestTimeout, TailIdleTimeout: o.tailIdleTimeout, - }, observabilityproxy.NewAuthorizer(client, o.authCacheTTL), observabilityproxy.NewStoreResolver(lister, namespaces.Informer().HasSynced, o.rootTenant), reg) + }, observabilityproxy.NewAuthorizer(client, o.authCacheTTL), observabilityproxy.NewStoreResolver(lister, namespaces.Informer().HasSynced, o.rootTenant), + observabilityproxy.NewApplicationKinds(appDefs.GetIndexer(), appDefs.HasSynced), reg) probes := http.NewServeMux() probes.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { _, _ = io.WriteString(w, "ok") }) probes.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) { if !ready.Load() { - http.Error(w, "namespace cache not synced", http.StatusServiceUnavailable) + http.Error(w, "namespace or application definition cache not synced", http.StatusServiceUnavailable) return } _, _ = io.WriteString(w, "ok") diff --git a/internal/observabilityproxy/server.go b/internal/observabilityproxy/server.go index eb540a271..617971bf2 100644 --- a/internal/observabilityproxy/server.go +++ b/internal/observabilityproxy/server.go @@ -73,13 +73,14 @@ type Server struct { cfg Config auth *Authorizer stores StoreLookup + kinds KindLookup client *http.Client limits *limiter now func() time.Time requests *prometheus.CounterVec } -func NewServer(cfg Config, auth *Authorizer, stores StoreLookup, reg prometheus.Registerer) *Server { +func NewServer(cfg Config, auth *Authorizer, stores StoreLookup, kinds KindLookup, reg prometheus.Registerer) *Server { if cfg.UpstreamURL == nil { domain := cfg.ClusterDomain cfg.UpstreamURL = func(store string) string { @@ -96,6 +97,7 @@ func NewServer(cfg Config, auth *Authorizer, stores StoreLookup, reg prometheus. cfg: cfg, auth: auth, stores: stores, + kinds: kinds, // No client timeout: it would cut live tails. Non-tail requests are // bounded by a context deadline instead. client: &http.Client{Transport: http.DefaultTransport.(*http.Transport).Clone()}, @@ -172,12 +174,7 @@ func writeError(w http.ResponseWriter, e *HTTPError) { http.Error(w, e.Message, e.Code) } -// serve answers a root it has no handler for before authenticating, so an -// unknown path costs no review calls. func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTTPError { - if route.Kind != NamespaceLogs && route.Kind != NamespaceMetrics { - return errorf(http.StatusNotFound, "not found") - } user, herr := s.authorize(r, route) if herr != nil { return herr @@ -190,8 +187,17 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT w.Header().Set("Retry-After", strconv.FormatInt(int64(math.Ceil(wait.Seconds())), 10)) return errorf(http.StatusTooManyRequests, "request rate limit exceeded") } - if route.Kind == NamespaceMetrics { + var kind string + if route.Kind == ApplicationLog || route.Kind == ApplicationMetrics { + if kind, herr = s.applicationKind(route.Plural); herr != nil { + return herr + } + } + switch route.Kind { + case NamespaceMetrics: return s.serveMetrics(w, r, route, user) + case ApplicationMetrics: + return errorf(http.StatusNotFound, "not found") } endpoint, ok := strings.CutPrefix(route.Endpoint, "select/logsql/") if !ok { @@ -214,6 +220,9 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT if herr != nil { return herr } + if kind != "" { + params.Add("extra_stream_filters", applicationStreamFilter(kind, route.Name)) + } store, err := s.stores.Resolve(route.Namespace) if err != nil { @@ -322,6 +331,26 @@ func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Rout return nil } +func (s *Server) applicationKind(plural string) (string, *HTTPError) { + kind, err := s.kinds.Kind(plural) + if err != nil { + slog.Error("mapping application plural to kind", "plural", plural, "error", err) + return "", errorf(http.StatusServiceUnavailable, "cannot resolve the application kind") + } + if kind == "" { + return "", errorf(http.StatusNotFound, "unknown application resource %q", plural) + } + return kind, nil +} + +// applicationStreamFilter selects the streams Fluent Bit labels with the +// pod's lineage. vlselect ANDs each extra_stream_filters value into the +// query and every subquery (Query.AddExtraFilters in lib/logstorage), so +// it narrows the namespace filter sent next to it. +func applicationStreamFilter(kind, name string) string { + return "{apps_cozystack_io_application_kind=" + strconv.Quote(kind) + ",apps_cozystack_io_application_name=" + strconv.Quote(name) + "}" +} + // post sends params as a form with only the given headers, so nothing the // client sent reaches the store, and tells an unreachable store (502) from // one that missed its deadline or, for a tail, its idle timeout (504). diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go new file mode 100644 index 000000000..0a76ab9a5 --- /dev/null +++ b/internal/observabilityproxy/server_app_test.go @@ -0,0 +1,114 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "io" + "net/http" + "net/url" + "slices" + "testing" + + authorizationv1 "k8s.io/api/authorization/v1" +) + +const appRoot = "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/" + +// appPolicy lets alice read every postgres in tenant-a, as the view role's +// apps.cozystack.io wildcard does, and bob only "a", as a grant narrowed +// with resourceNames does. +func appPolicy(user string, a *authorizationv1.ResourceAttributes) bool { + if a.Namespace != "tenant-a" || a.Verb != "get" || a.Group != "apps.cozystack.io" || a.Resource != "postgreses" || + (a.Subresource != "log" && a.Subresource != "metrics") { + return false + } + return user == "alice" || a.Name == "a" +} + +func TestServerApplicationLogsScope(t *testing.T) { + h := newPolicyHarness(t, Config{}, nil, appPolicy, nil) + hostile := url.Values{ + "query": {"error"}, + "field": {"level"}, + "extra_filters": {`{kubernetes_namespace_name="tenant-b"}`}, + "extra_stream_filters": {`{apps_cozystack_io_application_name=~".*"}`}, + "AccountID": {"9"}, + } + headers := map[string]string{"AccountID": "9", "ProjectID": "9"} + endpoints := []string{"query", "hits", "stats_query_range", "tail", "field_values", "stream_field_values", "facets"} + for _, ep := range endpoints { + resp := h.do(http.MethodPost, appRoot+"db.main/log/select/logsql/"+ep+"?extra_stream_filters=x", "good", hostile, headers) + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("%s answered %d", ep, resp.StatusCode) + } + } + want := authorizationv1.ResourceAttributes{Verb: "get", Group: "apps.cozystack.io", Resource: "postgreses", Subresource: "log", Name: "db.main", Namespace: "tenant-a"} + if got := h.kube.lastSAR.Load().ResourceAttributes; *got != want { + t.Fatalf("SubjectAccessReview = %+v, want %+v", *got, want) + } + reqs := h.requests() + if len(reqs) != len(endpoints) { + t.Fatalf("upstream saw %d requests, want %d", len(reqs), len(endpoints)) + } + // Two values: vlselect ANDs every extra_stream_filters value into the + // query and each of its subqueries. + wantFilters := []string{wantScope, `{apps_cozystack_io_application_kind="Postgres",apps_cozystack_io_application_name="db.main"}`} + for i, r := range reqs { + if r.path != "/select/logsql/"+endpoints[i] { + t.Errorf("upstream path = %s, want %s", r.path, endpoints[i]) + } + if got := r.form["extra_stream_filters"]; !slices.Equal(got, wantFilters) { + t.Errorf("%s: extra_stream_filters = %q, want %q", endpoints[i], got, wantFilters) + } + if r.form.Has("extra_filters") || r.form.Has("AccountID") || r.headers.Get("AccountID") != "0" || r.headers.Get("ProjectID") != "0" { + t.Errorf("%s: client scope reached the store: %v %v", endpoints[i], r.form, r.headers) + } + } +} + +func TestServerApplicationKinds(t *testing.T) { + tests := []struct { + name string + kinds KindLookup + path string + want int + }{ + {"unknown plural logs", nil, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/mysqls/a/log/select/logsql/query", http.StatusNotFound}, + {"unknown plural metrics", nil, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/mysqls/a/metrics/api/v1/query?query=up", http.StatusNotFound}, + {"not synced logs", newTestKinds(t, false), appRoot + "a/log/select/logsql/query", http.StatusServiceUnavailable}, + {"not synced metrics", newTestKinds(t, false), appRoot + "a/metrics/api/v1/query?query=up", http.StatusServiceUnavailable}, + {"ambiguous plural", newTestKinds(t, true, appDefinition("x", "Postgres", "postgreses"), appDefinition("y", "Pg", "postgreses")), appRoot + "a/log/select/logsql/query", http.StatusServiceUnavailable}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + h := newPolicyHarness(t, Config{}, nil, func(string, *authorizationv1.ResourceAttributes) bool { return true }, tt.kinds) + if got := h.status(http.MethodGet, tt.path, "good"); got != tt.want { + t.Fatalf("status = %d, want %d", got, tt.want) + } + if n := len(h.requests()); n != 0 { + t.Fatalf("upstream saw %d requests", n) + } + }) + } + // The kind is looked up only for an authorized caller. + h := newPolicyHarness(t, Config{}, nil, appPolicy, newTestKinds(t, false)) + if got := h.status(http.MethodGet, appRoot+"b/log/select/logsql/query", "bob-token"); got != http.StatusForbidden { + t.Fatalf("denied caller answered %d, want 403", got) + } +} diff --git a/internal/observabilityproxy/server_metrics_test.go b/internal/observabilityproxy/server_metrics_test.go index 1ef5e761e..1a3d1de10 100644 --- a/internal/observabilityproxy/server_metrics_test.go +++ b/internal/observabilityproxy/server_metrics_test.go @@ -203,7 +203,7 @@ func TestServerMetricsConcurrency(t *testing.T) { } func TestDefaultMetricsUpstreamURL(t *testing.T) { - s := NewServer(Config{ClusterDomain: "cozy.local", Metrics: MetricsConfig{RootTenant: "tenant-root"}}, nil, nil, nil) + s := NewServer(Config{ClusterDomain: "cozy.local", Metrics: MetricsConfig{RootTenant: "tenant-root"}}, nil, nil, nil, nil) if got, want := s.cfg.Metrics.UpstreamURL("shortterm"), "http://vmselect-shortterm.tenant-root.svc.cozy.local:8481/select/0/prometheus"; got != want { t.Fatalf("upstream = %s, want %s", got, want) } diff --git a/internal/observabilityproxy/server_test.go b/internal/observabilityproxy/server_test.go index 2670d0e3e..b541a18e1 100644 --- a/internal/observabilityproxy/server_test.go +++ b/internal/observabilityproxy/server_test.go @@ -55,10 +55,19 @@ func (m mapStores) Resolve(ns string) (string, error) { return m[ns], nil } // (alice) and "bob-token" (bob), both allowed list logs and metrics in // tenant-a only, and to an upstream served by handler. func newHarness(t *testing.T, cfg Config, handler http.HandlerFunc) *harness { - h := &harness{t: t} - h.kube = newFakeKube(map[string]string{"good": "alice", "bob-token": "bob"}, func(_ string, a *authorizationv1.ResourceAttributes) bool { + return newPolicyHarness(t, cfg, handler, func(_ string, a *authorizationv1.ResourceAttributes) bool { return a.Namespace == "tenant-a" && (a.Resource == "logs" || a.Resource == "metrics") - }) + }, nil) +} + +// newPolicyHarness is newHarness with the given SubjectAccessReview policy +// and plural-to-kind mapping; nil kinds knows only postgreses. +func newPolicyHarness(t *testing.T, cfg Config, handler http.HandlerFunc, allow func(string, *authorizationv1.ResourceAttributes) bool, kinds KindLookup) *harness { + h := &harness{t: t} + h.kube = newFakeKube(map[string]string{"good": "alice", "bob-token": "bob"}, allow) + if kinds == nil { + kinds = newTestKinds(t, true, appDefinition("postgres", "Postgres", "postgreses")) + } h.upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if err := r.ParseForm(); err != nil { t.Errorf("upstream ParseForm: %v", err) @@ -90,7 +99,7 @@ func newHarness(t *testing.T, cfg Config, handler http.HandlerFunc) *harness { if cfg.Logs == (LogsLimits{}) { cfg.Logs = testLimits() } - h.server = NewServer(cfg, NewAuthorizer(h.kube.client, 10*time.Second), mapStores{"tenant-a": "tenant-a-store"}, nil) + h.server = NewServer(cfg, NewAuthorizer(h.kube.client, 10*time.Second), mapStores{"tenant-a": "tenant-a-store"}, kinds, nil) h.server.now = func() time.Time { return testNow } h.proxy = httptest.NewServer(h.server) t.Cleanup(h.proxy.Close) @@ -154,8 +163,8 @@ func TestServerStatusCodes(t *testing.T) { {"unknown metrics endpoint", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/api/v1/status/tsdb", "good", http.StatusNotFound}, {"metrics in another namespace", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-b/metrics/api/v1/query?query=up", "good", http.StatusForbidden}, {"metrics without token", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/api/v1/query?query=up", "", http.StatusUnauthorized}, - {"application log not served yet", http.MethodGet, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/log/select/logsql/query", "good", http.StatusNotFound}, - {"application metrics not served yet", http.MethodGet, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/metrics/api/v1/query", "good", http.StatusNotFound}, + {"application log without a grant", http.MethodGet, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/log/select/logsql/query", "good", http.StatusForbidden}, + {"application metrics without a grant", http.MethodGet, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/metrics/api/v1/query", "good", http.StatusForbidden}, {"tenant_ids not proxied", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/logs/select/tenant_ids", "good", http.StatusNotFound}, {"unknown logsql endpoint", http.MethodGet, logsRoot + "stats_query", "good", http.StatusNotFound}, {"logs root itself", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/logs/", "good", http.StatusNotFound}, diff --git a/packages/system/observability-proxy/templates/rbac.yaml b/packages/system/observability-proxy/templates/rbac.yaml index 3c5392903..f3f67127c 100644 --- a/packages/system/observability-proxy/templates/rbac.yaml +++ b/packages/system/observability-proxy/templates/rbac.yaml @@ -15,6 +15,11 @@ rules: - apiGroups: [""] resources: ["namespaces"] verbs: ["list", "watch"] +# Application roots map the plural in the path to the kind the lineage +# labels carry, from an informer cache as well. +- apiGroups: ["cozystack.io"] + resources: ["applicationdefinitions"] + verbs: ["list", "watch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/packages/system/observability-proxy/tests/rbac_test.yaml b/packages/system/observability-proxy/tests/rbac_test.yaml index ef96c65d7..5382c2ab6 100644 --- a/packages/system/observability-proxy/tests/rbac_test.yaml +++ b/packages/system/observability-proxy/tests/rbac_test.yaml @@ -12,7 +12,7 @@ release: namespace: cozy-observability-proxy tests: - - it: lets the proxy review tokens and access, and list and watch namespaces + - it: lets the proxy review tokens and access, and list and watch namespaces and application definitions template: templates/rbac.yaml documentSelector: path: kind @@ -30,6 +30,9 @@ tests: - apiGroups: [""] resources: ["namespaces"] verbs: ["list", "watch"] + - apiGroups: ["cozystack.io"] + resources: ["applicationdefinitions"] + verbs: ["list", "watch"] - it: binds the proxy role to the proxy ServiceAccount in the release namespace template: templates/rbac.yaml