feat(observability-proxy): serve the application logs root

A viewer of one application gets that application's lines without
writing a namespace query and filtering by hand, and a grant narrowed to
one application by resourceNames is honoured on every logs endpoint,
tail and the pickers included. The namespace filter stays and the
application's identity is ANDed next to it as a second stream filter,
so the result can only be narrower than the namespace root.

The plural is mapped to its kind only after the caller is authorized,
so an anonymous client cannot probe which applications exist, and the
proxy is not ready until the ApplicationDefinitions cache is.

Assisted-by: LLM
Signed-off-by: ZverGuy <maximbel2003@gmail.com>
This commit is contained in:
ZverGuy committed 2026-11-10 04:03:24 +07:00
1 parent ecb5de6051
commit 2f600d5354
7 files changed
+192 -18

No files matched your search

+17 -3
View File
@@ -39,6 +39,8 @@ import (
"github.com/prometheus/client_golang/prometheus/collectors"
"github.com/prometheus/client_golang/prometheus/promhttp"
"k8s.io/apimachinery/pkg/util/validation"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/dynamic/dynamicinformer"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/cache"
@@ -178,13 +180,24 @@ func run(o options) error {
return fmt.Errorf("building client: %w", err)
}
dynClient, err := dynamic.NewForConfig(restConfig)
if err != nil {
return fmt.Errorf("building dynamic client: %w", err)
}
factory := informers.NewSharedInformerFactory(client, 0)
namespaces := factory.Core().V1().Namespaces()
lister := namespaces.Lister()
appDefs := dynamicinformer.NewDynamicSharedInformerFactory(dynClient, 0).
ForResource(observabilityproxy.ApplicationDefinitionsResource).Informer()
if err := appDefs.AddIndexers(observabilityproxy.ApplicationPluralIndexers()); err != nil {
return fmt.Errorf("indexing application definitions: %w", err)
}
factory.Start(ctx.Done())
go appDefs.Run(ctx.Done())
var ready atomic.Bool
go func() {
if cache.WaitForCacheSync(ctx.Done(), namespaces.Informer().HasSynced) {
if cache.WaitForCacheSync(ctx.Done(), namespaces.Informer().HasSynced, appDefs.HasSynced) {
ready.Store(true)
}
}()
@@ -198,13 +211,14 @@ func run(o options) error {
Limiter: o.limiter,
RequestTimeout: o.requestTimeout,
TailIdleTimeout: o.tailIdleTimeout,
}, observabilityproxy.NewAuthorizer(client, o.authCacheTTL), observabilityproxy.NewStoreResolver(lister, namespaces.Informer().HasSynced, o.rootTenant), reg)
}, observabilityproxy.NewAuthorizer(client, o.authCacheTTL), observabilityproxy.NewStoreResolver(lister, namespaces.Informer().HasSynced, o.rootTenant),
observabilityproxy.NewApplicationKinds(appDefs.GetIndexer(), appDefs.HasSynced), reg)
probes := http.NewServeMux()
probes.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { _, _ = io.WriteString(w, "ok") })
probes.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) {
if !ready.Load() {
http.Error(w, "namespace cache not synced", http.StatusServiceUnavailable)
http.Error(w, "namespace or application definition cache not synced", http.StatusServiceUnavailable)
return
}
_, _ = io.WriteString(w, "ok")
+36 -7
View File
@@ -73,13 +73,14 @@ type Server struct {
cfg Config
auth *Authorizer
stores StoreLookup
kinds KindLookup
client *http.Client
limits *limiter
now func() time.Time
requests *prometheus.CounterVec
}
func NewServer(cfg Config, auth *Authorizer, stores StoreLookup, reg prometheus.Registerer) *Server {
func NewServer(cfg Config, auth *Authorizer, stores StoreLookup, kinds KindLookup, reg prometheus.Registerer) *Server {
if cfg.UpstreamURL == nil {
domain := cfg.ClusterDomain
cfg.UpstreamURL = func(store string) string {
@@ -96,6 +97,7 @@ func NewServer(cfg Config, auth *Authorizer, stores StoreLookup, reg prometheus.
cfg: cfg,
auth: auth,
stores: stores,
kinds: kinds,
// No client timeout: it would cut live tails. Non-tail requests are
// bounded by a context deadline instead.
client: &http.Client{Transport: http.DefaultTransport.(*http.Transport).Clone()},
@@ -172,12 +174,7 @@ func writeError(w http.ResponseWriter, e *HTTPError) {
http.Error(w, e.Message, e.Code)
}
// serve answers a root it has no handler for before authenticating, so an
// unknown path costs no review calls.
func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTTPError {
if route.Kind != NamespaceLogs && route.Kind != NamespaceMetrics {
return errorf(http.StatusNotFound, "not found")
}
user, herr := s.authorize(r, route)
if herr != nil {
return herr
@@ -190,8 +187,17 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT
w.Header().Set("Retry-After", strconv.FormatInt(int64(math.Ceil(wait.Seconds())), 10))
return errorf(http.StatusTooManyRequests, "request rate limit exceeded")
}
if route.Kind == NamespaceMetrics {
var kind string
if route.Kind == ApplicationLog || route.Kind == ApplicationMetrics {
if kind, herr = s.applicationKind(route.Plural); herr != nil {
return herr
}
}
switch route.Kind {
case NamespaceMetrics:
return s.serveMetrics(w, r, route, user)
case ApplicationMetrics:
return errorf(http.StatusNotFound, "not found")
}
endpoint, ok := strings.CutPrefix(route.Endpoint, "select/logsql/")
if !ok {
@@ -214,6 +220,9 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT
if herr != nil {
return herr
}
if kind != "" {
params.Add("extra_stream_filters", applicationStreamFilter(kind, route.Name))
}
store, err := s.stores.Resolve(route.Namespace)
if err != nil {
@@ -322,6 +331,26 @@ func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Rout
return nil
}
func (s *Server) applicationKind(plural string) (string, *HTTPError) {
kind, err := s.kinds.Kind(plural)
if err != nil {
slog.Error("mapping application plural to kind", "plural", plural, "error", err)
return "", errorf(http.StatusServiceUnavailable, "cannot resolve the application kind")
}
if kind == "" {
return "", errorf(http.StatusNotFound, "unknown application resource %q", plural)
}
return kind, nil
}
// applicationStreamFilter selects the streams Fluent Bit labels with the
// pod's lineage. vlselect ANDs each extra_stream_filters value into the
// query and every subquery (Query.AddExtraFilters in lib/logstorage), so
// it narrows the namespace filter sent next to it.
func applicationStreamFilter(kind, name string) string {
return "{apps_cozystack_io_application_kind=" + strconv.Quote(kind) + ",apps_cozystack_io_application_name=" + strconv.Quote(name) + "}"
}
// post sends params as a form with only the given headers, so nothing the
// client sent reaches the store, and tells an unreachable store (502) from
// one that missed its deadline or, for a tail, its idle timeout (504).
@@ -0,0 +1,114 @@
/*
Copyright 2026 The Cozystack Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package observabilityproxy
import (
"io"
"net/http"
"net/url"
"slices"
"testing"
authorizationv1 "k8s.io/api/authorization/v1"
)
const appRoot = "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/"
// appPolicy lets alice read every postgres in tenant-a, as the view role's
// apps.cozystack.io wildcard does, and bob only "a", as a grant narrowed
// with resourceNames does.
func appPolicy(user string, a *authorizationv1.ResourceAttributes) bool {
if a.Namespace != "tenant-a" || a.Verb != "get" || a.Group != "apps.cozystack.io" || a.Resource != "postgreses" ||
(a.Subresource != "log" && a.Subresource != "metrics") {
return false
}
return user == "alice" || a.Name == "a"
}
func TestServerApplicationLogsScope(t *testing.T) {
h := newPolicyHarness(t, Config{}, nil, appPolicy, nil)
hostile := url.Values{
"query": {"error"},
"field": {"level"},
"extra_filters": {`{kubernetes_namespace_name="tenant-b"}`},
"extra_stream_filters": {`{apps_cozystack_io_application_name=~".*"}`},
"AccountID": {"9"},
}
headers := map[string]string{"AccountID": "9", "ProjectID": "9"}
endpoints := []string{"query", "hits", "stats_query_range", "tail", "field_values", "stream_field_values", "facets"}
for _, ep := range endpoints {
resp := h.do(http.MethodPost, appRoot+"db.main/log/select/logsql/"+ep+"?extra_stream_filters=x", "good", hostile, headers)
_, _ = io.Copy(io.Discard, resp.Body)
_ = resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("%s answered %d", ep, resp.StatusCode)
}
}
want := authorizationv1.ResourceAttributes{Verb: "get", Group: "apps.cozystack.io", Resource: "postgreses", Subresource: "log", Name: "db.main", Namespace: "tenant-a"}
if got := h.kube.lastSAR.Load().ResourceAttributes; *got != want {
t.Fatalf("SubjectAccessReview = %+v, want %+v", *got, want)
}
reqs := h.requests()
if len(reqs) != len(endpoints) {
t.Fatalf("upstream saw %d requests, want %d", len(reqs), len(endpoints))
}
// Two values: vlselect ANDs every extra_stream_filters value into the
// query and each of its subqueries.
wantFilters := []string{wantScope, `{apps_cozystack_io_application_kind="Postgres",apps_cozystack_io_application_name="db.main"}`}
for i, r := range reqs {
if r.path != "/select/logsql/"+endpoints[i] {
t.Errorf("upstream path = %s, want %s", r.path, endpoints[i])
}
if got := r.form["extra_stream_filters"]; !slices.Equal(got, wantFilters) {
t.Errorf("%s: extra_stream_filters = %q, want %q", endpoints[i], got, wantFilters)
}
if r.form.Has("extra_filters") || r.form.Has("AccountID") || r.headers.Get("AccountID") != "0" || r.headers.Get("ProjectID") != "0" {
t.Errorf("%s: client scope reached the store: %v %v", endpoints[i], r.form, r.headers)
}
}
}
func TestServerApplicationKinds(t *testing.T) {
tests := []struct {
name string
kinds KindLookup
path string
want int
}{
{"unknown plural logs", nil, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/mysqls/a/log/select/logsql/query", http.StatusNotFound},
{"unknown plural metrics", nil, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/mysqls/a/metrics/api/v1/query?query=up", http.StatusNotFound},
{"not synced logs", newTestKinds(t, false), appRoot + "a/log/select/logsql/query", http.StatusServiceUnavailable},
{"not synced metrics", newTestKinds(t, false), appRoot + "a/metrics/api/v1/query?query=up", http.StatusServiceUnavailable},
{"ambiguous plural", newTestKinds(t, true, appDefinition("x", "Postgres", "postgreses"), appDefinition("y", "Pg", "postgreses")), appRoot + "a/log/select/logsql/query", http.StatusServiceUnavailable},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
h := newPolicyHarness(t, Config{}, nil, func(string, *authorizationv1.ResourceAttributes) bool { return true }, tt.kinds)
if got := h.status(http.MethodGet, tt.path, "good"); got != tt.want {
t.Fatalf("status = %d, want %d", got, tt.want)
}
if n := len(h.requests()); n != 0 {
t.Fatalf("upstream saw %d requests", n)
}
})
}
// The kind is looked up only for an authorized caller.
h := newPolicyHarness(t, Config{}, nil, appPolicy, newTestKinds(t, false))
if got := h.status(http.MethodGet, appRoot+"b/log/select/logsql/query", "bob-token"); got != http.StatusForbidden {
t.Fatalf("denied caller answered %d, want 403", got)
}
}
@@ -203,7 +203,7 @@ func TestServerMetricsConcurrency(t *testing.T) {
}
func TestDefaultMetricsUpstreamURL(t *testing.T) {
s := NewServer(Config{ClusterDomain: "cozy.local", Metrics: MetricsConfig{RootTenant: "tenant-root"}}, nil, nil, nil)
s := NewServer(Config{ClusterDomain: "cozy.local", Metrics: MetricsConfig{RootTenant: "tenant-root"}}, nil, nil, nil, nil)
if got, want := s.cfg.Metrics.UpstreamURL("shortterm"), "http://vmselect-shortterm.tenant-root.svc.cozy.local:8481/select/0/prometheus"; got != want {
t.Fatalf("upstream = %s, want %s", got, want)
}
+15 -6
View File
@@ -55,10 +55,19 @@ func (m mapStores) Resolve(ns string) (string, error) { return m[ns], nil }
// (alice) and "bob-token" (bob), both allowed list logs and metrics in
// tenant-a only, and to an upstream served by handler.
func newHarness(t *testing.T, cfg Config, handler http.HandlerFunc) *harness {
h := &harness{t: t}
h.kube = newFakeKube(map[string]string{"good": "alice", "bob-token": "bob"}, func(_ string, a *authorizationv1.ResourceAttributes) bool {
return newPolicyHarness(t, cfg, handler, func(_ string, a *authorizationv1.ResourceAttributes) bool {
return a.Namespace == "tenant-a" && (a.Resource == "logs" || a.Resource == "metrics")
})
}, nil)
}
// newPolicyHarness is newHarness with the given SubjectAccessReview policy
// and plural-to-kind mapping; nil kinds knows only postgreses.
func newPolicyHarness(t *testing.T, cfg Config, handler http.HandlerFunc, allow func(string, *authorizationv1.ResourceAttributes) bool, kinds KindLookup) *harness {
h := &harness{t: t}
h.kube = newFakeKube(map[string]string{"good": "alice", "bob-token": "bob"}, allow)
if kinds == nil {
kinds = newTestKinds(t, true, appDefinition("postgres", "Postgres", "postgreses"))
}
h.upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseForm(); err != nil {
t.Errorf("upstream ParseForm: %v", err)
@@ -90,7 +99,7 @@ func newHarness(t *testing.T, cfg Config, handler http.HandlerFunc) *harness {
if cfg.Logs == (LogsLimits{}) {
cfg.Logs = testLimits()
}
h.server = NewServer(cfg, NewAuthorizer(h.kube.client, 10*time.Second), mapStores{"tenant-a": "tenant-a-store"}, nil)
h.server = NewServer(cfg, NewAuthorizer(h.kube.client, 10*time.Second), mapStores{"tenant-a": "tenant-a-store"}, kinds, nil)
h.server.now = func() time.Time { return testNow }
h.proxy = httptest.NewServer(h.server)
t.Cleanup(h.proxy.Close)
@@ -154,8 +163,8 @@ func TestServerStatusCodes(t *testing.T) {
{"unknown metrics endpoint", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/api/v1/status/tsdb", "good", http.StatusNotFound},
{"metrics in another namespace", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-b/metrics/api/v1/query?query=up", "good", http.StatusForbidden},
{"metrics without token", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/api/v1/query?query=up", "", http.StatusUnauthorized},
{"application log not served yet", http.MethodGet, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/log/select/logsql/query", "good", http.StatusNotFound},
{"application metrics not served yet", http.MethodGet, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/metrics/api/v1/query", "good", http.StatusNotFound},
{"application log without a grant", http.MethodGet, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/log/select/logsql/query", "good", http.StatusForbidden},
{"application metrics without a grant", http.MethodGet, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/metrics/api/v1/query", "good", http.StatusForbidden},
{"tenant_ids not proxied", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/logs/select/tenant_ids", "good", http.StatusNotFound},
{"unknown logsql endpoint", http.MethodGet, logsRoot + "stats_query", "good", http.StatusNotFound},
{"logs root itself", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/logs/", "good", http.StatusNotFound},
@@ -15,6 +15,11 @@ rules:
- apiGroups: [""]
resources: ["namespaces"]
verbs: ["list", "watch"]
# Application roots map the plural in the path to the kind the lineage
# labels carry, from an informer cache as well.
- apiGroups: ["cozystack.io"]
resources: ["applicationdefinitions"]
verbs: ["list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
@@ -12,7 +12,7 @@ release:
namespace: cozy-observability-proxy
tests:
- it: lets the proxy review tokens and access, and list and watch namespaces
- it: lets the proxy review tokens and access, and list and watch namespaces and application definitions
template: templates/rbac.yaml
documentSelector:
path: kind
@@ -30,6 +30,9 @@ tests:
- apiGroups: [""]
resources: ["namespaces"]
verbs: ["list", "watch"]
- apiGroups: ["cozystack.io"]
resources: ["applicationdefinitions"]
verbs: ["list", "watch"]
- it: binds the proxy role to the proxy ServiceAccount in the release namespace
template: templates/rbac.yaml