From f2c2abfe8852aabcc356e4c30daa95bc0b6cb00e Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 16 Jul 2026 21:15:25 +0300 Subject: [PATCH] feat(postgres): publish the CA trust anchor to tenants Declare CNPG's operator-created CA Secret as the postgres trust-anchor source (spec.caCert names {{ .release }}-ca), and add the generic internal.cozystack.io/tenant-ca selector to spec.secrets so the key-free .tenant-ca projection the extraction controller publishes becomes readable by the tenant. postgres is the first converged engine, and the hardest: CNPG creates the CA Secret itself, key-bearing, asynchronously, and offers no way to label it, so it exercises the name-driven leg end to end. The test drives the shipped ApplicationDefinition rather than a fixture, so the wiring and the controller cannot drift apart unseen: a CNPG-shaped key-bearing -ca appears, and the projection must be produced key-free AND selected for tenant visibility by the shipped selector. Signed-off-by: Aleksei Sviridkin --- hack/e2e-chainsaw/postgres/chainsaw-test.yaml | 115 +++++++++++++++ .../controller/cacert/postgres_wiring_test.go | 132 ++++++++++++++++++ packages/apps/postgres/README.md | 8 +- .../templates/dashboard-resourcemap.yaml | 30 ++++ packages/apps/postgres/templates/db.yaml | 9 +- .../postgres/tests/tenant_ca_rbac_test.yaml | 84 +++++++++++ .../system/postgres-rd/cozyrds/postgres.yaml | 13 ++ 7 files changed, 386 insertions(+), 5 deletions(-) create mode 100644 internal/controller/cacert/postgres_wiring_test.go create mode 100644 packages/apps/postgres/tests/tenant_ca_rbac_test.yaml diff --git a/hack/e2e-chainsaw/postgres/chainsaw-test.yaml b/hack/e2e-chainsaw/postgres/chainsaw-test.yaml index 1b8c60020..6a6c9d1e0 100644 --- a/hack/e2e-chainsaw/postgres/chainsaw-test.yaml +++ b/hack/e2e-chainsaw/postgres/chainsaw-test.yaml @@ -124,6 +124,121 @@ spec: apiVersion: batch/v1 kind: Job name: postgres-test-init-job + + # The trust anchor a client verifies against, checked against the live + # operator rather than against our own beliefs about it. + # + # This is the only independent witness the CA-extraction feature has. Its unit + # tests build CNPG's CA Secret from a fixture that encodes the same assumption + # the shipped ApplicationDefinition encodes ("{{ .release }}-ca", key ca.crt), + # so the two agree by construction and cannot catch each other being wrong. + # Only a real CNPG can say whether that name and key are right, and only a real + # apiserver can say whether the lineage webhook actually promotes the + # projection to a tenant-readable object. + - name: verify-tenant-ca-projection + try: + # Postgres never opts out of TLS: CNPG mints its CA regardless of the + # chart's tls flag, so the projection is expected on this plain fixture. + # + # The labels are the assertion that matters. tenant-ca is stamped by the + # controller; tenantresource is stamped by the LINEAGE WEBHOOK, computed + # from the ApplicationDefinition's spec.secrets — the new matchLabels + # include. It is "true" only if the whole chain held: definition selects the + # label, webhook admitted the object, verdict came out positive. Nothing + # short of a live cluster proves that link. + - assert: + timeout: 2m + resource: + apiVersion: v1 + kind: Secret + metadata: + name: postgres-test.tenant-ca + labels: + internal.cozystack.io/tenant-ca: "true" + internal.cozystack.io/tenantresource: "true" + type: Opaque + # The projection must be the CA and ONLY the CA. Everything below is + # deterministic once the assert above has passed, so it runs without retry. + - script: + content: | + set -eu + + # Exactly one key, and it is ca.crt. Any second key is a leak in the + # making: the source this was lifted from carries a private key. + keys=$(kubectl -n "$NAMESPACE" get secret postgres-test.tenant-ca \ + -o go-template='{{range $k, $v := .data}}{{$k}}{{"\n"}}{{end}}' | sort | tr '\n' ' ') + if [ "$keys" != "ca.crt " ]; then + echo "projection must hold exactly ca.crt, holds: $keys" >&2 + exit 1 + fi + + # It lifted the right bytes: the projection's ca.crt is CNPG's ca.crt. + # This is what pins "{{ .release }}-ca" to reality — if CNPG ever + # renames its CA Secret, the controller finds nothing and this fails. + src=$(kubectl -n "$NAMESPACE" get secret postgres-test-ca -o jsonpath='{.data.ca\.crt}') + proj=$(kubectl -n "$NAMESPACE" get secret postgres-test.tenant-ca -o jsonpath='{.data.ca\.crt}') + if [ -z "$src" ]; then + echo "CNPG's postgres-test-ca has no ca.crt — the declared source name or key is wrong" >&2 + exit 1 + fi + if [ "$src" != "$proj" ]; then + echo "projection does not carry CNPG's CA certificate" >&2 + exit 1 + fi + + # And the point of the whole exercise: the source carries the CA + # PRIVATE KEY, the projection must not. Asserting the source really + # does hold ca.key keeps this honest — if CNPG ever stopped storing it + # there, this check would otherwise pass while proving nothing. + if ! kubectl -n "$NAMESPACE" get secret postgres-test-ca \ + -o go-template='{{if .data}}{{range $k, $v := .data}}{{$k}}{{"\n"}}{{end}}{{end}}' | grep -qx 'ca.key'; then + echo "CNPG's CA Secret no longer carries ca.key — this check has stopped proving anything" >&2 + exit 1 + fi + if kubectl -n "$NAMESPACE" get secret postgres-test.tenant-ca \ + -o go-template='{{if .data}}{{range $k, $v := .data}}{{$k}}{{"\n"}}{{end}}{{end}}' | grep -qx 'ca.key'; then + echo "projection leaked the CA private key" >&2 + exit 1 + fi + + # Belt and braces on the bytes themselves: no PEM key header of any + # flavour survived into the tenant-readable object. + if kubectl -n "$NAMESPACE" get secret postgres-test.tenant-ca \ + -o jsonpath='{.data.ca\.crt}' | base64 -d | grep -qi 'BEGIN .*PRIVATE KEY'; then + echo "projection carries PEM private key material" >&2 + exit 1 + fi + + # The delivery path the README actually tells a tenant to use, run as + # the command the README prints. Everything above reads the raw Secret, + # which only an admin can do — it proves the projection's SHAPE and + # nothing about whether a tenant can reach it. + # + # core.cozystack.io/tenantsecrets serves only objects labelled + # tenantresource=true, and that label is the lineage webhook's verdict, + # computed from this app's ApplicationDefinition selectors. So this + # query 404s unless the whole chain held: the definition selects + # tenant-ca, the webhook admitted the projection, and the verdict came + # out positive. It is the one assertion that exercises the registry + # rather than inspecting its input. + anchor=$(kubectl -n "$NAMESPACE" get tenantsecret postgres-test.tenant-ca \ + -o jsonpath='{.data.ca\.crt}' | base64 -d) + if ! echo "$anchor" | grep -q 'BEGIN CERTIFICATE'; then + echo "the trust anchor is not served through core.cozystack.io/tenantsecrets" >&2 + exit 1 + fi + + # And the negative that gives the positive its meaning: the registry + # must NOT serve CNPG's key-bearing CA. It carries no tenant-ca label, + # so the definition does not select it and the webhook marks it + # tenantresource=false. If this ever starts resolving, the label path + # has stopped gating on the platform's verdict. + if kubectl -n "$NAMESPACE" get tenantsecret postgres-test-ca >/dev/null 2>&1; then + echo "CNPG's key-bearing CA is being served as a tenant secret" >&2 + exit 1 + fi + check: + ($error == null): true # The Postgres CR itself was applied by chainsaw and is deleted (and waited # on) automatically during cleanup — no explicit teardown step needed. --- diff --git a/internal/controller/cacert/postgres_wiring_test.go b/internal/controller/cacert/postgres_wiring_test.go new file mode 100644 index 000000000..3d5000ba0 --- /dev/null +++ b/internal/controller/cacert/postgres_wiring_test.go @@ -0,0 +1,132 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cacert + +import ( + "os" + "path/filepath" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/yaml" + + cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" +) + +// postgresApplicationDefinitionPath is the SHIPPED postgres ApplicationDefinition, +// relative to this package. The test loads it rather than hand-building a copy so +// that the wiring and the controller cannot drift apart silently: if the caCert +// declaration or the secrets selector in the YAML changes, this test sees it. +var postgresApplicationDefinitionPath = filepath.Join( + "..", "..", "..", "packages", "system", "postgres-rd", "cozyrds", "postgres.yaml") + +func loadPostgresApplicationDefinition(t *testing.T) *cozyv1alpha1.ApplicationDefinition { + t.Helper() + raw, err := os.ReadFile(postgresApplicationDefinitionPath) + if err != nil { + t.Fatalf("read shipped postgres ApplicationDefinition: %v", err) + } + def := &cozyv1alpha1.ApplicationDefinition{} + if err := yaml.Unmarshal(raw, def); err != nil { + t.Fatalf("decode shipped postgres ApplicationDefinition: %v", err) + } + return def +} + +// TestPostgresWiring_EndToEnd is the acceptance test for the first converged +// engine, driving the SHIPPED ApplicationDefinition rather than a fixture. +// +// postgres is the validation engine on purpose: CNPG creates its CA Secret +// itself — -ca, holding ca.crt next to the CA PRIVATE KEY — offers no +// way to label it, and mints it asynchronously. That is the hardest input the +// controller has to handle, so proving it end to end here proves the mechanism +// for the simpler engines too. +// +// Both halves of "a tenant can read ca.crt" are asserted: the controller +// produces the key-free .tenant-ca projection, AND the shipped +// spec.secrets selector matches it — which is what makes the lineage webhook +// stamp tenantresource=true and the tenant able to read it. +func TestPostgresWiring_EndToEnd(t *testing.T) { + def := loadPostgresApplicationDefinition(t) + + // The declaration the controller depends on must actually be in the shipped + // YAML, and must name CNPG's operator-created CA Secret. + if def.Spec.CACert == nil { + t.Fatal("the shipped postgres ApplicationDefinition declares no caCert; the name-driven leg has nothing to resolve") + } + if def.Spec.CACert.SourceSecretName != "{{ .release }}-ca" { + t.Errorf("caCert.sourceSecretName = %q, want CNPG's CA Secret \"{{ .release }}-ca\"", def.Spec.CACert.SourceSecretName) + } + + // CNPG's operator-created CA: ca.crt next to the CA private key, no publish + // label, named after the release ({{ .release }}-ca = postgres-mydb-ca). + cnpgCA := operatorSecret(testOperatorCA, map[string][]byte{ + caCertKey: []byte(testCA), + caKeyKey: []byte(testKey), + }) + c := fake.NewClientBuilder().WithScheme(newScheme(t)).WithObjects( + helmRelease(), def, cnpgCA, + ).Build() + + mustReconcile(t, c) + + // Half one: the projection exists, holds ca.crt alone, and leaks no key. + proj := mustProjection(t, c) + assertKeyFree(t, proj, testCA) + + // Half two: the shipped secrets selector actually selects that projection. + // Without a matching include, the lineage webhook stamps tenantresource=false + // and the projection, though correct, is never readable by the tenant. + if !selectedForTenant(t, def, proj) { + t.Errorf("the shipped postgres secrets selector does not select the trust anchor %q (labels %v); a tenant could not read ca.crt", + proj.Name, proj.Labels) + } + + // The CNPG source is left byte-for-byte as the operator wrote it — key and all. + src, ok := getSecret(t, c, testOperatorCA) + if !ok { + t.Fatal("the operator's CA Secret must be kept") + } + if len(src.Data) != 2 || string(src.Data[caKeyKey]) != testKey { + t.Errorf("the operator's CA Secret must not be modified, got %v", keysOf(src.Data)) + } +} + +// selectedForTenant reports whether any spec.secrets include selector in the +// definition matches the projection's labels — the same question the lineage +// webhook answers when it decides the tenantresource verdict. +// +// A resourceNames-only include carries an empty LabelSelector, which +// LabelSelectorAsSelector turns into "matches everything"; that would make the +// check pass vacuously, so empty selectors are skipped and only a real label +// match counts. +func selectedForTenant(t *testing.T, def *cozyv1alpha1.ApplicationDefinition, proj *corev1.Secret) bool { + t.Helper() + for _, inc := range def.Spec.Secrets.Include { + sel, err := metav1.LabelSelectorAsSelector(&inc.LabelSelector) + if err != nil { + t.Fatalf("bad include selector in the shipped definition: %v", err) + } + if !sel.Empty() && sel.Matches(labels.Set(proj.Labels)) { + return true + } + } + return false +} diff --git a/packages/apps/postgres/README.md b/packages/apps/postgres/README.md index 771ffc2cd..f6e279e4d 100644 --- a/packages/apps/postgres/README.md +++ b/packages/apps/postgres/README.md @@ -177,14 +177,18 @@ The tri-state `tls.enabled` controls whether the chart injects `serverAltDNSName **Retrieving the CA bundle** for client verification: -CNPG bundles the CA certificate in every user-credentials Secret it creates under the key `ca.crt`. Retrieve it from the `-credentials` Secret, which is already accessible to tenants via the dashboard RBAC: +The trust anchor is published as `.tenant-ca`: an object holding `ca.crt` and nothing else, created for every release and delivered to tenants through the `core.cozystack.io/tenantsecrets` API that the base tenant roles already grant. ```bash kubectl --context --namespace \ - get secret -credentials \ + get tenantsecret .tenant-ca \ --output jsonpath='{.data.ca\.crt}' | base64 --decode ``` +`.tenant-ca` is the only object that hands over the CA certificate without also handing over a private key, which is why it exists. CNPG creates its own `-ca` Secret, but that one stores the CA **private key** (`ca.key`) alongside the certificate — read access to it would let the holder issue certificates for anything, so it is never granted to a tenant. + +It is reached through `tenantsecrets` rather than by reading the Secret directly, and that is deliberate: `tenantsecrets` surfaces only objects the platform has vouched for (those the application's definition selects), whereas a direct grant on the name would convey whatever happens to occupy that name. + **Connecting with full verification** (psql example): ```bash diff --git a/packages/apps/postgres/templates/dashboard-resourcemap.yaml b/packages/apps/postgres/templates/dashboard-resourcemap.yaml index d979289fc..60fbe7316 100644 --- a/packages/apps/postgres/templates/dashboard-resourcemap.yaml +++ b/packages/apps/postgres/templates/dashboard-resourcemap.yaml @@ -20,6 +20,36 @@ rules: resourceNames: - {{ .Release.Name }}-credentials verbs: ["get", "list", "watch"] +# .tenant-ca is deliberately NOT granted here, and the reason is the +# difference between granting a NAME and granting an OBJECT. +# +# This Role names Secrets. A name grant conveys whatever occupies the name, and +# the CA-extraction controller cannot promise what that is: it refuses to +# overwrite a Secret at the canonical name that it did not create, precisely +# because such a Secret may hold key material, and surfaces the refusal as a +# CACertSecretCollision Warning. Granting the name would hand the tenant read on +# exactly the object the controller just refused to touch — the collision guard +# would still fire, and be useless. +# +# A dash-separated name would make that acute rather than theoretical: +# -tenant-ca is exactly what CNPG's -ca resolves to for a +# sibling app named "-tenant", so the name could hold a key-BEARING CA and +# this Role would convey it. The canonical name's dot rules that out by +# construction — no engine can produce a dotted name — leaving the general +# argument, which is enough on its own: a name grant is gated on a string, and +# the controller cannot promise what the string holds. +# +# The trust anchor reaches the tenant through core.cozystack.io/tenantsecrets +# instead, which the base tenant roles already grant and which gates on the +# internal.cozystack.io/tenantresource LABEL — computed by the lineage webhook +# from this app's ApplicationDefinition selectors, and stamped only on an object +# that carries internal.cozystack.io/tenant-ca. A foreign Secret squatting on the +# name carries neither, so it is never surfaced. The label path is gated on what +# the platform vouched for; a name grant is gated on a string. +# +# kafka's resourcemap does grant its CA Secrets by name. That is where this was +# almost copied from, and it is not a model to follow: the same pattern is how +# kafka's clients-CA private key became tenant-readable. - apiGroups: - cozystack.io resources: diff --git a/packages/apps/postgres/templates/db.yaml b/packages/apps/postgres/templates/db.yaml index a59b137d1..5f89b07f5 100644 --- a/packages/apps/postgres/templates/db.yaml +++ b/packages/apps/postgres/templates/db.yaml @@ -17,9 +17,12 @@ spec: # admission webhook, which is why the chart no longer renders a cert-manager # chain — the operator owns the chain end-to-end. # - # Trust anchor for clients: the ca.crt key present in every user-credentials - # Secret CNPG creates (e.g. -credentials). Mount that key into - # client pods to verify TLS. + # Trust anchor for clients: the .tenant-ca Secret, an Opaque object + # holding ca.crt and nothing else. The CA-extraction controller + # (internal/controller/cacert) publishes it per release by lifting ca.crt out + # of CNPG's own -ca — which cannot be handed to a client directly, + # because CNPG stores the CA PRIVATE KEY (ca.key) in it. Mount ca.crt from + # .tenant-ca into client pods to verify TLS. certificates: serverAltDNSNames: - {{ printf "%s.%s" .Release.Name .Values._namespace.host }} diff --git a/packages/apps/postgres/tests/tenant_ca_rbac_test.yaml b/packages/apps/postgres/tests/tenant_ca_rbac_test.yaml new file mode 100644 index 000000000..5293c34ba --- /dev/null +++ b/packages/apps/postgres/tests/tenant_ca_rbac_test.yaml @@ -0,0 +1,84 @@ +suite: postgres trust-anchor RBAC + +# What a client needs to verify this cluster's TLS is ca.crt — and nothing else. +# +# CNPG creates its own -ca Secret and stores the CA PRIVATE KEY +# (ca.key) in it alongside the certificate, so read access to it would let the +# holder issue certificates for anything. It can never be granted to a tenant. +# The CA-extraction controller exists to break that coupling: it lifts ca.crt +# alone into the key-free .tenant-ca. +# +# This suite pins how that object is NOT delivered. The trust anchor reaches the +# tenant through core.cozystack.io/tenantsecrets, which gates on the +# tenantresource label the lineage webhook computes from this app's definition — +# so it conveys an object the platform vouched for. This Role grants Secrets by +# NAME, which conveys whatever occupies the name, and the two are not +# interchangeable: +# +# - the controller refuses to overwrite a Secret at the canonical name that it +# did not create, because such a Secret may hold key material; +# - a name grant would then hand the tenant read on the very object that guard +# just refused to touch, so the guard would fire and buy nothing. +# +# This was acute while the projection was -tenant-ca: that name is what +# CNPG's -ca resolves to for a sibling app named "-tenant", so it +# could hold a key-BEARING CA. The canonical name's dot ended that by +# construction — application names are DNS-1035 labels and cannot contain a dot, +# so no engine can produce this name for any app name. Stated in the past tense +# deliberately: the dot is not decoration, and a future reader must not "tidy" it +# back into a dash on the strength of a comment that reads as if the collision +# were still live. + +release: + name: pg-test + namespace: tenant-test + +templates: + - templates/dashboard-resourcemap.yaml + +tests: + # The dashboard Role stays scoped to credentials. Adding the trust anchor here + # would look harmless and would defeat the collision guard, so its absence is + # pinned rather than left to reviewer memory. + - it: grants tenant use-subjects read on the credentials Secret only + documentIndex: 0 + asserts: + - isKind: + of: Role + - contains: + path: rules + content: + apiGroups: + - "" + resources: + - secrets + resourceNames: + - pg-test-credentials + verbs: ["get", "list", "watch"] + + # Index-free on purpose: an assertion bound to rules[1] follows the position + # rather than the rule, and would silently point elsewhere the moment a rule + # was inserted above it. Select the secrets rule by what it is. + - it: never grants a name that a key-bearing CA can occupy + documentIndex: 0 + asserts: + # CNPG's own CA. One word away from the projection, and it holds ca.key. + - notContains: + path: rules[?(@.resources[0] == "secrets")].resourceNames + content: pg-test-ca + # The canonical projection name. Not granted here BY DESIGN: it is delivered + # by label through tenantsecrets, which conveys an object the platform + # vouched for, rather than by name, which conveys whatever occupies it. + - notContains: + path: rules[?(@.resources[0] == "secrets")].resourceNames + content: pg-test.tenant-ca + + # The grant is only worth anything if it reaches the tenant's subjects. + - it: binds the Role to the tenant use-subjects + documentIndex: 1 + asserts: + - isKind: + of: RoleBinding + - equal: + path: roleRef.name + value: pg-test-dashboard-resources diff --git a/packages/system/postgres-rd/cozyrds/postgres.yaml b/packages/system/postgres-rd/cozyrds/postgres.yaml index 766019a88..81a0bbc20 100644 --- a/packages/system/postgres-rd/cozyrds/postgres.yaml +++ b/packages/system/postgres-rd/cozyrds/postgres.yaml @@ -34,11 +34,24 @@ spec: # helm.toolkit.fluxcd.io/name: "{reqs[0]['metadata','name']}" keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "replicas"], ["spec", "resources"], ["spec", "resourcesPreset"], ["spec", "size"], ["spec", "storageClass"], ["spec", "external"], ["spec", "version"], ["spec", "tls"], ["spec", "postgresql"], ["spec", "postgresql", "parameters"], ["spec", "postgresql", "parameters", "max_connections"], ["spec", "quorum"], ["spec", "quorum", "minSyncReplicas"], ["spec", "quorum", "maxSyncReplicas"], ["spec", "users"], ["spec", "databases"], ["spec", "backup"], ["spec", "backup", "enabled"], ["spec", "backup", "useSystemBucket"], ["spec", "backup", "retentionPolicy"], ["spec", "backup", "destinationPath"], ["spec", "backup", "endpointURL"], ["spec", "backup", "schedule"], ["spec", "backup", "s3AccessKey"], ["spec", "backup", "s3SecretKey"], ["spec", "backup", "s3CredentialsSecret"], ["spec", "backup", "s3CredentialsSecret", "name"], ["spec", "backup", "s3CredentialsSecret", "accessKeyIDKey"], ["spec", "backup", "s3CredentialsSecret", "secretAccessKeyKey"], ["spec", "backup", "endpointCA"], ["spec", "backup", "endpointCA", "name"], ["spec", "backup", "endpointCA", "key"], ["spec", "bootstrap"], ["spec", "bootstrap", "enabled"], ["spec", "bootstrap", "recoveryTime"], ["spec", "bootstrap", "oldName"], ["spec", "bootstrap", "serverName"]] + # CNPG creates its CA Secret itself — -ca, holding ca.crt next to the + # CA private key — and offers no way to label it, so the trust anchor is + # declared by name here. The CA-extraction controller lifts ca.crt alone into + # the key-free .tenant-ca Secret; the include selector below then + # makes that projection readable by the tenant. + caCert: + sourceSecretName: "{{ .release }}-ca" + sourceKey: ca.crt secrets: exclude: [] include: - resourceNames: - postgres-{{ .name }}-credentials + # The key-free trust anchor the CA-extraction controller publishes. This + # is the single, engine-agnostic selector every converged engine shares; + # it carries no per-release name templating. + - matchLabels: + internal.cozystack.io/tenant-ca: "true" services: exclude: [] include: