From ae29f07a13de60fa01fa2dc9115410a6d57a36db Mon Sep 17 00:00:00 2001 From: Timofei Larkin Date: Wed, 14 Oct 2026 12:29:46 +0200 Subject: [PATCH] feat(api): serve credentials with mint and revoke, and seed accounts The Credential view lists the account Secrets of the namespace by label and skips any no live HelmRelease owns, so no kind is served until its definition carries a credentials block. A mint replaces the stored value in one conditional write keyed on the Secret's resourceVersion and returns the plaintext once; a revoke discards it. Both record an Event on the application, which needs create on events. The application registry seeds an account Secret per declared account before it writes the HelmRelease, so the chart can reference them from the first render, prunes the Secrets of removed accounts after the write, and refuses to delete an application whose release is suspended, since Flux skips the uninstall and would leave the database running with the Secrets gone. The ApplicationDefinition reconciler computes the per-account valuesFrom entries with the function the API uses, so a release that predates the block gets them on its next reconcile. Schema defaulting moves to a package both can import. A definition declaring a format or an account source that is not built leaves its kind unserved rather than served with passwords the API cannot issue. Tenant roles gain get and list on credentials at view and create on mint and revoke at admin and for the tenant ServiceAccount. Attribution comes from the kube-apiserver audit log, so the operations docs get a policy for distributions that ship none, and monitoring-agents a value for an extra fluent-bit output so a SIEM can receive the log without dropping the VictoriaLogs outputs. Assisted-by: LLM Signed-off-by: Timofei Larkin --- cmd/backport-batch/test-inputs.txt | 1 + docs/operations/audit-log.md | 72 ++++ internal/apigate/parse.go | 6 +- internal/apigate/parse_test.go | 4 + .../applicationdefinition_helmreconciler.go | 38 +- ...plicationdefinition_helmreconciler_test.go | 50 +++ .../system/cozystack-api/templates/rbac.yaml | 5 + .../system/cozystack-api/tests/rbac_test.yaml | 19 +- .../templates/clusterroles.yaml | 28 ++ .../tests/clusterroles-credentials_test.yaml | 69 ++++ .../tests/fluent-bit_extra_outputs_test.yaml | 33 ++ packages/system/monitoring-agents/values.yaml | 9 + pkg/apiserver/apiserver.go | 24 +- pkg/cmd/server/start.go | 12 +- pkg/cmd/server/start_owner_test.go | 42 +++ pkg/config/config.go | 6 + pkg/credentials/store/event.go | 59 +++ pkg/credentials/store/resolve.go | 39 ++ pkg/credentials/store/store.go | 237 ++++++++++++ pkg/credentials/store/store_test.go | 248 +++++++++++++ pkg/registry/apps/application/rest.go | 60 +++- .../apps/application/rest_credentials.go | 180 ++++++++++ .../apps/application/rest_credentials_test.go | 219 ++++++++++++ pkg/registry/core/credential/rest.go | 336 ++++++++++++++++++ pkg/registry/core/credential/rest_test.go | 309 ++++++++++++++++ pkg/registry/core/credential/subresources.go | 210 +++++++++++ 26 files changed, 2282 insertions(+), 33 deletions(-) create mode 100644 docs/operations/audit-log.md create mode 100644 packages/system/cozystack-basics/tests/clusterroles-credentials_test.yaml create mode 100644 packages/system/monitoring-agents/tests/fluent-bit_extra_outputs_test.yaml create mode 100644 pkg/credentials/store/event.go create mode 100644 pkg/credentials/store/resolve.go create mode 100644 pkg/credentials/store/store.go create mode 100644 pkg/credentials/store/store_test.go create mode 100644 pkg/registry/apps/application/rest_credentials.go create mode 100644 pkg/registry/apps/application/rest_credentials_test.go create mode 100644 pkg/registry/core/credential/rest.go create mode 100644 pkg/registry/core/credential/rest_test.go create mode 100644 pkg/registry/core/credential/subresources.go diff --git a/cmd/backport-batch/test-inputs.txt b/cmd/backport-batch/test-inputs.txt index 4262b0e1f..d76bb0fab 100644 --- a/cmd/backport-batch/test-inputs.txt +++ b/cmd/backport-batch/test-inputs.txt @@ -16,6 +16,7 @@ go.mod go.sum # In-repo packages the tool and its tests import. +api/v1alpha1/ internal/apigate/ pkg/apis/apps/v1alpha1/ pkg/apis/apps/validation/ diff --git a/docs/operations/audit-log.md b/docs/operations/audit-log.md new file mode 100644 index 000000000..e9011ab2f --- /dev/null +++ b/docs/operations/audit-log.md @@ -0,0 +1,72 @@ +# Kubernetes Audit Log + +Cozystack attributes every issuance and revocation of a managed application password to the kube-apiserver audit log. A mint is a `create` on the `mint` subresource of a `core.cozystack.io` Credential, which kube-apiserver audits before it proxies the request to the Cozystack API, recording the caller, the time, the account in `objectRef` and the subresource that tells a mint from a revoke. kube-apiserver records neither the body nor the response of a request it proxies to an aggregated API, at any audit level, so the password shown once in the mint response cannot reach the log. The Cozystack API itself runs without audit flags, and it must stay that way or get a policy that keeps `credentials` below `RequestResponse`, because it is the one server that could write the password down. + +The `monitoring-agents` package tails `/var/log/audit/kube/*.log` on every node and ships it to the VictoriaLogs instance in the `tenant-root` tenant, tagged `audit.*`, where it is kept for a month by default. Only `tenant-root` can read that store. Every other tenant sees the last ten operations on each of its accounts in the Credential's `status.history`, and each mint and revoke also records a Kubernetes Event on the application, with reason `CredentialIssued` or `CredentialRevoked`, naming the account and the caller. + +## Distributions that ship no audit policy + +Talos Linux configures kube-apiserver with an audit policy and writes the log under `/var/log/audit/kube/` out of the box, which is where the default fluent-bit input reads it. A cluster built on a distribution that gives kube-apiserver no audit policy records nothing, so attribution is lost silently. Give it the policy below, or merge these rules into the policy it has, and point the log at the directory fluent-bit tails. + +```yaml +apiVersion: audit.k8s.io/v1 +kind: Policy +omitStages: + - RequestReceived +rules: + # Who issued or revoked which account, and when. Metadata is all there is to + # record for a proxied request, and all there must ever be for this resource. + - level: Metadata + resources: + - group: core.cozystack.io + resources: + - credentials + - credentials/mint + - credentials/revoke + # Account Secrets and the tenantsecrets view carry stored password forms; + # Metadata keeps their data out of the log. + - level: Metadata + resources: + - group: "" + resources: + - secrets + - group: core.cozystack.io + resources: + - tenantsecrets + - level: Metadata +``` + +``` +--audit-policy-file=/etc/kubernetes/audit-policy.yaml +--audit-log-path=/var/log/audit/kube/kube-apiserver.log +--audit-log-maxage=30 +--audit-log-maxbackup=10 +--audit-log-maxsize=100 +``` + +The log path has to be a file under the node's `/var/log/audit/kube/` directory, because the fluent-bit DaemonSet mounts the node's `/var/log` and tails that glob. A distribution that writes the log elsewhere needs `fluent-bit.config.inputs` in the `monitoring-agents` values changed to match. + +## Finding an issuance + +In VictoriaLogs, the stream fields of the audit output include `verb` and `user_username`, and the record keeps the whole audit event, so a mint is found by its subresource: + +``` +objectRef.subresource:mint AND objectRef.resource:credentials +``` + +The matching record names the Credential in `objectRef.name`, the caller in `user.username`, the impersonated user where Kubernetes impersonation was used, the source address, the response code and the authorization decision. The user is a person only when the dashboard signs in through OIDC; without it, the dashboard acts as the tenant ServiceAccount. + +## Shipping the log to a SIEM + +The fluent-bit outputs of `monitoring-agents` are one string, so replacing `config.outputs` to add a destination drops the VictoriaLogs outputs with it. Add an output under `fluent-bit.extraOutputs` instead, which is appended after them. The audit records carry the tag `audit.*`: + +```yaml +fluent-bit: + extraOutputs: | + [OUTPUT] + Name forward + Match audit.* + Host siem.example.org + Port 24224 + tls on +``` diff --git a/internal/apigate/parse.go b/internal/apigate/parse.go index 0fe248f6c..5fa007dc1 100644 --- a/internal/apigate/parse.go +++ b/internal/apigate/parse.go @@ -172,10 +172,12 @@ var storageVarGroups = map[string]string{ // coreV1alpha1Storage["tenantsecrets"] = ... // // The plural class allows hyphens so a future hyphenated resource name is not -// silently dropped. The apps storage map is indexed by a variable +// silently dropped, and a slash so a subresource registration such as +// `credentials/mint` is gated as a resource of its own rather than dropped. +// The apps storage map is indexed by a variable // (appsV1alpha1Storage[resConfig.Application.Plural]), not a string literal, so // it never matches here — the apps API is covered by the cozyrds instead. -var apiserverStorageRe = regexp.MustCompile(`(\w+Storage)\["([a-z0-9-]+)"\]`) +var apiserverStorageRe = regexp.MustCompile(`(\w+Storage)\["([a-z0-9/-]+)"\]`) // unmappedStoragePrefix labels resources whose storage variable prefix is not // in storageVarGroups. Rather than silently dropping them — which would let a diff --git a/internal/apigate/parse_test.go b/internal/apigate/parse_test.go index 8c6ba577d..9bd254f2b 100644 --- a/internal/apigate/parse_test.go +++ b/internal/apigate/parse_test.go @@ -190,6 +190,7 @@ metadata: func TestParseAPIServerStorages(t *testing.T) { src := ` coreV1alpha1Storage["tenantsecrets"] = cozyregistry.RESTInPeace(x) + coreV1alpha1Storage["credentials/mint"] = credentialstorage.NewMintREST(w) sdnV1alpha1Storage["securitygroups"] = cozyregistry.RESTInPeace(y) appsV1alpha1Storage[resConfig.Application.Plural] = cozyregistry.RESTInPeace(z) ` @@ -204,6 +205,9 @@ func TestParseAPIServerStorages(t *testing.T) { if got["securitygroups"] != "sdn.cozystack.io" { t.Fatalf("expected sdn securitygroups, got %v", got) } + if got["credentials/mint"] != "core.cozystack.io" { + t.Fatalf("expected the subresource registration to be gated, got %v", got) + } if _, leaked := got["Plural"]; leaked { t.Fatalf("apps dynamic registration must not be captured: %v", got) } diff --git a/internal/controller/applicationdefinition_helmreconciler.go b/internal/controller/applicationdefinition_helmreconciler.go index 91011500e..86d300839 100644 --- a/internal/controller/applicationdefinition_helmreconciler.go +++ b/internal/controller/applicationdefinition_helmreconciler.go @@ -7,7 +7,9 @@ import ( cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" "github.com/cozystack/cozystack/internal/shared/appdefowner" + "github.com/cozystack/cozystack/pkg/apis/apps/defaulting" "github.com/cozystack/cozystack/pkg/config" + "github.com/cozystack/cozystack/pkg/credentials/store" helmv2 "github.com/fluxcd/helm-controller/api/v2" corev1 "k8s.io/api/core/v1" @@ -151,14 +153,33 @@ func (r *ApplicationDefinitionHelmReconciler) updateHelmReleasesForAppDef(ctx co return errors.Join(errs...) } -// expectedValuesFrom returns the expected valuesFrom configuration for HelmReleases +// expectedValuesFrom returns the valuesFrom a release of a kind without a +// credentials block carries. func expectedValuesFrom() []helmv2.ValuesReference { - return []helmv2.ValuesReference{ - { - Kind: "Secret", - Name: "cozystack-values", - }, + return store.ValuesFrom(nil, "", nil) +} + +// releaseValuesFrom returns the valuesFrom a release should carry under its +// definition: the platform values, and for a kind whose block names a +// valuesPath one entry per account, so a release created before the block +// appeared gets the entries on its next reconcile. The accounts are resolved +// from the release's values with the definition's schema defaults applied, +// as the API resolves them on a write. An error means the values declare +// accounts the API would refuse, and leaves the release as it is. +func releaseValuesFrom(hr *helmv2.HelmRelease, appDef *cozyv1alpha1.ApplicationDefinition) ([]helmv2.ValuesReference, error) { + block := appDef.Spec.Credentials + if block == nil || block.ValuesPath == "" { + return store.ValuesFrom(block, hr.Name, nil), nil } + schema, err := defaulting.BuildSchema(appDef.Spec.Application.OpenAPISchema) + if err != nil { + return nil, fmt.Errorf("build schema: %w", err) + } + accounts, err := store.ResolveAccounts(block, schema, hr.Spec.Values) + if err != nil { + return nil, err + } + return store.ValuesFrom(block, hr.Name, accounts), nil } // valuesFromEqual compares two ValuesReference slices @@ -211,8 +232,9 @@ func (r *ApplicationDefinitionHelmReconciler) updateHelmReleaseChart(ctx context } // Check and update valuesFrom configuration - expected := expectedValuesFrom() - if !valuesFromEqual(hrCopy.Spec.ValuesFrom, expected) { + if expected, err := releaseValuesFrom(hr, appDef); err != nil { + logger.Error(err, "Skipping HelmRelease valuesFrom update: accounts cannot be resolved", "name", hr.Name, "namespace", hr.Namespace) + } else if !valuesFromEqual(hrCopy.Spec.ValuesFrom, expected) { logger.V(4).Info("Updating HelmRelease valuesFrom", "name", hr.Name, "namespace", hr.Namespace) hrCopy.Spec.ValuesFrom = expected updated = true diff --git a/internal/controller/applicationdefinition_helmreconciler_test.go b/internal/controller/applicationdefinition_helmreconciler_test.go index 33fc871b4..ac18211f6 100644 --- a/internal/controller/applicationdefinition_helmreconciler_test.go +++ b/internal/controller/applicationdefinition_helmreconciler_test.go @@ -10,8 +10,11 @@ import ( cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" "github.com/cozystack/cozystack/pkg/config" + "github.com/cozystack/cozystack/pkg/credentials/account" + "github.com/cozystack/cozystack/pkg/credentials/store" helmv2 "github.com/fluxcd/helm-controller/api/v2" "github.com/fluxcd/pkg/apis/kustomize" + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -959,3 +962,50 @@ func TestAppDefHelm_UpdateFailureRequeues(t *testing.T) { t.Fatalf("expected %s synced past the failed %s, got %+v", written, failed, got.Spec.HealthCheckExprs) } } + +// TestAppDefHelm_CredentialsValuesFrom pins that a definition with a +// credentials block naming a valuesPath gives every release of the kind one +// valuesFrom entry per account, resolved with the schema defaults applied, so +// a release created before the block appeared gets them on reconcile. +func TestAppDefHelm_CredentialsValuesFrom(t *testing.T) { + scheme := newAppDefHelmScheme(t) + appDef := vmInstanceAppDef(nil) + appDef.Spec.Application.OpenAPISchema = `{"type":"object","properties":{"backup":{"type":"object","default":{},"properties":{"enabled":{"type":"boolean","default":true}}}}}` + appDef.Spec.Credentials = &cozyv1alpha1.ApplicationDefinitionCredentials{ + Format: cozyv1alpha1.CredentialFormat{Name: "scram-sha-256"}, + ValuesPath: "_accounts", + Accounts: []cozyv1alpha1.CredentialAccountSource{ + {Kind: cozyv1alpha1.CredentialAccountsValuesMap, Path: "users"}, + {Kind: cozyv1alpha1.CredentialAccountsStatic, Name: "backup", When: "backup.enabled"}, + }, + } + hr := vmInstanceRelease(nil, nil) + hr.Spec.ValuesFrom = expectedValuesFrom() + hr.Spec.Values = &apiextv1.JSON{Raw: []byte(`{"users":{"web":{}}}`)} + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(appDef, hr).Build() + + r := &ApplicationDefinitionHelmReconciler{Client: fakeClient, Scheme: scheme} + if _, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Name: appDef.Name}}); err != nil { + t.Fatalf("unexpected error: %v", err) + } + got := &helmv2.HelmRelease{} + if err := fakeClient.Get(context.TODO(), types.NamespacedName{Name: hr.Name, Namespace: hr.Namespace}, got); err != nil { + t.Fatalf("get HR: %v", err) + } + want := store.ValuesFrom(appDef.Spec.Credentials, hr.Name, []account.Account{{Username: "web", Mapped: "web"}, {Username: "backup", Mapped: "backup"}}) + if !valuesFromEqual(got.Spec.ValuesFrom, want) { + t.Fatalf("valuesFrom = %+v, want %+v", got.Spec.ValuesFrom, want) + } + + // Idempotent once in line. + before := got.ResourceVersion + if _, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Name: appDef.Name}}); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if err := fakeClient.Get(context.TODO(), types.NamespacedName{Name: hr.Name, Namespace: hr.Namespace}, got); err != nil { + t.Fatalf("get HR: %v", err) + } + if got.ResourceVersion != before { + t.Fatalf("release rewritten with nothing to change: %q -> %q", before, got.ResourceVersion) + } +} diff --git a/packages/system/cozystack-api/templates/rbac.yaml b/packages/system/cozystack-api/templates/rbac.yaml index b5501b8cc..70f1bec97 100644 --- a/packages/system/cozystack-api/templates/rbac.yaml +++ b/packages/system/cozystack-api/templates/rbac.yaml @@ -18,6 +18,11 @@ rules: - apiGroups: [""] resources: ["secrets"] verbs: ["create", "update", "patch", "delete"] +# A mint or revoke records an Event on the application naming the account and +# the caller, never the value. +- apiGroups: [""] + resources: ["events"] + verbs: ["create"] - apiGroups: ["admissionregistration.k8s.io"] resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations", "validatingadmissionpolicies", "validatingadmissionpolicybindings"] verbs: ["get", "watch", "list"] diff --git a/packages/system/cozystack-api/tests/rbac_test.yaml b/packages/system/cozystack-api/tests/rbac_test.yaml index 18d4eb460..a919fedb8 100644 --- a/packages/system/cozystack-api/tests/rbac_test.yaml +++ b/packages/system/cozystack-api/tests/rbac_test.yaml @@ -16,8 +16,8 @@ release: # # The last two tests bound the role from two directions, and neither covers the # other. The count bound pins the list at exactly twenty-one: templates/rbac.yaml -# carries no template directives, so anything appended makes rules[21] exist and -# anything dropped takes rules[20] away, whatever the rule is called. It says +# carries no template directives, so anything appended makes rules[22] exist and +# anything dropped takes rules[21] away, whatever the rule is called. It says # nothing about a grant swapped in for an existing one, because the count does # not move under a swap. The spelling assert catches that case for the one rule # it names, and misses every other wording of it, because helm-unittest compares @@ -85,11 +85,20 @@ tests: resources: ["bucketclasses"] verbs: ["get", "watch", "list"] - - it: ships exactly the twenty-one rules this role is pinned to + - it: grants create on events for the credential issued and revoked records + asserts: + - contains: + path: rules + content: + apiGroups: [""] + resources: ["events"] + verbs: ["create"] + + - it: ships exactly the twenty-two rules this role is pinned to asserts: # Both directions. A dropped grant is the failure this suite is about: # the apiserver returns forbidden and List silently drops the dropdown. - exists: - path: rules[20] - - notExists: path: rules[21] + - notExists: + path: rules[22] diff --git a/packages/system/cozystack-basics/templates/clusterroles.yaml b/packages/system/cozystack-basics/templates/clusterroles.yaml index 5df762886..914ab5f5c 100644 --- a/packages/system/cozystack-basics/templates/clusterroles.yaml +++ b/packages/system/cozystack-basics/templates/clusterroles.yaml @@ -50,6 +50,19 @@ rules: - options - taps verbs: ["get", "list", "watch"] +- apiGroups: + - core.cozystack.io + resources: + - credentials + verbs: ["get", "list"] +# The tenant ServiceAccount is what the dashboard signs in with when OIDC is +# off, so it issues and revokes passwords as a tenant admin does. +- apiGroups: + - core.cozystack.io + resources: + - credentials/mint + - credentials/revoke + verbs: ["create"] --- # == view cluster role == # Aggregates all roles labeled for view access @@ -138,6 +151,13 @@ rules: - options - taps verbs: ["get", "list", "watch"] +{{/* A Credential is the record of one account: who set its password and when, + never the value. Mint and revoke are separate subresources granted at admin. */}} +- apiGroups: + - core.cozystack.io + resources: + - credentials + verbs: ["get", "list"] {{/* Read-only visibility into the objects that govern read-replica autoscaling of a tenant's own databases: the KEDA ScaledObject the chart renders into the tenant namespace and the keda-hpa- HorizontalPodAutoscaler KEDA creates beside @@ -301,6 +321,14 @@ rules: - update - patch - delete +# A mint replaces an account's password and shows the new one once, so it is +# an admin's write, not a reader's; a custom role can grant revoke alone. +- apiGroups: ["core.cozystack.io"] + resources: + - credentials/mint + - credentials/revoke + verbs: + - create --- # == super admin cluster role == # Aggregates admin + all roles labeled for super-admin access diff --git a/packages/system/cozystack-basics/tests/clusterroles-credentials_test.yaml b/packages/system/cozystack-basics/tests/clusterroles-credentials_test.yaml new file mode 100644 index 000000000..bf6d5c801 --- /dev/null +++ b/packages/system/cozystack-basics/tests/clusterroles-credentials_test.yaml @@ -0,0 +1,69 @@ +suite: cozystack-basics tenant ClusterRoles grant the credentials API + +templates: + - templates/clusterroles.yaml + +release: + name: cozystack-basics + namespace: cozy-system + +# A Credential is the view of one managed application account, and a mint on +# it replaces the password and shows the new one once. Reading the view is a +# view-tier grant; minting and revoking are admin writes, and the tenant +# ServiceAccount, which the dashboard signs in with when OIDC is off, holds +# both. The subresources are gated by verb create on their own resource +# names, so a grant on credentials alone issues nothing. +tests: + - it: cozy:tenant:view:base grants read on core.cozystack.io credentials + documentSelector: + path: metadata.name + value: cozy:tenant:view:base + asserts: + - contains: + path: rules + content: + apiGroups: ["core.cozystack.io"] + resources: ["credentials"] + verbs: ["get", "list"] + + - it: cozy:tenant:view:base does not grant mint or revoke + documentSelector: + path: metadata.name + value: cozy:tenant:view:base + asserts: + - notContains: + path: rules + content: + apiGroups: ["core.cozystack.io"] + resources: ["credentials/mint", "credentials/revoke"] + verbs: ["create"] + + - it: cozy:tenant:admin:base grants create on credentials/mint and credentials/revoke + documentSelector: + path: metadata.name + value: cozy:tenant:admin:base + asserts: + - contains: + path: rules + content: + apiGroups: ["core.cozystack.io"] + resources: ["credentials/mint", "credentials/revoke"] + verbs: ["create"] + + - it: cozy:tenant:base grants read on credentials and create on mint and revoke + documentSelector: + path: metadata.name + value: cozy:tenant:base + asserts: + - contains: + path: rules + content: + apiGroups: ["core.cozystack.io"] + resources: ["credentials"] + verbs: ["get", "list"] + - contains: + path: rules + content: + apiGroups: ["core.cozystack.io"] + resources: ["credentials/mint", "credentials/revoke"] + verbs: ["create"] diff --git a/packages/system/monitoring-agents/tests/fluent-bit_extra_outputs_test.yaml b/packages/system/monitoring-agents/tests/fluent-bit_extra_outputs_test.yaml new file mode 100644 index 000000000..e9ec52b34 --- /dev/null +++ b/packages/system/monitoring-agents/tests/fluent-bit_extra_outputs_test.yaml @@ -0,0 +1,33 @@ +suite: fluent-bit appends extra outputs after the VictoriaLogs outputs + +# config.outputs is one string, so an operator who needs a second destination +# for the audit log, such as a SIEM, could only replace the whole string and +# lose the VictoriaLogs outputs. extraOutputs is appended to them instead. +templates: + - charts/fluent-bit/templates/configmap.yaml +release: + name: monitoring-agents + namespace: cozy-monitoring +tests: + - it: renders the VictoriaLogs outputs alone by default + asserts: + - matchRegex: + path: data["fluent-bit.conf"] + pattern: 'Match audit\.\*\n\s+Host vlinsert-generic\.cozy-monitoring\.svc' + - notMatchRegex: + path: data["fluent-bit.conf"] + pattern: 'siem\.example\.org' + + - it: appends extraOutputs after the VictoriaLogs outputs + set: + fluent-bit: + extraOutputs: | + [OUTPUT] + Name forward + Match audit.* + Host siem.example.org + Port 24224 + asserts: + - matchRegex: + path: data["fluent-bit.conf"] + pattern: 'Match audit\.\*\n\s+Host vlinsert-generic\.cozy-monitoring\.svc[\s\S]*\[OUTPUT\]\n\s+Name forward\n\s+Match audit\.\*\n\s+Host siem\.example\.org\n\s+Port 24224' diff --git a/packages/system/monitoring-agents/values.yaml b/packages/system/monitoring-agents/values.yaml index b6c046ea5..531e6a57d 100644 --- a/packages/system/monitoring-agents/values.yaml +++ b/packages/system/monitoring-agents/values.yaml @@ -367,6 +367,12 @@ fluent-bit: - name: varlibdockercontainers mountPath: /var/lib/docker/containers readOnly: true + # Additional fluent-bit [OUTPUT] sections, appended after the VictoriaLogs + # outputs above and before the tenant routes. Overriding config.outputs + # replaces those outputs, so a security team shipping the audit log + # (tagged audit.*) to its own SIEM adds its output here instead. Rendered + # through tpl with the fluent-bit subchart's context. + extraOutputs: "" config: inputs: | [INPUT] @@ -438,6 +444,9 @@ fluent-bit: json_date_format iso8601 header AccountID 0 header ProjectID 0 + {{- with .Values.extraOutputs }} + {{ . | nindent 6 }} + {{- end }} {{- range $target, $_ := include "monitoring-agents.tenantLogRoutes" . | fromJson }} [OUTPUT] Name http diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index 06f024bc6..7af717527 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -47,6 +47,7 @@ import ( "github.com/cozystack/cozystack/pkg/config" cozyregistry "github.com/cozystack/cozystack/pkg/registry" applicationstorage "github.com/cozystack/cozystack/pkg/registry/apps/application" + credentialstorage "github.com/cozystack/cozystack/pkg/registry/core/credential" optionstorage "github.com/cozystack/cozystack/pkg/registry/core/option" tapstorage "github.com/cozystack/cozystack/pkg/registry/core/tap" tenantmodulestorage "github.com/cozystack/cozystack/pkg/registry/core/tenantmodule" @@ -213,6 +214,19 @@ func (c completedConfig) New() (*CozyServer, error) { if err != nil { return nil, fmt.Errorf("failed to build dynamic client: %w", err) } + // --- dynamically-configured, per-tenant resources --- + appsV1alpha1Storage := map[string]rest.Storage{} + // credentialKinds carries every converted application kind, by kind name, + // for the credentials view that is served from the core group below. + credentialKinds := map[string]credentialstorage.Kind{} + for _, resConfig := range c.ResourceConfig.Resources { + storage := applicationstorage.NewREST(cli, watchCli, &resConfig) + appsV1alpha1Storage[resConfig.Application.Plural] = cozyregistry.RESTInPeace(storage) + if block := storage.Credentials(); block != nil { + credentialKinds[resConfig.Application.Kind] = credentialstorage.Kind{Block: block, Accounts: storage.AccountsOf} + } + } + // --- static, cluster-scoped resource for core group --- coreV1alpha1Storage := map[string]rest.Storage{} coreV1alpha1Storage["tenantnamespaces"] = cozyregistry.RESTInPeace( @@ -233,6 +247,10 @@ func (c completedConfig) New() (*CozyServer, error) { coreV1alpha1Storage["taps"] = cozyregistry.RESTInPeace( tapstorage.NewREST(dyn), ) + credentials := credentialstorage.NewREST(cli, credentialKinds) + coreV1alpha1Storage["credentials"] = cozyregistry.RESTInPeace(credentials) + coreV1alpha1Storage["credentials/mint"] = credentialstorage.NewMintREST(credentials) + coreV1alpha1Storage["credentials/revoke"] = credentialstorage.NewRevokeREST(credentials) coreApiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(core.GroupName, Scheme, metav1.ParameterCodec, Codecs) coreApiGroupInfo.VersionedResourcesStorageMap["v1alpha1"] = coreV1alpha1Storage @@ -252,12 +270,6 @@ func (c completedConfig) New() (*CozyServer, error) { return nil, err } - // --- dynamically-configured, per-tenant resources --- - appsV1alpha1Storage := map[string]rest.Storage{} - for _, resConfig := range c.ResourceConfig.Resources { - storage := applicationstorage.NewREST(cli, watchCli, &resConfig) - appsV1alpha1Storage[resConfig.Application.Plural] = cozyregistry.RESTInPeace(storage) - } if err := InstallAppsAPIGroup(s.GenericAPIServer, appsV1alpha1Storage); err != nil { return nil, err } diff --git a/pkg/cmd/server/start.go b/pkg/cmd/server/start.go index 2a4d34e05..a92ee21c3 100644 --- a/pkg/cmd/server/start.go +++ b/pkg/cmd/server/start.go @@ -34,6 +34,7 @@ import ( sdnv1alpha1 "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1" "github.com/cozystack/cozystack/pkg/apiserver" "github.com/cozystack/cozystack/pkg/config" + "github.com/cozystack/cozystack/pkg/credentials/account" "github.com/spf13/cobra" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" @@ -288,6 +289,14 @@ func resourceConfigFrom(defs []v1alpha1.ApplicationDefinition, hrFlags helmRelea } resourceConfig := &config.ResourceConfig{} for _, crd := range owned { + // A credentials block the API cannot honour, declaring a format or a + // source kind that is not built or a format it does not know, leaves + // the kind unserved rather than served with passwords it cannot + // issue, as a kind with an invalid name schema refuses writes. + if err := account.Validate(crd.Spec.Credentials); err != nil { + fmt.Printf("Skipping ApplicationDefinition %q: credentials block cannot be served: %v\n", crd.Name, err) + continue + } resource, err := buildResourceFromCRD(crd, hrFlags) if err != nil { return nil, err @@ -408,7 +417,8 @@ func buildResourceFromCRD(crd v1alpha1.ApplicationDefinition, hrFlags helmReleas ShortNames: []string{}, // TODO: implement shortnames OpenAPISchema: crd.Spec.Application.OpenAPISchema, }, - Release: release, + Release: release, + Credentials: crd.Spec.Credentials, }, nil } diff --git a/pkg/cmd/server/start_owner_test.go b/pkg/cmd/server/start_owner_test.go index c1f62bd6b..9cdddbf07 100644 --- a/pkg/cmd/server/start_owner_test.go +++ b/pkg/cmd/server/start_owner_test.go @@ -62,3 +62,45 @@ func TestResourceConfigFromRegistersOwnerOnly(t *testing.T) { t.Fatalf("resources = %+v, want the owner's redis-artifact only", rc.Resources) } } + +// TestResourceConfigFromSkipsUnservableCredentials pins that a definition +// whose credentials block the API cannot honour leaves its kind unserved, +// while a block it can honour reaches the resource config. +func TestResourceConfigFromSkipsUnservableCredentials(t *testing.T) { + mk := func(name, kind string, block *v1alpha1.ApplicationDefinitionCredentials) v1alpha1.ApplicationDefinition { + d := v1alpha1.ApplicationDefinition{ObjectMeta: metav1.ObjectMeta{Name: name}} + d.Spec.Application.Kind = kind + d.Spec.Release.ChartRef = &helmv2.CrossNamespaceSourceReference{Kind: "ExternalArtifact", Name: name, Namespace: "cozy-system"} + d.Spec.Credentials = block + return d + } + served := &v1alpha1.ApplicationDefinitionCredentials{ + Format: v1alpha1.CredentialFormat{Name: "scram-sha-256"}, + Accounts: []v1alpha1.CredentialAccountSource{{Kind: v1alpha1.CredentialAccountsValuesMap, Path: "users"}}, + } + unbuilt := &v1alpha1.ApplicationDefinitionCredentials{ + Format: v1alpha1.CredentialFormat{Name: "apr1", Kind: v1alpha1.CredentialFormatWebhook}, + Accounts: []v1alpha1.CredentialAccountSource{{Kind: v1alpha1.CredentialAccountsValuesMap, Path: "users"}}, + } + rc, err := resourceConfigFrom([]v1alpha1.ApplicationDefinition{ + mk("postgres", "Postgres", served), + mk("foo", "Foo", unbuilt), + mk("redis", "Redis", nil), + }, helmReleaseFlagValues{}) + if err != nil { + t.Fatal(err) + } + kinds := map[string]*v1alpha1.ApplicationDefinitionCredentials{} + for _, r := range rc.Resources { + kinds[r.Application.Kind] = r.Credentials + } + if len(kinds) != 2 || kinds["Postgres"] == nil || kinds["Postgres"].Format.Name != "scram-sha-256" { + t.Fatalf("resources = %v, want Postgres with its block and Redis without", kinds) + } + if _, ok := kinds["Foo"]; ok { + t.Fatal("a kind with an unbuilt format must not be served") + } + if _, ok := kinds["Redis"]; !ok || kinds["Redis"] != nil { + t.Fatalf("Redis = %v, want served with no block", kinds["Redis"]) + } +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 0223fe407..5d743365f 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -24,6 +24,8 @@ import ( helmv2 "github.com/fluxcd/helm-controller/api/v2" "github.com/fluxcd/pkg/apis/kustomize" + + "github.com/cozystack/cozystack/api/v1alpha1" ) // ReleaseAnnotationPrefix is the prefix shared by every ApplicationDefinition @@ -195,6 +197,10 @@ type ResourceConfig struct { type Resource struct { Application ApplicationConfig `yaml:"application"` Release ReleaseConfig `yaml:"release"` + // Credentials is the kind's credentials block, copied from + // spec.credentials of its ApplicationDefinition at start-up. nil for a + // kind that is not converted to platform-minted credentials. + Credentials *v1alpha1.ApplicationDefinitionCredentials `yaml:"credentials,omitempty"` } // ApplicationConfig contains the application settings. diff --git a/pkg/credentials/store/event.go b/pkg/credentials/store/event.go new file mode 100644 index 000000000..f0089535e --- /dev/null +++ b/pkg/credentials/store/event.go @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: Apache-2.0 + +package store + +import ( + "context" + "fmt" + "time" + + helmv2 "github.com/fluxcd/helm-controller/api/v2" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + + appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1" +) + +// Event reasons the API records on an application for its credentials. +const ( + ReasonCredentialIssued = "CredentialIssued" + ReasonCredentialRevoked = "CredentialRevoked" +) + +// Emit records an Event on the application, named through the HelmRelease +// that backs it, saying which account was issued or revoked and by whom. It +// never carries a value. Events are best effort: a failure to record one is +// logged and does not fail the operation it describes, which the audit log +// records in any case. +func Emit(ctx context.Context, c client.Client, hr *helmv2.HelmRelease, reason, username, by string) { + now := metav1.NewTime(time.Now()) + verb := "issued" + if reason == ReasonCredentialRevoked { + verb = "revoked" + } + ev := &corev1.Event{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: hr.Labels[appsv1alpha1.ApplicationNameLabel] + ".", + Namespace: hr.Namespace, + }, + InvolvedObject: corev1.ObjectReference{ + APIVersion: appsv1alpha1.SchemeGroupVersion.String(), + Kind: hr.Labels[appsv1alpha1.ApplicationKindLabel], + Name: hr.Labels[appsv1alpha1.ApplicationNameLabel], + Namespace: hr.Namespace, + UID: hr.UID, + }, + Reason: reason, + Message: fmt.Sprintf("password of user %s %s by %s", username, verb, by), + Type: corev1.EventTypeNormal, + Source: corev1.EventSource{Component: "cozystack-api"}, + FirstTimestamp: now, + LastTimestamp: now, + Count: 1, + } + if err := c.Create(ctx, ev); err != nil { + klog.Warningf("failed to record %s event for %s/%s: %v", reason, hr.Namespace, hr.Name, err) + } +} diff --git a/pkg/credentials/store/resolve.go b/pkg/credentials/store/resolve.go new file mode 100644 index 000000000..4125a0735 --- /dev/null +++ b/pkg/credentials/store/resolve.go @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: Apache-2.0 + +package store + +import ( + "encoding/json" + "fmt" + + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" + + cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" + "github.com/cozystack/cozystack/pkg/apis/apps/defaulting" + "github.com/cozystack/cozystack/pkg/credentials/account" +) + +// ResolveAccounts resolves the tenant accounts an application declares from +// its values with the kind's schema defaults applied. Stored values carry no +// defaults, and a Static account switched on by a defaulted flag would +// otherwise go unseeded. The API resolves accounts this way on every write, +// and the ApplicationDefinition reconciler when it brings a release's +// valuesFrom in line with its definition. +func ResolveAccounts(block *cozyv1alpha1.ApplicationDefinitionCredentials, schema *structuralschema.Structural, values *apiextv1.JSON) ([]account.Account, error) { + if block == nil { + return nil, nil + } + m := map[string]any{} + if values != nil && len(values.Raw) > 0 { + if err := json.Unmarshal(values.Raw, &m); err != nil { + return nil, fmt.Errorf("decode values: %w", err) + } + } + if schema != nil { + if err := defaulting.Apply(&m, schema); err != nil { + return nil, fmt.Errorf("default values: %w", err) + } + } + return account.Resolve(block, m) +} diff --git a/pkg/credentials/store/store.go b/pkg/credentials/store/store.go new file mode 100644 index 000000000..bd2441e9d --- /dev/null +++ b/pkg/credentials/store/store.go @@ -0,0 +1,237 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package store writes and removes account Secrets for the Cozystack API: it +// seeds one per account when an application is written, prunes the ones a +// write removes, and builds the valuesFrom entries that carry a stored value +// into a chart whose operator cannot read a Secret. The mint itself lives in +// the credential registry; this package is what the application registry +// needs around a HelmRelease write. +package store + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + + helmv2 "github.com/fluxcd/helm-controller/api/v2" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" + appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1" + corev1alpha1 "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1" + "github.com/cozystack/cozystack/pkg/credentials/account" + "github.com/cozystack/cozystack/pkg/credentials/format" +) + +// App identifies the application whose account Secrets are written. +type App struct { + Namespace string + // Release is the HelmRelease name, prefix plus application name. + Release string + Group string + Kind string + Name string + Block *cozyv1alpha1.ApplicationDefinitionCredentials +} + +// Labels returns the labels every account Secret of the application carries. +func (a App) Labels() map[string]string { + return account.Labels(a.Block, a.Group, a.Kind, a.Name) +} + +// OwnerReference builds the HelmRelease owner reference the account Secrets +// carry, so they are collected with the release. +func OwnerReference(hr *helmv2.HelmRelease) metav1.OwnerReference { + return metav1.OwnerReference{ + APIVersion: helmv2.GroupVersion.String(), + Kind: helmv2.HelmReleaseKind, + Name: hr.Name, + UID: hr.UID, + } +} + +// SeedSecret builds the account Secret of a new account: the stored form of +// a random password that is discarded here, so the account exists in the +// engine while nobody can log in until a mint. owner may be nil on a create, +// where the HelmRelease does not exist yet and the reference is set after. +func SeedSecret(app App, acc account.Account, owner *metav1.OwnerReference, now metav1.Time, by string) (*corev1.Secret, error) { + password, err := format.GeneratePassword() + if err != nil { + return nil, err + } + stored, err := format.Encode(app.Block.Format.Name, password) + if err != nil { + return nil, err + } + sec := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: account.SecretName(app.Release, acc.Mapped), + Namespace: app.Namespace, + Labels: app.Labels(), + }, + Type: corev1.SecretTypeBasicAuth, + Data: map[string][]byte{ + account.UsernameKey: []byte(acc.Username), + account.PasswordKey: []byte(stored), + }, + } + if owner != nil { + sec.OwnerReferences = []metav1.OwnerReference{*owner} + } + account.WriteRecord(sec, account.Record{}.Apply(corev1alpha1.CredentialOperationCreate, now, by)) + return sec, nil +} + +// Theirs reports whether an existing Secret is the application's own account +// Secret for the account: it carries the account marker, holds the same +// username, and is owned by the live HelmRelease. On a create no HelmRelease +// exists yet, so a Secret that any HelmRelease owns is left over from a +// deleted application of the same name and is replaced, so the account gets a +// new identity, while one nobody owns is from a create that did not finish. +func Theirs(sec *corev1.Secret, acc account.Account, owner *metav1.OwnerReference) bool { + if !account.IsAccountSecret(sec) || string(sec.Data[account.UsernameKey]) != acc.Username { + return false + } + for _, ref := range sec.OwnerReferences { + if owner == nil && ref.Kind == helmv2.HelmReleaseKind { + return false + } + if owner != nil && ref.UID == owner.UID { + return true + } + } + return owner == nil +} + +// Ensure writes an account Secret for every account that has none, or whose +// Secret is not the application's, and returns the names of the Secrets it +// wrote. It never touches a Secret that is the application's own, so a +// password already minted survives every later write of the application. +func Ensure(ctx context.Context, c client.Client, app App, accounts []account.Account, owner *metav1.OwnerReference, now metav1.Time, by string) ([]string, error) { + var written []string + for _, acc := range accounts { + name := account.SecretName(app.Release, acc.Mapped) + cur := &corev1.Secret{} + err := c.Get(ctx, types.NamespacedName{Namespace: app.Namespace, Name: name}, cur) + switch { + case err == nil && Theirs(cur, acc, owner): + continue + case err == nil: + if err := c.Delete(ctx, cur, client.Preconditions{UID: &cur.UID}); err != nil && !apierrors.IsNotFound(err) { + return written, fmt.Errorf("replace account Secret %s: %w", name, err) + } + case !apierrors.IsNotFound(err): + return written, fmt.Errorf("read account Secret %s: %w", name, err) + } + sec, err := SeedSecret(app, acc, owner, now, by) + if err != nil { + return written, fmt.Errorf("seed account %s: %w", acc.Username, err) + } + if err := c.Create(ctx, sec); err != nil { + return written, fmt.Errorf("create account Secret %s: %w", name, err) + } + written = append(written, name) + } + return written, nil +} + +// SetOwner adds the HelmRelease as owner of the named account Secrets. It is +// the second half of a create, once the HelmRelease exists. +func SetOwner(ctx context.Context, c client.Client, namespace string, names []string, owner metav1.OwnerReference) error { + for _, name := range names { + sec := &corev1.Secret{} + if err := c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, sec); err != nil { + return fmt.Errorf("read account Secret %s: %w", name, err) + } + owned := false + for _, ref := range sec.OwnerReferences { + if ref.UID == owner.UID { + owned = true + } + } + if owned { + continue + } + patch := client.MergeFrom(sec.DeepCopy()) + sec.OwnerReferences = append(sec.OwnerReferences, owner) + if err := c.Patch(ctx, sec, patch); err != nil { + return fmt.Errorf("own account Secret %s: %w", name, err) + } + } + return nil +} + +// List returns the account Secrets of the application. +func List(ctx context.Context, c client.Client, app App) ([]corev1.Secret, error) { + list := &corev1.SecretList{} + if err := c.List(ctx, list, client.InNamespace(app.Namespace), client.MatchingLabels{ + account.AccountLabel: "true", + appsv1alpha1.ApplicationGroupLabel: app.Group, + appsv1alpha1.ApplicationKindLabel: app.Kind, + appsv1alpha1.ApplicationNameLabel: app.Name, + }); err != nil { + return nil, err + } + return list.Items, nil +} + +// Prune deletes the account Secrets of the application that no account in +// keep names, and returns the names it deleted. It runs after the HelmRelease +// write that removed the accounts, so the chart has already dropped them. +func Prune(ctx context.Context, c client.Client, app App, keep []account.Account) ([]string, error) { + kept := map[string]bool{} + for _, acc := range keep { + kept[account.SecretName(app.Release, acc.Mapped)] = true + } + secrets, err := List(ctx, c, app) + if err != nil { + return nil, err + } + var deleted []string + for i := range secrets { + sec := &secrets[i] + if kept[sec.Name] { + continue + } + if err := c.Delete(ctx, sec, client.Preconditions{UID: &sec.UID}); err != nil && !apierrors.IsNotFound(err) { + return deleted, fmt.Errorf("delete account Secret %s: %w", sec.Name, err) + } + deleted = append(deleted, sec.Name) + } + return deleted, nil +} + +// DigestKey is the values key one account's stored value lands under: a hex +// digest of the user name, because a user name fits neither a targetPath nor +// a valuesKey, while a digest fits both and leaves no order to keep aligned. +func DigestKey(username string) string { + sum := sha256.Sum256([]byte(username)) + return "u" + hex.EncodeToString(sum[:]) +} + +// ValuesFrom returns the HelmRelease valuesFrom entries of an application: +// the platform values first, as every release has them, then one entry per +// account for a kind with a valuesPath, reading the Secret's stored value into +// .. The API and the ApplicationDefinition reconciler +// both build the list here, so a release created before the block appeared +// gets the same entries on its next reconcile. +func ValuesFrom(block *cozyv1alpha1.ApplicationDefinitionCredentials, release string, accounts []account.Account) []helmv2.ValuesReference { + refs := []helmv2.ValuesReference{{Kind: "Secret", Name: "cozystack-values"}} + if block == nil || block.ValuesPath == "" { + return refs + } + for _, acc := range accounts { + refs = append(refs, helmv2.ValuesReference{ + Kind: "Secret", + Name: account.SecretName(release, acc.Mapped), + ValuesKey: account.PasswordKey, + TargetPath: block.ValuesPath + "." + DigestKey(acc.Username), + }) + } + return refs +} diff --git a/pkg/credentials/store/store_test.go b/pkg/credentials/store/store_test.go new file mode 100644 index 000000000..f82df48cf --- /dev/null +++ b/pkg/credentials/store/store_test.go @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: Apache-2.0 + +package store + +import ( + "context" + "strings" + "testing" + + helmv2 "github.com/fluxcd/helm-controller/api/v2" + corev1 "k8s.io/api/core/v1" + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" + "github.com/cozystack/cozystack/pkg/apis/apps/defaulting" + corev1alpha1 "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1" + "github.com/cozystack/cozystack/pkg/credentials/account" +) + +func newScheme(t *testing.T) *runtime.Scheme { + t.Helper() + s := runtime.NewScheme() + if err := corev1.AddToScheme(s); err != nil { + t.Fatal(err) + } + if err := helmv2.AddToScheme(s); err != nil { + t.Fatal(err) + } + return s +} + +func testBlock() *cozyv1alpha1.ApplicationDefinitionCredentials { + return &cozyv1alpha1.ApplicationDefinitionCredentials{ + Format: cozyv1alpha1.CredentialFormat{Name: "scram-sha-256"}, + SecretLabels: map[string]string{"k8s.mariadb.com/watch": ""}, + ValuesPath: "_accounts", + Accounts: []cozyv1alpha1.CredentialAccountSource{ + {Kind: cozyv1alpha1.CredentialAccountsValuesMap, Path: "users"}, + {Kind: cozyv1alpha1.CredentialAccountsStatic, Name: "backup", When: "backup.enabled"}, + {Kind: cozyv1alpha1.CredentialAccountsStatic, Name: "root", Owner: cozyv1alpha1.CredentialAccountOwnerPlatform}, + }, + } +} + +func testApp() App { + return App{Namespace: "tenant-foo", Release: "postgres-orders", Group: "apps.cozystack.io", Kind: "Postgres", Name: "orders", Block: testBlock()} +} + +func release(uid string) *helmv2.HelmRelease { + return &helmv2.HelmRelease{ObjectMeta: metav1.ObjectMeta{Name: "postgres-orders", Namespace: "tenant-foo", UID: types.UID(uid)}} +} + +func accounts(names ...string) []account.Account { + var out []account.Account + for _, n := range names { + out = append(out, account.Account{Username: n, Mapped: strings.ToLower(n)}) + } + return out +} + +func getSecret(t *testing.T, c client.Client, name string) *corev1.Secret { + t.Helper() + sec := &corev1.Secret{} + if err := c.Get(context.Background(), types.NamespacedName{Namespace: "tenant-foo", Name: name}, sec); err != nil { + t.Fatalf("get %s: %v", name, err) + } + return sec +} + +func TestEnsureSeedsAndKeepsTheirs(t *testing.T) { + ctx := context.Background() + c := fake.NewClientBuilder().WithScheme(newScheme(t)).Build() + app := testApp() + now := metav1.Now() + + written, err := Ensure(ctx, c, app, accounts("web"), nil, now, "jane") + if err != nil { + t.Fatal(err) + } + if len(written) != 1 || written[0] != "postgres-orders.web.account" { + t.Fatalf("written = %v", written) + } + sec := getSecret(t, c, "postgres-orders.web.account") + if sec.Type != corev1.SecretTypeBasicAuth || string(sec.Data[account.UsernameKey]) != "web" { + t.Fatalf("seeded Secret = %+v", sec) + } + if !strings.HasPrefix(string(sec.Data[account.PasswordKey]), "SCRAM-SHA-256$4096:") { + t.Fatalf("password not stored in the kind's format: %q", sec.Data[account.PasswordKey]) + } + if sec.Labels[account.AccountLabel] != "true" || sec.Labels["k8s.mariadb.com/watch"] != "" || sec.Labels["apps.cozystack.io/application.name"] != "orders" { + t.Fatalf("labels = %v", sec.Labels) + } + rec := account.ReadRecord(sec) + if rec.Origin != corev1alpha1.CredentialOriginNotIssued || rec.By != "jane" || len(rec.History) != 1 || rec.History[0].Operation != corev1alpha1.CredentialOperationCreate { + t.Fatalf("record = %+v", rec) + } + first := string(sec.Data[account.PasswordKey]) + + // A second create of the same application, as after an interrupted one, + // reuses the Secret nobody owns. + hr := release("uid-1") + if err := SetOwner(ctx, c, "tenant-foo", written, OwnerReference(hr)); err != nil { + t.Fatal(err) + } + owner := OwnerReference(hr) + written, err = Ensure(ctx, c, app, accounts("web", "api"), &owner, now, "jane") + if err != nil { + t.Fatal(err) + } + if len(written) != 1 || written[0] != "postgres-orders.api.account" { + t.Fatalf("an update seeds only the added account, written = %v", written) + } + sec = getSecret(t, c, "postgres-orders.web.account") + if string(sec.Data[account.PasswordKey]) != first { + t.Fatal("an owned Secret must survive a later write untouched") + } + api := getSecret(t, c, "postgres-orders.api.account") + if len(api.OwnerReferences) != 1 || api.OwnerReferences[0].UID != "uid-1" { + t.Fatalf("a Secret seeded on update carries the owner from the start: %v", api.OwnerReferences) + } +} + +func TestEnsureReplacesStale(t *testing.T) { + ctx := context.Background() + c := fake.NewClientBuilder().WithScheme(newScheme(t)).Build() + app := testApp() + now := metav1.Now() + + // Owned by a HelmRelease that is gone: a leftover of a deleted application + // of the same name, replaced on create so the account gets a new identity. + old := OwnerReference(release("uid-old")) + if _, err := Ensure(ctx, c, app, accounts("web"), &old, now, "jane"); err != nil { + t.Fatal(err) + } + before := getSecret(t, c, "postgres-orders.web.account") + written, err := Ensure(ctx, c, app, accounts("web"), nil, now, "jane") + if err != nil { + t.Fatal(err) + } + // The fake client assigns no UIDs, so a replacement shows as a new random + // password and no owner. + after := getSecret(t, c, "postgres-orders.web.account") + if len(written) != 1 || string(after.Data[account.PasswordKey]) == string(before.Data[account.PasswordKey]) || len(after.OwnerReferences) != 0 { + t.Fatalf("expected a fresh unowned Secret, written=%v owners=%v", written, after.OwnerReferences) + } + + // Same name, another username: not theirs either. + live := OwnerReference(release("uid-live")) + if err := SetOwner(ctx, c, "tenant-foo", written, live); err != nil { + t.Fatal(err) + } + written, err = Ensure(ctx, c, app, []account.Account{{Username: "Web", Mapped: "web"}}, &live, now, "jane") + if err != nil { + t.Fatal(err) + } + renamed := getSecret(t, c, "postgres-orders.web.account") + if len(written) != 1 || string(renamed.Data[account.PasswordKey]) == string(after.Data[account.PasswordKey]) || string(renamed.Data[account.UsernameKey]) != "Web" { + t.Fatalf("expected the Secret replaced for the new username, got %q", renamed.Data[account.UsernameKey]) + } +} + +func TestPruneDeletesRemovedOnly(t *testing.T) { + ctx := context.Background() + c := fake.NewClientBuilder().WithScheme(newScheme(t)).Build() + app := testApp() + owner := OwnerReference(release("uid-1")) + if _, err := Ensure(ctx, c, app, accounts("web", "api"), &owner, metav1.Now(), "jane"); err != nil { + t.Fatal(err) + } + // A Secret of another application in the namespace is never touched. + other := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "postgres-billing.web.account", Namespace: "tenant-foo", Labels: account.Labels(app.Block, app.Group, app.Kind, "billing")}} + if err := c.Create(ctx, other); err != nil { + t.Fatal(err) + } + deleted, err := Prune(ctx, c, app, accounts("api")) + if err != nil { + t.Fatal(err) + } + if len(deleted) != 1 || deleted[0] != "postgres-orders.web.account" { + t.Fatalf("deleted = %v", deleted) + } + left, err := List(ctx, c, app) + if err != nil { + t.Fatal(err) + } + if len(left) != 1 || left[0].Name != "postgres-orders.api.account" { + t.Fatalf("left = %v", left) + } + getSecret(t, c, "postgres-billing.web.account") +} + +func TestValuesFrom(t *testing.T) { + refs := ValuesFrom(nil, "postgres-orders", accounts("web")) + if len(refs) != 1 || refs[0].Name != "cozystack-values" { + t.Fatalf("a kind without a block carries the platform values only: %+v", refs) + } + block := testBlock() + block.ValuesPath = "" + if refs := ValuesFrom(block, "postgres-orders", accounts("web")); len(refs) != 1 { + t.Fatalf("a block without a valuesPath adds nothing: %+v", refs) + } + refs = ValuesFrom(testBlock(), "postgres-orders", []account.Account{{Username: "Web.1", Mapped: "web-1"}}) + if len(refs) != 2 { + t.Fatalf("refs = %+v", refs) + } + got := refs[1] + if got.Kind != "Secret" || got.Name != "postgres-orders.web-1.account" || got.ValuesKey != account.PasswordKey || got.TargetPath != "_accounts."+DigestKey("Web.1") { + t.Fatalf("ref = %+v", got) + } + if DigestKey("Web.1") == DigestKey("web.1") || !strings.HasPrefix(DigestKey("x"), "u") { + t.Fatal("the digest key is of the original user name") + } +} + +func TestResolveAccountsAppliesDefaults(t *testing.T) { + schema, err := defaulting.BuildSchema(`{"type":"object","properties":{"backup":{"type":"object","default":{},"properties":{"enabled":{"type":"boolean","default":true}}},"users":{"type":"object","additionalProperties":{"type":"object"}}}}`) + if err != nil { + t.Fatal(err) + } + values := &apiextv1.JSON{Raw: []byte(`{"users":{"web":{},"root":{}}}`)} + got, err := ResolveAccounts(testBlock(), schema, values) + if err != nil { + t.Fatal(err) + } + var names []string + for _, a := range got { + names = append(names, a.Username) + } + // root is a platform account and is skipped; backup is on by default. + if strings.Join(names, ",") != "web,backup" { + t.Fatalf("accounts = %v", names) + } + got, err = ResolveAccounts(testBlock(), nil, values) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 { + t.Fatalf("without the schema the defaulted flag is off: %v", got) + } + if got, err := ResolveAccounts(nil, schema, values); err != nil || got != nil { + t.Fatalf("no block, no accounts: %v %v", got, err) + } +} diff --git a/pkg/registry/apps/application/rest.go b/pkg/registry/apps/application/rest.go index 6cd544994..3e30a79ef 100644 --- a/pkg/registry/apps/application/rest.go +++ b/pkg/registry/apps/application/rest.go @@ -101,14 +101,13 @@ type REST struct { specSchema *structuralschema.Structural nameSchema *validation.NameSchema nameSchemaErr error + // credentials is the kind's credentials block, nil for a kind whose + // passwords the chart still generates. + credentials *cozyv1alpha1.ApplicationDefinitionCredentials } -// buildSpecSchema parses an OpenAPI-v3 JSON schema string and returns the -// structural-schema form used for defaulting. Returns (nil, nil) when raw -// is empty after trimming. Returns (nil, error) for malformed JSON, -// v1→internal conversion failures, or structural-schema construction -// errors — callers can decide whether to log-and-continue (NewREST does) -// or fail hard (tests do). +// buildSpecSchema parses an OpenAPI-v3 JSON schema string into the +// structural schema the registry defaults and null-strips against. func buildSpecSchema(raw string) (*structuralschema.Structural, error) { return defaulting.BuildSchema(raw) } @@ -144,6 +143,7 @@ func NewREST(c client.Client, w client.WithWatch, config *config.Resource) *REST specSchema: specSchema, nameSchema: nameSchema, nameSchemaErr: nameSchemaErr, + credentials: config.Credentials, } } @@ -226,6 +226,11 @@ func (r *REST) Create(ctx context.Context, obj runtime.Object, createValidation } } + accounts, err := r.accountsOf(app.Spec) + if err != nil { + return nil, r.invalidAccounts(app, err) + } + // Convert Application to HelmRelease helmRelease, err := r.ConvertApplicationToHelmRelease(app) if err != nil { @@ -246,14 +251,23 @@ func (r *REST) Create(ctx context.Context, obj runtime.Object, createValidation helmRelease.Labels[ApplicationNameLabel] = app.Name // Note: Annotations from config are not handled as r.releaseConfig.Annotations is undefined + // The account Secrets exist before the release that reads them, and the + // release owns them as soon as it exists. + seeded, err := r.seedBeforeCreate(ctx, app, helmRelease, accounts) + if err != nil { + return nil, err + } + klog.V(6).Infof("Creating HelmRelease %s in namespace %s", helmRelease.Name, app.Namespace) // Create HelmRelease in Kubernetes err = r.c.Create(ctx, helmRelease, registry.ClientCreateOptions(options)) if err != nil { klog.Errorf("Failed to create HelmRelease %s: %v", helmRelease.Name, err) + r.unseed(ctx, app.Namespace, seeded) return nil, registry.WrapPreservingStatus("failed to create HelmRelease", err, r.gvr.GroupResource(), app.Name) } + r.ownSeeded(ctx, helmRelease, seeded) // Convert the created HelmRelease back to Application convertedApp, err := r.ConvertHelmReleaseToApplication(ctx, helmRelease) @@ -575,6 +589,11 @@ func (r *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObje r.warnRemovedUserPasswords(ctx, app) r.warnUpgradeIntroducedCollisions(ctx, oldObj, app) + accounts, err := r.accountsOf(app.Spec) + if err != nil { + return nil, false, r.invalidAccounts(app, err) + } + // Convert Application to HelmRelease helmRelease, err := r.ConvertApplicationToHelmRelease(app) if err != nil { @@ -582,15 +601,20 @@ func (r *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObje return nil, false, fmt.Errorf("conversion error: %v", err) } - // Keep the conversion output pristine: the metadata assembly below reads the - // live object, so a conflict retry has to redo it from this copy rather than - // re-apply it on top of its own previous result. - converted := helmRelease.DeepCopy() - cur := &helmv2.HelmRelease{} if err := r.c.Get(ctx, client.ObjectKey{Namespace: helmRelease.Namespace, Name: helmRelease.Name}, cur, &client.GetOptions{Raw: &metav1.GetOptions{}}); err != nil { return nil, false, registry.WrapPreservingStatus("failed to fetch current HelmRelease", err, r.gvr.GroupResource(), name) } + + // Accounts the write adds get their Secrets before the release reads them. + if err := r.seedBeforeUpdate(ctx, app, helmRelease, cur, accounts); err != nil { + return nil, false, err + } + + // Keep the conversion output pristine: the metadata assembly below reads the + // live object, so a conflict retry has to redo it from this copy rather than + // re-apply it on top of its own previous result. + converted := helmRelease.DeepCopy() if helmRelease.ResourceVersion == "" { helmRelease.SetResourceVersion(cur.GetResourceVersion()) } @@ -646,6 +670,10 @@ func (r *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObje return nil, false, registry.WrapPreservingStatus("failed to update HelmRelease", err, r.gvr.GroupResource(), name) } + // Accounts the write removed are gone from the chart now, so their + // Secrets can go. + r.pruneAfterUpdate(ctx, app, accounts) + // Convert the updated HelmRelease back to Application convertedApp, err := r.ConvertHelmReleaseToApplication(ctx, helmRelease) if err != nil { @@ -694,6 +722,10 @@ func (r *REST) Delete(ctx context.Context, name string, deleteValidation rest.Va return nil, false, apierrors.NewNotFound(r.gvr.GroupResource(), name) } + if err := r.refuseDeleteSuspended(name, helmRelease); err != nil { + return nil, false, err + } + // Run the genericapiserver-supplied admission chain (validating webhooks // and ValidatingAdmissionPolicies) on the resolved Application before // removing the underlying HelmRelease. Custom REST handlers must invoke @@ -2198,7 +2230,13 @@ const ( // warnRemovedUserPasswords emits a client-facing admission warning for every // spec.users[].password still present on a Postgres or MariaDB resource. +// A kind with a credentials block gets the warning that points at the mint +// instead, whatever the kind. func (r *REST) warnRemovedUserPasswords(ctx context.Context, app *appsv1alpha1.Application) { + if r.credentials != nil { + r.warnMintedPasswords(ctx, app) + return + } if !removedUserPasswordKinds[r.kindName] || app == nil || app.Spec == nil || len(app.Spec.Raw) == 0 { return } diff --git a/pkg/registry/apps/application/rest_credentials.go b/pkg/registry/apps/application/rest_credentials.go new file mode 100644 index 000000000..d9fd02230 --- /dev/null +++ b/pkg/registry/apps/application/rest_credentials.go @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: Apache-2.0 + +package application + +import ( + "context" + "encoding/json" + "fmt" + + helmv2 "github.com/fluxcd/helm-controller/api/v2" + corev1 "k8s.io/api/core/v1" + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/apiserver/pkg/endpoints/request" + "k8s.io/apiserver/pkg/warning" + "k8s.io/klog/v2" + + cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" + appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1" + "github.com/cozystack/cozystack/pkg/credentials/account" + "github.com/cozystack/cozystack/pkg/credentials/store" +) + +// Credentials returns the kind's credentials block, nil for a kind that is +// not converted. +func (r *REST) Credentials() *cozyv1alpha1.ApplicationDefinitionCredentials { + return r.credentials +} + +// AccountsOf resolves the tenant accounts a HelmRelease declares, from its +// values with the kind's schema defaults applied. The stored values carry no +// defaults, and a Static account switched on by a defaulted flag would +// otherwise go unseeded. It is what the credentials registry asks on a mint of +// an account whose Secret is missing. +func (r *REST) AccountsOf(_ context.Context, hr *helmv2.HelmRelease) ([]account.Account, error) { + return r.accountsOf(hr.Spec.Values) +} + +func (r *REST) accountsOf(values *apiextv1.JSON) ([]account.Account, error) { + return store.ResolveAccounts(r.credentials, r.specSchema, values) +} + +// invalidAccounts turns an account resolution failure into the Invalid the +// write is refused with. +func (r *REST) invalidAccounts(app *appsv1alpha1.Application, err error) error { + return apierrors.NewInvalid(r.gvk.GroupKind(), app.Name, field.ErrorList{ + field.Invalid(field.NewPath("spec"), "", err.Error()), + }) +} + +func (r *REST) storeApp(namespace, name string) store.App { + return store.App{ + Namespace: namespace, + Release: r.releaseConfig.Prefix + name, + Group: r.gvk.Group, + Kind: r.kindName, + Name: name, + Block: r.credentials, + } +} + +// actor names the caller for the record an account Secret keeps. +func actor(ctx context.Context) string { + if user, ok := request.UserFrom(ctx); ok { + return user.GetName() + } + return "" +} + +// seedBeforeCreate writes the account Secrets of a new application, with no +// owner yet, and the valuesFrom entries that carry them into the chart. +// It returns the Secrets it wrote, for the owner to be set once the +// HelmRelease exists, or for removal if the HelmRelease is not created. +func (r *REST) seedBeforeCreate(ctx context.Context, app *appsv1alpha1.Application, hr *helmv2.HelmRelease, accounts []account.Account) ([]string, error) { + if r.credentials == nil { + return nil, nil + } + hr.Spec.ValuesFrom = store.ValuesFrom(r.credentials, hr.Name, accounts) + written, err := store.Ensure(ctx, r.c, r.storeApp(app.Namespace, app.Name), accounts, nil, metav1.Now(), actor(ctx)) + if err != nil { + r.unseed(ctx, app.Namespace, written) + return nil, apierrors.NewInternalError(fmt.Errorf("seed account Secrets: %w", err)) + } + return written, nil +} + +// unseed removes the Secrets a create wrote before its HelmRelease failed, so +// a retry starts clean. Best effort: a Secret nobody owns is reused by the +// retry in any case. +func (r *REST) unseed(ctx context.Context, namespace string, names []string) { + for _, name := range names { + sec := &corev1.Secret{} + sec.Namespace, sec.Name = namespace, name + if err := r.c.Delete(ctx, sec); err != nil && !apierrors.IsNotFound(err) { + klog.Warningf("failed to remove account Secret %s/%s after a failed create: %v", namespace, name, err) + } + } +} + +// ownSeeded gives the created HelmRelease the account Secrets seeded before +// it. The application exists by now, so a failure is reported as a warning +// rather than an error: the next write of the application retries it. +func (r *REST) ownSeeded(ctx context.Context, hr *helmv2.HelmRelease, names []string) { + if len(names) == 0 { + return + } + if err := store.SetOwner(ctx, r.c, hr.Namespace, names, store.OwnerReference(hr)); err != nil { + klog.Errorf("failed to own account Secrets of %s/%s: %v", hr.Namespace, hr.Name, err) + warning.AddWarning(ctx, "", fmt.Sprintf("account Secrets were written but not yet owned by the release: %v", err)) + } +} + +// seedBeforeUpdate writes the account Secrets an update adds, owned by the +// live HelmRelease from the start, and the valuesFrom entries of the result. +func (r *REST) seedBeforeUpdate(ctx context.Context, app *appsv1alpha1.Application, hr, cur *helmv2.HelmRelease, accounts []account.Account) error { + if r.credentials == nil { + return nil + } + hr.Spec.ValuesFrom = store.ValuesFrom(r.credentials, hr.Name, accounts) + owner := store.OwnerReference(cur) + if _, err := store.Ensure(ctx, r.c, r.storeApp(app.Namespace, app.Name), accounts, &owner, metav1.Now(), actor(ctx)); err != nil { + return apierrors.NewInternalError(fmt.Errorf("seed account Secrets: %w", err)) + } + return nil +} + +// pruneAfterUpdate deletes the account Secrets of the accounts the update +// removed, once the HelmRelease write has dropped them from the chart. +func (r *REST) pruneAfterUpdate(ctx context.Context, app *appsv1alpha1.Application, accounts []account.Account) { + if r.credentials == nil { + return + } + if _, err := store.Prune(ctx, r.c, r.storeApp(app.Namespace, app.Name), accounts); err != nil { + klog.Errorf("failed to prune account Secrets of %s/%s: %v", app.Namespace, app.Name, err) + warning.AddWarning(ctx, "", fmt.Sprintf("account Secrets of removed users were not all deleted: %v", err)) + } +} + +// refuseDeleteSuspended refuses to delete a converted application whose +// release is suspended: Flux skips the uninstall of a suspended release, so +// the database would keep running with its last passwords after the account +// Secrets are gone with the HelmRelease. +func (r *REST) refuseDeleteSuspended(name string, hr *helmv2.HelmRelease) error { + if r.credentials == nil || !hr.Spec.Suspend { + return nil + } + return apierrors.NewConflict(r.gvr.GroupResource(), name, fmt.Errorf( + "release %s is suspended, and deleting it would leave the database running after its account Secrets are gone; resume the release first", hr.Name)) +} + +// warnMintedPasswords warns about a password set in the values of a converted +// kind: the chart never reads it, since every password is issued by a mint. +func (r *REST) warnMintedPasswords(ctx context.Context, app *appsv1alpha1.Application) { + if r.credentials == nil || app == nil || app.Spec == nil || len(app.Spec.Raw) == 0 { + return + } + var values map[string]any + if err := json.Unmarshal(app.Spec.Raw, &values); err != nil { + return + } + release := r.releaseConfig.Prefix + app.Name + for _, path := range account.ValuesMapPaths(r.credentials) { + users, _ := account.Lookup(values, path).(map[string]any) + for _, user := range sortedKeys(users) { + u, _ := users[user].(map[string]any) + if _, present := u["password"]; !present { + continue + } + mapped, err := account.MapName(user) + if err != nil { + continue + } + warning.AddWarning(ctx, "", fmt.Sprintf( + "spec.%s[%q].password is ignored: passwords are issued by a mint on the credential %s and cannot be set from values.", + path, user, account.CredentialName(release, mapped))) + } + } +} diff --git a/pkg/registry/apps/application/rest_credentials_test.go b/pkg/registry/apps/application/rest_credentials_test.go new file mode 100644 index 000000000..125d74498 --- /dev/null +++ b/pkg/registry/apps/application/rest_credentials_test.go @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: Apache-2.0 + +package application + +import ( + "context" + "strings" + "testing" + + helmv2 "github.com/fluxcd/helm-controller/api/v2" + corev1 "k8s.io/api/core/v1" + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apiserver/pkg/authentication/user" + "k8s.io/apiserver/pkg/endpoints/request" + "k8s.io/apiserver/pkg/warning" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" + appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1" + "github.com/cozystack/cozystack/pkg/config" + "github.com/cozystack/cozystack/pkg/credentials/account" +) + +// The schema defaults backup.enabled to true, so the Static backup account +// is seeded unless the application turns it off. +const credentialsSchema = `{"type":"object","properties":{"backup":{"type":"object","default":{},"properties":{"enabled":{"type":"boolean","default":true}}},"users":{"type":"object","additionalProperties":{"type":"object","properties":{"password":{"type":"string"}}}}}}` + +func credentialsBlock() *cozyv1alpha1.ApplicationDefinitionCredentials { + return &cozyv1alpha1.ApplicationDefinitionCredentials{ + Format: cozyv1alpha1.CredentialFormat{Name: "scram-sha-256"}, + ValuesPath: "_accounts", + Accounts: []cozyv1alpha1.CredentialAccountSource{ + {Kind: cozyv1alpha1.CredentialAccountsValuesMap, Path: "users"}, + {Kind: cozyv1alpha1.CredentialAccountsStatic, Name: "backup", When: "backup.enabled"}, + {Kind: cozyv1alpha1.CredentialAccountsStatic, Name: "postgres", Owner: cozyv1alpha1.CredentialAccountOwnerPlatform}, + }, + } +} + +func newCredentialsREST(t *testing.T, objects ...client.Object) (*REST, client.Client) { + t.Helper() + scheme := runtime.NewScheme() + if err := helmv2.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + cfg := &config.Resource{ + Application: config.ApplicationConfig{Kind: postgresKind, Singular: "postgres", Plural: "postgreses", OpenAPISchema: credentialsSchema}, + Release: config.ReleaseConfig{Prefix: "postgres-"}, + Credentials: credentialsBlock(), + } + if err := appsv1alpha1.RegisterDynamicTypes(scheme, &config.ResourceConfig{Resources: []config.Resource{*cfg}}); err != nil { + t.Fatal(err) + } + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + return NewREST(c, c, cfg), c +} + +func credentialsCtx() context.Context { + return request.WithUser(request.WithNamespace(context.Background(), "tenant-foo"), &user.DefaultInfo{Name: "jane@example.org"}) +} + +func postgresApp(spec string) *appsv1alpha1.Application { + return &appsv1alpha1.Application{ + TypeMeta: metav1.TypeMeta{APIVersion: "apps.cozystack.io/v1alpha1", Kind: postgresKind}, + ObjectMeta: metav1.ObjectMeta{Name: "orders", Namespace: "tenant-foo"}, + Spec: &apiextv1.JSON{Raw: []byte(spec)}, + } +} + +func accountSecrets(t *testing.T, c client.Client) map[string]*corev1.Secret { + t.Helper() + list := &corev1.SecretList{} + if err := c.List(context.Background(), list, client.InNamespace("tenant-foo"), client.MatchingLabels{account.AccountLabel: "true"}); err != nil { + t.Fatal(err) + } + out := map[string]*corev1.Secret{} + for i := range list.Items { + out[list.Items[i].Name] = &list.Items[i] + } + return out +} + +func getRelease(t *testing.T, c client.Client) *helmv2.HelmRelease { + t.Helper() + hr := &helmv2.HelmRelease{} + if err := c.Get(context.Background(), types.NamespacedName{Namespace: "tenant-foo", Name: "postgres-orders"}, hr); err != nil { + t.Fatal(err) + } + return hr +} + +func TestCreateSeedsAccountsAndOwnsThem(t *testing.T) { + r, c := newCredentialsREST(t) + if _, err := r.Create(credentialsCtx(), postgresApp(`{"users":{"web":{},"postgres":{}}}`), nil, &metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + hr := getRelease(t, c) + secrets := accountSecrets(t, c) + // web from the values, backup from the defaulted flag; postgres is the + // platform's and gets no account Secret. + if len(secrets) != 2 || secrets["postgres-orders.web.account"] == nil || secrets["postgres-orders.backup.account"] == nil { + t.Fatalf("secrets = %v", keys(secrets)) + } + for name, sec := range secrets { + if len(sec.OwnerReferences) != 1 || sec.OwnerReferences[0].Name != hr.Name || sec.OwnerReferences[0].Kind != helmv2.HelmReleaseKind { + t.Fatalf("%s owners = %v", name, sec.OwnerReferences) + } + if account.ReadRecord(sec).By != "jane@example.org" { + t.Fatalf("%s record = %+v", name, account.ReadRecord(sec)) + } + } + if len(hr.Spec.ValuesFrom) != 3 || hr.Spec.ValuesFrom[0].Name != "cozystack-values" || hr.Spec.ValuesFrom[1].Name != "postgres-orders.web.account" || !strings.HasPrefix(hr.Spec.ValuesFrom[1].TargetPath, "_accounts.u") { + t.Fatalf("valuesFrom = %+v", hr.Spec.ValuesFrom) + } +} + +func TestCreateRefusesCollidingNames(t *testing.T) { + r, c := newCredentialsREST(t) + _, err := r.Create(credentialsCtx(), postgresApp(`{"users":{"web_app":{},"web-app":{}}}`), nil, &metav1.CreateOptions{}) + if !apierrors.IsInvalid(err) || !strings.Contains(err.Error(), `both map to the name "web-app"`) { + t.Fatalf("expected Invalid naming the collision, got %v", err) + } + if n := len(accountSecrets(t, c)); n != 0 { + t.Fatalf("a refused write seeds nothing, got %d Secrets", n) + } +} + +func TestCreateFailureRemovesSeeded(t *testing.T) { + existing := &helmv2.HelmRelease{ObjectMeta: metav1.ObjectMeta{Name: "postgres-orders", Namespace: "tenant-foo"}} + r, c := newCredentialsREST(t, existing) + if _, err := r.Create(credentialsCtx(), postgresApp(`{"users":{"web":{}}}`), nil, &metav1.CreateOptions{}); !apierrors.IsAlreadyExists(err) { + t.Fatalf("expected AlreadyExists, got %v", err) + } + if n := len(accountSecrets(t, c)); n != 0 { + t.Fatalf("Secrets seeded for a release that was not created must go, got %d", n) + } +} + +func TestUpdateSeedsAddedAndPrunesRemoved(t *testing.T) { + r, c := newCredentialsREST(t) + ctx := credentialsCtx() + if _, err := r.Create(ctx, postgresApp(`{"users":{"web":{},"api":{}}}`), nil, &metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + webBefore := accountSecrets(t, c)["postgres-orders.web.account"] + + updated := postgresApp(`{"users":{"web":{},"reports":{}},"backup":{"enabled":false}}`) + if _, _, err := r.Update(ctx, "orders", newDefaultUpdatedObjectInfo(updated), nil, nil, false, &metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + secrets := accountSecrets(t, c) + if len(secrets) != 2 || secrets["postgres-orders.web.account"] == nil || secrets["postgres-orders.reports.account"] == nil { + t.Fatalf("secrets = %v, want web kept, reports added, api and backup pruned", keys(secrets)) + } + if string(secrets["postgres-orders.web.account"].Data[account.PasswordKey]) != string(webBefore.Data[account.PasswordKey]) { + t.Fatal("a kept account's Secret must survive the update untouched") + } + hr := getRelease(t, c) + reports := secrets["postgres-orders.reports.account"] + if len(reports.OwnerReferences) != 1 || reports.OwnerReferences[0].UID != hr.UID { + t.Fatalf("added account's owner = %v", reports.OwnerReferences) + } + if len(hr.Spec.ValuesFrom) != 3 { + t.Fatalf("valuesFrom = %+v", hr.Spec.ValuesFrom) + } +} + +func TestDeleteRefusesSuspendedRelease(t *testing.T) { + r, c := newCredentialsREST(t) + ctx := credentialsCtx() + if _, err := r.Create(ctx, postgresApp(`{"users":{"web":{}}}`), nil, &metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + hr := getRelease(t, c) + hr.Spec.Suspend = true + if err := c.Update(ctx, hr); err != nil { + t.Fatal(err) + } + _, _, err := r.Delete(ctx, "orders", nil, &metav1.DeleteOptions{}) + if !apierrors.IsConflict(err) || !strings.Contains(err.Error(), "suspended") { + t.Fatalf("expected Conflict on a suspended release, got %v", err) + } + hr = getRelease(t, c) + hr.Spec.Suspend = false + if err := c.Update(ctx, hr); err != nil { + t.Fatal(err) + } + if _, _, err := r.Delete(ctx, "orders", nil, &metav1.DeleteOptions{}); err != nil { + t.Fatal(err) + } +} + +func TestWarnMintedPasswords(t *testing.T) { + r, _ := newCredentialsREST(t) + rec := &fakeWarningRecorder{} + ctx := warning.WithWarningRecorder(credentialsCtx(), rec) + if _, err := r.Create(ctx, postgresApp(`{"users":{"Web":{"password":"hackme"},"api":{}}}`), nil, &metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + if len(rec.warnings) != 1 || !strings.Contains(rec.warnings[0], `spec.users["Web"].password is ignored`) || !strings.Contains(rec.warnings[0], "postgres-orders.web") { + t.Fatalf("warnings = %v", rec.warnings) + } +} + +func keys(m map[string]*corev1.Secret) []string { + var out []string + for k := range m { + out = append(out, k) + } + return out +} diff --git a/pkg/registry/core/credential/rest.go b/pkg/registry/core/credential/rest.go new file mode 100644 index 000000000..d809f78af --- /dev/null +++ b/pkg/registry/core/credential/rest.go @@ -0,0 +1,336 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package credential serves core.cozystack.io/credentials: a view over the +// account Secrets the Cozystack API writes for the accounts of converted +// applications, with the mint and revoke subresources that are the only way +// to obtain or retire a password. The view shows the account's record and +// never the Secret's data. See design-proposals/user-secrets-api in +// cozystack/community. +package credential + +import ( + "context" + "fmt" + "net/http" + "time" + + helmv2 "github.com/fluxcd/helm-controller/api/v2" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metainternal "k8s.io/apimachinery/pkg/apis/meta/internalversion" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/duration" + "k8s.io/apiserver/pkg/endpoints/request" + "k8s.io/apiserver/pkg/registry/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + + cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" + appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1" + corev1alpha1 "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1" + "github.com/cozystack/cozystack/pkg/credentials/account" + "github.com/cozystack/cozystack/pkg/registry" + fieldfilter "github.com/cozystack/cozystack/pkg/registry/fields" + "github.com/cozystack/cozystack/pkg/registry/sorting" +) + +var ( + _ rest.Getter = &REST{} + _ rest.Lister = &REST{} + _ rest.TableConvertor = &REST{} + _ rest.Scoper = &REST{} + _ rest.SingularNameProvider = &REST{} +) + +const ( + singularName = "credential" + kindCredential = "Credential" + kindList = "CredentialList" +) + +// Kind is what the registry knows about one converted application kind: its +// credentials block, and how to resolve the accounts an application declares +// from the values its HelmRelease carries, which the application registry +// does with the kind's schema defaults applied. +type Kind struct { + Block *cozyv1alpha1.ApplicationDefinitionCredentials + Accounts func(ctx context.Context, hr *helmv2.HelmRelease) ([]account.Account, error) +} + +// REST serves the credentials view. +type REST struct { + c client.Client + gvr schema.GroupVersionResource + // kinds is keyed by the application kind the HelmRelease labels name. + kinds map[string]Kind + now func() time.Time +} + +// NewREST creates the credentials storage. kinds carries every converted +// application kind; a HelmRelease of a kind absent from it has no credentials. +func NewREST(c client.Client, kinds map[string]Kind) *REST { + return &REST{ + c: c, + gvr: schema.GroupVersionResource{ + Group: corev1alpha1.GroupName, + Version: "v1alpha1", + Resource: "credentials", + }, + kinds: kinds, + now: time.Now, + } +} + +func (*REST) NamespaceScoped() bool { return true } +func (*REST) GetSingularName() string { return singularName } +func (*REST) Destroy() {} +func (*REST) New() runtime.Object { return &corev1alpha1.Credential{} } +func (*REST) NewList() runtime.Object { return &corev1alpha1.CredentialList{} } +func (*REST) Kind() string { return kindCredential } +func (r *REST) GroupVersionKind(schema.GroupVersion) schema.GroupVersionKind { + return r.gvr.GroupVersion().WithKind(kindCredential) +} + +func nsFrom(ctx context.Context) (string, error) { + ns, ok := request.NamespaceFrom(ctx) + if !ok || ns == "" { + return "", apierrors.NewBadRequest("namespace required") + } + return ns, nil +} + +// resolved is one account the registry has located: its HelmRelease, its +// kind, and its Secret where one exists. +type resolved struct { + hr *helmv2.HelmRelease + kind Kind + mapped string + secret *corev1.Secret + // stale is a Secret at the account's name that is not the application's + // own, left from a deleted application of the same name; a mint replaces + // it rather than reusing it. + stale *corev1.Secret +} + +// resolve locates the account a Credential name points at. The HelmRelease +// must exist and belong to a converted kind, or the credential does not exist. +// A Secret that is not the application's own, by marker or by owner, counts +// as absent. +func (r *REST) resolve(ctx context.Context, namespace, name string) (*resolved, error) { + release, mapped, ok := account.SplitCredentialName(name) + if !ok { + return nil, apierrors.NewNotFound(r.gvr.GroupResource(), name) + } + hr := &helmv2.HelmRelease{} + if err := r.c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: release}, hr); err != nil { + if apierrors.IsNotFound(err) { + return nil, apierrors.NewNotFound(r.gvr.GroupResource(), name) + } + return nil, err + } + kind, ok := r.kinds[hr.Labels[appsv1alpha1.ApplicationKindLabel]] + if !ok || kind.Block == nil { + return nil, apierrors.NewNotFound(r.gvr.GroupResource(), name) + } + res := &resolved{hr: hr, kind: kind, mapped: mapped} + sec := &corev1.Secret{} + err := r.c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: account.SecretName(release, mapped)}, sec) + switch { + case err == nil && ownedBy(sec, hr): + res.secret = sec + case err == nil: + res.stale = sec + case !apierrors.IsNotFound(err): + return nil, err + } + return res, nil +} + +// ownedBy reports whether a Secret is an account Secret owned by the +// HelmRelease, which is what makes it the application's own. +func ownedBy(sec *corev1.Secret, hr *helmv2.HelmRelease) bool { + if !account.IsAccountSecret(sec) { + return false + } + for _, ref := range sec.OwnerReferences { + if ref.UID == hr.UID { + return true + } + } + return false +} + +// toCredential builds the view of one account Secret. +func toCredential(sec *corev1.Secret, hr *helmv2.HelmRelease) corev1alpha1.Credential { + rec := account.ReadRecord(sec) + _, mapped, _ := account.SplitCredentialName(sec.Name[:len(sec.Name)-len(account.SecretSuffix)]) + return corev1alpha1.Credential{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1alpha1.SchemeGroupVersion.String(), + Kind: kindCredential, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: account.CredentialName(hr.Name, mapped), + Namespace: sec.Namespace, + UID: sec.UID, + ResourceVersion: sec.ResourceVersion, + CreationTimestamp: sec.CreationTimestamp, + }, + Spec: corev1alpha1.CredentialSpec{ + Application: corev1alpha1.CredentialApplicationRef{ + APIGroup: hr.Labels[appsv1alpha1.ApplicationGroupLabel], + Kind: hr.Labels[appsv1alpha1.ApplicationKindLabel], + Name: hr.Labels[appsv1alpha1.ApplicationNameLabel], + }, + User: string(sec.Data[account.UsernameKey]), + }, + Status: corev1alpha1.CredentialStatus{ + Password: corev1alpha1.CredentialPasswordState{Origin: rec.Origin, At: rec.At, By: rec.By}, + History: rec.History, + }, + } +} + +func (r *REST) Get(ctx context.Context, name string, _ *metav1.GetOptions) (runtime.Object, error) { + ns, err := nsFrom(ctx) + if err != nil { + return nil, err + } + res, err := r.resolve(ctx, ns, name) + if err != nil { + return nil, err + } + if res.secret == nil { + return nil, apierrors.NewNotFound(r.gvr.GroupResource(), name) + } + cred := toCredential(res.secret, res.hr) + return &cred, nil +} + +func (r *REST) List(ctx context.Context, opts *metainternal.ListOptions) (runtime.Object, error) { + ns, err := nsFrom(ctx) + if err != nil { + return nil, err + } + out := &corev1alpha1.CredentialList{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1alpha1.SchemeGroupVersion.String(), + Kind: kindList, + }, + } + fieldFilter, err := fieldfilter.ParseFieldSelector(opts.FieldSelector) + if err != nil { + return nil, err + } + if fieldFilter.Namespace != "" && fieldFilter.Namespace != ns { + return out, nil + } + + sel := labels.NewSelector() + req, _ := labels.NewRequirement(account.AccountLabel, selection.Equals, []string{"true"}) + sel = sel.Add(*req) + if opts.LabelSelector != nil { + reqs, selectable := opts.LabelSelector.Requirements() + if !selectable { + return out, nil + } + sel = sel.Add(reqs...) + } + + list := &corev1.SecretList{} + if err := r.c.List(ctx, list, &client.ListOptions{Namespace: ns, LabelSelector: sel}); err != nil { + return nil, err + } + releases := map[string]*helmv2.HelmRelease{} + for i := range list.Items { + sec := &list.Items[i] + hr, err := r.liveOwner(ctx, sec, releases) + if err != nil { + return nil, err + } + if hr == nil { + continue + } + cred := toCredential(sec, hr) + if !fieldFilter.MatchesName(cred.Name) { + continue + } + out.Items = append(out.Items, cred) + } + listRV := list.ResourceVersion + if listRV == "" { + listRV, _ = registry.MaxResourceVersion(list) + } + out.ResourceVersion = listRV + sorting.ByNamespacedName[corev1alpha1.Credential, *corev1alpha1.Credential](out.Items) + return out, nil +} + +// liveOwner returns the HelmRelease that owns an account Secret, provided it +// exists and belongs to a converted kind; nil otherwise, so a Secret no live +// release owns is not listed. +func (r *REST) liveOwner(ctx context.Context, sec *corev1.Secret, cache map[string]*helmv2.HelmRelease) (*helmv2.HelmRelease, error) { + for _, ref := range sec.OwnerReferences { + if ref.Kind != helmv2.HelmReleaseKind { + continue + } + hr, seen := cache[ref.Name] + if !seen { + hr = &helmv2.HelmRelease{} + if err := r.c.Get(ctx, types.NamespacedName{Namespace: sec.Namespace, Name: ref.Name}, hr); err != nil { + if !apierrors.IsNotFound(err) { + return nil, err + } + hr = nil + } + cache[ref.Name] = hr + } + if hr == nil || hr.UID != ref.UID { + continue + } + if kind, ok := r.kinds[hr.Labels[appsv1alpha1.ApplicationKindLabel]]; !ok || kind.Block == nil { + continue + } + return hr, nil + } + return nil, nil +} + +func (r *REST) ConvertToTable(ctx context.Context, object runtime.Object, _ runtime.Object) (*metav1.Table, error) { + table := &metav1.Table{ + ColumnDefinitions: []metav1.TableColumnDefinition{ + {Name: "Name", Type: "string", Format: "name"}, + {Name: "Application", Type: "string"}, + {Name: "User", Type: "string"}, + {Name: "Origin", Type: "string"}, + {Name: "Age", Type: "string"}, + }, + } + now := r.now() + row := func(c *corev1alpha1.Credential) metav1.TableRow { + age := "" + if !c.CreationTimestamp.IsZero() { + age = duration.HumanDuration(now.Sub(c.CreationTimestamp.Time)) + } + return metav1.TableRow{ + Cells: []any{c.Name, c.Spec.Application.Kind + "/" + c.Spec.Application.Name, c.Spec.User, string(c.Status.Password.Origin), age}, + Object: runtime.RawExtension{Object: c}, + } + } + switch v := object.(type) { + case *corev1alpha1.Credential: + table.Rows = append(table.Rows, row(v)) + case *corev1alpha1.CredentialList: + table.ResourceVersion = v.ResourceVersion + for i := range v.Items { + table.Rows = append(table.Rows, row(&v.Items[i])) + } + default: + return nil, apierrors.NewGenericServerResponse(http.StatusInternalServerError, "", r.gvr.GroupResource(), "", fmt.Sprintf("unexpected object type %T", object), 0, false) + } + return table, nil +} diff --git a/pkg/registry/core/credential/rest_test.go b/pkg/registry/core/credential/rest_test.go new file mode 100644 index 000000000..ba2f264b8 --- /dev/null +++ b/pkg/registry/core/credential/rest_test.go @@ -0,0 +1,309 @@ +// SPDX-License-Identifier: Apache-2.0 + +package credential + +import ( + "context" + "strings" + "testing" + "time" + + helmv2 "github.com/fluxcd/helm-controller/api/v2" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metainternal "k8s.io/apimachinery/pkg/apis/meta/internalversion" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apiserver/pkg/authentication/user" + "k8s.io/apiserver/pkg/endpoints/request" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" + appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1" + corev1alpha1 "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1" + "github.com/cozystack/cozystack/pkg/credentials/account" + "github.com/cozystack/cozystack/pkg/credentials/store" +) + +const ns = "tenant-foo" + +func testBlock() *cozyv1alpha1.ApplicationDefinitionCredentials { + return &cozyv1alpha1.ApplicationDefinitionCredentials{ + Format: cozyv1alpha1.CredentialFormat{Name: "sha256"}, + Accounts: []cozyv1alpha1.CredentialAccountSource{ + {Kind: cozyv1alpha1.CredentialAccountsValuesMap, Path: "users"}, + }, + } +} + +// fixture is a fake cluster with one converted Postgres application whose +// declared accounts are fixed by the test. +type fixture struct { + c client.Client + r *REST + hr *helmv2.HelmRelease + ctx context.Context + declare []account.Account +} + +func newFixture(t *testing.T, objs ...client.Object) *fixture { + t.Helper() + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := helmv2.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + hr := &helmv2.HelmRelease{ObjectMeta: metav1.ObjectMeta{ + Name: "postgres-orders", Namespace: ns, UID: "hr-uid", + Labels: map[string]string{ + appsv1alpha1.ApplicationGroupLabel: appsv1alpha1.GroupName, + appsv1alpha1.ApplicationKindLabel: "Postgres", + appsv1alpha1.ApplicationNameLabel: "orders", + }, + }} + f := &fixture{hr: hr} + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(append(objs, hr)...).Build() + f.c = c + f.r = NewREST(c, map[string]Kind{"Postgres": { + Block: testBlock(), + Accounts: func(context.Context, *helmv2.HelmRelease) ([]account.Account, error) { return f.declare, nil }, + }}) + f.r.now = func() time.Time { return time.Date(2026, 10, 2, 9, 14, 7, 0, time.UTC) } + f.ctx = request.WithUser(request.WithNamespace(context.Background(), ns), &user.DefaultInfo{Name: "jane@example.org"}) + return f +} + +func (f *fixture) app() store.App { + return store.App{Namespace: ns, Release: "postgres-orders", Group: appsv1alpha1.GroupName, Kind: "Postgres", Name: "orders", Block: testBlock()} +} + +// seed writes an owned account Secret for user and returns it as stored. +func (f *fixture) seed(t *testing.T, username string) *corev1.Secret { + t.Helper() + owner := store.OwnerReference(f.hr) + mapped, _ := account.MapName(username) + sec, err := store.SeedSecret(f.app(), account.Account{Username: username, Mapped: mapped}, &owner, metav1.NewTime(f.r.now()), "seeder") + if err != nil { + t.Fatal(err) + } + if err := f.c.Create(f.ctx, sec); err != nil { + t.Fatal(err) + } + return f.secret(t, sec.Name) +} + +func (f *fixture) secret(t *testing.T, name string) *corev1.Secret { + t.Helper() + sec := &corev1.Secret{} + if err := f.c.Get(f.ctx, types.NamespacedName{Namespace: ns, Name: name}, sec); err != nil { + t.Fatalf("get %s: %v", name, err) + } + return sec +} + +func (f *fixture) mint(t *testing.T, name, precondition string) (*corev1alpha1.CredentialRequest, error) { + t.Helper() + req := &corev1alpha1.CredentialRequest{} + if precondition != "" { + req.Spec.Preconditions = &corev1alpha1.CredentialRequestPreconditions{ResourceVersion: precondition} + } + out, err := NewMintREST(f.r).Create(f.ctx, name, req, nil, &metav1.CreateOptions{}) + if err != nil { + return nil, err + } + return out.(*corev1alpha1.CredentialRequest), nil +} + +func (f *fixture) events(t *testing.T) []corev1.Event { + t.Helper() + list := &corev1.EventList{} + if err := f.c.List(f.ctx, list, client.InNamespace(ns)); err != nil { + t.Fatal(err) + } + return list.Items +} + +func TestGetShowsRecordNeverData(t *testing.T) { + f := newFixture(t) + sec := f.seed(t, "Web") + + obj, err := f.r.Get(f.ctx, "postgres-orders.web", &metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + cred := obj.(*corev1alpha1.Credential) + if cred.Name != "postgres-orders.web" || cred.Spec.User != "Web" || cred.Spec.Application.Name != "orders" || cred.Spec.Application.Kind != "Postgres" { + t.Fatalf("credential = %+v", cred) + } + if cred.ResourceVersion != sec.ResourceVersion { + t.Fatalf("resourceVersion is the Secret's: %q != %q", cred.ResourceVersion, sec.ResourceVersion) + } + if cred.Status.Password.Origin != corev1alpha1.CredentialOriginNotIssued || cred.Status.Password.By != "seeder" { + t.Fatalf("status = %+v", cred.Status) + } + table, err := f.r.ConvertToTable(f.ctx, cred, nil) + if err != nil { + t.Fatal(err) + } + if len(table.Rows) != 1 || table.Rows[0].Cells[1] != "Postgres/orders" || table.Rows[0].Cells[3] != "NotIssued" { + t.Fatalf("table = %+v", table.Rows) + } + for _, name := range []string{"postgres-orders.nobody", "postgres-orders", "postgres-orders.web.account"} { + if _, err := f.r.Get(f.ctx, name, &metav1.GetOptions{}); !apierrors.IsNotFound(err) { + t.Errorf("Get(%q) = %v, want NotFound", name, err) + } + } +} + +func TestListSkipsSecretsNoLiveReleaseOwns(t *testing.T) { + gone := &helmv2.HelmRelease{ObjectMeta: metav1.ObjectMeta{Name: "postgres-gone", Namespace: ns, UID: "gone-uid"}} + f := newFixture(t) + f.seed(t, "web") + // Owned by a release that no longer exists. + orphan, _ := store.SeedSecret(store.App{Namespace: ns, Release: "postgres-gone", Group: appsv1alpha1.GroupName, Kind: "Postgres", Name: "gone", Block: testBlock()}, account.Account{Username: "web", Mapped: "web"}, ptr(store.OwnerReference(gone)), metav1.Now(), "x") + if err := f.c.Create(f.ctx, orphan); err != nil { + t.Fatal(err) + } + // A Secret of an unconverted kind carrying the marker by accident. + other := &helmv2.HelmRelease{ObjectMeta: metav1.ObjectMeta{Name: "redis-cache", Namespace: ns, UID: "redis-uid", Labels: map[string]string{appsv1alpha1.ApplicationKindLabel: "Redis", appsv1alpha1.ApplicationNameLabel: "cache"}}} + if err := f.c.Create(f.ctx, other); err != nil { + t.Fatal(err) + } + stray, _ := store.SeedSecret(store.App{Namespace: ns, Release: "redis-cache", Kind: "Redis", Name: "cache", Block: testBlock()}, account.Account{Username: "default", Mapped: "default"}, ptr(store.OwnerReference(other)), metav1.Now(), "x") + if err := f.c.Create(f.ctx, stray); err != nil { + t.Fatal(err) + } + + obj, err := f.r.List(f.ctx, &metainternal.ListOptions{}) + if err != nil { + t.Fatal(err) + } + list := obj.(*corev1alpha1.CredentialList) + if len(list.Items) != 1 || list.Items[0].Name != "postgres-orders.web" { + t.Fatalf("list = %+v", list.Items) + } +} + +func TestMintReplacesPasswordOnceAndRecords(t *testing.T) { + f := newFixture(t) + before := f.seed(t, "web") + + res, err := f.mint(t, "postgres-orders.web", before.ResourceVersion) + if err != nil { + t.Fatal(err) + } + if res.Status.Username != "web" || len(res.Status.Password) != 32 || res.Status.IssuedAt.Time != f.r.now() { + t.Fatalf("response = %+v", res.Status) + } + after := f.secret(t, before.Name) + if string(after.Data[account.PasswordKey]) == string(before.Data[account.PasswordKey]) { + t.Fatal("stored value unchanged") + } + if strings.Contains(string(after.Data[account.PasswordKey]), res.Status.Password) { + t.Fatal("the plaintext must not be stored") + } + rec := account.ReadRecord(after) + if rec.Origin != corev1alpha1.CredentialOriginMinted || rec.By != "jane@example.org" || len(rec.History) != 2 || rec.History[0].Operation != corev1alpha1.CredentialOperationMint { + t.Fatalf("record = %+v", rec) + } + evs := f.events(t) + if len(evs) != 1 || evs[0].Reason != store.ReasonCredentialIssued || evs[0].InvolvedObject.Kind != "Postgres" || evs[0].InvolvedObject.Name != "orders" { + t.Fatalf("events = %+v", evs) + } + if evs[0].Message != "password of user web issued by jane@example.org" || strings.Contains(evs[0].Message, res.Status.Password) { + t.Fatalf("event message = %q", evs[0].Message) + } + + // The stale precondition is the one from before the mint: conflict, + // naming who issued in between, and nothing written. + _, err = f.mint(t, "postgres-orders.web", before.ResourceVersion) + if !apierrors.IsConflict(err) || !strings.Contains(err.Error(), "password issued by jane@example.org at 2026-10-02T09:14:07Z") { + t.Fatalf("expected conflict naming the issuer, got %v", err) + } + if string(f.secret(t, before.Name).Data[account.PasswordKey]) != string(after.Data[account.PasswordKey]) { + t.Fatal("a refused mint must not write") + } + // No precondition: always proceeds. + if _, err := f.mint(t, "postgres-orders.web", ""); err != nil { + t.Fatal(err) + } +} + +func TestRevokeReturnsNoPassword(t *testing.T) { + f := newFixture(t) + before := f.seed(t, "web") + out, err := NewRevokeREST(f.r).Create(f.ctx, "postgres-orders.web", &corev1alpha1.CredentialRequest{}, nil, &metav1.CreateOptions{}) + if err != nil { + t.Fatal(err) + } + res := out.(*corev1alpha1.CredentialRequest) + if res.Status.Password != "" || res.Status.Username != "web" { + t.Fatalf("response = %+v", res.Status) + } + after := f.secret(t, before.Name) + if string(after.Data[account.PasswordKey]) == string(before.Data[account.PasswordKey]) { + t.Fatal("revoke must replace the stored value") + } + if rec := account.ReadRecord(after); rec.Origin != corev1alpha1.CredentialOriginRevoked { + t.Fatalf("record = %+v", rec) + } + if evs := f.events(t); len(evs) != 1 || evs[0].Reason != store.ReasonCredentialRevoked { + t.Fatalf("events = %+v", evs) + } +} + +func TestMintCreatesMissingDeclaredAccount(t *testing.T) { + f := newFixture(t) + f.declare = []account.Account{{Username: "Web", Mapped: "web"}} + + // Not declared: does not exist. + if _, err := f.mint(t, "postgres-orders.api", ""); !apierrors.IsNotFound(err) { + t.Fatalf("undeclared account: %v", err) + } + // A stale Secret at the name, from a deleted application, is replaced. + gone := &helmv2.HelmRelease{ObjectMeta: metav1.ObjectMeta{Name: "postgres-orders", Namespace: ns, UID: "old-uid"}} + stale, _ := store.SeedSecret(f.app(), account.Account{Username: "Web", Mapped: "web"}, ptr(store.OwnerReference(gone)), metav1.Now(), "x") + if err := f.c.Create(f.ctx, stale); err != nil { + t.Fatal(err) + } + if _, err := f.r.Get(f.ctx, "postgres-orders.web", &metav1.GetOptions{}); !apierrors.IsNotFound(err) { + t.Fatalf("a stale Secret must not be served: %v", err) + } + // A precondition on a missing Secret is ignored. + res, err := f.mint(t, "postgres-orders.web", "12345") + if err != nil { + t.Fatal(err) + } + if res.Status.Username != "Web" { + t.Fatalf("username must be the declared one: %+v", res.Status) + } + sec := f.secret(t, "postgres-orders.web.account") + if len(sec.OwnerReferences) != 1 || sec.OwnerReferences[0].UID != f.hr.UID || string(sec.Data[account.UsernameKey]) != "Web" { + t.Fatalf("created Secret = %+v", sec.ObjectMeta) + } + rec := account.ReadRecord(sec) + if rec.Origin != corev1alpha1.CredentialOriginMinted || len(rec.History) != 1 { + t.Fatalf("record = %+v", rec) + } +} + +func TestUnconvertedKindHasNoCredentials(t *testing.T) { + f := newFixture(t) + f.hr.Labels[appsv1alpha1.ApplicationKindLabel] = "Redis" + if err := f.c.Update(f.ctx, f.hr); err != nil { + t.Fatal(err) + } + f.seed(t, "web") + if _, err := f.r.Get(f.ctx, "postgres-orders.web", &metav1.GetOptions{}); !apierrors.IsNotFound(err) { + t.Fatalf("Get = %v", err) + } + if _, err := f.mint(t, "postgres-orders.web", ""); !apierrors.IsNotFound(err) { + t.Fatalf("mint = %v", err) + } +} + +func ptr[T any](v T) *T { return &v } diff --git a/pkg/registry/core/credential/subresources.go b/pkg/registry/core/credential/subresources.go new file mode 100644 index 000000000..5dbf2b8e4 --- /dev/null +++ b/pkg/registry/core/credential/subresources.go @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: Apache-2.0 + +package credential + +import ( + "context" + "fmt" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apiserver/pkg/endpoints/request" + "k8s.io/apiserver/pkg/registry/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + + appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1" + corev1alpha1 "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1" + "github.com/cozystack/cozystack/pkg/credentials/account" + "github.com/cozystack/cozystack/pkg/credentials/format" + "github.com/cozystack/cozystack/pkg/credentials/store" +) + +var ( + _ rest.NamedCreater = &OperationREST{} + _ rest.Scoper = &OperationREST{} + _ rest.GroupVersionKindProvider = &OperationREST{} +) + +// OperationREST is the mint or the revoke subresource of a Credential. Both +// take a CredentialRequest, replace the account's stored value with that of a +// fresh random password in one conditional write of the account Secret, and +// answer with a CredentialRequest. A mint returns the password, once; a +// revoke discards it, so nobody holds a working one until the next mint. +type OperationREST struct { + parent *REST + op corev1alpha1.CredentialOperation +} + +// NewMintREST and NewRevokeREST return the two subresource storages of a +// credentials registry. +func NewMintREST(parent *REST) *OperationREST { + return &OperationREST{parent: parent, op: corev1alpha1.CredentialOperationMint} +} + +func NewRevokeREST(parent *REST) *OperationREST { + return &OperationREST{parent: parent, op: corev1alpha1.CredentialOperationRevoke} +} + +func (*OperationREST) New() runtime.Object { return &corev1alpha1.CredentialRequest{} } +func (*OperationREST) Destroy() {} +func (*OperationREST) NamespaceScoped() bool { return true } +func (o *OperationREST) GroupVersionKind(schema.GroupVersion) schema.GroupVersionKind { + return o.parent.gvr.GroupVersion().WithKind("CredentialRequest") +} + +// Create performs the operation on the named Credential. +func (o *OperationREST) Create(ctx context.Context, name string, obj runtime.Object, createValidation rest.ValidateObjectFunc, _ *metav1.CreateOptions) (runtime.Object, error) { + req, ok := obj.(*corev1alpha1.CredentialRequest) + if !ok { + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected CredentialRequest, got %T", obj)) + } + ns, err := nsFrom(ctx) + if err != nil { + return nil, err + } + user, ok := request.UserFrom(ctx) + if !ok { + return nil, apierrors.NewUnauthorized("user missing in context") + } + if createValidation != nil { + if err := createValidation(ctx, req); err != nil { + return nil, err + } + } + + r := o.parent + res, err := r.resolve(ctx, ns, name) + if err != nil { + return nil, err + } + + var precondition string + if req.Spec.Preconditions != nil { + precondition = req.Spec.Preconditions.ResourceVersion + } + + // The Secret's version is the Credential's, so a caller that read the + // credential and sends that version back learns, as a conflict, that + // somebody minted in between instead of silently killing the password that + // person just received. A missing Secret takes no precondition. + if res.secret != nil && precondition != "" && precondition != res.secret.ResourceVersion { + rec := account.ReadRecord(res.secret) + return nil, apierrors.NewConflict(r.gvr.GroupResource(), name, fmt.Errorf( + "credential %s changed: password %s by %s at %s", name, describe(rec.Origin), rec.By, rec.At.UTC().Format("2006-01-02T15:04:05Z"))) + } + + password, err := format.GeneratePassword() + if err != nil { + return nil, apierrors.NewInternalError(err) + } + stored, err := format.Encode(res.kind.Block.Format.Name, password) + if err != nil { + return nil, apierrors.NewInternalError(err) + } + now := metav1.NewTime(r.now()) + by := user.GetName() + + var username string + if res.secret != nil { + username = string(res.secret.Data[account.UsernameKey]) + sec := res.secret.DeepCopy() + sec.Data[account.PasswordKey] = []byte(stored) + account.WriteRecord(sec, account.ReadRecord(sec).Apply(o.op, now, by)) + // The update carries the Secret's own resourceVersion, so two concurrent + // operations cannot both land: the second gets a conflict and no password. + if err := r.c.Update(ctx, sec); err != nil { + if apierrors.IsConflict(err) { + return nil, apierrors.NewConflict(r.gvr.GroupResource(), name, fmt.Errorf("credential %s changed while the request was in flight", name)) + } + return nil, err + } + } else { + // A declared account whose Secret is missing, as after a conversion + // that left the release out, gets a Secret with a new identity. One that + // is not declared does not exist. + acc, err := o.declared(ctx, res) + if err != nil { + return nil, err + } + username = acc.Username + app := store.App{ + Namespace: ns, Release: res.hr.Name, + Group: res.hr.Labels[appsv1alpha1.ApplicationGroupLabel], + Kind: res.hr.Labels[appsv1alpha1.ApplicationKindLabel], + Name: res.hr.Labels[appsv1alpha1.ApplicationNameLabel], + Block: res.kind.Block, + } + if res.stale != nil { + if err := r.c.Delete(ctx, res.stale, client.Preconditions{UID: &res.stale.UID}); err != nil && !apierrors.IsNotFound(err) { + return nil, err + } + } + owner := store.OwnerReference(res.hr) + sec, err := store.SeedSecret(app, acc, &owner, now, by) + if err != nil { + return nil, apierrors.NewInternalError(err) + } + sec.Data[account.PasswordKey] = []byte(stored) + account.WriteRecord(sec, account.Record{}.Apply(o.op, now, by)) + if err := r.c.Create(ctx, sec); err != nil { + if apierrors.IsAlreadyExists(err) { + return nil, apierrors.NewConflict(r.gvr.GroupResource(), name, fmt.Errorf("credential %s was created while the request was in flight", name)) + } + return nil, err + } + } + + reason := store.ReasonCredentialIssued + if o.op == corev1alpha1.CredentialOperationRevoke { + reason = store.ReasonCredentialRevoked + password = "" + } + store.Emit(ctx, r.c, res.hr, reason, username, by) + + return &corev1alpha1.CredentialRequest{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1alpha1.SchemeGroupVersion.String(), + Kind: "CredentialRequest", + }, + Status: corev1alpha1.CredentialRequestStatus{ + Username: username, + Password: password, + IssuedAt: now, + }, + }, nil +} + +// declared finds the declared account a Credential name maps to. A name no +// declared user maps to is NotFound; one two declared users map to is a +// Conflict, since the write that declared them should have been refused. +func (o *OperationREST) declared(ctx context.Context, res *resolved) (account.Account, error) { + r := o.parent + name := account.CredentialName(res.hr.Name, res.mapped) + if res.kind.Accounts == nil { + return account.Account{}, apierrors.NewNotFound(r.gvr.GroupResource(), name) + } + accounts, err := res.kind.Accounts(ctx, res.hr) + if err != nil { + return account.Account{}, apierrors.NewConflict(r.gvr.GroupResource(), name, err) + } + for _, acc := range accounts { + if acc.Mapped == res.mapped { + return acc, nil + } + } + return account.Account{}, apierrors.NewNotFound(r.gvr.GroupResource(), name) +} + +func describe(origin corev1alpha1.CredentialOrigin) string { + switch origin { + case corev1alpha1.CredentialOriginMinted: + return "issued" + case corev1alpha1.CredentialOriginRevoked: + return "revoked" + case corev1alpha1.CredentialOriginMigrated: + return "migrated" + } + return "seeded" +}