diff --git a/hack/e2e-apps/kubernetes-oidc.bats b/hack/e2e-apps/kubernetes-oidc-system.bats similarity index 100% rename from hack/e2e-apps/kubernetes-oidc.bats rename to hack/e2e-apps/kubernetes-oidc-system.bats diff --git a/packages/apps/kubernetes/templates/oidc-kubeconfig-secret.yaml b/packages/apps/kubernetes/templates/oidc-kubeconfig-secret.yaml new file mode 100644 index 000000000..14206cd25 --- /dev/null +++ b/packages/apps/kubernetes/templates/oidc-kubeconfig-secret.yaml @@ -0,0 +1,31 @@ +{{- /* + Pre-render the OIDC kubeconfig Secret as a chart-owned resource so + `helm uninstall` cleans it up. The bootstrap Job patches the `data` + field at runtime with the actual kubeconfig (it cannot be rendered + at template time because the tenant CA is generated asynchronously + by Kamaji). + + Emitted only for `mode: System`; in `CustomConfig` mode the chart + does not know the issuer/clientId — the operator distributes the + OIDC kubeconfig out-of-band from their own IdP configuration. + + The placeholder content keeps the Secret addressable by the + dashboard immediately on install; the bootstrap Job swaps it for a + real kubeconfig as soon as the Kamaji-issued admin kubeconfig is + available. +*/}} +{{- if eq .Values.oidc.mode "System" }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Release.Name }}-oidc-kubeconfig + namespace: {{ .Release.Namespace }} + labels: + {{- include "kubernetes.labels" . | nindent 4 }} +type: Opaque +stringData: + kubeconfig: | + # awaiting OIDC bootstrap Job — the tenant CA is generated by + # Kamaji asynchronously; the Job populates this Secret as soon as + # the admin kubeconfig appears. +{{- end }} diff --git a/packages/apps/kubernetes/templates/oidc-rbac-job.yaml b/packages/apps/kubernetes/templates/oidc-rbac-job.yaml index 8a03a3d28..d5ac305f8 100644 --- a/packages/apps/kubernetes/templates/oidc-rbac-job.yaml +++ b/packages/apps/kubernetes/templates/oidc-rbac-job.yaml @@ -10,14 +10,26 @@ 2. Remove ClusterRoleBindings labelled by the release that are no longer in `users[]` — toggling a user out of the list revokes their kubectl access on the next reconcile. - 3. For `mode: System`, also write a ready-to-use OIDC kubeconfig - Secret in the management cluster's tenant namespace - (`-oidc-kubeconfig`) wired to `kubectl oidc-login` so - a user only needs to run `kubectl --kubeconfig … get pods` to - trigger the browser flow. + 3. For `mode: System`, patch the pre-rendered + `-oidc-kubeconfig` Secret with a ready-to-use + kubeconfig wired to `kubectl oidc-login` (the tenant CA is + generated by Kamaji asynchronously, so the Job extracts it + from the admin kubeconfig at runtime). - Rendered only when `spec.oidc.mode` is not None. Skipped entirely - for `mode: None`. + Lifecycle: + - ServiceAccount, Role, RoleBinding are RELEASE-OWNED (no + `helm.sh/hook` annotations). They are created on `helm install` + and deleted on `helm uninstall`, so both the post-install + bootstrap hook AND the pre-delete cleanup hook find a working + SA. This avoids the race where a hook-scoped SA is deleted + before pre-delete fires. + - The bootstrap Job itself stays a hook so `helm upgrade` + re-runs it on every reconcile (picks up `users[]` changes). + - The Secret it patches is pre-rendered in + `oidc-kubeconfig-secret.yaml` and is also release-owned, so + `helm uninstall` cleans it up. + + Rendered only when `spec.oidc.mode` is not None. Pod is labelled `policy.cozystack.io/allow-to-apiserver: "true"` so the Cilium policy lets it reach the tenant apiserver through @@ -34,10 +46,6 @@ kind: ServiceAccount metadata: name: {{ .Release.Name }}-oidc-bootstrap namespace: {{ .Release.Namespace }} - annotations: - "helm.sh/hook": post-install,post-upgrade - "helm.sh/hook-weight": "0" - "helm.sh/hook-delete-policy": before-hook-creation labels: {{- include "kubernetes.labels" . | nindent 4 }} --- @@ -46,36 +54,20 @@ kind: Role metadata: name: {{ .Release.Name }}-oidc-bootstrap namespace: {{ .Release.Namespace }} - annotations: - "helm.sh/hook": post-install,post-upgrade - "helm.sh/hook-weight": "1" - "helm.sh/hook-delete-policy": before-hook-creation labels: {{- include "kubernetes.labels" . | nindent 4 }} rules: - - apiGroups: [""] - resources: ["secrets"] - resourceNames: - - {{ .Release.Name }}-admin-kubeconfig - verbs: ["get"] - apiGroups: [""] resources: ["secrets"] resourceNames: - {{ .Release.Name }}-oidc-kubeconfig - verbs: ["get", "create", "update", "patch"] - - apiGroups: [""] - resources: ["secrets"] - verbs: ["create"] + verbs: ["get", "patch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: {{ .Release.Name }}-oidc-bootstrap namespace: {{ .Release.Namespace }} - annotations: - "helm.sh/hook": post-install,post-upgrade - "helm.sh/hook-weight": "2" - "helm.sh/hook-delete-policy": before-hook-creation labels: {{- include "kubernetes.labels" . | nindent 4 }} roleRef: @@ -151,7 +143,7 @@ spec: - /bin/sh - -c - | - set -eu + set -euo pipefail echo "Waiting for the tenant admin kubeconfig to appear..." deadline=$(( $(date +%s) + 600 )) until [ -s "$KUBECONFIG_TENANT" ]; do @@ -192,9 +184,10 @@ spec: {{- end }} echo "Pruning stale ClusterRoleBindings labelled by this release..." - for crb in $(kubectl --kubeconfig "$KUBECONFIG_TENANT" get clusterrolebindings \ - --selector=app.kubernetes.io/managed-by=cozystack-oidc,app.kubernetes.io/instance=$RELEASE \ - --output=jsonpath='{.items[*].metadata.name}'); do + EXISTING=$(kubectl --kubeconfig "$KUBECONFIG_TENANT" get clusterrolebindings \ + --selector=app.kubernetes.io/managed-by=cozystack-oidc,app.kubernetes.io/instance=$RELEASE \ + --output=jsonpath='{.items[*].metadata.name}') + for crb in $EXISTING; do if ! grep --line-regexp --quiet --fixed-strings "$crb" "$DESIRED_FILE"; then echo " removing $crb" kubectl --kubeconfig "$KUBECONFIG_TENANT" delete clusterrolebinding "$crb" --ignore-not-found @@ -203,11 +196,11 @@ spec: rm --force "$DESIRED_FILE" if [ "$OIDC_MODE" = "System" ]; then - echo "Writing the OIDC kubeconfig Secret on the management cluster..." + echo "Patching the OIDC kubeconfig Secret on the management cluster..." CA_DATA=$(kubectl --kubeconfig "$KUBECONFIG_TENANT" config view --raw \ --output=jsonpath='{.clusters[0].cluster.certificate-authority-data}') if [ -z "$CA_DATA" ]; then - echo "WARNING: tenant admin kubeconfig has no certificate-authority-data; OIDC kubeconfig will not be written" >&2 + echo "WARNING: tenant admin kubeconfig has no certificate-authority-data; leaving OIDC kubeconfig Secret as the install-time placeholder" >&2 else KUBECONFIG_FILE=$(mktemp) cat > "$KUBECONFIG_FILE" <