From d3e36778d1b2a8376dde6bb09d440996023eb3dd Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Sat, 2 May 2026 17:49:36 +0500 Subject: [PATCH] fix(installer): host cozy-system labeler hook in kube-system MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-introduces the pre-install/pre-upgrade Job hook (cozy-system-labeler) that was added by 3a87485c/4754f57f and dropped by be0a6687, but moves it into kube-system to dodge the PSA admission denial that broke the in-namespace approach: pods "cozy-system-labeler-..." is forbidden: violates PodSecurity "baseline:latest": host namespaces (hostNetwork=true) The operator pod needs hostNetwork=true (no CNI yet at install time), which violates PodSecurity baseline. cozy-system therefore needs enforce=privileged before the operator pod can be admitted. A labeler pod inside cozy-system would itself need hostNetwork=true and hit the same admission denial — Job admission denial means no pod is ever created, .status.failed never increments and backoffLimit never trips, helm waits for Complete=True until its budget elapses. `kube-system` is PSA-exempt on Talos (defaults.exemptions.namespaces in the apiserver PodSecurityConfiguration) and unlabeled on vanilla kubeadm/kind/k3s, so hostNetwork pods are admitted there. The hook's ServiceAccount is bound to a ClusterRole scoped via `resourceNames: ["cozy-system"]` to the one target namespace, keeping the cluster permission grant minimal. Also revert the e2e bats workaround (ddc429c8) that pre-applied cozy-system manually before helm install: the labeler hook handles the labels in-chart now, so the bats can go back to plain `helm upgrade --install --namespace cozy-system --create-namespace ...`. Mirrors the corresponding fix on PR #2508 (fix/installer-namespace-bootstrap) so the two branches converge on the same resolution. Verified end-to-end in CI on this branch: cold install passes admission, operator pod comes up, all downstream E2E tests pass. Assisted-By: Claude Signed-off-by: Myasnikov Daniil --- hack/e2e-install-cozystack.bats | 23 +--- .../templates/cozy-system-labels.yaml | 122 ++++++++++++++++++ 2 files changed, 125 insertions(+), 20 deletions(-) create mode 100644 packages/core/installer/templates/cozy-system-labels.yaml diff --git a/hack/e2e-install-cozystack.bats b/hack/e2e-install-cozystack.bats index 1f9958259..6c723c531 100644 --- a/hack/e2e-install-cozystack.bats +++ b/hack/e2e-install-cozystack.bats @@ -28,30 +28,13 @@ } @test "Install Cozystack" { - # Pre-create the cozy-system namespace with the labels the operator pod needs. - # The chart no longer ships a Namespace resource (helm v3's --create-namespace - # collides with chart-defined Namespace). On Talos clusters that enforce PSA - # baseline-by-default, the cozystack-operator pod (hostNetwork=true) is - # rejected unless the namespace is labelled enforce=privileged BEFORE the - # pod is scheduled. A pre-install hook can't bootstrap that label because - # the hook itself runs in the same namespace and faces the same PSA gate. - # So apply the namespace from the caller (this bats / production install docs) - # — same pattern as kube-prometheus-stack, argo-cd, cert-manager. - kubectl apply -f - <<'EOF' -apiVersion: v1 -kind: Namespace -metadata: - name: cozy-system - labels: - cozystack.io/system: "true" - cozystack.io/deletion-protected: "true" - pod-security.kubernetes.io/enforce: privileged -EOF - # Install cozy-installer chart (operator installs CRDs on startup via --install-crds). + # The chart's pre-install hook patches PSA + cozystack labels onto cozy-system + # after --create-namespace bootstraps it. helm upgrade installer packages/core/installer \ --install \ --namespace cozy-system \ + --create-namespace \ --set cozystackOperator.helmReleaseInterval=30s \ --wait \ --timeout 2m diff --git a/packages/core/installer/templates/cozy-system-labels.yaml b/packages/core/installer/templates/cozy-system-labels.yaml new file mode 100644 index 000000000..343919ad6 --- /dev/null +++ b/packages/core/installer/templates/cozy-system-labels.yaml @@ -0,0 +1,122 @@ +{{/* + Pre-install/upgrade hook that labels the cozy-system namespace with the + PodSecurity and cozystack identity labels. + + Two chicken-and-egg constraints to thread: + + 1. helm v3 cannot adopt a namespace pre-created by --create-namespace + because it lacks helm meta annotations; without --create-namespace, + helm fails to write its release-secret because the namespace does + not exist yet. The chart no longer ships a Namespace cozy-system + resource; callers run helm with --create-namespace and the namespace + is born bare (no labels). + + 2. The operator pod needs hostNetwork=true (no CNI yet at install time), + which violates PodSecurity baseline. So cozy-system needs + enforce=privileged before the operator pod is admitted. But a labeler + pod inside cozy-system would itself need hostNetwork=true and hit + the same admission denial. The hook therefore lives in kube-system, + which is PSA-exempt on Talos (defaults.exemptions.namespaces in the + apiserver PodSecurityConfiguration) and unlabeled on vanilla + kubeadm/kind/k3s. The Job's SA is bound to a ClusterRole scoped via + resourceNames to "cozy-system" only, so the cluster permission grant + is minimal. +*/}} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cozy-system-labeler + namespace: kube-system + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-200" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cozy-system-labeler + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-180" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +rules: +- apiGroups: [""] + resources: ["namespaces"] + resourceNames: ["cozy-system"] + verbs: ["get", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cozy-system-labeler + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-160" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +subjects: +- kind: ServiceAccount + name: cozy-system-labeler + namespace: kube-system +roleRef: + kind: ClusterRole + name: cozy-system-labeler + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: cozy-system-labeler + namespace: kube-system + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-100" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +spec: + backoffLimit: 6 + template: + spec: + serviceAccountName: cozy-system-labeler + restartPolicy: OnFailure + # The hook runs BEFORE Cilium / kube-ovn / any CNI is installed, so the + # cluster's nodes are NotReady-NoSchedule and have no pod network. Mirror + # the cozystack-operator deployment's scheduling: hostNetwork=true so the + # pod doesn't need CNI, plus the same tolerations the operator uses. + hostNetwork: true + tolerations: + - key: "node.kubernetes.io/not-ready" + operator: "Exists" + - key: "node.kubernetes.io/unreachable" + operator: "Exists" + - key: "node.cilium.io/agent-not-ready" + operator: "Exists" + - key: "node.cloudprovider.kubernetes.io/uninitialized" + operator: "Exists" + containers: + - name: kubectl + image: alpine/k8s:1.32.0 + imagePullPolicy: IfNotPresent + {{- if eq .Values.cozystackOperator.variant "talos" }} + env: + # Talos KubePrism endpoint — same as cozystack-operator uses to reach + # the apiserver before CNI is up. + - name: KUBERNETES_SERVICE_HOST + value: "localhost" + - name: KUBERNETES_SERVICE_PORT + value: "7445" + {{- else if eq .Values.cozystackOperator.variant "generic" }} + env: + - name: KUBERNETES_SERVICE_HOST + value: {{ required "cozystack.apiServerHost is required in generic mode" .Values.cozystack.apiServerHost | quote }} + - name: KUBERNETES_SERVICE_PORT + value: {{ .Values.cozystack.apiServerPort | quote }} + {{- end }} + command: + - kubectl + - label + - --overwrite + - namespace + - cozy-system + - cozystack.io/system=true + - pod-security.kubernetes.io/enforce=privileged