mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 20:23:39 +09:00
cert-manager runs with enableCertificateOwnerRef: false, so the <release>-talos-ca and <release>-talos-tls-cert Secrets carry no ownerReference to their Certificates and outlive the release, keeping the CA private key of a cluster that no longer exists. A new cluster created under the same name then adopts that stale CA instead of issuing its own trust root. The <release>-oidc-bootstrap Job is a hook whose only delete policy is before-hook-creation. Hooks are not part of the release manifest, so uninstall never removes it. The pre-delete hook deletes the two Certificates before their Secrets, because deleting a Secret while its Certificate exists makes cert-manager re-issue it. It runs after the TenantControlPlane is gone, since the control plane's CSR signer mounts the CA. The Role grants each delete by resourceName only. Assisted-by: LLM Signed-off-by: Ray Low <crossrt@gmail.com>
338 lines
13 KiB
YAML
338 lines
13 KiB
YAML
suite: pre-delete cleanup hook (issue #3062)
|
|
|
|
# These tests pin the behaviour that prevents the Kamaji
|
|
# <release>-datastore-config Secret from blocking namespace deletion:
|
|
# the pre-delete Job must wait for the TenantControlPlane to be gone and
|
|
# then strip the finalizer.kamaji.clastix.io/datastore-secret finalizer,
|
|
# and the cleanup Role must be allowed to patch exactly that one Secret.
|
|
|
|
templates:
|
|
- templates/delete.yaml
|
|
|
|
release:
|
|
name: test-k8s
|
|
namespace: tenant-test
|
|
|
|
values:
|
|
- values-ci.yaml
|
|
|
|
tests:
|
|
- it: cleanup Role can get/patch only the datastore-config Secret
|
|
documentSelector:
|
|
path: kind
|
|
value: Role
|
|
asserts:
|
|
- contains:
|
|
path: rules
|
|
content:
|
|
apiGroups:
|
|
- ""
|
|
resources:
|
|
- secrets
|
|
resourceNames:
|
|
- test-k8s-datastore-config
|
|
verbs:
|
|
- get
|
|
- patch
|
|
|
|
- it: pre-delete Job waits for the TenantControlPlane before clearing the Secret
|
|
documentSelector:
|
|
path: kind
|
|
value: Job
|
|
asserts:
|
|
# TCP deletion must wait so Kamaji can clear the finalizer itself first.
|
|
- matchRegex:
|
|
path: spec.template.spec.containers[0].command[2]
|
|
pattern: 'delete tenantcontrolplanes\.kamaji\.clastix\.io test-k8s[\s\S]*--wait=true'
|
|
|
|
- it: pre-delete Job strips the leftover datastore-secret finalizer as a backstop
|
|
documentSelector:
|
|
path: kind
|
|
value: Job
|
|
asserts:
|
|
- matchRegex:
|
|
path: spec.template.spec.containers[0].command[2]
|
|
pattern: 'patch secret test-k8s-datastore-config'
|
|
- matchRegex:
|
|
path: spec.template.spec.containers[0].command[2]
|
|
pattern: '"finalizers":null'
|
|
|
|
- it: pre-delete Job clears the finalizer only after confirming the TenantControlPlane is gone
|
|
documentSelector:
|
|
path: kind
|
|
value: Job
|
|
asserts:
|
|
# The finalizer patch must be gated on the TCP being absent (#3078): if the
|
|
# Step 4 delete timed out with the TCP still alive, we must not strip the
|
|
# finalizer Kamaji still legitimately owns.
|
|
- matchRegex:
|
|
path: spec.template.spec.containers[0].command[2]
|
|
pattern: 'get tenantcontrolplanes\.kamaji\.clastix\.io test-k8s[\s\S]*still present[\s\S]*patch secret test-k8s-datastore-config'
|
|
|
|
# The remaining tests pin the wall-clock bounds on every delete in the hook.
|
|
# kubectl defaults --wait to true and reads --timeout=0 as a week, so a delete
|
|
# written with neither flag is one stuck finalizer away from parking this hook
|
|
# for 168 hours.
|
|
|
|
- it: every delete in the hook carries a wall-clock bound
|
|
documentSelector:
|
|
path: kind
|
|
value: Job
|
|
asserts:
|
|
# Each pattern walks only continuation lines — the ones ending in a
|
|
# backslash — so it stays inside the command it names. A cross-line
|
|
# wildcard reaches a flag belonging to the next command instead, and RE2,
|
|
# which helm-unittest matches with, has no lookahead to exclude the echo
|
|
# in between.
|
|
#
|
|
# None of the three waits, so none of them needs a guard to swallow an
|
|
# expired one — which matters, because a guard cannot tell a timeout from
|
|
# a rejected request and would report both as cleanup that happened.
|
|
#
|
|
# Step 3. What matters downstream is the TenantControlPlane, which Step 4
|
|
# deletes and waits for; waiting here only adds a second place for a CAPI
|
|
# finalizer to stall.
|
|
- matchRegex:
|
|
path: spec.template.spec.containers[0].command[2]
|
|
pattern: 'delete kamajicontrolplanes\.controlplane\.cluster\.x-k8s\.io test-k8s(?:[^\n]*\\\n)*[^\n]*--wait=false'
|
|
# Steps 5 and 6: nothing after them reads the result, and a DELETE that
|
|
# has been accepted is not made more complete by watching it. --wait=false
|
|
# and an expired --wait leave the same cluster state, one of them sooner.
|
|
- matchRegex:
|
|
path: spec.template.spec.containers[0].command[2]
|
|
pattern: 'delete datavolumes(?:[^\n]*\\\n)*[^\n]*--wait=false'
|
|
- matchRegex:
|
|
path: spec.template.spec.containers[0].command[2]
|
|
pattern: 'delete services(?:[^\n]*\\\n)*[^\n]*--wait=false'
|
|
# And the two that were already bounded stay bounded. --wait=false and
|
|
# --timeout bound the waiting, not the request: --request-timeout is still
|
|
# 0 on every call here, so an apiserver that stops answering hangs this
|
|
# hook the same as before. That is a different failure from the stuck
|
|
# finalizer these bounds are for, and it wants one answer across all the
|
|
# calls rather than a flag on the deletes.
|
|
- matchRegex:
|
|
path: spec.template.spec.containers[0].command[2]
|
|
pattern: 'delete helmreleases\.helm\.toolkit\.fluxcd\.io(?:[^\n]*\\\n)*[^\n]*--timeout=[1-9][0-9]*s'
|
|
- matchRegex:
|
|
path: spec.template.spec.containers[0].command[2]
|
|
pattern: 'delete tenantcontrolplanes\.kamaji\.clastix\.io test-k8s(?:[^\n]*\\\n)*[^\n]*--timeout=[1-9][0-9]*s'
|
|
|
|
- it: cleanup Role keeps watch on exactly the two resources whose deletes wait
|
|
documentSelector:
|
|
path: kind
|
|
value: Role
|
|
asserts:
|
|
# kubectl opens a watch only on the wait path, so watch is granted to the
|
|
# two resources Steps 2 and 4 wait on and to no others. Removing it from
|
|
# either of those does not error — the delete hangs instead, which is the
|
|
# failure mode that is hardest to read from the outside, so it is pinned
|
|
# rather than left to the reviewer of the next RBAC change.
|
|
#
|
|
# Whole-rule contains, not per-index verb assertions. notContains passes
|
|
# when its path does not resolve, so an index-addressed "and nowhere else"
|
|
# goes quiet the moment the rules list is reordered or shortened — the
|
|
# failure it is written to prevent would take the assertion with it.
|
|
# Matching the entire rule pins the verb set in both directions and names
|
|
# the resource when it fails; lengthEqual is what closes the remaining
|
|
# hole, a tenth rule appended past everything asserted here.
|
|
- lengthEqual:
|
|
path: rules
|
|
count: 9
|
|
- contains:
|
|
path: rules
|
|
content:
|
|
apiGroups: ["helm.toolkit.fluxcd.io"]
|
|
resources: ["helmreleases"]
|
|
verbs: ["get", "list", "watch", "patch", "delete"]
|
|
- contains:
|
|
path: rules
|
|
content:
|
|
apiGroups: ["kamaji.clastix.io"]
|
|
resources: ["tenantcontrolplanes"]
|
|
verbs: ["get", "list", "watch", "delete"]
|
|
- contains:
|
|
path: rules
|
|
content:
|
|
apiGroups: ["controlplane.cluster.x-k8s.io"]
|
|
resources: ["kamajicontrolplanes"]
|
|
verbs: ["get", "list", "delete"]
|
|
- contains:
|
|
path: rules
|
|
content:
|
|
apiGroups: ["cdi.kubevirt.io"]
|
|
resources: ["datavolumes"]
|
|
verbs: ["get", "list", "delete"]
|
|
- contains:
|
|
path: rules
|
|
content:
|
|
apiGroups: [""]
|
|
resources: ["services"]
|
|
verbs: ["get", "list", "delete"]
|
|
|
|
# Step 7 removes what outlives the release: the two Talos PKI Secrets, which
|
|
# cert-manager issues without an ownerReference to their Certificates, and the
|
|
# OIDC bootstrap Job, a hook whose only delete policy is before-hook-creation.
|
|
- it: deletes the Talos Certificates before their Secrets, so cert-manager cannot re-issue them
|
|
documentSelector:
|
|
path: kind
|
|
value: Job
|
|
asserts:
|
|
- matchRegex:
|
|
path: spec.template.spec.containers[0].command[2]
|
|
pattern: 'delete certificates\.cert-manager\.io \\\n\s+test-k8s-talos-ca test-k8s-talos-tls-cert(?:[^\n]*\\\n)*[^\n]*--wait=false'
|
|
- matchRegex:
|
|
path: spec.template.spec.containers[0].command[2]
|
|
pattern: 'delete secrets \\\n\s+test-k8s-talos-ca test-k8s-talos-tls-cert(?:[^\n]*\\\n)*[^\n]*--wait=false'
|
|
- notMatchRegex:
|
|
path: spec.template.spec.containers[0].command[2]
|
|
pattern: 'delete secrets \\\n\s+test-k8s-talos-ca[\s\S]*delete certificates\.cert-manager\.io'
|
|
|
|
- it: deletes the orphaned OIDC bootstrap Job
|
|
documentSelector:
|
|
path: kind
|
|
value: Job
|
|
asserts:
|
|
- matchRegex:
|
|
path: spec.template.spec.containers[0].command[2]
|
|
pattern: 'delete jobs\.batch \\\n\s+test-k8s-oidc-bootstrap(?:[^\n]*\\\n)*[^\n]*--wait=false'
|
|
|
|
- it: cleanup Role can delete only the Step 7 objects, by name
|
|
documentSelector:
|
|
path: kind
|
|
value: Role
|
|
asserts:
|
|
- contains:
|
|
path: rules
|
|
content:
|
|
apiGroups: ["cert-manager.io"]
|
|
resources: ["certificates"]
|
|
resourceNames: ["test-k8s-talos-ca", "test-k8s-talos-tls-cert"]
|
|
verbs: ["get", "delete"]
|
|
- contains:
|
|
path: rules
|
|
content:
|
|
apiGroups: [""]
|
|
resources: ["secrets"]
|
|
resourceNames: ["test-k8s-talos-ca", "test-k8s-talos-tls-cert"]
|
|
verbs: ["get", "delete"]
|
|
- contains:
|
|
path: rules
|
|
content:
|
|
apiGroups: ["batch"]
|
|
resources: ["jobs"]
|
|
resourceNames: ["test-k8s-oidc-bootstrap"]
|
|
verbs: ["get", "delete"]
|
|
|
|
- it: pre-delete Job makes a single attempt
|
|
documentSelector:
|
|
path: kind
|
|
value: Job
|
|
asserts:
|
|
# restartPolicy is Never, so the default backoffLimit of 6 allows seven
|
|
# pods in series, each re-running every step from the top and each paying
|
|
# its own image pull. That outlives the uninstall budget, and helm gives
|
|
# up on its own timeout without reporting which step failed.
|
|
#
|
|
# Zero and not one: both remaining waits are guarded, so an attempt can
|
|
# spend its full 240s without failing, and a retry starts from Step 1 with
|
|
# the clock already there. On a nodeless tenant it also re-pays Step 4's
|
|
# 120s, because attempt 1 leaves the TenantControlPlane still terminating.
|
|
- equal:
|
|
path: spec.backoffLimit
|
|
value: 0
|
|
- equal:
|
|
path: spec.template.spec.restartPolicy
|
|
value: Never
|
|
|
|
# Step 1b exists only on the proxmox substrate: a PVC there is a disk on the
|
|
# hypervisor that only the CSI controller can free, and Step 2 removes that
|
|
# controller. The Role gains the tenant's admin kubeconfig for it, by name.
|
|
- it: releases Proxmox volumes before the driver goes, on proxmox only
|
|
set: &proxmox
|
|
substrate: proxmox
|
|
_namespace:
|
|
etcd: etcd-test
|
|
proxmox:
|
|
dnsServers: ["8.8.8.8"]
|
|
ipv4Config:
|
|
addresses: ["10.0.0.160-10.0.0.170"]
|
|
gateway: "10.0.0.1"
|
|
prefix: 24
|
|
ccm:
|
|
credentialsSecretName: proxmox-ccm-creds
|
|
csi:
|
|
enabled: true
|
|
credentialsSecretName: proxmox-csi-creds
|
|
documentSelector:
|
|
path: kind
|
|
value: Job
|
|
asserts:
|
|
- matchRegex:
|
|
path: spec.template.spec.containers[0].command[2]
|
|
pattern: 'Step 1b: Releasing Proxmox volumes while the CSI driver is still running'
|
|
- matchRegex:
|
|
path: spec.template.spec.containers[0].command[2]
|
|
pattern: 'get --raw /readyz'
|
|
|
|
- it: has no volume-release step on kubevirt, where the CSI controller stays until the cluster goes
|
|
documentSelector:
|
|
path: kind
|
|
value: Job
|
|
asserts:
|
|
- notMatchRegex:
|
|
path: spec.template.spec.containers[0].command[2]
|
|
pattern: 'Step 1b'
|
|
|
|
- it: lets the cleanup Role read the tenant admin kubeconfig on proxmox, and nothing more
|
|
set: *proxmox
|
|
documentSelector:
|
|
path: kind
|
|
value: Role
|
|
asserts:
|
|
- contains:
|
|
path: rules
|
|
content:
|
|
apiGroups:
|
|
- ""
|
|
resources:
|
|
- secrets
|
|
resourceNames:
|
|
- test-k8s-datastore-config
|
|
- test-k8s-admin-kubeconfig
|
|
verbs:
|
|
- get
|
|
- patch
|
|
|
|
- it: deletes the claims before the Pods that hold them, so nothing can mount one again
|
|
set: *proxmox
|
|
documentSelector:
|
|
path: kind
|
|
value: Job
|
|
asserts:
|
|
- matchRegex:
|
|
path: spec.template.spec.containers[0].command[2]
|
|
pattern: 'delete pvc "\$pname"[\s\S]*delete pod "\$pod"'
|
|
- notMatchRegex:
|
|
path: spec.template.spec.containers[0].command[2]
|
|
pattern: 'delete pod "\$pod"[\s\S]*delete pvc "\$pname"'
|
|
|
|
# A JSONPath filter on the claim name fails for the whole namespace as soon as
|
|
# one Pod mounts two claims, and the error is swallowed, so no Pod is deleted
|
|
# and the disks outlive the driver. The loop lists every Pod with its claims
|
|
# and matches in the shell; hack/proxmox-step1b-pod-match_test.bats runs it.
|
|
- it: finds the Pods that hold a claim without a JSONPath filter
|
|
set: *proxmox
|
|
documentSelector:
|
|
path: kind
|
|
value: Job
|
|
asserts:
|
|
- notMatchRegex:
|
|
path: spec.template.spec.containers[0].command[2]
|
|
pattern: '\[\?\(@\.spec\.volumes'
|
|
- matchRegex:
|
|
path: spec.template.spec.containers[0].command[2]
|
|
pattern: 'range \.spec\.volumes\[\*\]\}\{" "\}\{\.persistentVolumeClaim\.claimName\}'
|
|
- matchRegex:
|
|
path: spec.template.spec.containers[0].command[2]
|
|
pattern: 'case " \$claims " in'
|