From 4f35af6694116c6b5ac96482e7f110e0d28208db Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Mon, 9 Nov 2026 11:18:49 +0100 Subject: [PATCH] fix(monitoring-agents): let the release uninstall when the VictoriaMetrics operator is gone first Disabling the Monitoring Agents addon of a tenant cluster removes this release, the VictoriaMetrics operator and the VPA CRDs together, and Flux deletes them in parallel without honouring dependsOn. When the operator goes first, nothing clears apps.victoriametrics.com/finalizer on the VMAgent and the other VictoriaMetrics objects this chart created, so they stay Terminating and the release sits in UninstallFailed for good. Clearing those finalizers has to happen inside the uninstall and must never be what blocks it, and a live operator still has to get the first chance to finalize its objects. Only objects of this release may be touched, and release names are unique only within a namespace, so the release name alone does not identify them. kubectl loads in-cluster credentials only while the client config is otherwise default, which a --request-timeout breaks. A VPA recorded in the release manifest while its CRD is not served cannot be mapped at uninstall either. Assisted-by: LLM Signed-off-by: Mattia Eleuteri --- hack/monitoring-agents-vm-cleanup-hook.bats | 312 ++++++++++++++++++ .../templates/vm-cleanup-hook.yaml | 167 ++++++++++ .../monitoring-agents/templates/vpa.yaml | 6 + .../tests/uninstall_test.yaml | 141 ++++++++ packages/system/monitoring-agents/values.yaml | 12 + 5 files changed, 638 insertions(+) create mode 100644 hack/monitoring-agents-vm-cleanup-hook.bats create mode 100644 packages/system/monitoring-agents/templates/vm-cleanup-hook.yaml create mode 100644 packages/system/monitoring-agents/tests/uninstall_test.yaml diff --git a/hack/monitoring-agents-vm-cleanup-hook.bats b/hack/monitoring-agents-vm-cleanup-hook.bats new file mode 100644 index 000000000..290a91314 --- /dev/null +++ b/hack/monitoring-agents-vm-cleanup-hook.bats @@ -0,0 +1,312 @@ +#!/usr/bin/env bats +# Behavioural tests for the monitoring-agents pre-delete hook: +# +# packages/system/monitoring-agents/templates/vm-cleanup-hook.yaml +# +# The script is extracted from the rendered chart and RUN against a fake +# kubectl that keeps the VictoriaMetrics objects in a state file, with fake +# `date` and `sleep` so the 60s grace period passes in an instant. The +# helm-unittest suite in packages/system/monitoring-agents/tests/ +# uninstall_test.yaml matches the script text instead, and stayed green when the +# ownership condition was removed, when the finalizer patch was made a no-op and +# when the polling condition was inverted. Each of those turns this file red. +# +# What is pinned here: +# * only objects whose Helm annotations name this release, by name and +# namespace both, are deleted or patched: an object of another release, of +# a same-named release in another namespace, or one Helm does not own, is +# left alone; +# * with the operator gone, the hook waits before touching finalizers, then +# clears them, and nothing of this release is left behind; +# * with the operator alive, it finalizes the objects itself and the hook +# never patches a finalizer; +# * a kind whose CRD is gone, or a release with nothing left, exits 0, since a +# failing pre-delete hook would itself block the uninstall. +# +# Run from the repo root with make bats-unit-tests. Direct status checks +# retain compatibility with cozytest, which has no Bats run/setup helpers. +# Inline cleanup preserves Bats' TAP handler on failed assertions. + +load test_helper + +MA_CHART=packages/system/monitoring-agents +MA_RELEASE=monitoring-agents +MA_NAMESPACE=cozy-monitoring + +# render_hook +# Extract the named Job's shell script from the rendered chart into . The +# non-empty check must be an early `return 1`: hack/cozytest.sh rewrites every +# line matching ^}$ into `return 0` plus `}`, helpers included, so a check left +# in last-command position has its status discarded. +render_hook() { + helm template "$MA_RELEASE" "$MA_CHART" --namespace "$MA_NAMESPACE" \ + | yq eval "select(.kind == \"Job\" and .metadata.name == \"$1\") | .spec.template.spec.containers[0].command[2]" - > "$2" + [ -s "$2" ] || return 1 +} + +# write_fakes +# Emit fake `kubectl`, `date` and `sleep` into . +# +# kubectl reads and writes $STATE, one object per line as +# kind|release-name|release-namespace|namespace|name, where the two release +# fields are the Helm annotations (empty when Helm does not own the object). +# `get` prints this kind's objects in the hook's jsonpath format, or fails for a +# kind listed in $MISSING_KINDS as kubectl does when the CRD is gone. `delete` +# removes the object only when $OPERATOR_LIVE is 1, standing for an operator +# that finalizes it; otherwise the object stays, Terminating. `patch` removes it +# only when the payload actually clears the finalizers. Every call goes to $KLOG. +# +# date advances 1s per call from $CLOCK, so the 60s wait ends after sixty +# of polls, and records each time it returns in $KLOG; sleep records itself, +# and with $SLEEP_ADVANCES=1 also moves $CLOCK by its argument. The log thus +# shows how much fake time passed before each patch. +# No line below is a bare column-0 `}`, so cozytest.sh's parser leaves it alone. +write_fakes() { + cat > "$1/kubectl" <<'KEOF' +#!/bin/sh +echo "kubectl $*" >> "$KLOG" +drop() { + awk -F'|' -v k="$1" -v ns="$2" -v n="$3" '!($1 == k && $4 == ns && $5 == n)' "$STATE" > "$STATE.new" + mv "$STATE.new" "$STATE"; } +case "$1" in + get) + kind=${2%%.*} + case " ${MISSING_KINDS:-} " in + *" $kind "*) + echo "error: the server doesn't have a resource type \"$kind\"" >&2 + exit 1 ;; + esac + awk -F'|' -v k="$kind" '$1 == k { print $2 "|" $3 "|" $4 "|" $5 }' "$STATE" + exit 0 ;; + -n) + ns=$2 verb=$3 kind=${4%%.*} name=$5 + case "$verb" in + delete) + [ "${OPERATOR_LIVE:-0}" = 1 ] && drop "$kind" "$ns" "$name" + echo "$kind \"$name\" deleted" + exit 0 ;; + patch) + case "$*" in + *'"finalizers":null'*) drop "$kind" "$ns" "$name" ;; + esac + echo "$kind \"$name\" patched" + exit 0 ;; + esac ;; +esac +echo "fake kubectl: unexpected call: $*" >&2 +exit 64 +KEOF + cat > "$1/date" <<'KEOF' +#!/bin/sh +c=$(cat "$CLOCK" 2>/dev/null || echo 0) +echo $((c + 1)) > "$CLOCK" +echo "date $((1000000 + c))" >> "$KLOG" +echo $((1000000 + c)) +KEOF + cat > "$1/sleep" <<'KEOF' +#!/bin/sh +echo "sleep $*" >> "$KLOG" +if [ "${SLEEP_ADVANCES:-0}" = 1 ]; then + c=$(cat "$CLOCK" 2>/dev/null || echo 0) + echo $((c + $1)) > "$CLOCK" +fi +KEOF + chmod +x "$1/kubectl" "$1/date" "$1/sleep" +} + +# write_state +# Three objects of this release in two namespaces; a VMRule of another release; +# a VMRule whose release name matches but whose release namespace does not, so +# only the pair of annotations tells it apart; and a VMServiceScrape that +# carries no Helm annotations. +write_state() { + cat > "$1" < +# Run the rendered hook with the fakes first on PATH; its status lands in $rc. +run_hook() { + export KLOG="$1/calls" STATE="$1/state" CLOCK="$1/clock" + : > "$KLOG" + rc=0 + PATH="$1:$PATH" sh "$1/hook.sh" > "$1/out" 2> "$1/err" || rc=$? +} + +@test "a render that produces no script fails instead of passing as a no-op" { + # Guards the guard: every test below runs whatever render_hook produced, so an + # empty render would satisfy the exit-0 assertions without executing a line of + # the hook. + tmp=$(mktemp -d) + if render_hook no-such-job "$tmp/hook.sh"; then + echo "FAIL: render_hook reported success for a Job name the chart never emits" + false + fi + rm -rf "$tmp" +} + +@test "vm cleanup hook clears the finalizers of this release's objects only, after the grace period" { + tmp=$(mktemp -d) + write_fakes "$tmp" + write_state "$tmp/state" + render_hook vm-objects-cleanup "$tmp/hook.sh" + export OPERATOR_LIVE=0 MISSING_KINDS="" + run_hook "$tmp" + + if [ "$rc" -ne 0 ]; then + echo "FAIL: hook exited $rc" + cat "$tmp/out" "$tmp/err" + false + fi + for obj in "vmagents.operator.victoriametrics.com vmagent" \ + "vmrules.operator.victoriametrics.com alerts-node" \ + "vmservicescrapes.operator.victoriametrics.com operator"; do + if ! grep -q -- " delete $obj --wait=false" "$tmp/calls"; then + echo "FAIL: $obj was not deleted" + cat "$tmp/calls" + false + fi + done + # The objects of this release must be gone, which the fake only does for a + # patch that really clears the finalizers. + if grep -q "^[a-z]*|$MA_RELEASE|$MA_NAMESPACE|" "$tmp/state"; then + echo "FAIL: objects of this release are still there, Terminating" + cat "$tmp/state" "$tmp/calls" + false + fi + # Another release's object, one of a same-named release in another + # namespace, and an object Helm does not own are never touched. + if grep -E -q -- ' (delete|patch) [a-z.]+ (foreign-rule|foreign-ns-rule|unmanaged-scrape)( |$)' "$tmp/calls"; then + echo "FAIL: the hook touched an object this release does not own" + cat "$tmp/calls" + false + fi + if [ "$(grep -c '|' "$tmp/state")" -ne 3 ]; then + echo "FAIL: the three objects this release does not own should remain" + cat "$tmp/state" + false + fi + # The operator gets its chance first: the hook polls and sleeps before the + # first patch, and does not stop waiting before the 60s are up. + first_patch=$(grep -n ' patch ' "$tmp/calls" | head -1 | cut -d: -f1) + if [ -z "$first_patch" ]; then + echo "FAIL: no finalizer was cleared" + cat "$tmp/calls" + false + fi + # The first date call sets the deadline; the last one before the first patch + # is the time the hook gave up waiting. + start=$(grep '^date ' "$tmp/calls" | head -1 | cut -d' ' -f2) + waited_until=$(head -n "$first_patch" "$tmp/calls" | grep '^date ' | tail -1 | cut -d' ' -f2) + if [ -z "$start" ] || [ $((waited_until - start)) -lt 60 ]; then + echo "FAIL: finalizers cleared after $((waited_until - start))s; the 60s grace period was cut short" + cat "$tmp/calls" + false + fi + rm -rf "$tmp" +} + +@test "vm cleanup hook clears the finalizers before its Job's deadline" { + # The grace-period test above keeps sleep free so a shortened wait shows; here + # sleep costs its argument, so a poll interval long enough to overshoot the + # Job's activeDeadlineSeconds fails instead of passing. + tmp=$(mktemp -d) + write_fakes "$tmp" + write_state "$tmp/state" + render_hook vm-objects-cleanup "$tmp/hook.sh" + job_deadline=$(helm template "$MA_RELEASE" "$MA_CHART" --namespace "$MA_NAMESPACE" \ + | yq eval 'select(.kind == "Job" and .metadata.name == "vm-objects-cleanup") | .spec.activeDeadlineSeconds' -) + export OPERATOR_LIVE=0 MISSING_KINDS="" SLEEP_ADVANCES=1 + run_hook "$tmp" + unset SLEEP_ADVANCES + + if [ "$rc" -ne 0 ]; then + echo "FAIL: hook exited $rc" + cat "$tmp/out" "$tmp/err" + false + fi + first_patch=$(grep -n ' patch ' "$tmp/calls" | head -1 | cut -d: -f1) + start=$(grep '^date ' "$tmp/calls" | head -1 | cut -d' ' -f2) + waited_until=$(head -n "${first_patch:-0}" "$tmp/calls" | grep '^date ' | tail -1 | cut -d' ' -f2) + if [ -z "$first_patch" ] || [ -z "$job_deadline" ] || [ $((waited_until - start)) -ge "$job_deadline" ]; then + echo "FAIL: first finalizer patch at $((waited_until - start))s, Job deadline ${job_deadline}s" + cat "$tmp/calls" + false + fi + rm -rf "$tmp" +} + +@test "vm cleanup hook leaves the finalizers to a live operator" { + tmp=$(mktemp -d) + write_fakes "$tmp" + write_state "$tmp/state" + render_hook vm-objects-cleanup "$tmp/hook.sh" + export OPERATOR_LIVE=1 MISSING_KINDS="" + run_hook "$tmp" + + if [ "$rc" -ne 0 ]; then + echo "FAIL: hook exited $rc" + cat "$tmp/out" "$tmp/err" + false + fi + if ! grep -q ' delete vmagents.operator.victoriametrics.com vmagent ' "$tmp/calls"; then + echo "FAIL: the VMAgent was not deleted" + cat "$tmp/calls" + false + fi + if grep -q ' patch ' "$tmp/calls"; then + echo "FAIL: the hook cleared finalizers the live operator was handling" + cat "$tmp/calls" + false + fi + rm -rf "$tmp" +} + +@test "vm cleanup hook exits 0 and cleans the other kinds when a CRD is gone" { + tmp=$(mktemp -d) + write_fakes "$tmp" + write_state "$tmp/state" + render_hook vm-objects-cleanup "$tmp/hook.sh" + export OPERATOR_LIVE=0 MISSING_KINDS="vmrules vmpodscrapes vmnodescrapes" + run_hook "$tmp" + + if [ "$rc" -ne 0 ]; then + echo "FAIL: hook exited $rc with a CRD gone, which would block the uninstall" + cat "$tmp/out" "$tmp/err" + false + fi + if grep -q "^vmagents|$MA_RELEASE|" "$tmp/state"; then + echo "FAIL: the VMAgent was left Terminating because another kind's CRD was gone" + cat "$tmp/state" "$tmp/calls" + false + fi + rm -rf "$tmp" +} + +@test "vm cleanup hook exits 0 without deleting anything when the release owns nothing" { + tmp=$(mktemp -d) + write_fakes "$tmp" + printf 'vmrules|other|%s|%s|foreign-rule\nvmrules|%s|other-namespace|%s|foreign-ns-rule\n' \ + "$MA_NAMESPACE" "$MA_NAMESPACE" "$MA_RELEASE" "$MA_NAMESPACE" > "$tmp/state" + render_hook vm-objects-cleanup "$tmp/hook.sh" + export OPERATOR_LIVE=0 MISSING_KINDS="" + run_hook "$tmp" + + if [ "$rc" -ne 0 ]; then + echo "FAIL: hook exited $rc with nothing to clean" + cat "$tmp/out" "$tmp/err" + false + fi + if grep -E -q ' (delete|patch) ' "$tmp/calls"; then + echo "FAIL: the hook deleted or patched with nothing of this release left" + cat "$tmp/calls" + false + fi + rm -rf "$tmp" +} diff --git a/packages/system/monitoring-agents/templates/vm-cleanup-hook.yaml b/packages/system/monitoring-agents/templates/vm-cleanup-hook.yaml new file mode 100644 index 000000000..efdbd1158 --- /dev/null +++ b/packages/system/monitoring-agents/templates/vm-cleanup-hook.yaml @@ -0,0 +1,167 @@ +{{- /* + The VictoriaMetrics operator puts its finalizer on the VMAgent and is the only + thing that ever removes it. The operator ships as a separate release that is + gated on the same addon flag, so disabling the addon removes both at once and + Flux does not order deletions: when the operator goes first, the VMAgent stays + Terminating and the uninstall never finishes. The other kinds are handled the + same way in case an operator version finalizes them too; for those the patch + is a no-op. + + So delete them before Helm does, give a live operator a bounded chance to + finalize them itself, and only then clear the finalizers by hand. The VMAgent's + Deployment, Service and Secrets carry owner references and are garbage + collected once it is gone. + + Cluster-scoped RBAC because the objects span two namespaces, and a Role in the + operator's namespace could not be created if that namespace were already + gone. The hook always exits 0: a cleanup that cannot complete must not be what + keeps the release from uninstalling. +*/ -}} +{{- $name := "vm-objects-cleanup" }} +{{- $img := printf "%s:%s" .Values.kubectlImage.repository .Values.kubectlImage.tag }} +{{- with .Values.kubectlImage.digest }}{{- $img = printf "%s@%s" $img . }}{{- end }} +{{- $kinds := list "vmagents" "vmrules" "vmservicescrapes" "vmpodscrapes" "vmnodescrapes" }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $name }} + namespace: {{ .Release.Namespace }} + annotations: + helm.sh/hook: pre-delete + helm.sh/hook-weight: "-10" + helm.sh/hook-delete-policy: hook-succeeded,hook-failed,before-hook-creation +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Release.Namespace }}-{{ $name }} + annotations: + helm.sh/hook: pre-delete + helm.sh/hook-weight: "-10" + helm.sh/hook-delete-policy: hook-succeeded,hook-failed,before-hook-creation +rules: + - apiGroups: ["operator.victoriametrics.com"] + resources: + {{- toYaml $kinds | nindent 6 }} + verbs: ["get", "list", "delete", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Release.Namespace }}-{{ $name }} + annotations: + helm.sh/hook: pre-delete + helm.sh/hook-weight: "-10" + helm.sh/hook-delete-policy: hook-succeeded,hook-failed,before-hook-creation +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Release.Namespace }}-{{ $name }} +subjects: + - kind: ServiceAccount + name: {{ $name }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ $name }} + namespace: {{ .Release.Namespace }} + annotations: + helm.sh/hook: pre-delete + helm.sh/hook-weight: "0" + helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation +spec: + # Bounds every kubectl call as well: --request-timeout would make kubectl + # skip the in-cluster credentials (hack/check-hook-request-timeout.bats). + # Well inside the HelmRelease's 10m uninstall budget, and above the 60s + # wait plus one call per object on either side of it. + activeDeadlineSeconds: 180 + backoffLimit: 0 + ttlSecondsAfterFinished: 600 + template: + spec: + serviceAccountName: {{ $name }} + restartPolicy: Never + securityContext: + runAsNonRoot: true + # The image user is the non-numeric `nonroot`, which the kubelet cannot + # check against runAsNonRoot. + runAsUser: 65532 + seccompProfile: + type: RuntimeDefault + volumes: + - name: home + emptyDir: {} + containers: + - name: cleanup + image: {{ $img | quote }} + imagePullPolicy: {{ .Values.kubectlImage.pullPolicy }} + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + # kubectl writes its discovery cache under $HOME. + env: + - name: HOME + value: /tmp + volumeMounts: + - name: home + mountPath: /tmp + command: + - /bin/sh + - -c + - | + rel={{ .Release.Name | quote }} + relns={{ .Release.Namespace | quote }} + kinds={{ join " " $kinds | quote }} + + # Selected by Helm's ownership annotations rather than by name, so + # the alert rules added under alerts/ are covered without a list + # to keep in step. A kind whose CRD is gone just prints an error. + owned() { + for k in $kinds; do + kubectl get "$k.operator.victoriametrics.com" -A \ + -o jsonpath='{range .items[*]}{.metadata.annotations.meta\.helm\.sh/release-name}{"|"}{.metadata.annotations.meta\.helm\.sh/release-namespace}{"|"}{.metadata.namespace}{"|"}{.metadata.name}{"\n"}{end}' | + while IFS='|' read -r r rns ns n; do + if [ "$r" = "$rel" ] && [ "$rns" = "$relns" ]; then + echo "$k.operator.victoriametrics.com $ns $n" + fi + done + done + } + + objs=$(owned) + if [ -z "$objs" ]; then + echo "no VictoriaMetrics objects left for $relns/$rel" + exit 0 + fi + + echo "$objs" | while read -r k ns n; do + kubectl -n "$ns" delete "$k" "$n" --wait=false + done + + deadline=$(( $(date +%s) + 60 )) + while [ "$(date +%s)" -lt "$deadline" ]; do + sleep 2 + objs=$(owned) + if [ -z "$objs" ]; then + echo "the operator finalized every object" + exit 0 + fi + done + + echo "finalizers still held after 60s; clearing them by hand" + echo "$objs" | while read -r k ns n; do + kubectl -n "$ns" patch "$k" "$n" --type=merge \ + -p '{"metadata":{"finalizers":null}}' + done + exit 0 diff --git a/packages/system/monitoring-agents/templates/vpa.yaml b/packages/system/monitoring-agents/templates/vpa.yaml index 9a31fab94..1ac949f30 100644 --- a/packages/system/monitoring-agents/templates/vpa.yaml +++ b/packages/system/monitoring-agents/templates/vpa.yaml @@ -6,6 +6,11 @@ Initial sets requests at creation without evicting, removing the trigger. Do not switch back to Auto without fixing the underlying Cilium IP-reuse race. */ -}} +{{- /* + Rendered only when the CRD is served: a release manifest that records a VPA + the cluster cannot map is one Helm can no longer uninstall. +*/ -}} +{{- if .Capabilities.APIVersions.Has "autoscaling.k8s.io/v1/VerticalPodAutoscaler" }} apiVersion: autoscaling.k8s.io/v1 kind: VerticalPodAutoscaler metadata: @@ -34,3 +39,4 @@ spec: maxAllowed: cpu: 4000m memory: 6G +{{- end }} diff --git a/packages/system/monitoring-agents/tests/uninstall_test.yaml b/packages/system/monitoring-agents/tests/uninstall_test.yaml new file mode 100644 index 000000000..df8a3eaa6 --- /dev/null +++ b/packages/system/monitoring-agents/tests/uninstall_test.yaml @@ -0,0 +1,141 @@ +suite: the release uninstalls whatever order its dependencies go in + +# Disabling the addon removes this release, the VictoriaMetrics operator and the +# VPA CRDs together, and Flux deletes them in parallel. When the operator goes +# first, nothing clears its finalizer on the objects this chart created and the +# uninstall hangs; when the VPA CRD goes first, a release manifest recording a +# VPA cannot be built and the uninstall fails. +release: + name: cozy-monitoring-agents + namespace: cozy-monitoring +templates: + - templates/vm-cleanup-hook.yaml + - templates/vpa.yaml +tests: + - it: runs before Helm deletes anything, bounded and with one attempt + template: templates/vm-cleanup-hook.yaml + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.annotations["helm.sh/hook"] + value: pre-delete + - equal: + path: metadata.namespace + value: cozy-monitoring + - equal: + path: spec.activeDeadlineSeconds + value: 180 + - equal: + path: spec.backoffLimit + value: 0 + - matchRegex: + path: spec.template.spec.containers[0].image + pattern: ^docker\.io/clastix/kubectl:v[0-9.]+@sha256:[0-9a-f]{64}$ + + - it: its RBAC exists only for the hook and is removed with it + template: templates/vm-cleanup-hook.yaml + documentSelector: + path: kind + value: ClusterRole + asserts: + - equal: + path: metadata.annotations["helm.sh/hook"] + value: pre-delete + - equal: + path: metadata.annotations["helm.sh/hook-weight"] + value: "-10" + - matchRegex: + path: metadata.annotations["helm.sh/hook-delete-policy"] + pattern: hook-succeeded + + - it: may delete and clear finalizers on every VictoriaMetrics kind the chart renders + template: templates/vm-cleanup-hook.yaml + documentSelector: + path: kind + value: ClusterRole + asserts: + - equal: + path: rules[0].apiGroups + value: ["operator.victoriametrics.com"] + - contains: + path: rules[0].resources + content: vmagents + - contains: + path: rules[0].resources + content: vmrules + - contains: + path: rules[0].resources + content: vmservicescrapes + - contains: + path: rules[0].resources + content: vmpodscrapes + - contains: + path: rules[0].resources + content: vmnodescrapes + - equal: + path: rules[0].verbs + value: ["get", "list", "delete", "patch"] + - lengthEqual: + path: rules + count: 1 + + - it: grants that role to the hook's own service account + template: templates/vm-cleanup-hook.yaml + documentSelector: + path: kind + value: ClusterRoleBinding + asserts: + - equal: + path: roleRef.name + value: cozy-monitoring-vm-objects-cleanup + - equal: + path: subjects[0] + value: + kind: ServiceAccount + name: vm-objects-cleanup + namespace: cozy-monitoring + + - it: touches only objects this release owns, and clears their finalizers + template: templates/vm-cleanup-hook.yaml + documentSelector: + path: kind + value: Job + asserts: + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: rel="cozy-monitoring-agents" + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: relns="cozy-monitoring" + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: meta\\\.helm\\\.sh/release-name + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: 'delete "\$k" "\$n" --wait=false' + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: '"finalizers":null' + # A pre-delete hook that fails aborts the uninstall it exists to unblock. + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: exit 0\s*$ + + - it: the VPA is rendered when its CRD is served + template: templates/vpa.yaml + capabilities: + apiVersions: + - autoscaling.k8s.io/v1/VerticalPodAutoscaler + asserts: + - hasDocuments: + count: 1 + - isKind: + of: VerticalPodAutoscaler + + - it: the VPA stays out of the release manifest when its CRD is not served + template: templates/vpa.yaml + asserts: + - hasDocuments: + count: 0 diff --git a/packages/system/monitoring-agents/values.yaml b/packages/system/monitoring-agents/values.yaml index 78a6ec9fd..bd1b501c8 100644 --- a/packages/system/monitoring-agents/values.yaml +++ b/packages/system/monitoring-agents/values.yaml @@ -552,3 +552,15 @@ prometheus-node-exporter: memory: 32Mi limits: memory: 256Mi + +# Image for the pre-delete hook that releases the VictoriaMetrics objects (see +# templates/vm-cleanup-hook.yaml). Only needs a kubectl binary. Digest-pinned so +# an upstream retag does not change what runs across the fleet; refresh with +# `docker manifest inspect docker.io/clastix/kubectl:v1.32`. Same pin as the +# etcd-operator hook. +# renovate: datasource=docker depName=docker.io/clastix/kubectl +kubectlImage: + repository: docker.io/clastix/kubectl + tag: v1.32 + digest: sha256:b9ef7d8dbe65bcc81a46c09b8dc7543103055021c4f43287bf59e92a8f4fe05c + pullPolicy: IfNotPresent