feat(platform): move Strimzi resources to the v1 API before Strimzi 1.x

Strimzi 1.0 removed v1beta2 from its CRDs, and the API server refuses
a CRD that drops a version its status still lists as stored. Every
stored object has to be rewritten as v1 before the new operator
arrives, which needs the CRDs of Strimzi 0.49 to 0.51 that serve both
versions and Strimzi's own conversion tool. The 0.45 operator is
stopped first: Strimzi supports the conversion only under an operator
that knows v1, and 0.45 keeps rewriting the Kafka status through
v1beta2.

Two references to v1beta2 outlive the stored objects. Helm builds the
objects of a release before it upgrades or uninstalls it, so every
stored revision of every Kafka application is rewritten to v1. And the
API server refuses server-side apply to an object whose managedFields
still name a version the CRD no longer serves; when helm-controller
recovers by wiping them, a field a later chart drops is never removed.
Those entries move to v1 as well.

Strimzi 1.x also runs neither ZooKeeper nor Kafka 3.x. A ZooKeeper
cluster, or a shared broker pool still being drained, left by a
platform that moved past v1.7 before its ZooKeeper to KRaft migration
finished cannot be finished afterwards, so the migration stops
instead; Kafka applications on v3.8 or v3.9 move to v4.3.

Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin <f@lex.la>

hack/migration-60-strimzi-v1-conversion.bats

@@ -0,0 +1,513 @@
+#!/usr/bin/env bats
+# -----------------------------------------------------------------------------
+# Unit tests for platform migration 60 --> 61 (move the Strimzi custom
+# resources to the v1 API, refuse ZooKeeper-based Kafkas, move Kafka 3.x
+# applications to v4.3). See
+# packages/core/platform/images/migrations/migrations/60 for the mechanism.
+#
+# The fake kubectl keeps the storedVersions every Strimzi CRD reports in a
+# state file, and the conversion Job rewrites it when it succeeds, so the
+# check after the Job reads what the Job did.
+#
+# The harness (docker, jq baked onto the migrations base image, the explicit
+# `return` in run_migration) is the one hack/migration-55-fluxcd-orphan.bats
+# describes; see there for why each piece is load-bearing.
+#
+# Run with: hack/cozytest.sh hack/migration-60-strimzi-v1-conversion.bats
+# -----------------------------------------------------------------------------
+
+load test_helper
+
+FIXTURES="$PWD/hack/testdata/migration-60-strimzi-v1"
+FAKE_FIXTURES="$FIXTURES"
+MIG_DIR="$PWD/packages/core/platform/images/migrations/migrations"
+ALPINE=$(sed -n 's/^FROM \(alpine:[^ ]*\).*$/\1/p' \
+  "$PWD/packages/core/platform/images/migrations/Dockerfile" | head -1)
+TESTIMG="cozystack-migration60-test:$(printf '%s' "$ALPINE" | sed 's/[^a-zA-Z0-9]/-/g')"
+WORKROOT="${TMPDIR:-/tmp}/cozy-migration-60-$$"
+
+cozy_cleanup() {
+  rm -rf "$WORKROOT"
+  return 0
+}
+
+# The wait for the conversion Job counts only its sleeps, so a zero poll
+# interval would let a wait that misses the end of the Job spin for good. A
+# one-second poll under a three-second timeout turns that into a quick failure.
+run_migration() {
+  _run_migration_rc=0
+  docker run --rm --network none \
+    --user "$(id -u):$(id -g)" \
+    -v "$MIG_DIR:/migrations:ro" \
+    -v "$FIXTURES:/fakebin:ro" \
+    -v "$WORK:/work" \
+    -e PATH=/fakebin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
+    -e FAKE_CMDLOG=/work/cmdlog \
+    -e FAKE_STATE=/work/state \
+    -e FAKE_APPS=/fakebin/apps.txt \
+    -e FAKE_RELEASES=/fakebin/kafka-releases.txt \
+    -e FAKE_KAFKAS="/fakebin/${FAKE_KAFKAS_FILE:-kafkas-kraft.json}" \
+    -e FAKE_NO_STRIMZI="${FAKE_NO_STRIMZI-}" \
+    -e FAKE_JOB="${FAKE_JOB-}" \
+    -e FAKE_PATCH_FAIL="${FAKE_PATCH_FAIL-}" \
+    -e FAKE_MANAGED_CONFLICT="${FAKE_MANAGED_CONFLICT-}" \
+    -e FAKE_REBALANCES="${FAKE_REBALANCES-}" \
+    -e FAKE_CRD_READ_FAIL="${FAKE_CRD_READ_FAIL-}" \
+    -e FAKE_APIRES_FAIL="${FAKE_APIRES_FAIL-}" \
+    -e FAKE_SECRET_LIST_FAIL="${FAKE_SECRET_LIST_FAIL-}" \
+    -e FAKE_RELEASE_REWRITTEN_BACK="${FAKE_RELEASE_REWRITTEN_BACK-}" \
+    -e FAKE_OLD_JOB="${FAKE_OLD_JOB-}" \
+    -e FAKE_REBALANCE_FAIL="${FAKE_REBALANCE_FAIL-}" \
+    -e FAKE_OPERATOR_READ_FAIL="${FAKE_OPERATOR_READ_FAIL-}" \
+    -e FAKE_SERVED="${FAKE_SERVED-}" \
+    -e FAKE_MANAGED_PATCH_FAIL="${FAKE_MANAGED_PATCH_FAIL-}" \
+    -e FAKE_PULL="${FAKE_PULL-}" \
+    -e STRIMZI_CRD_FILE="${STRIMZI_CRD_FILE-/fakebin/strimzi-crds-0.51.0.yaml}" \
+    -e POLL_INTERVAL=1 \
+    -e CONVERSION_TIMEOUT="${CONVERSION_TIMEOUT-3}" \
+    "$TESTIMG" "/migrations/$1" || _run_migration_rc=$?
+  return "$_run_migration_rc"
+}
+
+prep() {
+  docker info >/dev/null 2>&1 || {
+    echo "docker is required: these tests run migration 60 inside a jq-enabled" >&2
+    echo "build of $ALPINE, the migrations image's base." >&2
+    return 1
+  }
+  docker build -q -t "$TESTIMG" - >/dev/null <<DOCKERFILE
+FROM $ALPINE
+RUN apk add --no-cache jq
+DOCKERFILE
+  chmod +x "$FIXTURES/kubectl"
+  mkdir -p "$WORKROOT"
+  WORK=$(mktemp -d "$WORKROOT/XXXXXX")
+  mkdir -p "$WORK/state"
+  printf '["v1beta2"]' > "$WORK/state/stored"
+  FAKE_CMDLOG="$WORK/cmdlog"
+  : > "$FAKE_CMDLOG"
+  unset FAKE_NO_STRIMZI FAKE_JOB FAKE_PATCH_FAIL FAKE_KAFKAS_FILE FAKE_MANAGED_CONFLICT FAKE_MANAGED_PATCH_FAIL FAKE_REBALANCES STRIMZI_CRD_FILE FAKE_CRD_READ_FAIL FAKE_APIRES_FAIL FAKE_SECRET_LIST_FAIL FAKE_RELEASE_REWRITTEN_BACK FAKE_OLD_JOB FAKE_REBALANCE_FAIL FAKE_OPERATOR_READ_FAIL FAKE_SERVED FAKE_PULL CONVERSION_TIMEOUT || true
+  return 0
+}
+
+release_json() {
+  base64 -d < "$WORK/state/release-$1" | base64 -d | gzip -dc
+}
+
+@test "converts to v1, moves Kafka 3.x applications to v4.3 and stamps 61" {
+  prep
+  rc=0
+  run_migration 60 >"$WORK/out" 2>&1 || rc=$?
+  cat "$WORK/out"; cat "$FAKE_CMDLOG"
+  [ "$rc" -eq 0 ]
+
+  grep -qxF "APPLY-CRDS /fakebin/strimzi-crds-0.51.0.yaml" "$FAKE_CMDLOG"
+  # convert-resource is the init container, so crd-upgrade cannot start first.
+  grep -qxF "APPLY-CONVERSION convert-resource crd-upgrade" "$FAKE_CMDLOG"
+  crds_line=$(grep -nxF "APPLY-CRDS /fakebin/strimzi-crds-0.51.0.yaml" "$FAKE_CMDLOG" | cut -d: -f1)
+  job_line=$(grep -nxF "APPLY-CONVERSION convert-resource crd-upgrade" "$FAKE_CMDLOG" | cut -d: -f1)
+  [ "$crds_line" -lt "$job_line" ]
+  # The 0.45 operator is stopped before anything is converted.
+  scale_line=$(grep -nxF "SCALE-OPERATOR 0" "$FAKE_CMDLOG" | cut -d: -f1)
+  [ "$scale_line" -lt "$crds_line" ]
+  # The entity operators only after the cluster operator, or it scales them back.
+  entity_line=$(grep -nxF "SCALE-ENTITY tenant-a/kafka-one-entity-operator" "$FAKE_CMDLOG" | cut -d: -f1)
+  [ "$scale_line" -lt "$entity_line" ]
+  [ "$entity_line" -lt "$crds_line" ]
+
+  grep -qxF 'PATCH tenant-a/one {"spec":{"version":"v4.3"}}' "$FAKE_CMDLOG"
+  grep -qxF 'PATCH tenant-b/two {"spec":{"version":"v4.3"}}' "$FAKE_CMDLOG"
+  if grep -qE "^PATCH tenant-(c|d)/" "$FAKE_CMDLOG"; then echo "patched an application that needs no version change" >&2; return 1; fi
+
+  grep -qxF "DELETE clusterrole strimzi-v1-api-conversion" "$FAKE_CMDLOG"
+  [ "$(tail -1 "$FAKE_CMDLOG")" = "STAMP 61" ]
+  rm -rf "$WORK"
+}
+
+# Helm builds the objects of the deployed release before it upgrades or
+# uninstalls it, and fails on an API version the server no longer serves.
+@test "rewrites the deployed Helm release of each Kafka to the v1 API" {
+  prep
+  rc=0
+  run_migration 60 >"$WORK/out" 2>&1 || rc=$?
+  cat "$WORK/out"; cat "$FAKE_CMDLOG"
+  [ "$rc" -eq 0 ]
+
+  grep -qxF "PATCH-RELEASE tenant-a/sh.helm.release.v1.kafka-one.v3" "$FAKE_CMDLOG"
+  grep -qxF "PATCH-RELEASE tenant-a/sh.helm.release.v1.kafka-one.v4" "$FAKE_CMDLOG"
+  # An application whose install failed has a release and no Kafka CR.
+  grep -qxF "PATCH-RELEASE tenant-f/sh.helm.release.v1.kafka-failed-install.v1" "$FAKE_CMDLOG"
+  # The applications leave Kafka 3.x before any release is rewritten.
+  version_line=$(grep -nF 'PATCH tenant-a/one ' "$FAKE_CMDLOG" | cut -d: -f1)
+  first_release_line=$(grep -nF 'PATCH-RELEASE ' "$FAKE_CMDLOG" | head -1 | cut -d: -f1)
+  [ "$version_line" -lt "$first_release_line" ]
+  release_json v4 > "$WORK/release.json"
+  [ "$(jq -r '.manifest' "$WORK/release.json" | grep -c '^apiVersion: kafka.strimzi.io/v1$')" -eq 2 ]
+  if jq -r '.manifest' "$WORK/release.json" | grep -qF 'kafka.strimzi.io/v1beta2'; then echo "v1beta2 left in the release manifest" >&2; return 1; fi
+  # Only the manifest is rewritten; everything else in the release stays as it was.
+  jq 'del(.manifest)' "$FAKE_FIXTURES/release-kafka-one.json" > "$WORK/want"
+  jq 'del(.manifest)' "$WORK/release.json" > "$WORK/got"
+  cmp "$WORK/want" "$WORK/got"
+  release_line=$(grep -nxF "PATCH-RELEASE tenant-a/sh.helm.release.v1.kafka-one.v3" "$FAKE_CMDLOG" | cut -d: -f1)
+  stamp_line=$(grep -nxF "STAMP 61" "$FAKE_CMDLOG" | cut -d: -f1)
+  [ "$release_line" -lt "$stamp_line" ]
+
+  : > "$FAKE_CMDLOG"
+  rc=0
+  run_migration 60 >"$WORK/out" 2>&1 || rc=$?
+  [ "$rc" -eq 0 ]
+  if grep -qF "PATCH-RELEASE" "$FAKE_CMDLOG"; then echo "a second run rewrote the release again" >&2; return 1; fi
+  rm -rf "$WORK"
+}
+
+# Once the CRDs serve only v1, the API server refuses a server-side apply to
+# an object whose managedFields still name v1beta2.
+@test "moves managedFields entries recorded under old API versions to v1" {
+  prep
+  rc=0
+  run_migration 60 >"$WORK/out" 2>&1 || rc=$?
+  cat "$WORK/out"; cat "$FAKE_CMDLOG"
+  [ "$rc" -eq 0 ]
+  grep -qxF "PATCH-MANAGED tenant-a/kafka-one rv=100 helm-controller=kafka.strimzi.io/v1,strimzi-cluster-operator=kafka.strimzi.io/v1" "$FAKE_CMDLOG"
+  # The status entry keeps its subresource and its fields.
+  [ "$(jq -r '.items[0].metadata.managedFields[1].subresource' "$WORK/state/managed")" = "status" ]
+  if grep -qF "PATCH-MANAGED tenant-b/" "$FAKE_CMDLOG"; then echo "patched an object already on v1" >&2; return 1; fi
+
+  : > "$FAKE_CMDLOG"
+  rc=0
+  run_migration 60 >"$WORK/out" 2>&1 || rc=$?
+  [ "$rc" -eq 0 ]
+  if grep -qF "PATCH-MANAGED" "$FAKE_CMDLOG"; then echo "a second run patched managedFields again" >&2; return 1; fi
+  rm -rf "$WORK"
+}
+
+@test "a managedFields patch that loses a race is rebuilt from a fresh read" {
+  prep
+  export FAKE_MANAGED_CONFLICT=1
+  rc=0
+  run_migration 60 >"$WORK/out" 2>&1 || rc=$?
+  cat "$WORK/out"; cat "$FAKE_CMDLOG"
+  [ "$rc" -eq 0 ]
+  grep -qxF "PATCH-MANAGED tenant-a/kafka-one rv=100 helm-controller=kafka.strimzi.io/v1,strimzi-cluster-operator=kafka.strimzi.io/v1" "$FAKE_CMDLOG"
+  grep -qxF "PATCH-MANAGED tenant-a/kafka-one rv=101 helm-controller=kafka.strimzi.io/v1,strimzi-cluster-operator=kafka.strimzi.io/v1" "$FAKE_CMDLOG"
+  [ "$(jq -r '.items[0].metadata.managedFields[0].apiVersion' "$WORK/state/managed")" = "kafka.strimzi.io/v1" ]
+  grep -qxF "STAMP 61" "$FAKE_CMDLOG"
+  rm -rf "$WORK"
+}
+
+@test "a managedFields patch that fails for another reason aborts before stamping" {
+  prep
+  export FAKE_MANAGED_PATCH_FAIL=1
+  rc=0
+  run_migration 60 >"$WORK/out" 2>&1 || rc=$?
+  cat "$WORK/out"; cat "$FAKE_CMDLOG"
+  [ "$rc" -ne 0 ]
+  [ "$(grep -c "PATCH-MANAGED" "$FAKE_CMDLOG")" -eq 1 ]
+  if grep -qF "STAMP" "$FAKE_CMDLOG"; then echo "stamped after a failed managedFields patch" >&2; return 1; fi
+  rm -rf "$WORK"
+}
+
+@test "a cluster without Strimzi only stamps the version" {
+  prep
+  export FAKE_NO_STRIMZI=1
+  rc=0
+  run_migration 60 >"$WORK/out" 2>&1 || rc=$?
+  cat "$WORK/out"; cat "$FAKE_CMDLOG"
+  [ "$rc" -eq 0 ]
+  if grep -qE "APPLY-CRDS|APPLY-CONVERSION|PATCH" "$FAKE_CMDLOG"; then echo "touched Strimzi on a cluster without it" >&2; return 1; fi
+  grep -qxF "STAMP 61" "$FAKE_CMDLOG"
+  rm -rf "$WORK"
+}
+
+@test "a ZooKeeper-based or still-migrating Kafka stops the migration before it changes anything" {
+  prep
+  export FAKE_KAFKAS_FILE=kafkas-zk.json
+  rc=0
+  run_migration 60 >"$WORK/out" 2>&1 || rc=$?
+  cat "$WORK/out"; cat "$FAKE_CMDLOG"
+  [ "$rc" -ne 0 ]
+  grep -qxF "  tenant-c/kafka-zk" "$WORK/out"
+  grep -qxF "  tenant-d/kafka-mid" "$WORK/out"
+  if grep -qF "tenant-a/kafka-one" "$WORK/out"; then echo "named a KRaft cluster as ZooKeeper-based" >&2; return 1; fi
+  grep -qF "Cozystack v1.7" "$WORK/out"
+  if grep -qE "APPLY-CRDS|APPLY-CONVERSION|PATCH|STAMP|SCALE-OPERATOR" "$FAKE_CMDLOG"; then echo "changed the cluster despite a ZooKeeper Kafka" >&2; return 1; fi
+  rm -rf "$WORK"
+}
+
+@test "a drain of the v1.7 ZooKeeper migration still in flight stops the migration" {
+  prep
+  export FAKE_REBALANCES="tenant-a/kafka-one-drain-kafka,tenant-a/my-own-rebalance"
+  rc=0
+  run_migration 60 >"$WORK/out" 2>&1 || rc=$?
+  cat "$WORK/out"; cat "$FAKE_CMDLOG"
+  [ "$rc" -ne 0 ]
+  grep -qxF "  tenant-a/kafka-one-drain-kafka" "$WORK/out"
+  if grep -qxF "  tenant-a/my-own-rebalance" "$WORK/out"; then echo "named an unrelated KafkaRebalance" >&2; return 1; fi
+  if grep -qE "SCALE-OPERATOR|APPLY-CRDS|STAMP" "$FAKE_CMDLOG"; then echo "changed the cluster despite an unfinished drain" >&2; return 1; fi
+  rm -rf "$WORK"
+}
+
+# Run by hand outside the migrations image, the CRD file is easy to miss;
+# the operators must not be stopped for a run that cannot go on.
+@test "a missing CRD file aborts before the operators are stopped" {
+  prep
+  export STRIMZI_CRD_FILE=/nonexistent/strimzi-crds-0.51.0.yaml
+  rc=0
+  run_migration 60 >"$WORK/out" 2>&1 || rc=$?
+  cat "$WORK/out"; cat "$FAKE_CMDLOG"
+  [ "$rc" -ne 0 ]
+  grep -qF "/nonexistent/strimzi-crds-0.51.0.yaml is not readable" "$WORK/out"
+  if grep -qE "SCALE-|APPLY-CRDS|STAMP" "$FAKE_CMDLOG"; then echo "changed the cluster without the CRD file" >&2; return 1; fi
+  rm -rf "$WORK"
+}
+
+@test "a failed conversion Job aborts with its logs, before any application changes" {
+  prep
+  export FAKE_JOB=fail
+  rc=0
+  run_migration 60 >"$WORK/out" 2>&1 || rc=$?
+  cat "$WORK/out"; cat "$FAKE_CMDLOG"
+  [ "$rc" -ne 0 ]
+  grep -qF "fake conversion log" "$WORK/out"
+  # The failure ends the wait at once, not at the timeout.
+  [ "$(grep -cF "get job strimzi-v1-api-conversion -o jsonpath={.status.succeeded}" "$FAKE_CMDLOG")" -eq 1 ]
+  if grep -qE "PATCH|STAMP" "$FAKE_CMDLOG"; then echo "went on after the conversion failed" >&2; return 1; fi
+  rm -rf "$WORK"
+}
+
+@test "a Job that leaves a non-v1 stored version behind aborts" {
+  prep
+  export FAKE_JOB=noop
+  rc=0
+  run_migration 60 >"$WORK/out" 2>&1 || rc=$?
+  cat "$WORK/out"; cat "$FAKE_CMDLOG"
+  [ "$rc" -ne 0 ]
+  grep -qF 'kafkas.kafka.strimzi.io: ["v1beta2"]' "$WORK/out"
+  if grep -qE "PATCH|STAMP" "$FAKE_CMDLOG"; then echo "went on with v1beta2 still stored" >&2; return 1; fi
+  rm -rf "$WORK"
+}
+
+@test "a re-run after the conversion skips it and still moves the versions" {
+  prep
+  printf '["v1"]' > "$WORK/state/stored"
+  rc=0
+  run_migration 60 >"$WORK/out" 2>&1 || rc=$?
+  cat "$WORK/out"; cat "$FAKE_CMDLOG"
+  [ "$rc" -eq 0 ]
+  if grep -qE "APPLY-CRDS|APPLY-CONVERSION|kafkas.v1beta2" "$FAKE_CMDLOG"; then echo "converted again" >&2; return 1; fi
+  # The CRDs still serve v1beta2, so the operator may be 0.45: it is stopped.
+  grep -qxF "SCALE-OPERATOR 0" "$FAKE_CMDLOG"
+  grep -qxF 'PATCH tenant-a/one {"spec":{"version":"v4.3"}}' "$FAKE_CMDLOG"
+  grep -qxF "STAMP 61" "$FAKE_CMDLOG"
+  rm -rf "$WORK"
+}
+
+@test "a failing application patch aborts before stamping" {
+  prep
+  export FAKE_PATCH_FAIL=1
+  rc=0
+  run_migration 60 >"$WORK/out" 2>&1 || rc=$?
+  cat "$WORK/out"; cat "$FAKE_CMDLOG"
+  [ "$rc" -ne 0 ]
+  if grep -qF "STAMP" "$FAKE_CMDLOG"; then echo "stamped after a failed patch" >&2; return 1; fi
+  rm -rf "$WORK"
+}
+
+# The migration runs once: a read that fails must stop it before the stamp,
+# never pass for "nothing there".
+@test "a failed read of the CRD stored versions aborts before stamping" {
+  prep
+  export FAKE_CRD_READ_FAIL=1
+  rc=0
+  run_migration 60 >"$WORK/out" 2>&1 || rc=$?
+  cat "$WORK/out"; cat "$FAKE_CMDLOG"
+  [ "$rc" -ne 0 ]
+  if grep -qE "STAMP|SCALE-|APPLY-CRDS|PATCH" "$FAKE_CMDLOG"; then echo "went on after a failed CRD read" >&2; return 1; fi
+  rm -rf "$WORK"
+}
+
+@test "a failed read of the API resources aborts before stamping" {
+  prep
+  export FAKE_APIRES_FAIL=1
+  rc=0
+  run_migration 60 >"$WORK/out" 2>&1 || rc=$?
+  cat "$WORK/out"; cat "$FAKE_CMDLOG"
+  [ "$rc" -ne 0 ]
+  if grep -qE "STAMP|PATCH-RELEASE" "$FAKE_CMDLOG"; then echo "went on after a failed API resources read" >&2; return 1; fi
+  rm -rf "$WORK"
+}
+
+@test "a failed Helm release listing aborts before stamping" {
+  prep
+  export FAKE_SECRET_LIST_FAIL=1
+  rc=0
+  run_migration 60 >"$WORK/out" 2>&1 || rc=$?
+  cat "$WORK/out"; cat "$FAKE_CMDLOG"
+  [ "$rc" -ne 0 ]
+  if grep -qF "STAMP" "$FAKE_CMDLOG"; then echo "stamped after a failed release listing" >&2; return 1; fi
+  rm -rf "$WORK"
+}
+
+@test "a release that names v1beta2 again after the rewrite stops the stamp" {
+  prep
+  export FAKE_RELEASE_REWRITTEN_BACK=1
+  rc=0
+  run_migration 60 >"$WORK/out" 2>&1 || rc=$?
+  cat "$WORK/out"; cat "$FAKE_CMDLOG"
+  [ "$rc" -ne 0 ]
+  grep -qxF "  Helm release tenant-a/sh.helm.release.v1.kafka-one.v3" "$WORK/out"
+  if grep -qF "STAMP" "$FAKE_CMDLOG"; then echo "stamped with a v1beta2 release left" >&2; return 1; fi
+  rm -rf "$WORK"
+}
+
+# A Job failed on its deadline can still have a pod terminating, and
+# convert-resource writes until that pod is gone.
+@test "a finished Job of an earlier attempt is removed with its pods before the CRDs are touched" {
+  for state in failed succeeded; do
+    prep
+    export FAKE_OLD_JOB=$state
+    rc=0
+    run_migration 60 >"$WORK/out" 2>&1 || rc=$?
+    cat "$WORK/out"; cat "$FAKE_CMDLOG"
+    [ "$rc" -eq 0 ]
+    delete_line=$(grep -nxF "DELETE job strimzi-v1-api-conversion foreground" "$FAKE_CMDLOG" | head -1 | cut -d: -f1)
+    scale_line=$(grep -nxF "SCALE-OPERATOR 0" "$FAKE_CMDLOG" | cut -d: -f1)
+    crds_line=$(grep -nF "APPLY-CRDS" "$FAKE_CMDLOG" | cut -d: -f1)
+    job_line=$(grep -nF "APPLY-CONVERSION" "$FAKE_CMDLOG" | cut -d: -f1)
+    [ "$delete_line" -lt "$scale_line" ]
+    [ "$delete_line" -lt "$crds_line" ]
+    [ "$crds_line" -lt "$job_line" ]
+    rm -rf "$WORK"
+  done
+}
+
+@test "a Job of an earlier attempt that fails while waited for is removed with its pods and started over" {
+  prep
+  export FAKE_OLD_JOB=running FAKE_JOB=fail
+  rc=0
+  run_migration 60 >"$WORK/out" 2>&1 || rc=$?
+  cat "$WORK/out"; cat "$FAKE_CMDLOG"
+  # The new Job fails as well, so the run stops after starting it.
+  [ "$rc" -ne 0 ]
+  # Each failure ends its wait at once, not at the timeout.
+  [ "$(grep -cF "get job strimzi-v1-api-conversion -o jsonpath={.status.succeeded}" "$FAKE_CMDLOG")" -eq 2 ]
+  delete_line=$(grep -nxF "DELETE job strimzi-v1-api-conversion foreground" "$FAKE_CMDLOG" | head -1 | cut -d: -f1)
+  crds_line=$(grep -nF "APPLY-CRDS" "$FAKE_CMDLOG" | cut -d: -f1)
+  [ "$delete_line" -lt "$crds_line" ]
+  grep -qF "APPLY-CONVERSION" "$FAKE_CMDLOG"
+  rm -rf "$WORK"
+}
+
+# Without a deadline, a Job whose pod never starts never fails, and every
+# retry waits on it and exits. The deadline must cover the wait of the run
+# that started the Job plus the wait of the retry after it.
+@test "the conversion Job carries a deadline of at least two waits for it" {
+  prep
+  export CONVERSION_TIMEOUT=100
+  rc=0
+  run_migration 60 >"$WORK/out" 2>&1 || rc=$?
+  cat "$WORK/out"; cat "$FAKE_CMDLOG"
+  [ "$rc" -eq 0 ]
+  deadline=$(sed -n 's/^CONVERSION-DEADLINE //p' "$FAKE_CMDLOG")
+  [ -n "$deadline" ]
+  [ "$deadline" -ge 200 ]
+  rm -rf "$WORK"
+}
+
+# A pod the API server refused never fails, so the Job reports its deadline
+# only in its Failed condition.
+@test "a Job of an earlier attempt failed on its deadline without a pod is started over" {
+  prep
+  export FAKE_OLD_JOB=running FAKE_JOB=nopod CONVERSION_TIMEOUT=0
+  rc=0
+  run_migration 60 >"$WORK/out" 2>&1 || rc=$?
+  cat "$WORK/out"; cat "$FAKE_CMDLOG"
+  [ "$rc" -ne 0 ]
+  if grep -qF "is still running; retry once it finishes" "$WORK/out"; then echo "took a failed Job for a running one" >&2; return 1; fi
+  delete_line=$(grep -nxF "DELETE job strimzi-v1-api-conversion foreground" "$FAKE_CMDLOG" | head -1 | cut -d: -f1)
+  crds_line=$(grep -nF "APPLY-CRDS" "$FAKE_CMDLOG" | cut -d: -f1)
+  [ "$delete_line" -lt "$crds_line" ]
+  grep -qF "APPLY-CONVERSION" "$FAKE_CMDLOG"
+  rm -rf "$WORK"
+}
+
+@test "a conversion pod that cannot pull its image is named, with what to do" {
+  prep
+  export FAKE_JOB=hang FAKE_PULL=1 CONVERSION_TIMEOUT=0
+  rc=0
+  run_migration 60 >"$WORK/out" 2>&1 || rc=$?
+  cat "$WORK/out"; cat "$FAKE_CMDLOG"
+  [ "$rc" -ne 0 ]
+  grep -qF "cannot pull quay.io/strimzi/operator:0.51.0@sha256:" "$WORK/out"
+  grep -qF "mirror" "$WORK/out"
+  if grep -qE "PATCH|STAMP" "$FAKE_CMDLOG"; then echo "went on without a conversion" >&2; return 1; fi
+  rm -rf "$WORK"
+}
+
+# A retry after the hook timed out must not touch the CRDs under a Job that
+# is still converting.
+@test "a Job of an earlier attempt still running is waited for before the CRDs are touched" {
+  prep
+  export FAKE_OLD_JOB=running
+  rc=0
+  run_migration 60 >"$WORK/out" 2>&1 || rc=$?
+  cat "$WORK/out"; cat "$FAKE_CMDLOG"
+  [ "$rc" -eq 0 ]
+  grep -qF "still running" "$WORK/out"
+  # It converted everything, so there is nothing left to apply or start.
+  if grep -qE "APPLY-CRDS|APPLY-CONVERSION" "$FAKE_CMDLOG"; then echo "touched the CRDs under a running Job" >&2; return 1; fi
+  grep -qxF "DELETE clusterrole strimzi-v1-api-conversion" "$FAKE_CMDLOG"
+  grep -qxF "STAMP 61" "$FAKE_CMDLOG"
+  rm -rf "$WORK"
+}
+
+@test "a Job of an earlier attempt still running after the wait aborts before the CRDs are touched" {
+  prep
+  export FAKE_OLD_JOB=running FAKE_JOB=hang CONVERSION_TIMEOUT=0
+  rc=0
+  run_migration 60 >"$WORK/out" 2>&1 || rc=$?
+  cat "$WORK/out"; cat "$FAKE_CMDLOG"
+  [ "$rc" -ne 0 ]
+  grep -qF "is still running; retry once it finishes" "$WORK/out"
+  if grep -qE "SCALE-|APPLY-CRDS|DELETE|STAMP" "$FAKE_CMDLOG"; then echo "went on under a running Job" >&2; return 1; fi
+  rm -rf "$WORK"
+}
+
+@test "a failed read of the KafkaRebalances aborts before the operators are stopped" {
+  prep
+  export FAKE_REBALANCE_FAIL=1
+  rc=0
+  run_migration 60 >"$WORK/out" 2>&1 || rc=$?
+  cat "$WORK/out"; cat "$FAKE_CMDLOG"
+  [ "$rc" -ne 0 ]
+  if grep -qE "SCALE-|APPLY-CRDS|STAMP" "$FAKE_CMDLOG"; then echo "went on after a failed KafkaRebalance read" >&2; return 1; fi
+  rm -rf "$WORK"
+}
+
+# Converting under a 0.45 operator the migration failed to see would let it
+# record v1beta2 managedFields again after the rewrite.
+@test "a failed read of the cluster operator Deployment aborts before anything is converted" {
+  prep
+  export FAKE_OPERATOR_READ_FAIL=1
+  rc=0
+  run_migration 60 >"$WORK/out" 2>&1 || rc=$?
+  cat "$WORK/out"; cat "$FAKE_CMDLOG"
+  [ "$rc" -ne 0 ]
+  if grep -qE "APPLY-CRDS|APPLY-CONVERSION|STAMP" "$FAKE_CMDLOG"; then echo "converted without knowing whether the 0.45 operator runs" >&2; return 1; fi
+  rm -rf "$WORK"
+}
+
+# Run by hand after the kafka-operator release brought Strimzi 1.x: nothing
+# would scale its operator back up.
+@test "a re-run once the CRDs serve only v1 leaves the operators running" {
+  prep
+  printf '["v1"]' > "$WORK/state/stored"
+  export FAKE_SERVED=v1
+  rc=0
+  run_migration 60 >"$WORK/out" 2>&1 || rc=$?
+  cat "$WORK/out"; cat "$FAKE_CMDLOG"
+  [ "$rc" -eq 0 ]
+  if grep -qF "SCALE-" "$FAKE_CMDLOG"; then echo "stopped the 1.x operator" >&2; return 1; fi
+  grep -qxF "STAMP 61" "$FAKE_CMDLOG"
+  rm -rf "$WORK"
+}

hack/testdata/migration-60-strimzi-v1/apps.txt

@@ -0,0 +1,4 @@
+tenant-a one v3.9
+tenant-b two v3.8
+tenant-c three v4.2
+tenant-d four 

hack/testdata/migration-60-strimzi-v1/kafka-releases.txt

@@ -0,0 +1,2 @@
+tenant-a kafka-one
+tenant-b 

hack/testdata/migration-60-strimzi-v1/kafkas-kraft.json

@@ -0,0 +1,4 @@
+{"items":[
+ {"metadata":{"namespace":"tenant-a","name":"kafka-one","annotations":{"strimzi.io/kraft":"enabled"}},"spec":{"kafka":{"version":"3.9.0"}},"status":{"kafkaMetadataState":"KRaft"}},
+ {"metadata":{"namespace":"tenant-b","name":"kafka-two"},"spec":{"kafka":{"version":"3.8.1"}}}
+]}

hack/testdata/migration-60-strimzi-v1/kafkas-zk.json

@@ -0,0 +1,5 @@
+{"items":[
+ {"metadata":{"namespace":"tenant-a","name":"kafka-one"},"spec":{"kafka":{"version":"3.9.0"}},"status":{"kafkaMetadataState":"KRaft"}},
+ {"metadata":{"namespace":"tenant-c","name":"kafka-zk"},"spec":{"kafka":{"version":"3.9.0"},"zookeeper":{"replicas":3}},"status":{"kafkaMetadataState":"ZooKeeper"}},
+ {"metadata":{"namespace":"tenant-d","name":"kafka-mid"},"spec":{"kafka":{"version":"3.9.0"}},"status":{"kafkaMetadataState":"KRaftPostMigration"}}
+]}

hack/testdata/migration-60-strimzi-v1/kubectl

@@ -0,0 +1,249 @@
+#!/bin/sh
+# Fake kubectl for hack/migration-60-strimzi-v1-conversion.bats. It serves only
+# the calls migration 60 and lib/cozystack-version.sh make, and logs each one
+# to $FAKE_CMDLOG.
+#
+#   FAKE_CMDLOG        file to append a one-line record of each call to
+#   FAKE_STATE         directory; its "stored" file holds the storedVersions
+#                      every Strimzi CRD reports, and the conversion Job
+#                      rewrites it to ["v1"] when it succeeds
+#   FAKE_NO_STRIMZI    non-empty => the Kafka CRD is absent
+#   FAKE_KAFKAS        fixture served for the Strimzi Kafka list
+#   FAKE_JOB           succeed (default) | fail | noop (succeeds, converts nothing)
+#                      | hang (neither succeeds nor fails)
+#                      | nopod (Failed condition, no failed pod: the pod was
+#                      never created)
+#   FAKE_PULL          non-empty => the conversion pod cannot pull its image
+#   FAKE_PATCH_FAIL    non-empty => every application patch exits 1
+#   FAKE_CRD_READ_FAIL, FAKE_APIRES_FAIL, FAKE_SECRET_LIST_FAIL: that read
+#                      exits 1
+#   FAKE_RELEASE_REWRITTEN_BACK: release patches do not stick
+#   FAKE_REBALANCES    comma-separated ns/name served for the KafkaRebalance list
+#   The deployed Helm release of kafka-one is release-kafka-one.json, encoded
+#   the way Helm stores it; a patch of its Secret is written to
+#   $FAKE_STATE/release, so a second run reads what the first one wrote.
+set -u
+
+log() { [ -n "${FAKE_CMDLOG:-}" ] && printf '%s\n' "$*" >> "$FAKE_CMDLOG"; }
+log "KUBECTL $*"
+
+args="$*"
+arg_after() { k="$1"; shift; p=""; for a in "$@"; do [ "$p" = "$k" ] && { printf '%s' "$a"; return; }; p="$a"; done; }
+
+case "$args" in
+  *"apply --filename -"*)
+    manifest=$(cat)
+    v=$(printf '%s\n' "$manifest" | sed -n 's/^  version: "\{0,1\}\([0-9]\{1,\}\)"\{0,1\}$/\1/p' | head -1)
+    log "STAMP ${v:-unknown}"
+    exit 0
+    ;;
+  *"apply -f -"*)
+    manifest=$(cat)
+    case "$manifest" in
+      *"kind: Job"*)
+        order=$(printf '%s\n' "$manifest" | sed -n 's/.*v1-api-conversion.sh", "\([a-z-]*\)".*/\1/p' | tr '\n' ' ')
+        log "APPLY-CONVERSION ${order% }"
+        log "CONVERSION-DEADLINE $(printf '%s\n' "$manifest" | sed -n 's/^  activeDeadlineSeconds: \([0-9]*\)$/\1/p')"
+        exit 0
+        ;;
+    esac
+    log "APPLY-UNKNOWN"
+    exit 1
+    ;;
+  *"apply --server-side"*)
+    log "APPLY-CRDS $(arg_after -f "$@")"
+    exit 0
+    ;;
+  *"get crd kafkas.kafka.strimzi.io --ignore-not-found -o name"*)
+    [ -n "${FAKE_NO_STRIMZI:-}" ] || echo "customresourcedefinition.apiextensions.k8s.io/kafkas.kafka.strimzi.io"
+    exit 0
+    ;;
+  *"get crd kafkas.kafka.strimzi.io -o jsonpath={.spec.versions[*].name}"*)
+    printf '%s' "${FAKE_SERVED:-v1 v1beta2}"
+    exit 0
+    ;;
+  *"get crd "*"storedVersions"*)
+    if [ -n "${FAKE_CRD_READ_FAIL:-}" ]; then echo "Error from server: etcdserver: request timed out" >&2; exit 1; fi
+    cat "$FAKE_STATE/stored"
+    exit 0
+    ;;
+  *"get kafkarebalances.v1beta2.kafka.strimzi.io"*)
+    if [ -n "${FAKE_REBALANCE_FAIL:-}" ]; then echo "Error from server: etcdserver: request timed out" >&2; exit 1; fi
+    [ -z "${FAKE_REBALANCES:-}" ] || printf '%s\n' "$FAKE_REBALANCES" | tr ',' '\n'
+    exit 0
+    ;;
+  *"get kafkas.v1beta2.kafka.strimzi.io"*)
+    cat "$FAKE_KAFKAS"
+    exit 0
+    ;;
+  *"get kafkas.v1.kafka.strimzi.io"*)
+    cat "$FAKE_RELEASES"
+    exit 0
+    ;;
+  *"get secret -l owner=helm,name=kafka-one -o"*)
+    if [ -n "${FAKE_SECRET_LIST_FAIL:-}" ]; then echo "Error from server (Forbidden): secrets is forbidden" >&2; exit 1; fi
+    # v4 stands for a failed upgrade: still the latest revision, so uninstall
+    # builds it.
+    echo "sh.helm.release.v1.kafka-one.v3 sh.helm.release.v1.kafka-one.v4"
+    exit 0
+    ;;
+  *"get secret -l owner=helm,name=kafka-failed-install -o"*)
+    echo "sh.helm.release.v1.kafka-failed-install.v1"
+    exit 0
+    ;;
+  *"get secret sh.helm.release.v1.kafka-failed-install.v1 -o jsonpath={.data.release}"*)
+    if [ ! -f "$FAKE_STATE/release-failed" ]; then
+      gzip -c "$(dirname "$0")/release-kafka-one.json" | base64 | tr -d '\n' | base64 | tr -d '\n' > "$FAKE_STATE/release-failed"
+    fi
+    cat "$FAKE_STATE/release-failed"
+    exit 0
+    ;;
+  *"patch secret sh.helm.release.v1.kafka-failed-install.v1 --type merge --patch-file "*)
+    jq -r '.data.release' "$(arg_after --patch-file "$@")" > "$FAKE_STATE/release-failed"
+    log "PATCH-RELEASE tenant-f/sh.helm.release.v1.kafka-failed-install.v1"
+    exit 0
+    ;;
+  *"get deployment --all-namespaces -l app.kubernetes.io/name=entity-operator"*)
+    [ -n "${FAKE_NO_OPERATOR:-}" ] || echo "tenant-a kafka-one-entity-operator"
+    exit 0
+    ;;
+  *"scale deployment kafka-one-entity-operator --replicas=0"*)
+    log "SCALE-ENTITY tenant-a/kafka-one-entity-operator"
+    exit 0
+    ;;
+  *"get secret -l owner=helm,name="*)
+    exit 0
+    ;;
+  *"get secret sh.helm.release.v1.kafka-one.v"*" -o jsonpath={.data.release}"*)
+    rev=$(arg_after secret "$@"); rev=${rev##*.}
+    if [ ! -f "$FAKE_STATE/release-$rev" ]; then
+      gzip -c "$(dirname "$0")/release-kafka-one.json" | base64 | tr -d '\n' | base64 | tr -d '\n' > "$FAKE_STATE/release-$rev"
+    fi
+    cat "$FAKE_STATE/release-$rev"
+    exit 0
+    ;;
+  *"patch secret sh.helm.release.v1.kafka-one.v"*" --type merge --patch-file "*)
+    rev=$(arg_after secret "$@"); rev=${rev##*.}
+    # FAKE_RELEASE_REWRITTEN_BACK: a Helm upgrade that was already running
+    # stores its v1beta2 revision over the rewrite.
+    [ -n "${FAKE_RELEASE_REWRITTEN_BACK:-}" ] || jq -r '.data.release' "$(arg_after --patch-file "$@")" > "$FAKE_STATE/release-$rev"
+    log "PATCH-RELEASE tenant-a/sh.helm.release.v1.kafka-one.$rev"
+    exit 0
+    ;;
+  *"get deployment strimzi-cluster-operator --ignore-not-found -o name"*)
+    if [ -n "${FAKE_OPERATOR_READ_FAIL:-}" ]; then echo "Error from server: etcdserver: request timed out" >&2; exit 1; fi
+    [ -n "${FAKE_NO_OPERATOR:-}" ] || echo "deployment.apps/strimzi-cluster-operator"
+    exit 0
+    ;;
+  *"scale deployment strimzi-cluster-operator --replicas=0"*)
+    log "SCALE-OPERATOR 0"
+    exit 0
+    ;;
+  *"wait --for=delete pod -l name=strimzi-cluster-operator"*)
+    exit 0
+    ;;
+  *"get kafkas.kafka.strimzi.io --all-namespaces --show-managed-fields -o json"*)
+    if [ -f "$FAKE_STATE/managed" ]; then cat "$FAKE_STATE/managed"; else cat "$(dirname "$0")/managed-kafkas.json"; fi
+    exit 0
+    ;;
+  *"--all-namespaces --show-managed-fields -o json"*)
+    echo '{"items":[]}'
+    exit 0
+    ;;
+  *"get kafkas.kafka.strimzi.io "*" --show-managed-fields -o json"*)
+    ns=$(arg_after -n "$@"); name=$(arg_after kafkas.kafka.strimzi.io "$@")
+    src="$FAKE_STATE/managed"; [ -f "$src" ] || src="$(dirname "$0")/managed-kafkas.json"
+    jq --arg ns "$ns" --arg n "$name" '.items[] | select(.metadata.namespace == $ns and .metadata.name == $n)' "$src"
+    exit 0
+    ;;
+  *"patch kafkas.kafka.strimzi.io "*"--patch-file "*)
+    ns=$(arg_after -n "$@"); name=$(arg_after kafkas.kafka.strimzi.io "$@")
+    patch=$(arg_after --patch-file "$@")
+    src="$FAKE_STATE/managed"; [ -f "$src" ] || src="$(dirname "$0")/managed-kafkas.json"
+    rv=$(jq -r '.metadata.resourceVersion' "$patch")
+    log "PATCH-MANAGED $ns/$name rv=$rv $(jq -r '[.metadata.managedFields[] | "\(.manager)=\(.apiVersion)"] | join(",")' "$patch")"
+    # FAKE_MANAGED_CONFLICT: a concurrent write bumps the object before the
+    # first patch lands, so that patch carries a stale resourceVersion.
+    if [ -n "${FAKE_MANAGED_CONFLICT:-}" ] && [ ! -f "$FAKE_STATE/bumped" ]; then
+      : > "$FAKE_STATE/bumped"
+      jq --arg ns "$ns" --arg n "$name" '(.items[] | select(.metadata.namespace == $ns and .metadata.name == $n) | .metadata.resourceVersion) |= (tonumber + 1 | tostring)' "$src" > "$FAKE_STATE/managed.new" && mv "$FAKE_STATE/managed.new" "$FAKE_STATE/managed"
+    fi
+    if [ -n "${FAKE_MANAGED_PATCH_FAIL:-}" ]; then echo "Error from server (Forbidden): forbidden" >&2; exit 1; fi
+    src="$FAKE_STATE/managed"; [ -f "$src" ] || src="$(dirname "$0")/managed-kafkas.json"
+    cur=$(jq -r --arg ns "$ns" --arg n "$name" '.items[] | select(.metadata.namespace == $ns and .metadata.name == $n) | .metadata.resourceVersion' "$src")
+    if [ "$cur" != "$rv" ]; then
+      echo "Error from server (Conflict): Operation cannot be fulfilled on kafkas.kafka.strimzi.io \"$name\": the object has been modified; please apply your changes to the latest version and try again" >&2
+      exit 1
+    fi
+    jq --arg ns "$ns" --arg n "$name" --slurpfile p "$patch" \
+      '(.items[] | select(.metadata.namespace == $ns and .metadata.name == $n) | .metadata) |= (.managedFields = $p[0].metadata.managedFields | .resourceVersion = (.resourceVersion | tonumber + 1 | tostring))' \
+      "$src" > "$FAKE_STATE/managed.new" && mv "$FAKE_STATE/managed.new" "$FAKE_STATE/managed"
+    exit 0
+    ;;
+  *"wait --for=condition=Established"*)
+    exit 0
+    ;;
+  *"get job strimzi-v1-api-conversion --ignore-not-found -o jsonpath={.metadata.name} {.status.succeeded}{.status.failed}"*)
+    # FAKE_OLD_JOB: a Job an earlier attempt left behind, running, succeeded
+    # or failed.
+    case "${FAKE_OLD_JOB:-}" in
+      running) printf 'strimzi-v1-api-conversion ' ;;
+      succeeded|failed) printf 'strimzi-v1-api-conversion 1' ;;
+    esac
+    exit 0
+    ;;
+  *"get job strimzi-v1-api-conversion -o jsonpath={.status.succeeded}"*)
+    case "${FAKE_JOB:-succeed}" in
+      succeed) printf '["v1"]' > "$FAKE_STATE/stored"; echo 1 ;;
+      noop) echo 1 ;;
+    esac
+    exit 0
+    ;;
+  *"get job strimzi-v1-api-conversion -o jsonpath={.status.failed}"*)
+    case "${FAKE_JOB:-succeed}" in
+      fail) printf '1' ;;
+    esac
+    case "$args" in
+      *'conditions[?(@.type=="Failed")]'*)
+        case "${FAKE_JOB:-succeed}" in fail|nopod) printf ' True' ;; esac
+        ;;
+    esac
+    exit 0
+    ;;
+  *"get pod -l batch.kubernetes.io/job-name=strimzi-v1-api-conversion -o jsonpath="*)
+    [ -z "${FAKE_PULL:-}" ] || printf ' ImagePullBackOff'
+    exit 0
+    ;;
+  *" logs "*)
+    echo "fake conversion log"
+    exit 0
+    ;;
+  "delete "*|*" delete "*)
+    cascade=""
+    case "$args" in *"--cascade=foreground"*) cascade=" foreground" ;; esac
+    log "DELETE $(arg_after delete "$@") $(arg_after "$(arg_after delete "$@")" "$@")$cascade"
+    exit 0
+    ;;
+  *"api-resources --api-group=apps.cozystack.io"*)
+    if [ -n "${FAKE_APIRES_FAIL:-}" ]; then echo "error: unable to retrieve the complete list of server APIs" >&2; exit 1; fi
+    echo "kafkas.apps.cozystack.io"
+    exit 0
+    ;;
+  *"get kafkas.apps.cozystack.io"*" kafka-"*)
+    # Release names of the applications; failed-install has no Kafka CR.
+    printf '%s\n' 'tenant-a kafka-one' 'tenant-f kafka-failed-install'
+    exit 0
+    ;;
+  *"get kafkas.apps.cozystack.io"*)
+    cat "$FAKE_APPS"
+    exit 0
+    ;;
+  *"patch kafkas.apps.cozystack.io"*)
+    if [ -n "${FAKE_PATCH_FAIL:-}" ]; then echo "Error from server (Conflict)" >&2; exit 1; fi
+    log "PATCH $(arg_after -n "$@")/$(arg_after kafkas.apps.cozystack.io "$@") $(arg_after -p "$@")"
+    exit 0
+    ;;
+esac
+log "UNMODELLED"
+printf 'fake kubectl: unmodelled call: %s\n' "$args" >&2
+exit 1

hack/testdata/migration-60-strimzi-v1/managed-kafkas.json

@@ -0,0 +1,54 @@
+{
+ "items": [
+  {
+   "metadata": {
+    "namespace": "tenant-a",
+    "name": "kafka-one",
+    "managedFields": [
+     {
+      "manager": "helm-controller",
+      "operation": "Apply",
+      "apiVersion": "kafka.strimzi.io/v1beta2",
+      "fieldsType": "FieldsV1",
+      "fieldsV1": {
+       "f:spec": {
+        "f:kafka": {
+         "f:version": {}
+        }
+       }
+      }
+     },
+     {
+      "manager": "strimzi-cluster-operator",
+      "operation": "Update",
+      "apiVersion": "kafka.strimzi.io/v1beta2",
+      "subresource": "status",
+      "fieldsType": "FieldsV1",
+      "fieldsV1": {
+       "f:status": {}
+      }
+     }
+    ],
+    "resourceVersion": "100"
+   }
+  },
+  {
+   "metadata": {
+    "namespace": "tenant-b",
+    "name": "kafka-two",
+    "managedFields": [
+     {
+      "manager": "helm-controller",
+      "operation": "Apply",
+      "apiVersion": "kafka.strimzi.io/v1",
+      "fieldsType": "FieldsV1",
+      "fieldsV1": {
+       "f:spec": {}
+      }
+     }
+    ],
+    "resourceVersion": "101"
+   }
+  }
+ ]
+}

hack/testdata/migration-60-strimzi-v1/release-kafka-one.json

@@ -0,0 +1 @@
+{"name":"kafka-one","version":3,"info":{"status":"deployed"},"manifest":"---\n# Source: kafka/templates/kafka.yaml\napiVersion: kafka.strimzi.io/v1beta2\nkind: Kafka\nmetadata:\n  name: kafka-one\n---\n# Source: kafka/templates/topics.yaml\napiVersion: kafka.strimzi.io/v1beta2\nkind: KafkaTopic\nmetadata:\n  name: kafka-one-orders\n---\napiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: kafka-one-metrics\n","hooks":[{"name":"kafka-one-kafka-migration","manifest":"apiVersion: batch/v1\nkind: Job\n# script applies apiVersion: kafka.strimzi.io/v1beta2\n"}]}

hack/testdata/migration-60-strimzi-v1/strimzi-crds-0.51.0.yaml

@@ -0,0 +1 @@
+# Stand-in for strimzi-crds-0.51.0.yaml; the fake kubectl never reads it.

packages/core/platform/images/migrations/Dockerfile

@@ -29,6 +29,14 @@ RUN set -eux; \
     echo "${sha256}  /usr/bin/etcd-migrate" | sha256sum -c -; \
     chmod +x /usr/bin/etcd-migrate
 
+# CRDs of Strimzi 0.51.0, the last release serving both v1beta2 and v1.
+# Migration 60 applies them so Strimzi's conversion tool can move every stored
+# object to v1 before the kafka-operator release installs the v1-only CRDs.
+RUN mkdir /strimzi-crds && \
+    curl -fsSL -o /strimzi-crds/strimzi-crds-0.51.0.yaml \
+      https://github.com/strimzi/strimzi-kafka-operator/releases/download/0.51.0/strimzi-crds-0.51.0.yaml && \
+    echo "a11257e67985a946a6facfb17d729aac5670b89675d410d13162fe455a5ac7a8  /strimzi-crds/strimzi-crds-0.51.0.yaml" | sha256sum -c -
+
 COPY migrations /migrations
 # v1alpha2 etcd CRDs, vendored into the build context by `make image-migrations`.
 # migration 50 applies these before adoption so it does not deadlock waiting for

packages/core/platform/images/migrations/migrations/60

@@ -0,0 +1,473 @@
+#!/bin/sh
+# Migration 60 --> 61
+# Move the Strimzi custom resources to the v1 API before Strimzi 1.x arrives.
+#
+# Strimzi 1.0 removed the v1beta2 API from its CRDs. The kafka-operator
+# release replaces the CRDs on upgrade (upgradeCRDs: CreateReplace), and the
+# API server refuses a CRD whose spec.versions no longer lists a version that
+# status.storedVersions still records. So every stored object has to be
+# rewritten as v1, and storedVersions narrowed to [v1], before the new
+# operator arrives. That needs CRDs serving both versions, which only Strimzi
+# 0.49 to 0.51 ship: the 0.51.0 CRDs are baked into this image (see the
+# Dockerfile). The conversion itself is Strimzi's v1-api-conversion tool, run
+# as a Job from the 0.51.0 operator image: convert-resource rewrites the
+# objects, crd-upgrade switches the storage version and narrows
+# storedVersions.
+#
+# Strimzi 0.46 also removed ZooKeeper. A ZooKeeper-based Kafka that did not
+# reach KRaft on the release carrying the chart's ZooKeeper to KRaft
+# migration cannot be run, converted or migrated by anything after this
+# point, so this migration refuses to proceed while one is left.
+#
+# Strimzi 1.2 runs neither Kafka 3.8 nor 3.9, so every Kafka application on
+# v3.8 or v3.9 is moved to v4.3. Until the new kafka chart arrives, the
+# release renders with the old one, which rejects v4.3, and the HelmRelease
+# reports a values error; no Helm action runs in that window.
+#
+# Ordering: this runs as the platform chart's pre-upgrade hook, before the
+# upgrade re-points cozystack-packages, so on a variant that runs migrations
+# neither the new operator nor the new kafka chart is rendered before it. The
+# default variant renders no migration hook, and there these steps are manual.
+#
+# Re-runnable: every step tolerates having run before.
+
+set -euo pipefail
+
+# shellcheck source=lib/cozystack-version.sh
+. "$(dirname "$0")/lib/cozystack-version.sh"
+
+STRIMZI_CRD_FILE="${STRIMZI_CRD_FILE:-/strimzi-crds/strimzi-crds-0.51.0.yaml}"
+CONVERSION_IMAGE="${CONVERSION_IMAGE:-quay.io/strimzi/operator:0.51.0@sha256:ef3e406aa09449dd0241d53cdc81e15e58bf59b51f2425fb64384a179d676b1d}"
+CONVERSION_NS="${CONVERSION_NS:-cozy-kafka-operator}"
+CONVERSION_NAME=strimzi-v1-api-conversion
+CONVERSION_TIMEOUT="${CONVERSION_TIMEOUT:-1800}"
+# The run that starts the Job waits up to CONVERSION_TIMEOUT for it. As the
+# platform hook, a run is cut off sooner by the Helm upgrade timeout
+# (cozystack-operator --helmrelease-upgrade-timeout, 10m by default), and the
+# retry, the next pod of the hook Job or a new hook Job once helm-controller
+# retries the upgrade, waits up to CONVERSION_TIMEOUT again for the same Job.
+# The Job is not owned by the hook Job and nothing else bounds it, so the
+# deadline is the only limit: a conversion gets two waits, and a Job whose pod
+# never starts then fails and the next run starts the conversion over.
+CONVERSION_DEADLINE=$((CONVERSION_TIMEOUT * 2))
+POLL_INTERVAL="${POLL_INTERVAL:-5}"
+KAFKA_TARGET_VERSION=v4.3
+
+# The CRDs the conversion tool rewrites. kafkamirrormakers.kafka.strimzi.io
+# is not among them: MirrorMaker 1 left in Strimzi 0.46 and its CRD is left
+# behind untouched.
+STRIMZI_CRDS="kafkas.kafka.strimzi.io kafkanodepools.kafka.strimzi.io
+kafkaconnects.kafka.strimzi.io kafkaconnectors.kafka.strimzi.io
+kafkabridges.kafka.strimzi.io kafkamirrormaker2s.kafka.strimzi.io
+kafkarebalances.kafka.strimzi.io kafkatopics.kafka.strimzi.io
+kafkausers.kafka.strimzi.io strimzipodsets.core.strimzi.io"
+
+# --ignore-not-found separates a cluster without Strimzi (no output, rc 0)
+# from an API error, which aborts under set -e.
+crd=$(kubectl get crd kafkas.kafka.strimzi.io --ignore-not-found -o name)
+if [ -z "$crd" ]; then
+  echo "Strimzi is not installed; nothing to convert"
+  stamp_cozystack_version 61
+  exit 0
+fi
+
+# The running 0.45 operator must not write to the resources from here on.
+# Strimzi supports the conversion only under an operator that knows v1
+# (0.49 or later), and 0.45 rewrites the Kafka status through v1beta2 on every
+# reconcile, because the v1 schema prunes its kafkaMetadataState field. Each
+# such write records a v1beta2 managedFields entry again after the rewrite
+# below. The kafka-operator release brings the operator back at Strimzi 1.2:
+# its chart sets the replica count, so the upgrade restores it. Kafka pods keep
+# running while no operator reconciles them.
+stop_old_operator() {
+  operator=$(kubectl -n "$CONVERSION_NS" get deployment strimzi-cluster-operator --ignore-not-found -o name)
+  if [ -n "$operator" ]; then
+    kubectl -n "$CONVERSION_NS" scale deployment strimzi-cluster-operator --replicas=0
+    kubectl -n "$CONVERSION_NS" wait --for=delete pod -l name=strimzi-cluster-operator --timeout=300s
+  fi
+  # The topic and user operators of each cluster write KafkaTopic and
+  # KafkaUser status through v1beta2 the same way. Only now that the cluster
+  # operator is gone does a scale-down stick; the 1.2 operator scales them
+  # back up when it rolls them onto its own image.
+  kubectl get deployment --all-namespaces -l app.kubernetes.io/name=entity-operator,app.kubernetes.io/managed-by=strimzi-cluster-operator \
+    -o jsonpath='{range .items[*]}{.metadata.namespace}{" "}{.metadata.name}{"\n"}{end}' |
+  while read -r ns name; do
+    kubectl -n "$ns" scale deployment "$name" --replicas=0
+  done
+  }
+
+# Called as an if condition, where set -e does not apply: a failed read exits
+# here, or it would pass for an absent CRD.
+stored_v1_only() {
+  for c in $STRIMZI_CRDS; do
+    if ! v=$(kubectl get crd "$c" --ignore-not-found -o jsonpath='{.status.storedVersions}'); then
+      echo "ERROR: reading the stored versions of $c failed" >&2
+      exit 1
+    fi
+    if [ -n "$v" ] && [ "$v" != '["v1"]' ]; then
+      return 1
+    fi
+  done
+  return 0
+  }
+
+# A Job whose pod the API server refused fails on its deadline with no failed
+# pod to count, so its Failed condition is read as well.
+job_failed() {
+  f=$(kubectl -n "$CONVERSION_NS" get job "$CONVERSION_NAME" -o jsonpath='{.status.failed} {.status.conditions[?(@.type=="Failed")].status}')
+  case "$f" in
+    [1-9]*|*True*) return 0 ;;
+  esac
+  return 1
+  }
+
+# wait_for_conversion: 0 once the conversion Job has succeeded; on a failure
+# or a timeout, print its logs and return 1.
+wait_for_conversion() {
+  echo "Waiting for the conversion Job"
+  waited=0
+  pull_reported=
+  while :; do
+    succeeded=$(kubectl -n "$CONVERSION_NS" get job "$CONVERSION_NAME" -o jsonpath='{.status.succeeded}')
+    # A pod that cannot pull its image leaves no logs to print, and its Job
+    # fails only at the deadline.
+    if [ -z "$pull_reported" ]; then
+      waiting=$(kubectl -n "$CONVERSION_NS" get pod -l "batch.kubernetes.io/job-name=$CONVERSION_NAME" \
+        -o jsonpath='{.items[*].status.initContainerStatuses[*].state.waiting.reason} {.items[*].status.containerStatuses[*].state.waiting.reason}' || true)
+      case "$waiting" in
+        *ErrImagePull*|*ImagePullBackOff*)
+          echo "ERROR: the conversion pod cannot pull $CONVERSION_IMAGE. A cluster that cannot reach quay.io needs that image mirrored under the same name and digest, for example through a registry mirror for quay.io in the container runtime of the nodes. The Job fails ${CONVERSION_DEADLINE}s after it started, and the next run of the migration starts it again." >&2
+          pull_reported=1
+          ;;
+      esac
+    fi
+    if [ "${succeeded:-0}" -ge 1 ]; then
+      kubectl -n "$CONVERSION_NS" logs "job/$CONVERSION_NAME" --all-containers --prefix || true
+      return 0
+    fi
+    if job_failed || [ "$waited" -ge "$CONVERSION_TIMEOUT" ]; then
+      echo "ERROR: the Strimzi v1 API conversion Job did not succeed; its logs:" >&2
+      kubectl -n "$CONVERSION_NS" logs "job/$CONVERSION_NAME" --all-containers --prefix >&2 || true
+      return 1
+    fi
+    sleep "$POLL_INTERVAL"
+    waited=$((waited + POLL_INTERVAL))
+  done
+  }
+
+# The Job's ClusterRole can patch CRDs and write ConfigMaps everywhere, so
+# none of it outlives the conversion, whichever path reaches here.
+remove_conversion_job() {
+  kubectl -n "$CONVERSION_NS" delete job "$CONVERSION_NAME" --ignore-not-found --wait=true
+  kubectl delete clusterrolebinding "$CONVERSION_NAME" --ignore-not-found
+  kubectl delete clusterrole "$CONVERSION_NAME" --ignore-not-found
+  kubectl -n "$CONVERSION_NS" delete serviceaccount "$CONVERSION_NAME" --ignore-not-found
+  }
+
+# The platform hook can time out while the Job converts, and the retry kills
+# the hook at whatever point it reached. A Job still running then has to
+# finish before anything here touches the CRDs, or the CRD apply below would
+# switch the storage version back under it. Its outcome is read from the CRDs
+# afterwards, not from the Job.
+job_state=$(kubectl -n "$CONVERSION_NS" get job "$CONVERSION_NAME" --ignore-not-found -o jsonpath='{.metadata.name} {.status.succeeded}{.status.failed}{.status.conditions[?(@.type=="Failed")].status}')
+case "$job_state" in
+  "$CONVERSION_NAME ")
+    echo "A conversion Job from an earlier attempt is still running"
+    # backoffLimit is 0, so a Job reporting no failure has not finished.
+    if ! wait_for_conversion; then
+      if ! job_failed; then
+        echo "ERROR: the conversion Job of an earlier attempt is still running; retry once it finishes" >&2
+        exit 1
+      fi
+    fi
+    ;;
+esac
+# A Job failed on its deadline can still have a pod terminating, and
+# convert-resource writes until it is gone. A foreground deletion returns
+# only once the Job's pods are.
+if [ -n "$job_state" ]; then
+  kubectl -n "$CONVERSION_NS" delete job "$CONVERSION_NAME" --ignore-not-found --cascade=foreground --wait=true
+fi
+
+if stored_v1_only; then
+  echo "Every Strimzi CRD already stores only v1; skipping the conversion"
+  # The 0.45 operator can only still be running while the CRDs serve
+  # v1beta2. Once they serve v1 alone the operator is 1.x, and a run by hand
+  # must not stop it: no Helm upgrade would bring it back.
+  served=$(kubectl get crd kafkas.kafka.strimzi.io -o jsonpath='{.spec.versions[*].name}')
+  case " $served " in
+    *" v1beta2 "*) stop_old_operator ;;
+  esac
+  remove_conversion_job
+else
+  # A Kafka that is not KRaft yet still has its ZooKeeper section, or reports a
+  # metadata state other than KRaft while the ZooKeeper to KRaft migration
+  # runs. Read through v1beta2, which every Strimzi this can meet serves.
+  zk=$(kubectl get kafkas.v1beta2.kafka.strimzi.io --all-namespaces -o json | jq -r '
+    .items[]
+    | select(.spec.zookeeper != null or ((.status.kafkaMetadataState // "KRaft") != "KRaft"))
+    | "\(.metadata.namespace)/\(.metadata.name)"')
+  if [ -n "$zk" ]; then
+    echo "ERROR: these Kafka clusters still run on ZooKeeper:" >&2
+    printf '%s\n' "$zk" | sed 's/^/  /' >&2
+    echo "Strimzi 1.x cannot run them. Upgrade through Cozystack v1.7, whose kafka chart migrates every instance to KRaft, and wait until each one reports Ready before upgrading past it." >&2
+    exit 1
+  fi
+
+  # The v1.7 kafka chart freed the shared "kafka" pool name by draining a
+  # migrated cluster onto its own pool through a KafkaRebalance named
+  # <release>-drain-kafka, and removed the rebalance once done. The new chart
+  # no longer drives that drain, so one still in flight would be left half
+  # done, with the brokers spread over two pools.
+  rebalances=$(kubectl get kafkarebalances.v1beta2.kafka.strimzi.io --all-namespaces \
+    -o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}')
+  drains=$(printf '%s\n' "$rebalances" | grep -- '-drain-kafka$' || true)
+  if [ -n "$drains" ]; then
+    echo "ERROR: a ZooKeeper to KRaft migration of Cozystack v1.7 is still draining a shared broker pool:" >&2
+    printf '%s\n' "$drains" | sed 's/^/  /' >&2
+    echo "Let the kafka application upgrade on v1.7 finish, which removes these KafkaRebalances, before upgrading past it." >&2
+    exit 1
+  fi
+
+  if [ ! -r "$STRIMZI_CRD_FILE" ]; then
+    echo "ERROR: $STRIMZI_CRD_FILE is not readable; set STRIMZI_CRD_FILE to strimzi-crds-0.51.0.yaml from the Strimzi 0.51.0 release" >&2
+    exit 1
+  fi
+  stop_old_operator
+  # These CRDs mark v1beta2 as the storage version, so a re-run after a
+  # partial crd-upgrade switches storage back and converts from the start.
+  echo "Applying the Strimzi 0.51.0 CRDs, which serve both v1beta2 and v1"
+  kubectl apply --server-side --force-conflicts --field-manager=cozystack-migration-60 -f "$STRIMZI_CRD_FILE"
+  for c in $STRIMZI_CRDS; do
+    kubectl wait --for=condition=Established --timeout=120s "crd/$c"
+  done
+
+  # Whatever an earlier attempt left has finished by now, and its result was
+  # undone by the CRD apply above, so the conversion starts over.
+  kubectl -n "$CONVERSION_NS" delete job "$CONVERSION_NAME" --ignore-not-found --wait=true
+  kubectl apply -f - <<EOF
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: $CONVERSION_NAME
+  namespace: $CONVERSION_NS
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: $CONVERSION_NAME
+rules:
+  - apiGroups: ["kafka.strimzi.io"]
+    resources: ["kafkas", "kafkanodepools", "kafkaconnects", "kafkaconnectors", "kafkabridges", "kafkamirrormaker2s", "kafkarebalances", "kafkatopics", "kafkausers"]
+    verbs: ["get", "list", "patch", "update"]
+  - apiGroups: ["core.strimzi.io"]
+    resources: ["strimzipodsets"]
+    verbs: ["get", "list", "patch", "update"]
+  - apiGroups: ["apiextensions.k8s.io"]
+    resources: ["customresourcedefinitions", "customresourcedefinitions/status"]
+    verbs: ["get", "list", "patch", "update"]
+  # The tool moves a KafkaBridge metrics configuration into a ConfigMap.
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get", "list", "create", "patch", "update"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: $CONVERSION_NAME
+subjects:
+  - kind: ServiceAccount
+    name: $CONVERSION_NAME
+    namespace: $CONVERSION_NS
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: $CONVERSION_NAME
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: $CONVERSION_NAME
+  namespace: $CONVERSION_NS
+spec:
+  backoffLimit: 0
+  activeDeadlineSeconds: $CONVERSION_DEADLINE
+  template:
+    metadata:
+      labels:
+        policy.cozystack.io/allow-to-apiserver: "true"
+    spec:
+      serviceAccountName: $CONVERSION_NAME
+      restartPolicy: Never
+      securityContext:
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault
+      # crd-upgrade must not start before every object is converted.
+      initContainers:
+        - name: convert-resource
+          image: $CONVERSION_IMAGE
+          command: ["/opt/v1-api-conversion/bin/v1-api-conversion.sh", "convert-resource", "--all-namespaces", "--log-level", "INFO"]
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+      containers:
+        - name: crd-upgrade
+          image: $CONVERSION_IMAGE
+          command: ["/opt/v1-api-conversion/bin/v1-api-conversion.sh", "crd-upgrade"]
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+EOF
+
+  if ! wait_for_conversion; then
+    exit 1
+  fi
+
+  if ! stored_v1_only; then
+    echo "ERROR: the conversion Job succeeded but a Strimzi CRD still records a stored version other than v1" >&2
+    for c in $STRIMZI_CRDS; do
+      echo "  $c: $(kubectl get crd "$c" --ignore-not-found -o jsonpath='{.status.storedVersions}')" >&2
+    done
+    exit 1
+  fi
+
+  remove_conversion_job
+fi
+
+# Moving the applications off Kafka 3.x first makes the old chart reject
+# their values, so no Helm action can store a new v1beta2 revision behind the
+# rewrite below.
+app_resources=$(kubectl api-resources --api-group=apps.cozystack.io -o name)
+has_kafka_apps() { printf '%s\n' "$app_resources" | grep -q '^kafkas\.'; }
+if has_kafka_apps; then
+  kubectl get kafkas.apps.cozystack.io --all-namespaces \
+    -o jsonpath='{range .items[*]}{.metadata.namespace}{" "}{.metadata.name}{" "}{.spec.version}{"\n"}{end}' |
+  while read -r ns name version; do
+    case "$version" in
+      v3.8|v3.9)
+        echo "Moving Kafka $ns/$name from $version to $KAFKA_TARGET_VERSION"
+        kubectl -n "$ns" patch kafkas.apps.cozystack.io "$name" --type merge \
+          -p "{\"spec\":{\"version\":\"$KAFKA_TARGET_VERSION\"}}"
+        ;;
+    esac
+  done
+fi
+
+# Helm builds the objects of a stored release before it upgrades,
+# uninstalls or rolls back to it, and fails on an API version the server no
+# longer serves, so every Kafka release would be stuck once the v1-only CRDs
+# land. Uninstall reads the latest revision whatever its status, and upgrade
+# falls back to the last failed one, so every revision of each release is
+# rewritten, the way helm-mapkubeapis does: the Secret holds base64 of
+# gzipped JSON, inside the base64 of the Secret data. A release whose install
+# failed has no Kafka CR, so the releases are listed from the applications as
+# well. The chart's hooks carry no Strimzi kinds, so only the manifest needs
+# the rewrite.
+work=$(mktemp -d)
+trap 'rm -rf "$work"' EXIT
+
+list_releases() {
+  kubectl get kafkas.v1.kafka.strimzi.io --all-namespaces \
+    -o jsonpath='{range .items[*]}{.metadata.namespace}{" "}{.metadata.annotations.meta\.helm\.sh/release-name}{"\n"}{end}' > "$work/releases.all"
+  if has_kafka_apps; then
+    kubectl get kafkas.apps.cozystack.io --all-namespaces \
+      -o jsonpath='{range .items[*]}{.metadata.namespace}{" kafka-"}{.metadata.name}{"\n"}{end}' >> "$work/releases.all"
+  fi
+  sort -u "$work/releases.all" > "$work/releases"
+  }
+
+# scan_releases rewrite|check: rewrite every revision that still names
+# v1beta2, or list the ones that do.
+scan_releases() {
+  list_releases
+  while read -r ns release; do
+    [ -n "$release" ] || continue
+    secrets=$(kubectl -n "$ns" get secret -l "owner=helm,name=$release" -o jsonpath='{.items[*].metadata.name}')
+    for secret in $secrets; do
+      kubectl -n "$ns" get secret "$secret" -o jsonpath='{.data.release}' | base64 -d | base64 -d | gzip -dc > "$work/release.json"
+      if ! jq -e '.manifest | contains("apiVersion: kafka.strimzi.io/v1beta2")' "$work/release.json" > /dev/null; then
+        continue
+      fi
+      if [ "$1" = check ]; then
+        echo "  Helm release $ns/$secret"
+        continue
+      fi
+      echo "Rewriting Helm release $ns/$secret to kafka.strimzi.io/v1"
+      jq -c '.manifest |= gsub("apiVersion: kafka.strimzi.io/v1beta2\n"; "apiVersion: kafka.strimzi.io/v1\n")' "$work/release.json" |
+        gzip -c | base64 | tr -d '\n' | base64 | tr -d '\n' > "$work/release.b64"
+      jq -n --rawfile r "$work/release.b64" '{data: {release: $r}}' > "$work/patch.json"
+      kubectl -n "$ns" patch secret "$secret" --type merge --patch-file "$work/patch.json"
+    done
+  done < "$work/releases"
+  }
+
+# The objects keep managedFields entries recorded under the old API versions,
+# and once the CRDs serve only v1 the API server refuses every server-side
+# apply to such an object ("request to convert CR to an invalid group/version",
+# kubernetes/kubernetes#111937). helm-controller recovers by wiping the
+# entries, and the fields Helm owned pass to "before-first-apply", so a field a
+# later chart drops is never removed. Rewrite the entries to v1 while both
+# versions are still served; the conversion is None, so field paths are
+# spelled the same under either, and an entry for a field v1 dropped only
+# names a field that no longer exists.
+#
+# The patch replaces the whole managedFields list, so a patch built from a
+# stale read could drop an entry a concurrent writer added in between. The
+# patch carries the resourceVersion it was built from: the API server then
+# answers a concurrent change with a conflict, and the object is read again.
+stale_fields='[.metadata.managedFields[]? | select(.apiVersion | test("/v1(alpha1|beta1|beta2)$"))] | length > 0'
+
+# list_stale_managed <crd>: the objects of that kind with entries to rewrite.
+list_stale_managed() {
+  kubectl get "$1" --all-namespaces --show-managed-fields -o json |
+    jq -r ".items[] | select($stale_fields) | \"\\(.metadata.namespace) \\(.metadata.name)\"" > "$work/managed"
+  }
+
+rewrite_managed() {
+  for c in $STRIMZI_CRDS; do
+    list_stale_managed "$c"
+    while read -r ns name; do
+      echo "Moving the managedFields of $c $ns/$name to v1"
+      attempt=0
+      while :; do
+        kubectl -n "$ns" get "$c" "$name" --show-managed-fields -o json > "$work/object.json"
+        if ! jq -e "$stale_fields" "$work/object.json" > /dev/null; then
+          break
+        fi
+        jq '{metadata: {resourceVersion: .metadata.resourceVersion, managedFields: (.metadata.managedFields | map(.apiVersion |= sub("/v1(alpha1|beta1|beta2)$"; "/v1")))}}' \
+          "$work/object.json" > "$work/managed-patch.json"
+        if kubectl -n "$ns" patch "$c" "$name" --type merge --patch-file "$work/managed-patch.json" 2> "$work/patch.err"; then
+          break
+        fi
+        attempt=$((attempt + 1))
+        if ! grep -q 'the object has been modified' "$work/patch.err" || [ "$attempt" -ge 5 ]; then
+          cat "$work/patch.err" >&2
+          exit 1
+        fi
+      done
+    done < "$work/managed"
+  done
+  }
+
+scan_releases rewrite
+rewrite_managed
+
+# A Helm upgrade already running when the releases were rewritten stores its
+# own revision when it finishes, and that one may name v1beta2 again. Scan
+# once more and refuse to stamp the version on any leftover: the hook then
+# fails and runs the migration again.
+scan_releases check > "$work/leftover"
+for c in $STRIMZI_CRDS; do
+  list_stale_managed "$c"
+  sed "s|^|  managedFields of $c |" "$work/managed" >> "$work/leftover"
+done
+if [ -s "$work/leftover" ]; then
+  echo "ERROR: these still name a removed Strimzi API version:" >&2
+  cat "$work/leftover" >&2
+  exit 1
+fi
+
+stamp_cozystack_version 61

packages/core/platform/values.yaml

@@ -14,7 +14,7 @@ migrations:
   # refuses to advance past a migration file missing from the image (exit 1), so
   # a stale pin fails loudly rather than silently skipping the etcd adoption.
   image: ghcr.io/cozystack/cozystack/platform-migrations:v1.6.0@sha256:6777042e3e9c6bad76e7d96fb64baa115061975f39061cb3ede84e21d0e2213f
-  targetVersion: 60
+  targetVersion: 61
   # Adopt legacy etcd clusters onto the v1alpha2 operator WITHOUT the mandatory
   # pre-adoption safety snapshot (migration 50). Leave false unless migration 50
   # has actually refused to proceed and you accept the risk: adoption rewires