diff --git a/hack/migration-61-strimzi-v1-conversion.bats b/hack/migration-61-strimzi-v1-conversion.bats new file mode 100644 index 000000000..6b82dd5c7 --- /dev/null +++ b/hack/migration-61-strimzi-v1-conversion.bats @@ -0,0 +1,1013 @@ +#!/usr/bin/env bats +# ----------------------------------------------------------------------------- +# Unit tests for platform migration 61 --> 62 (move the Strimzi custom +# resources to the v1 API, refuse ZooKeeper-based Kafkas, move Kafka 3.x +# applications to v4.3). See +# packages/core/platform/images/migrations/migrations/61 for the mechanism. +# +# The fake kubectl keeps the storedVersions every Strimzi CRD reports in a +# state file, and the conversion Job rewrites it when it succeeds, so the +# check after the Job reads what the Job did. +# +# The harness (docker, jq baked onto the migrations base image, the explicit +# `return` in run_migration) is the one hack/migration-55-fluxcd-orphan.bats +# describes; see there for why each piece is load-bearing. +# +# Run with: hack/cozytest.sh hack/migration-61-strimzi-v1-conversion.bats +# ----------------------------------------------------------------------------- + +load test_helper + +FIXTURES="$PWD/hack/testdata/migration-61-strimzi-v1" +FAKE_FIXTURES="$FIXTURES" +MIG_DIR="$PWD/packages/core/platform/images/migrations/migrations" +ALPINE=$(sed -n 's/^FROM \(alpine:[^ ]*\).*$/\1/p' \ + "$PWD/packages/core/platform/images/migrations/Dockerfile" | head -1) +TESTIMG="cozystack-migration61-test:$(printf '%s' "$ALPINE" | sed 's/[^a-zA-Z0-9]/-/g')" +WORKROOT="${TMPDIR:-/tmp}/cozy-migration-61-$$" + +cozy_cleanup() { + rm -rf "$WORKROOT" + return 0 +} + +# The wait for the conversion Job counts only its sleeps, so a zero poll +# interval would let a wait that misses the end of the Job spin for good. A +# one-second poll under a three-second timeout turns that into a quick failure. +run_migration() { + _run_migration_rc=0 + docker run --rm --network none \ + --user "$(id -u):$(id -g)" \ + -v "$MIG_DIR:/migrations:ro" \ + -v "$FIXTURES:/fakebin:ro" \ + -v "$WORK:/work" \ + -e PATH=/fakebin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + -e FAKE_CMDLOG=/work/cmdlog \ + -e FAKE_STATE=/work/state \ + -e FAKE_APPS=/fakebin/apps.txt \ + -e FAKE_RELEASES=/fakebin/kafka-releases.txt \ + -e FAKE_KAFKAS="${FAKE_KAFKAS_OVERRIDE:-/fakebin/${FAKE_KAFKAS_FILE:-kafkas-kraft.json}}" \ + -e FAKE_KAFKA_LIST_FAIL="${FAKE_KAFKA_LIST_FAIL-}" \ + -e FAKE_CONVERSION_CHANGES_SPEC="${FAKE_CONVERSION_CHANGES_SPEC-}" \ + -e FAKE_NO_STRIMZI="${FAKE_NO_STRIMZI-}" \ + -e FAKE_JOB="${FAKE_JOB-}" \ + -e FAKE_PATCH_FAIL="${FAKE_PATCH_FAIL-}" \ + -e FAKE_MANAGED_CONFLICT="${FAKE_MANAGED_CONFLICT-}" \ + -e FAKE_REBALANCES="${FAKE_REBALANCES-}" \ + -e FAKE_CRD_READ_FAIL="${FAKE_CRD_READ_FAIL-}" \ + -e FAKE_APIRES_FAIL="${FAKE_APIRES_FAIL-}" \ + -e FAKE_APPS_ABSENT="${FAKE_APPS_ABSENT-}" \ + -e FAKE_DISCOVERY_PARTIAL="${FAKE_DISCOVERY_PARTIAL-}" \ + -e FAKE_SECRET_LIST_FAIL="${FAKE_SECRET_LIST_FAIL-}" \ + -e FAKE_RELEASE_REWRITTEN_BACK="${FAKE_RELEASE_REWRITTEN_BACK-}" \ + -e FAKE_RELEASE_PENDING="${FAKE_RELEASE_PENDING-}" \ + -e FAKE_OLD_JOB="${FAKE_OLD_JOB-}" \ + -e FAKE_REBALANCE_FAIL="${FAKE_REBALANCE_FAIL-}" \ + -e FAKE_OPERATOR_READ_FAIL="${FAKE_OPERATOR_READ_FAIL-}" \ + -e FAKE_SERVED="${FAKE_SERVED-}" \ + -e FAKE_MANAGED_PATCH_FAIL="${FAKE_MANAGED_PATCH_FAIL-}" \ + -e FAKE_PULL="${FAKE_PULL-}" \ + -e FAKE_HELMRELEASES="${FAKE_HELMRELEASES-}" \ + -e FAKE_KAFKATOPICS="${FAKE_KAFKATOPICS-}" \ + -e FAKE_STRIMZI_OBJECTS="${FAKE_STRIMZI_OBJECTS-}" \ + -e FAKE_STRIMZI_LIST_FAIL="${FAKE_STRIMZI_LIST_FAIL-}" \ + -e STRIMZI_CRD_FILE="${STRIMZI_CRD_FILE-/fakebin/strimzi-crds-0.51.0.yaml}" \ + -e POLL_INTERVAL=1 \ + -e CONVERSION_TIMEOUT="${CONVERSION_TIMEOUT-3}" \ + "${MIG_IMAGE:-$TESTIMG}" "/migrations/$1" || _run_migration_rc=$? + return "$_run_migration_rc" +} + +prep() { + docker info >/dev/null 2>&1 || { + echo "docker is required: these tests run migration 61 inside a jq-enabled" >&2 + echo "build of $ALPINE, the migrations image's base." >&2 + return 1 + } + docker build -q -t "$TESTIMG" - >/dev/null < "$WORK/state/stored" + FAKE_CMDLOG="$WORK/cmdlog" + : > "$FAKE_CMDLOG" + unset FAKE_CONVERSION_CHANGES_SPEC FAKE_KAFKAS_OVERRIDE FAKE_KAFKA_LIST_FAIL FAKE_NO_STRIMZI FAKE_JOB FAKE_PATCH_FAIL FAKE_KAFKAS_FILE FAKE_MANAGED_CONFLICT FAKE_MANAGED_PATCH_FAIL FAKE_REBALANCES STRIMZI_CRD_FILE FAKE_CRD_READ_FAIL FAKE_APIRES_FAIL FAKE_SECRET_LIST_FAIL FAKE_RELEASE_REWRITTEN_BACK FAKE_OLD_JOB FAKE_REBALANCE_FAIL FAKE_OPERATOR_READ_FAIL FAKE_SERVED FAKE_PULL FAKE_HELMRELEASES FAKE_KAFKATOPICS FAKE_STRIMZI_OBJECTS FAKE_STRIMZI_LIST_FAIL FAKE_APPS_ABSENT FAKE_DISCOVERY_PARTIAL CONVERSION_TIMEOUT || true + return 0 +} + +release_json() { + base64 -d < "$WORK/state/release-$1" | base64 -d | gzip -dc +} + +@test "converts to v1, moves Kafka 3.x applications to v4.3 and stamps 62" { + prep + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + + grep -qxF "APPLY-CRDS /fakebin/strimzi-crds-0.51.0.yaml" "$FAKE_CMDLOG" + # convert-resource is the init container, so crd-upgrade cannot start first. + grep -qxF "APPLY-CONVERSION convert-resource crd-upgrade" "$FAKE_CMDLOG" + crds_line=$(grep -nxF "APPLY-CRDS /fakebin/strimzi-crds-0.51.0.yaml" "$FAKE_CMDLOG" | cut -d: -f1) + job_line=$(grep -nxF "APPLY-CONVERSION convert-resource crd-upgrade" "$FAKE_CMDLOG" | cut -d: -f1) + [ "$crds_line" -lt "$job_line" ] + # The 0.45 operator is stopped before anything is converted. + scale_line=$(grep -nxF "SCALE-OPERATOR 0" "$FAKE_CMDLOG" | cut -d: -f1) + [ "$scale_line" -lt "$crds_line" ] + # The entity operators only after the cluster operator, or it scales them back. + entity_line=$(grep -nxF "SCALE-ENTITY tenant-a/kafka-one-entity-operator" "$FAKE_CMDLOG" | cut -d: -f1) + [ "$scale_line" -lt "$entity_line" ] + # A terminating topic or user operator still writes status through v1beta2. + wait_line=$(grep -nxF "WAIT-ENTITY tenant-a/strimzi.io/cluster=kafka-one,strimzi.io/name=kafka-one-entity-operator" "$FAKE_CMDLOG" | cut -d: -f1) + [ "$entity_line" -lt "$wait_line" ] + [ "$wait_line" -lt "$crds_line" ] + + grep -qxF 'PATCH tenant-a/one {"spec":{"version":"v4.3"}}' "$FAKE_CMDLOG" + grep -qxF 'PATCH tenant-b/two {"spec":{"version":"v4.3"}}' "$FAKE_CMDLOG" + if grep -qE "^PATCH tenant-(c|d)/" "$FAKE_CMDLOG"; then echo "patched an application that needs no version change" >&2; return 1; fi + + grep -qxF "DELETE clusterrole strimzi-v1-api-conversion" "$FAKE_CMDLOG" + [ "$(tail -1 "$FAKE_CMDLOG")" = "STAMP 62" ] + rm -rf "$WORK" +} + +# The README documents a manual run outside the image. On Ubuntu /bin/sh is a +# dash without pipefail. +@test "runs on Ubuntu, where /bin/sh is dash" { + prep + docker build -q -t cozystack-migration61-test:ubuntu - >/dev/null <<'DOCKERFILE' +FROM ubuntu:24.04@sha256:2198e4c65076e9cb0b3d695219df667c0e3b0072da64ca3c2648aca17dcecaf5 +RUN apt-get update && apt-get install -y --no-install-recommends jq && rm -rf /var/lib/apt/lists/* +DOCKERFILE + rc=0 + MIG_IMAGE=cozystack-migration61-test:ubuntu run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + [ "$(tail -1 "$FAKE_CMDLOG")" = "STAMP 62" ] +} + +# Helm builds the objects of the deployed release before it upgrades or +# uninstalls it, and fails on an API version the server no longer serves. +@test "rewrites the deployed Helm release of each Kafka to the v1 API" { + prep + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + + grep -qxF "PATCH-RELEASE tenant-a/sh.helm.release.v1.kafka-one.v3" "$FAKE_CMDLOG" + grep -qxF "PATCH-RELEASE tenant-a/sh.helm.release.v1.kafka-one.v4" "$FAKE_CMDLOG" + # An application whose install failed has a release and no Kafka CR. + grep -qxF "PATCH-RELEASE tenant-f/sh.helm.release.v1.kafka-failed-install.v1" "$FAKE_CMDLOG" + # The applications leave Kafka 3.x before any release is rewritten. + version_line=$(grep -nF 'PATCH tenant-a/one ' "$FAKE_CMDLOG" | cut -d: -f1) + first_release_line=$(grep -nF 'PATCH-RELEASE ' "$FAKE_CMDLOG" | head -1 | cut -d: -f1) + [ "$version_line" -lt "$first_release_line" ] + release_json v4 > "$WORK/release.json" + [ "$(jq -r '.manifest' "$WORK/release.json" | grep -c '^apiVersion: kafka.strimzi.io/v1$')" -eq 2 ] + if jq -r '.manifest' "$WORK/release.json" | grep -qF 'kafka.strimzi.io/v1beta2'; then echo "v1beta2 left in the release manifest" >&2; return 1; fi + # Only the manifest is rewritten; everything else in the release stays as it was. + jq 'del(.manifest)' "$FAKE_FIXTURES/release-kafka-one.json" > "$WORK/want" + jq 'del(.manifest)' "$WORK/release.json" > "$WORK/got" + cmp "$WORK/want" "$WORK/got" + release_line=$(grep -nxF "PATCH-RELEASE tenant-a/sh.helm.release.v1.kafka-one.v3" "$FAKE_CMDLOG" | cut -d: -f1) + stamp_line=$(grep -nxF "STAMP 62" "$FAKE_CMDLOG" | cut -d: -f1) + [ "$release_line" -lt "$stamp_line" ] + + : > "$FAKE_CMDLOG" + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + [ "$rc" -eq 0 ] + if grep -qF "PATCH-RELEASE" "$FAKE_CMDLOG"; then echo "a second run rewrote the release again" >&2; return 1; fi + rm -rf "$WORK" +} + +# Once the CRDs serve only v1, the API server refuses a server-side apply to +# an object whose managedFields still name v1beta2. +@test "moves managedFields entries recorded under old API versions to v1" { + prep + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + grep -qxF "PATCH-MANAGED tenant-a/kafka-one rv=100 helm-controller=kafka.strimzi.io/v1,strimzi-cluster-operator=kafka.strimzi.io/v1" "$FAKE_CMDLOG" + # The status entry keeps its subresource and its fields. + [ "$(jq -r '.items[0].metadata.managedFields[1].subresource' "$WORK/state/managed")" = "status" ] + if grep -qF "PATCH-MANAGED tenant-b/" "$FAKE_CMDLOG"; then echo "patched an object already on v1" >&2; return 1; fi + + : > "$FAKE_CMDLOG" + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + [ "$rc" -eq 0 ] + if grep -qF "PATCH-MANAGED" "$FAKE_CMDLOG"; then echo "a second run patched managedFields again" >&2; return 1; fi + rm -rf "$WORK" +} + +@test "a managedFields patch that loses a race is rebuilt from a fresh read" { + prep + export FAKE_MANAGED_CONFLICT=1 + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + grep -qxF "PATCH-MANAGED tenant-a/kafka-one rv=100 helm-controller=kafka.strimzi.io/v1,strimzi-cluster-operator=kafka.strimzi.io/v1" "$FAKE_CMDLOG" + grep -qxF "PATCH-MANAGED tenant-a/kafka-one rv=101 helm-controller=kafka.strimzi.io/v1,strimzi-cluster-operator=kafka.strimzi.io/v1" "$FAKE_CMDLOG" + [ "$(jq -r '.items[0].metadata.managedFields[0].apiVersion' "$WORK/state/managed")" = "kafka.strimzi.io/v1" ] + grep -qxF "STAMP 62" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +@test "a managedFields patch that fails for another reason aborts before stamping" { + prep + export FAKE_MANAGED_PATCH_FAIL=1 + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + [ "$(grep -c "PATCH-MANAGED" "$FAKE_CMDLOG")" -eq 1 ] + if grep -qF "STAMP" "$FAKE_CMDLOG"; then echo "stamped after a failed managedFields patch" >&2; return 1; fi + rm -rf "$WORK" +} + +@test "a cluster without Strimzi only stamps the version" { + prep + export FAKE_NO_STRIMZI=1 + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + if grep -qE "APPLY-CRDS|APPLY-CONVERSION|PATCH" "$FAKE_CMDLOG"; then echo "touched Strimzi on a cluster without it" >&2; return 1; fi + grep -qxF "STAMP 62" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +@test "a ZooKeeper-based or still-migrating Kafka stops the migration before it changes anything" { + prep + export FAKE_KAFKAS_FILE=kafkas-zk.json + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + grep -qxF " tenant-c/kafka-zk: ZooKeeper section in spec" "$WORK/out" + grep -qxF " tenant-d/kafka-mid: metadata state KRaftPostMigration" "$WORK/out" + if grep -qF "tenant-a/kafka-one" "$WORK/out"; then echo "named a KRaft cluster as ZooKeeper-based" >&2; return 1; fi + grep -qF "Cozystack v1.7" "$WORK/out" + if grep -qE "APPLY-CRDS|APPLY-CONVERSION|PATCH|STAMP|SCALE-OPERATOR" "$FAKE_CMDLOG"; then echo "changed the cluster despite a ZooKeeper Kafka" >&2; return 1; fi + rm -rf "$WORK" +} + +@test "a drain of the v1.7 ZooKeeper migration still in flight stops the migration" { + prep + export FAKE_REBALANCES="tenant-a/kafka-one-drain-kafka,tenant-a/my-own-rebalance" + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + grep -qxF " tenant-a/kafka-one-drain-kafka" "$WORK/out" + if grep -qxF " tenant-a/my-own-rebalance" "$WORK/out"; then echo "named an unrelated KafkaRebalance" >&2; return 1; fi + if grep -qE "SCALE-OPERATOR|APPLY-CRDS|STAMP" "$FAKE_CMDLOG"; then echo "changed the cluster despite an unfinished drain" >&2; return 1; fi + rm -rf "$WORK" +} + +# Run by hand outside the migrations image, the CRD file is easy to miss; +# the operators must not be stopped for a run that cannot go on. +@test "a missing CRD file aborts before the operators are stopped" { + prep + export STRIMZI_CRD_FILE=/nonexistent/strimzi-crds-0.51.0.yaml + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + grep -qF "/nonexistent/strimzi-crds-0.51.0.yaml is not readable" "$WORK/out" + if grep -qE "SCALE-|APPLY-CRDS|STAMP" "$FAKE_CMDLOG"; then echo "changed the cluster without the CRD file" >&2; return 1; fi + rm -rf "$WORK" +} + +@test "a failed conversion Job aborts with its logs, before any application changes" { + prep + export FAKE_JOB=fail + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + grep -qF "fake conversion log" "$WORK/out" + # The failure ends the wait at once, not at the timeout. + [ "$(grep -cF "get job strimzi-v1-api-conversion -o jsonpath={.status.succeeded}" "$FAKE_CMDLOG")" -eq 1 ] + if grep -qE "PATCH|STAMP" "$FAKE_CMDLOG"; then echo "went on after the conversion failed" >&2; return 1; fi + # The failed Job stays for diagnosis; the rights it ran with do not. + grep -qxF "DELETE clusterrolebinding strimzi-v1-api-conversion" "$FAKE_CMDLOG" + grep -qxF "DELETE clusterrole strimzi-v1-api-conversion" "$FAKE_CMDLOG" + grep -qxF "DELETE serviceaccount strimzi-v1-api-conversion" "$FAKE_CMDLOG" + [ "$(sed -n '/^APPLY-CONVERSION/,$p' "$FAKE_CMDLOG" | grep -cF "DELETE job strimzi-v1-api-conversion")" -eq 0 ] + rm -rf "$WORK" +} + +@test "a Job that leaves a non-v1 stored version behind aborts" { + prep + export FAKE_JOB=noop + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + grep -qF 'kafkas.kafka.strimzi.io: ["v1beta2"]' "$WORK/out" + if grep -qE "PATCH|STAMP" "$FAKE_CMDLOG"; then echo "went on with v1beta2 still stored" >&2; return 1; fi + grep -qxF "DELETE clusterrolebinding strimzi-v1-api-conversion" "$FAKE_CMDLOG" + grep -qxF "DELETE clusterrole strimzi-v1-api-conversion" "$FAKE_CMDLOG" + grep -qxF "DELETE serviceaccount strimzi-v1-api-conversion" "$FAKE_CMDLOG" + [ "$(sed -n '/^APPLY-CONVERSION/,$p' "$FAKE_CMDLOG" | grep -cF "DELETE job strimzi-v1-api-conversion")" -eq 0 ] + rm -rf "$WORK" +} + +@test "a re-run after the conversion skips it and still moves the versions" { + prep + printf '["v1"]' > "$WORK/state/stored" + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + if grep -qE "APPLY-CRDS|APPLY-CONVERSION" "$FAKE_CMDLOG"; then echo "converted again" >&2; return 1; fi + # The CRDs still serve v1beta2, so the operator may be 0.45: it is stopped. + grep -qxF "SCALE-OPERATOR 0" "$FAKE_CMDLOG" + # Nothing slow follows on this path, so the managedFields rewrite has to + # wait for the entity operators to be gone. + wait_line=$(grep -nF "WAIT-ENTITY tenant-a/" "$FAKE_CMDLOG" | cut -d: -f1) + managed_line=$(grep -nF "PATCH-MANAGED " "$FAKE_CMDLOG" | head -1 | cut -d: -f1) + [ "$wait_line" -lt "$managed_line" ] + grep -qxF 'PATCH tenant-a/one {"spec":{"version":"v4.3"}}' "$FAKE_CMDLOG" + grep -qxF "STAMP 62" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +@test "a failing application patch aborts before stamping" { + prep + export FAKE_PATCH_FAIL=1 + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + if grep -qF "STAMP" "$FAKE_CMDLOG"; then echo "stamped after a failed patch" >&2; return 1; fi + rm -rf "$WORK" +} + +# The migration runs once: a read that fails must stop it before the stamp, +# never pass for "nothing there". +@test "a failed read of the CRD stored versions aborts before stamping" { + prep + export FAKE_CRD_READ_FAIL=1 + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + if grep -qE "STAMP|SCALE-|APPLY-CRDS|PATCH" "$FAKE_CMDLOG"; then echo "went on after a failed CRD read" >&2; return 1; fi + rm -rf "$WORK" +} + +@test "a failed read of the apps.cozystack.io API aborts before stamping" { + prep + export FAKE_APIRES_FAIL=1 + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + grep -qF "reading the apps.cozystack.io API failed" "$WORK/out" + if grep -qE "STAMP|PATCH-RELEASE" "$FAKE_CMDLOG"; then echo "went on after a failed apps.cozystack.io read" >&2; return 1; fi + rm -rf "$WORK" +} + +@test "another API group failing discovery does not stop the migration" { + prep + export FAKE_DISCOVERY_PARTIAL=1 + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + grep -qxF 'PATCH tenant-a/one {"spec":{"version":"v4.3"}}' "$FAKE_CMDLOG" + grep -qxF "STAMP 62" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +@test "a cluster without the apps.cozystack.io API skips the applications and stamps" { + prep + export FAKE_APPS_ABSENT=1 + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + if grep -qF "kafkas.apps.cozystack.io" "$FAKE_CMDLOG"; then echo "read Kafka applications without their API" >&2; return 1; fi + grep -qxF "STAMP 62" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +@test "a failed Helm release listing aborts before stamping" { + prep + export FAKE_SECRET_LIST_FAIL=1 + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + if grep -qF "STAMP" "$FAKE_CMDLOG"; then echo "stamped after a failed release listing" >&2; return 1; fi + rm -rf "$WORK" +} + +@test "a release that names v1beta2 again after the rewrite stops the stamp" { + prep + export FAKE_RELEASE_REWRITTEN_BACK=1 + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + grep -qxF " Helm release tenant-a/sh.helm.release.v1.kafka-one.v3" "$WORK/out" + if grep -qF "STAMP" "$FAKE_CMDLOG"; then echo "stamped with a v1beta2 release left" >&2; return 1; fi + rm -rf "$WORK" +} + +# A Helm action that started before the rewrite stores the releases it read +# when it finishes, so a revision still pending blocks the stamp as well. +@test "a Helm action still running on a Kafka release stops the stamp" { + prep + export FAKE_RELEASE_PENDING=1 + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + grep -qxF " Helm release tenant-a/sh.helm.release.v1.kafka-one.v4 is pending-upgrade" "$WORK/out" + if grep -qF "STAMP" "$FAKE_CMDLOG"; then echo "stamped while a Helm action was running" >&2; return 1; fi + rm -rf "$WORK" +} + +# A Job failed on its deadline can still have a pod terminating, and +# convert-resource writes until that pod is gone. +@test "a finished Job of an earlier attempt is removed with its pods before the CRDs are touched" { + for state in failed succeeded; do + prep + export FAKE_OLD_JOB=$state + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + delete_line=$(grep -nxF "DELETE job strimzi-v1-api-conversion foreground" "$FAKE_CMDLOG" | head -1 | cut -d: -f1) + scale_line=$(grep -nxF "SCALE-OPERATOR 0" "$FAKE_CMDLOG" | cut -d: -f1) + crds_line=$(grep -nF "APPLY-CRDS" "$FAKE_CMDLOG" | cut -d: -f1) + job_line=$(grep -nF "APPLY-CONVERSION" "$FAKE_CMDLOG" | cut -d: -f1) + [ "$delete_line" -lt "$scale_line" ] + [ "$delete_line" -lt "$crds_line" ] + [ "$crds_line" -lt "$job_line" ] + rm -rf "$WORK" + done +} + +@test "a Job of an earlier attempt that fails while waited for is removed with its pods and started over" { + prep + export FAKE_OLD_JOB=running FAKE_JOB=fail + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + # The new Job fails as well, so the run stops after starting it. + [ "$rc" -ne 0 ] + # Each failure ends its wait at once, not at the timeout. + [ "$(grep -cF "get job strimzi-v1-api-conversion -o jsonpath={.status.succeeded}" "$FAKE_CMDLOG")" -eq 2 ] + delete_line=$(grep -nxF "DELETE job strimzi-v1-api-conversion foreground" "$FAKE_CMDLOG" | head -1 | cut -d: -f1) + crds_line=$(grep -nF "APPLY-CRDS" "$FAKE_CMDLOG" | cut -d: -f1) + [ "$delete_line" -lt "$crds_line" ] + grep -qF "APPLY-CONVERSION" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +# Without a deadline, a Job whose pod never starts never fails, and every +# retry waits on it and exits. The deadline must cover the wait of the run +# that started the Job plus the wait of the retry after it. +@test "the conversion Job carries a deadline of at least two waits for it" { + prep + export CONVERSION_TIMEOUT=100 + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + deadline=$(sed -n 's/^CONVERSION-DEADLINE //p' "$FAKE_CMDLOG") + [ -n "$deadline" ] + [ "$deadline" -ge 200 ] + rm -rf "$WORK" +} + +# A pod the API server refused never fails, so the Job reports its deadline +# only in its Failed condition. +@test "a Job of an earlier attempt failed on its deadline without a pod is started over" { + prep + export FAKE_OLD_JOB=running FAKE_JOB=nopod CONVERSION_TIMEOUT=0 + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + if grep -qF "is still running; retry once it finishes" "$WORK/out"; then echo "took a failed Job for a running one" >&2; return 1; fi + delete_line=$(grep -nxF "DELETE job strimzi-v1-api-conversion foreground" "$FAKE_CMDLOG" | head -1 | cut -d: -f1) + crds_line=$(grep -nF "APPLY-CRDS" "$FAKE_CMDLOG" | cut -d: -f1) + [ "$delete_line" -lt "$crds_line" ] + grep -qF "APPLY-CONVERSION" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +@test "a conversion pod that cannot pull its image is named, with what to do" { + prep + export FAKE_JOB=hang FAKE_PULL=1 CONVERSION_TIMEOUT=0 + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + grep -qF "cannot pull quay.io/strimzi/operator:0.51.0@sha256:" "$WORK/out" + grep -qF "mirror" "$WORK/out" + if grep -qE "PATCH|STAMP" "$FAKE_CMDLOG"; then echo "went on without a conversion" >&2; return 1; fi + rm -rf "$WORK" +} + +@test "a conversion Job still running when the wait times out keeps its rights for the retry" { + prep + export FAKE_JOB=hang CONVERSION_TIMEOUT=0 + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + if sed -n '/^APPLY-CONVERSION/,$p' "$FAKE_CMDLOG" | grep -qF "DELETE"; then echo "removed what the running Job needs" >&2; return 1; fi + rm -rf "$WORK" +} + +# A retry after the hook timed out must not touch the CRDs under a Job that +# is still converting. +@test "a Job of an earlier attempt still running is waited for before the CRDs are touched" { + prep + export FAKE_OLD_JOB=running + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + grep -qF "still running" "$WORK/out" + # It converted everything, so there is nothing left to apply or start. + if grep -qE "APPLY-CRDS|APPLY-CONVERSION" "$FAKE_CMDLOG"; then echo "touched the CRDs under a running Job" >&2; return 1; fi + grep -qxF "DELETE clusterrole strimzi-v1-api-conversion" "$FAKE_CMDLOG" + grep -qxF "STAMP 62" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +@test "a Job of an earlier attempt still running after the wait aborts before the CRDs are touched" { + prep + export FAKE_OLD_JOB=running FAKE_JOB=hang CONVERSION_TIMEOUT=0 + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + grep -qF "is still running; retry once it finishes" "$WORK/out" + if grep -qE "SCALE-|APPLY-CRDS|DELETE|STAMP" "$FAKE_CMDLOG"; then echo "went on under a running Job" >&2; return 1; fi + rm -rf "$WORK" +} + +@test "a failed read of the KafkaRebalances aborts before the operators are stopped" { + prep + export FAKE_REBALANCE_FAIL=1 + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + if grep -qE "SCALE-|APPLY-CRDS|STAMP" "$FAKE_CMDLOG"; then echo "went on after a failed KafkaRebalance read" >&2; return 1; fi + rm -rf "$WORK" +} + +# Converting under a 0.45 operator the migration failed to see would let it +# record v1beta2 managedFields again after the rewrite. +@test "a failed read of the cluster operator Deployment aborts before anything is converted" { + prep + export FAKE_OPERATOR_READ_FAIL=1 + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + if grep -qE "APPLY-CRDS|APPLY-CONVERSION|STAMP" "$FAKE_CMDLOG"; then echo "converted without knowing whether the 0.45 operator runs" >&2; return 1; fi + rm -rf "$WORK" +} + +# Run by hand after the kafka-operator release brought Strimzi 1.x: nothing +# would scale its operator back up. +@test "a re-run once the CRDs serve only v1 leaves the operators running" { + prep + printf '["v1"]' > "$WORK/state/stored" + export FAKE_SERVED=v1 + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + if grep -qF "SCALE-" "$FAKE_CMDLOG"; then echo "stopped the 1.x operator" >&2; return 1; fi + grep -qxF "STAMP 62" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +@test "the KRaft gate rejects missing and transitional state on both storage paths" { + for stored in '["v1beta2"]' '["v1"]'; do + for state in '""' '"PreKRaft"' '"KRaftMigration"' '"KRaftDualWriting"' '"KRaftPostMigration"' '"ZooKeeper"' false; do + prep + printf '%s' "$stored" > "$WORK/state/stored" + jq --argjson state "$state" '.items[0].status.kafkaMetadataState = $state | .items[1].status = {}' "$FIXTURES/kafkas-kraft.json" > "$WORK/kafkas.json" + export FAKE_KAFKAS_OVERRIDE=/work/kafkas.json + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + grep -qF "tenant-a/kafka-one" "$WORK/out" + grep -qF "tenant-b/kafka-two" "$WORK/out" + if grep -qE "SCALE-|APPLY-|PATCH|STAMP|DELETE" "$FAKE_CMDLOG"; then echo "mutated before proving KRaft" >&2; return 1; fi + done + done +} + +@test "a healthy modern Kafka without legacy state passes on both served APIs" { + for served in 'v1 v1beta2' v1; do + prep + export FAKE_SERVED="$served" + [ "$served" != v1 ] || printf '["v1"]' > "$WORK/state/stored" + broker_version=4.1.1 + [ "$served" != v1 ] || broker_version=4.3.1 + jq --arg broker "$broker_version" '.items |= map(.metadata.generation = 7 | .spec.kafka.version = $broker | .status = {kafkaVersion: $broker, kafkaMetadataVersion: "4.1-IV1", observedGeneration: 7, conditions: [{type: "Ready", status: "True"}]})' "$FIXTURES/kafkas-kraft.json" > "$WORK/kafkas.json" + export FAKE_KAFKAS_OVERRIDE=/work/kafkas.json + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + grep -qxF "STAMP 62" "$FAKE_CMDLOG" + if [ "$served" = v1 ]; then + if grep -qE "SCALE-|APPLY-CRDS|v1beta2.kafka.strimzi.io" "$FAKE_CMDLOG"; then echo "touched legacy API on a v1-only rerun" >&2; return 1; fi + fi + done +} + +@test "a NotReady or malformed modern status cannot prove KRaft" { + for change in 'del(.status)' '.status = null' '.status.kafkaMetadataVersion = ""' '.status.kafkaMetadataVersion = false' '.status.kafkaMetadataVersion = 41' '.status.kafkaMetadataVersion = []' '.status.conditions = [{type:"NotReady",status:"True"}]' '.status.conditions = [{type:"Ready",status:"False"}]' '.status.kafkaMetadataState = "KRaftPostMigration"' '.spec.zookeeper = {replicas:3}'; do + prep + jq '.items |= map(.metadata.generation = 7 | .status = {kafkaMetadataVersion:"4.1-IV1",observedGeneration:7,conditions:[{type:"Ready",status:"True"}]})' "$FIXTURES/kafkas-kraft.json" | jq ".items[0] |= ($change)" > "$WORK/kafkas.json" + export FAKE_KAFKAS_OVERRIDE=/work/kafkas.json + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + grep -qF "tenant-a/kafka-one" "$WORK/out" + if grep -qE "SCALE-|APPLY-|PATCH|STAMP|DELETE" "$FAKE_CMDLOG"; then echo "accepted unproven modern status" >&2; return 1; fi + done +} + +@test "the KRaft gate reports every failing Kafka and its Helm release" { + prep + jq '.items |= map(.status = {} | .metadata.annotations["meta.helm.sh/release-name"] = ("release-" + .metadata.name))' "$FIXTURES/kafkas-kraft.json" > "$WORK/kafkas.json" + export FAKE_KAFKAS_OVERRIDE=/work/kafkas.json + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + grep -qF "tenant-a/kafka-one" "$WORK/out" + grep -qF "tenant-a/release-kafka-one" "$WORK/out" + grep -qF "tenant-b/kafka-two" "$WORK/out" + grep -qF "tenant-b/release-kafka-two" "$WORK/out" +} + +@test "a failed or malformed Kafka list fails closed before mutation" { + for response in failed empty invalid null '{}' '{"items":null}' '{"items":{}}' '{"items":[null]}' '{"items":[{}]}' '{"items":[]} {"items":[]}'; do + prep + export FAKE_KAFKAS_OVERRIDE=/work/kafkas.json + case "$response" in + failed) export FAKE_KAFKA_LIST_FAIL=1; printf '{"items":[]}' > "$WORK/kafkas.json" ;; + empty) : > "$WORK/kafkas.json" ;; + *) printf '%s' "$response" > "$WORK/kafkas.json" ;; + esac + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + if grep -qE "SCALE-|APPLY-|PATCH|STAMP|DELETE" "$FAKE_CMDLOG"; then echo "accepted a failed or malformed list" >&2; return 1; fi + done +} + +@test "an empty Kafka list passes the KRaft gate" { + prep + printf '{"items":[]}' > "$WORK/kafkas.json" + export FAKE_KAFKAS_OVERRIDE=/work/kafkas.json + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + grep -qxF "STAMP 62" "$FAKE_CMDLOG" +} + +@test "a previous conversion Job is fenced before the KRaft gate rejects an unproven Kafka" { + for state in running failed succeeded; do + prep + export FAKE_OLD_JOB="$state" + jq '.items[0].status = {}' "$FIXTURES/kafkas-kraft.json" > "$WORK/kafkas.json" + export FAKE_KAFKAS_OVERRIDE=/work/kafkas.json + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + grep -qF "tenant-a/kafka-one" "$WORK/out" + if [ "$state" = running ]; then + wait_line=$(grep -nF 'get job strimzi-v1-api-conversion -o jsonpath={.status.succeeded}' "$FAKE_CMDLOG" | head -1 | cut -d: -f1) + gate_line=$(grep -nF 'get kafkas.v1beta2.kafka.strimzi.io' "$FAKE_CMDLOG" | head -1 | cut -d: -f1) + [ "$wait_line" -lt "$gate_line" ] + fi + if grep -qE "SCALE-|APPLY-|PATCH|STAMP|DELETE" "$FAKE_CMDLOG"; then echo "mutated after an unproven Kafka" >&2; return 1; fi + done +} + +@test "an unfinished drain blocks an already-v1 rerun" { + prep + printf '["v1"]' > "$WORK/state/stored" + export FAKE_SERVED=v1 FAKE_REBALANCES=tenant-a/kafka-one-drain-kafka + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + grep -qF "tenant-a/kafka-one-drain-kafka" "$WORK/out" + if grep -qE "SCALE-|APPLY-|PATCH|STAMP|DELETE" "$FAKE_CMDLOG"; then echo "mutated despite a pending drain" >&2; return 1; fi +} + +@test "a completed conversion with a newer spec generation does not block recovery of a proven KRaft cluster" { + prep + jq '.items |= map( + .metadata.generation = 7 + | .spec.kafka.version = "4.1.1" + | .spec.kafka.listeners = [{name:"plain",port:9092,type:"internal",tls:false}] + | .spec.kafka.replicas = 3 + | .spec.entityOperator.topicOperator.reconciliationIntervalSeconds = 90 + | .status = {kafkaMetadataVersion:"4.1-IV1",observedGeneration:7,conditions:[{type:"Ready",status:"True"}]})' "$FIXTURES/kafkas-kraft.json" > "$WORK/kafkas.json" + export FAKE_KAFKAS_OVERRIDE=/work/kafkas.json FAKE_OLD_JOB=running FAKE_CONVERSION_CHANGES_SPEC=1 + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + jq --exit-status 'all(.items[]; .metadata.generation == 8 and .status.observedGeneration == 7 and .spec.kafka.replicas == null and .spec.entityOperator.topicOperator.reconciliationIntervalSeconds == null and .spec.entityOperator.topicOperator.reconciliationIntervalMs == 90000)' "$WORK/state/kafkas-converted.json" + jq '.items | map(.status)' "$WORK/kafkas.json" > "$WORK/status-before.json" + jq '.items | map(.status)' "$WORK/state/kafkas-converted.json" > "$WORK/status-after.json" + cmp "$WORK/status-before.json" "$WORK/status-after.json" + [ "$rc" -eq 0 ] + grep -qxF "STAMP 62" "$FAKE_CMDLOG" + if grep -qE "APPLY-CRDS|APPLY-CONVERSION" "$FAKE_CMDLOG"; then echo "restarted an already completed conversion" >&2; return 1; fi +} + +# The converter deletes status.kafkaMetadataState, so anything the gate +# accepts before the conversion must still pass on a retry after it. +@test "an explicit KRaft state without a metadata version does not pass the gate" { + prep + jq '.items |= map(.status = {kafkaMetadataState:"KRaft",kafkaMetadataVersion:"3.9-IV0",conditions:[{type:"Ready",status:"True"}]}) | .items[0] |= del(.status.kafkaMetadataVersion)' "$FIXTURES/kafkas-kraft.json" > "$WORK/kafkas.json" + export FAKE_KAFKAS_OVERRIDE=/work/kafkas.json + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + grep -qxF " tenant-a/kafka-one: no kafkaMetadataVersion" "$WORK/out" + if grep -qE "SCALE-|APPLY-|PATCH|STAMP|DELETE|ANNOTATE" "$FAKE_CMDLOG"; then echo "changed the cluster for an unproven Kafka" >&2; return 1; fi +} + +# KRaft is the final Strimzi metadata state, so it needs no Ready. The +# converter deletes the state, so the proof is recorded on the Kafka before the +# operators stop, and a retry after the conversion accepts it from there. +@test "a NotReady Kafka in the KRaft state passes, before and after the converter drops its state" { + prep + jq '.items |= map(.status = {kafkaMetadataState:"KRaft",kafkaMetadataVersion:"3.9-IV0",conditions:[{type:"Ready",status:"True"}]}) | .items[0].status.conditions = [{type:"NotReady",status:"True"}]' "$FIXTURES/kafkas-kraft.json" > "$WORK/kafkas.json" + export FAKE_KAFKAS_OVERRIDE=/work/kafkas.json FAKE_CONVERSION_CHANGES_SPEC=1 FAKE_PATCH_FAIL=1 + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + annotate_line=$(grep -nxF "ANNOTATE tenant-a/kafka-one cozystack.io/kraft-verified=3.9-IV0" "$FAKE_CMDLOG" | cut -d: -f1) + grep -qxF "ANNOTATE tenant-b/kafka-two cozystack.io/kraft-verified=3.9-IV0" "$FAKE_CMDLOG" + scale_line=$(grep -nxF "SCALE-OPERATOR 0" "$FAKE_CMDLOG" | cut -d: -f1) + [ "$annotate_line" -lt "$scale_line" ] + jq --exit-status '.items[0] | .status.kafkaMetadataState == null and .metadata.annotations["cozystack.io/kraft-verified"] == "3.9-IV0"' "$WORK/state/kafkas-converted.json" + + unset FAKE_PATCH_FAIL + : > "$FAKE_CMDLOG" + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + grep -qxF "STAMP 62" "$FAKE_CMDLOG" + if grep -qF "ANNOTATE" "$FAKE_CMDLOG"; then echo "annotated an already recorded Kafka again" >&2; return 1; fi +} + +@test "a Kafka accepted before the conversion is still accepted after the converter drops its state" { + prep + jq '.items |= map(.status = {kafkaMetadataState:"KRaft",kafkaMetadataVersion:"3.9-IV0",conditions:[{type:"Ready",status:"True"}]})' "$FIXTURES/kafkas-kraft.json" > "$WORK/kafkas.json" + export FAKE_KAFKAS_OVERRIDE=/work/kafkas.json FAKE_OLD_JOB=running FAKE_CONVERSION_CHANGES_SPEC=1 + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + jq --exit-status 'all(.items[]; .status.kafkaMetadataState == null)' "$WORK/state/kafkas-converted.json" + [ "$rc" -eq 0 ] + grep -qxF "STAMP 62" "$FAKE_CMDLOG" +} + +@test "a Kafka application topic with a config Kafka 4 removed stops the migration before it changes anything" { + prep + cat > "$WORK/helmreleases.json" <<'JSON' +{"items":[ + {"metadata":{"namespace":"tenant-a","name":"kafka-one","labels":{"apps.cozystack.io/application.kind":"Kafka","apps.cozystack.io/application.name":"one"}}, + "spec":{"values":{"topics":[ + {"name":"legacy","config":{"message.format.version":"3.0","message.timestamp.difference.max.ms":"1000","retention.ms":"1"}}, + {"name":"clean","config":{"retention.ms":"1"}}, + {"name":"bare"}]}}}, + {"metadata":{"namespace":"tenant-b","name":"kafka-two","labels":{"apps.cozystack.io/application.kind":"Kafka"}}, + "spec":{"values":{"topics":[{"name":"old","config":{"message.downconversion.enable":true}}]}}}, + {"metadata":{"namespace":"tenant-c","name":"kafka-three","labels":{"apps.cozystack.io/application.kind":"Kafka"}}, + "spec":{}} +]} +JSON + export FAKE_HELMRELEASES=/work/helmreleases.json + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + grep -qF -- "-l apps.cozystack.io/application.kind=Kafka" "$FAKE_CMDLOG" + grep -qxF " tenant-a/one: topic legacy sets message.format.version" "$WORK/out" + grep -qxF " tenant-a/one: topic legacy sets message.timestamp.difference.max.ms" "$WORK/out" + grep -qxF " tenant-b/kafka-two: topic old sets message.downconversion.enable" "$WORK/out" + if grep -qE "clean|bare|retention" "$WORK/out"; then echo "named a topic without a removed config" >&2; return 1; fi + grep -qF "before running this migration" "$WORK/out" + if grep -qE "SCALE-|APPLY-|PATCH|STAMP|DELETE|ANNOTATE" "$FAKE_CMDLOG"; then echo "changed the cluster despite a removed topic config" >&2; return 1; fi + rm -rf "$WORK" +} + +@test "a topic below the Kafka 4 segment.bytes floor stops the migration before it changes anything" { + prep + cat > "$WORK/helmreleases.json" <<'JSON' +{"items":[ + {"metadata":{"namespace":"tenant-a","name":"kafka-one","labels":{"apps.cozystack.io/application.kind":"Kafka","apps.cozystack.io/application.name":"one"}}, + "spec":{"values":{"topics":[ + {"name":"small","config":{"segment.bytes":524288}}, + {"name":"padded","config":{"segment.bytes":" 524288"}}, + {"name":"plus","config":{"segment.bytes":"+524288"}}, + {"name":"zeros","config":{"segment.bytes":"02000000"}}, + {"name":"exp","config":{"segment.bytes":"1e3"}}, + {"name":"floor","config":{"segment.bytes":"1048576"}}]}}} +]} +JSON + cat > "$WORK/kafkatopics.json" <<'JSON' +{"items":[ + {"metadata":{"namespace":"tenant-b","name":"by-hand"},"spec":{"topicName":"raw.events","config":{"segment.bytes":"65536"}}}, + {"metadata":{"namespace":"tenant-b","name":"named-by-cr"},"spec":{"config":{"segment.bytes":100}}}, + {"metadata":{"namespace":"tenant-b","name":"fine"},"spec":{"config":{"segment.bytes":1073741824}}}, + {"metadata":{"namespace":"tenant-b","name":"bare"},"spec":{}}, + {"metadata":{"namespace":"tenant-b","name":"padded-cr"},"spec":{"config":{"segment.bytes":"\t+100 "}}}, + {"metadata":{"namespace":"tenant-b","name":"negative-cr"},"spec":{"config":{"segment.bytes":"-2000000"}}} +]} +JSON + export FAKE_HELMRELEASES=/work/helmreleases.json FAKE_KAFKATOPICS=/work/kafkatopics.json + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + grep -qxF " tenant-a/one: topic small sets segment.bytes=524288, below the Kafka 4 minimum of 1048576" "$WORK/out" + grep -qxF " tenant-b/kafkatopic by-hand: topic raw.events sets segment.bytes=65536, below the Kafka 4 minimum of 1048576" "$WORK/out" + grep -qxF " tenant-b/kafkatopic named-by-cr: topic named-by-cr sets segment.bytes=100, below the Kafka 4 minimum of 1048576" "$WORK/out" + # Kafka trims the value and reads it with Integer.parseInt, which takes a + # sign and reads leading zeros as decimal. + grep -qxF " tenant-a/one: topic padded sets segment.bytes=524288, below the Kafka 4 minimum of 1048576" "$WORK/out" + grep -qxF " tenant-a/one: topic plus sets segment.bytes=524288, below the Kafka 4 minimum of 1048576" "$WORK/out" + grep -qxF " tenant-b/kafkatopic padded-cr: topic padded-cr sets segment.bytes=100, below the Kafka 4 minimum of 1048576" "$WORK/out" + grep -qxF " tenant-b/kafkatopic negative-cr: topic negative-cr sets segment.bytes=-2000000, below the Kafka 4 minimum of 1048576" "$WORK/out" + if grep -qE "floor|fine|bare|zeros|exp" "$WORK/out"; then echo "named a topic at or above the floor" >&2; return 1; fi + if grep -qE "SCALE-|APPLY-|PATCH|STAMP|DELETE|ANNOTATE" "$FAKE_CMDLOG"; then echo "changed the cluster despite a topic below the floor" >&2; return 1; fi + rm -rf "$WORK" +} + +@test "a failed read of the KafkaTopics aborts before the operators are stopped" { + prep + export FAKE_KAFKATOPICS=/work/missing.json + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + grep -qF "get kafkatopics.v1beta2.kafka.strimzi.io" "$FAKE_CMDLOG" + if grep -qE "SCALE-|APPLY-|PATCH|STAMP|DELETE|ANNOTATE" "$FAKE_CMDLOG"; then echo "went on after a failed KafkaTopic read" >&2; return 1; fi + rm -rf "$WORK" +} + +@test "a failed read of the Kafka application HelmReleases aborts before the operators are stopped" { + prep + export FAKE_HELMRELEASES=/work/missing.json + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + grep -qF "helmreleases.helm.toolkit.fluxcd.io" "$FAKE_CMDLOG" + if grep -qE "SCALE-|APPLY-|PATCH|STAMP|DELETE|ANNOTATE" "$FAKE_CMDLOG"; then echo "went on after a failed HelmRelease read" >&2; return 1; fi + rm -rf "$WORK" +} + +# Strimzi's conversion tool refuses these objects, and convert-resource stops +# at the first one, after the operators are stopped. +refused_objects() { + jq '.items[0].spec.kafka += {resources: {requests: {cpu: "1"}}, authorization: {type: "keycloak"}, listeners: [{name: "oauth", authentication: {type: "oauth"}}, {name: "custom", authentication: {type: "custom", secrets: [{secretName: "s", key: "k"}]}}]} + | .items[1].spec.kafka += {authorization: {type: "opa"}, listeners: [{name: "custom", authentication: {type: "custom", sasl: true}}, {name: "tls", authentication: {type: "tls"}}]} + | .items += [.items[1] | .metadata = {namespace: "tenant-c", name: "kafka-clean"} | .spec.kafka.authorization = {type: "simple"}]' \ + "$FIXTURES/kafkas-kraft.json" > "$WORK/kafkas.json" + mkdir -p "$WORK/objects" + cat > "$WORK/objects/kafkaconnects.json" <<'JSON' +{"items":[ + {"metadata":{"namespace":"tenant-a","name":"connect-bad"},"spec":{"authentication":{"type":"oauth"},"externalConfiguration":{"env":[]},"build":{"output":{"additionalKanikoOptions":["--a"],"additionalBuildOptions":["--b"]}}}}, + {"metadata":{"namespace":"tenant-a","name":"connect-clean"},"spec":{"authentication":{"type":"tls"},"build":{"output":{"additionalKanikoOptions":["--a"]}}}}, + {"metadata":{"namespace":"tenant-a","name":"connect-build-clean"},"spec":{"build":{"output":{"additionalBuildOptions":["--b"]}}}} +]} +JSON + cat > "$WORK/objects/kafkabridges.json" <<'JSON' +{"items":[ + {"metadata":{"namespace":"tenant-a","name":"bridge-bad"},"spec":{"authentication":{"type":"oauth"}}}, + {"metadata":{"namespace":"tenant-a","name":"bridge-clean"},"spec":{"authentication":{"type":"scram-sha-512"}}} +]} +JSON + cat > "$WORK/objects/kafkamirrormaker2s.json" <<'JSON' +{"items":[ + {"metadata":{"namespace":"tenant-a","name":"mm2-old-bad"},"spec":{"externalConfiguration":{"env":[]},"connectCluster":"missing","clusters":[{"alias":"src","authentication":{"type":"oauth"}},{"alias":"dst"}],"mirrors":[{"sourceCluster":"nope","targetCluster":"other"}]}}, + {"metadata":{"namespace":"tenant-a","name":"mm2-no-clusters"},"spec":{"mirrors":[{}]}}, + {"metadata":{"namespace":"tenant-a","name":"mm2-new-oauth"},"spec":{"target":{"alias":"dst","authentication":{"type":"oauth"}},"mirrors":[{"source":{"alias":"src","authentication":{"type":"oauth"}}}]}}, + {"metadata":{"namespace":"tenant-a","name":"mm2-old-clean"},"spec":{"connectCluster":"dst","clusters":[{"alias":"src"},{"alias":"dst"}],"mirrors":[{"sourceCluster":"src","targetCluster":"dst"}]}}, + {"metadata":{"namespace":"tenant-a","name":"mm2-new-clean"},"spec":{"target":{"alias":"dst"},"mirrors":[{"source":{"alias":"src"},"targetCluster":"dst"}]}} +]} +JSON + cat > "$WORK/objects/kafkausers.json" <<'JSON' +{"items":[ + {"metadata":{"namespace":"tenant-a","name":"user-no-spec"}}, + {"metadata":{"namespace":"tenant-a","name":"user-clean"},"spec":{"authentication":{"type":"tls"}}} +]} +JSON + cat > "$WORK/objects/kafkanodepools.json" <<'JSON' +{"items":[{"metadata":{"namespace":"tenant-a","name":"pool-null-spec"},"spec":null}]} +JSON + export FAKE_KAFKAS_OVERRIDE=/work/kafkas.json FAKE_STRIMZI_OBJECTS=/work/objects +} + +@test "an object the Strimzi v1 conversion refuses stops the migration before the operators are stopped" { + prep + refused_objects + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + while IFS= read -r want; do + grep -qxF " $want" "$WORK/out" || { echo "not reported: $want" >&2; return 1; } + done <<'LINES' +kafkas.kafka.strimzi.io tenant-a/kafka-one: .spec.kafka.resources (use a KafkaNodePool) +kafkas.kafka.strimzi.io tenant-a/kafka-one: OAuth authentication on a listener +kafkas.kafka.strimzi.io tenant-a/kafka-one: secrets in a custom listener authentication +kafkas.kafka.strimzi.io tenant-a/kafka-one: Keycloak authorization +kafkas.kafka.strimzi.io tenant-b/kafka-two: Open Policy Agent authorization +kafkaconnects.kafka.strimzi.io tenant-a/connect-bad: OAuth client authentication +kafkaconnects.kafka.strimzi.io tenant-a/connect-bad: .spec.externalConfiguration +kafkaconnects.kafka.strimzi.io tenant-a/connect-bad: both .spec.build.output.additionalKanikoOptions and additionalBuildOptions +kafkabridges.kafka.strimzi.io tenant-a/bridge-bad: OAuth client authentication +kafkamirrormaker2s.kafka.strimzi.io tenant-a/mm2-old-bad: .spec.externalConfiguration +kafkamirrormaker2s.kafka.strimzi.io tenant-a/mm2-old-bad: OAuth client authentication +kafkamirrormaker2s.kafka.strimzi.io tenant-a/mm2-old-bad: .spec.connectCluster is not an alias in .spec.clusters +kafkamirrormaker2s.kafka.strimzi.io tenant-a/mm2-old-bad: a mirror whose targetCluster is not the target cluster +kafkamirrormaker2s.kafka.strimzi.io tenant-a/mm2-old-bad: a mirror whose sourceCluster is not an alias in .spec.clusters +kafkamirrormaker2s.kafka.strimzi.io tenant-a/mm2-no-clusters: no target cluster in .spec.target, or in .spec.connectCluster and .spec.clusters +kafkamirrormaker2s.kafka.strimzi.io tenant-a/mm2-no-clusters: a mirror without .source or .sourceCluster +kafkamirrormaker2s.kafka.strimzi.io tenant-a/mm2-new-oauth: OAuth client authentication +kafkausers.kafka.strimzi.io tenant-a/user-no-spec: no .spec +kafkanodepools.kafka.strimzi.io tenant-a/pool-null-spec: no .spec +LINES + [ "$(grep -c '^ kafkas.kafka.strimzi.io tenant-b/kafka-two: ' "$WORK/out")" -eq 1 ] + if grep -qF "clean" "$WORK/out"; then echo "named an object the conversion accepts" >&2; return 1; fi + if grep -qE "SCALE-|APPLY-|PATCH|STAMP|DELETE|ANNOTATE" "$FAKE_CMDLOG"; then echo "changed the cluster despite an object the conversion refuses" >&2; return 1; fi + rm -rf "$WORK" +} + +@test "a failed read of a Strimzi kind for the conversion refusal check aborts before the operators are stopped" { + prep + export FAKE_STRIMZI_LIST_FAIL=kafkaconnects + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + grep -qF "get kafkaconnects.v1beta2.kafka.strimzi.io --all-namespaces -o json" "$FAKE_CMDLOG" + if grep -qE "SCALE-|APPLY-|PATCH|STAMP|DELETE|ANNOTATE" "$FAKE_CMDLOG"; then echo "went on after a failed Strimzi list read" >&2; return 1; fi + rm -rf "$WORK" +} + +# The refusal check copies what the 0.51.0 tool rejects, so a different tool +# version needs that list read again from its source. +@test "the conversion image stays on Strimzi 0.51.0, the version the refusal check was read from" { + grep -qE '^CONVERSION_IMAGE="\$\{CONVERSION_IMAGE:-quay\.io/strimzi/operator:0\.51\.0@sha256:[0-9a-f]{64}\}"$' "$MIG_DIR/61" +} + +# With the CRDs serving v1 alone, every object is stored as v1 and no +# conversion runs. +@test "objects the conversion would refuse do not block a re-run once the CRDs serve only v1" { + prep + refused_objects + export FAKE_SERVED=v1 + printf '["v1"]' > "$WORK/state/stored" + rc=0 + run_migration 61 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + grep -qxF "STAMP 62" "$FAKE_CMDLOG" + rm -rf "$WORK" +} diff --git a/hack/testdata/migration-61-strimzi-v1/apps.txt b/hack/testdata/migration-61-strimzi-v1/apps.txt new file mode 100644 index 000000000..deb2852f1 --- /dev/null +++ b/hack/testdata/migration-61-strimzi-v1/apps.txt @@ -0,0 +1,4 @@ +tenant-a one v3.9 +tenant-b two v3.8 +tenant-c three v4.2 +tenant-d four diff --git a/hack/testdata/migration-61-strimzi-v1/kafka-releases.txt b/hack/testdata/migration-61-strimzi-v1/kafka-releases.txt new file mode 100644 index 000000000..e617bdcb9 --- /dev/null +++ b/hack/testdata/migration-61-strimzi-v1/kafka-releases.txt @@ -0,0 +1,2 @@ +tenant-a kafka-one +tenant-b diff --git a/hack/testdata/migration-61-strimzi-v1/kafkas-kraft.json b/hack/testdata/migration-61-strimzi-v1/kafkas-kraft.json new file mode 100644 index 000000000..9bb3daab3 --- /dev/null +++ b/hack/testdata/migration-61-strimzi-v1/kafkas-kraft.json @@ -0,0 +1,4 @@ +{"items":[ + {"metadata":{"namespace":"tenant-a","name":"kafka-one","annotations":{"strimzi.io/kraft":"enabled"}},"spec":{"kafka":{"version":"3.9.0"}},"status":{"kafkaMetadataState":"KRaft","kafkaMetadataVersion":"3.9-IV0","conditions":[{"type":"Ready","status":"True"}]}}, + {"metadata":{"namespace":"tenant-b","name":"kafka-two"},"spec":{"kafka":{"version":"3.8.1"}},"status":{"kafkaMetadataState":"KRaft","kafkaMetadataVersion":"3.9-IV0","conditions":[{"type":"Ready","status":"True"}]}} +]} diff --git a/hack/testdata/migration-61-strimzi-v1/kafkas-zk.json b/hack/testdata/migration-61-strimzi-v1/kafkas-zk.json new file mode 100644 index 000000000..1c0611fae --- /dev/null +++ b/hack/testdata/migration-61-strimzi-v1/kafkas-zk.json @@ -0,0 +1,5 @@ +{"items":[ + {"metadata":{"namespace":"tenant-a","name":"kafka-one"},"spec":{"kafka":{"version":"3.9.0"}},"status":{"kafkaMetadataState":"KRaft","kafkaMetadataVersion":"3.9-IV0","conditions":[{"type":"Ready","status":"True"}]}}, + {"metadata":{"namespace":"tenant-c","name":"kafka-zk"},"spec":{"kafka":{"version":"3.9.0"},"zookeeper":{"replicas":3}},"status":{"kafkaMetadataState":"ZooKeeper"}}, + {"metadata":{"namespace":"tenant-d","name":"kafka-mid"},"spec":{"kafka":{"version":"3.9.0"}},"status":{"kafkaMetadataState":"KRaftPostMigration"}} +]} diff --git a/hack/testdata/migration-61-strimzi-v1/kubectl b/hack/testdata/migration-61-strimzi-v1/kubectl new file mode 100755 index 000000000..e2d899bad --- /dev/null +++ b/hack/testdata/migration-61-strimzi-v1/kubectl @@ -0,0 +1,342 @@ +#!/bin/sh +# Fake kubectl for hack/migration-61-strimzi-v1-conversion.bats. It serves only +# the calls migration 61 and lib/cozystack-version.sh make, and logs each one +# to $FAKE_CMDLOG. +# +# FAKE_CMDLOG file to append a one-line record of each call to +# FAKE_STATE directory; its "stored" file holds the storedVersions +# every Strimzi CRD reports, and the conversion Job +# rewrites it to ["v1"] when it succeeds +# FAKE_NO_STRIMZI non-empty => the Kafka CRD is absent +# FAKE_KAFKAS fixture served for the Strimzi Kafka list; annotations +# written to it are kept in $FAKE_STATE +# FAKE_JOB succeed (default) | fail | noop (succeeds, converts nothing) +# | hang (neither succeeds nor fails) +# | nopod (Failed condition, no failed pod: the pod was +# never created) +# FAKE_PULL non-empty => the conversion pod cannot pull its image +# FAKE_PATCH_FAIL non-empty => every application patch exits 1 +# FAKE_CRD_READ_FAIL, FAKE_APIRES_FAIL, FAKE_SECRET_LIST_FAIL: that read +# exits 1 +# FAKE_APPS_ABSENT non-empty => the apps.cozystack.io API is not installed +# FAKE_DISCOVERY_PARTIAL non-empty => another API group fails discovery, so +# api-resources exits 1 after listing what it found +# FAKE_RELEASE_REWRITTEN_BACK: release patches do not stick +# FAKE_RELEASE_PENDING: revision v4 of kafka-one is pending-upgrade, a +# Helm action that has not finished +# FAKE_REBALANCES comma-separated ns/name served for the KafkaRebalance list +# FAKE_HELMRELEASES fixture served for the Kafka application HelmRelease +# list; none by default, and a missing file fails the read +# FAKE_KAFKATOPICS fixture served for the KafkaTopic list, the same way +# FAKE_STRIMZI_OBJECTS directory; .json in it is served for the JSON +# list of that Strimzi kind other than Kafka and +# KafkaTopic, an empty list when it is absent +# FAKE_STRIMZI_LIST_FAIL plural of one such kind whose JSON list read exits 1 +# The deployed Helm release of kafka-one is release-kafka-one.json, encoded +# the way Helm stores it; a patch of its Secret is written to +# $FAKE_STATE/release, so a second run reads what the first one wrote. +set -u + +log() { [ -n "${FAKE_CMDLOG:-}" ] && printf '%s\n' "$*" >> "$FAKE_CMDLOG"; } +log "KUBECTL $*" + +args="$*" +# The Kafka list as it stands: converted, annotated, or the fixture. +kafkas_src() { + for f in "$FAKE_STATE/kafkas-converted.json" "$FAKE_STATE/kafkas-live.json"; do + [ -f "$f" ] && { printf '%s' "$f"; return; } + done + printf '%s' "$FAKE_KAFKAS" +} +arg_after() { k="$1"; shift; p=""; for a in "$@"; do [ "$p" = "$k" ] && { printf '%s' "$a"; return; }; p="$a"; done; } + +case "$args" in + *"apply --filename -"*) + manifest=$(cat) + v=$(printf '%s\n' "$manifest" | sed -n 's/^ version: "\{0,1\}\([0-9]\{1,\}\)"\{0,1\}$/\1/p' | head -1) + log "STAMP ${v:-unknown}" + exit 0 + ;; + *"apply -f -"*) + manifest=$(cat) + case "$manifest" in + *"kind: Job"*) + order=$(printf '%s\n' "$manifest" | sed -n 's/.*v1-api-conversion.sh", "\([a-z-]*\)".*/\1/p' | tr '\n' ' ') + log "APPLY-CONVERSION ${order% }" + log "CONVERSION-DEADLINE $(printf '%s\n' "$manifest" | sed -n 's/^ activeDeadlineSeconds: \([0-9]*\)$/\1/p')" + exit 0 + ;; + esac + log "APPLY-UNKNOWN" + exit 1 + ;; + *"apply --server-side"*) + log "APPLY-CRDS $(arg_after -f "$@")" + exit 0 + ;; + *"get crd kafkas.kafka.strimzi.io --ignore-not-found -o name"*) + [ -n "${FAKE_NO_STRIMZI:-}" ] || echo "customresourcedefinition.apiextensions.k8s.io/kafkas.kafka.strimzi.io" + exit 0 + ;; + *"get crd kafkas.kafka.strimzi.io -o jsonpath={.spec.versions[*].name}"*) + printf '%s' "${FAKE_SERVED:-v1 v1beta2}" + exit 0 + ;; + *"get crd "*"storedVersions"*) + if [ -n "${FAKE_CRD_READ_FAIL:-}" ]; then echo "Error from server: etcdserver: request timed out" >&2; exit 1; fi + cat "$FAKE_STATE/stored" + exit 0 + ;; + *" annotate kafkas.v1beta2.kafka.strimzi.io "*|*" annotate kafkas.v1.kafka.strimzi.io "*) + ns=$(arg_after -n "$@"); kind=$(arg_after annotate "$@"); name=$(arg_after "$kind" "$@") + kv=""; for a in "$@"; do case "$a" in cozystack.io/*=*) kv="$a" ;; esac; done + log "ANNOTATE $ns/$name $kv" + src=$(kafkas_src); dst="$FAKE_STATE/kafkas-live.json" + [ "$src" = "$FAKE_STATE/kafkas-converted.json" ] && dst="$src" + jq --arg ns "$ns" --arg name "$name" --arg k "${kv%%=*}" --arg v "${kv#*=}" \ + '.items |= map(if .metadata.namespace == $ns and .metadata.name == $name then .metadata.annotations[$k] = $v else . end)' \ + "$src" > "$dst.new" && mv "$dst.new" "$dst" + exit 0 + ;; + *"get "*".kafka.strimzi.io --all-namespaces -o json"|*"get strimzipodsets."*".core.strimzi.io --all-namespaces -o json") + resource=$(arg_after get "$@") + case "${resource%%.*}" in + kafkas|kafkatopics) ;; + *) + if [ "${resource%%.*}" = kafkarebalances ] && [ -n "${FAKE_REBALANCE_FAIL:-}" ]; then echo "Error from server: etcdserver: request timed out" >&2; exit 1; fi + if [ "${resource%%.*}" = "${FAKE_STRIMZI_LIST_FAIL:-}" ]; then echo "Error from server: etcdserver: request timed out" >&2; exit 1; fi + f="${FAKE_STRIMZI_OBJECTS:-/nonexistent}/${resource%%.*}.json" + if [ -f "$f" ]; then cat "$f"; else echo '{"items":[]}'; fi + exit 0 + ;; + esac + ;; +esac +case "$args" in + *"get kafkarebalances.v1beta2.kafka.strimzi.io"*|*"get kafkarebalances.v1.kafka.strimzi.io"*) + if [ -n "${FAKE_REBALANCE_FAIL:-}" ]; then echo "Error from server: etcdserver: request timed out" >&2; exit 1; fi + [ -z "${FAKE_REBALANCES:-}" ] || printf '%s\n' "$FAKE_REBALANCES" | tr ',' '\n' + exit 0 + ;; + *"get helmreleases.helm.toolkit.fluxcd.io --all-namespaces -l apps.cozystack.io/application.kind=Kafka -o json"*) + if [ -n "${FAKE_HELMRELEASES:-}" ]; then cat "$FAKE_HELMRELEASES" || exit 1; else echo '{"items":[]}'; fi + exit 0 + ;; + *"get kafkatopics.v1beta2.kafka.strimzi.io --all-namespaces -o json"*|*"get kafkatopics.v1.kafka.strimzi.io --all-namespaces -o json"*) + if [ -n "${FAKE_KAFKATOPICS:-}" ]; then cat "$FAKE_KAFKATOPICS" || exit 1; else echo '{"items":[]}'; fi + exit 0 + ;; + *"get kafkas.v1beta2.kafka.strimzi.io"*) + if [ -n "${FAKE_KAFKA_LIST_FAIL:-}" ]; then echo "Error from server: Kafka list failed" >&2; exit 1; fi + case "${FAKE_SERVED:-v1 v1beta2}" in v1) echo "v1beta2 is not served" >&2; exit 1 ;; esac + cat "$(kafkas_src)" + exit 0 + ;; + *"get kafkas.v1.kafka.strimzi.io --all-namespaces -o json"|*"get kafkas.v1.kafka.strimzi.io --all-namespaces --output json") + if [ -n "${FAKE_KAFKA_LIST_FAIL:-}" ]; then echo "Error from server: Kafka list failed" >&2; exit 1; fi + cat "$(kafkas_src)" + exit 0 + ;; + *"get kafkas.v1.kafka.strimzi.io"*) + cat "$FAKE_RELEASES" + exit 0 + ;; + *"get secret -l owner=helm,name=kafka-one -o"*) + if [ -n "${FAKE_SECRET_LIST_FAIL:-}" ]; then echo "Error from server (Forbidden): secrets is forbidden" >&2; exit 1; fi + # v4 stands for a failed upgrade: still the latest revision, so uninstall + # builds it. + echo "sh.helm.release.v1.kafka-one.v3 sh.helm.release.v1.kafka-one.v4" + exit 0 + ;; + *"get secret -l owner=helm,name=kafka-failed-install -o"*) + echo "sh.helm.release.v1.kafka-failed-install.v1" + exit 0 + ;; + *"get secret sh.helm.release.v1.kafka-failed-install.v1 -o jsonpath={.data.release}"*) + if [ ! -f "$FAKE_STATE/release-failed" ]; then + gzip -c "$(dirname "$0")/release-kafka-one.json" | base64 | tr -d '\n' | base64 | tr -d '\n' > "$FAKE_STATE/release-failed" + fi + cat "$FAKE_STATE/release-failed" + exit 0 + ;; + *"patch secret sh.helm.release.v1.kafka-failed-install.v1 --type merge --patch-file "*) + jq -r '.data.release' "$(arg_after --patch-file "$@")" > "$FAKE_STATE/release-failed" + log "PATCH-RELEASE tenant-f/sh.helm.release.v1.kafka-failed-install.v1" + exit 0 + ;; + *"get deployment --all-namespaces -l app.kubernetes.io/name=entity-operator"*) + [ -n "${FAKE_NO_OPERATOR:-}" ] || echo "tenant-a kafka-one-entity-operator" + exit 0 + ;; + *"scale deployment kafka-one-entity-operator --replicas=0"*) + log "SCALE-ENTITY tenant-a/kafka-one-entity-operator" + exit 0 + ;; + *"get deployment kafka-one-entity-operator -o json"*) + echo '{"spec":{"selector":{"matchLabels":{"strimzi.io/cluster":"kafka-one","strimzi.io/name":"kafka-one-entity-operator"}}}}' + exit 0 + ;; + *"wait --for=delete pod -l strimzi.io/"*) + log "WAIT-ENTITY $(arg_after -n "$@")/$(arg_after -l "$@")" + exit 0 + ;; + *"get secret -l owner=helm,name="*) + exit 0 + ;; + *"get secret sh.helm.release.v1.kafka-one.v"*" -o jsonpath={.data.release}"*) + rev=$(arg_after secret "$@"); rev=${rev##*.} + if [ ! -f "$FAKE_STATE/release-$rev" ]; then + status=deployed + [ "$rev" != v4 ] || [ -z "${FAKE_RELEASE_PENDING:-}" ] || status=pending-upgrade + jq -c --arg s "$status" '.info.status = $s' "$(dirname "$0")/release-kafka-one.json" | + gzip -c | base64 | tr -d '\n' | base64 | tr -d '\n' > "$FAKE_STATE/release-$rev" + fi + cat "$FAKE_STATE/release-$rev" + exit 0 + ;; + *"patch secret sh.helm.release.v1.kafka-one.v"*" --type merge --patch-file "*) + rev=$(arg_after secret "$@"); rev=${rev##*.} + # FAKE_RELEASE_REWRITTEN_BACK: a Helm upgrade that was already running + # stores its v1beta2 revision over the rewrite. + [ -n "${FAKE_RELEASE_REWRITTEN_BACK:-}" ] || jq -r '.data.release' "$(arg_after --patch-file "$@")" > "$FAKE_STATE/release-$rev" + log "PATCH-RELEASE tenant-a/sh.helm.release.v1.kafka-one.$rev" + exit 0 + ;; + *"get deployment strimzi-cluster-operator --ignore-not-found -o name"*) + if [ -n "${FAKE_OPERATOR_READ_FAIL:-}" ]; then echo "Error from server: etcdserver: request timed out" >&2; exit 1; fi + [ -n "${FAKE_NO_OPERATOR:-}" ] || echo "deployment.apps/strimzi-cluster-operator" + exit 0 + ;; + *"scale deployment strimzi-cluster-operator --replicas=0"*) + log "SCALE-OPERATOR 0" + exit 0 + ;; + *"wait --for=delete pod -l name=strimzi-cluster-operator"*) + exit 0 + ;; + *"get kafkas.kafka.strimzi.io --all-namespaces --show-managed-fields -o json"*) + if [ -f "$FAKE_STATE/managed" ]; then cat "$FAKE_STATE/managed"; else cat "$(dirname "$0")/managed-kafkas.json"; fi + exit 0 + ;; + *"--all-namespaces --show-managed-fields -o json"*) + echo '{"items":[]}' + exit 0 + ;; + *"get kafkas.kafka.strimzi.io "*" --show-managed-fields -o json"*) + ns=$(arg_after -n "$@"); name=$(arg_after kafkas.kafka.strimzi.io "$@") + src="$FAKE_STATE/managed"; [ -f "$src" ] || src="$(dirname "$0")/managed-kafkas.json" + jq --arg ns "$ns" --arg n "$name" '.items[] | select(.metadata.namespace == $ns and .metadata.name == $n)' "$src" + exit 0 + ;; + *"patch kafkas.kafka.strimzi.io "*"--patch-file "*) + ns=$(arg_after -n "$@"); name=$(arg_after kafkas.kafka.strimzi.io "$@") + patch=$(arg_after --patch-file "$@") + src="$FAKE_STATE/managed"; [ -f "$src" ] || src="$(dirname "$0")/managed-kafkas.json" + rv=$(jq -r '.metadata.resourceVersion' "$patch") + log "PATCH-MANAGED $ns/$name rv=$rv $(jq -r '[.metadata.managedFields[] | "\(.manager)=\(.apiVersion)"] | join(",")' "$patch")" + # FAKE_MANAGED_CONFLICT: a concurrent write bumps the object before the + # first patch lands, so that patch carries a stale resourceVersion. + if [ -n "${FAKE_MANAGED_CONFLICT:-}" ] && [ ! -f "$FAKE_STATE/bumped" ]; then + : > "$FAKE_STATE/bumped" + jq --arg ns "$ns" --arg n "$name" '(.items[] | select(.metadata.namespace == $ns and .metadata.name == $n) | .metadata.resourceVersion) |= (tonumber + 1 | tostring)' "$src" > "$FAKE_STATE/managed.new" && mv "$FAKE_STATE/managed.new" "$FAKE_STATE/managed" + fi + if [ -n "${FAKE_MANAGED_PATCH_FAIL:-}" ]; then echo "Error from server (Forbidden): forbidden" >&2; exit 1; fi + src="$FAKE_STATE/managed"; [ -f "$src" ] || src="$(dirname "$0")/managed-kafkas.json" + cur=$(jq -r --arg ns "$ns" --arg n "$name" '.items[] | select(.metadata.namespace == $ns and .metadata.name == $n) | .metadata.resourceVersion' "$src") + if [ "$cur" != "$rv" ]; then + echo "Error from server (Conflict): Operation cannot be fulfilled on kafkas.kafka.strimzi.io \"$name\": the object has been modified; please apply your changes to the latest version and try again" >&2 + exit 1 + fi + jq --arg ns "$ns" --arg n "$name" --slurpfile p "$patch" \ + '(.items[] | select(.metadata.namespace == $ns and .metadata.name == $n) | .metadata) |= (.managedFields = $p[0].metadata.managedFields | .resourceVersion = (.resourceVersion | tonumber + 1 | tostring))' \ + "$src" > "$FAKE_STATE/managed.new" && mv "$FAKE_STATE/managed.new" "$FAKE_STATE/managed" + exit 0 + ;; + *"wait --for=condition=Established"*) + exit 0 + ;; + *"get job strimzi-v1-api-conversion --ignore-not-found -o jsonpath={.metadata.name} {.status.succeeded}{.status.failed}"*) + # FAKE_OLD_JOB: a Job an earlier attempt left behind, running, succeeded + # or failed. + case "${FAKE_OLD_JOB:-}" in + running) printf 'strimzi-v1-api-conversion ' ;; + succeeded|failed) printf 'strimzi-v1-api-conversion 1' ;; + esac + exit 0 + ;; + *"get job strimzi-v1-api-conversion -o jsonpath={.status.succeeded}"*) + case "${FAKE_JOB:-succeed}" in + succeed) + if [ -n "${FAKE_CONVERSION_CHANGES_SPEC:-}" ] && [ ! -f "$FAKE_STATE/kafkas-converted.json" ]; then + jq '.items |= map( + .apiVersion = "kafka.strimzi.io/v1" + | .metadata.generation += 1 + | del(.spec.kafka.replicas, .spec.kafka.storage) + | del(.status.kafkaMetadataState) + | if .spec.entityOperator.topicOperator.reconciliationIntervalSeconds != null then + .spec.entityOperator.topicOperator |= + (.reconciliationIntervalMs = (.reconciliationIntervalSeconds * 1000) + | del(.reconciliationIntervalSeconds)) + else . end)' "$(kafkas_src)" > "$FAKE_STATE/kafkas-converted.json" + fi + printf '["v1"]' > "$FAKE_STATE/stored" + echo 1 + ;; + noop) echo 1 ;; + esac + exit 0 + ;; + *"get job strimzi-v1-api-conversion -o jsonpath={.status.failed}"*) + case "${FAKE_JOB:-succeed}" in + fail) printf '1' ;; + esac + case "$args" in + *'conditions[?(@.type=="Failed")]'*) + case "${FAKE_JOB:-succeed}" in fail|nopod) printf ' True' ;; esac + ;; + esac + exit 0 + ;; + *"get pod -l batch.kubernetes.io/job-name=strimzi-v1-api-conversion -o jsonpath="*) + [ -z "${FAKE_PULL:-}" ] || printf ' ImagePullBackOff' + exit 0 + ;; + *" logs "*) + echo "fake conversion log" + exit 0 + ;; + "delete "*|*" delete "*) + cascade="" + case "$args" in *"--cascade=foreground"*) cascade=" foreground" ;; esac + log "DELETE $(arg_after delete "$@") $(arg_after "$(arg_after delete "$@")" "$@")$cascade" + exit 0 + ;; + *"api-resources --api-group=apps.cozystack.io"*) + echo "kafkas.apps.cozystack.io" + if [ -n "${FAKE_DISCOVERY_PARTIAL:-}" ]; then echo "error: unable to retrieve the complete list of server APIs: metrics.k8s.io/v1beta1: the server is currently unable to handle the request" >&2; exit 1; fi + exit 0 + ;; + *"get --raw /apis/apps.cozystack.io/v1alpha1"*) + if [ -n "${FAKE_APIRES_FAIL:-}" ]; then echo "Error from server (ServiceUnavailable): the server is currently unable to handle the request" >&2; exit 1; fi + if [ -n "${FAKE_APPS_ABSENT:-}" ]; then echo "Error from server (NotFound): the server could not find the requested resource" >&2; exit 1; fi + echo '{"kind":"APIResourceList","groupVersion":"apps.cozystack.io/v1alpha1","resources":[{"name":"kafkas","kind":"Kafka"},{"name":"kafkas/status","kind":"Kafka"}]}' + exit 0 + ;; + *"get kafkas.apps.cozystack.io"*" kafka-"*) + # Release names of the applications; failed-install has no Kafka CR. + printf '%s\n' 'tenant-a kafka-one' 'tenant-f kafka-failed-install' + exit 0 + ;; + *"get kafkas.apps.cozystack.io"*) + cat "$FAKE_APPS" + exit 0 + ;; + *"patch kafkas.apps.cozystack.io"*) + if [ -n "${FAKE_PATCH_FAIL:-}" ]; then echo "Error from server (Conflict)" >&2; exit 1; fi + log "PATCH $(arg_after -n "$@")/$(arg_after kafkas.apps.cozystack.io "$@") $(arg_after -p "$@")" + exit 0 + ;; +esac +log "UNMODELLED" +printf 'fake kubectl: unmodelled call: %s\n' "$args" >&2 +exit 1 diff --git a/hack/testdata/migration-61-strimzi-v1/managed-kafkas.json b/hack/testdata/migration-61-strimzi-v1/managed-kafkas.json new file mode 100644 index 000000000..bea24efd5 --- /dev/null +++ b/hack/testdata/migration-61-strimzi-v1/managed-kafkas.json @@ -0,0 +1,54 @@ +{ + "items": [ + { + "metadata": { + "namespace": "tenant-a", + "name": "kafka-one", + "managedFields": [ + { + "manager": "helm-controller", + "operation": "Apply", + "apiVersion": "kafka.strimzi.io/v1beta2", + "fieldsType": "FieldsV1", + "fieldsV1": { + "f:spec": { + "f:kafka": { + "f:version": {} + } + } + } + }, + { + "manager": "strimzi-cluster-operator", + "operation": "Update", + "apiVersion": "kafka.strimzi.io/v1beta2", + "subresource": "status", + "fieldsType": "FieldsV1", + "fieldsV1": { + "f:status": {} + } + } + ], + "resourceVersion": "100" + } + }, + { + "metadata": { + "namespace": "tenant-b", + "name": "kafka-two", + "managedFields": [ + { + "manager": "helm-controller", + "operation": "Apply", + "apiVersion": "kafka.strimzi.io/v1", + "fieldsType": "FieldsV1", + "fieldsV1": { + "f:spec": {} + } + } + ], + "resourceVersion": "101" + } + } + ] +} \ No newline at end of file diff --git a/hack/testdata/migration-61-strimzi-v1/release-kafka-one.json b/hack/testdata/migration-61-strimzi-v1/release-kafka-one.json new file mode 100644 index 000000000..f995a4926 --- /dev/null +++ b/hack/testdata/migration-61-strimzi-v1/release-kafka-one.json @@ -0,0 +1 @@ +{"name":"kafka-one","version":3,"info":{"status":"deployed"},"manifest":"---\n# Source: kafka/templates/kafka.yaml\napiVersion: kafka.strimzi.io/v1beta2\nkind: Kafka\nmetadata:\n name: kafka-one\n---\n# Source: kafka/templates/topics.yaml\napiVersion: kafka.strimzi.io/v1beta2\nkind: KafkaTopic\nmetadata:\n name: kafka-one-orders\n---\napiVersion: v1\nkind: ConfigMap\nmetadata:\n name: kafka-one-metrics\n","hooks":[{"name":"kafka-one-kafka-migration","manifest":"apiVersion: batch/v1\nkind: Job\n# script applies apiVersion: kafka.strimzi.io/v1beta2\n"}]} \ No newline at end of file diff --git a/hack/testdata/migration-61-strimzi-v1/strimzi-crds-0.51.0.yaml b/hack/testdata/migration-61-strimzi-v1/strimzi-crds-0.51.0.yaml new file mode 100644 index 000000000..76a5f108a --- /dev/null +++ b/hack/testdata/migration-61-strimzi-v1/strimzi-crds-0.51.0.yaml @@ -0,0 +1 @@ +# Stand-in for strimzi-crds-0.51.0.yaml; the fake kubectl never reads it. diff --git a/packages/core/platform/images/migrations/Dockerfile b/packages/core/platform/images/migrations/Dockerfile index 6084a9e5e..7a6bcf6b5 100644 --- a/packages/core/platform/images/migrations/Dockerfile +++ b/packages/core/platform/images/migrations/Dockerfile @@ -29,6 +29,14 @@ RUN set -eux; \ echo "${sha256} /usr/bin/etcd-migrate" | sha256sum -c -; \ chmod +x /usr/bin/etcd-migrate +# CRDs of Strimzi 0.51.0, the last release serving both v1beta2 and v1. +# Migration 61 applies them so Strimzi's conversion tool can move every stored +# object to v1 before the kafka-operator release installs the v1-only CRDs. +RUN mkdir /strimzi-crds && \ + curl -fsSL -o /strimzi-crds/strimzi-crds-0.51.0.yaml \ + https://github.com/strimzi/strimzi-kafka-operator/releases/download/0.51.0/strimzi-crds-0.51.0.yaml && \ + echo "a11257e67985a946a6facfb17d729aac5670b89675d410d13162fe455a5ac7a8 /strimzi-crds/strimzi-crds-0.51.0.yaml" | sha256sum -c - + COPY migrations /migrations # v1alpha2 etcd CRDs, vendored into the build context by `make image-migrations`. # migration 50 applies these before adoption so it does not deadlock waiting for diff --git a/packages/core/platform/images/migrations/migrations/61 b/packages/core/platform/images/migrations/migrations/61 new file mode 100755 index 000000000..c70895a61 --- /dev/null +++ b/packages/core/platform/images/migrations/migrations/61 @@ -0,0 +1,661 @@ +#!/bin/bash +# Migration 61 --> 62 +# Move the Strimzi custom resources to the v1 API before Strimzi 1.x arrives. +# +# Strimzi 1.0 removed the v1beta2 API from its CRDs. The kafka-operator +# release replaces the CRDs on upgrade (upgradeCRDs: CreateReplace), and the +# API server refuses a CRD whose spec.versions no longer lists a version that +# status.storedVersions still records. So every stored object has to be +# rewritten as v1, and storedVersions narrowed to [v1], before the new +# operator arrives. That needs CRDs serving both versions, which only Strimzi +# 0.49 to 0.51 ship: the 0.51.0 CRDs are baked into this image (see the +# Dockerfile). The conversion itself is Strimzi's v1-api-conversion tool, run +# as a Job from the 0.51.0 operator image: convert-resource rewrites the +# objects, crd-upgrade switches the storage version and narrows +# storedVersions. +# +# Strimzi 0.46 also removed ZooKeeper. A ZooKeeper-based Kafka that did not +# reach KRaft on the release carrying the chart's ZooKeeper to KRaft +# migration cannot be run, converted or migrated by anything after this +# point, so this migration refuses to proceed while one is left. +# +# Strimzi 1.2 runs neither Kafka 3.8 nor 3.9, so every Kafka application on +# v3.8 or v3.9 is moved to v4.3. Until the new kafka chart arrives, the +# release renders with the old one, which rejects v4.3, and the HelmRelease +# reports a values error; no Helm action runs in that window. +# +# Ordering: this runs as the platform chart's pre-upgrade hook, before the +# upgrade re-points cozystack-packages, so on a variant that runs migrations +# neither the new operator nor the new kafka chart is rendered before it. The +# default variant renders no migration hook, and there these steps are manual. +# +# Re-runnable: every step tolerates having run before. + +set -euo pipefail + +# shellcheck source=lib/cozystack-version.sh +. "$(dirname "$0")/lib/cozystack-version.sh" + +STRIMZI_CRD_FILE="${STRIMZI_CRD_FILE:-/strimzi-crds/strimzi-crds-0.51.0.yaml}" +CONVERSION_IMAGE="${CONVERSION_IMAGE:-quay.io/strimzi/operator:0.51.0@sha256:ef3e406aa09449dd0241d53cdc81e15e58bf59b51f2425fb64384a179d676b1d}" +CONVERSION_NS="${CONVERSION_NS:-cozy-kafka-operator}" +CONVERSION_NAME=strimzi-v1-api-conversion +CONVERSION_TIMEOUT="${CONVERSION_TIMEOUT:-1800}" +# The run that starts the Job waits up to CONVERSION_TIMEOUT for it. As the +# platform hook, a run is cut off sooner by the Helm upgrade timeout +# (cozystack-operator --helmrelease-upgrade-timeout, 10m by default), and the +# retry, the next pod of the hook Job or a new hook Job once helm-controller +# retries the upgrade, waits up to CONVERSION_TIMEOUT again for the same Job. +# The Job is not owned by the hook Job and nothing else bounds it, so the +# deadline is the only limit: a conversion gets two waits, and a Job whose pod +# never starts then fails and the next run starts the conversion over. +CONVERSION_DEADLINE=$((CONVERSION_TIMEOUT * 2)) +POLL_INTERVAL="${POLL_INTERVAL:-5}" +KAFKA_TARGET_VERSION=v4.3 + +# The CRDs the conversion tool rewrites. kafkamirrormakers.kafka.strimzi.io +# is not among them: MirrorMaker 1 left in Strimzi 0.46 and its CRD is left +# behind untouched. +STRIMZI_CRDS="kafkas.kafka.strimzi.io kafkanodepools.kafka.strimzi.io +kafkaconnects.kafka.strimzi.io kafkaconnectors.kafka.strimzi.io +kafkabridges.kafka.strimzi.io kafkamirrormaker2s.kafka.strimzi.io +kafkarebalances.kafka.strimzi.io kafkatopics.kafka.strimzi.io +kafkausers.kafka.strimzi.io strimzipodsets.core.strimzi.io" + +# --ignore-not-found separates a cluster without Strimzi (no output, rc 0) +# from an API error, which aborts under set -e. +crd=$(kubectl get crd kafkas.kafka.strimzi.io --ignore-not-found -o name) +if [ -z "$crd" ]; then + echo "Strimzi is not installed; nothing to convert" + stamp_cozystack_version 62 + exit 0 +fi + +# The running 0.45 operator must not write to the resources from here on. +# Strimzi supports the conversion only under an operator that knows v1 +# (0.49 or later), and 0.45 rewrites the Kafka status through v1beta2 on every +# reconcile, because the v1 schema prunes its kafkaMetadataState field. Each +# such write records a v1beta2 managedFields entry again after the rewrite +# below. The kafka-operator release brings the operator back at Strimzi 1.2: +# its chart sets the replica count, so the upgrade restores it. Kafka pods keep +# running while no operator reconciles them. +stop_old_operator() { + operator=$(kubectl -n "$CONVERSION_NS" get deployment strimzi-cluster-operator --ignore-not-found -o name) + if [ -n "$operator" ]; then + kubectl -n "$CONVERSION_NS" scale deployment strimzi-cluster-operator --replicas=0 + kubectl -n "$CONVERSION_NS" wait --for=delete pod -l name=strimzi-cluster-operator --timeout=300s + fi + # The topic and user operators of each cluster write KafkaTopic and + # KafkaUser status through v1beta2 the same way, also while they shut down. + # Only now that the cluster operator is gone does a scale-down stick; the + # 1.2 operator scales them back up when it rolls them onto its own image. + kubectl get deployment --all-namespaces -l app.kubernetes.io/name=entity-operator,app.kubernetes.io/managed-by=strimzi-cluster-operator \ + -o jsonpath='{range .items[*]}{.metadata.namespace}{" "}{.metadata.name}{"\n"}{end}' | + while read -r ns name; do + kubectl -n "$ns" scale deployment "$name" --replicas=0 + # An empty selector would wait on every pod in the namespace. + selector=$(kubectl -n "$ns" get deployment "$name" -o json | + jq -er '.spec.selector.matchLabels // {} | to_entries | map("\(.key)=\(.value)") | join(",") | select(length > 0)') + kubectl -n "$ns" wait --for=delete pod -l "$selector" --timeout=300s + done + } + +# Called as an if condition, where set -e does not apply: a failed read exits +# here, or it would pass for an absent CRD. +stored_v1_only() { + for c in $STRIMZI_CRDS; do + if ! v=$(kubectl get crd "$c" --ignore-not-found -o jsonpath='{.status.storedVersions}'); then + echo "ERROR: reading the stored versions of $c failed" >&2 + exit 1 + fi + if [ -n "$v" ] && [ "$v" != '["v1"]' ]; then + return 1 + fi + done + return 0 + } + +# A Job whose pod the API server refused fails on its deadline with no failed +# pod to count, so its Failed condition is read as well. +job_failed() { + f=$(kubectl -n "$CONVERSION_NS" get job "$CONVERSION_NAME" -o jsonpath='{.status.failed} {.status.conditions[?(@.type=="Failed")].status}') + case "$f" in + [1-9]*|*True*) return 0 ;; + esac + return 1 + } + +# wait_for_conversion: 0 once the conversion Job has succeeded; on a failure +# or a timeout, print its logs and return 1. +wait_for_conversion() { + echo "Waiting for the conversion Job" + waited=0 + pull_reported= + while :; do + succeeded=$(kubectl -n "$CONVERSION_NS" get job "$CONVERSION_NAME" -o jsonpath='{.status.succeeded}') + # A pod that cannot pull its image leaves no logs to print, and its Job + # fails only at the deadline. + if [ -z "$pull_reported" ]; then + waiting=$(kubectl -n "$CONVERSION_NS" get pod -l "batch.kubernetes.io/job-name=$CONVERSION_NAME" \ + -o jsonpath='{.items[*].status.initContainerStatuses[*].state.waiting.reason} {.items[*].status.containerStatuses[*].state.waiting.reason}' || true) + case "$waiting" in + *ErrImagePull*|*ImagePullBackOff*) + echo "ERROR: the conversion pod cannot pull $CONVERSION_IMAGE. A cluster that cannot reach quay.io needs that image mirrored under the same name and digest, for example through a registry mirror for quay.io in the container runtime of the nodes. The Job fails ${CONVERSION_DEADLINE}s after it started, and the next run of the migration starts it again." >&2 + pull_reported=1 + ;; + esac + fi + if [ "${succeeded:-0}" -ge 1 ]; then + kubectl -n "$CONVERSION_NS" logs "job/$CONVERSION_NAME" --all-containers --prefix || true + return 0 + fi + if job_failed || [ "$waited" -ge "$CONVERSION_TIMEOUT" ]; then + echo "ERROR: the Strimzi v1 API conversion Job did not succeed; its logs:" >&2 + kubectl -n "$CONVERSION_NS" logs "job/$CONVERSION_NAME" --all-containers --prefix >&2 || true + return 1 + fi + sleep "$POLL_INTERVAL" + waited=$((waited + POLL_INTERVAL)) + done + } + +# The Job's ClusterRole can patch CRDs and write ConfigMaps everywhere, so +# none of it outlives a conversion that has finished, whether it succeeded or +# failed. A failed Job itself stays for diagnosis. A Job still running when +# the wait times out keeps its rights: the retry waits for that same Job. +remove_conversion_rbac() { + kubectl delete clusterrolebinding "$CONVERSION_NAME" --ignore-not-found + kubectl delete clusterrole "$CONVERSION_NAME" --ignore-not-found + kubectl -n "$CONVERSION_NS" delete serviceaccount "$CONVERSION_NAME" --ignore-not-found + } + +remove_conversion_job() { + kubectl -n "$CONVERSION_NS" delete job "$CONVERSION_NAME" --ignore-not-found --wait=true + remove_conversion_rbac + } + +# The platform hook can time out while the Job converts, and the retry kills +# the hook at whatever point it reached. A Job still running then has to +# finish before anything here touches the CRDs, or the CRD apply below would +# switch the storage version back under it. Its outcome is read from the CRDs +# afterwards, not from the Job. +job_state=$(kubectl -n "$CONVERSION_NS" get job "$CONVERSION_NAME" --ignore-not-found -o jsonpath='{.metadata.name} {.status.succeeded}{.status.failed}{.status.conditions[?(@.type=="Failed")].status}') +case "$job_state" in + "$CONVERSION_NAME ") + echo "A conversion Job from an earlier attempt is still running" + # backoffLimit is 0, so a Job reporting no failure has not finished. + if ! wait_for_conversion; then + if ! job_failed; then + echo "ERROR: the conversion Job of an earlier attempt is still running; retry once it finishes" >&2 + exit 1 + fi + fi + ;; +esac +# A previous conversion can still be changing these objects, so the gate runs +# after its wait but before a new operator stop or any CRD mutation. +served=$(kubectl get crd kafkas.kafka.strimzi.io -o jsonpath='{.spec.versions[*].name}') +case " $served " in + *" v1beta2 "*) kafka_api=v1beta2 ;; + *" v1 "*) kafka_api=v1 ;; + *) echo "ERROR: no supported Strimzi Kafka API is served" >&2; exit 1 ;; +esac +# KRaft is the last Strimzi metadata state, so the state with a metadata +# version proves KRaft. The conversion deletes kafkaMetadataState, so without +# it the proof is Ready=True, or the annotation a previous run recorded below. +# Ready because the assembly operator retains kafkaMetadataVersion on failure: +# https://github.com/strimzi/strimzi-kafka-operator/blob/0.51.0/cluster-operator/src/main/java/io/strimzi/operator/cluster/operator/assembly/KafkaAssemblyOperator.java#L189-L192 +KRAFT_ANNOTATION=cozystack.io/kraft-verified +kafkas=$(kubectl get "kafkas.$kafka_api.kafka.strimzi.io" --all-namespaces -o json) +unproven=$(printf '%s' "$kafkas" | jq --slurp --exit-status --raw-output --arg verified "$KRAFT_ANNOTATION" ' + def nonempty_string: type == "string" and length > 0; + def unproven_reason: + if .spec.zookeeper != null then "ZooKeeper section in spec" + elif (.status | type) != "object" then "no status" + elif .status.kafkaMetadataState != null and .status.kafkaMetadataState != "KRaft" + then "metadata state \(.status.kafkaMetadataState)" + elif (.status.kafkaMetadataVersion | nonempty_string | not) then "no kafkaMetadataVersion" + elif .status.kafkaMetadataState == "KRaft" or + any(.status.conditions[]?; .type == "Ready" and .status == "True") or + (.metadata.annotations[$verified] | nonempty_string) + then null + else "not Ready" end; + if length != 1 or (.[0] | type) != "object" or (.[0].items | type) != "array" + then error("invalid Strimzi Kafka list") else .[0].items end + | map( + if type != "object" or (.metadata.namespace | nonempty_string | not) or + (.metadata.name | nonempty_string | not) or (.spec | type) != "object" + then error("invalid Strimzi Kafka object") else . end + | unproven_reason as $reason + | select($reason != null) + | "\(.metadata.namespace)/\(.metadata.name)" + + (if (.metadata.annotations["meta.helm.sh/release-name"] | nonempty_string) + then " (Helm release: \(.metadata.namespace)/\(.metadata.annotations["meta.helm.sh/release-name"]))" + else "" end) + ": \($reason)") + | join("\n")') +if [ -n "$unproven" ]; then + echo "ERROR: KRaft is not confirmed for these Kafka clusters:" >&2 + printf '%s\n' "$unproven" | sed 's/^/ /' >&2 + echo "Strimzi 1.x cannot run ZooKeeper or an unfinished migration. Upgrade through Cozystack v1.7 and let each Kafka finish its KRaft migration and report Ready before upgrading past it. A Kafka created after this migration stopped the Strimzi operators has no status until the upgrade finishes: delete that application and create it again afterwards." >&2 + exit 1 +fi + +# The v1.7 chart removes its drain rebalance only when the shared pool is free. +rebalances=$(kubectl get "kafkarebalances.$kafka_api.kafka.strimzi.io" --all-namespaces \ + -o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}') +drains=$(printf '%s\n' "$rebalances" | grep -- '-drain-kafka$' || true) +if [ -n "$drains" ]; then + echo "ERROR: a ZooKeeper to KRaft migration of Cozystack v1.7 is still draining a shared broker pool:" >&2 + printf '%s\n' "$drains" | sed 's/^/ /' >&2 + echo "Let the kafka application upgrade on v1.7 finish, which removes these KafkaRebalances, before upgrading past it." >&2 + exit 1 +fi + +# Kafka 4.2/4.3 reject these topic configs that Kafka 3.x accepted, and the +# new kafka chart refuses to render them, so such an application would stay on +# its pre-upgrade spec under an operator that cannot run it; a broker whose log +# holds a segment.bytes below the floor does not start. The same limits are in +# packages/apps/kafka/templates/topics.yaml, +# packages/core/platform/templates/_helpers.tpl and +# packages/system/backupstrategy-controller/files/kafka-backup.sh. +# Application values are read from the HelmReleases, the same source as the +# platform chart's guard; KafkaTopics cover topics created outside the chart. +SEGMENT_BYTES_MIN=1048576 +# shellcheck disable=SC2016 # a jq program, expanded by jq +rejected_topic_configs=' + (("message.downconversion.enable", "message.format.version", "message.timestamp.difference.max.ms") + as $key | select($config | has($key)) | "\($owner): topic \($topic) sets \($key)"), + ($config["segment.bytes"] + # Kafka trims characters up to U+0020 and reads the rest with + # Integer.parseInt, which allows a sign and leading zeros. + | (if type == "number" then . + elif type == "string" then + capture("^[\\x00-\\x20]*(?:\\+|(?-))?0*(?[0-9]+)[\\x00-\\x20]*$") + | "\(.sign // "")\(.digits)" | tonumber + else empty end) + | select(. < $min) + | "\($owner): topic \($topic) sets segment.bytes=\(.), below the Kafka 4 minimum of \($min)")' +releases=$(kubectl get helmreleases.helm.toolkit.fluxcd.io --all-namespaces -l apps.cozystack.io/application.kind=Kafka -o json) +topics=$(kubectl get "kafkatopics.$kafka_api.kafka.strimzi.io" --all-namespaces -o json) +rejected=$(printf '%s' "$releases" | jq --raw-output --argjson min "$SEGMENT_BYTES_MIN" ' + .items[] + | "\(.metadata.namespace)/\(.metadata.labels["apps.cozystack.io/application.name"] // .metadata.name)" as $owner + | (.spec.values.topics // []) | if type == "array" then .[] else empty end + | select(type == "object") | .name as $topic + | (.config | if type == "object" then . else {} end) as $config + | '"$rejected_topic_configs") +rejected_crs=$(printf '%s' "$topics" | jq --raw-output --argjson min "$SEGMENT_BYTES_MIN" ' + .items[] + | "\(.metadata.namespace)/kafkatopic \(.metadata.name)" as $owner + | (.spec.topicName // .metadata.name) as $topic + | (.spec.config | if type == "object" then . else {} end) as $config + | '"$rejected_topic_configs") +rejected=$(printf '%s\n%s' "$rejected" "$rejected_crs" | sed '/^$/d') +if [ -n "$rejected" ]; then + echo "ERROR: Kafka 4 rejects topic configs these Kafka applications and KafkaTopics still set:" >&2 + printf '%s\n' "$rejected" | sed 's/^/ /' >&2 + echo "Remove the configs Kafka 4 removed and raise segment.bytes to the minimum before running this migration." >&2 + exit 1 +fi + +# Strimzi's conversion tool refuses some v1beta2 objects outright, and +# convert-resource stops at the first one, with the operators stopped and the +# objects it reached before already converted. These are the refusals of the +# tool in CONVERSION_IMAGE, read from the strimzi-kafka-operator 0.51.0 source +# (v1-api-conversion/src/main/java/io/strimzi/kafka/api/conversion/v1/converter/): +# a missing .spec on every kind, and what KafkaConversions, +# SharedConversions, MirrorMaker2Conversions and the Conversion.move of +# KafkaConnectConverter reject. Re-read them when CONVERSION_IMAGE moves off +# 0.51.0; a Bats case fails on that change. The tool cannot do a dry run here +# instead: it is Java, and this image has no JVM. Once the CRDs serve v1 +# alone, every object is stored as v1 and nothing is converted. +# shellcheck disable=SC2016 # a jq program, expanded by jq +conversion_refusals=' + def oauth: .authentication.type? == "oauth"; + .spec as $s + | if $s == null then "no .spec" + elif $crd == "kafkas.kafka.strimzi.io" then + (select($s.kafka.resources != null) | ".spec.kafka.resources (use a KafkaNodePool)"), + (select(any($s.kafka.listeners[]?; oauth)) | "OAuth authentication on a listener"), + (select(any($s.kafka.listeners[]?; .authentication.type? == "custom" and .authentication.secrets != null)) + | "secrets in a custom listener authentication"), + (select($s.kafka.authorization.type? == "keycloak") | "Keycloak authorization"), + (select($s.kafka.authorization.type? == "opa") | "Open Policy Agent authorization") + elif $crd == "kafkabridges.kafka.strimzi.io" then + select($s | oauth) | "OAuth client authentication" + elif $crd == "kafkaconnects.kafka.strimzi.io" then + (select($s | oauth) | "OAuth client authentication"), + (select($s.externalConfiguration != null) | ".spec.externalConfiguration"), + (select($s.build.output.additionalKanikoOptions != null and $s.build.output.additionalBuildOptions != null) + | "both .spec.build.output.additionalKanikoOptions and additionalBuildOptions") + elif $crd == "kafkamirrormaker2s.kafka.strimzi.io" then + [$s.clusters[]?.alias] as $aliases + | (if $s.target != null then $s.target.alias else $s.connectCluster end) as $target + | (select($s.externalConfiguration != null) | ".spec.externalConfiguration"), + (select(any($s.clusters[]?, $s.target, $s.mirrors[]?.source; oauth)) | "OAuth client authentication"), + (select($s.target == null and ($aliases == [] or $s.connectCluster == null)) + | "no target cluster in .spec.target, or in .spec.connectCluster and .spec.clusters"), + (select($s.target == null and $aliases != [] and $s.connectCluster != null and + ($aliases | index([$s.connectCluster])) == null) + | ".spec.connectCluster is not an alias in .spec.clusters"), + ($s.mirrors[]? | + (select(.targetCluster != null and .targetCluster != $target) + | "a mirror whose targetCluster is not the target cluster"), + (select(.source == null and ($aliases == [] or .sourceCluster == null)) + | "a mirror without .source or .sourceCluster"), + (select(.source == null and $aliases != [] and .sourceCluster != null and + (.sourceCluster as $source | $aliases | index([$source])) == null) + | "a mirror whose sourceCluster is not an alias in .spec.clusters")) + else empty end' +case " $served " in + *" v1beta2 "*) + refused= + for c in $STRIMZI_CRDS; do + list=$(kubectl get "${c%%.*}.$kafka_api.${c#*.}" --all-namespaces -o json) + found=$(printf '%s' "$list" | jq --raw-output --arg crd "$c" '.items[] + | "\($crd) \(.metadata.namespace)/\(.metadata.name)" as $object + | [ '"$conversion_refusals"' ] | unique | .[] | "\($object): \(.)"') + refused=$(printf '%s\n%s' "$refused" "$found" | sed '/^$/d') + done + if [ -n "$refused" ]; then + echo "ERROR: the Strimzi v1 API conversion refuses these objects:" >&2 + printf '%s\n' "$refused" | sed 's/^/ /' >&2 + echo "Change each one as the Strimzi v1 API requires and run this migration again." >&2 + exit 1 + fi + ;; +esac + +# Record the proof before anything converts, so a retry after the conversion +# accepts a Kafka that passed only on its KRaft state. +printf '%s' "$kafkas" | jq --raw-output --arg verified "$KRAFT_ANNOTATION" '.items[] + | select(.metadata.annotations[$verified] | type == "string" and length > 0 | not) + | "\(.metadata.namespace) \(.metadata.name) \(.status.kafkaMetadataVersion)"' | +while read -r ns name version; do + kubectl -n "$ns" annotate "kafkas.$kafka_api.kafka.strimzi.io" "$name" --overwrite "$KRAFT_ANNOTATION=$version" +done + +# A Job failed on its deadline can still have a pod terminating, and +# convert-resource writes until it is gone. A foreground deletion returns +# only once the Job's pods are. +if [ -n "$job_state" ]; then + kubectl -n "$CONVERSION_NS" delete job "$CONVERSION_NAME" --ignore-not-found --cascade=foreground --wait=true +fi + +if stored_v1_only; then + echo "Every Strimzi CRD already stores only v1; skipping the conversion" + # The 0.45 operator can only still be running while the CRDs serve + # v1beta2. Once they serve v1 alone the operator is 1.x, and a run by hand + # must not stop it: no Helm upgrade would bring it back. + case " $served " in + *" v1beta2 "*) stop_old_operator ;; + esac + remove_conversion_job +else + if [ ! -r "$STRIMZI_CRD_FILE" ]; then + echo "ERROR: $STRIMZI_CRD_FILE is not readable; set STRIMZI_CRD_FILE to strimzi-crds-0.51.0.yaml from the Strimzi 0.51.0 release" >&2 + exit 1 + fi + stop_old_operator + # These CRDs mark v1beta2 as the storage version, so a re-run after a + # partial crd-upgrade switches storage back and converts from the start. + echo "Applying the Strimzi 0.51.0 CRDs, which serve both v1beta2 and v1" + kubectl apply --server-side --force-conflicts --field-manager=cozystack-migration-61 -f "$STRIMZI_CRD_FILE" + for c in $STRIMZI_CRDS; do + kubectl wait --for=condition=Established --timeout=120s "crd/$c" + done + + # Whatever an earlier attempt left has finished by now, and its result was + # undone by the CRD apply above, so the conversion starts over. + kubectl -n "$CONVERSION_NS" delete job "$CONVERSION_NAME" --ignore-not-found --wait=true + kubectl apply -f - <&2 + for c in $STRIMZI_CRDS; do + echo " $c: $(kubectl get crd "$c" --ignore-not-found -o jsonpath='{.status.storedVersions}')" >&2 + done + remove_conversion_rbac + exit 1 + fi + + remove_conversion_job +fi + +# Moving the applications off Kafka 3.x first makes the old chart reject +# their values, so no Helm action can store a new v1beta2 revision behind the +# rewrite below. +# kubectl api-resources fails when any API group fails discovery, so only +# this group is read: NotFound means it is not installed, any other error +# aborts. +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +if app_group=$(kubectl get --raw /apis/apps.cozystack.io/v1alpha1 2>"$work/app-group.err"); then + kafka_apps=$(printf '%s' "$app_group" | jq --raw-output 'any(.resources[]?; .name == "kafkas")') +elif grep -qF '(NotFound)' "$work/app-group.err"; then + kafka_apps=false +else + echo "ERROR: reading the apps.cozystack.io API failed: $(cat "$work/app-group.err")" >&2 + exit 1 +fi +has_kafka_apps() { [ "$kafka_apps" = true ]; } +if has_kafka_apps; then + kubectl get kafkas.apps.cozystack.io --all-namespaces \ + -o jsonpath='{range .items[*]}{.metadata.namespace}{" "}{.metadata.name}{" "}{.spec.version}{"\n"}{end}' | + while read -r ns name version; do + case "$version" in + v3.8|v3.9) + echo "Moving Kafka $ns/$name from $version to $KAFKA_TARGET_VERSION" + kubectl -n "$ns" patch kafkas.apps.cozystack.io "$name" --type merge \ + -p "{\"spec\":{\"version\":\"$KAFKA_TARGET_VERSION\"}}" + ;; + esac + done +fi + +# Helm builds the objects of a stored release before it upgrades, +# uninstalls or rolls back to it, and fails on an API version the server no +# longer serves, so every Kafka release would be stuck once the v1-only CRDs +# land. Uninstall reads the latest revision whatever its status, and upgrade +# falls back to the last failed one, so every revision of each release is +# rewritten, the way helm-mapkubeapis does: the Secret holds base64 of +# gzipped JSON, inside the base64 of the Secret data. A release whose install +# failed has no Kafka CR, so the releases are listed from the applications as +# well. The chart's hooks carry no Strimzi kinds, so only the manifest needs +# the rewrite. + +list_releases() { + kubectl get kafkas.v1.kafka.strimzi.io --all-namespaces \ + -o jsonpath='{range .items[*]}{.metadata.namespace}{" "}{.metadata.annotations.meta\.helm\.sh/release-name}{"\n"}{end}' > "$work/releases.all" + if has_kafka_apps; then + kubectl get kafkas.apps.cozystack.io --all-namespaces \ + -o jsonpath='{range .items[*]}{.metadata.namespace}{" kafka-"}{.metadata.name}{"\n"}{end}' >> "$work/releases.all" + fi + sort -u "$work/releases.all" > "$work/releases" + } + +# scan_releases rewrite|check: rewrite every revision that still names +# v1beta2, or list the ones that do and the ones still pending. +scan_releases() { + list_releases + while read -r ns release; do + [ -n "$release" ] || continue + secrets=$(kubectl -n "$ns" get secret -l "owner=helm,name=$release" -o jsonpath='{.items[*].metadata.name}') + for secret in $secrets; do + kubectl -n "$ns" get secret "$secret" -o jsonpath='{.data.release}' | base64 -d | base64 -d | gzip -dc > "$work/release.json" + if [ "$1" = check ]; then + status=$(jq -r '.info.status' "$work/release.json") + case "$status" in + pending-*) + echo " Helm release $ns/$secret is $status" + continue + ;; + esac + fi + if ! jq -e '.manifest | contains("apiVersion: kafka.strimzi.io/v1beta2")' "$work/release.json" > /dev/null; then + continue + fi + if [ "$1" = check ]; then + echo " Helm release $ns/$secret" + continue + fi + echo "Rewriting Helm release $ns/$secret to kafka.strimzi.io/v1" + jq -c '.manifest |= gsub("apiVersion: kafka.strimzi.io/v1beta2\n"; "apiVersion: kafka.strimzi.io/v1\n")' "$work/release.json" | + gzip -c | base64 | tr -d '\n' | base64 | tr -d '\n' > "$work/release.b64" + jq -n --rawfile r "$work/release.b64" '{data: {release: $r}}' > "$work/patch.json" + kubectl -n "$ns" patch secret "$secret" --type merge --patch-file "$work/patch.json" + done + done < "$work/releases" + } + +# The objects keep managedFields entries recorded under the old API versions, +# and once the CRDs serve only v1 the API server refuses every server-side +# apply to such an object ("request to convert CR to an invalid group/version", +# kubernetes/kubernetes#111937). helm-controller recovers by wiping the +# entries, and the fields Helm owned pass to "before-first-apply", so a field a +# later chart drops is never removed. Rewrite the entries to v1 while both +# versions are still served; the conversion is None, so field paths are +# spelled the same under either, and an entry for a field v1 dropped only +# names a field that no longer exists. +# +# The patch replaces the whole managedFields list, so a patch built from a +# stale read could drop an entry a concurrent writer added in between. The +# patch carries the resourceVersion it was built from: the API server then +# answers a concurrent change with a conflict, and the object is read again. +stale_fields='[.metadata.managedFields[]? | select(.apiVersion | test("/v1(alpha1|beta1|beta2)$"))] | length > 0' + +# list_stale_managed : the objects of that kind with entries to rewrite. +list_stale_managed() { + kubectl get "$1" --all-namespaces --show-managed-fields -o json | + jq -r ".items[] | select($stale_fields) | \"\\(.metadata.namespace) \\(.metadata.name)\"" > "$work/managed" + } + +rewrite_managed() { + for c in $STRIMZI_CRDS; do + list_stale_managed "$c" + while read -r ns name; do + echo "Moving the managedFields of $c $ns/$name to v1" + attempt=0 + while :; do + kubectl -n "$ns" get "$c" "$name" --show-managed-fields -o json > "$work/object.json" + if ! jq -e "$stale_fields" "$work/object.json" > /dev/null; then + break + fi + jq '{metadata: {resourceVersion: .metadata.resourceVersion, managedFields: (.metadata.managedFields | map(.apiVersion |= sub("/v1(alpha1|beta1|beta2)$"; "/v1")))}}' \ + "$work/object.json" > "$work/managed-patch.json" + if kubectl -n "$ns" patch "$c" "$name" --type merge --patch-file "$work/managed-patch.json" 2> "$work/patch.err"; then + break + fi + attempt=$((attempt + 1)) + if ! grep -q 'the object has been modified' "$work/patch.err" || [ "$attempt" -ge 5 ]; then + cat "$work/patch.err" >&2 + exit 1 + fi + done + done < "$work/managed" + done + } + +scan_releases rewrite +rewrite_managed + +# A Helm action already running when the releases were rewritten keeps the +# releases it read in memory and stores them when it finishes: its own +# revision, and the previous one as superseded, both naming v1beta2 again. +# Scan once more and refuse to stamp the version while a revision is still +# pending or names v1beta2: the hook then fails and runs the migration again. +scan_releases check > "$work/leftover" +for c in $STRIMZI_CRDS; do + list_stale_managed "$c" + sed "s|^| managedFields of $c |" "$work/managed" >> "$work/leftover" +done +if [ -s "$work/leftover" ]; then + echo "ERROR: these still name a removed Strimzi API version, or a Helm action on them has not finished:" >&2 + cat "$work/leftover" >&2 + echo "A pending release clears once helm-controller finishes or unlocks that HelmRelease action." >&2 + exit 1 +fi + +stamp_cozystack_version 62 diff --git a/packages/core/platform/values.yaml b/packages/core/platform/values.yaml index 8ea4a054a..bb690233c 100644 --- a/packages/core/platform/values.yaml +++ b/packages/core/platform/values.yaml @@ -14,7 +14,7 @@ migrations: # refuses to advance past a migration file missing from the image (exit 1), so # a stale pin fails loudly rather than silently skipping the etcd adoption. image: ghcr.io/cozystack/cozystack/platform-migrations:v1.6.0@sha256:6777042e3e9c6bad76e7d96fb64baa115061975f39061cb3ede84e21d0e2213f - targetVersion: 61 + targetVersion: 62 # Adopt legacy etcd clusters onto the v1alpha2 operator WITHOUT the mandatory # pre-adoption safety snapshot (migration 50). Leave false unless migration 50 # has actually refused to proceed and you accept the risk: adoption rewires