diff --git a/hack/migration-60-strimzi-v1-conversion.bats b/hack/migration-60-strimzi-v1-conversion.bats new file mode 100644 index 000000000..362695af2 --- /dev/null +++ b/hack/migration-60-strimzi-v1-conversion.bats @@ -0,0 +1,513 @@ +#!/usr/bin/env bats +# ----------------------------------------------------------------------------- +# Unit tests for platform migration 60 --> 61 (move the Strimzi custom +# resources to the v1 API, refuse ZooKeeper-based Kafkas, move Kafka 3.x +# applications to v4.3). See +# packages/core/platform/images/migrations/migrations/60 for the mechanism. +# +# The fake kubectl keeps the storedVersions every Strimzi CRD reports in a +# state file, and the conversion Job rewrites it when it succeeds, so the +# check after the Job reads what the Job did. +# +# The harness (docker, jq baked onto the migrations base image, the explicit +# `return` in run_migration) is the one hack/migration-55-fluxcd-orphan.bats +# describes; see there for why each piece is load-bearing. +# +# Run with: hack/cozytest.sh hack/migration-60-strimzi-v1-conversion.bats +# ----------------------------------------------------------------------------- + +load test_helper + +FIXTURES="$PWD/hack/testdata/migration-60-strimzi-v1" +FAKE_FIXTURES="$FIXTURES" +MIG_DIR="$PWD/packages/core/platform/images/migrations/migrations" +ALPINE=$(sed -n 's/^FROM \(alpine:[^ ]*\).*$/\1/p' \ + "$PWD/packages/core/platform/images/migrations/Dockerfile" | head -1) +TESTIMG="cozystack-migration60-test:$(printf '%s' "$ALPINE" | sed 's/[^a-zA-Z0-9]/-/g')" +WORKROOT="${TMPDIR:-/tmp}/cozy-migration-60-$$" + +cozy_cleanup() { + rm -rf "$WORKROOT" + return 0 +} + +# The wait for the conversion Job counts only its sleeps, so a zero poll +# interval would let a wait that misses the end of the Job spin for good. A +# one-second poll under a three-second timeout turns that into a quick failure. +run_migration() { + _run_migration_rc=0 + docker run --rm --network none \ + --user "$(id -u):$(id -g)" \ + -v "$MIG_DIR:/migrations:ro" \ + -v "$FIXTURES:/fakebin:ro" \ + -v "$WORK:/work" \ + -e PATH=/fakebin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + -e FAKE_CMDLOG=/work/cmdlog \ + -e FAKE_STATE=/work/state \ + -e FAKE_APPS=/fakebin/apps.txt \ + -e FAKE_RELEASES=/fakebin/kafka-releases.txt \ + -e FAKE_KAFKAS="/fakebin/${FAKE_KAFKAS_FILE:-kafkas-kraft.json}" \ + -e FAKE_NO_STRIMZI="${FAKE_NO_STRIMZI-}" \ + -e FAKE_JOB="${FAKE_JOB-}" \ + -e FAKE_PATCH_FAIL="${FAKE_PATCH_FAIL-}" \ + -e FAKE_MANAGED_CONFLICT="${FAKE_MANAGED_CONFLICT-}" \ + -e FAKE_REBALANCES="${FAKE_REBALANCES-}" \ + -e FAKE_CRD_READ_FAIL="${FAKE_CRD_READ_FAIL-}" \ + -e FAKE_APIRES_FAIL="${FAKE_APIRES_FAIL-}" \ + -e FAKE_SECRET_LIST_FAIL="${FAKE_SECRET_LIST_FAIL-}" \ + -e FAKE_RELEASE_REWRITTEN_BACK="${FAKE_RELEASE_REWRITTEN_BACK-}" \ + -e FAKE_OLD_JOB="${FAKE_OLD_JOB-}" \ + -e FAKE_REBALANCE_FAIL="${FAKE_REBALANCE_FAIL-}" \ + -e FAKE_OPERATOR_READ_FAIL="${FAKE_OPERATOR_READ_FAIL-}" \ + -e FAKE_SERVED="${FAKE_SERVED-}" \ + -e FAKE_MANAGED_PATCH_FAIL="${FAKE_MANAGED_PATCH_FAIL-}" \ + -e FAKE_PULL="${FAKE_PULL-}" \ + -e STRIMZI_CRD_FILE="${STRIMZI_CRD_FILE-/fakebin/strimzi-crds-0.51.0.yaml}" \ + -e POLL_INTERVAL=1 \ + -e CONVERSION_TIMEOUT="${CONVERSION_TIMEOUT-3}" \ + "$TESTIMG" "/migrations/$1" || _run_migration_rc=$? + return "$_run_migration_rc" +} + +prep() { + docker info >/dev/null 2>&1 || { + echo "docker is required: these tests run migration 60 inside a jq-enabled" >&2 + echo "build of $ALPINE, the migrations image's base." >&2 + return 1 + } + docker build -q -t "$TESTIMG" - >/dev/null < "$WORK/state/stored" + FAKE_CMDLOG="$WORK/cmdlog" + : > "$FAKE_CMDLOG" + unset FAKE_NO_STRIMZI FAKE_JOB FAKE_PATCH_FAIL FAKE_KAFKAS_FILE FAKE_MANAGED_CONFLICT FAKE_MANAGED_PATCH_FAIL FAKE_REBALANCES STRIMZI_CRD_FILE FAKE_CRD_READ_FAIL FAKE_APIRES_FAIL FAKE_SECRET_LIST_FAIL FAKE_RELEASE_REWRITTEN_BACK FAKE_OLD_JOB FAKE_REBALANCE_FAIL FAKE_OPERATOR_READ_FAIL FAKE_SERVED FAKE_PULL CONVERSION_TIMEOUT || true + return 0 +} + +release_json() { + base64 -d < "$WORK/state/release-$1" | base64 -d | gzip -dc +} + +@test "converts to v1, moves Kafka 3.x applications to v4.3 and stamps 61" { + prep + rc=0 + run_migration 60 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + + grep -qxF "APPLY-CRDS /fakebin/strimzi-crds-0.51.0.yaml" "$FAKE_CMDLOG" + # convert-resource is the init container, so crd-upgrade cannot start first. + grep -qxF "APPLY-CONVERSION convert-resource crd-upgrade" "$FAKE_CMDLOG" + crds_line=$(grep -nxF "APPLY-CRDS /fakebin/strimzi-crds-0.51.0.yaml" "$FAKE_CMDLOG" | cut -d: -f1) + job_line=$(grep -nxF "APPLY-CONVERSION convert-resource crd-upgrade" "$FAKE_CMDLOG" | cut -d: -f1) + [ "$crds_line" -lt "$job_line" ] + # The 0.45 operator is stopped before anything is converted. + scale_line=$(grep -nxF "SCALE-OPERATOR 0" "$FAKE_CMDLOG" | cut -d: -f1) + [ "$scale_line" -lt "$crds_line" ] + # The entity operators only after the cluster operator, or it scales them back. + entity_line=$(grep -nxF "SCALE-ENTITY tenant-a/kafka-one-entity-operator" "$FAKE_CMDLOG" | cut -d: -f1) + [ "$scale_line" -lt "$entity_line" ] + [ "$entity_line" -lt "$crds_line" ] + + grep -qxF 'PATCH tenant-a/one {"spec":{"version":"v4.3"}}' "$FAKE_CMDLOG" + grep -qxF 'PATCH tenant-b/two {"spec":{"version":"v4.3"}}' "$FAKE_CMDLOG" + if grep -qE "^PATCH tenant-(c|d)/" "$FAKE_CMDLOG"; then echo "patched an application that needs no version change" >&2; return 1; fi + + grep -qxF "DELETE clusterrole strimzi-v1-api-conversion" "$FAKE_CMDLOG" + [ "$(tail -1 "$FAKE_CMDLOG")" = "STAMP 61" ] + rm -rf "$WORK" +} + +# Helm builds the objects of the deployed release before it upgrades or +# uninstalls it, and fails on an API version the server no longer serves. +@test "rewrites the deployed Helm release of each Kafka to the v1 API" { + prep + rc=0 + run_migration 60 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + + grep -qxF "PATCH-RELEASE tenant-a/sh.helm.release.v1.kafka-one.v3" "$FAKE_CMDLOG" + grep -qxF "PATCH-RELEASE tenant-a/sh.helm.release.v1.kafka-one.v4" "$FAKE_CMDLOG" + # An application whose install failed has a release and no Kafka CR. + grep -qxF "PATCH-RELEASE tenant-f/sh.helm.release.v1.kafka-failed-install.v1" "$FAKE_CMDLOG" + # The applications leave Kafka 3.x before any release is rewritten. + version_line=$(grep -nF 'PATCH tenant-a/one ' "$FAKE_CMDLOG" | cut -d: -f1) + first_release_line=$(grep -nF 'PATCH-RELEASE ' "$FAKE_CMDLOG" | head -1 | cut -d: -f1) + [ "$version_line" -lt "$first_release_line" ] + release_json v4 > "$WORK/release.json" + [ "$(jq -r '.manifest' "$WORK/release.json" | grep -c '^apiVersion: kafka.strimzi.io/v1$')" -eq 2 ] + if jq -r '.manifest' "$WORK/release.json" | grep -qF 'kafka.strimzi.io/v1beta2'; then echo "v1beta2 left in the release manifest" >&2; return 1; fi + # Only the manifest is rewritten; everything else in the release stays as it was. + jq 'del(.manifest)' "$FAKE_FIXTURES/release-kafka-one.json" > "$WORK/want" + jq 'del(.manifest)' "$WORK/release.json" > "$WORK/got" + cmp "$WORK/want" "$WORK/got" + release_line=$(grep -nxF "PATCH-RELEASE tenant-a/sh.helm.release.v1.kafka-one.v3" "$FAKE_CMDLOG" | cut -d: -f1) + stamp_line=$(grep -nxF "STAMP 61" "$FAKE_CMDLOG" | cut -d: -f1) + [ "$release_line" -lt "$stamp_line" ] + + : > "$FAKE_CMDLOG" + rc=0 + run_migration 60 >"$WORK/out" 2>&1 || rc=$? + [ "$rc" -eq 0 ] + if grep -qF "PATCH-RELEASE" "$FAKE_CMDLOG"; then echo "a second run rewrote the release again" >&2; return 1; fi + rm -rf "$WORK" +} + +# Once the CRDs serve only v1, the API server refuses a server-side apply to +# an object whose managedFields still name v1beta2. +@test "moves managedFields entries recorded under old API versions to v1" { + prep + rc=0 + run_migration 60 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + grep -qxF "PATCH-MANAGED tenant-a/kafka-one rv=100 helm-controller=kafka.strimzi.io/v1,strimzi-cluster-operator=kafka.strimzi.io/v1" "$FAKE_CMDLOG" + # The status entry keeps its subresource and its fields. + [ "$(jq -r '.items[0].metadata.managedFields[1].subresource' "$WORK/state/managed")" = "status" ] + if grep -qF "PATCH-MANAGED tenant-b/" "$FAKE_CMDLOG"; then echo "patched an object already on v1" >&2; return 1; fi + + : > "$FAKE_CMDLOG" + rc=0 + run_migration 60 >"$WORK/out" 2>&1 || rc=$? + [ "$rc" -eq 0 ] + if grep -qF "PATCH-MANAGED" "$FAKE_CMDLOG"; then echo "a second run patched managedFields again" >&2; return 1; fi + rm -rf "$WORK" +} + +@test "a managedFields patch that loses a race is rebuilt from a fresh read" { + prep + export FAKE_MANAGED_CONFLICT=1 + rc=0 + run_migration 60 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + grep -qxF "PATCH-MANAGED tenant-a/kafka-one rv=100 helm-controller=kafka.strimzi.io/v1,strimzi-cluster-operator=kafka.strimzi.io/v1" "$FAKE_CMDLOG" + grep -qxF "PATCH-MANAGED tenant-a/kafka-one rv=101 helm-controller=kafka.strimzi.io/v1,strimzi-cluster-operator=kafka.strimzi.io/v1" "$FAKE_CMDLOG" + [ "$(jq -r '.items[0].metadata.managedFields[0].apiVersion' "$WORK/state/managed")" = "kafka.strimzi.io/v1" ] + grep -qxF "STAMP 61" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +@test "a managedFields patch that fails for another reason aborts before stamping" { + prep + export FAKE_MANAGED_PATCH_FAIL=1 + rc=0 + run_migration 60 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + [ "$(grep -c "PATCH-MANAGED" "$FAKE_CMDLOG")" -eq 1 ] + if grep -qF "STAMP" "$FAKE_CMDLOG"; then echo "stamped after a failed managedFields patch" >&2; return 1; fi + rm -rf "$WORK" +} + +@test "a cluster without Strimzi only stamps the version" { + prep + export FAKE_NO_STRIMZI=1 + rc=0 + run_migration 60 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + if grep -qE "APPLY-CRDS|APPLY-CONVERSION|PATCH" "$FAKE_CMDLOG"; then echo "touched Strimzi on a cluster without it" >&2; return 1; fi + grep -qxF "STAMP 61" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +@test "a ZooKeeper-based or still-migrating Kafka stops the migration before it changes anything" { + prep + export FAKE_KAFKAS_FILE=kafkas-zk.json + rc=0 + run_migration 60 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + grep -qxF " tenant-c/kafka-zk" "$WORK/out" + grep -qxF " tenant-d/kafka-mid" "$WORK/out" + if grep -qF "tenant-a/kafka-one" "$WORK/out"; then echo "named a KRaft cluster as ZooKeeper-based" >&2; return 1; fi + grep -qF "Cozystack v1.7" "$WORK/out" + if grep -qE "APPLY-CRDS|APPLY-CONVERSION|PATCH|STAMP|SCALE-OPERATOR" "$FAKE_CMDLOG"; then echo "changed the cluster despite a ZooKeeper Kafka" >&2; return 1; fi + rm -rf "$WORK" +} + +@test "a drain of the v1.7 ZooKeeper migration still in flight stops the migration" { + prep + export FAKE_REBALANCES="tenant-a/kafka-one-drain-kafka,tenant-a/my-own-rebalance" + rc=0 + run_migration 60 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + grep -qxF " tenant-a/kafka-one-drain-kafka" "$WORK/out" + if grep -qxF " tenant-a/my-own-rebalance" "$WORK/out"; then echo "named an unrelated KafkaRebalance" >&2; return 1; fi + if grep -qE "SCALE-OPERATOR|APPLY-CRDS|STAMP" "$FAKE_CMDLOG"; then echo "changed the cluster despite an unfinished drain" >&2; return 1; fi + rm -rf "$WORK" +} + +# Run by hand outside the migrations image, the CRD file is easy to miss; +# the operators must not be stopped for a run that cannot go on. +@test "a missing CRD file aborts before the operators are stopped" { + prep + export STRIMZI_CRD_FILE=/nonexistent/strimzi-crds-0.51.0.yaml + rc=0 + run_migration 60 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + grep -qF "/nonexistent/strimzi-crds-0.51.0.yaml is not readable" "$WORK/out" + if grep -qE "SCALE-|APPLY-CRDS|STAMP" "$FAKE_CMDLOG"; then echo "changed the cluster without the CRD file" >&2; return 1; fi + rm -rf "$WORK" +} + +@test "a failed conversion Job aborts with its logs, before any application changes" { + prep + export FAKE_JOB=fail + rc=0 + run_migration 60 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + grep -qF "fake conversion log" "$WORK/out" + # The failure ends the wait at once, not at the timeout. + [ "$(grep -cF "get job strimzi-v1-api-conversion -o jsonpath={.status.succeeded}" "$FAKE_CMDLOG")" -eq 1 ] + if grep -qE "PATCH|STAMP" "$FAKE_CMDLOG"; then echo "went on after the conversion failed" >&2; return 1; fi + rm -rf "$WORK" +} + +@test "a Job that leaves a non-v1 stored version behind aborts" { + prep + export FAKE_JOB=noop + rc=0 + run_migration 60 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + grep -qF 'kafkas.kafka.strimzi.io: ["v1beta2"]' "$WORK/out" + if grep -qE "PATCH|STAMP" "$FAKE_CMDLOG"; then echo "went on with v1beta2 still stored" >&2; return 1; fi + rm -rf "$WORK" +} + +@test "a re-run after the conversion skips it and still moves the versions" { + prep + printf '["v1"]' > "$WORK/state/stored" + rc=0 + run_migration 60 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + if grep -qE "APPLY-CRDS|APPLY-CONVERSION|kafkas.v1beta2" "$FAKE_CMDLOG"; then echo "converted again" >&2; return 1; fi + # The CRDs still serve v1beta2, so the operator may be 0.45: it is stopped. + grep -qxF "SCALE-OPERATOR 0" "$FAKE_CMDLOG" + grep -qxF 'PATCH tenant-a/one {"spec":{"version":"v4.3"}}' "$FAKE_CMDLOG" + grep -qxF "STAMP 61" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +@test "a failing application patch aborts before stamping" { + prep + export FAKE_PATCH_FAIL=1 + rc=0 + run_migration 60 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + if grep -qF "STAMP" "$FAKE_CMDLOG"; then echo "stamped after a failed patch" >&2; return 1; fi + rm -rf "$WORK" +} + +# The migration runs once: a read that fails must stop it before the stamp, +# never pass for "nothing there". +@test "a failed read of the CRD stored versions aborts before stamping" { + prep + export FAKE_CRD_READ_FAIL=1 + rc=0 + run_migration 60 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + if grep -qE "STAMP|SCALE-|APPLY-CRDS|PATCH" "$FAKE_CMDLOG"; then echo "went on after a failed CRD read" >&2; return 1; fi + rm -rf "$WORK" +} + +@test "a failed read of the API resources aborts before stamping" { + prep + export FAKE_APIRES_FAIL=1 + rc=0 + run_migration 60 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + if grep -qE "STAMP|PATCH-RELEASE" "$FAKE_CMDLOG"; then echo "went on after a failed API resources read" >&2; return 1; fi + rm -rf "$WORK" +} + +@test "a failed Helm release listing aborts before stamping" { + prep + export FAKE_SECRET_LIST_FAIL=1 + rc=0 + run_migration 60 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + if grep -qF "STAMP" "$FAKE_CMDLOG"; then echo "stamped after a failed release listing" >&2; return 1; fi + rm -rf "$WORK" +} + +@test "a release that names v1beta2 again after the rewrite stops the stamp" { + prep + export FAKE_RELEASE_REWRITTEN_BACK=1 + rc=0 + run_migration 60 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + grep -qxF " Helm release tenant-a/sh.helm.release.v1.kafka-one.v3" "$WORK/out" + if grep -qF "STAMP" "$FAKE_CMDLOG"; then echo "stamped with a v1beta2 release left" >&2; return 1; fi + rm -rf "$WORK" +} + +# A Job failed on its deadline can still have a pod terminating, and +# convert-resource writes until that pod is gone. +@test "a finished Job of an earlier attempt is removed with its pods before the CRDs are touched" { + for state in failed succeeded; do + prep + export FAKE_OLD_JOB=$state + rc=0 + run_migration 60 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + delete_line=$(grep -nxF "DELETE job strimzi-v1-api-conversion foreground" "$FAKE_CMDLOG" | head -1 | cut -d: -f1) + scale_line=$(grep -nxF "SCALE-OPERATOR 0" "$FAKE_CMDLOG" | cut -d: -f1) + crds_line=$(grep -nF "APPLY-CRDS" "$FAKE_CMDLOG" | cut -d: -f1) + job_line=$(grep -nF "APPLY-CONVERSION" "$FAKE_CMDLOG" | cut -d: -f1) + [ "$delete_line" -lt "$scale_line" ] + [ "$delete_line" -lt "$crds_line" ] + [ "$crds_line" -lt "$job_line" ] + rm -rf "$WORK" + done +} + +@test "a Job of an earlier attempt that fails while waited for is removed with its pods and started over" { + prep + export FAKE_OLD_JOB=running FAKE_JOB=fail + rc=0 + run_migration 60 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + # The new Job fails as well, so the run stops after starting it. + [ "$rc" -ne 0 ] + # Each failure ends its wait at once, not at the timeout. + [ "$(grep -cF "get job strimzi-v1-api-conversion -o jsonpath={.status.succeeded}" "$FAKE_CMDLOG")" -eq 2 ] + delete_line=$(grep -nxF "DELETE job strimzi-v1-api-conversion foreground" "$FAKE_CMDLOG" | head -1 | cut -d: -f1) + crds_line=$(grep -nF "APPLY-CRDS" "$FAKE_CMDLOG" | cut -d: -f1) + [ "$delete_line" -lt "$crds_line" ] + grep -qF "APPLY-CONVERSION" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +# Without a deadline, a Job whose pod never starts never fails, and every +# retry waits on it and exits. The deadline must cover the wait of the run +# that started the Job plus the wait of the retry after it. +@test "the conversion Job carries a deadline of at least two waits for it" { + prep + export CONVERSION_TIMEOUT=100 + rc=0 + run_migration 60 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + deadline=$(sed -n 's/^CONVERSION-DEADLINE //p' "$FAKE_CMDLOG") + [ -n "$deadline" ] + [ "$deadline" -ge 200 ] + rm -rf "$WORK" +} + +# A pod the API server refused never fails, so the Job reports its deadline +# only in its Failed condition. +@test "a Job of an earlier attempt failed on its deadline without a pod is started over" { + prep + export FAKE_OLD_JOB=running FAKE_JOB=nopod CONVERSION_TIMEOUT=0 + rc=0 + run_migration 60 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + if grep -qF "is still running; retry once it finishes" "$WORK/out"; then echo "took a failed Job for a running one" >&2; return 1; fi + delete_line=$(grep -nxF "DELETE job strimzi-v1-api-conversion foreground" "$FAKE_CMDLOG" | head -1 | cut -d: -f1) + crds_line=$(grep -nF "APPLY-CRDS" "$FAKE_CMDLOG" | cut -d: -f1) + [ "$delete_line" -lt "$crds_line" ] + grep -qF "APPLY-CONVERSION" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +@test "a conversion pod that cannot pull its image is named, with what to do" { + prep + export FAKE_JOB=hang FAKE_PULL=1 CONVERSION_TIMEOUT=0 + rc=0 + run_migration 60 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + grep -qF "cannot pull quay.io/strimzi/operator:0.51.0@sha256:" "$WORK/out" + grep -qF "mirror" "$WORK/out" + if grep -qE "PATCH|STAMP" "$FAKE_CMDLOG"; then echo "went on without a conversion" >&2; return 1; fi + rm -rf "$WORK" +} + +# A retry after the hook timed out must not touch the CRDs under a Job that +# is still converting. +@test "a Job of an earlier attempt still running is waited for before the CRDs are touched" { + prep + export FAKE_OLD_JOB=running + rc=0 + run_migration 60 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + grep -qF "still running" "$WORK/out" + # It converted everything, so there is nothing left to apply or start. + if grep -qE "APPLY-CRDS|APPLY-CONVERSION" "$FAKE_CMDLOG"; then echo "touched the CRDs under a running Job" >&2; return 1; fi + grep -qxF "DELETE clusterrole strimzi-v1-api-conversion" "$FAKE_CMDLOG" + grep -qxF "STAMP 61" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +@test "a Job of an earlier attempt still running after the wait aborts before the CRDs are touched" { + prep + export FAKE_OLD_JOB=running FAKE_JOB=hang CONVERSION_TIMEOUT=0 + rc=0 + run_migration 60 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + grep -qF "is still running; retry once it finishes" "$WORK/out" + if grep -qE "SCALE-|APPLY-CRDS|DELETE|STAMP" "$FAKE_CMDLOG"; then echo "went on under a running Job" >&2; return 1; fi + rm -rf "$WORK" +} + +@test "a failed read of the KafkaRebalances aborts before the operators are stopped" { + prep + export FAKE_REBALANCE_FAIL=1 + rc=0 + run_migration 60 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + if grep -qE "SCALE-|APPLY-CRDS|STAMP" "$FAKE_CMDLOG"; then echo "went on after a failed KafkaRebalance read" >&2; return 1; fi + rm -rf "$WORK" +} + +# Converting under a 0.45 operator the migration failed to see would let it +# record v1beta2 managedFields again after the rewrite. +@test "a failed read of the cluster operator Deployment aborts before anything is converted" { + prep + export FAKE_OPERATOR_READ_FAIL=1 + rc=0 + run_migration 60 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + if grep -qE "APPLY-CRDS|APPLY-CONVERSION|STAMP" "$FAKE_CMDLOG"; then echo "converted without knowing whether the 0.45 operator runs" >&2; return 1; fi + rm -rf "$WORK" +} + +# Run by hand after the kafka-operator release brought Strimzi 1.x: nothing +# would scale its operator back up. +@test "a re-run once the CRDs serve only v1 leaves the operators running" { + prep + printf '["v1"]' > "$WORK/state/stored" + export FAKE_SERVED=v1 + rc=0 + run_migration 60 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + if grep -qF "SCALE-" "$FAKE_CMDLOG"; then echo "stopped the 1.x operator" >&2; return 1; fi + grep -qxF "STAMP 61" "$FAKE_CMDLOG" + rm -rf "$WORK" +} diff --git a/hack/testdata/migration-60-strimzi-v1/apps.txt b/hack/testdata/migration-60-strimzi-v1/apps.txt new file mode 100644 index 000000000..deb2852f1 --- /dev/null +++ b/hack/testdata/migration-60-strimzi-v1/apps.txt @@ -0,0 +1,4 @@ +tenant-a one v3.9 +tenant-b two v3.8 +tenant-c three v4.2 +tenant-d four diff --git a/hack/testdata/migration-60-strimzi-v1/kafka-releases.txt b/hack/testdata/migration-60-strimzi-v1/kafka-releases.txt new file mode 100644 index 000000000..e617bdcb9 --- /dev/null +++ b/hack/testdata/migration-60-strimzi-v1/kafka-releases.txt @@ -0,0 +1,2 @@ +tenant-a kafka-one +tenant-b diff --git a/hack/testdata/migration-60-strimzi-v1/kafkas-kraft.json b/hack/testdata/migration-60-strimzi-v1/kafkas-kraft.json new file mode 100644 index 000000000..30c547e9e --- /dev/null +++ b/hack/testdata/migration-60-strimzi-v1/kafkas-kraft.json @@ -0,0 +1,4 @@ +{"items":[ + {"metadata":{"namespace":"tenant-a","name":"kafka-one","annotations":{"strimzi.io/kraft":"enabled"}},"spec":{"kafka":{"version":"3.9.0"}},"status":{"kafkaMetadataState":"KRaft"}}, + {"metadata":{"namespace":"tenant-b","name":"kafka-two"},"spec":{"kafka":{"version":"3.8.1"}}} +]} diff --git a/hack/testdata/migration-60-strimzi-v1/kafkas-zk.json b/hack/testdata/migration-60-strimzi-v1/kafkas-zk.json new file mode 100644 index 000000000..21fa1e8b7 --- /dev/null +++ b/hack/testdata/migration-60-strimzi-v1/kafkas-zk.json @@ -0,0 +1,5 @@ +{"items":[ + {"metadata":{"namespace":"tenant-a","name":"kafka-one"},"spec":{"kafka":{"version":"3.9.0"}},"status":{"kafkaMetadataState":"KRaft"}}, + {"metadata":{"namespace":"tenant-c","name":"kafka-zk"},"spec":{"kafka":{"version":"3.9.0"},"zookeeper":{"replicas":3}},"status":{"kafkaMetadataState":"ZooKeeper"}}, + {"metadata":{"namespace":"tenant-d","name":"kafka-mid"},"spec":{"kafka":{"version":"3.9.0"}},"status":{"kafkaMetadataState":"KRaftPostMigration"}} +]} diff --git a/hack/testdata/migration-60-strimzi-v1/kubectl b/hack/testdata/migration-60-strimzi-v1/kubectl new file mode 100755 index 000000000..a544113f4 --- /dev/null +++ b/hack/testdata/migration-60-strimzi-v1/kubectl @@ -0,0 +1,249 @@ +#!/bin/sh +# Fake kubectl for hack/migration-60-strimzi-v1-conversion.bats. It serves only +# the calls migration 60 and lib/cozystack-version.sh make, and logs each one +# to $FAKE_CMDLOG. +# +# FAKE_CMDLOG file to append a one-line record of each call to +# FAKE_STATE directory; its "stored" file holds the storedVersions +# every Strimzi CRD reports, and the conversion Job +# rewrites it to ["v1"] when it succeeds +# FAKE_NO_STRIMZI non-empty => the Kafka CRD is absent +# FAKE_KAFKAS fixture served for the Strimzi Kafka list +# FAKE_JOB succeed (default) | fail | noop (succeeds, converts nothing) +# | hang (neither succeeds nor fails) +# | nopod (Failed condition, no failed pod: the pod was +# never created) +# FAKE_PULL non-empty => the conversion pod cannot pull its image +# FAKE_PATCH_FAIL non-empty => every application patch exits 1 +# FAKE_CRD_READ_FAIL, FAKE_APIRES_FAIL, FAKE_SECRET_LIST_FAIL: that read +# exits 1 +# FAKE_RELEASE_REWRITTEN_BACK: release patches do not stick +# FAKE_REBALANCES comma-separated ns/name served for the KafkaRebalance list +# The deployed Helm release of kafka-one is release-kafka-one.json, encoded +# the way Helm stores it; a patch of its Secret is written to +# $FAKE_STATE/release, so a second run reads what the first one wrote. +set -u + +log() { [ -n "${FAKE_CMDLOG:-}" ] && printf '%s\n' "$*" >> "$FAKE_CMDLOG"; } +log "KUBECTL $*" + +args="$*" +arg_after() { k="$1"; shift; p=""; for a in "$@"; do [ "$p" = "$k" ] && { printf '%s' "$a"; return; }; p="$a"; done; } + +case "$args" in + *"apply --filename -"*) + manifest=$(cat) + v=$(printf '%s\n' "$manifest" | sed -n 's/^ version: "\{0,1\}\([0-9]\{1,\}\)"\{0,1\}$/\1/p' | head -1) + log "STAMP ${v:-unknown}" + exit 0 + ;; + *"apply -f -"*) + manifest=$(cat) + case "$manifest" in + *"kind: Job"*) + order=$(printf '%s\n' "$manifest" | sed -n 's/.*v1-api-conversion.sh", "\([a-z-]*\)".*/\1/p' | tr '\n' ' ') + log "APPLY-CONVERSION ${order% }" + log "CONVERSION-DEADLINE $(printf '%s\n' "$manifest" | sed -n 's/^ activeDeadlineSeconds: \([0-9]*\)$/\1/p')" + exit 0 + ;; + esac + log "APPLY-UNKNOWN" + exit 1 + ;; + *"apply --server-side"*) + log "APPLY-CRDS $(arg_after -f "$@")" + exit 0 + ;; + *"get crd kafkas.kafka.strimzi.io --ignore-not-found -o name"*) + [ -n "${FAKE_NO_STRIMZI:-}" ] || echo "customresourcedefinition.apiextensions.k8s.io/kafkas.kafka.strimzi.io" + exit 0 + ;; + *"get crd kafkas.kafka.strimzi.io -o jsonpath={.spec.versions[*].name}"*) + printf '%s' "${FAKE_SERVED:-v1 v1beta2}" + exit 0 + ;; + *"get crd "*"storedVersions"*) + if [ -n "${FAKE_CRD_READ_FAIL:-}" ]; then echo "Error from server: etcdserver: request timed out" >&2; exit 1; fi + cat "$FAKE_STATE/stored" + exit 0 + ;; + *"get kafkarebalances.v1beta2.kafka.strimzi.io"*) + if [ -n "${FAKE_REBALANCE_FAIL:-}" ]; then echo "Error from server: etcdserver: request timed out" >&2; exit 1; fi + [ -z "${FAKE_REBALANCES:-}" ] || printf '%s\n' "$FAKE_REBALANCES" | tr ',' '\n' + exit 0 + ;; + *"get kafkas.v1beta2.kafka.strimzi.io"*) + cat "$FAKE_KAFKAS" + exit 0 + ;; + *"get kafkas.v1.kafka.strimzi.io"*) + cat "$FAKE_RELEASES" + exit 0 + ;; + *"get secret -l owner=helm,name=kafka-one -o"*) + if [ -n "${FAKE_SECRET_LIST_FAIL:-}" ]; then echo "Error from server (Forbidden): secrets is forbidden" >&2; exit 1; fi + # v4 stands for a failed upgrade: still the latest revision, so uninstall + # builds it. + echo "sh.helm.release.v1.kafka-one.v3 sh.helm.release.v1.kafka-one.v4" + exit 0 + ;; + *"get secret -l owner=helm,name=kafka-failed-install -o"*) + echo "sh.helm.release.v1.kafka-failed-install.v1" + exit 0 + ;; + *"get secret sh.helm.release.v1.kafka-failed-install.v1 -o jsonpath={.data.release}"*) + if [ ! -f "$FAKE_STATE/release-failed" ]; then + gzip -c "$(dirname "$0")/release-kafka-one.json" | base64 | tr -d '\n' | base64 | tr -d '\n' > "$FAKE_STATE/release-failed" + fi + cat "$FAKE_STATE/release-failed" + exit 0 + ;; + *"patch secret sh.helm.release.v1.kafka-failed-install.v1 --type merge --patch-file "*) + jq -r '.data.release' "$(arg_after --patch-file "$@")" > "$FAKE_STATE/release-failed" + log "PATCH-RELEASE tenant-f/sh.helm.release.v1.kafka-failed-install.v1" + exit 0 + ;; + *"get deployment --all-namespaces -l app.kubernetes.io/name=entity-operator"*) + [ -n "${FAKE_NO_OPERATOR:-}" ] || echo "tenant-a kafka-one-entity-operator" + exit 0 + ;; + *"scale deployment kafka-one-entity-operator --replicas=0"*) + log "SCALE-ENTITY tenant-a/kafka-one-entity-operator" + exit 0 + ;; + *"get secret -l owner=helm,name="*) + exit 0 + ;; + *"get secret sh.helm.release.v1.kafka-one.v"*" -o jsonpath={.data.release}"*) + rev=$(arg_after secret "$@"); rev=${rev##*.} + if [ ! -f "$FAKE_STATE/release-$rev" ]; then + gzip -c "$(dirname "$0")/release-kafka-one.json" | base64 | tr -d '\n' | base64 | tr -d '\n' > "$FAKE_STATE/release-$rev" + fi + cat "$FAKE_STATE/release-$rev" + exit 0 + ;; + *"patch secret sh.helm.release.v1.kafka-one.v"*" --type merge --patch-file "*) + rev=$(arg_after secret "$@"); rev=${rev##*.} + # FAKE_RELEASE_REWRITTEN_BACK: a Helm upgrade that was already running + # stores its v1beta2 revision over the rewrite. + [ -n "${FAKE_RELEASE_REWRITTEN_BACK:-}" ] || jq -r '.data.release' "$(arg_after --patch-file "$@")" > "$FAKE_STATE/release-$rev" + log "PATCH-RELEASE tenant-a/sh.helm.release.v1.kafka-one.$rev" + exit 0 + ;; + *"get deployment strimzi-cluster-operator --ignore-not-found -o name"*) + if [ -n "${FAKE_OPERATOR_READ_FAIL:-}" ]; then echo "Error from server: etcdserver: request timed out" >&2; exit 1; fi + [ -n "${FAKE_NO_OPERATOR:-}" ] || echo "deployment.apps/strimzi-cluster-operator" + exit 0 + ;; + *"scale deployment strimzi-cluster-operator --replicas=0"*) + log "SCALE-OPERATOR 0" + exit 0 + ;; + *"wait --for=delete pod -l name=strimzi-cluster-operator"*) + exit 0 + ;; + *"get kafkas.kafka.strimzi.io --all-namespaces --show-managed-fields -o json"*) + if [ -f "$FAKE_STATE/managed" ]; then cat "$FAKE_STATE/managed"; else cat "$(dirname "$0")/managed-kafkas.json"; fi + exit 0 + ;; + *"--all-namespaces --show-managed-fields -o json"*) + echo '{"items":[]}' + exit 0 + ;; + *"get kafkas.kafka.strimzi.io "*" --show-managed-fields -o json"*) + ns=$(arg_after -n "$@"); name=$(arg_after kafkas.kafka.strimzi.io "$@") + src="$FAKE_STATE/managed"; [ -f "$src" ] || src="$(dirname "$0")/managed-kafkas.json" + jq --arg ns "$ns" --arg n "$name" '.items[] | select(.metadata.namespace == $ns and .metadata.name == $n)' "$src" + exit 0 + ;; + *"patch kafkas.kafka.strimzi.io "*"--patch-file "*) + ns=$(arg_after -n "$@"); name=$(arg_after kafkas.kafka.strimzi.io "$@") + patch=$(arg_after --patch-file "$@") + src="$FAKE_STATE/managed"; [ -f "$src" ] || src="$(dirname "$0")/managed-kafkas.json" + rv=$(jq -r '.metadata.resourceVersion' "$patch") + log "PATCH-MANAGED $ns/$name rv=$rv $(jq -r '[.metadata.managedFields[] | "\(.manager)=\(.apiVersion)"] | join(",")' "$patch")" + # FAKE_MANAGED_CONFLICT: a concurrent write bumps the object before the + # first patch lands, so that patch carries a stale resourceVersion. + if [ -n "${FAKE_MANAGED_CONFLICT:-}" ] && [ ! -f "$FAKE_STATE/bumped" ]; then + : > "$FAKE_STATE/bumped" + jq --arg ns "$ns" --arg n "$name" '(.items[] | select(.metadata.namespace == $ns and .metadata.name == $n) | .metadata.resourceVersion) |= (tonumber + 1 | tostring)' "$src" > "$FAKE_STATE/managed.new" && mv "$FAKE_STATE/managed.new" "$FAKE_STATE/managed" + fi + if [ -n "${FAKE_MANAGED_PATCH_FAIL:-}" ]; then echo "Error from server (Forbidden): forbidden" >&2; exit 1; fi + src="$FAKE_STATE/managed"; [ -f "$src" ] || src="$(dirname "$0")/managed-kafkas.json" + cur=$(jq -r --arg ns "$ns" --arg n "$name" '.items[] | select(.metadata.namespace == $ns and .metadata.name == $n) | .metadata.resourceVersion' "$src") + if [ "$cur" != "$rv" ]; then + echo "Error from server (Conflict): Operation cannot be fulfilled on kafkas.kafka.strimzi.io \"$name\": the object has been modified; please apply your changes to the latest version and try again" >&2 + exit 1 + fi + jq --arg ns "$ns" --arg n "$name" --slurpfile p "$patch" \ + '(.items[] | select(.metadata.namespace == $ns and .metadata.name == $n) | .metadata) |= (.managedFields = $p[0].metadata.managedFields | .resourceVersion = (.resourceVersion | tonumber + 1 | tostring))' \ + "$src" > "$FAKE_STATE/managed.new" && mv "$FAKE_STATE/managed.new" "$FAKE_STATE/managed" + exit 0 + ;; + *"wait --for=condition=Established"*) + exit 0 + ;; + *"get job strimzi-v1-api-conversion --ignore-not-found -o jsonpath={.metadata.name} {.status.succeeded}{.status.failed}"*) + # FAKE_OLD_JOB: a Job an earlier attempt left behind, running, succeeded + # or failed. + case "${FAKE_OLD_JOB:-}" in + running) printf 'strimzi-v1-api-conversion ' ;; + succeeded|failed) printf 'strimzi-v1-api-conversion 1' ;; + esac + exit 0 + ;; + *"get job strimzi-v1-api-conversion -o jsonpath={.status.succeeded}"*) + case "${FAKE_JOB:-succeed}" in + succeed) printf '["v1"]' > "$FAKE_STATE/stored"; echo 1 ;; + noop) echo 1 ;; + esac + exit 0 + ;; + *"get job strimzi-v1-api-conversion -o jsonpath={.status.failed}"*) + case "${FAKE_JOB:-succeed}" in + fail) printf '1' ;; + esac + case "$args" in + *'conditions[?(@.type=="Failed")]'*) + case "${FAKE_JOB:-succeed}" in fail|nopod) printf ' True' ;; esac + ;; + esac + exit 0 + ;; + *"get pod -l batch.kubernetes.io/job-name=strimzi-v1-api-conversion -o jsonpath="*) + [ -z "${FAKE_PULL:-}" ] || printf ' ImagePullBackOff' + exit 0 + ;; + *" logs "*) + echo "fake conversion log" + exit 0 + ;; + "delete "*|*" delete "*) + cascade="" + case "$args" in *"--cascade=foreground"*) cascade=" foreground" ;; esac + log "DELETE $(arg_after delete "$@") $(arg_after "$(arg_after delete "$@")" "$@")$cascade" + exit 0 + ;; + *"api-resources --api-group=apps.cozystack.io"*) + if [ -n "${FAKE_APIRES_FAIL:-}" ]; then echo "error: unable to retrieve the complete list of server APIs" >&2; exit 1; fi + echo "kafkas.apps.cozystack.io" + exit 0 + ;; + *"get kafkas.apps.cozystack.io"*" kafka-"*) + # Release names of the applications; failed-install has no Kafka CR. + printf '%s\n' 'tenant-a kafka-one' 'tenant-f kafka-failed-install' + exit 0 + ;; + *"get kafkas.apps.cozystack.io"*) + cat "$FAKE_APPS" + exit 0 + ;; + *"patch kafkas.apps.cozystack.io"*) + if [ -n "${FAKE_PATCH_FAIL:-}" ]; then echo "Error from server (Conflict)" >&2; exit 1; fi + log "PATCH $(arg_after -n "$@")/$(arg_after kafkas.apps.cozystack.io "$@") $(arg_after -p "$@")" + exit 0 + ;; +esac +log "UNMODELLED" +printf 'fake kubectl: unmodelled call: %s\n' "$args" >&2 +exit 1 diff --git a/hack/testdata/migration-60-strimzi-v1/managed-kafkas.json b/hack/testdata/migration-60-strimzi-v1/managed-kafkas.json new file mode 100644 index 000000000..bea24efd5 --- /dev/null +++ b/hack/testdata/migration-60-strimzi-v1/managed-kafkas.json @@ -0,0 +1,54 @@ +{ + "items": [ + { + "metadata": { + "namespace": "tenant-a", + "name": "kafka-one", + "managedFields": [ + { + "manager": "helm-controller", + "operation": "Apply", + "apiVersion": "kafka.strimzi.io/v1beta2", + "fieldsType": "FieldsV1", + "fieldsV1": { + "f:spec": { + "f:kafka": { + "f:version": {} + } + } + } + }, + { + "manager": "strimzi-cluster-operator", + "operation": "Update", + "apiVersion": "kafka.strimzi.io/v1beta2", + "subresource": "status", + "fieldsType": "FieldsV1", + "fieldsV1": { + "f:status": {} + } + } + ], + "resourceVersion": "100" + } + }, + { + "metadata": { + "namespace": "tenant-b", + "name": "kafka-two", + "managedFields": [ + { + "manager": "helm-controller", + "operation": "Apply", + "apiVersion": "kafka.strimzi.io/v1", + "fieldsType": "FieldsV1", + "fieldsV1": { + "f:spec": {} + } + } + ], + "resourceVersion": "101" + } + } + ] +} \ No newline at end of file diff --git a/hack/testdata/migration-60-strimzi-v1/release-kafka-one.json b/hack/testdata/migration-60-strimzi-v1/release-kafka-one.json new file mode 100644 index 000000000..f995a4926 --- /dev/null +++ b/hack/testdata/migration-60-strimzi-v1/release-kafka-one.json @@ -0,0 +1 @@ +{"name":"kafka-one","version":3,"info":{"status":"deployed"},"manifest":"---\n# Source: kafka/templates/kafka.yaml\napiVersion: kafka.strimzi.io/v1beta2\nkind: Kafka\nmetadata:\n name: kafka-one\n---\n# Source: kafka/templates/topics.yaml\napiVersion: kafka.strimzi.io/v1beta2\nkind: KafkaTopic\nmetadata:\n name: kafka-one-orders\n---\napiVersion: v1\nkind: ConfigMap\nmetadata:\n name: kafka-one-metrics\n","hooks":[{"name":"kafka-one-kafka-migration","manifest":"apiVersion: batch/v1\nkind: Job\n# script applies apiVersion: kafka.strimzi.io/v1beta2\n"}]} \ No newline at end of file diff --git a/hack/testdata/migration-60-strimzi-v1/strimzi-crds-0.51.0.yaml b/hack/testdata/migration-60-strimzi-v1/strimzi-crds-0.51.0.yaml new file mode 100644 index 000000000..76a5f108a --- /dev/null +++ b/hack/testdata/migration-60-strimzi-v1/strimzi-crds-0.51.0.yaml @@ -0,0 +1 @@ +# Stand-in for strimzi-crds-0.51.0.yaml; the fake kubectl never reads it. diff --git a/packages/core/platform/images/migrations/Dockerfile b/packages/core/platform/images/migrations/Dockerfile index 6084a9e5e..c5023c286 100644 --- a/packages/core/platform/images/migrations/Dockerfile +++ b/packages/core/platform/images/migrations/Dockerfile @@ -29,6 +29,14 @@ RUN set -eux; \ echo "${sha256} /usr/bin/etcd-migrate" | sha256sum -c -; \ chmod +x /usr/bin/etcd-migrate +# CRDs of Strimzi 0.51.0, the last release serving both v1beta2 and v1. +# Migration 60 applies them so Strimzi's conversion tool can move every stored +# object to v1 before the kafka-operator release installs the v1-only CRDs. +RUN mkdir /strimzi-crds && \ + curl -fsSL -o /strimzi-crds/strimzi-crds-0.51.0.yaml \ + https://github.com/strimzi/strimzi-kafka-operator/releases/download/0.51.0/strimzi-crds-0.51.0.yaml && \ + echo "a11257e67985a946a6facfb17d729aac5670b89675d410d13162fe455a5ac7a8 /strimzi-crds/strimzi-crds-0.51.0.yaml" | sha256sum -c - + COPY migrations /migrations # v1alpha2 etcd CRDs, vendored into the build context by `make image-migrations`. # migration 50 applies these before adoption so it does not deadlock waiting for diff --git a/packages/core/platform/images/migrations/migrations/60 b/packages/core/platform/images/migrations/migrations/60 new file mode 100755 index 000000000..15aa10e7f --- /dev/null +++ b/packages/core/platform/images/migrations/migrations/60 @@ -0,0 +1,473 @@ +#!/bin/sh +# Migration 60 --> 61 +# Move the Strimzi custom resources to the v1 API before Strimzi 1.x arrives. +# +# Strimzi 1.0 removed the v1beta2 API from its CRDs. The kafka-operator +# release replaces the CRDs on upgrade (upgradeCRDs: CreateReplace), and the +# API server refuses a CRD whose spec.versions no longer lists a version that +# status.storedVersions still records. So every stored object has to be +# rewritten as v1, and storedVersions narrowed to [v1], before the new +# operator arrives. That needs CRDs serving both versions, which only Strimzi +# 0.49 to 0.51 ship: the 0.51.0 CRDs are baked into this image (see the +# Dockerfile). The conversion itself is Strimzi's v1-api-conversion tool, run +# as a Job from the 0.51.0 operator image: convert-resource rewrites the +# objects, crd-upgrade switches the storage version and narrows +# storedVersions. +# +# Strimzi 0.46 also removed ZooKeeper. A ZooKeeper-based Kafka that did not +# reach KRaft on the release carrying the chart's ZooKeeper to KRaft +# migration cannot be run, converted or migrated by anything after this +# point, so this migration refuses to proceed while one is left. +# +# Strimzi 1.2 runs neither Kafka 3.8 nor 3.9, so every Kafka application on +# v3.8 or v3.9 is moved to v4.3. Until the new kafka chart arrives, the +# release renders with the old one, which rejects v4.3, and the HelmRelease +# reports a values error; no Helm action runs in that window. +# +# Ordering: this runs as the platform chart's pre-upgrade hook, before the +# upgrade re-points cozystack-packages, so on a variant that runs migrations +# neither the new operator nor the new kafka chart is rendered before it. The +# default variant renders no migration hook, and there these steps are manual. +# +# Re-runnable: every step tolerates having run before. + +set -euo pipefail + +# shellcheck source=lib/cozystack-version.sh +. "$(dirname "$0")/lib/cozystack-version.sh" + +STRIMZI_CRD_FILE="${STRIMZI_CRD_FILE:-/strimzi-crds/strimzi-crds-0.51.0.yaml}" +CONVERSION_IMAGE="${CONVERSION_IMAGE:-quay.io/strimzi/operator:0.51.0@sha256:ef3e406aa09449dd0241d53cdc81e15e58bf59b51f2425fb64384a179d676b1d}" +CONVERSION_NS="${CONVERSION_NS:-cozy-kafka-operator}" +CONVERSION_NAME=strimzi-v1-api-conversion +CONVERSION_TIMEOUT="${CONVERSION_TIMEOUT:-1800}" +# The run that starts the Job waits up to CONVERSION_TIMEOUT for it. As the +# platform hook, a run is cut off sooner by the Helm upgrade timeout +# (cozystack-operator --helmrelease-upgrade-timeout, 10m by default), and the +# retry, the next pod of the hook Job or a new hook Job once helm-controller +# retries the upgrade, waits up to CONVERSION_TIMEOUT again for the same Job. +# The Job is not owned by the hook Job and nothing else bounds it, so the +# deadline is the only limit: a conversion gets two waits, and a Job whose pod +# never starts then fails and the next run starts the conversion over. +CONVERSION_DEADLINE=$((CONVERSION_TIMEOUT * 2)) +POLL_INTERVAL="${POLL_INTERVAL:-5}" +KAFKA_TARGET_VERSION=v4.3 + +# The CRDs the conversion tool rewrites. kafkamirrormakers.kafka.strimzi.io +# is not among them: MirrorMaker 1 left in Strimzi 0.46 and its CRD is left +# behind untouched. +STRIMZI_CRDS="kafkas.kafka.strimzi.io kafkanodepools.kafka.strimzi.io +kafkaconnects.kafka.strimzi.io kafkaconnectors.kafka.strimzi.io +kafkabridges.kafka.strimzi.io kafkamirrormaker2s.kafka.strimzi.io +kafkarebalances.kafka.strimzi.io kafkatopics.kafka.strimzi.io +kafkausers.kafka.strimzi.io strimzipodsets.core.strimzi.io" + +# --ignore-not-found separates a cluster without Strimzi (no output, rc 0) +# from an API error, which aborts under set -e. +crd=$(kubectl get crd kafkas.kafka.strimzi.io --ignore-not-found -o name) +if [ -z "$crd" ]; then + echo "Strimzi is not installed; nothing to convert" + stamp_cozystack_version 61 + exit 0 +fi + +# The running 0.45 operator must not write to the resources from here on. +# Strimzi supports the conversion only under an operator that knows v1 +# (0.49 or later), and 0.45 rewrites the Kafka status through v1beta2 on every +# reconcile, because the v1 schema prunes its kafkaMetadataState field. Each +# such write records a v1beta2 managedFields entry again after the rewrite +# below. The kafka-operator release brings the operator back at Strimzi 1.2: +# its chart sets the replica count, so the upgrade restores it. Kafka pods keep +# running while no operator reconciles them. +stop_old_operator() { + operator=$(kubectl -n "$CONVERSION_NS" get deployment strimzi-cluster-operator --ignore-not-found -o name) + if [ -n "$operator" ]; then + kubectl -n "$CONVERSION_NS" scale deployment strimzi-cluster-operator --replicas=0 + kubectl -n "$CONVERSION_NS" wait --for=delete pod -l name=strimzi-cluster-operator --timeout=300s + fi + # The topic and user operators of each cluster write KafkaTopic and + # KafkaUser status through v1beta2 the same way. Only now that the cluster + # operator is gone does a scale-down stick; the 1.2 operator scales them + # back up when it rolls them onto its own image. + kubectl get deployment --all-namespaces -l app.kubernetes.io/name=entity-operator,app.kubernetes.io/managed-by=strimzi-cluster-operator \ + -o jsonpath='{range .items[*]}{.metadata.namespace}{" "}{.metadata.name}{"\n"}{end}' | + while read -r ns name; do + kubectl -n "$ns" scale deployment "$name" --replicas=0 + done + } + +# Called as an if condition, where set -e does not apply: a failed read exits +# here, or it would pass for an absent CRD. +stored_v1_only() { + for c in $STRIMZI_CRDS; do + if ! v=$(kubectl get crd "$c" --ignore-not-found -o jsonpath='{.status.storedVersions}'); then + echo "ERROR: reading the stored versions of $c failed" >&2 + exit 1 + fi + if [ -n "$v" ] && [ "$v" != '["v1"]' ]; then + return 1 + fi + done + return 0 + } + +# A Job whose pod the API server refused fails on its deadline with no failed +# pod to count, so its Failed condition is read as well. +job_failed() { + f=$(kubectl -n "$CONVERSION_NS" get job "$CONVERSION_NAME" -o jsonpath='{.status.failed} {.status.conditions[?(@.type=="Failed")].status}') + case "$f" in + [1-9]*|*True*) return 0 ;; + esac + return 1 + } + +# wait_for_conversion: 0 once the conversion Job has succeeded; on a failure +# or a timeout, print its logs and return 1. +wait_for_conversion() { + echo "Waiting for the conversion Job" + waited=0 + pull_reported= + while :; do + succeeded=$(kubectl -n "$CONVERSION_NS" get job "$CONVERSION_NAME" -o jsonpath='{.status.succeeded}') + # A pod that cannot pull its image leaves no logs to print, and its Job + # fails only at the deadline. + if [ -z "$pull_reported" ]; then + waiting=$(kubectl -n "$CONVERSION_NS" get pod -l "batch.kubernetes.io/job-name=$CONVERSION_NAME" \ + -o jsonpath='{.items[*].status.initContainerStatuses[*].state.waiting.reason} {.items[*].status.containerStatuses[*].state.waiting.reason}' || true) + case "$waiting" in + *ErrImagePull*|*ImagePullBackOff*) + echo "ERROR: the conversion pod cannot pull $CONVERSION_IMAGE. A cluster that cannot reach quay.io needs that image mirrored under the same name and digest, for example through a registry mirror for quay.io in the container runtime of the nodes. The Job fails ${CONVERSION_DEADLINE}s after it started, and the next run of the migration starts it again." >&2 + pull_reported=1 + ;; + esac + fi + if [ "${succeeded:-0}" -ge 1 ]; then + kubectl -n "$CONVERSION_NS" logs "job/$CONVERSION_NAME" --all-containers --prefix || true + return 0 + fi + if job_failed || [ "$waited" -ge "$CONVERSION_TIMEOUT" ]; then + echo "ERROR: the Strimzi v1 API conversion Job did not succeed; its logs:" >&2 + kubectl -n "$CONVERSION_NS" logs "job/$CONVERSION_NAME" --all-containers --prefix >&2 || true + return 1 + fi + sleep "$POLL_INTERVAL" + waited=$((waited + POLL_INTERVAL)) + done + } + +# The Job's ClusterRole can patch CRDs and write ConfigMaps everywhere, so +# none of it outlives the conversion, whichever path reaches here. +remove_conversion_job() { + kubectl -n "$CONVERSION_NS" delete job "$CONVERSION_NAME" --ignore-not-found --wait=true + kubectl delete clusterrolebinding "$CONVERSION_NAME" --ignore-not-found + kubectl delete clusterrole "$CONVERSION_NAME" --ignore-not-found + kubectl -n "$CONVERSION_NS" delete serviceaccount "$CONVERSION_NAME" --ignore-not-found + } + +# The platform hook can time out while the Job converts, and the retry kills +# the hook at whatever point it reached. A Job still running then has to +# finish before anything here touches the CRDs, or the CRD apply below would +# switch the storage version back under it. Its outcome is read from the CRDs +# afterwards, not from the Job. +job_state=$(kubectl -n "$CONVERSION_NS" get job "$CONVERSION_NAME" --ignore-not-found -o jsonpath='{.metadata.name} {.status.succeeded}{.status.failed}{.status.conditions[?(@.type=="Failed")].status}') +case "$job_state" in + "$CONVERSION_NAME ") + echo "A conversion Job from an earlier attempt is still running" + # backoffLimit is 0, so a Job reporting no failure has not finished. + if ! wait_for_conversion; then + if ! job_failed; then + echo "ERROR: the conversion Job of an earlier attempt is still running; retry once it finishes" >&2 + exit 1 + fi + fi + ;; +esac +# A Job failed on its deadline can still have a pod terminating, and +# convert-resource writes until it is gone. A foreground deletion returns +# only once the Job's pods are. +if [ -n "$job_state" ]; then + kubectl -n "$CONVERSION_NS" delete job "$CONVERSION_NAME" --ignore-not-found --cascade=foreground --wait=true +fi + +if stored_v1_only; then + echo "Every Strimzi CRD already stores only v1; skipping the conversion" + # The 0.45 operator can only still be running while the CRDs serve + # v1beta2. Once they serve v1 alone the operator is 1.x, and a run by hand + # must not stop it: no Helm upgrade would bring it back. + served=$(kubectl get crd kafkas.kafka.strimzi.io -o jsonpath='{.spec.versions[*].name}') + case " $served " in + *" v1beta2 "*) stop_old_operator ;; + esac + remove_conversion_job +else + # A Kafka that is not KRaft yet still has its ZooKeeper section, or reports a + # metadata state other than KRaft while the ZooKeeper to KRaft migration + # runs. Read through v1beta2, which every Strimzi this can meet serves. + zk=$(kubectl get kafkas.v1beta2.kafka.strimzi.io --all-namespaces -o json | jq -r ' + .items[] + | select(.spec.zookeeper != null or ((.status.kafkaMetadataState // "KRaft") != "KRaft")) + | "\(.metadata.namespace)/\(.metadata.name)"') + if [ -n "$zk" ]; then + echo "ERROR: these Kafka clusters still run on ZooKeeper:" >&2 + printf '%s\n' "$zk" | sed 's/^/ /' >&2 + echo "Strimzi 1.x cannot run them. Upgrade through Cozystack v1.7, whose kafka chart migrates every instance to KRaft, and wait until each one reports Ready before upgrading past it." >&2 + exit 1 + fi + + # The v1.7 kafka chart freed the shared "kafka" pool name by draining a + # migrated cluster onto its own pool through a KafkaRebalance named + # -drain-kafka, and removed the rebalance once done. The new chart + # no longer drives that drain, so one still in flight would be left half + # done, with the brokers spread over two pools. + rebalances=$(kubectl get kafkarebalances.v1beta2.kafka.strimzi.io --all-namespaces \ + -o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}') + drains=$(printf '%s\n' "$rebalances" | grep -- '-drain-kafka$' || true) + if [ -n "$drains" ]; then + echo "ERROR: a ZooKeeper to KRaft migration of Cozystack v1.7 is still draining a shared broker pool:" >&2 + printf '%s\n' "$drains" | sed 's/^/ /' >&2 + echo "Let the kafka application upgrade on v1.7 finish, which removes these KafkaRebalances, before upgrading past it." >&2 + exit 1 + fi + + if [ ! -r "$STRIMZI_CRD_FILE" ]; then + echo "ERROR: $STRIMZI_CRD_FILE is not readable; set STRIMZI_CRD_FILE to strimzi-crds-0.51.0.yaml from the Strimzi 0.51.0 release" >&2 + exit 1 + fi + stop_old_operator + # These CRDs mark v1beta2 as the storage version, so a re-run after a + # partial crd-upgrade switches storage back and converts from the start. + echo "Applying the Strimzi 0.51.0 CRDs, which serve both v1beta2 and v1" + kubectl apply --server-side --force-conflicts --field-manager=cozystack-migration-60 -f "$STRIMZI_CRD_FILE" + for c in $STRIMZI_CRDS; do + kubectl wait --for=condition=Established --timeout=120s "crd/$c" + done + + # Whatever an earlier attempt left has finished by now, and its result was + # undone by the CRD apply above, so the conversion starts over. + kubectl -n "$CONVERSION_NS" delete job "$CONVERSION_NAME" --ignore-not-found --wait=true + kubectl apply -f - <&2 + for c in $STRIMZI_CRDS; do + echo " $c: $(kubectl get crd "$c" --ignore-not-found -o jsonpath='{.status.storedVersions}')" >&2 + done + exit 1 + fi + + remove_conversion_job +fi + +# Moving the applications off Kafka 3.x first makes the old chart reject +# their values, so no Helm action can store a new v1beta2 revision behind the +# rewrite below. +app_resources=$(kubectl api-resources --api-group=apps.cozystack.io -o name) +has_kafka_apps() { printf '%s\n' "$app_resources" | grep -q '^kafkas\.'; } +if has_kafka_apps; then + kubectl get kafkas.apps.cozystack.io --all-namespaces \ + -o jsonpath='{range .items[*]}{.metadata.namespace}{" "}{.metadata.name}{" "}{.spec.version}{"\n"}{end}' | + while read -r ns name version; do + case "$version" in + v3.8|v3.9) + echo "Moving Kafka $ns/$name from $version to $KAFKA_TARGET_VERSION" + kubectl -n "$ns" patch kafkas.apps.cozystack.io "$name" --type merge \ + -p "{\"spec\":{\"version\":\"$KAFKA_TARGET_VERSION\"}}" + ;; + esac + done +fi + +# Helm builds the objects of a stored release before it upgrades, +# uninstalls or rolls back to it, and fails on an API version the server no +# longer serves, so every Kafka release would be stuck once the v1-only CRDs +# land. Uninstall reads the latest revision whatever its status, and upgrade +# falls back to the last failed one, so every revision of each release is +# rewritten, the way helm-mapkubeapis does: the Secret holds base64 of +# gzipped JSON, inside the base64 of the Secret data. A release whose install +# failed has no Kafka CR, so the releases are listed from the applications as +# well. The chart's hooks carry no Strimzi kinds, so only the manifest needs +# the rewrite. +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +list_releases() { + kubectl get kafkas.v1.kafka.strimzi.io --all-namespaces \ + -o jsonpath='{range .items[*]}{.metadata.namespace}{" "}{.metadata.annotations.meta\.helm\.sh/release-name}{"\n"}{end}' > "$work/releases.all" + if has_kafka_apps; then + kubectl get kafkas.apps.cozystack.io --all-namespaces \ + -o jsonpath='{range .items[*]}{.metadata.namespace}{" kafka-"}{.metadata.name}{"\n"}{end}' >> "$work/releases.all" + fi + sort -u "$work/releases.all" > "$work/releases" + } + +# scan_releases rewrite|check: rewrite every revision that still names +# v1beta2, or list the ones that do. +scan_releases() { + list_releases + while read -r ns release; do + [ -n "$release" ] || continue + secrets=$(kubectl -n "$ns" get secret -l "owner=helm,name=$release" -o jsonpath='{.items[*].metadata.name}') + for secret in $secrets; do + kubectl -n "$ns" get secret "$secret" -o jsonpath='{.data.release}' | base64 -d | base64 -d | gzip -dc > "$work/release.json" + if ! jq -e '.manifest | contains("apiVersion: kafka.strimzi.io/v1beta2")' "$work/release.json" > /dev/null; then + continue + fi + if [ "$1" = check ]; then + echo " Helm release $ns/$secret" + continue + fi + echo "Rewriting Helm release $ns/$secret to kafka.strimzi.io/v1" + jq -c '.manifest |= gsub("apiVersion: kafka.strimzi.io/v1beta2\n"; "apiVersion: kafka.strimzi.io/v1\n")' "$work/release.json" | + gzip -c | base64 | tr -d '\n' | base64 | tr -d '\n' > "$work/release.b64" + jq -n --rawfile r "$work/release.b64" '{data: {release: $r}}' > "$work/patch.json" + kubectl -n "$ns" patch secret "$secret" --type merge --patch-file "$work/patch.json" + done + done < "$work/releases" + } + +# The objects keep managedFields entries recorded under the old API versions, +# and once the CRDs serve only v1 the API server refuses every server-side +# apply to such an object ("request to convert CR to an invalid group/version", +# kubernetes/kubernetes#111937). helm-controller recovers by wiping the +# entries, and the fields Helm owned pass to "before-first-apply", so a field a +# later chart drops is never removed. Rewrite the entries to v1 while both +# versions are still served; the conversion is None, so field paths are +# spelled the same under either, and an entry for a field v1 dropped only +# names a field that no longer exists. +# +# The patch replaces the whole managedFields list, so a patch built from a +# stale read could drop an entry a concurrent writer added in between. The +# patch carries the resourceVersion it was built from: the API server then +# answers a concurrent change with a conflict, and the object is read again. +stale_fields='[.metadata.managedFields[]? | select(.apiVersion | test("/v1(alpha1|beta1|beta2)$"))] | length > 0' + +# list_stale_managed : the objects of that kind with entries to rewrite. +list_stale_managed() { + kubectl get "$1" --all-namespaces --show-managed-fields -o json | + jq -r ".items[] | select($stale_fields) | \"\\(.metadata.namespace) \\(.metadata.name)\"" > "$work/managed" + } + +rewrite_managed() { + for c in $STRIMZI_CRDS; do + list_stale_managed "$c" + while read -r ns name; do + echo "Moving the managedFields of $c $ns/$name to v1" + attempt=0 + while :; do + kubectl -n "$ns" get "$c" "$name" --show-managed-fields -o json > "$work/object.json" + if ! jq -e "$stale_fields" "$work/object.json" > /dev/null; then + break + fi + jq '{metadata: {resourceVersion: .metadata.resourceVersion, managedFields: (.metadata.managedFields | map(.apiVersion |= sub("/v1(alpha1|beta1|beta2)$"; "/v1")))}}' \ + "$work/object.json" > "$work/managed-patch.json" + if kubectl -n "$ns" patch "$c" "$name" --type merge --patch-file "$work/managed-patch.json" 2> "$work/patch.err"; then + break + fi + attempt=$((attempt + 1)) + if ! grep -q 'the object has been modified' "$work/patch.err" || [ "$attempt" -ge 5 ]; then + cat "$work/patch.err" >&2 + exit 1 + fi + done + done < "$work/managed" + done + } + +scan_releases rewrite +rewrite_managed + +# A Helm upgrade already running when the releases were rewritten stores its +# own revision when it finishes, and that one may name v1beta2 again. Scan +# once more and refuse to stamp the version on any leftover: the hook then +# fails and runs the migration again. +scan_releases check > "$work/leftover" +for c in $STRIMZI_CRDS; do + list_stale_managed "$c" + sed "s|^| managedFields of $c |" "$work/managed" >> "$work/leftover" +done +if [ -s "$work/leftover" ]; then + echo "ERROR: these still name a removed Strimzi API version:" >&2 + cat "$work/leftover" >&2 + exit 1 +fi + +stamp_cozystack_version 61 diff --git a/packages/core/platform/values.yaml b/packages/core/platform/values.yaml index 747f4c904..b050f21c6 100644 --- a/packages/core/platform/values.yaml +++ b/packages/core/platform/values.yaml @@ -14,7 +14,7 @@ migrations: # refuses to advance past a migration file missing from the image (exit 1), so # a stale pin fails loudly rather than silently skipping the etcd adoption. image: ghcr.io/cozystack/cozystack/platform-migrations:v1.6.0@sha256:6777042e3e9c6bad76e7d96fb64baa115061975f39061cb3ede84e21d0e2213f - targetVersion: 60 + targetVersion: 61 # Adopt legacy etcd clusters onto the v1alpha2 operator WITHOUT the mandatory # pre-adoption safety snapshot (migration 50). Leave false unless migration 50 # has actually refused to proceed and you accept the risk: adoption rewires