diff --git a/api/apps/v1alpha1/kafka/types.go b/api/apps/v1alpha1/kafka/types.go index a69cece1b..16e4ce04e 100644 --- a/api/apps/v1alpha1/kafka/types.go +++ b/api/apps/v1alpha1/kafka/types.go @@ -24,8 +24,8 @@ type ConfigSpec struct { // TLS configuration. Strimzi manages the cluster PKI automatically (no cert-manager is involved for this chart): the operator auto-creates `-cluster-ca-cert` and `-clients-ca-cert` secrets, both exposed for client trust setup. The internal TLS listener on 9093 is always on; this toggle only controls the external listener on 9094. // +kubebuilder:default:={} Tls TLS `json:"tls"` - // Kafka version to deploy. Upgrade-only: once a cluster's KRaft metadata is at a given version, Strimzi refuses an in-place downgrade, so lowering this on a running cluster (e.g. v3.9 back to v3.8) leaves the Kafka CR stuck in a reconcile error. Pick the target version at creation and only ever raise it. - // +kubebuilder:default:="v3.9" + // Kafka version to deploy. Upgrade-only: once a cluster's KRaft metadata is at a given version, Strimzi refuses an in-place downgrade, so lowering this on a running cluster (e.g. v4.3 back to v4.2) leaves the Kafka CR stuck in a reconcile error. Pick the target version at creation and only ever raise it. + // +kubebuilder:default:="v4.3" Version Version `json:"version"` // Topics configuration. // +kubebuilder:default:={} @@ -36,7 +36,7 @@ type ConfigSpec struct { } type Kafka struct { - // Persistent Volume size for KRaft controller metadata (used during ZK-to-KRaft migration). + // Persistent Volume size for KRaft controller metadata. // +kubebuilder:default:="5Gi" ControllerStorageSize resource.Quantity `json:"controllerStorageSize,omitempty"` // Number of Kafka replicas. @@ -83,5 +83,5 @@ type Topic struct { // +kubebuilder:validation:Enum="t1.nano";"t1.micro";"t1.small";"t1.medium";"t1.large";"t1.xlarge";"t1.2xlarge";"t1.4xlarge";"c1.nano";"c1.micro";"c1.small";"c1.medium";"c1.large";"c1.xlarge";"c1.2xlarge";"c1.4xlarge";"s1.nano";"s1.micro";"s1.small";"s1.medium";"s1.large";"s1.xlarge";"s1.2xlarge";"s1.4xlarge";"u1.nano";"u1.micro";"u1.small";"u1.medium";"u1.large";"u1.xlarge";"u1.2xlarge";"u1.4xlarge";"m1.nano";"m1.micro";"m1.small";"m1.medium";"m1.large";"m1.xlarge";"m1.2xlarge";"m1.4xlarge";"nano";"micro";"small";"medium";"large";"xlarge";"2xlarge" type ResourcesPreset string -// +kubebuilder:validation:Enum="v3.9";"v3.8" +// +kubebuilder:validation:Enum="v4.3";"v4.2" type Version string diff --git a/docs/agents/e2e-testing.md b/docs/agents/e2e-testing.md index c5bcd091e..f5c793125 100644 --- a/docs/agents/e2e-testing.md +++ b/docs/agents/e2e-testing.md @@ -205,7 +205,7 @@ The suite is pinned to Chainsaw **v0.2.15** (the latest release as of May 2026); 3. Imperative-only waits live in a `script` step; any bare `sleep` carries a `TODO(e2e-replace-fixed-timeouts):` justification, unless it is a measurement interval, which §2 carves out by name and which takes no marker because no event can replace it. 4. No test-level `EXIT`/`RETURN` trap — rely on Chainsaw cleanup; a self-contained trap is allowed inside a single `script` step (port-forward / temp dir), and inside an explicit subshell within a BATS `@test`, where it does not displace the runner's own handler. A BATS file that still carries test-level traps declares the count itself in `# EXIT-TRAP DEBT: N`; do not ask for one to be removed without checking that line, because the count is exact in both directions. 5. Controller-created artifacts the test cannot reclaim are pruned explicitly; nested tenants delete child → parent with a wait-for-deletion between. -6. An HR-Ready assert has a floor: the release's own install timeout, not only the app's boot time. Below it a failed install reports as a bare assert timeout, and above it by less than a start margin it can, so **a number under the floor buys a shorter red run at the price of the diagnosis**. Whether to leave a suite under it is a per-suite call, made against what that suite's red runs cost, unless no budget inside the E2E job's cap can clear the floor at all, as with kafka's 7h install timeout; either way, a Chainsaw assert left under it says why in-line. Derive the floor from [Sizing an HR-Ready budget](#sizing-an-hr-ready-budget) rather than copying a neighbour's number, and justify a raise in-line. The same goes for any other wait longer than the ones around it, HR-Ready or not: it carries its reason in-line. +6. An HR-Ready assert has a floor: the release's own install timeout, not only the app's boot time. Below it a failed install reports as a bare assert timeout, and above it by less than a start margin it can, so **a number under the floor buys a shorter red run at the price of the diagnosis**. Whether to leave a suite under it is a per-suite call, made against what that suite's red runs cost; either way, a Chainsaw assert left under it says why in-line. Derive the floor from [Sizing an HR-Ready budget](#sizing-an-hr-ready-budget) rather than copying a neighbour's number, and justify a raise in-line. The same goes for any other wait longer than the ones around it, HR-Ready or not: it carries its reason in-line. 7. Failure path attaches scoped diagnostics via a `catch:` block, never a silent pass. The node-join deadline is no exception, and no longer has one: with the soft-red marker gone it fails its suite like any other deadline and its diagnostics run from the same `catch:`. 8. If it touches HelmRelease install/upgrade behavior, add the `status.history` guard: fail the run on a `failed` or `uninstalled` Snapshot of a release whose install strategy is `RetryOnFailure`, fail it on a parent whose strategy is anything else, and report the footprint of every other release without failing. What is reusable today is the predicate and the jsonpath in `hack/e2e-chainsaw/_lib/remediation-guard.sh`; the policy that weighs the footprint against `.spec.install.strategy.name`, judges an empty history against the release's own readiness, and keeps each read's exit status apart from its output lives in `cozy_guard_helmrelease` in `hack/e2e-chainsaw/_lib/run-kubernetes.sh` — read it before rebuilding any of that for another suite. If the suite has a failure path that exits before the guard, give that path `cozy_report_helmrelease_remediation` as well, or the runs the guard exists for are the runs it does not see. 9. A test that creates a tenant `Kubernetes` cluster waits for its release to reach `Ready` **before the assertions it ends on**, with a `--timeout` the guard can read as positive and no longer than a day. Reads earlier in the run are expected and are not what this bounds. The mechanism, how to size the budget and what the guard does with it are under "Do not mask cleanup or teardown failures". One thing to carry from there rather than look up: the naming is part of the rule, not a detail of the check, so an end-assertion added under any name outside the `cozy_assert_*` / `cozy_switch_and_assert_*` convention is invisible to it and passes before the wait without reddening anything — as long as one recognised call site is left behind it. Rename every recognised call and the guard reds instead, on finding none at all, which is the one shape of this mistake it does catch. @@ -245,7 +245,7 @@ A HelmRelease install waits through kstatus unless the ApplicationDefinition car The first floor is the app's own first boot. For a chart that sets `startupProbe` it is `initialDelaySeconds + (failureThreshold - 1) * periodSeconds`; for a chart that does not, it is whatever its operator needs, and only a measurement will say. mariadb is the case in this tree where this floor is real rather than nominal: its chart grants a 310s startup budget, so a wait that fails earlier fails a boot the probe was still willing to wait for. -The second floor is the release's own install timeout, and it is usually the binding one. Where to read it depends on who created the release, and getting this wrong under-budgets rather than over-budgets. For a release cozystack-api generated from an ApplicationDefinition the default is 10m (`pkg/cmd/server/start.go`), overridden per definition by `release.cozystack.io/helm-install-timeout` — `kubernetes-rd` at 20m, `kubernetes-nodes-rd`, `tenant-rd` and `monitoring-rd` at 15m, `kafka-rd` at 7h, a floor no suite clears inside the E2E job's 215-minute cap; the first two also disable the wait, so for `kubernetes` the override bounds its blocking post-install hook rather than a kstatus poll, and `kubernetes-nodes`, which renders no post-install hook, has nothing in its install for the override to bound. For a HelmRelease an app chart renders itself there is no ApplicationDefinition to read and that default never applies: the floor is the `spec.install.timeout` on the template that renders it, which takes precedence over its `spec.timeout`, and where a template sets neither, helm-controller's own documented default of 5m applies instead. There is deliberately no count of that population here. Read the value off the template you are waiting on: they range from 5m to 30m, they are spread across `packages/apps`, `packages/extra` and `packages/system`, and a release carrying `spec.kubeConfig` installs into a tenant cluster and is not one of these at all. A number in this sentence would be an obligation to revisit it whenever a template is added, and its three qualifiers are each easy to drop from a recursive grep. The nested `-system` releases the app charts wrap their workloads in are all of this second kind. A HelmRelease that cozystack-operator builds for a `Package` is a third kind: its `spec.install.timeout` comes from the operator's `--helmrelease-install-timeout`, 10m unless `cozystackOperator.helmReleaseInstallTimeout` in the installer chart overrides it. At or below whichever floor applies, a failed install is reported as a bare assert timeout, because the release has not yet recorded why it failed. +The second floor is the release's own install timeout, and it is usually the binding one. Where to read it depends on who created the release, and getting this wrong under-budgets rather than over-budgets. For a release cozystack-api generated from an ApplicationDefinition the default is 10m (`pkg/cmd/server/start.go`), overridden per definition by `release.cozystack.io/helm-install-timeout` — `kubernetes-rd` at 20m, `kubernetes-nodes-rd`, `tenant-rd` and `monitoring-rd` at 15m; the first two also disable the wait, so for `kubernetes` the override bounds its blocking post-install hook rather than a kstatus poll, and `kubernetes-nodes`, which renders no post-install hook, has nothing in its install for the override to bound. For a HelmRelease an app chart renders itself there is no ApplicationDefinition to read and that default never applies: the floor is the `spec.install.timeout` on the template that renders it, which takes precedence over its `spec.timeout`, and where a template sets neither, helm-controller's own documented default of 5m applies instead. There is deliberately no count of that population here. Read the value off the template you are waiting on: they range from 5m to 30m, they are spread across `packages/apps`, `packages/extra` and `packages/system`, and a release carrying `spec.kubeConfig` installs into a tenant cluster and is not one of these at all. A number in this sentence would be an obligation to revisit it whenever a template is added, and its three qualifiers are each easy to drop from a recursive grep. The nested `-system` releases the app charts wrap their workloads in are all of this second kind. A HelmRelease that cozystack-operator builds for a `Package` is a third kind: its `spec.install.timeout` comes from the operator's `--helmrelease-install-timeout`, 10m unless `cozystackOperator.helmReleaseInstallTimeout` in the installer chart overrides it. At or below whichever floor applies, a failed install is reported as a bare assert timeout, because the release has not yet recorded why it failed. What holds the install open is anything in the rendered manifests kstatus reports InProgress, which is wider than a `Ready` condition. kstatus carries built-in rules for a list of kinds — the workloads, but also `Service`, `Pod`, `PersistentVolumeClaim` and `CustomResourceDefinition` — and falls through to reading `Ready` only for kinds it has no rule for. Read that list rather than recalling it: `legacyTypes` in `pkg/kstatus/status/core.go` of `fluxcd/cli-utils`, at whatever version the helm-controller image in `internal/fluxinstall/manifests/fluxcd.yaml` was built against — nothing in this repository vendors it, so there is no local copy to open and the version has to come from that image. It does not divide the way intuition suggests, because `Secret`, `ConfigMap` and `batch/CronJob` are in the table and map to always-ready, so being in it is not the same as gating on it. The practical consequence is that "renders no CR publishing `Ready`" does not exempt a release, and `tcp-balancer` and `vpn` show why: each renders a `WorkloadMonitor`, whose status type declares no `conditions` at all, so kstatus falls past the `Ready` fallback and reads that CR as Current — and then gates the install on the Deployment beside it through a built-in rule. The CR is present and contributes nothing; the workload is what holds the install open. diff --git a/examples/backups/kafka-metadata/00-helpers.sh b/examples/backups/kafka-metadata/00-helpers.sh index 8d77fba64..08a44b8bc 100755 --- a/examples/backups/kafka-metadata/00-helpers.sh +++ b/examples/backups/kafka-metadata/00-helpers.sh @@ -63,7 +63,7 @@ export CA_MOUNT_DIR="${CA_MOUNT_DIR:-/etc/ssl/kafka-backup-ca}" # the target broker's own image at reconcile time and renders it as the # strategy's .ClientImage. Override this only to match your operator's image if # the CLI it carries differs. -export KAFKA_IMAGE="${KAFKA_IMAGE:-quay.io/strimzi/kafka:0.45.1-rc1-kafka-3.9.1@sha256:ba52ed046b1dccdbd96f4e68057ce014d862a7c9c1fc670760c023b9aa09f23f}" +export KAFKA_IMAGE="${KAFKA_IMAGE:-quay.io/strimzi/kafka:1.2.0-kafka-4.3.1@sha256:fef34b5438e8556cc08c01f3e254e47346f061b53a4e38d4289853777e0ea7f1}" export KAFKA_BIN="${KAFKA_BIN:-/opt/kafka/bin}" # Name of that long-lived CLI Pod; cleanup.sh removes it. export KAFKA_CLI_POD="${KAFKA_CLI_POD:-kafka-cli}" diff --git a/examples/backups/kafka-metadata/05-kafka-src.yaml b/examples/backups/kafka-metadata/05-kafka-src.yaml index 041a8dc26..b905e12c5 100644 --- a/examples/backups/kafka-metadata/05-kafka-src.yaml +++ b/examples/backups/kafka-metadata/05-kafka-src.yaml @@ -14,8 +14,4 @@ spec: replicas: 1 size: 2Gi resourcesPreset: "c1.small" - zookeeper: - replicas: 1 - size: 1Gi - resourcesPreset: "c1.small" topics: [] diff --git a/examples/backups/kafka-metadata/20-kafka-target.yaml b/examples/backups/kafka-metadata/20-kafka-target.yaml index 7642f81bf..ff7e30b84 100644 --- a/examples/backups/kafka-metadata/20-kafka-target.yaml +++ b/examples/backups/kafka-metadata/20-kafka-target.yaml @@ -12,8 +12,4 @@ spec: replicas: 1 size: 2Gi resourcesPreset: "c1.small" - zookeeper: - replicas: 1 - size: 1Gi - resourcesPreset: "c1.small" topics: [] diff --git a/examples/backups/kafka/00-helpers.sh b/examples/backups/kafka/00-helpers.sh index 09b03fd6c..5ae81170a 100755 --- a/examples/backups/kafka/00-helpers.sh +++ b/examples/backups/kafka/00-helpers.sh @@ -92,8 +92,8 @@ resolve_kafka_image() { # is being assumed. On an air-gapped or operator-less cluster - the persona # the README's override note is for - that literal is unreachable, so the # run must set KAFKA_IMAGE to an image the nodes can actually pull. - echo "! could not read STRIMZI_KAFKA_IMAGES from deploy/strimzi-cluster-operator in cozy-kafka-operator; assuming quay.io/strimzi/kafka:0.45.1-rc1-kafka-3.9.1. Set KAFKA_IMAGE explicitly if that image is wrong or unreachable (e.g. air-gapped)." >&2 - echo "quay.io/strimzi/kafka:0.45.1-rc1-kafka-3.9.1" + echo "! could not read STRIMZI_KAFKA_IMAGES from deploy/strimzi-cluster-operator in cozy-kafka-operator; assuming quay.io/strimzi/kafka:1.2.0-kafka-4.3.1. Set KAFKA_IMAGE explicitly if that image is wrong or unreachable (e.g. air-gapped)." >&2 + echo "quay.io/strimzi/kafka:1.2.0-kafka-4.3.1" } export KAFKA_IMAGE="${KAFKA_IMAGE:-$(resolve_kafka_image)}" export KAFKA_BIN="${KAFKA_BIN:-/opt/kafka/bin}" @@ -215,7 +215,7 @@ seed_topic() { printf "k-%s\torder-%s\n" "$i" "$i" i=$((i + 1)) done | "$BIN"/kafka-console-producer.sh --bootstrap-server "$BOOT" \ - --topic "$TOPIC" --property parse.key=true + --topic "$TOPIC" --reader-property parse.key=true ' } @@ -231,7 +231,7 @@ seed_decoy_topic() { printf "d-%s\tdecoy-%s\n" "$i" "$i" i=$((i + 1)) done | "$BIN"/kafka-console-producer.sh --bootstrap-server "$BOOT" \ - --topic "$DECOY_TOPIC" --property parse.key=true + --topic "$DECOY_TOPIC" --reader-property parse.key=true ' } @@ -287,7 +287,7 @@ topic_dump() { [ "${end:-0}" -gt 0 ] || continue "$BIN"/kafka-console-consumer.sh --bootstrap-server "$BOOT" --topic "$TOPIC" \ --partition "$p" --offset 0 --max-messages "$end" --timeout-ms 60000 \ - --property print.key=true --property print.timestamp=false \ + --formatter-property print.key=true --formatter-property print.timestamp=false \ | sed "s/^/$p\t/" done ' diff --git a/examples/backups/kafka/01-create-strategy.sh b/examples/backups/kafka/01-create-strategy.sh index 26b8d95d9..7c57c1f65 100755 --- a/examples/backups/kafka/01-create-strategy.sh +++ b/examples/backups/kafka/01-create-strategy.sh @@ -324,7 +324,7 @@ spec: if [ "\${n}" -gt 0 ]; then "\${BIN}"/kafka-console-consumer.sh --bootstrap-server "\${BOOT}" \ --topic "\${t}" --partition "\${p}" --offset "\${begin}" --max-messages "\${n}" \ - --timeout-ms 120000 --property print.key=true --property print.timestamp=false \ + --timeout-ms 120000 --formatter-property print.key=true --formatter-property print.timestamp=false \ > "\${WORK}/data-\${t}-\${p}.tsv" # kafka-console-consumer exits 0 even on a short read: a # --timeout-ms expiry is swallowed inside ConsoleConsumer, @@ -509,7 +509,7 @@ spec: [ "\${mt}" = "\${t}" ] || continue [ "\${me}" -gt "\${mb}" ] || continue "\${BIN}"/kafka-console-producer.sh --bootstrap-server "\${BOOT}" \ - --topic "\${t}" --property parse.key=true < "\${WORK}/data-\${t}-\${mp}.tsv" + --topic "\${t}" --reader-property parse.key=true < "\${WORK}/data-\${t}-\${mp}.tsv" done < "\${WORK}/manifest.txt" echo " \${t}: replayed" done diff --git a/examples/backups/kafka/cleanup.sh b/examples/backups/kafka/cleanup.sh index 0e3340a48..7f06bab82 100755 --- a/examples/backups/kafka/cleanup.sh +++ b/examples/backups/kafka/cleanup.sh @@ -33,7 +33,7 @@ kubectl delete backupclass.backups.cozystack.io "$BACKUPCLASS_NAME" --ignore-not kubectl delete job.strategy.backups.cozystack.io "$STRATEGY_NAME" --ignore-not-found # `kubectl delete` returns as soon as deletionTimestamp is set, while the -# HelmRelease uninstall behind each Kafka app (two StatefulSets, ZooKeeper and +# HelmRelease uninstall behind each Kafka app (its broker and controller pods and # their PVCs) is still draining. Returning here lets the next suite start into a # namespace whose predecessor is still uninstalling - the neighbouring # chainsaw/kafka Test's PVC-reclaim step contends with exactly that. Wait for diff --git a/hack/e2e-chainsaw/kafka-metadata/chainsaw-test.yaml b/hack/e2e-chainsaw/kafka-metadata/chainsaw-test.yaml index c8b559e97..076015dc2 100644 --- a/hack/e2e-chainsaw/kafka-metadata/chainsaw-test.yaml +++ b/hack/e2e-chainsaw/kafka-metadata/chainsaw-test.yaml @@ -39,7 +39,7 @@ spec: kind: HelmRelease name: kafka-kafka-meta-src - describe: - apiVersion: kafka.strimzi.io/v1beta2 + apiVersion: kafka.strimzi.io/v1 kind: Kafka name: kafka-kafka-meta-src - describe: @@ -72,7 +72,7 @@ spec: check: ($error == null): true # 40m budget: two single-broker Strimzi clusters (source + to-copy target, - # each boots ZooKeeper then the broker and passes readiness), the metadata + # each boots its controller and broker and passes readiness), the metadata # backup, and the in-place + to-copy restores with their verifications. # run-all.sh fails fast on terminal BackupJob/RestoreJob phases and stalled # HelmReleases rather than polling out the budget. @@ -119,7 +119,7 @@ spec: - error: timeout: 12m resource: - apiVersion: kafka.strimzi.io/v1beta2 + apiVersion: kafka.strimzi.io/v1 kind: KafkaNodePool metadata: labels: @@ -127,7 +127,7 @@ spec: - error: timeout: 12m resource: - apiVersion: kafka.strimzi.io/v1beta2 + apiVersion: kafka.strimzi.io/v1 kind: KafkaNodePool metadata: labels: diff --git a/hack/e2e-chainsaw/kafka/chainsaw-test.yaml b/hack/e2e-chainsaw/kafka/chainsaw-test.yaml index dfeb9da2e..9f0edbbe7 100644 --- a/hack/e2e-chainsaw/kafka/chainsaw-test.yaml +++ b/hack/e2e-chainsaw/kafka/chainsaw-test.yaml @@ -3,7 +3,7 @@ # The chart deploys Kafka in KRaft mode (no ZooKeeper): a combined-free layout # of separate broker + controller KafkaNodePools. This test asserts the KRaft # topology renders and converges (both node pools present, the Strimzi Kafka CR -# reaches kafkaMetadataState=KRaft) and that no ZooKeeper resources remain. +# is Ready) and that no ZooKeeper resources remain. # The last step deletes the app itself and checks that no node pool or Strimzi # pod of this cluster survives the uninstall. apiVersion: chainsaw.kyverno.io/v1alpha1 @@ -46,9 +46,9 @@ spec: - name: wait-ready try: - # Deliberately under the floor: kafka-rd sets a 7h install timeout, which - # no budget inside the E2E job's cap can clear, so a failed install here - # reports as a bare assert timeout and the catch block's describe of the + # Deliberately under the floor, kafka-rd's default 10m install timeout: a + # healthy cluster is Ready in a few minutes, so a failed install ends the + # run early as a bare assert timeout, and the catch block's describe of the # HelmRelease is where its state is read. - assert: timeout: 6m @@ -73,7 +73,7 @@ spec: - assert: timeout: 5m resource: - apiVersion: kafka.strimzi.io/v1beta2 + apiVersion: kafka.strimzi.io/v1 kind: KafkaTopic metadata: name: kafka-test-test-results @@ -91,7 +91,7 @@ spec: - assert: timeout: 2m resource: - apiVersion: kafka.strimzi.io/v1beta2 + apiVersion: kafka.strimzi.io/v1 kind: KafkaNodePool metadata: labels: @@ -102,7 +102,7 @@ spec: - assert: timeout: 2m resource: - apiVersion: kafka.strimzi.io/v1beta2 + apiVersion: kafka.strimzi.io/v1 kind: KafkaNodePool metadata: labels: @@ -113,15 +113,13 @@ spec: - assert: timeout: 5m resource: - apiVersion: kafka.strimzi.io/v1beta2 + apiVersion: kafka.strimzi.io/v1 kind: Kafka metadata: name: kafka-test - annotations: - strimzi.io/kraft: enabled - strimzi.io/node-pools: enabled status: - kafkaMetadataState: KRaft + (conditions[?type == 'Ready']): + - status: "True" # KRaft mode ships no ZooKeeper: the client Service must never exist. - error: resource: @@ -161,7 +159,7 @@ spec: # period, which the default error timeout covers. - error: resource: - apiVersion: kafka.strimzi.io/v1beta2 + apiVersion: kafka.strimzi.io/v1 kind: KafkaNodePool metadata: labels: @@ -227,11 +225,11 @@ spec: kind: HelmRelease name: kafka-kafka-test - describe: - apiVersion: kafka.strimzi.io/v1beta2 + apiVersion: kafka.strimzi.io/v1 kind: Kafka name: kafka-kafka-test - describe: - apiVersion: kafka.strimzi.io/v1beta2 + apiVersion: kafka.strimzi.io/v1 kind: Kafka name: kafka-kafka-restore - describe: @@ -341,14 +339,14 @@ spec: try: - error: resource: - apiVersion: kafka.strimzi.io/v1beta2 + apiVersion: kafka.strimzi.io/v1 kind: KafkaNodePool metadata: labels: strimzi.io/cluster: kafka-kafka-test - error: resource: - apiVersion: kafka.strimzi.io/v1beta2 + apiVersion: kafka.strimzi.io/v1 kind: KafkaNodePool metadata: labels: diff --git a/hack/e2e-chainsaw/kafka/kafka.yaml b/hack/e2e-chainsaw/kafka/kafka.yaml index 9f1e25f14..3c107f27b 100644 --- a/hack/e2e-chainsaw/kafka/kafka.yaml +++ b/hack/e2e-chainsaw/kafka/kafka.yaml @@ -3,7 +3,7 @@ kind: Kafka metadata: name: test spec: - version: "v3.9" + version: "v4.3" external: false kafka: size: 10Gi diff --git a/hack/kafka-migration-hook.bats b/hack/kafka-migration-hook.bats index 65925385f..8ba753e75 100644 --- a/hack/kafka-migration-hook.bats +++ b/hack/kafka-migration-hook.bats @@ -1,19 +1,10 @@ #!/usr/bin/env bats -# Behavioural tests for the ZooKeeper->KRaft migration hook of -# packages/apps/kafka. The shell script is extracted from the rendered Job and -# executed against a stub kubectl, so the fail-closed reads, the atomic -# namespace-collision guard, the node-pool creation, the finalize guard and the -# "free the kafka name" drain are exercised for real. The helm-unittest cases in +# Behavioural tests for the node-pool hook of packages/apps/kafka. The shell +# script is extracted from the rendered Job and executed against a stub +# kubectl, so what the hook actually applies, and whether a failed apply fails +# the Helm upgrade, are exercised for real. The helm-unittest cases in # packages/apps/kafka/tests/migration-hook_test.yaml only match the script's -# source TEXT; they cannot say whether a stuck migration refuses to stamp -# kraft=enabled, or whether the drain drives the rebalance to completion before -# deleting the pool. -# -# The hook manages the node pools itself (no regular KafkaNodePool manifest), so -# it runs on install AND upgrade. `helm template` (no live lookup) resolves the -# broker pool to the fresh "b-" name; tests that must exercise the -# migration "kafka" pool (collision guard, create-CAS, name-freeing) force the -# resolved value with force_kafka_pool. +# source TEXT. # # Run from the repo root with make bats-unit-tests. Direct assertions and # inline cleanup retain compatibility with cozytest. Its translator rewrites @@ -30,87 +21,21 @@ render_script() { --show-only templates/migration-hook.yaml 2>"$1.err" | yq 'select(.kind == "Job") | .spec.template.spec.containers[0].args[0]' - > "$1" if [ ! -s "$1" ]; then - echo "FAIL: the chart rendered no migration Job script" >&2 + echo "FAIL: the chart rendered no node-pool Job script" >&2 grep -v 'found symbolic link in path' "$1.err" >&2 || true return 1 fi } -# Force the render-resolved broker pool from "b-" to "kafka", so the -# collision / create-CAS / name-freeing branches (gated on brokerPool == "kafka") -# run. `helm template` cannot seed the live lookup that resolves it. -force_kafka_pool() { - sed 's/brokerPool="b-[0-9a-f]\{8\}"/brokerPool="kafka"/' "$1" > "$1.k" && mv "$1.k" "$1" - } - make_bin() { mkdir -p "$1" - printf '#!/bin/sh\nexit 0\n' > "$1/sleep" - chmod 0755 "$1/sleep" cat > "$1/kubectl" <<'STUB' #!/bin/sh -args="$*" -echo "$args" >> "${KUBECTL_LOG:-/dev/null}" - -# Order matters: match the more specific jsonpath reads before the generic -# owner/kraft/found reads that share substrings. -case "$args" in - *"create -f -"*) - cat >> "${KUBECTL_LOG:-/dev/null}" - [ -n "${STUB_CREATE_FAIL:-}" ] && { echo "AlreadyExists" >&2; exit 1; } - exit 0 - ;; +echo "$*" >> "${KUBECTL_LOG:-/dev/null}" +case "$*" in *"apply "*"-f -"*) cat >> "${KUBECTL_LOG:-/dev/null}" - exit 0 - ;; - *"range .items"*) - # kafkas list for the "other ZooKeeper clusters remain" count. - [ -n "${STUB_OTHERS_LIST_FAIL:-}" ] && { echo "apiserver error" >&2; exit 1; } - printf '%s\n' "${STUB_OTHERS_LIST:-}" - exit 0 - ;; - *nodeIds*) - printf '%s' "${STUB_KAFKA_IDS:-}" - exit 0 - ;; - *].reason*) - # KafkaRebalance NotReady reason (retriable vs terminal). - printf '%s' "${STUB_REBALANCE_REASON:-}" - exit 0 - ;; - *conditions*) - # KafkaRebalance status.conditions[*].type, advanced via STUB_REBALANCE. - i=$(cat "${REBALANCE_COUNTER}" 2>/dev/null) - [ -n "$i" ] || i=0 - i=$((i + 1)) - echo "$i" > "${REBALANCE_COUNTER}" - printf '%s\n' "${STUB_REBALANCE:-}" | awk -v n="$i" '{a[NR]=$0} END{print (n<=NR)?a[n]:a[NR]}' - exit 0 - ;; - *kafkaMetadataState*) - [ -n "${STUB_STATE_FAIL:-}" ] && { echo "apiserver error" >&2; exit 1; } - i=$(cat "${STATE_COUNTER}" 2>/dev/null) - [ -n "$i" ] || i=0 - i=$((i + 1)) - echo "$i" > "${STATE_COUNTER}" - printf '%s\n' "${STUB_STATES:-}" | awk -v n="$i" '{a[NR]=$0} END{print (n<=NR)?a[n]:a[NR]}' - exit 0 - ;; - *cluster}*) - [ -n "${STUB_OWNER_FAIL:-}" ] && { echo "apiserver error" >&2; exit 1; } - printf '%s\n' "${STUB_OWNER:-}" - exit 0 - ;; - *kraft}*) - [ -n "${STUB_KRAFT_FAIL:-}" ] && { echo "apiserver error" >&2; exit 1; } - printf '%s\n' "${STUB_KRAFT:-}" - exit 0 - ;; - *--ignore-not-found*) - [ -n "${STUB_FOUND_FAIL:-}" ] && { echo "apiserver error" >&2; exit 1; } - printf '%s' "${STUB_FOUND:-}" - [ -n "${STUB_FOUND:-}" ] && echo + [ -n "${STUB_APPLY_FAIL:-}" ] && { echo "apiserver error" >&2; exit 1; } exit 0 ;; esac @@ -128,38 +53,26 @@ setup_case() { run_hook() { rc=0 : > "$tmp/kubectl.log" - : > "$tmp/state.counter" - : > "$tmp/rebalance.counter" KUBECONFIG=/dev/null \ KUBECTL_LOG="$tmp/kubectl.log" \ - STATE_COUNTER="$tmp/state.counter" \ - REBALANCE_COUNTER="$tmp/rebalance.counter" \ - STUB_FOUND="${STUB_FOUND:-}" \ - STUB_FOUND_FAIL="${STUB_FOUND_FAIL:-}" \ - STUB_KRAFT="${STUB_KRAFT:-}" \ - STUB_STATES="${STUB_STATES:-}" \ - STUB_OWNER="${STUB_OWNER:-}" \ - STUB_OWNER_FAIL="${STUB_OWNER_FAIL:-}" \ - STUB_KRAFT_FAIL="${STUB_KRAFT_FAIL:-}" \ - STUB_STATE_FAIL="${STUB_STATE_FAIL:-}" \ - STUB_CREATE_FAIL="${STUB_CREATE_FAIL:-}" \ - STUB_OTHERS_LIST="${STUB_OTHERS_LIST:-}" \ - STUB_OTHERS_LIST_FAIL="${STUB_OTHERS_LIST_FAIL:-}" \ - STUB_KAFKA_IDS="${STUB_KAFKA_IDS:-}" \ - STUB_REBALANCE="${STUB_REBALANCE:-}" \ - STUB_REBALANCE_REASON="${STUB_REBALANCE_REASON:-}" \ + STUB_APPLY_FAIL="${STUB_APPLY_FAIL:-}" \ PATH="$tmp/bin:$PATH" sh "$tmp/s.sh" > "$tmp/out" 2>&1 || rc=$? [ -s "$tmp/kubectl.log" ] || return 1 } -@test "a NotFound Kafka CR is a fresh install: pools ensured, no migration" { +@test "the hook applies the broker and the controller pool through the v1 API" { setup_case - STUB_FOUND="" run_hook + run_hook [ "$rc" = 0 ] - grep -qF 'fresh install' "$tmp/out" - if grep -qE 'kraft=(migration|enabled)|node-pools=enabled' "$tmp/kubectl.log"; then - echo "FAIL: the fresh-install path drove the migration state machine" + applies=$(grep -c 'apply --openapi-patch=false -f -' "$tmp/kubectl.log") + [ "$applies" = 2 ] + pools=$(grep -c '^apiVersion: kafka.strimzi.io/v1$' "$tmp/kubectl.log") + [ "$pools" = 2 ] + grep -qE '^ *- broker$' "$tmp/kubectl.log" + grep -qE '^ *- controller$' "$tmp/kubectl.log" + if grep -qE 'v1beta2|strimzi.io/kraft|KafkaRebalance|(^| )(annotate|delete|patch|create) ' "$tmp/kubectl.log"; then + echo "FAIL: the hook did more than apply the two pools" cat "$tmp/kubectl.log" return 1 fi @@ -167,267 +80,17 @@ run_hook() { rm -rf "$tmp" } -@test "a read error reading the Kafka CR fails closed, never mistaken for fresh" { +@test "a failed pool apply fails the hook" { setup_case - STUB_FOUND_FAIL=1 run_hook + STUB_APPLY_FAIL=1 run_hook [ "$rc" != 0 ] - grep -qF 'reading Kafka CR failed' "$tmp/out" - if grep -qF 'fresh install' "$tmp/out"; then - echo "FAIL: an apiserver error was read as a fresh install" - cat "$tmp/out" - return 1 - fi - - rm -rf "$tmp" -} - -@test "an already-KRaft cluster on b- is a no-op with nothing to free" { - setup_case - STUB_FOUND="kafka.kafka.strimzi.io/test-kafka" STUB_KRAFT="enabled" run_hook - - [ "$rc" = 0 ] - grep -qF 'already KRaft' "$tmp/out" - grep -qF 'nothing to free' "$tmp/out" - if grep -qE 'kraft=(migration|enabled)|node-pools=enabled' "$tmp/kubectl.log"; then - echo "FAIL: an already-KRaft cluster was driven through the state machine" + applies=$(grep -c 'apply --openapi-patch=false -f -' "$tmp/kubectl.log") + if [ "$applies" != 1 ]; then + echo "FAIL: the hook went on after the broker pool apply failed ($applies applies)" cat "$tmp/kubectl.log" return 1 fi rm -rf "$tmp" } - -@test "a kafka pool owned by another cluster fails fast before any mutation" { - setup_case - force_kafka_pool "$tmp/s.sh" - STUB_FOUND="kafka.kafka.strimzi.io/test-kafka" STUB_KRAFT="" \ - STUB_STATES="ZooKeeper" STUB_OWNER="other-kafka" run_hook - - [ "$rc" != 0 ] - grep -qF 'already belongs to cluster' "$tmp/out" - if grep -qE 'apply .*-f -|create -f -|kraft=|node-pools=' "$tmp/kubectl.log"; then - echo "FAIL: a foreign-owned kafka pool did not abort before mutating" - cat "$tmp/kubectl.log" - return 1 - fi - - rm -rf "$tmp" -} - -@test "a failed read of the kafka pool owner fails closed" { - setup_case - force_kafka_pool "$tmp/s.sh" - STUB_FOUND="kafka.kafka.strimzi.io/test-kafka" STUB_KRAFT="" \ - STUB_STATES="ZooKeeper" STUB_OWNER_FAIL=1 run_hook - - [ "$rc" != 0 ] - grep -qF 'owner label failed' "$tmp/out" - if grep -qE 'apply .*-f -|create -f -|kraft=|node-pools=' "$tmp/kubectl.log"; then - echo "FAIL: proceeded to mutate the cluster after a failed owner read" - cat "$tmp/kubectl.log" - return 1 - fi - - rm -rf "$tmp" -} - -@test "a failed read of the kraft annotation fails closed" { - setup_case - STUB_FOUND="kafka.kafka.strimzi.io/test-kafka" STUB_KRAFT_FAIL=1 run_hook - - [ "$rc" != 0 ] - grep -qF 'strimzi.io/kraft annotation failed' "$tmp/out" - if grep -qE 'kraft=(migration|enabled)|node-pools=enabled' "$tmp/kubectl.log"; then - echo "FAIL: drove the migration after a failed kraft-annotation read" - cat "$tmp/kubectl.log" - return 1 - fi - - rm -rf "$tmp" -} - -@test "a classic ZooKeeper cluster migrates and finalizes to KRaft" { - setup_case - STUB_FOUND="kafka.kafka.strimzi.io/test-kafka" STUB_KRAFT="" STUB_OWNER="" \ - STUB_STATES="ZooKeeper -KRaftPostMigration -KRaft" run_hook - - [ "$rc" = 0 ] - grep -qF 'Migration complete' "$tmp/out" - grep -qF 'node-pools=enabled' "$tmp/kubectl.log" - grep -qF 'kraft=migration' "$tmp/kubectl.log" - grep -qF 'kraft=enabled' "$tmp/kubectl.log" - - rm -rf "$tmp" -} - -@test "a migration stuck before KRaftPostMigration refuses to stamp kraft=enabled" { - setup_case - STUB_FOUND="kafka.kafka.strimzi.io/test-kafka" STUB_KRAFT="" STUB_OWNER="" \ - STUB_STATES="ZooKeeper" run_hook - - [ "$rc" != 0 ] - grep -qF 'Refusing to apply' "$tmp/out" - if grep -qF 'kraft=enabled' "$tmp/kubectl.log"; then - echo "FAIL: kraft=enabled was stamped while the cluster was still on ZooKeeper" - cat "$tmp/kubectl.log" - return 1 - fi - - rm -rf "$tmp" -} - -@test "a retry already past the ZooKeeper phase does not re-issue kraft=migration" { - setup_case - STUB_FOUND="kafka.kafka.strimzi.io/test-kafka" STUB_KRAFT="" STUB_OWNER="" \ - STUB_STATES="KRaftPostMigration -KRaftPostMigration -KRaft" run_hook - - [ "$rc" = 0 ] - grep -qF 'not re-issuing kraft=migration' "$tmp/out" - grep -qF 'kraft=enabled' "$tmp/kubectl.log" - if grep -qF 'kraft=migration' "$tmp/kubectl.log"; then - echo "FAIL: kraft=migration was re-issued from a post-ZooKeeper state" - cat "$tmp/kubectl.log" - return 1 - fi - - rm -rf "$tmp" -} - -# --- Name-freeing (multi Kafka per namespace) ------------------------------- - -@test "the sole migrated cluster keeps the kafka pool, no drain" { - setup_case - force_kafka_pool "$tmp/s.sh" - # Already KRaft on kafka; no other ZooKeeper clusters in the namespace. - STUB_FOUND="kafka.kafka.strimzi.io/test-kafka" STUB_KRAFT="enabled" \ - STUB_OTHERS_LIST="test-kafka=enabled" run_hook - - [ "$rc" = 0 ] - grep -qF 'keeping the "kafka" pool' "$tmp/out" - # No drain: no rebalance, no cruiseControl, no pool deletion. - if grep -qE 'KafkaRebalance|cruiseControl|delete .*kafkanodepools' "$tmp/kubectl.log"; then - echo "FAIL: a sole cluster ran the name-freeing drain" - cat "$tmp/kubectl.log" - return 1 - fi - - rm -rf "$tmp" -} - -@test "a failed read of the other-clusters list fails closed, not mistaken for none" { - setup_case - force_kafka_pool "$tmp/s.sh" - # Already KRaft on kafka; the list read that decides whether others wait errors. - STUB_FOUND="kafka.kafka.strimzi.io/test-kafka" STUB_KRAFT="enabled" \ - STUB_OTHERS_LIST_FAIL=1 run_hook - - [ "$rc" != 0 ] - grep -qF 'could not list Kafka CRs' "$tmp/out" - if grep -qF 'keeping the "kafka" pool' "$tmp/out"; then - echo "FAIL: a failed list read was mistaken for 'no other clusters' and kept the pool" - cat "$tmp/out" - return 1 - fi - - rm -rf "$tmp" -} - -@test "a migrated cluster with others waiting drains and frees the kafka name" { - setup_case - force_kafka_pool "$tmp/s.sh" - # Already KRaft on kafka; one other cluster still on ZooKeeper. - STUB_FOUND="kafka.kafka.strimzi.io/test-kafka" STUB_KRAFT="enabled" \ - STUB_OTHERS_LIST="test-kafka=enabled -other-zk=" \ - STUB_KAFKA_IDS="[0,1,2]" \ - STUB_REBALANCE="PendingProposal -ProposalReady -Ready" run_hook - - [ "$rc" = 0 ] - grep -qF 'freeing the "kafka" pool name' "$tmp/out" - grep -qF 'Freed the "kafka" pool name' "$tmp/out" - # Full drain sequence, in order: target pool, Cruise Control, rebalance, - # approval, delete kafka pool + PVCs. - grep -qF '1000-1099' "$tmp/kubectl.log" # applied the b- target pool - grep -qF 'cruiseControl' "$tmp/kubectl.log" - grep -qF 'rebalance=approve' "$tmp/kubectl.log" - grep -qF 'delete kafkanodepools.kafka.strimzi.io kafka' "$tmp/kubectl.log" - grep -qF 'delete pvc data-0-test-kafka-kafka-0' "$tmp/kubectl.log" - grep -qF 'delete pvc data-0-test-kafka-kafka-2' "$tmp/kubectl.log" - - rm -rf "$tmp" -} - -@test "the drain refuses to delete the kafka pool if the rebalance never completes" { - setup_case - force_kafka_pool "$tmp/s.sh" - STUB_FOUND="kafka.kafka.strimzi.io/test-kafka" STUB_KRAFT="enabled" \ - STUB_OTHERS_LIST="test-kafka=enabled -other-zk=" \ - STUB_KAFKA_IDS="[0,1,2]" \ - STUB_REBALANCE="PendingProposal" run_hook - - [ "$rc" != 0 ] - grep -qF 'will resume on the next retry' "$tmp/out" - # Must NOT delete the pool (which still holds all the data) before the drain. - if grep -qF 'delete kafkanodepools.kafka.strimzi.io kafka' "$tmp/kubectl.log"; then - echo "FAIL: deleted the kafka pool before the rebalance finished — data loss" - cat "$tmp/kubectl.log" - return 1 - fi - - rm -rf "$tmp" -} - -# Cruise Control rejects proposals transiently while it settles — the pod/cert -# are still coming up (retriable connect), or the capacity model over-estimates -# load from noisy samples and trips a goal. The drain must refresh the proposal -# and keep polling, never treat a NotReady as terminal. Covers both reasons. -@test "the drain refreshes a transient-connection NotReady and proceeds" { - setup_case - force_kafka_pool "$tmp/s.sh" - STUB_FOUND="kafka.kafka.strimzi.io/test-kafka" STUB_KRAFT="enabled" \ - STUB_OTHERS_LIST="test-kafka=enabled -other-zk=" \ - STUB_KAFKA_IDS="[0,1,2]" \ - STUB_REBALANCE_REASON="CruiseControlRetriableConnectionException" \ - STUB_REBALANCE="PendingProposal -NotReady -ProposalReady -Ready" run_hook - - [ "$rc" = 0 ] - grep -qF 'refreshing the proposal' "$tmp/out" - grep -qF 'rebalance=refresh' "$tmp/kubectl.log" - grep -qF 'Freed the "kafka" pool name' "$tmp/out" - grep -qF 'delete kafkanodepools.kafka.strimzi.io kafka' "$tmp/kubectl.log" - - rm -rf "$tmp" -} - -@test "the drain refreshes and recovers from a capacity-goal NotReady" { - setup_case - force_kafka_pool "$tmp/s.sh" - STUB_FOUND="kafka.kafka.strimzi.io/test-kafka" STUB_KRAFT="enabled" \ - STUB_OTHERS_LIST="test-kafka=enabled -other-zk=" \ - STUB_KAFKA_IDS="[0,1,2]" \ - STUB_REBALANCE_REASON="OptimizationFailureException" \ - STUB_REBALANCE="PendingProposal -NotReady -ProposalReady -Ready" run_hook - - [ "$rc" = 0 ] - grep -qF 'refreshing the proposal' "$tmp/out" - grep -qF 'rebalance=refresh' "$tmp/kubectl.log" - grep -qF 'Freed the "kafka" pool name' "$tmp/out" - - rm -rf "$tmp" -} diff --git a/hack/kafka-pre-delete-hook.bats b/hack/kafka-pre-delete-hook.bats index 0ab408294..e8af58aec 100644 --- a/hack/kafka-pre-delete-hook.bats +++ b/hack/kafka-pre-delete-hook.bats @@ -135,7 +135,7 @@ KEOF # gone. Failing here would wedge the uninstall over nothing to clean up. Both # RESTMapper wordings are covered because kubectl picks between them. for msg in 'error: the server doesn'"'"'t have a resource type "kafkatopics"' \ - 'error: no matches for kind "KafkaTopic" in version "kafka.strimzi.io/v1beta2"'; do + 'error: no matches for kind "KafkaTopic" in version "kafka.strimzi.io/v1"'; do tmp=$(mktemp -d) write_fake_kubectl "$tmp" render_hook "$KAFKA_CHART" kafka-test tenant-test kafka-test-pre-delete "$tmp/hook.sh" @@ -289,7 +289,7 @@ KEOF @test "kafka pre-delete hook exits 0 when the KafkaNodePool CRD is absent" { for msg in 'error: the server doesn'"'"'t have a resource type "kafkanodepools"' \ - 'error: no matches for kind "KafkaNodePool" in version "kafka.strimzi.io/v1beta2"'; do + 'error: no matches for kind "KafkaNodePool" in version "kafka.strimzi.io/v1"'; do tmp=$(mktemp -d) write_fake_kubectl "$tmp" render_hook "$KAFKA_CHART" kafka-test tenant-test kafka-test-pre-delete "$tmp/hook.sh" diff --git a/hack/kafka-topic-backup-strategy.bats b/hack/kafka-topic-backup-strategy.bats index ce233e0d8..57830d8a7 100644 --- a/hack/kafka-topic-backup-strategy.bats +++ b/hack/kafka-topic-backup-strategy.bats @@ -116,7 +116,9 @@ topic=""; p=""; off=0; max=0 while [ $# -gt 0 ]; do case "$1" in --topic) topic="$2"; shift ;; --partition) p="$2"; shift ;; --offset) off="$2"; shift ;; - --max-messages) max="$2"; shift ;; --bootstrap-server|--timeout-ms|--property) shift ;; + --max-messages) max="$2"; shift ;; --bootstrap-server|--timeout-ms|--formatter-property) shift ;; + # Kafka 4 prints this on stdout, ahead of the records. + --property) echo "Option --property is deprecated and will be removed in a future version. Use --formatter-property instead."; shift ;; esac shift done @@ -131,7 +133,10 @@ STUB set -eu topic="" while [ $# -gt 0 ]; do - case "$1" in --topic) topic="$2"; shift ;; --bootstrap-server|--property) shift ;; esac + case "$1" in + --topic) topic="$2"; shift ;; --bootstrap-server|--reader-property) shift ;; + --property) echo "Warning: --property is deprecated and will be removed in a future version. Use --reader-property instead."; shift ;; + esac shift done [ -d "$BROKER/$topic" ] || { echo "no such topic $topic" >&2; exit 1; } diff --git a/packages/apps/kafka/Chart.yaml b/packages/apps/kafka/Chart.yaml index 7acada07e..9d88b84fe 100644 --- a/packages/apps/kafka/Chart.yaml +++ b/packages/apps/kafka/Chart.yaml @@ -4,4 +4,4 @@ description: Managed Kafka service icon: /logos/kafka.svg type: application version: 0.0.0 # Placeholder, the actual version will be automatically set during the build process -appVersion: "3.9.0" +appVersion: "4.3.1" diff --git a/packages/apps/kafka/README.md b/packages/apps/kafka/README.md index 651e6e8d2..9872d554b 100644 --- a/packages/apps/kafka/README.md +++ b/packages/apps/kafka/README.md @@ -1,5 +1,27 @@ # Managed Kafka Service +## Node pools + +The chart runs Kafka in KRaft mode with two `KafkaNodePool` resources, one for brokers and one for controllers. The pools are not regular chart manifests: a `pre-install,pre-upgrade` Helm hook applies them on every install and upgrade, before the main release apply. Every pool that exists today was created by that hook and is absent from every release manifest. Follow a run with `kubectl logs -n job/-kafka-migration`. + +Each pool gets a short, release-hashed name (`b-` for brokers, `c-` for controllers). `KafkaNodePool` names are namespace-unique and each release hashes differently, so several Kafka instances can live in one namespace. The hash keeps the pool name at 10 characters, so the derived `--` pod hostname grows only with the release name and the node ID. At the 44-character `name` limit it fits the 63-character limit only while every node ID is a single digit; controllers take node IDs from `kafka.replicas` upward. + +A cluster migrated from ZooKeeper by an earlier chart keeps its broker pool named `kafka`: that pool adopted the cluster's original `-kafka-N` brokers and their data. The controller pool pins its node IDs above the broker range (`strimzi.io/next-node-ids`), so the broker pool keeps `[0..replicas-1]`. + +### Deletion and PVC retention + +Both node pools set `deleteClaim: false`, so deleting a Kafka release intentionally leaves its data PVCs (`data-0---N`) behind. The trade-off is that the PVCs keep consuming tenant quota until removed by hand, and recreating a same-named Kafka rebinds the stale volumes, whose on-disk cluster id will not match the new cluster (`InconsistentClusterIdException`, CrashLoop). To truly start over, delete the leftover PVCs before recreating: `kubectl -n delete pvc -l strimzi.io/cluster=`. + +### Upgrading from ZooKeeper and from Kafka 3.x + +The operator behind this chart (Strimzi 1.x) runs neither ZooKeeper nor Kafka 3.x. A ZooKeeper-based instance must reach KRaft on the Cozystack release that ships the ZooKeeper to KRaft migration (v1.7) before the platform moves past it. On every platform variant, the platform upgrade refuses to switch package sources while Strimzi still serves `v1beta2` and any Kafka lacks confirmed KRaft state. On the variants that run platform migrations, migration 61 checks the same evidence again, including on a repeat run after API conversion. Every Kafka must report a nonempty `kafkaMetadataVersion` and either `kafkaMetadataState: KRaft` or, without a state, `Ready=True` or the `cozystack.io/kraft-verified` annotation that migration 61 records on every Kafka it accepts before it stops the Strimzi operators. A `kafkaMetadataState` other than `KRaft`, a ZooKeeper section or a missing status blocks the upgrade, and the error names the reason for each Kafka. The API conversion deletes `kafkaMetadataState` and may change the spec, and neither invalidates that evidence on a retry. The migration also moves every instance on `v3.8` or `v3.9` to `v4.3`; Strimzi then rolls the brokers onto Kafka 4.3. Elsewhere those steps are manual, and the platform upgrade refuses to start while Strimzi still stores `v1beta2`: before upgrading the platform, run `/migrations/61` from this release's `platform-migrations` image with cluster-admin credentials. That run is needed on every such cluster that has the kafka-operator package installed, through the `paas` bundle or `bundles.enabledPackages`, even if it runs no Kafka: the Strimzi CRDs list `v1beta2` in their stored versions whether or not any object exists. The run stamps platform version 62 in the `cozystack-version` ConfigMap of `cozy-system`, and the migration runner starts from that stamp, so if the cluster later moves to an `isp-*` variant, the platform migrations numbered below 62 do not run there. Before it changes anything, the run checks the Kafkas, their topics and every Strimzi object, and refuses with the name and reason of each one it cannot move; it can be repeated after a failure. It then stops the Strimzi operators and converts with a Job from `quay.io/strimzi/operator:0.51.0`, pinned by digest, so a cluster that cannot reach quay.io needs that image mirrored under the same name and digest before the run. Until the platform upgrade the operators stay stopped, so a broker pod that is evicted or deleted is not recreated, and the HelmRelease of every application moved to `v4.3` reports a values error, because the old chart accepts only `v3.8` and `v3.9`: upgrade the platform right after the run. Outside that image the script, `packages/core/platform/images/migrations/migrations/61`, needs the `lib/` directory next to it and `kubectl`, `jq`, `gzip` and `base64`, plus `STRIMZI_CRD_FILE` pointing at `strimzi-crds-0.51.0.yaml` from the [Strimzi 0.51.0 release](https://github.com/strimzi/strimzi-kafka-operator/releases/tag/0.51.0). + +Kafka 4 no longer accepts clients older than Kafka 2.1: upgrade such clients before the platform upgrade. Kafka 4 also removed the `message.format.version`, `message.downconversion.enable` and `message.timestamp.difference.max.ms` topic settings, and it requires `segment.bytes` of at least 1048576 (1 MiB) where Kafka 3.x accepted much smaller values; a broker whose log holds a topic below that does not start. The platform upgrade refuses to start while a Kafka application sets one of them, migration 61 also refuses while a KafkaTopic does, and the chart refuses to render such a topic, so remove the removed settings and raise `segment.bytes` in the `config` of every topic before the upgrade. A topic created through the Kafka admin API without a KafkaTopic is not visible to these checks. + +### No downgrade + +Neither lowering the `version` field on a running cluster nor rolling the app back to an earlier chart is supported: Strimzi refuses an in-place downgrade of the KRaft metadata version, and an earlier chart renders resources for an operator API this platform no longer serves. Take a backup before upgrading. + ## Parameters ### Common parameters @@ -9,7 +31,7 @@ | `external` | Enable external access from outside the cluster. | `bool` | `false` | | `tls` | TLS configuration. Strimzi manages the cluster PKI automatically (no cert-manager is involved for this chart): the operator auto-creates `-cluster-ca-cert` and `-clients-ca-cert` secrets, both exposed for client trust setup. The internal TLS listener on 9093 is always on; this toggle only controls the external listener on 9094. | `object` | `{}` | | `tls.enabled` | Enable TLS on the external listener. When unset, inherits the value of `external` (TLS is on when external access is enabled). Warning: setting this to false while external is true exposes Kafka over plaintext on a public IP via LoadBalancer. Strimzi does not provide authentication on this listener unless SCRAM, mTLS, or OAuth is separately configured. Use only in controlled networks. | `*bool` | `null` | -| `version` | Kafka version to deploy. Upgrade-only: once a cluster's KRaft metadata is at a given version, Strimzi refuses an in-place downgrade, so lowering this on a running cluster (e.g. v3.9 back to v3.8) leaves the Kafka CR stuck in a reconcile error. Pick the target version at creation and only ever raise it. | `string` | `v3.9` | +| `version` | Kafka version to deploy. Upgrade-only: once a cluster's KRaft metadata is at a given version, Strimzi refuses an in-place downgrade, so lowering this on a running cluster (e.g. v4.3 back to v4.2) leaves the Kafka CR stuck in a reconcile error. Pick the target version at creation and only ever raise it. | `string` | `v4.3` | ### Application-specific parameters @@ -35,7 +57,7 @@ | `kafka.resourcesPreset` | Default sizing preset used when `resources` is omitted. | `string` | `c1.small` | | `kafka.size` | Persistent Volume size for Kafka. | `quantity` | `10Gi` | | `kafka.storageClass` | StorageClass used to store the Kafka data. | `string` | `""` | -| `kafka.controllerStorageSize` | Persistent Volume size for KRaft controller metadata (used during ZK-to-KRaft migration). | `quantity` | `5Gi` | +| `kafka.controllerStorageSize` | Persistent Volume size for KRaft controller metadata. | `quantity` | `5Gi` | ## Parameter examples and reference @@ -76,47 +98,3 @@ topics: partitions: 1 replicas: 3 ``` - -## ZooKeeper to KRaft Migration - -The chart is pure KRaft — it ships a Kafka CR annotated `strimzi.io/kraft: enabled` with no `spec.zookeeper` block. The broker and controller `KafkaNodePool` resources are **not** regular chart manifests: they are created and reconciled by the migration Job (a `pre-install,pre-upgrade` Helm hook) on every install and upgrade. This is deliberate. The hook has to create the pools and walk Strimzi's migration state machine *before* the main release apply, and a resource a hook creates cannot also be a normal chart manifest without Helm failing the migrating upgrade — its 3-way merge finds the pool in the new manifest and live in the cluster but absent from the previous (ZooKeeper) revision, and aborts with `original object KafkaNodePool ... not found`, leaving the release stuck `failed` even though the cluster migrated fine. Keeping the pools hook-owned avoids that whole class of conflict. - -Existing ZooKeeper-based instances are migrated automatically on the next chart upgrade. The Job is idempotent and safe to re-run. - -### How it works - -1. The Job runs on install and upgrade. It first ensures both `KafkaNodePool`s exist (creating or updating them), then decides what to do from the Kafka CR. -2. It inspects the Kafka CR: a genuine NotFound means a fresh install (the pools are ensured and the main apply creates the KRaft Kafka CR — nothing to migrate); a CR already annotated `strimzi.io/kraft=enabled` or in `status.kafkaMetadataState: KRaft` means done; any other read error aborts (fail closed) rather than risk driving a live ZooKeeper cluster. Otherwise (typically `ZooKeeper` state) it migrates. -3. For a migrating cluster the broker pool is named exactly `kafka` — Strimzi derives broker and PVC names as `--`, so only that name reuses the existing `-kafka-N` brokers and their data in place — while the controller pool pins its node IDs above the broker range (`strimzi.io/next-node-ids`) so the broker pool keeps `[0..replicas-1]` and Strimzi cannot hand the existing broker IDs to the fresh controllers (which would recreate the brokers empty and orphan the data). Fresh clusters use the release-hashed `b-` broker pool instead. -4. It annotates the Kafka CR with `strimzi.io/node-pools=enabled` and `strimzi.io/kraft=migration`, then polls `status.kafkaMetadataState` and waits for the migration to reach `KRaftPostMigration | PreKRaft | KRaft`. -5. Only once that safe state is reached does it flip the annotation to `strimzi.io/kraft=enabled` and wait until the state reaches `KRaft`; if the wait times out in an intermediate state the Job aborts (`exit 1`) so Helm fails the upgrade and the next reconcile retries from where the migration left off. -6. When the Job succeeds, the main release apply settles the KRaft Kafka CR, which already matches the migrated state. - -### Observability and escape hatches - -- Tail the Job logs to follow migration progress: `kubectl logs -n job/-kafka-migration` -- Monitor `status.kafkaMetadataState` on the Kafka CR directly. -- Until the cluster reaches `PreKRaft`, Strimzi's `rollback` annotation stays available as a manual escape hatch: `kubectl annotate kafka strimzi.io/kraft=rollback --overwrite` (back to `KRaftDualWriting`), then `strimzi.io/kraft=disabled` to return to `ZooKeeper`; then delete the failed Job and retry. Past `PreKRaft` the state machine accepts only `enabled`. - -### Multiple Kafka clusters per namespace - -Fresh KRaft clusters can coexist in one namespace: each one's node pools get short, release-hashed names (`b-` for brokers, `c-` for controllers), unique because `KafkaNodePool` object names are namespace-unique and each release hashes differently, while every pod/PVC/Service is already prefixed by the (unique) cluster name. The hash keeps the pool name a fixed length rather than embedding the release name — which Strimzi already prepends as the cluster name — so the derived `--` pod hostname stays within Kubernetes' 63-character limit even for long release names. So you can deploy any number of new Kafka instances side by side. - -The ZooKeeper→KRaft migration path is more constrained, but still fully automatic. To adopt an existing cluster's `-kafka-N` brokers and their data in place, the broker pool must be named exactly `kafka`, and that fixed name is namespace-unique — so co-namespaced ZooKeeper clusters cannot all migrate at once, they must serialize on the `kafka` name (Strimzi's own guidance is one Kafka per namespace for this reason — [strimzi/strimzi-kafka-operator discussions/11120](https://github.com/orgs/strimzi/discussions/11120)). The migration hook handles that serialization automatically: whichever cluster's upgrade wins the `kafka` name (claimed atomically via `kubectl create`) migrates in place; a cluster that loses the race fails its hook and its next reconcile retries. Once a migrated cluster sees other ZooKeeper clusters still waiting, it frees the name — it stands up its own `b-` broker pool, drains all data off the `kafka`-pool brokers with a Cruise Control `KafkaRebalance` (`mode: remove-brokers`, done online, no downtime), deletes the drained `kafka` pool, and the next waiting cluster's retry adopts the freed name. The last cluster to migrate keeps `kafka`. So N co-namespaced ZooKeeper clusters all reach KRaft with no manual steps — sequentially, and with one full data drain per cluster except the last. Cruise Control is enabled only for the drain and removed after. Fresh KRaft clusters are never involved in any of this. - -### Deletion and PVC retention - -Both node pools set `deleteClaim: false`, so deleting a Kafka release intentionally leaves its data PVCs (`data-0---N`) behind — this protects data across the migration and against accidental deletion. The trade-off is that the PVCs keep consuming tenant quota until removed by hand, and recreating a same-named Kafka rebinds the stale volumes, whose on-disk cluster id will not match the new cluster (`InconsistentClusterIdException`, CrashLoop). To truly start over, delete the leftover PVCs before recreating: `kubectl -n delete pvc -l strimzi.io/cluster=`. - -### No downgrade across the KRaft boundary - -Migration to KRaft is a point of no return once the cluster passes `PreKRaft` (before that, the escape hatch above still works). After it, Strimzi has no KRaft→ZooKeeper rollback, so neither lowering the `version` field on a running cluster nor rolling the whole app back to a pre-KRaft chart is supported: rolling the chart back would re-render the `spec.zookeeper` objects over live KRaft storage and drop the `strimzi.io/kraft` / `strimzi.io/node-pools` annotations — that does not restore ZooKeeper, it **damages** the Kafka cluster. Take a backup first, and never downgrade the kafka app or the platform once a cluster has reached KRaft (`strimzi.io/kraft=enabled`). - -**This chart release is a mandatory stop on the upgrade path.** A cluster that skips it arrives at a future Strimzi (0.46+) still on ZooKeeper, and 0.46 removed the ZooKeeper→KRaft migration entirely — there is then no supported path off ZooKeeper. Make sure every Kafka in the platform has reached KRaft on this release before upgrading past it. - -Budget a maintenance window of **up to a few hours per instance**: the in-place migration is tens of minutes (broker rolling restarts), and when several Kafkas share a namespace each one but the last also drains its full dataset off the shared `kafka` pool via a Cruise Control rebalance to free the name, which scales with data size. - -### Important notes - -- **Strimzi 0.45 is the last version supporting ZooKeeper.** Future Strimzi releases only support KRaft. -- The `kafka.controllerStorageSize` parameter controls PV size for the new KRaft controller nodes (default: `5Gi`). diff --git a/packages/apps/kafka/files/versions.yaml b/packages/apps/kafka/files/versions.yaml index caf329aee..64dedf4f9 100644 --- a/packages/apps/kafka/files/versions.yaml +++ b/packages/apps/kafka/files/versions.yaml @@ -1,2 +1,2 @@ -"v3.9": "3.9.0" -"v3.8": "3.8.1" +"v4.3": "4.3.1" +"v4.2": "4.2.1" diff --git a/packages/apps/kafka/templates/_versions.tpl b/packages/apps/kafka/templates/_versions.tpl index 30c63b3ef..dc8bbf91b 100644 --- a/packages/apps/kafka/templates/_versions.tpl +++ b/packages/apps/kafka/templates/_versions.tpl @@ -16,23 +16,17 @@ deterministic token — a role letter plus an 8-hex digest of the release — which is unique within the namespace (each release hashes differently) and adds a fixed ~10 characters regardless of release length. The broker pool of a - MIGRATED cluster is the one exception: it stays exactly "kafka" to adopt the - existing -kafka-N brokers and their data (see kafka.brokerPoolName). + cluster migrated from ZooKeeper is the one exception: it stays exactly "kafka", + the pool that adopted its -kafka-N brokers (see kafka.brokerPoolName). */ -}} {{- define "kafka.poolHash" -}} {{- substr 0 8 (sha256sum .Release.Name) -}} {{- end -}} {{- /* - The controller pool name is a pure function of the release — no adoption/ - self-ownership lookup like kafka.brokerPoolName has, on purpose. A KRaft - controller pool is ALWAYS created new: a fresh install makes one, and a - ZooKeeper cluster has no controller pool to adopt (KRaft controllers do not - exist under ZooKeeper). So there is never a pre-existing, differently-named - controller pool for this name to preserve — the broker's "kafka"-adoption - branch has no controller analogue because the underlying situation it handles - (live brokers to reuse in place) has no controller counterpart. No released - chart ships a controller pool under any other name, so no upgrade renames one. + The controller pool name is a pure function of the release, with no ownership + lookup like kafka.brokerPoolName has: no released chart ships a controller + pool under any other name, so no upgrade renames one. */ -}} {{- define "kafka.controllerPoolName" -}} {{- printf "c-%s" (include "kafka.poolHash" .) -}} @@ -41,7 +35,7 @@ {{- /* KRaft controller quorum — fixed at creation, never re-derived on a day-2 edit. - The quorum must be static: Strimzi 0.45 cannot scale a controller node pool, so + The quorum must be static: this chart does not resize a live KRaft quorum, so flipping it (e.g. because a tenant raised kafka.replicas from 1 to 3) would wedge the reconcile or lose the metadata quorum. So the size is pinned once: if the controller pool already exists, keep its current replica count; only on first @@ -57,81 +51,30 @@ applied — either way, computing the initial value is correct. */ -}} {{- define "kafka.controllerReplicas" -}} -{{- $existing := lookup "kafka.strimzi.io/v1beta2" "KafkaNodePool" .Release.Namespace (include "kafka.controllerPoolName" .) -}} +{{- $existing := lookup "kafka.strimzi.io/v1" "KafkaNodePool" .Release.Namespace (include "kafka.controllerPoolName" .) -}} {{- $pinned := dig "spec" "replicas" 0 ($existing | default dict) -}} {{- if $pinned -}}{{ int $pinned }}{{- else if le (int .Values.kafka.replicas) 1 -}}1{{- else -}}3{{- end -}} {{- end -}} {{- /* - Broker KafkaNodePool name — a short release-hashed token for fresh installs so - several Kafka clusters can coexist in one namespace, but exactly "kafka" for a - cluster being migrated from ZooKeeper. + Broker KafkaNodePool name: "kafka" for a cluster migrated from ZooKeeper, whose + pool of that name adopted its -kafka-N brokers and their PVCs, and + the release-hashed "b-" for every other cluster. - Strimzi derives node and PVC names as --. A fresh KRaft - cluster has no prior state to preserve, so its broker pool is named "b-" - — unique within the namespace (each release hashes differently), letting - multiple fresh clusters live side by side without doubling the release name - into every pod hostname. A pre-node-pool ZooKeeper cluster, however, has - brokers "-kafka-N", and only a pool named exactly "kafka" adopts those - brokers (and their PVCs) in place during migration. - - Resolution (lookup runs at render time, before the pre-upgrade migration Job): - - this cluster already owns a "kafka" pool -> "kafka" (migrated, steady state) - - this cluster already owns a "b-" pool -> "b-" (fresh, steady state) - - an existing non-KRaft CR with no release pool yet -> "kafka" (classic ZK about to migrate) - - nothing exists -> "b-" (fresh install) - - Safety of the lookup: Helm propagates any non-NotFound read error and fails the - whole render (pkg/engine/lookup_func.go returns an empty result only on - apierrors.IsNotFound), so a transient apiserver/RBAC miss makes the reconcile - retry — it never silently resolves a live migrated cluster to "b-" and - lets a later upgrade prune the "kafka" pool. An empty lookup means genuine - absence: a real fresh install, or a client-side render (helm template/unittest) - that is never applied. - - Because "kafka" is a fixed, namespace-unique name, co-namespaced ZooKeeper->KRaft - migrations run one at a time: while another cluster owns the "kafka" pool, this - cluster's render fails closed and its HelmRelease retries. The migration hook - drains the first cluster off the shared pool with a Cruise Control rebalance and - frees the name, then the waiting cluster retries into it — so every co-namespaced - Kafka reaches KRaft, not just the first (strimzi/strimzi-kafka-operator - discussions/11120). Fresh KRaft clusters are never affected. lookup returns - nothing during `helm template`, so unit tests resolve to the fresh "b-" - default. + An empty lookup means genuine absence: Helm propagates any non-NotFound read + error and fails the render (pkg/engine/lookup_func.go returns an empty result + only on apierrors.IsNotFound), so a transient apiserver/RBAC miss never + resolves a migrated cluster to "b-", which would make the hook create a + second, empty broker pool. lookup returns nothing during `helm template`, so + unit tests resolve to "b-". */ -}} {{- define "kafka.brokerPoolName" -}} -{{- $release := .Release.Name -}} -{{- $ns := .Release.Namespace -}} -{{- $brokerFresh := printf "b-%s" (include "kafka.poolHash" .) -}} -{{- $name := $brokerFresh -}} -{{- $kp := lookup "kafka.strimzi.io/v1beta2" "KafkaNodePool" $ns "kafka" -}} -{{- $kpOwner := "" -}} -{{- if $kp -}}{{- $kpOwner = dig "metadata" "labels" "strimzi.io/cluster" "" $kp -}}{{- end -}} -{{- if and $kp (eq $kpOwner $release) -}} - {{- /* Migrated, steady state: this cluster owns the adopted "kafka" pool. */ -}} - {{- $name = "kafka" -}} +{{- $kp := lookup "kafka.strimzi.io/v1" "KafkaNodePool" .Release.Namespace "kafka" -}} +{{- if and $kp (eq (dig "metadata" "labels" "strimzi.io/cluster" "" $kp) .Release.Name) -}} +kafka {{- else -}} - {{- $existingCR := lookup "kafka.strimzi.io/v1beta2" "Kafka" $ns $release -}} - {{- $ownBrokerPool := lookup "kafka.strimzi.io/v1beta2" "KafkaNodePool" $ns $brokerFresh -}} - {{- $kraftAnn := "" -}} - {{- if $existingCR -}}{{- $kraftAnn = dig "metadata" "annotations" "strimzi.io/kraft" "" $existingCR -}}{{- end -}} - {{- if $ownBrokerPool -}} - {{- /* Fresh KRaft cluster, steady state: it already owns its "b-" pool. */ -}} - {{- $name = $brokerFresh -}} - {{- else if and $existingCR (ne $kraftAnn "enabled") -}} - {{- /* Classic ZooKeeper cluster about to migrate: adopt its brokers as "kafka". */ -}} - {{- if and $kp (ne $kpOwner "") (ne $kpOwner $release) -}} - {{- fail (printf "Kafka %q is waiting to migrate: namespace %q still has the shared \"kafka\" broker pool owned by cluster %q, which is migrating first. Strimzi node pool names are namespace-unique, so co-namespaced ZooKeeper->KRaft migrations run one at a time (see strimzi/strimzi-kafka-operator discussions/11120): %q drains off the shared pool and frees the name, then this HelmRelease retries into it automatically — no manual action needed. Fresh KRaft clusters are unaffected." $release $ns $kpOwner $kpOwner) -}} - {{- end -}} - {{- $name = "kafka" -}} - {{- end -}} - {{- /* else: fresh install (no CR, no own pool) -> "b-". A transient read - error can't reach here silently: Helm propagates every non-NotFound error - and fails the render, so an empty lookup means genuine absence (a real - fresh install, or a client-side helm-template render that is never - applied), never a flaked read of a live migrated cluster's "kafka" pool. */ -}} +{{- printf "b-%s" (include "kafka.poolHash" .) -}} {{- end -}} -{{- $name -}} {{- end -}} {{- /* diff --git a/packages/apps/kafka/templates/kafka.yaml b/packages/apps/kafka/templates/kafka.yaml index 167a43b05..e1b5dc0ed 100644 --- a/packages/apps/kafka/templates/kafka.yaml +++ b/packages/apps/kafka/templates/kafka.yaml @@ -17,16 +17,13 @@ {{- $showExternal := .Values.external -}} {{- $kafkaVersion := include "kafka.versionMap" . -}} {{- $entityOperatorMemory := "512Mi" -}} -apiVersion: kafka.strimzi.io/v1beta2 +apiVersion: kafka.strimzi.io/v1 kind: Kafka metadata: name: {{ .Release.Name }} labels: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} - annotations: - strimzi.io/node-pools: enabled - strimzi.io/kraft: enabled spec: kafka: version: {{ $kafkaVersion }} @@ -65,6 +62,13 @@ spec: default.replication.factor: 3 min.insync.replicas: 2 {{- end }} + template: + pod: + metadata: + labels: + # The nodes read their certificates from Secrets through the + # Kubernetes API (Strimzi's strimzisecrets config provider). + policy.cozystack.io/allow-to-apiserver: "true" metricsConfig: type: jmxPrometheusExporter valueFrom: diff --git a/packages/apps/kafka/templates/migration-hook.yaml b/packages/apps/kafka/templates/migration-hook.yaml index 997b28ea1..646a6ae1d 100644 --- a/packages/apps/kafka/templates/migration-hook.yaml +++ b/packages/apps/kafka/templates/migration-hook.yaml @@ -1,32 +1,11 @@ {{- /* - Node pools are managed entirely by this hook, NOT by a regular chart manifest. - - Reason: an in-place ZooKeeper->KRaft migration must create the broker and - controller KafkaNodePools BEFORE the main release apply (Strimzi has to walk - the migration state machine while the old ZooKeeper Kafka CR is still live). - If the pools were also regular chart manifests, Helm's main apply would try to - reconcile a resource that exists in the cluster but was absent from the - previous (ZooKeeper) release revision, and fail the upgrade with - "original object KafkaNodePool ... not found" — leaving the HelmRelease stuck - in a failed state even though the Kafka cluster migrated fine. Keeping the - pools out of the main manifest and applying them here (idempotently, on every - install and upgrade) avoids that whole class of Helm 3-way-merge conflict. - - Multiple ZooKeeper Kafkas per namespace: only a pool named exactly "kafka" - adopts a ZooKeeper cluster's existing -kafka-N brokers in place, and - that name is namespace-unique, so migrations serialize on it. To let every - co-namespaced cluster reach KRaft, a cluster that took "kafka" and finds other - ZooKeeper clusters still waiting frees the name after migrating: it stands up - its own release-hashed "b-" broker pool, drains all data off the "kafka" - pool brokers with a Cruise Control KafkaRebalance, deletes the "kafka" pool, - and the next waiting cluster's retry grabs the freed name. The last one to - migrate keeps "kafka". Cruise Control is enabled only for the drain and removed - after (kafka.yaml never sets spec.cruiseControl, so Helm does not fight it). + Node pools are applied by this hook, not as regular chart manifests. Every + pool that exists today was created here and is absent from every release + manifest; applying them before the main release apply also keeps the Kafka + CR from being reconciled against pools that do not exist yet. */ -}} {{- $controllerReplicas := include "kafka.controllerReplicas" . }} {{- $brokerPool := include "kafka.brokerPoolName" . }} -{{- $freshBrokerPool := printf "b-%s" (include "kafka.poolHash" .) }} -{{- $cruiseControlMemory := "1Gi" }} --- apiVersion: v1 kind: ServiceAccount @@ -47,17 +26,8 @@ metadata: helm.sh/hook-delete-policy: before-hook-creation rules: - apiGroups: ["kafka.strimzi.io"] - # kafkanodepools: delete is needed to remove the drained "kafka" pool when - # freeing the name; kafkarebalances drive the Cruise Control drain. - resources: ["kafkas", "kafkanodepools", "kafkarebalances"] - verbs: ["get", "list", "watch", "create", "patch", "update", "delete"] -- apiGroups: [""] - # The drained "kafka" pool's PVCs (deleteClaim:false keeps them) are removed - # once empty, so freeing the name does not leak storage. "watch" is needed - # because `kubectl delete` waits on a watch by default — without it the delete - # still succeeds but logs noisy "cannot watch persistentvolumeclaims" errors. - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "delete"] + resources: ["kafkanodepools"] + verbs: ["get", "create", "patch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -88,13 +58,7 @@ metadata: helm.sh/hook-delete-policy: before-hook-creation spec: backoffLimit: 0 - # Freeing the "kafka" name drains the whole dataset off the old brokers with a - # Cruise Control rebalance, which can take hours on a large cluster. The Job - # deadline must cover that; if it is exceeded the Job is killed, Helm fails the - # upgrade, and the next reconcile re-runs this Job, which resumes from where it - # left off (every phase below is idempotent). The release's helm timeout must - # exceed this deadline or the migration is aborted before it can finish. - activeDeadlineSeconds: 21600 + activeDeadlineSeconds: 300 template: metadata: labels: @@ -113,14 +77,6 @@ spec: ns={{ .Release.Namespace | quote }} name={{ .Release.Name | quote }} brokerPool={{ $brokerPool | quote }} - freshBrokerPool={{ $freshBrokerPool | quote }} - rebalance="${name}-drain-kafka" - # Fully qualify the resources: the plural "kafkas" also exists in - # this chart's own apps.cozystack.io group, so a bare "kafka" could - # resolve to the tenant kind, which this hook's Role does not grant. - kafka=kafkas.kafka.strimzi.io - np=kafkanodepools.kafka.strimzi.io - kr=kafkarebalances.kafka.strimzi.io # Every apply here passes --openapi-patch=false. Without it, applying # an object that already exists makes kubectl download the cluster's @@ -129,17 +85,15 @@ spec: # while this container runs under a tenant LimitRange default of # 128Mi. With it kubectl builds the same merge patch from the # last-applied annotation. Not --server-side: fields that - # "kubectl create" set on the "kafka" pool stay co-owned by that - # manager, so a server-side apply would never remove them. - # ---- Node-pool manifests (broker pool reused for the create-CAS) ---- + # "kubectl create" set on a pool stay co-owned by that manager, so + # a server-side apply would never remove them. broker_manifest=$(cat <-kafka-N brokers and their data in - # place); "b-" for a fresh cluster. Resolved at render time - # by kafka.brokerPoolName. + # "kafka" for a cluster migrated from ZooKeeper (that pool adopted + # its -kafka-N brokers), "b-" otherwise. Resolved + # at render time by kafka.brokerPoolName. name: ${brokerPool} namespace: ${ns} labels: @@ -153,9 +107,7 @@ spec: roles: - broker resources: {{- include "cozy-lib.resources.defaultingSanitize" (list .Values.kafka.resourcesPreset .Values.kafka.resources $) | nindent 18 }} - # A pool template replaces spec.kafka.template whole, for every - # pool here: nothing set on the Kafka CR's template reaches these - # pods (KafkaPool.java at 0.45.1-rc1: https://github.com/strimzi/strimzi-kafka-operator/blob/0.45.1-rc1/cluster-operator/src/main/java/io/strimzi/operator/cluster/model/KafkaPool.java#L196-L218). + # JAVA_TOOL_OPTIONS: see kafka.javaToolOptions. template: kafkaContainer: env: @@ -172,17 +124,15 @@ spec: {{- with .Values.kafka.storageClass }} class: {{ . }} {{- end }} - # deleteClaim:false is load-bearing here. Strimzi's PvcReconciler - # deletes a pool's PVCs when the pool is removed, so with - # deleteClaim:true the drain's deletion of the "kafka" pool (and - # a plain pool teardown) would destroy the tenant's data instead - # of leaving the volumes to be adopted / cleaned up deliberately. + # Strimzi's PvcReconciler deletes a pool's PVCs when the pool + # is removed; deleteClaim:false leaves the tenant's data on a + # pool teardown. deleteClaim: false MANIFEST ) controller_manifest=$(cat <-kafka-N brokers and their PVCs in place. - # Without this Strimzi hands the controller pool the lowest free - # IDs — the broker IDs — recreating brokers empty and orphaning - # the tenant's data. + # Controller node IDs start above the broker range, leaving + # [0..replicas-1] to the broker pool. A cluster migrated from + # ZooKeeper relies on it: its brokers kept their original IDs. strimzi.io/next-node-ids: "[{{ .Values.kafka.replicas }}-999]" spec: replicas: {{ $controllerReplicas }} @@ -224,307 +171,8 @@ spec: MANIFEST ) - # ---- Ensure both node pools exist ---- - # Broker pool ownership FIRST, before any apply, so a "kafka" pool - # owned by another cluster aborts without mutating anything. For a - # fresh cluster the name is the release-unique "b-" and there - # is no collision — a plain apply is safe. For a cluster migrating - # from ZooKeeper the name is the namespace-unique "kafka"; enforce - # one migration per namespace ATOMICALLY. A plain read-then-apply - # would race: two ZooKeeper Kafkas in one namespace upgraded - # together could both observe no owner and both apply, the second - # silently re-parenting the first's brokers. Instead, when the pool - # is absent, "kubectl create" is the atomic arbiter: exactly one - # concurrent creator wins; the loser's create fails, re-reads the - # owner, and aborts (fail fast — it retries once the winner frees - # the name). - if [ "$brokerPool" = "kafka" ]; then - if ! owner=$(kubectl -n "$ns" get "$np" kafka --ignore-not-found -o jsonpath='{.metadata.labels.strimzi\.io/cluster}' 2>&1); then - echo "ERROR: reading the \"kafka\" node pool owner label failed: $owner" >&2 - exit 1 - fi - if [ -z "$owner" ]; then - echo "Creating broker pool \"kafka\"..." - if echo "$broker_manifest" | kubectl create -f - 2>/tmp/createerr; then - owner="$name" - else - if ! owner=$(kubectl -n "$ns" get "$np" kafka --ignore-not-found -o jsonpath='{.metadata.labels.strimzi\.io/cluster}' 2>&1); then - echo "ERROR: re-reading the \"kafka\" node pool owner after create failure failed: $owner" >&2 - exit 1 - fi - if [ -z "$owner" ]; then - echo "ERROR: creating broker pool \"kafka\" failed: $(cat /tmp/createerr)" >&2 - exit 1 - fi - fi - fi - if [ "$owner" != "$name" ]; then - echo "ERROR: KafkaNodePool 'kafka' in namespace '$ns' already belongs to cluster '$owner'." >&2 - echo "Its ZooKeeper->KRaft migration is in progress and will free the name when done; this cluster's upgrade will retry and proceed then. (Strimzi node pool names are namespace-unique, so migrations serialize on the \"kafka\" name.)" >&2 - exit 1 - fi - fi echo "Ensuring broker KafkaNodePool \"$brokerPool\" exists..." echo "$broker_manifest" | kubectl apply --openapi-patch=false -f - echo "Ensuring controller KafkaNodePool exists..." echo "$controller_manifest" | kubectl apply --openapi-patch=false -f - - - # ---- Decide what to do from the Kafka CR ---- - # Fail closed. --ignore-not-found makes kubectl exit 0 with empty - # output on a genuine NotFound (fresh install: the Kafka CR is - # created by the main release apply, after this hook) and exit - # non-zero only on a real error (apiserver/RBAC/timeout). - if ! found=$(kubectl -n "$ns" get "$kafka" "$name" --ignore-not-found -o name 2>&1); then - echo "ERROR: reading Kafka CR failed: $found" >&2 - exit 1 - fi - if [ -z "$found" ]; then - echo "No existing Kafka CR — fresh install. Node pools ensured; the main release apply will create the KRaft Kafka CR. Nothing to migrate." - exit 0 - fi - - # Read the current KRaft annotation / metadata state (fail closed). - if ! kraft=$(kubectl -n "$ns" get "$kafka" "$name" -o jsonpath='{.metadata.annotations.strimzi\.io/kraft}' 2>&1); then - echo "ERROR: reading the strimzi.io/kraft annotation failed: $kraft" >&2 - exit 1 - fi - if ! state=$(kubectl -n "$ns" get "$kafka" "$name" -o jsonpath='{.status.kafkaMetadataState}' 2>&1); then - echo "ERROR: reading kafkaMetadataState failed: $state" >&2 - exit 1 - fi - echo "Current strimzi.io/kraft='${kraft}', kafkaMetadataState='${state}'." - - # ---- Migrate a ZooKeeper cluster (skip if already KRaft) ---- - if [ "$kraft" != "enabled" ] && [ "$state" != "KRaft" ]; then - # Enabling node pools is idempotent. Only issue kraft=migration - # from a genuine pre-migration state: on a retry the cluster may - # already be at PreKRaft/KRaftPostMigration, where re-annotating - # "migration" is an invalid backward transition Strimzi rejects. - echo "Enabling node pools on the Kafka CR..." - kubectl -n "$ns" annotate "$kafka" "$name" strimzi.io/node-pools=enabled --overwrite - case "$state" in - KRaftMigration|KRaftDualWriting|KRaftPostMigration|PreKRaft|KRaft) - echo "Migration already past the ZooKeeper phase (state: $state); not re-issuing kraft=migration." ;; - *) - echo "Starting KRaft migration (kraft=migration)..." - kubectl -n "$ns" annotate "$kafka" "$name" strimzi.io/kraft=migration --overwrite ;; - esac - - echo "Waiting for migration to reach KRaftPostMigration..." - migration_ready=false - for i in $(seq 1 240); do - state=$(kubectl -n "$ns" get "$kafka" "$name" -o jsonpath='{.status.kafkaMetadataState}' || echo "") - case "$state" in - KRaftPostMigration|PreKRaft|KRaft) - echo "Reached state: $state" - migration_ready=true - break - ;; - *) - echo "[$i/240] State: ${state}, waiting..." - sleep 5 - ;; - esac - done - - # Strimzi only permits strimzi.io/kraft=enabled once the cluster - # has reached KRaftPostMigration. Applying it earlier — while - # brokers still depend on ZooKeeper — is unsafe and irreversible. - # If the wait timed out in an intermediate state, abort without - # finalising so Helm fails the upgrade and the next reconcile - # retries from where the migration left off. - if [ "$migration_ready" != "true" ]; then - echo "ERROR: migration did not reach KRaftPostMigration (last state: ${state:-unknown})." - echo "Refusing to apply strimzi.io/kraft=enabled in an unsafe state." - exit 1 - fi - - echo "Finalising migration to KRaft..." - kubectl -n "$ns" annotate "$kafka" "$name" strimzi.io/kraft=enabled --overwrite - - echo "Waiting for KRaft state..." - migrated=false - for i in $(seq 1 120); do - state=$(kubectl -n "$ns" get "$kafka" "$name" -o jsonpath='{.status.kafkaMetadataState}' || echo "") - if [ "$state" = "KRaft" ]; then - echo "Migration complete." - migrated=true - break - fi - echo "[$i/120] State: ${state}, waiting..." - sleep 5 - done - if [ "$migrated" != "true" ]; then - echo "ERROR: migration did not complete in time." - exit 1 - fi - else - echo "Kafka is already KRaft — no migration needed." - fi - - # ---- Free the "kafka" name for the next waiting cluster ---- - # Only relevant if THIS cluster ended up owning the shared "kafka" - # pool (a migrated ZooKeeper cluster). If other ZooKeeper Kafkas - # remain in the namespace, drain this cluster off "kafka" onto its - # release-hashed "b-" pool and delete "kafka" so the next one - # can adopt the name. Every step is idempotent so a killed Job - # resumes on retry. - if [ "$brokerPool" != "kafka" ]; then - echo "This cluster runs on \"$brokerPool\" (not the shared \"kafka\" pool); nothing to free." - exit 0 - fi - # Reading fails open if the pipeline swallows the kubectl error: an - # empty list is indistinguishable from a failed read, and both would - # count 0 and keep the pool, stranding any co-namespaced ZK cluster - # still waiting. Capture the read status separately, like the drain - # read below. - if ! others_raw=$(kubectl -n "$ns" get "$kafka" -o jsonpath='{range .items[*]}{.metadata.name}={.metadata.annotations.strimzi\.io/kraft}{"\n"}{end}' 2>&1); then - echo "ERROR: could not list Kafka CRs in $ns to decide whether to free the \"kafka\" pool: $others_raw" >&2 - exit 1 - fi - others=$(printf '%s\n' "$others_raw" | awk -F= -v me="$name" '$1!="" && $1!=me && $2!="enabled"{c++} END{print c+0}') - if [ "$others" -eq 0 ]; then - echo "No other ZooKeeper Kafka clusters in $ns; keeping the \"kafka\" pool. Done." - exit 0 - fi - echo "$others other ZooKeeper Kafka cluster(s) remain in $ns; freeing the \"kafka\" pool name..." - - # F1: stand up this cluster's own release-hashed broker pool with - # node IDs pinned high, clear of the existing broker/controller IDs. - echo "Creating target broker pool \"$freshBrokerPool\"..." - cat </dev/null || echo "") - if [ -z "$kafkaIds" ] || [ "$kafkaIds" = "[]" ]; then - echo "ERROR: could not read the \"kafka\" pool node IDs; refusing to drain blindly." >&2 - exit 1 - fi - echo "Draining brokers $kafkaIds via Cruise Control rebalance..." - cat </dev/null || echo "") - case " $st " in - *" Ready "*) - echo "Rebalance complete." - rebalanced=true - break ;; - *" ProposalReady "*) - echo "[$i] Proposal ready; approving..." - kubectl -n "$ns" annotate "$kr" "$rebalance" strimzi.io/rebalance=approve --overwrite || true - sleep 15 ;; - *" NotReady "*) - # A NotReady rebalance is not treated as terminal. Cruise - # Control rejects proposals transiently while it settles: a - # just-enabled CC is unreachable for a minute or two (its pod - # and cruise-control-certs Secret are still coming up — - # CruiseControlRetriableConnectionException), and its capacity - # model can briefly over-estimate load from noisy samples and - # trip an optimization goal even with skipHardGoalCheck. Both - # clear once CC's metric windows age out, so refresh the - # proposal (recompute against current metrics) and keep - # polling. This also covers resume: a re-run of the Job finds a - # stale NotReady rebalance and refreshes it instead of reading - # its old failure forever. Only the loop timeout below is - # terminal — then the Job fails and the next reconcile retries. - reason=$(kubectl -n "$ns" get "$kr" "$rebalance" -o jsonpath='{.status.conditions[*].reason}' 2>/dev/null || echo "") - echo "[$i] Rebalance NotReady (${reason}); refreshing the proposal and waiting for Cruise Control to settle..." - kubectl -n "$ns" annotate "$kr" "$rebalance" strimzi.io/rebalance=refresh --overwrite || true - sleep 15 ;; - *) - echo "[$i] Rebalance state: '${st}', waiting..." - sleep 15 ;; - esac - done - if [ "$rebalanced" != "true" ]; then - echo "ERROR: rebalance did not complete within the Job deadline; will resume on the next retry." - exit 1 - fi - - # F4/F5: the drained "kafka" brokers now hold no partitions — remove - # the pool and its (now empty) PVCs. After this the render-time - # kafka.brokerPoolName resolves to "$freshBrokerPool" (this cluster - # owns it), so the cluster stays there on every future reconcile. - echo "Deleting the drained \"kafka\" pool and its PVCs..." - kubectl -n "$ns" delete "$np" kafka --ignore-not-found - for id in $(echo "$kafkaIds" | tr -d '[] ' | tr ',' ' '); do - kubectl -n "$ns" delete pvc "data-0-${name}-kafka-${id}" --ignore-not-found - done - - # F6: tear Cruise Control back down and clean up the rebalance CR. - echo "Removing Cruise Control and the rebalance CR..." - kubectl -n "$ns" patch "$kafka" "$name" --type json -p '[{"op":"remove","path":"/spec/cruiseControl"}]' 2>/dev/null || true - kubectl -n "$ns" delete "$kr" "$rebalance" --ignore-not-found - - echo "Freed the \"kafka\" pool name; this cluster now runs on \"$freshBrokerPool\". The next waiting cluster's retry will adopt it." - exit 0 diff --git a/packages/apps/kafka/templates/topics.yaml b/packages/apps/kafka/templates/topics.yaml index 712b5d670..2456f5018 100644 --- a/packages/apps/kafka/templates/topics.yaml +++ b/packages/apps/kafka/templates/topics.yaml @@ -1,6 +1,39 @@ +{{- /* + Topic configs Kafka 3.x accepted and Kafka 4.2/4.3 reject. A KafkaTopic + carrying one stays NotReady while the release reports Ready, and a broker + whose log holds a segment.bytes below the floor does not start. The same + limits are in packages/core/platform/templates/_helpers.tpl, + packages/core/platform/images/migrations/migrations/61 and + packages/system/backupstrategy-controller/files/kafka-backup.sh. +*/}} +{{- $segmentBytesMin := 1048576 }} {{- range $topic := .Values.topics }} +{{- $config := $topic.config | default dict }} +{{- range $key := list "message.downconversion.enable" "message.format.version" "message.timestamp.difference.max.ms" }} +{{- if hasKey $config $key }} +{{- fail (printf "topic %q sets %s, which Kafka 4 removed: delete it from the topic's config" $topic.name $key) }} +{{- end }} +{{- end }} +{{- $segmentBytes := get $config "segment.bytes" }} +{{- if or (kindIs "float64" $segmentBytes) (kindIs "int64" $segmentBytes) (kindIs "int" $segmentBytes) }} +{{- $segmentBytes = printf "%d" (int64 $segmentBytes) }} +{{- end }} +{{- /* + Kafka trims characters up to U+0020 and reads the rest with + Integer.parseInt: a sign is allowed and leading zeros are decimal, where + int64 would read them as octal. The digits are compared as text, so a + string too long for int64 is not misread either. +*/}} +{{- $integer := `^[\x00-\x20]*(?:\+|(-))?0*([0-9]+)[\x00-\x20]*$` }} +{{- if and (kindIs "string" $segmentBytes) (regexMatch $integer $segmentBytes) }} +{{- $value := regexReplaceAll $integer $segmentBytes "${1}${2}" }} +{{- $min := toString $segmentBytesMin }} +{{- if or (hasPrefix "-" $value) (lt (len $value) (len $min)) (and (eq (len $value) (len $min)) (lt $value $min)) }} +{{- fail (printf "topic %q sets segment.bytes=%s, below the Kafka 4 minimum of %d: raise it" $topic.name $value $segmentBytesMin) }} +{{- end }} +{{- end }} --- -apiVersion: kafka.strimzi.io/v1beta2 +apiVersion: kafka.strimzi.io/v1 kind: KafkaTopic metadata: name: "{{ $.Release.Name }}-{{ kebabcase $topic.name }}" diff --git a/packages/apps/kafka/tests/jvm_sizing_test.yaml b/packages/apps/kafka/tests/jvm_sizing_test.yaml index f4569a1e2..223131814 100644 --- a/packages/apps/kafka/tests/jvm_sizing_test.yaml +++ b/packages/apps/kafka/tests/jvm_sizing_test.yaml @@ -19,9 +19,6 @@ tests: - matchRegex: path: spec.template.spec.containers[0].args[0] pattern: "roles:\\s+- controller\\s+resources:\\s+limits:(\\s+\\S+: \\S+)+\\s+requests:(\\s+\\S+: \\S+)+(\\s+#[^\\n]*)*\\s+template:\\s+kafkaContainer:\\s+env:\\s+- name: JAVA_TOOL_OPTIONS\\s+value: \"-XX:MaxRAM=1073741824 -XX:ActiveProcessorCount=1\"\\s+storage:" - - matchRegex: - path: spec.template.spec.containers[0].args[0] - pattern: "next-node-ids: \"\\[1000-1099\\]\"\\s+spec:\\s+replicas: \\d+\\s+roles:\\s+- broker\\s+resources:\\s+limits:(\\s+\\S+: \\S+)+\\s+requests:(\\s+\\S+: \\S+)+(\\s+#[^\\n]*)*\\s+template:\\s+kafkaContainer:\\s+env:\\s+- name: JAVA_TOOL_OPTIONS\\s+value: \"-XX:MaxRAM=1073741824 -XX:ActiveProcessorCount=1\"\\s+storage:" # Strimzi moves jvmOptions -XX into KAFKA_JVM_PERFORMANCE_OPTS, and # kafka-run-class.sh then drops its G1 settings. @@ -64,19 +61,6 @@ tests: - matchRegex: path: spec.template.spec.containers[0].args[0] pattern: "roles:\\s+- controller\\s+resources:\\s+limits:(\\s+\\S+: \\S+)+\\s+requests:(\\s+\\S+: \\S+)+(\\s+#[^\\n]*)*\\s+template:\\s+kafkaContainer:\\s+env:\\s+- name: JAVA_TOOL_OPTIONS\\s+value: \"-XX:MaxRAM=2147483648 -XX:ActiveProcessorCount=2\"\\s+storage:" - - matchRegex: - path: spec.template.spec.containers[0].args[0] - pattern: "next-node-ids: \"\\[1000-1099\\]\"\\s+spec:\\s+replicas: \\d+\\s+roles:\\s+- broker\\s+resources:\\s+limits:(\\s+\\S+: \\S+)+\\s+requests:(\\s+\\S+: \\S+)+(\\s+#[^\\n]*)*\\s+template:\\s+kafkaContainer:\\s+env:\\s+- name: JAVA_TOOL_OPTIONS\\s+value: \"-XX:MaxRAM=2147483648 -XX:ActiveProcessorCount=2\"\\s+storage:" - - # The whole patch, closing brace included: a merge patch kubectl cannot - # parse fails the hook before the drain starts. - - it: passes the drain-time Cruise Control memory limit to its JVM - template: templates/migration-hook.yaml - documentIndex: 3 - asserts: - - matchRegex: - path: spec.template.spec.containers[0].args[0] - pattern: "\\-\\-type\\ merge\\ \\-p\\ '\\{\"spec\":\\{\"cruiseControl\":\\{\"resources\":\\{\"limits\":\\{\"memory\":\"1Gi\"\\},\"requests\":\\{\"cpu\":\"200m\",\"memory\":\"768Mi\"\\}\\},\"template\":\\{\"cruiseControlContainer\":\\{\"env\":\\[\\{\"name\":\"JAVA_TOOL_OPTIONS\",\"value\":\"\\-XX:MaxRAM=1073741824\"\\}\\]\\}\\}\\}\\}\\}'\n" - it: passes the entity operator memory limit to its JVMs template: templates/kafka.yaml diff --git a/packages/apps/kafka/tests/kafka_test.yaml b/packages/apps/kafka/tests/kafka_test.yaml index dc3f406ab..0fd5a0488 100644 --- a/packages/apps/kafka/tests/kafka_test.yaml +++ b/packages/apps/kafka/tests/kafka_test.yaml @@ -8,7 +8,7 @@ tests: # KRaft mode (new inst.) # ########################## - - it: renders Kafka CR with KRaft annotations for new instance + - it: renders a v1 Kafka CR without the KRaft annotations Strimzi ignores release: name: test-kafka namespace: tenant-test @@ -17,12 +17,12 @@ tests: count: 1 - isKind: of: Kafka - - equal: + - isAPIVersion: + of: kafka.strimzi.io/v1 + - notExists: path: metadata.annotations["strimzi.io/kraft"] - value: enabled - - equal: + - notExists: path: metadata.annotations["strimzi.io/node-pools"] - value: enabled - it: sets correct name and labels release: @@ -40,25 +40,35 @@ tests: # Version # ########################## - - it: sets Kafka version from version map (default v3.9) + - it: sets Kafka version from version map (default v4.3) release: name: test-kafka namespace: tenant-test asserts: - equal: path: spec.kafka.version - value: "3.9.0" + value: "4.3.1" - - it: resolves v3.8 version + - it: resolves v4.2 version release: name: test-kafka namespace: tenant-test set: - version: "v3.8" + version: "v4.2" asserts: - equal: path: spec.kafka.version - value: "3.8.1" + value: "4.2.1" + + - it: refuses a Kafka 3.x version the operator no longer runs + release: + name: test-kafka + namespace: tenant-test + set: + version: "v3.9" + asserts: + - failedTemplate: + errorPattern: "value must be one of 'v4.3', 'v4.2'" ########################## # No ZooKeeper / node-pool managed fields # @@ -283,3 +293,14 @@ tests: - equal: path: spec.kafka.metricsConfig.valueFrom.configMapKeyRef.key value: kafka-metrics-config.yml + + # Strimzi 1.x Kafka nodes read their certificates from Secrets through the + # Kubernetes API, which tenant network policy opens only to labelled pods. + - it: lets the Kafka pods reach the Kubernetes API + release: + name: test-kafka + namespace: tenant-test + asserts: + - equal: + path: spec.kafka.template.pod.metadata.labels["policy.cozystack.io/allow-to-apiserver"] + value: "true" diff --git a/packages/apps/kafka/tests/migration-hook_test.yaml b/packages/apps/kafka/tests/migration-hook_test.yaml index 36255fab4..5b1daff11 100644 --- a/packages/apps/kafka/tests/migration-hook_test.yaml +++ b/packages/apps/kafka/tests/migration-hook_test.yaml @@ -63,85 +63,9 @@ tests: value: test-kafka-kafka-migration documentIndex: 3 - - it: grants permission to patch kafkas and kafkanodepools - release: - name: test-kafka - namespace: tenant-test - asserts: - - contains: - path: rules[0].apiGroups - content: kafka.strimzi.io - documentIndex: 1 - - contains: - path: rules[0].resources - content: kafkas - documentIndex: 1 - - contains: - path: rules[0].resources - content: kafkanodepools - documentIndex: 1 - - contains: - path: rules[0].verbs - content: patch - documentIndex: 1 - - contains: - path: rules[0].verbs - content: create - documentIndex: 1 - # Name-freeing needs to drive rebalances, delete the drained pool, and - # clean up its PVCs. - - contains: - path: rules[0].resources - content: kafkarebalances - documentIndex: 1 - - contains: - path: rules[0].verbs - content: delete - documentIndex: 1 - - contains: - path: rules[1].resources - content: persistentvolumeclaims - documentIndex: 1 - - # When it holds the shared "kafka" pool and other ZooKeeper clusters remain, - # the hook drains all data onto its own "b-" pool via a Cruise Control - # KafkaRebalance and only then deletes the "kafka" pool — so the name frees for - # the next cluster without losing data. - - it: frees the kafka pool name by draining via a rebalance - release: - name: test-kafka - namespace: tenant-test - asserts: - - matchRegex: - path: spec.template.spec.containers[0].args[0] - pattern: 'other ZooKeeper Kafka cluster' - documentIndex: 3 - - matchRegex: - path: spec.template.spec.containers[0].args[0] - pattern: 'kind: KafkaRebalance' - documentIndex: 3 - - matchRegex: - path: spec.template.spec.containers[0].args[0] - pattern: 'mode: remove-brokers' - documentIndex: 3 - # A mandatory broker evacuation must not be blocked by capacity hard goals. - - matchRegex: - path: spec.template.spec.containers[0].args[0] - pattern: 'skipHardGoalCheck: true' - documentIndex: 3 - - matchRegex: - path: spec.template.spec.containers[0].args[0] - pattern: 'next-node-ids: "\[1000-1099\]"' - documentIndex: 3 - # The pool is deleted only after the rebalance is Ready — never before. - - matchRegex: - path: spec.template.spec.containers[0].args[0] - pattern: 'Rebalance complete[\s\S]*delete "\$np" kafka' - documentIndex: 3 - # The hook creates the broker pool under the render-resolved name - # (kafka.brokerPoolName): "kafka" for a migrating ZooKeeper cluster, "b-" - # for a fresh one. The manifest references the shell var so both resolve here. + # (kafka.brokerPoolName): "kafka" for a cluster migrated from ZooKeeper, + # "b-" otherwise. The manifest references the shell var so both resolve here. - it: creates the broker node pool under the resolved pool name release: name: test-kafka @@ -168,73 +92,6 @@ tests: pattern: 'brokerPool="b-[0-9a-f]{8}"' documentIndex: 3 - # CRITICAL adoption invariant: a cluster migrating from ZooKeeper (an existing - # Kafka CR without kraft=enabled) must resolve the broker pool to exactly - # "kafka" so Strimzi reuses the existing -kafka-N brokers and PVCs. - - it: resolves the broker pool to kafka for a migrating ZooKeeper cluster - release: - name: test-kafka - namespace: tenant-test - kubernetesProvider: - scheme: - "kafka.strimzi.io/v1beta2/Kafka": - gvr: - group: kafka.strimzi.io - version: v1beta2 - resource: kafkas - namespaced: true - objects: - - apiVersion: kafka.strimzi.io/v1beta2 - kind: Kafka - metadata: - name: test-kafka - namespace: tenant-test - asserts: - - matchRegex: - path: spec.template.spec.containers[0].args[0] - pattern: 'brokerPool="kafka"' - documentIndex: 3 - - # Render-time serialization of co-namespaced migrations: while a "kafka" pool is - # owned by a DIFFERENT cluster that is migrating first, kafka.brokerPoolName - # fails this ZooKeeper cluster's render so its HelmRelease retries until the - # first cluster drains off the shared pool and frees the name (the hook's - # runtime create-CAS is the second line of defense). - - it: fails the render when a foreign cluster still owns the kafka pool - release: - name: test-kafka - namespace: tenant-test - kubernetesProvider: - scheme: - "kafka.strimzi.io/v1beta2/Kafka": - gvr: - group: kafka.strimzi.io - version: v1beta2 - resource: kafkas - namespaced: true - "kafka.strimzi.io/v1beta2/KafkaNodePool": - gvr: - group: kafka.strimzi.io - version: v1beta2 - resource: kafkanodepools - namespaced: true - objects: - - apiVersion: kafka.strimzi.io/v1beta2 - kind: Kafka - metadata: - name: test-kafka - namespace: tenant-test - - apiVersion: kafka.strimzi.io/v1beta2 - kind: KafkaNodePool - metadata: - name: kafka - namespace: tenant-test - labels: - strimzi.io/cluster: other-cluster - asserts: - - failedTemplate: - errorPattern: "broker pool owned by cluster .* migrating first" - # The controller pool the hook creates must use the short hashed name # kafka.controllerPoolName, in lockstep with every selector helper. - it: creates the controller node pool with the release-hashed name @@ -268,133 +125,6 @@ tests: pattern: 'strimzi.io/next-node-ids: "\[3-999\]"' documentIndex: 3 - # The migration Job must never apply strimzi.io/kraft=enabled unless the - # cluster actually reached KRaftPostMigration — flipping it earlier is unsafe - # and irreversible. Guard the finalize step behind an explicit readiness flag - # that aborts (exit 1) on timeout, so Helm fails and the next reconcile retries. - - it: guards the KRaft finalize step behind a reached-safe-state check - release: - name: test-kafka - namespace: tenant-test - asserts: - - matchRegex: - path: spec.template.spec.containers[0].args[0] - pattern: "migration_ready=false" - documentIndex: 3 - - matchRegex: - path: spec.template.spec.containers[0].args[0] - pattern: 'if \[ "\$migration_ready" != "true" \]; then' - documentIndex: 3 - - matchRegex: - path: spec.template.spec.containers[0].args[0] - pattern: "Refusing to apply strimzi.io/kraft=enabled in an unsafe state" - documentIndex: 3 - - # Fail closed: a real read error (non-zero from the --ignore-not-found get) - # must abort, never be mistaken for a fresh install. - - it: fails closed on non-NotFound errors reading the Kafka CR - release: - name: test-kafka - namespace: tenant-test - asserts: - - matchRegex: - path: spec.template.spec.containers[0].args[0] - pattern: 'if ! found=\$\(kubectl .* --ignore-not-found -o name' - documentIndex: 3 - - matchRegex: - path: spec.template.spec.containers[0].args[0] - pattern: "ERROR: reading Kafka CR failed" - documentIndex: 3 - - # Idempotent: on a retry the cluster may already be past ZooKeeper; do not - # re-issue kraft=migration (an invalid backward transition Strimzi rejects). - - it: does not re-issue kraft=migration once past the ZooKeeper phase - release: - name: test-kafka - namespace: tenant-test - asserts: - - matchRegex: - path: spec.template.spec.containers[0].args[0] - pattern: "KRaftMigration\\|KRaftDualWriting\\|KRaftPostMigration\\|PreKRaft\\|KRaft\\)" - documentIndex: 3 - - matchRegex: - path: spec.template.spec.containers[0].args[0] - pattern: "not re-issuing kraft=migration" - documentIndex: 3 - - # ATOMIC one-migration-per-namespace guard: an absent "kafka" pool is created - # via `kubectl create` (the atomic arbiter of a concurrent race), and a "kafka" - # pool owned by another cluster is refused. - - it: guards the shared kafka pool name atomically via create, refusing a foreign owner - release: - name: test-kafka - namespace: tenant-test - asserts: - - matchRegex: - path: spec.template.spec.containers[0].args[0] - pattern: 'if \[ "\$brokerPool" = "kafka" \]; then' - documentIndex: 3 - - matchRegex: - path: spec.template.spec.containers[0].args[0] - pattern: 'kubectl create -f -' - documentIndex: 3 - - matchRegex: - path: spec.template.spec.containers[0].args[0] - pattern: "already belongs to cluster" - documentIndex: 3 - - # The bare plural "kafkas" is ambiguous (also apps.cozystack.io); the hook must - # fully qualify the Strimzi resource or it can hit the tenant kind and Forbidden. - - it: fully qualifies the Strimzi Kafka and KafkaNodePool resources - release: - name: test-kafka - namespace: tenant-test - asserts: - - matchRegex: - path: spec.template.spec.containers[0].args[0] - pattern: "kafka=kafkas\\.kafka\\.strimzi\\.io" - documentIndex: 3 - - matchRegex: - path: spec.template.spec.containers[0].args[0] - pattern: "np=kafkanodepools\\.kafka\\.strimzi\\.io" - documentIndex: 3 - - notMatchRegex: - path: spec.template.spec.containers[0].args[0] - pattern: 'annotate kafka "' - documentIndex: 3 - - # NotFound must be structural (--ignore-not-found), never a stderr substring - # match that a stray "... not found" error could trip into skipping a live ZK. - - it: classifies NotFound structurally, not by stderr substring - release: - name: test-kafka - namespace: tenant-test - asserts: - - matchRegex: - path: spec.template.spec.containers[0].args[0] - pattern: "--ignore-not-found" - documentIndex: 3 - - notMatchRegex: - path: spec.template.spec.containers[0].args[0] - pattern: "grep -qiE.*not.*found" - documentIndex: 3 - - # A KRaft-native / already-migrated CR (kraft=enabled) must be a no-op, never - # driven backwards into kraft=migration. - - it: treats a kraft=enabled cluster as done - release: - name: test-kafka - namespace: tenant-test - asserts: - - matchRegex: - path: spec.template.spec.containers[0].args[0] - pattern: 'if \[ "\$kraft" != "enabled" \] && \[ "\$state" != "KRaft" \]' - documentIndex: 3 - - matchRegex: - path: spec.template.spec.containers[0].args[0] - pattern: "Kafka is already KRaft" - documentIndex: 3 - # Preserve failed-Job logs for a post-mortem: only before-hook-creation, never # hook-succeeded (which would delete the previous attempt's pod logs). - it: keeps only before-hook-creation on the Job hook-delete-policy @@ -452,11 +182,100 @@ tests: release: name: test-kafka namespace: tenant-test - documentIndex: 3 + documentSelector: + path: kind + value: Job asserts: - notMatchRegex: path: spec.template.spec.containers[0].args[0] pattern: "kubectl[^|\\n]* apply (-f|--filename|--server-side)" - matchRegex: path: spec.template.spec.containers[0].args[0] - pattern: "echo \"\\$broker_manifest\" \\| kubectl apply --openapi-patch=false -f -\\n[\\s\\S]*echo \"\\$controller_manifest\" \\| kubectl apply --openapi-patch=false -f -\\n[\\s\\S]*<KRaft pre-upgrade migration Job (packages/apps/kafka/templates/migration-hook.yaml) - # can run for hours: the migration itself is tens of minutes (broker rolling - # restarts through the Strimzi state machine), and when several Kafkas share a - # namespace the Job also drains the whole dataset off the shared "kafka" pool - # via a Cruise Control rebalance to free the name for the next cluster. - # helm-controller waits on the pre-upgrade hook within the release timeout - # (default 10m — far below this). Raise it above the Job's - # activeDeadlineSeconds (21600s = 6h) so a real drain is not aborted and - # thrashed. Keep this above that Job deadline if either changes. - release.cozystack.io/helm-install-timeout: "7h" spec: application: kind: Kafka plural: kafkas singular: kafka openAPISchema: |- - {"title":"Chart Values","type":"object","x-cozystack-name":{"type":"string","description":"Kafka cluster name. The KRaft controller pods are named `kafka--c--` with an 8-character hash, and that pod hostname must fit the 63-character DNS-1123 label limit even for the smallest cluster.","maxLength":44},"properties":{"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"tls":{"description":"TLS configuration. Strimzi manages the cluster PKI automatically (no cert-manager is involved for this chart): the operator auto-creates `-cluster-ca-cert` and `-clients-ca-cert` secrets, both exposed for client trust setup. The internal TLS listener on 9093 is always on; this toggle only controls the external listener on 9094.","type":"object","default":{},"properties":{"enabled":{"description":"Enable TLS on the external listener. When unset, inherits the value of `external` (TLS is on when external access is enabled). Warning: setting this to false while external is true exposes Kafka over plaintext on a public IP via LoadBalancer. Strimzi does not provide authentication on this listener unless SCRAM, mTLS, or OAuth is separately configured. Use only in controlled networks.","type":"boolean"}}},"version":{"description":"Kafka version to deploy. Upgrade-only: once a cluster's KRaft metadata is at a given version, Strimzi refuses an in-place downgrade, so lowering this on a running cluster (e.g. v3.9 back to v3.8) leaves the Kafka CR stuck in a reconcile error. Pick the target version at creation and only ever raise it.","type":"string","default":"v3.9","enum":["v3.9","v3.8"]},"topics":{"description":"Topics configuration.","type":"array","default":[],"items":{"type":"object","required":["name","partitions","replicas"],"properties":{"config":{"description":"Topic configuration.","type":"object","x-kubernetes-preserve-unknown-fields":true},"name":{"description":"Topic name.","type":"string"},"partitions":{"description":"Number of partitions.","type":"integer"},"replicas":{"description":"Number of replicas.","type":"integer"}}}},"kafka":{"description":"Kafka configuration.","type":"object","default":{},"required":["replicas","resourcesPreset","size","storageClass"],"properties":{"controllerStorageSize":{"description":"Persistent Volume size for KRaft controller metadata (used during ZK-to-KRaft migration).","default":"5Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"replicas":{"description":"Number of Kafka replicas.","type":"integer","default":3},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"c1.small","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume size for Kafka.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the Kafka data.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}}}}}} + {"title":"Chart Values","type":"object","x-cozystack-name":{"type":"string","description":"Kafka cluster name. The KRaft controller pods are named `kafka--c--` with an 8-character hash, and that pod hostname must fit the 63-character DNS-1123 label limit even for the smallest cluster.","maxLength":44},"properties":{"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"tls":{"description":"TLS configuration. Strimzi manages the cluster PKI automatically (no cert-manager is involved for this chart): the operator auto-creates `-cluster-ca-cert` and `-clients-ca-cert` secrets, both exposed for client trust setup. The internal TLS listener on 9093 is always on; this toggle only controls the external listener on 9094.","type":"object","default":{},"properties":{"enabled":{"description":"Enable TLS on the external listener. When unset, inherits the value of `external` (TLS is on when external access is enabled). Warning: setting this to false while external is true exposes Kafka over plaintext on a public IP via LoadBalancer. Strimzi does not provide authentication on this listener unless SCRAM, mTLS, or OAuth is separately configured. Use only in controlled networks.","type":"boolean"}}},"version":{"description":"Kafka version to deploy. Upgrade-only: once a cluster's KRaft metadata is at a given version, Strimzi refuses an in-place downgrade, so lowering this on a running cluster (e.g. v4.3 back to v4.2) leaves the Kafka CR stuck in a reconcile error. Pick the target version at creation and only ever raise it.","type":"string","default":"v4.3","enum":["v4.3","v4.2"]},"topics":{"description":"Topics configuration.","type":"array","default":[],"items":{"type":"object","required":["name","partitions","replicas"],"properties":{"config":{"description":"Topic configuration.","type":"object","x-kubernetes-preserve-unknown-fields":true},"name":{"description":"Topic name.","type":"string"},"partitions":{"description":"Number of partitions.","type":"integer"},"replicas":{"description":"Number of replicas.","type":"integer"}}}},"kafka":{"description":"Kafka configuration.","type":"object","default":{},"required":["replicas","resourcesPreset","size","storageClass"],"properties":{"controllerStorageSize":{"description":"Persistent Volume size for KRaft controller metadata.","default":"5Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"replicas":{"description":"Number of Kafka replicas.","type":"integer","default":3},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"c1.small","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume size for Kafka.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the Kafka data.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}}}}}} release: prefix: kafka- labels: