mirror of
https://github.com/cozystack/cozystack.git
synced 2026-10-23 03:27:50 +09:00
Aleksei Sviridkin
7002397e40
The Kafka suites passed while the node pools of the apps they had deleted, and the brokers those pools own, kept running into the suites that came after. Each suite now checks, after its apps are gone, that no KafkaNodePool and no pod labelled strimzi.io/cluster=<cluster> remains in the namespace. The kafka suite deletes its app explicitly so it can look afterwards. The metadata roundtrip's cleanup does not wait for the uninstalls, so its checks carry a budget sized from the pre-delete Job's deadline. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <f@lex.la>
348 lines
14 KiB
YAML
348 lines
14 KiB
YAML
# Chainsaw port of hack/e2e-apps/kafka.bats ("Create Kafka"), KRaft edition.
|
|
#
|
|
# The chart deploys Kafka in KRaft mode (no ZooKeeper): a combined-free layout
|
|
# of separate broker + controller KafkaNodePools. This test asserts the KRaft
|
|
# topology renders and converges (both node pools present, the Strimzi Kafka CR
|
|
# reaches kafkaMetadataState=KRaft) and that no ZooKeeper resources remain.
|
|
# The last step deletes the app itself and checks that no node pool or Strimzi
|
|
# pod of this cluster survives the uninstall.
|
|
apiVersion: chainsaw.kyverno.io/v1alpha1
|
|
kind: Test
|
|
metadata:
|
|
name: kafka
|
|
spec:
|
|
# A KRaft teardown (broker + controller pools, entity operator, their PVCs)
|
|
# has more to drain than the shared 5m cleanup default allows once a CI run
|
|
# is contended, so give this suite a wider cleanup budget — teardown latency,
|
|
# not the assertions, was deciding the verdict.
|
|
timeouts:
|
|
cleanup: 8m
|
|
catch:
|
|
- events: {}
|
|
- describe:
|
|
apiVersion: helm.toolkit.fluxcd.io/v2
|
|
kind: HelmRelease
|
|
name: kafka-test
|
|
- podLogs:
|
|
selector: app.kubernetes.io/instance=kafka-test
|
|
tail: 50
|
|
steps:
|
|
- name: create-kafka
|
|
try:
|
|
- apply:
|
|
file: kafka.yaml
|
|
|
|
- name: wait-ready
|
|
try:
|
|
- assert:
|
|
timeout: 6m
|
|
resource:
|
|
apiVersion: helm.toolkit.fluxcd.io/v2
|
|
kind: HelmRelease
|
|
metadata:
|
|
name: kafka-test
|
|
status:
|
|
(conditions[?type == 'Ready']):
|
|
- status: "True"
|
|
- assert:
|
|
timeout: 5m
|
|
resource:
|
|
apiVersion: apps.cozystack.io/v1alpha1
|
|
kind: Kafka
|
|
metadata:
|
|
name: test
|
|
status:
|
|
(conditions[?type == 'Ready']):
|
|
- status: "True"
|
|
- assert:
|
|
timeout: 5m
|
|
resource:
|
|
apiVersion: kafka.strimzi.io/v1beta2
|
|
kind: KafkaTopic
|
|
metadata:
|
|
name: kafka-test-test-results
|
|
spec:
|
|
topicName: testResults
|
|
(config == null): true
|
|
status:
|
|
(conditions[?type == 'Ready']):
|
|
- status: "True"
|
|
|
|
- name: verify-kraft
|
|
try:
|
|
# Fresh pools use short release-hashed names (b-<hash>/c-<hash>), so match by
|
|
# the cluster label + role rather than a hard-coded name.
|
|
- assert:
|
|
timeout: 2m
|
|
resource:
|
|
apiVersion: kafka.strimzi.io/v1beta2
|
|
kind: KafkaNodePool
|
|
metadata:
|
|
labels:
|
|
strimzi.io/cluster: kafka-test
|
|
spec:
|
|
roles:
|
|
- broker
|
|
- assert:
|
|
timeout: 2m
|
|
resource:
|
|
apiVersion: kafka.strimzi.io/v1beta2
|
|
kind: KafkaNodePool
|
|
metadata:
|
|
labels:
|
|
strimzi.io/cluster: kafka-test
|
|
spec:
|
|
roles:
|
|
- controller
|
|
- assert:
|
|
timeout: 5m
|
|
resource:
|
|
apiVersion: kafka.strimzi.io/v1beta2
|
|
kind: Kafka
|
|
metadata:
|
|
name: kafka-test
|
|
annotations:
|
|
strimzi.io/kraft: enabled
|
|
strimzi.io/node-pools: enabled
|
|
status:
|
|
kafkaMetadataState: KRaft
|
|
# KRaft mode ships no ZooKeeper: the client Service must never exist.
|
|
- error:
|
|
resource:
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: kafka-test-zookeeper-client
|
|
- assert:
|
|
timeout: 3m
|
|
resource:
|
|
apiVersion: v1
|
|
kind: Endpoints
|
|
metadata:
|
|
name: kafka-test-kafka-bootstrap
|
|
(subsets[0].addresses[0].ip != null): true
|
|
|
|
# The node pools are created by the migration hook, outside the release
|
|
# manifest, and each owns the StrimziPodSet running its brokers, so only the
|
|
# chart's pre-delete hook removes them. Deleting the app explicitly here,
|
|
# rather than leaving it to Chainsaw's cleanup, is what lets the test look at
|
|
# the namespace afterwards. The pod check selects on strimzi.io/cluster on
|
|
# purpose, so it covers every pod Strimzi runs for this cluster (brokers,
|
|
# controllers, entity operator), not only the brokers.
|
|
- name: verify-uninstall-removes-node-pools
|
|
try:
|
|
# Waits for the app, and so the HelmRelease uninstall behind it, to finish.
|
|
# 8m, the same budget and for the same reason as timeouts.cleanup above:
|
|
# this delete is the teardown that cleanup used to perform.
|
|
- delete:
|
|
timeout: 8m
|
|
ref:
|
|
apiVersion: apps.cozystack.io/v1alpha1
|
|
kind: Kafka
|
|
name: test
|
|
# The pre-delete hook deleted the pools before the delete above returned;
|
|
# the pods then go through garbage collection and their termination grace
|
|
# period, which the default error timeout covers.
|
|
- error:
|
|
resource:
|
|
apiVersion: kafka.strimzi.io/v1beta2
|
|
kind: KafkaNodePool
|
|
metadata:
|
|
labels:
|
|
strimzi.io/cluster: kafka-test
|
|
- error:
|
|
resource:
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata:
|
|
labels:
|
|
strimzi.io/cluster: kafka-test
|
|
catch:
|
|
# A failed error op already names each object still matching. This adds
|
|
# the StrimziPodSets between the pools and the pods, and the pods' phase.
|
|
- script:
|
|
timeout: 1m
|
|
content: |
|
|
kubectl -n "$NAMESPACE" --request-timeout=15s \
|
|
get kafkanodepools.kafka.strimzi.io,strimzipodsets.core.strimzi.io,pods \
|
|
-l strimzi.io/cluster=kafka-test -o wide || true
|
|
---
|
|
# Logical topic-data backup -> in-place + to-copy RestoreJob round-trip,
|
|
# driving the validated example scripts (examples/backups/kafka) as the harness
|
|
# so the test and the documented flow can never drift. run-all.sh runs Bucket
|
|
# -> two-broker source Kafka + a seeded 3-partition, 2-replica topic -> Job
|
|
# strategy + BackupClass -> ad-hoc BackupJob (freeze-end-offset drain to S3,
|
|
# waits Succeeded) -> in-place restore (deletes the topic, restores, asserts
|
|
# count, content and replication factor, then a second RestoreJob into the
|
|
# populated topic that must be refused) -> empty two-broker target Kafka +
|
|
# to-copy RestoreJob (waits Succeeded) and asserts the records round-tripped
|
|
# through S3 into the restored copy. It follows the same
|
|
# harness-driving shape as mariadb-2-backup-roundtrip
|
|
# (hack/e2e-chainsaw/mariadb/chainsaw-test.yaml): a single run step pre-cleans
|
|
# any leftover state, drives run-all.sh, and drains in an always-reached
|
|
# finally.
|
|
#
|
|
# Runs in tenant-root, not the suite default tenant-test: the S3 client here
|
|
# is the generic Job strategy Pod INSIDE the tenant, and the e2e tenant is
|
|
# created isolated (hack/e2e-install-cozystack.bats) — its Cilium egress
|
|
# allowlist blocks a tenant-test Pod from reaching tenant-root's seaweedfs,
|
|
# while a tenant-root Pod reaches it over same-tenant egress. The harness
|
|
# targets the in-cluster endpoint (S3_ENDPOINT below) instead of the BucketInfo
|
|
# external ingress URL, which in CI is an unroutable placeholder. The strategy's
|
|
# curl verifies TLS against the seaweedfs CA the example copies into its S3
|
|
# Secret; the S3 listener serves the chart's client certificate
|
|
# (packages/system/seaweedfs), whose SANs cover '*.tenant-root',
|
|
# '*.tenant-root.svc' and '*.tenant-root.svc.cluster.local', so the 2-label
|
|
# name would verify too — .svc is simply the fully-qualified in-cluster form.
|
|
# KAFKA_IMAGE is resolved from the running operator so the backup Pod reuses
|
|
# the exact Kafka image already cached on the nodes (no extra pull, no version
|
|
# skew).
|
|
apiVersion: chainsaw.kyverno.io/v1alpha1
|
|
kind: Test
|
|
metadata:
|
|
name: kafka-2-backup-roundtrip
|
|
spec:
|
|
namespace: tenant-root
|
|
skipDelete: true
|
|
catch:
|
|
- events: {}
|
|
- describe:
|
|
apiVersion: helm.toolkit.fluxcd.io/v2
|
|
kind: HelmRelease
|
|
name: kafka-kafka-test
|
|
- describe:
|
|
apiVersion: kafka.strimzi.io/v1beta2
|
|
kind: Kafka
|
|
name: kafka-kafka-test
|
|
- describe:
|
|
apiVersion: kafka.strimzi.io/v1beta2
|
|
kind: Kafka
|
|
name: kafka-kafka-restore
|
|
- describe:
|
|
apiVersion: backups.cozystack.io/v1alpha1
|
|
kind: BackupJob
|
|
- describe:
|
|
apiVersion: backups.cozystack.io/v1alpha1
|
|
kind: RestoreJob
|
|
# The drain/replay runs in backupstrategy-controller-created Job Pods; a
|
|
# failed S3 upload/connect or a SigV4 rejection prints there, not on the
|
|
# broker. Those Pods carry job.strategy.backups.cozystack.io/mode=backup|
|
|
# restore (never app.kubernetes.io/instance) and are not TTL-reaped, so they
|
|
# are still present at catch time - select on that label to capture the
|
|
# actual backup/restore error.
|
|
- podLogs:
|
|
selector: job.strategy.backups.cozystack.io/mode
|
|
tail: 200
|
|
# Broker Pods too, for the drain/replay's server side. The example uses its
|
|
# default app names (kafka-test source, kafka-restore to-copy target); the
|
|
# Cozystack chart prefixes the Strimzi cluster with "kafka-", so the source
|
|
# cluster is kafka-kafka-test and the to-copy target is kafka-kafka-restore.
|
|
- podLogs:
|
|
selector: app.kubernetes.io/instance=kafka-kafka-test
|
|
tail: 200
|
|
- podLogs:
|
|
selector: app.kubernetes.io/instance=kafka-kafka-restore
|
|
tail: 200
|
|
steps:
|
|
- name: run
|
|
try:
|
|
# Pre-clean: tenant-root is a shared, persistent namespace with fixed
|
|
# resource names, so a run killed before the finally below leaves stale
|
|
# state behind. A leftover BackupJob still carrying status.phase=Succeeded
|
|
# from a prior run would satisfy `wait_for_field ... Succeeded` instantly
|
|
# and pass without a real backup ever running. cleanup.sh is idempotent
|
|
# (--ignore-not-found), so this is a no-op on a clean namespace.
|
|
- script:
|
|
timeout: 10m
|
|
content: |
|
|
set -eu
|
|
cd ../../..
|
|
NAMESPACE="$NAMESPACE" examples/backups/kafka/cleanup.sh
|
|
check:
|
|
($error == null): true
|
|
# The 40m budget is the whole documented flow: bucket provisioning, TWO
|
|
# two-broker Strimzi clusters (source + to-copy target, each pulls the
|
|
# Kafka image, boots ZooKeeper then the brokers, and passes readiness), the
|
|
# freeze-end-offset drain to S3, the in-place restore, the refused restore
|
|
# into the populated topic (three strategy Pod attempts, each stopping at
|
|
# the emptiness check), and the to-copy restore with its verification.
|
|
# run-all.sh fails fast on terminal BackupJob/RestoreJob phases and stalled
|
|
# HelmReleases rather than polling out the budget.
|
|
- script:
|
|
timeout: 40m
|
|
content: |
|
|
set -eu
|
|
cd ../../..
|
|
[ -x examples/backups/kafka/run-all.sh ] || { echo "kafka backup example scripts not found at examples/backups/kafka" >&2; exit 1; }
|
|
# Reuse a Kafka image the operator already caches on the nodes, so the
|
|
# backup Pod hits a warm image cache. STRIMZI_KAFKA_IMAGES is a
|
|
# newline-separated "version=image" map (one entry per supported Kafka
|
|
# version), so split on whitespace, drop the "version=" prefix and take
|
|
# the last (newest) image rather than concatenating every line into one
|
|
# malformed reference. The reference is taken as-is, tag or digest.
|
|
KAFKA_IMAGE=$(kubectl -n cozy-kafka-operator get deploy strimzi-cluster-operator \
|
|
-o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="STRIMZI_KAFKA_IMAGES")].value}' \
|
|
| tr '[:space:]' '\n' | grep '=' | sed 's/^[^=]*=//' | tail -n1)
|
|
[ -n "$KAFKA_IMAGE" ] || { echo "could not resolve Kafka image from strimzi-cluster-operator" >&2; exit 1; }
|
|
COZY_E2E_BACKUP_PREFLIGHT=1 NAMESPACE="$NAMESPACE" KAFKA_IMAGE="$KAFKA_IMAGE" \
|
|
S3_ENDPOINT="https://seaweedfs-s3.tenant-root.svc:8333" \
|
|
examples/backups/kafka/run-all.sh
|
|
check:
|
|
($error == null): true
|
|
finally:
|
|
# Removes every resource the harness created (chainsaw did not apply them,
|
|
# so they are not part of automatic cleanup).
|
|
- description: backup-flow cleanup
|
|
script:
|
|
timeout: 10m
|
|
content: |
|
|
set -eu
|
|
cd ../../..
|
|
NAMESPACE="$NAMESPACE" examples/backups/kafka/cleanup.sh
|
|
|
|
# cleanup.sh above deleted both Kafka apps and waited for them to go, so their
|
|
# uninstalls have run, and the node pools the migration hook created outside
|
|
# the release manifest must be gone with them. The pod checks select on
|
|
# strimzi.io/cluster on purpose, so they cover every pod Strimzi runs for each
|
|
# cluster, not only the brokers. What is left is garbage collection and the
|
|
# pods' termination grace period, which the default error timeout covers.
|
|
- name: verify-uninstall-removes-node-pools
|
|
try:
|
|
- error:
|
|
resource:
|
|
apiVersion: kafka.strimzi.io/v1beta2
|
|
kind: KafkaNodePool
|
|
metadata:
|
|
labels:
|
|
strimzi.io/cluster: kafka-kafka-test
|
|
- error:
|
|
resource:
|
|
apiVersion: kafka.strimzi.io/v1beta2
|
|
kind: KafkaNodePool
|
|
metadata:
|
|
labels:
|
|
strimzi.io/cluster: kafka-kafka-restore
|
|
- error:
|
|
resource:
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata:
|
|
labels:
|
|
strimzi.io/cluster: kafka-kafka-test
|
|
- error:
|
|
resource:
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata:
|
|
labels:
|
|
strimzi.io/cluster: kafka-kafka-restore
|
|
catch:
|
|
# A failed error op already names each object still matching. This adds
|
|
# the StrimziPodSets between the pools and the pods, and the pods' phase.
|
|
- script:
|
|
timeout: 1m
|
|
content: |
|
|
kubectl -n "$NAMESPACE" --request-timeout=15s \
|
|
get kafkanodepools.kafka.strimzi.io,strimzipodsets.core.strimzi.io,pods \
|
|
-l 'strimzi.io/cluster in (kafka-kafka-test,kafka-kafka-restore)' -o wide || true
|