mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-07 04:23:50 +09:00
Myasnikov Daniil
adc430e27c
`kubectl wait <kind> <name>` errors immediately with NotFound if the
resource doesn't exist yet — even with --for=condition or --for=jsonpath.
The redis test in CI run 25043350705 failed in 1 second for exactly this
reason: the redis-failover operator hadn't created the PVC by the time
the test waited for it. Previously the 3x retry on `Run E2E tests`
masked this race; with retry dropped, every such call is a flake risk.
Add a small `until kubectl get` existence backstop before each
kubectl wait, matching the pattern already established for HRs in
commit 66888c91.
33 backstops across 12 files:
bucket.bats — bucketclaims, bucketaccesses x2
clickhouse.bats — statefulset 0-0 (0-1 already covered)
harbor.bats — deploy x3, bucketclaims, bucketaccesses
kafka.bats — kafkas
mariadb.bats — statefulset, deployment
mongodb.bats — statefulset
openbao.bats — sts, pvc
postgres.bats — job.batch
qdrant.bats — sts, pvc
redis.bats — pvc, deploy, sts (the trigger)
vminstance.bats — dv, pvc, vm (with 120s for KubeVirt latency)
e2e-install-cozystack.bats — apiservices, sts/etcd, vmalert,
vmalertmanager, vlclusters, vmcluster,
clusters.postgresql.cnpg.io,
deploy/grafana-deployment, namespace
Same pattern, same 60s default timeout for the existence wait (120s for
nested-virt resources). Once the resource exists, the original wait
timeout takes over.
Run-kubernetes.sh has the same race shape on several waits (nfs, kamaji,
machinedeployment, etc.) — out of scope here; flagged for follow-up.
Signed-off-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
65 lines
3.1 KiB
Bash
65 lines
3.1 KiB
Bash
#!/usr/bin/env bats
|
|
|
|
@test "Create OpenBAO (standalone)" {
|
|
name='test'
|
|
kubectl -n tenant-test delete openbao.apps.cozystack.io $name --ignore-not-found --timeout=2m || true
|
|
kubectl apply -f- <<EOF
|
|
apiVersion: apps.cozystack.io/v1alpha1
|
|
kind: OpenBAO
|
|
metadata:
|
|
name: $name
|
|
namespace: tenant-test
|
|
spec:
|
|
replicas: 1
|
|
size: 10Gi
|
|
storageClass: ""
|
|
resourcesPreset: "small"
|
|
resources: {}
|
|
external: false
|
|
ui: true
|
|
EOF
|
|
# Wait for the operator to materialise the HelmRelease before kubectl wait
|
|
# kicks in (kubectl wait errors immediately if the object does not exist yet).
|
|
timeout 60 sh -ec "until kubectl -n tenant-test get hr openbao-$name >/dev/null 2>&1; do sleep 2; done"
|
|
kubectl -n tenant-test wait hr openbao-$name --timeout=60s --for=condition=ready
|
|
kubectl -n tenant-test wait hr openbao-$name-system --timeout=120s --for=condition=ready
|
|
|
|
# Wait for container to be started (pod Running does not guarantee container is ready for exec on slow CI)
|
|
if ! timeout 120 sh -ec "until kubectl -n tenant-test get pod openbao-$name-0 --output jsonpath='{.status.containerStatuses[0].started}' 2>/dev/null | grep -q true; do sleep 5; done"; then
|
|
echo "=== DEBUG: Container did not start in time ===" >&2
|
|
kubectl -n tenant-test describe pod openbao-$name-0 >&2 || true
|
|
kubectl -n tenant-test logs openbao-$name-0 --previous >&2 || true
|
|
kubectl -n tenant-test logs openbao-$name-0 >&2 || true
|
|
return 1
|
|
fi
|
|
|
|
# Wait for OpenBAO API to accept connections
|
|
# bao status exit codes: 0 = unsealed, 1 = error/not ready, 2 = sealed but responsive
|
|
if ! timeout 60 sh -ec "until kubectl -n tenant-test exec openbao-$name-0 -- bao status >/dev/null 2>&1; rc=\$?; test \$rc -eq 0 -o \$rc -eq 2; do sleep 3; done"; then
|
|
echo "=== DEBUG: OpenBAO API did not become responsive ===" >&2
|
|
kubectl -n tenant-test describe pod openbao-$name-0 >&2 || true
|
|
kubectl -n tenant-test logs openbao-$name-0 --previous >&2 || true
|
|
kubectl -n tenant-test logs openbao-$name-0 >&2 || true
|
|
return 1
|
|
fi
|
|
|
|
# Initialize OpenBAO (single key share for testing simplicity)
|
|
init_output=$(kubectl -n tenant-test exec openbao-$name-0 -- bao operator init -key-shares=1 -key-threshold=1 -format=json)
|
|
unseal_key=$(echo "$init_output" | jq -r '.unseal_keys_b64[0]')
|
|
if [ -z "$unseal_key" ] || [ "$unseal_key" = "null" ]; then
|
|
echo "Failed to extract unseal key. Init output: $init_output" >&2
|
|
return 1
|
|
fi
|
|
|
|
# Unseal OpenBAO
|
|
kubectl -n tenant-test exec openbao-$name-0 -- bao operator unseal "$unseal_key"
|
|
|
|
# Now wait for pod to become ready (readiness probe checks seal status)
|
|
timeout 60 sh -ec "until kubectl -n tenant-test get sts openbao-$name >/dev/null 2>&1; do sleep 2; done"
|
|
kubectl -n tenant-test wait sts openbao-$name --timeout=90s --for=jsonpath='{.status.readyReplicas}'=1
|
|
timeout 60 sh -ec "until kubectl -n tenant-test get pvc data-openbao-$name-0 >/dev/null 2>&1; do sleep 2; done"
|
|
kubectl -n tenant-test wait pvc data-openbao-$name-0 --timeout=50s --for=jsonpath='{.status.phase}'=Bound
|
|
kubectl -n tenant-test delete openbao.apps.cozystack.io $name
|
|
kubectl -n tenant-test delete pvc data-openbao-$name-0 --ignore-not-found
|
|
}
|