mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-05 03:27:07 +09:00
Flux helm-controller v1.5.0 (shipped in Flux v2.8) replaced the legacy
Helm readiness logic with kstatus-based health checking, which polls
Deployment/StatefulSet/Service status and waits for kstatus to assess
rollout-ready. This typically adds 10-30s of latency between resource
apply and HR condition Ready=True compared to v2.7.
Our per-app bats tests had `kubectl wait hr/... --for=condition=ready
--timeout=` budgets of 20s, 30s, 60s, 100s — sized for v2.7's faster
readiness flip and previously masked by the 3x retry on Run E2E tests
(dropped in #2558). Under v2.8 with no retry, every app that runs after
the first few hit its HR wait timeout exactly, surfacing as 13 of 16
test failures on PR #2619.
Standardise on 5m for HR Ready across all app tests. Tight enough to
catch genuine bring-up failures within the per-app job budget, loose
enough to absorb kstatus dispatch jitter on a management cluster that
is concurrently reconciling multiple tenant-test HRs.
Post-HR-Ready checks for app-specific resources (StatefulSet/Service/
PVC/operator CRs) remain — for CR-based apps kstatus only observes
that the chart applied, not that the operator finished. Audit of
which of those are actually redundant under v2.8 kstatus is queued
as a follow-up.
Also bumps the 10s "kamajicontrolplane appears" wait in
run-kubernetes.sh to 2m, and the kubernetes-addon HR Ready loop from
1m to 5m for the same reason.
Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
(cherry picked from commit 5ef02ee7b3)
114 lines
4.5 KiB
Bash
114 lines
4.5 KiB
Bash
#!/usr/bin/env bats
|
|
|
|
# The etcd chart pins the Helm release name to 'etcd' via
|
|
# packages/extra/etcd/templates/check-release-name.yaml, so every test
|
|
# must apply its Etcd with metadata.name: etcd. setup() clears any
|
|
# prior run so tests remain independent despite the singleton name.
|
|
|
|
setup() {
|
|
kubectl -n tenant-test delete etcd.apps.cozystack.io --all --ignore-not-found --timeout=2m
|
|
# HelmRelease teardown is async relative to the CR deletion above; wait for
|
|
# downstream resources so the next test starts from a clean state.
|
|
kubectl -n tenant-test wait hr/etcd --for=delete --timeout=2m --ignore-not-found
|
|
kubectl -n tenant-test wait secret/etcd-s3-creds --for=delete --timeout=1m --ignore-not-found
|
|
kubectl -n tenant-test wait etcdbackupschedule.etcd.aenix.io/etcd --for=delete --timeout=1m --ignore-not-found
|
|
}
|
|
|
|
dump_diagnostics() {
|
|
echo "# --- diagnostics ---" >&3
|
|
kubectl -n tenant-test get etcdcluster,etcdbackupschedule,cronjob -o wide >&3 2>&1 || true
|
|
kubectl -n tenant-test describe etcdbackupschedule etcd >&3 2>&1 || true
|
|
kubectl -n cozy-etcd-operator logs -l app.kubernetes.io/name=etcd-operator --tail=100 >&3 2>&1 || true
|
|
}
|
|
|
|
# Wait until the etcd HelmRelease is reconciled by Flux and its
|
|
# condition=ready is set. kubectl wait fails immediately on a missing
|
|
# resource, so poll for existence first.
|
|
wait_etcd_hr_ready() {
|
|
timeout 60 sh -ec 'until kubectl -n tenant-test get hr/etcd >/dev/null 2>&1; do sleep 2; done'
|
|
kubectl -n tenant-test wait hr/etcd --timeout=5m --for=condition=ready
|
|
}
|
|
|
|
@test "Create Etcd" {
|
|
kubectl apply -f- <<EOF
|
|
apiVersion: apps.cozystack.io/v1alpha1
|
|
kind: Etcd
|
|
metadata:
|
|
name: etcd
|
|
namespace: tenant-test
|
|
spec:
|
|
size: 1Gi
|
|
replicas: 3
|
|
storageClass: ""
|
|
resources:
|
|
cpu: 100m
|
|
memory: 128Mi
|
|
EOF
|
|
wait_etcd_hr_ready || { dump_diagnostics; false; }
|
|
kubectl -n tenant-test wait etcdcluster.etcd.aenix.io etcd --timeout=180s --for=jsonpath='{.status.conditions[?(@.type=="Ready")].status}'=True || { dump_diagnostics; false; }
|
|
}
|
|
|
|
@test "Create Etcd with empty backup block (disabled by default)" {
|
|
kubectl apply -f- <<EOF
|
|
apiVersion: apps.cozystack.io/v1alpha1
|
|
kind: Etcd
|
|
metadata:
|
|
name: etcd
|
|
namespace: tenant-test
|
|
spec:
|
|
size: 1Gi
|
|
replicas: 3
|
|
storageClass: ""
|
|
resources:
|
|
cpu: 100m
|
|
memory: 128Mi
|
|
backup: {}
|
|
EOF
|
|
wait_etcd_hr_ready || { dump_diagnostics; false; }
|
|
# With backup disabled, neither the schedule nor the secret should be created.
|
|
! kubectl -n tenant-test get etcdbackupschedule.etcd.aenix.io etcd 2>/dev/null
|
|
! kubectl -n tenant-test get secret etcd-s3-creds 2>/dev/null
|
|
}
|
|
|
|
@test "Create Etcd with backup schedule" {
|
|
kubectl apply -f- <<EOF
|
|
apiVersion: apps.cozystack.io/v1alpha1
|
|
kind: Etcd
|
|
metadata:
|
|
name: etcd
|
|
namespace: tenant-test
|
|
spec:
|
|
size: 1Gi
|
|
replicas: 3
|
|
storageClass: ""
|
|
resources:
|
|
cpu: 100m
|
|
memory: 128Mi
|
|
backup:
|
|
enabled: true
|
|
# Schedule is chosen far in the future so no Jobs fire during the
|
|
# ~6-minute test window; this test only checks that the chart renders
|
|
# EtcdBackupSchedule/Secret and that the etcd-operator materializes a
|
|
# CronJob — it does NOT verify that backups reach S3. The endpoint
|
|
# below intentionally resolves nowhere to keep the test self-contained.
|
|
schedule: "0 0 1 1 *"
|
|
destinationPath: "s3://test-bucket/etcd-backups/"
|
|
endpointURL: "http://no-such-endpoint.invalid:9000"
|
|
region: "us-east-1"
|
|
forcePathStyle: true
|
|
s3AccessKey: "e2e-access-key"
|
|
s3SecretKey: "e2e-secret-key"
|
|
successfulJobsHistoryLimit: 1
|
|
failedJobsHistoryLimit: 1
|
|
EOF
|
|
wait_etcd_hr_ready || { dump_diagnostics; false; }
|
|
kubectl -n tenant-test wait etcdcluster.etcd.aenix.io etcd --timeout=180s --for=jsonpath='{.status.conditions[?(@.type=="Ready")].status}'=True || { dump_diagnostics; false; }
|
|
kubectl -n tenant-test get etcdbackupschedule.etcd.aenix.io etcd || { dump_diagnostics; false; }
|
|
# Verify the region field propagated to the EtcdBackupSchedule.
|
|
REGION=$(kubectl -n tenant-test get etcdbackupschedule.etcd.aenix.io etcd -o jsonpath='{.spec.destination.s3.region}')
|
|
[ "$REGION" = "us-east-1" ]
|
|
kubectl -n tenant-test get secret etcd-s3-creds -o jsonpath='{.data.AWS_ACCESS_KEY_ID}' | base64 -d | grep -q '^e2e-access-key$'
|
|
# The etcd-operator generates a CronJob from the EtcdBackupSchedule. Wait for it.
|
|
timeout 120 sh -ec "until [ \"\$(kubectl -n tenant-test get cronjob -l etcd.aenix.io/etcdbackupschedule-name=etcd -o jsonpath='{.items[*].metadata.name}' 2>/dev/null)\" != '' ]; do sleep 5; done" || { dump_diagnostics; false; }
|
|
}
|