Files
cozystack/hack/e2e-chainsaw/foundationdb/chainsaw-test.yaml
T
Aleksei Sviridkin 9e6cb7d0cc fix(tests): let HR-Ready asserts outlive the install they wait on
cozystack-api gives every release it generates a 600s install timeout,
and these asserts gave up at 5-6m. A failed install was therefore
reported as a bare assert timeout while the release was still
installing and had not yet recorded why it failed.

Raise them to 11m, above that floor, so the release's own reason is on
record before Chainsaw gives up. The cost is up to six more minutes on
a red run of each suite; a green run is unchanged. vm-instance-test
stays: vm-instance-rd disables the install wait, so its release goes
Ready as soon as helm has applied.

Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin <f@lex.la>
2026-10-01 17:40:21 +03:00

235 lines
9.4 KiB
YAML

# Chainsaw port of hack/e2e-apps/foundationdb.bats ("Create DB FoundationDB").
#
# The dense block of `until kubectl get ... jsonpath | grep` status probes
# collapses into one assert on the FoundationDBCluster status tree. Pod-count
# (by process class + phase) and PVC-size checks stay as polling scripts —
# they count List results, which an assert cannot express.
apiVersion: chainsaw.kyverno.io/v1alpha1
kind: Test
metadata:
name: foundationdb
spec:
catch:
- events: {}
- describe:
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
name: foundationdb-test
- describe:
apiVersion: apps.foundationdb.org/v1beta2
kind: FoundationDBCluster
name: foundationdb-test
- podLogs:
selector: foundationdb.org/fdb-cluster-name=foundationdb-test
tail: 50
steps:
- name: create-foundationdb
try:
- apply:
file: foundationdb.yaml
- name: wait-helmrelease-ready
try:
# 11m: above the 600s install timeout cozystack-api gives this release;
# see "Sizing an HR-Ready budget" in docs/agents/e2e-testing.md.
- assert:
timeout: 11m
resource:
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: foundationdb-test
status:
(conditions[?type == 'Ready']):
- status: "True"
- name: assert-cluster-status
try:
- assert:
timeout: 5m
resource:
apiVersion: apps.foundationdb.org/v1beta2
kind: FoundationDBCluster
metadata:
name: foundationdb-test
status:
configured: true
hasListenIPsForAllPods: true
databaseConfiguration:
logs: 3
proxies: 3
redundancy_mode: double
resolvers: 1
storage_engine: ssd-2
usable_regions: 1
health:
available: true
fullReplication: true
# volumeClaimTemplate size is pinned on the CR spec.
- assert:
resource:
apiVersion: apps.foundationdb.org/v1beta2
kind: FoundationDBCluster
metadata:
name: foundationdb-test
spec:
processes:
general:
volumeClaimTemplate:
spec:
resources:
requests:
storage: 1Gi
- name: verify-processes-and-security-context
try:
- script:
# Three sequential inner deadlines -- 240s for the process counts, then
# 120s for connectionString and 120s for generations.reconciled -- sum to
# 480s. Pre-existing overrun, found by applying the same .chainsaw.yaml
# rule to the step next door: at 5m the op fired first and took the whole
# script out by SIGKILL, so a slow-but-recovering cluster reported
# `signal: killed` instead of naming the stage that was still waiting.
timeout: 9m
content: |
set -eu
ns="$NAMESPACE"
cl=foundationdb-test
sel="foundationdb.org/fdb-cluster-name=$cl"
count_running() {
kubectl -n "$ns" get pods -l "$sel,foundationdb.org/fdb-process-class=$1" \
--field-selector=status.phase=Running --no-headers 2>/dev/null | wc -l
}
deadline=$(( $(date +%s) + 240 ))
while [ "$(date +%s)" -lt "$deadline" ]; do
s=$(count_running storage); l=$(count_running log); c=$(count_running cluster_controller)
echo "storage=$s log=$l cluster_controller=$c"
if [ "$s" -eq 3 ] && [ "$l" -ge 1 ] && [ "$c" -eq 1 ]; then break; fi
sleep 10
done
test "$(count_running storage)" -eq 3
test "$(count_running log)" -ge 1
test "$(count_running cluster_controller)" -eq 1
# securityContext: storage pod must run as the non-root 4059 uid/gid.
pod=$(kubectl -n "$ns" get pods -l "$sel,foundationdb.org/fdb-process-class=storage" \
--no-headers | head -n1 | awk '{print $1}')
kubectl -n "$ns" get pod "$pod" -o jsonpath='{.spec.containers[0].securityContext.runAsUser}' | grep -qx 4059
kubectl -n "$ns" get pod "$pod" -o jsonpath='{.spec.containers[0].securityContext.runAsGroup}' | grep -qx 4059
# Status fields that are nil early on (so they live here, not in the
# assert, where contains()/numeric-compare on nil hard-errors).
fdb() { kubectl -n "$ns" get foundationdbclusters.apps.foundationdb.org "$cl" -o jsonpath="$1"; }
deadline=$(( $(date +%s) + 120 ))
until fdb '{.status.connectionString}' | grep -qE '@.*\.svc\.cozy\.local'; do
[ "$(date +%s)" -ge "$deadline" ] && { echo "connectionString not set" >&2; fdb '{.status.connectionString}' >&2; exit 1; }
sleep 5
done
fdb '{.status.desiredProcessGroups}' | grep -qE '^[0-9]+$'
# generations.reconciled is stamped by the operator after the
# post-bounce uptime gate and can trail the fdbcli-derived health
# fields the previous assert converged on; retry briefly instead of
# a one-shot read (mirrors main's widened budgets, a79f8009f).
deadline=$(( $(date +%s) + 120 ))
until fdb '{.status.generations.reconciled}' | grep -qE '^[0-9]+$'; do
[ "$(date +%s)" -ge "$deadline" ] && { echo "generations.reconciled not stamped" >&2; fdb '{.status.generations}' >&2; exit 1; }
sleep 5
done
check:
($error == null): true
- name: refresh-and-assert-full-health
try:
- script:
# 270s poll deadline + one 25s in-loop read that may start just under it
# + one 25s diagnostic read on the failure branch = 320s of inner bounds.
# Keep the op above that sum, per .chainsaw.yaml: at 5m Chainsaw SIGKILLs
# the script instead, and the diagnostic that the failure branch exists
# to print is exactly what gets lost.
timeout: 6m
content: |
set -eu
ns="$NAMESPACE"
cl=foundationdb-test
pod=$(kubectl -n "$ns" get pods \
-l "foundationdb.org/fdb-cluster-name=$cl,foundationdb.org/fdb-process-class=storage" \
-o jsonpath='{.items[0].metadata.name}')
status_json() {
timeout -k 5 20 kubectl -n "$ns" exec "$pod" -c foundationdb -- \
fdbcli -C /var/fdb/data/fdb.cluster --exec 'status json' 2>/dev/null
}
database_healthy() {
status_json | jq -e '.client.database_status.healthy == true' >/dev/null
}
# The v2.30 operator can finish its initial reconciliation from a
# transient status_incomplete response, then never requeue solely to
# refresh health. Wait on the authoritative machine-readable status
# before asking the operator for exactly one fresh read.
deadline=$(( $(date +%s) + 270 ))
until database_healthy; do
if [ "$(date +%s)" -ge "$deadline" ]; then
echo "FoundationDB machine status did not become fully healthy" >&2
status_json | jq '{databaseStatus: .client.database_status, messages: .cluster.messages}' >&2 || true
exit 1
fi
# TODO(e2e-replace-fixed-timeouts): fdbcli health has no watchable
# Kubernetes object; replace this poll when the operator exposes one.
sleep 5
done
kubectl -n "$ns" annotate foundationdbclusters.apps.foundationdb.org "$cl" \
"e2e.cozystack.io/health-refresh=$(date +%s)" --overwrite >/dev/null
check:
($error == null): true
- assert:
timeout: 3m
resource:
apiVersion: apps.foundationdb.org/v1beta2
kind: FoundationDBCluster
metadata:
name: foundationdb-test
status:
health:
healthy: true
- name: verify-monitoring-and-storage
try:
# Dashboard resource map ConfigMap is created by the controller.
- assert:
timeout: 2m
resource:
apiVersion: v1
kind: ConfigMap
metadata:
name: foundationdb-test-resourcemap
- script:
timeout: 2m
content: |
set -eu
ns="$NAMESPACE"
cl=foundationdb-test
# WorkloadMonitor is created and configured with replicas=3.
kubectl -n "$ns" get workloadmonitor "$cl"
kubectl -n "$ns" get workloadmonitor "$cl" -o jsonpath='{.spec.replicas}' | grep -qx 3
# PVCs: at least 3, all requesting and reporting 1Gi.
count=$(kubectl -n "$ns" get pvc -l "foundationdb.org/fdb-cluster-name=$cl" --no-headers | wc -l)
test "$count" -ge 3
kubectl -n "$ns" get pvc -l "foundationdb.org/fdb-cluster-name=$cl" \
-o jsonpath='{.items[*].spec.resources.requests.storage}' | grep -q '1Gi'
kubectl -n "$ns" get pvc -l "foundationdb.org/fdb-cluster-name=$cl" \
-o jsonpath='{.items[*].status.capacity.storage}' | grep -q '1Gi'
check:
($error == null): true
- name: verify-cleanup
try:
- delete:
ref:
apiVersion: apps.cozystack.io/v1alpha1
kind: FoundationDB
name: test
- error:
timeout: 2m
resource:
apiVersion: apps.foundationdb.org/v1beta2
kind: FoundationDBCluster
metadata:
name: foundationdb-test