mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 20:23:39 +09:00
cozystack-api gives every release it generates a 600s install timeout, and these asserts gave up at 5-6m. A failed install was therefore reported as a bare assert timeout while the release was still installing and had not yet recorded why it failed. Raise them to 11m, above that floor, so the release's own reason is on record before Chainsaw gives up. The cost is up to six more minutes on a red run of each suite; a green run is unchanged. vm-instance-test stays: vm-instance-rd disables the install wait, so its release goes Ready as soon as helm has applied. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <f@lex.la>
235 lines
9.4 KiB
YAML
235 lines
9.4 KiB
YAML
# Chainsaw port of hack/e2e-apps/foundationdb.bats ("Create DB FoundationDB").
|
|
#
|
|
# The dense block of `until kubectl get ... jsonpath | grep` status probes
|
|
# collapses into one assert on the FoundationDBCluster status tree. Pod-count
|
|
# (by process class + phase) and PVC-size checks stay as polling scripts —
|
|
# they count List results, which an assert cannot express.
|
|
apiVersion: chainsaw.kyverno.io/v1alpha1
|
|
kind: Test
|
|
metadata:
|
|
name: foundationdb
|
|
spec:
|
|
catch:
|
|
- events: {}
|
|
- describe:
|
|
apiVersion: helm.toolkit.fluxcd.io/v2
|
|
kind: HelmRelease
|
|
name: foundationdb-test
|
|
- describe:
|
|
apiVersion: apps.foundationdb.org/v1beta2
|
|
kind: FoundationDBCluster
|
|
name: foundationdb-test
|
|
- podLogs:
|
|
selector: foundationdb.org/fdb-cluster-name=foundationdb-test
|
|
tail: 50
|
|
steps:
|
|
- name: create-foundationdb
|
|
try:
|
|
- apply:
|
|
file: foundationdb.yaml
|
|
|
|
- name: wait-helmrelease-ready
|
|
try:
|
|
# 11m: above the 600s install timeout cozystack-api gives this release;
|
|
# see "Sizing an HR-Ready budget" in docs/agents/e2e-testing.md.
|
|
- assert:
|
|
timeout: 11m
|
|
resource:
|
|
apiVersion: helm.toolkit.fluxcd.io/v2
|
|
kind: HelmRelease
|
|
metadata:
|
|
name: foundationdb-test
|
|
status:
|
|
(conditions[?type == 'Ready']):
|
|
- status: "True"
|
|
|
|
- name: assert-cluster-status
|
|
try:
|
|
- assert:
|
|
timeout: 5m
|
|
resource:
|
|
apiVersion: apps.foundationdb.org/v1beta2
|
|
kind: FoundationDBCluster
|
|
metadata:
|
|
name: foundationdb-test
|
|
status:
|
|
configured: true
|
|
hasListenIPsForAllPods: true
|
|
databaseConfiguration:
|
|
logs: 3
|
|
proxies: 3
|
|
redundancy_mode: double
|
|
resolvers: 1
|
|
storage_engine: ssd-2
|
|
usable_regions: 1
|
|
health:
|
|
available: true
|
|
fullReplication: true
|
|
# volumeClaimTemplate size is pinned on the CR spec.
|
|
- assert:
|
|
resource:
|
|
apiVersion: apps.foundationdb.org/v1beta2
|
|
kind: FoundationDBCluster
|
|
metadata:
|
|
name: foundationdb-test
|
|
spec:
|
|
processes:
|
|
general:
|
|
volumeClaimTemplate:
|
|
spec:
|
|
resources:
|
|
requests:
|
|
storage: 1Gi
|
|
|
|
- name: verify-processes-and-security-context
|
|
try:
|
|
- script:
|
|
# Three sequential inner deadlines -- 240s for the process counts, then
|
|
# 120s for connectionString and 120s for generations.reconciled -- sum to
|
|
# 480s. Pre-existing overrun, found by applying the same .chainsaw.yaml
|
|
# rule to the step next door: at 5m the op fired first and took the whole
|
|
# script out by SIGKILL, so a slow-but-recovering cluster reported
|
|
# `signal: killed` instead of naming the stage that was still waiting.
|
|
timeout: 9m
|
|
content: |
|
|
set -eu
|
|
ns="$NAMESPACE"
|
|
cl=foundationdb-test
|
|
sel="foundationdb.org/fdb-cluster-name=$cl"
|
|
count_running() {
|
|
kubectl -n "$ns" get pods -l "$sel,foundationdb.org/fdb-process-class=$1" \
|
|
--field-selector=status.phase=Running --no-headers 2>/dev/null | wc -l
|
|
}
|
|
deadline=$(( $(date +%s) + 240 ))
|
|
while [ "$(date +%s)" -lt "$deadline" ]; do
|
|
s=$(count_running storage); l=$(count_running log); c=$(count_running cluster_controller)
|
|
echo "storage=$s log=$l cluster_controller=$c"
|
|
if [ "$s" -eq 3 ] && [ "$l" -ge 1 ] && [ "$c" -eq 1 ]; then break; fi
|
|
sleep 10
|
|
done
|
|
test "$(count_running storage)" -eq 3
|
|
test "$(count_running log)" -ge 1
|
|
test "$(count_running cluster_controller)" -eq 1
|
|
# securityContext: storage pod must run as the non-root 4059 uid/gid.
|
|
pod=$(kubectl -n "$ns" get pods -l "$sel,foundationdb.org/fdb-process-class=storage" \
|
|
--no-headers | head -n1 | awk '{print $1}')
|
|
kubectl -n "$ns" get pod "$pod" -o jsonpath='{.spec.containers[0].securityContext.runAsUser}' | grep -qx 4059
|
|
kubectl -n "$ns" get pod "$pod" -o jsonpath='{.spec.containers[0].securityContext.runAsGroup}' | grep -qx 4059
|
|
# Status fields that are nil early on (so they live here, not in the
|
|
# assert, where contains()/numeric-compare on nil hard-errors).
|
|
fdb() { kubectl -n "$ns" get foundationdbclusters.apps.foundationdb.org "$cl" -o jsonpath="$1"; }
|
|
deadline=$(( $(date +%s) + 120 ))
|
|
until fdb '{.status.connectionString}' | grep -qE '@.*\.svc\.cozy\.local'; do
|
|
[ "$(date +%s)" -ge "$deadline" ] && { echo "connectionString not set" >&2; fdb '{.status.connectionString}' >&2; exit 1; }
|
|
sleep 5
|
|
done
|
|
fdb '{.status.desiredProcessGroups}' | grep -qE '^[0-9]+$'
|
|
# generations.reconciled is stamped by the operator after the
|
|
# post-bounce uptime gate and can trail the fdbcli-derived health
|
|
# fields the previous assert converged on; retry briefly instead of
|
|
# a one-shot read (mirrors main's widened budgets, a79f8009f).
|
|
deadline=$(( $(date +%s) + 120 ))
|
|
until fdb '{.status.generations.reconciled}' | grep -qE '^[0-9]+$'; do
|
|
[ "$(date +%s)" -ge "$deadline" ] && { echo "generations.reconciled not stamped" >&2; fdb '{.status.generations}' >&2; exit 1; }
|
|
sleep 5
|
|
done
|
|
check:
|
|
($error == null): true
|
|
|
|
- name: refresh-and-assert-full-health
|
|
try:
|
|
- script:
|
|
# 270s poll deadline + one 25s in-loop read that may start just under it
|
|
# + one 25s diagnostic read on the failure branch = 320s of inner bounds.
|
|
# Keep the op above that sum, per .chainsaw.yaml: at 5m Chainsaw SIGKILLs
|
|
# the script instead, and the diagnostic that the failure branch exists
|
|
# to print is exactly what gets lost.
|
|
timeout: 6m
|
|
content: |
|
|
set -eu
|
|
ns="$NAMESPACE"
|
|
cl=foundationdb-test
|
|
pod=$(kubectl -n "$ns" get pods \
|
|
-l "foundationdb.org/fdb-cluster-name=$cl,foundationdb.org/fdb-process-class=storage" \
|
|
-o jsonpath='{.items[0].metadata.name}')
|
|
status_json() {
|
|
timeout -k 5 20 kubectl -n "$ns" exec "$pod" -c foundationdb -- \
|
|
fdbcli -C /var/fdb/data/fdb.cluster --exec 'status json' 2>/dev/null
|
|
}
|
|
database_healthy() {
|
|
status_json | jq -e '.client.database_status.healthy == true' >/dev/null
|
|
}
|
|
# The v2.30 operator can finish its initial reconciliation from a
|
|
# transient status_incomplete response, then never requeue solely to
|
|
# refresh health. Wait on the authoritative machine-readable status
|
|
# before asking the operator for exactly one fresh read.
|
|
deadline=$(( $(date +%s) + 270 ))
|
|
until database_healthy; do
|
|
if [ "$(date +%s)" -ge "$deadline" ]; then
|
|
echo "FoundationDB machine status did not become fully healthy" >&2
|
|
status_json | jq '{databaseStatus: .client.database_status, messages: .cluster.messages}' >&2 || true
|
|
exit 1
|
|
fi
|
|
# TODO(e2e-replace-fixed-timeouts): fdbcli health has no watchable
|
|
# Kubernetes object; replace this poll when the operator exposes one.
|
|
sleep 5
|
|
done
|
|
kubectl -n "$ns" annotate foundationdbclusters.apps.foundationdb.org "$cl" \
|
|
"e2e.cozystack.io/health-refresh=$(date +%s)" --overwrite >/dev/null
|
|
check:
|
|
($error == null): true
|
|
- assert:
|
|
timeout: 3m
|
|
resource:
|
|
apiVersion: apps.foundationdb.org/v1beta2
|
|
kind: FoundationDBCluster
|
|
metadata:
|
|
name: foundationdb-test
|
|
status:
|
|
health:
|
|
healthy: true
|
|
|
|
- name: verify-monitoring-and-storage
|
|
try:
|
|
# Dashboard resource map ConfigMap is created by the controller.
|
|
- assert:
|
|
timeout: 2m
|
|
resource:
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: foundationdb-test-resourcemap
|
|
- script:
|
|
timeout: 2m
|
|
content: |
|
|
set -eu
|
|
ns="$NAMESPACE"
|
|
cl=foundationdb-test
|
|
# WorkloadMonitor is created and configured with replicas=3.
|
|
kubectl -n "$ns" get workloadmonitor "$cl"
|
|
kubectl -n "$ns" get workloadmonitor "$cl" -o jsonpath='{.spec.replicas}' | grep -qx 3
|
|
# PVCs: at least 3, all requesting and reporting 1Gi.
|
|
count=$(kubectl -n "$ns" get pvc -l "foundationdb.org/fdb-cluster-name=$cl" --no-headers | wc -l)
|
|
test "$count" -ge 3
|
|
kubectl -n "$ns" get pvc -l "foundationdb.org/fdb-cluster-name=$cl" \
|
|
-o jsonpath='{.items[*].spec.resources.requests.storage}' | grep -q '1Gi'
|
|
kubectl -n "$ns" get pvc -l "foundationdb.org/fdb-cluster-name=$cl" \
|
|
-o jsonpath='{.items[*].status.capacity.storage}' | grep -q '1Gi'
|
|
check:
|
|
($error == null): true
|
|
|
|
- name: verify-cleanup
|
|
try:
|
|
- delete:
|
|
ref:
|
|
apiVersion: apps.cozystack.io/v1alpha1
|
|
kind: FoundationDB
|
|
name: test
|
|
- error:
|
|
timeout: 2m
|
|
resource:
|
|
apiVersion: apps.foundationdb.org/v1beta2
|
|
kind: FoundationDBCluster
|
|
metadata:
|
|
name: foundationdb-test
|