diff --git a/hack/e2e-chainsaw/mariadb/chainsaw-test.yaml b/hack/e2e-chainsaw/mariadb/chainsaw-test.yaml index 6e7f72433..c5f1f4303 100644 --- a/hack/e2e-chainsaw/mariadb/chainsaw-test.yaml +++ b/hack/e2e-chainsaw/mariadb/chainsaw-test.yaml @@ -43,8 +43,18 @@ spec: name: mariadb-test spec: (ports[0].port): 3306 + # The chart hands the operator a MariaDB CR and ships no built-in workload, + # so the HelmRelease above goes Ready as soon as helm applies the manifests + # -- it never waits for a pod. This assert is the first gate that does: an + # endpoint address appears only once a replica has passed its startup and + # then its readiness probe. The startup probe grants each replica 310s of + # first-boot budget (see packages/apps/mariadb/templates/mariadb.yaml for + # why), so a ceiling under that would fail runs the probe was still willing + # to wait for. One ready address satisfies this assert, so it has to clear + # one budget, not two, plus the operator reconcile, PVC bind and image pull + # that precede it. 10m leaves room for all of that. - assert: - timeout: 3m + timeout: 10m resource: apiVersion: v1 kind: Endpoints @@ -142,8 +152,12 @@ spec: name: mariadb-single spec: (ports[0].port): 3306 + # Same gate as the replicated case above -- the HelmRelease does not wait + # for a pod, so this is where first boot is actually covered. One replica + # means one 310s startup budget, plus the operator reconcile, PVC bind and + # image pull ahead of it. - assert: - timeout: 3m + timeout: 10m resource: apiVersion: v1 kind: Endpoints diff --git a/packages/apps/mariadb/templates/mariadb.yaml b/packages/apps/mariadb/templates/mariadb.yaml index e6925591d..8c141e801 100644 --- a/packages/apps/mariadb/templates/mariadb.yaml +++ b/packages/apps/mariadb/templates/mariadb.yaml @@ -12,6 +12,28 @@ spec: port: 3306 + # The operator leaves failureThreshold unset on the startup probe, so it + # defaults to 3. The kubelet only begins probing once initialDelaySeconds has + # elapsed and kills the container on the Nth consecutive failure, so the + # budget is initialDelay + (failureThreshold - 1) * period, here just 40s -- + # the same budget as the liveness probe the startup probe exists to defer. + # A fresh datadir bootstrap (mysql_install_db, then the entrypoint's + # temporary server applying the root password) does not reliably finish + # inside that budget under a constrained CPU limit or slow replicated + # storage. Overrunning it is not merely a slow start: the kubelet kills the + # container mid-bootstrap, leaving a datadir that already has the mysql + # system tables but never received the root grant. On restart the entrypoint + # keys DATABASE_ALREADY_EXISTS off that directory, skips initialization for + # good, and the server comes up with the install-time root account -- local + # only and passwordless -- instead of the credentials the operator holds. So + # every later probe fails with "Access denied for user 'root'" and the pod + # crash-loops permanently with no path back. A threshold of 30 lifts the + # budget to 310s. Widen only the startup budget; the liveness and readiness + # probes keep the operator defaults, so a server that hangs after a + # successful start is still caught as quickly as before. + startupProbe: + failureThreshold: 30 + replicas: {{ .Values.replicas }} replicasAllowEvenNumber: true affinity: diff --git a/packages/apps/mariadb/tests/startup_probe_test.yaml b/packages/apps/mariadb/tests/startup_probe_test.yaml new file mode 100644 index 000000000..d6df7e0f4 --- /dev/null +++ b/packages/apps/mariadb/tests/startup_probe_test.yaml @@ -0,0 +1,52 @@ +suite: MariaDB startup probe budget + +# Regression cover for the widened startup budget. The rationale lives with +# the field itself, in templates/mariadb.yaml. + +templates: + - templates/mariadb.yaml + +tests: + - it: widens the startup probe budget for a single-replica instance + release: + name: mariadb-test + namespace: tenant-test + set: + replicas: 1 + asserts: + - equal: + path: spec.startupProbe.failureThreshold + value: 30 + + - it: widens the startup probe budget for a replicated instance + release: + name: mariadb-test + namespace: tenant-test + set: + replicas: 2 + asserts: + - equal: + path: spec.startupProbe.failureThreshold + value: 30 + + - it: leaves liveness and readiness probes to the operator defaults + release: + name: mariadb-test + namespace: tenant-test + asserts: + - notExists: + path: spec.livenessProbe + - notExists: + path: spec.readinessProbe + + - it: overrides only thresholds so the operator keeps its own probe handler + release: + name: mariadb-test + namespace: tenant-test + asserts: + - notExists: + path: spec.startupProbe.exec + - notExists: + path: spec.startupProbe.httpGet + - notExists: + path: spec.startupProbe.tcpSocket