From bfc1fbd555695d8cee954050837bd22bc8a7dc7e Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 25 Jun 2026 13:39:34 +0300 Subject: [PATCH] fix(seaweedfs): raise master probe timeouts above the cluster-status backoff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The master's /cluster/status:9333 handler calls topo.Leader(), which spins an exponential backoff with MaxElapsedTime=20s before returning 200 whenever the node has no leader in its view — e.g. a master that was briefly the initial raft leader during bootstrap and then was demoted. The chart's master liveness and readiness probes both httpGet /cluster/status with timeoutSeconds: 10, which is shorter than that worst-case backoff. Such a master is then liveness-killed roughly every 137s into a permanent CrashLoop, the master StatefulSet never reaches full readiness, and the helm --wait install times out. Raise both master probe timeouts to 30s so each probe outlasts the 20s backoff (the volume component already uses 30s). The override sets only timeoutSeconds and relies on Helm's deep-merge to keep the rest of the subchart probe defaults. Both probes must be raised: leaving readiness at 10s would keep the master from becoming Ready and still stall the StatefulSet. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- .../tests/master_probe_timeout_test.yaml | 64 +++++++++++++++++++ packages/system/seaweedfs/values.yaml | 13 ++++ 2 files changed, 77 insertions(+) create mode 100644 packages/system/seaweedfs/tests/master_probe_timeout_test.yaml diff --git a/packages/system/seaweedfs/tests/master_probe_timeout_test.yaml b/packages/system/seaweedfs/tests/master_probe_timeout_test.yaml new file mode 100644 index 000000000..dabcfb963 --- /dev/null +++ b/packages/system/seaweedfs/tests/master_probe_timeout_test.yaml @@ -0,0 +1,64 @@ +suite: seaweedfs master probe timeouts + +# Pins the master probe timeout overrides that keep master-0 alive during raft +# bootstrap. The master's /cluster/status handler calls topo.Leader(), which +# backs off up to ~20s (MaxElapsedTime=20s) while the node has no leader in +# view — e.g. a master that was briefly the initial raft leader then was demoted. +# The subchart default 10s probe timeout is shorter than that worst case, so such +# a master is liveness-killed into a permanent CrashLoop and the master +# StatefulSet never reaches full readiness. Both probes must allow more than the +# 20s backoff. The overlay overrides only timeoutSeconds and relies on Helm's +# deep-merge to keep every other subchart probe field, so this also guards that +# the merge does not drop the surrounding probe configuration. + +templates: + - charts/seaweedfs/templates/master/master-statefulset.yaml + +release: + name: seaweedfs + namespace: cozy-seaweedfs + +tests: + - it: raises both master probe timeouts above the cluster-status backoff + asserts: + - equal: + path: spec.template.spec.containers[0].livenessProbe.timeoutSeconds + value: 30 + - equal: + path: spec.template.spec.containers[0].readinessProbe.timeoutSeconds + value: 30 + + - it: preserves the other master probe fields through the deep-merge + asserts: + # liveness: only timeoutSeconds is overridden, the rest survive + - equal: + path: spec.template.spec.containers[0].livenessProbe.httpGet.path + value: /cluster/status + - equal: + path: spec.template.spec.containers[0].livenessProbe.failureThreshold + value: 4 + - equal: + path: spec.template.spec.containers[0].livenessProbe.successThreshold + value: 1 + - equal: + path: spec.template.spec.containers[0].livenessProbe.initialDelaySeconds + value: 20 + - equal: + path: spec.template.spec.containers[0].livenessProbe.periodSeconds + value: 30 + # readiness: only timeoutSeconds is overridden, the rest survive + - equal: + path: spec.template.spec.containers[0].readinessProbe.httpGet.path + value: /cluster/status + - equal: + path: spec.template.spec.containers[0].readinessProbe.failureThreshold + value: 100 + - equal: + path: spec.template.spec.containers[0].readinessProbe.successThreshold + value: 2 + - equal: + path: spec.template.spec.containers[0].readinessProbe.initialDelaySeconds + value: 10 + - equal: + path: spec.template.spec.containers[0].readinessProbe.periodSeconds + value: 45 diff --git a/packages/system/seaweedfs/values.yaml b/packages/system/seaweedfs/values.yaml index ac31fe81d..62612eae3 100644 --- a/packages/system/seaweedfs/values.yaml +++ b/packages/system/seaweedfs/values.yaml @@ -32,6 +32,19 @@ seaweedfs: type: "emptyDir" logs: type: "" + # The master's /cluster/status handler calls topo.Leader(), which backs off + # up to ~20s (MaxElapsedTime=20s) while the node has no leader in view — e.g. + # a master that was briefly the initial raft leader then was demoted during + # bootstrap. The default 10s probe timeout is shorter than that worst case, so + # such a master is liveness-killed every ~137s into a permanent CrashLoop and + # the master StatefulSet never reaches full readiness, timing out the install. + # Both probes must allow more than the 20s backoff (the volume component in + # this chart already uses 30s). Override only timeoutSeconds; Helm deep-merges + # these onto the subchart defaults, keeping enabled/path/initialDelay/etc. + livenessProbe: + timeoutSeconds: 30 + readinessProbe: + timeoutSeconds: 30 volume: replicas: 2 resizeHook: