From a70a1cf83b5d94f18c7e4854a57bcb22dc4edc1c Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Thu, 5 Nov 2026 21:03:43 +0300 Subject: [PATCH] fix(seaweedfs): retry the data-plane release instead of rolling it back The admin volume and the two lifecycle pods are new demands on tenant quota, so an upgrade into a namespace with no headroom can wait out its timeout. Flux's default remediation then rolls the release back, and with unlimited retries it does so on every attempt: each rollback drops the [grpc.s3] section and the s3 restart annotation, each retry brings them back, and the s3 pods restart twice per cycle until quota frees up. RetryOnFailure keeps what was applied and retries it, as the parent release that the apiserver builds for the application already does. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../extra/seaweedfs/templates/seaweedfs.yaml | 11 ++++++++++ .../extra/seaweedfs/tests/lifecycle_test.yaml | 22 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/packages/extra/seaweedfs/templates/seaweedfs.yaml b/packages/extra/seaweedfs/templates/seaweedfs.yaml index c111a80c5..a2cf02c5b 100644 --- a/packages/extra/seaweedfs/templates/seaweedfs.yaml +++ b/packages/extra/seaweedfs/templates/seaweedfs.yaml @@ -214,10 +214,21 @@ spec: namespace: cozy-system interval: 30s timeout: 10m + {{- /* Flux remediates a failed install by uninstalling and a failed upgrade + by rolling back, and with retries: -1 it does so on every attempt. A + release waiting on tenant quota for the admin volume would then roll + the s3 pods back and forth with the [grpc.s3] restart annotation. + RetryOnFailure keeps what was applied and retries it. */}} install: + strategy: + name: RetryOnFailure + retryInterval: 30s remediation: retries: -1 upgrade: + strategy: + name: RetryOnFailure + retryInterval: 30s remediation: retries: -1 valuesFrom: diff --git a/packages/extra/seaweedfs/tests/lifecycle_test.yaml b/packages/extra/seaweedfs/tests/lifecycle_test.yaml index a040e0ae6..07aeb573e 100644 --- a/packages/extra/seaweedfs/tests/lifecycle_test.yaml +++ b/packages/extra/seaweedfs/tests/lifecycle_test.yaml @@ -194,3 +194,25 @@ tests: asserts: - hasDocuments: count: 0 + + # The admin PVC and the two lifecycle pods can leave an upgrade waiting on a + # full tenant quota. Under Flux's default rollback remediation each failed + # attempt would roll back the [grpc.s3] section and its restart annotation + # and the next one would bring them back, restarting the s3 pods on every + # cycle. Retrying in place keeps what was applied while the release waits. + - it: retries the data-plane release in place instead of rolling it back + template: templates/seaweedfs.yaml + set: *platform + documentSelector: + path: metadata.name + value: seaweedfs-system + asserts: + - equal: + path: spec.install.strategy.name + value: RetryOnFailure + - equal: + path: spec.upgrade.strategy.name + value: RetryOnFailure + - equal: + path: spec.upgrade.remediation.retries + value: -1