From 45a273eba57fc0b6dc5e2926811d25ce7b5a458b Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin <f@lex.la>
Date: Wed, 24 Jun 2026 23:38:38 +0300
Subject: [PATCH] fix(monitoring-agents): relax fluent-bit probes for loaded
 installs

The fluent-bit DaemonSet crash-loops on a loaded install, timing out its
HelmRelease and cascading into velero and backupstrategy-controller
DependencyNotReady failures. The subchart default liveness/readiness probes
(initialDelaySeconds 0, timeoutSeconds 1, failureThreshold 3) target the HTTP
server on :2020, but that server does not bind until AFTER the kubernetes
filter's 'testing connectivity with API server' step. Under a concurrent
install the kube-API can be unreachable for ~10s, delaying the :2020 bind,
while the single-threaded HTTP server can miss a 1s timeout when the engine is
busy. The probes then fire into a connection-refused port and liveness-kill
fluent-bit into a restart loop.

Give both probes load headroom (initialDelaySeconds 15, timeoutSeconds 5,
failureThreshold 6) so the API-gated late bind and busy-engine blips are
tolerated. The httpGet target is unchanged (path / on port http = :2020). Add
a helm-unittest pinning the relaxed timing and the preserved target.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
---
 .../tests/fluent-bit_probes_test.yaml         | 53 +++++++++++++++++++
 packages/system/monitoring-agents/values.yaml | 19 +++++++
 2 files changed, 72 insertions(+)
 create mode 100644 packages/system/monitoring-agents/tests/fluent-bit_probes_test.yaml

diff --git a/packages/system/monitoring-agents/tests/fluent-bit_probes_test.yaml b/packages/system/monitoring-agents/tests/fluent-bit_probes_test.yaml
new file mode 100644
index 000000000..3816091a6
--- /dev/null
+++ b/packages/system/monitoring-agents/tests/fluent-bit_probes_test.yaml
@@ -0,0 +1,53 @@
+suite: fluent-bit liveness/readiness probes carry load headroom
+
+# Regression guard for the fluent-bit crash loop on a loaded install. The
+# subchart default probes (initialDelaySeconds 0, timeoutSeconds 1,
+# failureThreshold 3) fire into a connection-refused :2020 before fluent-bit's
+# HTTP server binds — that bind happens only AFTER the kubernetes filter's
+# "testing connectivity with API server" step, which the kube-API can gate for
+# ~10s under a concurrent install. The aggressive defaults then liveness-kill
+# fluent-bit into a restart loop, timing out the HelmRelease. The umbrella
+# values relax both probes. Pin the relaxed timing AND the unchanged target
+# (httpGet path / on port http = :2020) — a wrong path/port would make the
+# probe always fail, strictly worse than the bug it fixes.
+release:
+  name: monitoring-agents
+  namespace: cozy-monitoring
+tests:
+  - it: liveness probe keeps :2020 target and gains load headroom
+    template: charts/fluent-bit/templates/daemonset.yaml
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.httpGet.path
+          value: /
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.httpGet.port
+          value: http
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.initialDelaySeconds
+          value: 15
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.periodSeconds
+          value: 10
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.timeoutSeconds
+          value: 5
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.failureThreshold
+          value: 6
+
+  - it: readiness probe keeps :2020 target and gains load headroom
+    template: charts/fluent-bit/templates/daemonset.yaml
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.httpGet.path
+          value: /
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.httpGet.port
+          value: http
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.timeoutSeconds
+          value: 5
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.failureThreshold
+          value: 6
diff --git a/packages/system/monitoring-agents/values.yaml b/packages/system/monitoring-agents/values.yaml
index bcdee0e0b..4357db276 100644
--- a/packages/system/monitoring-agents/values.yaml
+++ b/packages/system/monitoring-agents/values.yaml
@@ -287,9 +287,28 @@ vmagent:
 fluent-bit:
   rbac:
     eventsAccess: true
+  # Relax probe timing to survive a loaded install. Fluent-bit's HTTP server on
+  # :2020 (the probe target) does not bind until AFTER the kubernetes filter's
+  # "testing connectivity with API server" step; under concurrent install the
+  # kube-API can be unreachable for ~10s, delaying the :2020 bind, and the
+  # single-threaded HTTP server can miss a 1s timeout while the engine is busy.
+  # The subchart defaults (initialDelaySeconds 0, timeoutSeconds 1,
+  # failureThreshold 3) then liveness-kill fluent-bit into a crash loop. httpGet
+  # stays on path / port http (:2020) — same target as the subchart defaults.
+  livenessProbe:
+    httpGet:
+      path: /
+      port: http
+    initialDelaySeconds: 15
+    periodSeconds: 10
+    timeoutSeconds: 5
+    failureThreshold: 6
   readinessProbe:
     httpGet:
       path: /
+      port: http
+    timeoutSeconds: 5
+    failureThreshold: 6
   daemonSetVolumes:
     - name: varlog
       hostPath: