Merge origin/main into refactor/build-matrix-2937

Resolve conflict in .github/workflows/pull-requests.yaml: keep the new
per-package build matrix and fold in main's docker.io->mirror.gcr.io
base-image routing (#3042). Apply buildkitd-config: hack/buildkitd.toml
to all three new build legs (build, build-talos, finalize) so the mirror
fix covers the whole decomposed pipeline, not just the conflicted leg.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Signed-off-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>

.github/renovate.json

@@ -4,7 +4,7 @@
     "config:recommended",
     ":semanticCommits"
   ],
-  "enabledManagers": ["gomod"],
+  "enabledManagers": ["gomod", "dockerfile", "github-actions"],
   "postUpdateOptions": ["gomodTidy", "gomodUpdateImportPaths"],
   "labels": ["automated", "do-not-merge/hold"],
   "automerge": false,
@@ -22,6 +22,18 @@
       "matchManagers": ["gomod"],
       "matchDepTypes": ["golang"],
       "enabled": false
+    },
+    {
+      "description": "Pin Dockerfile base images and GitHub Actions to immutable digests so OpenSSF Scorecard Pinned-Dependencies passes and supply-chain refs stay reproducible. The tag is kept in front of the digest so it stays human-readable and trackable.",
+      "matchManagers": ["dockerfile", "github-actions"],
+      "pinDigests": true
+    },
+    {
+      "description": "Batch the initial pin and all digest-only refreshes (same tag, rebuilt upstream) into a single weekly PR. Without this, frequently-rebuilt bases (alpine, debian, golang, distroless) would trickle in one rebuild-triggering PR each, loading the shared CI runner. Real tag bumps (major/minor/patch) are intentionally left out of this group so they get individual review.",
+      "matchManagers": ["dockerfile", "github-actions"],
+      "matchUpdateTypes": ["digest", "pinDigest"],
+      "groupName": "container image & action digests",
+      "schedule": ["before 6am on monday"]
     }
   ]
 }

.github/workflows/pull-requests.yaml

@@ -138,11 +138,20 @@ jobs:
           DOCKER_CONFIG: ${{ runner.temp }}/.docker
       # docker-container driver: isolated buildkit per job (no shared-buildkit
       # contention) and a prerequisite for the mode=max cache reads (#2938).
-      - name: Set up Buildx (docker-container driver)
+      # Its buildkitd also routes docker.io base-image pulls through
+      # mirror.gcr.io (hack/buildkitd.toml): the Build step authenticates only
+      # to the push registry (OCIR), so base images (golang/ubuntu/nginx/...)
+      # would otherwise be pulled from Docker Hub anonymously and intermittently
+      # trip its anonymous-pull rate limit; the mirror sidesteps that and falls
+      # back to Docker Hub on a miss/outage. Create the builder under the SAME
+      # DOCKER_CONFIG the Build step reads (runner.temp/.docker, holding the
+      # OCIR auth) so buildx forwards push/cache auth to the container builder.
+      - name: Set up Buildx (docker-container driver, docker.io mirror)
         id: buildx
         uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
         with:
           driver: docker-container
+          buildkitd-config: hack/buildkitd.toml
         env:
           DOCKER_CONFIG: ${{ runner.temp }}/.docker
       - name: Build ${{ matrix.pkg }}
@@ -150,6 +159,9 @@ jobs:
         env:
           PKG: ${{ matrix.pkg }}
           DOCKER_CONFIG: ${{ runner.temp }}/.docker
+          # Build on the docker-container builder created above so base-image
+          # pulls go through the docker.io mirror (BUILDX_ARGS adds
+          # --builder=$(BUILDER)).
           BUILDER: ${{ steps.buildx.outputs.name }}
           # Unique per (PR, commit) so concurrent PR builds never push the same
           # tag. PR images are consumed by digest; :latest belongs to releases.
@@ -208,11 +220,15 @@ jobs:
           registry: iad.ocir.io
         env:
           DOCKER_CONFIG: ${{ runner.temp }}/.docker
-      - name: Set up Buildx (docker-container driver)
+      # buildkitd-config routes docker.io base-image pulls through mirror.gcr.io
+      # (hack/buildkitd.toml) so the talos image build dodges Docker Hub's
+      # anonymous-pull rate limit; see the build job for the full rationale.
+      - name: Set up Buildx (docker-container driver, docker.io mirror)
         id: buildx
         uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
         with:
           driver: docker-container
+          buildkitd-config: hack/buildkitd.toml
         env:
           DOCKER_CONFIG: ${{ runner.temp }}/.docker
       - name: Build Talos image (matchbox + installer tarball)
@@ -316,11 +332,15 @@ jobs:
           registry: iad.ocir.io
         env:
           DOCKER_CONFIG: ${{ runner.temp }}/.docker
-      - name: Set up Buildx (docker-container driver)
+      # buildkitd-config routes docker.io base-image pulls through mirror.gcr.io
+      # (hack/buildkitd.toml) so the installer build dodges Docker Hub's
+      # anonymous-pull rate limit; see the build job for the full rationale.
+      - name: Set up Buildx (docker-container driver, docker.io mirror)
         id: buildx
         uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
         with:
           driver: docker-container
+          buildkitd-config: hack/buildkitd.toml
         env:
           DOCKER_CONFIG: ${{ runner.temp }}/.docker
       # Builds the operator image AND flux-pushes the now-fully-patched packages

cmd/cozystack-operator/main.go

@@ -30,6 +30,7 @@ import (
 
 	cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
 	helmv2 "github.com/fluxcd/helm-controller/api/v2"
+	fluxmeta "github.com/fluxcd/pkg/apis/meta"
 	sourcev1 "github.com/fluxcd/source-controller/api/v1"
 	sourcewatcherv1beta1 "github.com/fluxcd/source-watcher/api/v2/v1beta1"
 	corev1 "k8s.io/api/core/v1"
@@ -95,6 +96,7 @@ func main() {
 	var platformSourceURL string
 	var platformSourceName string
 	var platformSourceRef string
+	var platformSourceSecret string
 
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
@@ -138,6 +140,7 @@ func main() {
 	flag.StringVar(&platformSourceURL, "platform-source-url", "", "Platform source URL (oci:// or https://). If specified, generates OCIRepository or GitRepository resource.")
 	flag.StringVar(&platformSourceName, "platform-source-name", "cozystack-platform", "Name for the generated platform source resource and PackageSource")
 	flag.StringVar(&platformSourceRef, "platform-source-ref", "", "Reference specification as key=value pairs (e.g., 'branch=main' or 'digest=sha256:...,tag=v1.0'). For OCI: digest, semver, semverFilter, tag. For Git: branch, tag, semver, name, commit.")
+	flag.StringVar(&platformSourceSecret, "platform-source-secret", "", "Name of the Secret in cozy-system namespace containing credentials for the platform source. Sets spec.secretRef on the generated OCIRepository or GitRepository. Secret type depends on the source kind: kubernetes.io/dockerconfigjson for OCI (oci://...); Opaque with username+password (or bearerToken) for Git over HTTPS; Opaque with identity (PEM private key) and known_hosts for Git over SSH.")
 	flag.StringVar(&cozyValuesSecretName, "cozy-values-secret-name", "cozystack-values", "The name of the secret containing cluster-wide configuration values.")
 	flag.StringVar(&cozyValuesSecretNamespace, "cozy-values-secret-namespace", "cozy-system", "The namespace of the secret containing cluster-wide configuration values.")
 	flag.StringVar(&cozyValuesNamespaceSelector, "cozy-values-namespace-selector", "cozystack.io/system=true", "The label selector for namespaces where the cluster-wide configuration values must be replicated.")
@@ -256,12 +259,12 @@ func main() {
 
 	// Generate and install platform source resource if specified
 	if platformSourceURL != "" {
-		setupLog.Info("Generating platform source resource", "url", platformSourceURL, "name", platformSourceName, "ref", platformSourceRef)
+		setupLog.Info("Generating platform source resource", "url", platformSourceURL, "name", platformSourceName, "ref", platformSourceRef, "secret", platformSourceSecret)
 		installCtx, installCancel := context.WithTimeout(mgrCtx, 2*time.Minute)
 		defer installCancel()
 
 		// Use direct client for pre-start operations (cache is not ready yet)
-		if err := installPlatformSourceResource(installCtx, directClient, platformSourceURL, platformSourceName, platformSourceRef); err != nil {
+		if err := installPlatformSourceResource(installCtx, directClient, platformSourceURL, platformSourceName, platformSourceRef, platformSourceSecret); err != nil {
 			setupLog.Error(err, "failed to install platform source resource")
 			os.Exit(1)
 		} else {
@@ -375,7 +378,7 @@ func main() {
 
 // installPlatformSourceResource generates and installs a Flux source resource (OCIRepository or GitRepository)
 // based on the platform source URL
-func installPlatformSourceResource(ctx context.Context, k8sClient client.Client, sourceURL, resourceName, refSpec string) error {
+func installPlatformSourceResource(ctx context.Context, k8sClient client.Client, sourceURL, resourceName, refSpec, secretName string) error {
 	logger := log.FromContext(ctx)
 
 	// Parse the source URL to determine type
@@ -393,12 +396,12 @@ func installPlatformSourceResource(ctx context.Context, k8sClient client.Client,
 	var obj client.Object
 	switch sourceType {
 	case "oci":
-		obj, err = generateOCIRepository(resourceName, repoURL, refMap)
+		obj, err = generateOCIRepository(resourceName, repoURL, refMap, secretName)
 		if err != nil {
 			return fmt.Errorf("failed to generate OCIRepository: %w", err)
 		}
 	case "git":
-		obj, err = generateGitRepository(resourceName, repoURL, refMap)
+		obj, err = generateGitRepository(resourceName, repoURL, refMap, secretName)
 		if err != nil {
 			return fmt.Errorf("failed to generate GitRepository: %w", err)
 		}
@@ -558,7 +561,7 @@ func validateGitRef(refMap map[string]string) error {
 }
 
 // generateOCIRepository creates an OCIRepository resource
-func generateOCIRepository(name, repoURL string, refMap map[string]string) (*sourcev1.OCIRepository, error) {
+func generateOCIRepository(name, repoURL string, refMap map[string]string, secretName string) (*sourcev1.OCIRepository, error) {
 	if err := validateOCIRef(refMap); err != nil {
 		return nil, err
 	}
@@ -588,11 +591,18 @@ func generateOCIRepository(name, repoURL string, refMap map[string]string) (*sou
 		}
 	}
 
+	// Set secretRef if secret name is provided
+	if secretName != "" {
+		obj.Spec.SecretRef = &fluxmeta.LocalObjectReference{
+			Name: secretName,
+		}
+	}
+
 	return obj, nil
 }
 
 // generateGitRepository creates a GitRepository resource
-func generateGitRepository(name, repoURL string, refMap map[string]string) (*sourcev1.GitRepository, error) {
+func generateGitRepository(name, repoURL string, refMap map[string]string, secretName string) (*sourcev1.GitRepository, error) {
 	if err := validateGitRef(refMap); err != nil {
 		return nil, err
 	}
@@ -623,6 +633,13 @@ func generateGitRepository(name, repoURL string, refMap map[string]string) (*sou
 		}
 	}
 
+	// Set secretRef if secret name is provided
+	if secretName != "" {
+		obj.Spec.SecretRef = &fluxmeta.LocalObjectReference{
+			Name: secretName,
+		}
+	}
+
 	return obj, nil
 }
 

cmd/cozystack-operator/main_test.go

@@ -487,7 +487,7 @@ func TestValidateGitRef(t *testing.T) {
 
 func TestGenerateOCIRepository(t *testing.T) {
 	refMap := map[string]string{"tag": "v1.0", "digest": "sha256:abc123"}
-	obj, err := generateOCIRepository("my-repo", "oci://registry.example.com/repo", refMap)
+	obj, err := generateOCIRepository("my-repo", "oci://registry.example.com/repo", refMap, "")
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -510,10 +510,13 @@ func TestGenerateOCIRepository(t *testing.T) {
 	if obj.Spec.Reference.Digest != "sha256:abc123" {
 		t.Errorf("expected digest %q, got %q", "sha256:abc123", obj.Spec.Reference.Digest)
 	}
+	if obj.Spec.SecretRef != nil {
+		t.Errorf("expected SecretRef to be nil when secretName is empty, got %+v", obj.Spec.SecretRef)
+	}
 }
 
 func TestGenerateOCIRepository_NoRef(t *testing.T) {
-	obj, err := generateOCIRepository("my-repo", "oci://registry.example.com/repo", map[string]string{})
+	obj, err := generateOCIRepository("my-repo", "oci://registry.example.com/repo", map[string]string{}, "")
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -523,15 +526,28 @@ func TestGenerateOCIRepository_NoRef(t *testing.T) {
 }
 
 func TestGenerateOCIRepository_InvalidRef(t *testing.T) {
-	_, err := generateOCIRepository("my-repo", "oci://registry.example.com/repo", map[string]string{"branch": "main"})
+	_, err := generateOCIRepository("my-repo", "oci://registry.example.com/repo", map[string]string{"branch": "main"}, "")
 	if err == nil {
 		t.Fatal("expected error for invalid OCI ref key, got nil")
 	}
 }
 
+func TestGenerateOCIRepository_WithSecret(t *testing.T) {
+	obj, err := generateOCIRepository("my-repo", "oci://registry.example.com/repo", map[string]string{"tag": "v1.0"}, "my-registry-creds")
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if obj.Spec.SecretRef == nil {
+		t.Fatal("expected SecretRef to be set when secretName is non-empty")
+	}
+	if obj.Spec.SecretRef.Name != "my-registry-creds" {
+		t.Errorf("expected SecretRef.Name %q, got %q", "my-registry-creds", obj.Spec.SecretRef.Name)
+	}
+}
+
 func TestGenerateGitRepository(t *testing.T) {
 	refMap := map[string]string{"branch": "main", "commit": "abc1234def5678"}
-	obj, err := generateGitRepository("my-repo", "https://github.com/user/repo", refMap)
+	obj, err := generateGitRepository("my-repo", "https://github.com/user/repo", refMap, "")
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -554,10 +570,13 @@ func TestGenerateGitRepository(t *testing.T) {
 	if obj.Spec.Reference.Commit != "abc1234def5678" {
 		t.Errorf("expected commit %q, got %q", "abc1234def5678", obj.Spec.Reference.Commit)
 	}
+	if obj.Spec.SecretRef != nil {
+		t.Errorf("expected SecretRef to be nil when secretName is empty, got %+v", obj.Spec.SecretRef)
+	}
 }
 
 func TestGenerateGitRepository_NoRef(t *testing.T) {
-	obj, err := generateGitRepository("my-repo", "https://github.com/user/repo", map[string]string{})
+	obj, err := generateGitRepository("my-repo", "https://github.com/user/repo", map[string]string{}, "")
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -567,8 +586,21 @@ func TestGenerateGitRepository_NoRef(t *testing.T) {
 }
 
 func TestGenerateGitRepository_InvalidRef(t *testing.T) {
-	_, err := generateGitRepository("my-repo", "https://github.com/user/repo", map[string]string{"digest": "sha256:abc"})
+	_, err := generateGitRepository("my-repo", "https://github.com/user/repo", map[string]string{"digest": "sha256:abc"}, "")
 	if err == nil {
 		t.Fatal("expected error for invalid Git ref key, got nil")
 	}
 }
+
+func TestGenerateGitRepository_WithSecret(t *testing.T) {
+	obj, err := generateGitRepository("my-repo", "https://github.com/user/repo", map[string]string{"branch": "main"}, "my-git-creds")
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if obj.Spec.SecretRef == nil {
+		t.Fatal("expected SecretRef to be set when secretName is non-empty")
+	}
+	if obj.Spec.SecretRef.Name != "my-git-creds" {
+		t.Errorf("expected SecretRef.Name %q, got %q", "my-git-creds", obj.Spec.SecretRef.Name)
+	}
+}

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/cozystack/cozystack-scheduler/pkg/apis v0.1.1
 	github.com/emicklei/dot v1.10.0
 	github.com/fluxcd/helm-controller/api v1.4.3
+	github.com/fluxcd/pkg/apis/meta v1.23.0
 	github.com/fluxcd/source-controller/api v1.7.4
 	github.com/fluxcd/source-watcher/api/v2 v2.0.3
 	github.com/go-logr/logr v1.4.3
@@ -55,7 +56,6 @@ require (
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fluxcd/pkg/apis/acl v0.9.0 // indirect
 	github.com/fluxcd/pkg/apis/kustomize v1.13.0 // indirect
-	github.com/fluxcd/pkg/apis/meta v1.23.0 // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.9.0 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect

hack/buildkitd.toml

@@ -0,0 +1,21 @@
+# BuildKit daemon config for CI builds.
+#
+# Route docker.io base-image pulls through mirror.gcr.io, Google's public
+# pull-through cache for Docker Hub. The Build job authenticates only to the
+# push registry (OCIR), so every base image (golang, alpine, ubuntu, nginx, ...)
+# is otherwise pulled from Docker Hub ANONYMOUSLY and is subject to Docker Hub's
+# anonymous-pull rate limit -- which intermittently fails the build at
+# auth.docker.io/token with the base image left unresolved.
+#
+# mirror.gcr.io needs no auth, carries far higher limits, and transparently
+# falls through to Docker Hub on a cache miss. This config only takes effect on
+# a buildx builder created with `--config hack/buildkitd.toml` (CI wires this via
+# docker/setup-buildx-action `buildkitd-config:`); a plain local `make build` on
+# the default builder is unaffected.
+#
+# FAIL-SAFE: if mirror.gcr.io is unreachable, BuildKit logs "trying next host"
+# and falls back to registry-1.docker.io, so the build still succeeds (verified:
+# a build pointed at a non-resolving mirror fell back to Docker Hub and exited 0).
+# This strictly reduces build failures; it never introduces a new hard dependency.
+[registry."docker.io"]
+  mirrors = ["mirror.gcr.io"]

hack/capture-dataplane.bats

@@ -0,0 +1,224 @@
+#!/usr/bin/env bats
+# -----------------------------------------------------------------------------
+# Unit tests for the pure decision/parsing helpers in
+# hack/e2e-capture-dataplane.sh -- specifically the LoadBalancer-datapath
+# section, which fires for a Service type=LoadBalancer whose external IP is
+# unreachable while its backend stays Ready (the NotReady-pod path never sees
+# this case). Only the pure logic is unit-testable here:
+#
+#   - lb_filter_services      -- keep only LoadBalancer rows with an ingress IP;
+#   - lb_first_ready_endpoint -- pick the first addressed, non-NotReady endpoint;
+#   - lb_announcer_node       -- the speaker node of the most recent announce;
+#   - lb_capture_decision     -- capture only when every probe failed.
+#
+# The kubectl exec / tcpdump capture itself is not unit-testable (it needs a
+# live cluster); these tests pin the derivations that decide WHICH node to
+# capture on and WHETHER to capture at all, fed mock kubectl/log output.
+#
+# Strategy: the script is sourced once with E2E_CAPTURE_DATAPLANE_LIB set, which
+# the script's sourcing guard honours by defining the helpers and returning
+# before it touches $1 or runs any capture -- so no cluster is required and the
+# capture body never executes. Each @test then calls the helpers directly and
+# asserts with `[ ... ]`, matching this repo's plain-shell bats convention (no
+# `run` helper). Mock IPs use the RFC 5737 / RFC 3849 documentation ranges.
+#
+# Title syntax constraints (inherited from cozytest.sh's awk parser):
+#   - Titles delimited by ASCII double quotes; embedded quotes truncate.
+#   - Only [A-Za-z0-9] from the title survives into the function name, so keep
+#     titles distinctive in their alphanumeric run.
+#
+# Run with: hack/cozytest.sh hack/capture-dataplane.bats
+#           (or `bats hack/capture-dataplane.bats` if the bats binary is
+#           installed; cozytest.sh is the CI path.)
+# -----------------------------------------------------------------------------
+
+HACK_DIR="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")" && pwd)"
+SCRIPT="$HACK_DIR/e2e-capture-dataplane.sh"
+
+# Load the pure helpers. The guard returns before the capture body, so this is
+# side-effect-free and needs no cluster.
+E2E_CAPTURE_DATAPLANE_LIB=1
+# shellcheck source=/dev/null
+. "$SCRIPT"
+
+@test "lb_filter_services keeps only LoadBalancer rows that have an ingress IP" {
+  rows="$(printf '%s\n' \
+    'tenant|app|LoadBalancer|192.0.2.50|80|31000|Cluster' \
+    'kube-system|kube-dns|ClusterIP||53||Cluster' \
+    'tenant|pending|LoadBalancer||443|31443|Local' \
+    'tenant|db|LoadBalancer|192.0.2.51|5432|31543|Local')"
+
+  out="$(printf '%s\n' "$rows" | lb_filter_services)"
+
+  [ "$(printf '%s\n' "$out" | grep -c .)" -eq 2 ]
+  printf '%s\n' "$out" | grep -q '^tenant|app|LoadBalancer|192.0.2.50|'
+  printf '%s\n' "$out" | grep -q '^tenant|db|LoadBalancer|192.0.2.51|'
+  ! printf '%s\n' "$out" | grep -q 'kube-dns'
+  ! printf '%s\n' "$out" | grep -q 'pending'
+}
+
+@test "lb_first_ready_endpoint picks the first addressed endpoint that is not NotReady" {
+  eps="$(printf '%s\n' \
+    '||tenant|virt-launcher-x|true' \
+    '192.0.2.60|worker-1|tenant|virt-launcher-x|false' \
+    '192.0.2.61|worker-2|tenant|virt-launcher-y|true' \
+    '192.0.2.62|worker-3|tenant|virt-launcher-z|true')"
+
+  out="$(printf '%s\n' "$eps" | lb_first_ready_endpoint)"
+
+  [ "$out" = "192.0.2.61|worker-2|tenant|virt-launcher-y" ]
+}
+
+@test "lb_first_ready_endpoint treats a blank ready column as ready" {
+  eps="$(printf '%s\n' '192.0.2.70|worker-9|tenant|app-0|')"
+
+  out="$(printf '%s\n' "$eps" | lb_first_ready_endpoint)"
+
+  [ "$out" = "192.0.2.70|worker-9|tenant|app-0" ]
+}
+
+@test "lb_announcer_node returns the speaker node of the most recent announce" {
+  logs="$(printf '%s\t%s\n' \
+    'node-a' '{"event":"serviceAnnounced","ips":["192.0.2.50"],"node":"node-a"}' \
+    'node-b' '{"event":"serviceAnnounced","ips":["192.0.2.50"],"node":"node-b"}')"
+
+  out="$(printf '%s\n' "$logs" | lb_announcer_node '192.0.2.50')"
+
+  [ "$out" = "node-b" ]
+}
+
+@test "lb_announcer_node ignores withdraw lines and announces for other IPs" {
+  logs="$(printf '%s\t%s\n' \
+    'node-x' '{"event":"serviceAnnounced","ips":["198.51.100.9"],"node":"node-x"}' \
+    'node-a' '{"event":"serviceAnnounced","ips":["192.0.2.50"],"node":"node-a"}' \
+    'node-b' '{"event":"serviceWithdrawn","ips":["192.0.2.50"],"node":"node-b"}' \
+    'node-c' '{"event":"serviceAnnounced","ips":["192.0.2.50"],"node":"node-c"}')"
+
+  out="$(printf '%s\n' "$logs" | lb_announcer_node '192.0.2.50')"
+
+  [ "$out" = "node-c" ]
+}
+
+@test "lb_announcer_node emits nothing when the IP was never announced" {
+  logs="$(printf '%s\t%s\n' \
+    'node-a' '{"event":"serviceAnnounced","ips":["198.51.100.9"],"node":"node-a"}')"
+
+  out="$(printf '%s\n' "$logs" | lb_announcer_node '192.0.2.50')"
+
+  [ -z "$out" ]
+}
+
+@test "lb_announcer_node excludes a node whose last own event is a withdraw" {
+  # Case A: node-a announces the IP then withdraws it and never re-announces.
+  # Its last own IP-event is the withdraw, so it is NOT the current announcer and
+  # nothing must be reported. The pre-polish logic kept node-a (a withdraw did
+  # not retract a prior announce), so this pins the failover fix.
+  logs="$(printf '%s\t%s\n' \
+    'node-a' '{"event":"serviceAnnounced","ips":["192.0.2.50"],"node":"node-a"}' \
+    'node-a' '{"event":"serviceWithdrawn","ips":["192.0.2.50"],"node":"node-a"}')"
+
+  out="$(printf '%s\n' "$logs" | lb_announcer_node '192.0.2.50')"
+
+  [ -z "$out" ]
+}
+
+@test "lb_announcer_node ignores a stale owner that appears later in concat order" {
+  # Case B: node-c is the true current owner (announce, never withdrawn). Later
+  # in the input -- speaker logs are concatenated per-pod, NOT globally time-
+  # sorted -- a stale node-a announce+withdraw block appears. node-a's last own
+  # event is a withdraw, so the announcer is node-c, NOT the later-in-input
+  # node-a. The pre-polish logic returned node-a (most recent announce by concat
+  # order), so this pins robustness to concat order.
+  logs="$(printf '%s\t%s\n' \
+    'node-c' '{"event":"serviceAnnounced","ips":["192.0.2.50"],"node":"node-c"}' \
+    'node-a' '{"event":"serviceAnnounced","ips":["192.0.2.50"],"node":"node-a"}' \
+    'node-a' '{"event":"serviceWithdrawn","ips":["192.0.2.50"],"node":"node-a"}')"
+
+  out="$(printf '%s\n' "$logs" | lb_announcer_node '192.0.2.50')"
+
+  [ "$out" = "node-c" ]
+}
+
+@test "lb_announcer_node matches the IP as a whole token not a substring" {
+  # Case C: querying 192.0.2.5 must NOT match a 192.0.2.50 announce (the
+  # pre-polish index()/substring match returned node-d here). The exact 192.0.2.5
+  # announce on node-e is matched and wins.
+  logs="$(printf '%s\t%s\n' \
+    'node-d' '{"event":"serviceAnnounced","ips":["192.0.2.50"],"node":"node-d"}' \
+    'node-e' '{"event":"serviceAnnounced","ips":["192.0.2.5"],"node":"node-e"}')"
+
+  no_substr="$(printf '%s\n' "$logs" | lb_announcer_node '192.0.2.5')"
+  no_match="$(printf '%s\t%s\n' \
+    'node-d' '{"event":"serviceAnnounced","ips":["192.0.2.50"],"node":"node-d"}' \
+    | lb_announcer_node '192.0.2.5')"
+
+  [ "$no_substr" = "node-e" ]
+  [ -z "$no_match" ]
+}
+
+@test "lb_capture_decision returns capture when every probe failed" {
+  [ "$(printf 'fail\nfail\nfail\n' | lb_capture_decision)" = "capture" ]
+  [ "$(printf 'fail\n' | lb_capture_decision)" = "capture" ]
+}
+
+@test "lb_capture_decision returns skip when any probe succeeded" {
+  [ "$(printf 'fail\nok\nfail\n' | lb_capture_decision)" = "skip" ]
+  [ "$(printf 'ok\n' | lb_capture_decision)" = "skip" ]
+}
+
+@test "lb_capture_decision returns skip when no probe ran at all" {
+  [ "$(printf '' | lb_capture_decision)" = "skip" ]
+}
+
+@test "lb_budget_ok yes below the cap and no once it is reached" {
+  [ "$(lb_budget_ok 0 6)" = "yes" ]
+  [ "$(lb_budget_ok 5 6)" = "yes" ]
+  [ "$(lb_budget_ok 6 6)" = "no" ]
+  [ "$(lb_budget_ok 7 6)" = "no" ]
+}
+
+@test "capture budget counts captured LBs so a late broken LB is still captured" {
+  # The MAX_LBS cap must bound LBs actually CAPTURED, not LBs enumerated: reachable
+  # (skipped) LBs must not consume the budget, so a broken LB enumerated after many
+  # reachable ones is still characterised. This mirrors the per-LB loop in
+  # capture_lb_datapath, which only consults lb_budget_ok / increments the counter
+  # on the capture branch -- here driven by a fixed decision sequence so no cluster
+  # is needed. With the pre-polish "increment per enumerated LB" logic and max=2,
+  # the loop would have broken at the 3rd LB and never reached the broken one.
+  max=2
+  captured=0
+  last=""
+  for decision in skip skip skip skip skip capture; do
+    if [ "$decision" != "capture" ]; then
+      last="skip"
+      continue
+    fi
+    if [ "$(lb_budget_ok "$captured" "$max")" != "yes" ]; then
+      last="dropped"
+      continue
+    fi
+    captured=$((captured + 1))
+    last="captured"
+  done
+
+  [ "$captured" -eq 1 ]
+  [ "$last" = "captured" ]
+}
+
+@test "capture budget stops after the cap worth of broken LBs" {
+  # Once MAX_LBS broken LBs are captured, further unreachable LBs are not
+  # characterised (left to the outer wall-clock backstop). max=2, three broken.
+  max=2
+  captured=0
+  dropped=0
+  for decision in capture capture capture; do
+    if [ "$(lb_budget_ok "$captured" "$max")" != "yes" ]; then
+      dropped=$((dropped + 1))
+      continue
+    fi
+    captured=$((captured + 1))
+  done
+
+  [ "$captured" -eq 2 ]
+  [ "$dropped" -eq 1 ]
+}

hack/check-applicationdefinition-crd-ordering.bats

@@ -0,0 +1,62 @@
+#!/usr/bin/env bats
+# Chart-wide invariant for packages/core/platform/sources:
+#
+# Every *-application PackageSource that ships a `system/*-rd` component renders
+# an ApplicationDefinition (cozystack.io/v1alpha1). That CRD is installed by the
+# application-definition-crd component of cozystack.cozystack-engine. Without an
+# explicit `dependsOn: cozystack.cozystack-engine`, a fresh install where Flux
+# happens to reconcile the *-application source before cozystack-engine fails to
+# render with `no matches for kind "ApplicationDefinition" in version
+# "cozystack.io/v1alpha1"`, which cascades into a broad install timeout.
+#
+# This invariant is generic: any future *-application source that adds a
+# system/*-rd component but forgets the cozystack-engine edge fails here, even
+# though the individual source files are not enumerated below.
+#
+# Requires: yq (mikefarah v4+), jq. Both are available on the project's CI
+# runners and on the maintainer workstation.
+
+REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)"
+SOURCES_DIR="$REPO_ROOT/packages/core/platform/sources"
+
+@test "every PackageSource with a system/*-rd component dependsOn cozystack.cozystack-engine" {
+  command -v yq >/dev/null || { echo "yq (mikefarah v4+) is required" >&2; exit 1; }
+  command -v jq >/dev/null || { echo "jq is required" >&2; exit 1; }
+
+  # yq streams one JSON object per source document; jq -s slurps the stream so
+  # every PackageSource variant can be evaluated as a single collection.
+  json="$(yq --output-format=json eval-all '.' "$SOURCES_DIR"/*.yaml)"
+
+  # A "-rd variant" is any variant whose components include a system/*-rd path.
+  rd_total="$(printf '%s' "$json" | jq -s '
+    [ .[]
+      | select(.kind == "PackageSource")
+      | (.spec.variants // [])[]
+      | select((.components // []) | any(.path // "" | test("^system/[a-z0-9-]+-rd$")))
+    ] | length
+  ')"
+
+  # Guard against a vacuous pass (empty glob / yq breakage): the platform ships
+  # many -rd sources, so discovering zero means this test itself is broken.
+  [ "$rd_total" -ge 1 ] || { echo "Discovered zero system/*-rd variants under $SOURCES_DIR - test is misconfigured" >&2; exit 1; }
+
+  offenders="$(printf '%s' "$json" | jq -rs '
+    [ .[]
+      | select(.kind == "PackageSource")
+      | .metadata.name as $name
+      | (.spec.variants // [])[]
+      | select((.components // []) | any(.path // "" | test("^system/[a-z0-9-]+-rd$")))
+      | select(((.dependsOn // []) | any(. == "cozystack.cozystack-engine")) | not)
+      | "\($name) [variant: \(.name)]"
+    ] | .[]
+  ')"
+
+  if [ -n "$offenders" ]; then
+    echo "PackageSources with a system/*-rd component but missing the cozystack.cozystack-engine dependsOn edge:" >&2
+    printf '%s\n' "$offenders" >&2
+    echo "Fix: add '- cozystack.cozystack-engine' to each listed variant's dependsOn (mirror sources/gateway-application.yaml)." >&2
+    exit 1
+  fi
+
+  echo "Invariant holds: all $rd_total system/*-rd PackageSource variant(s) dependsOn cozystack.cozystack-engine"
+}

hack/cozystack-version-stamp.bats

@@ -0,0 +1,143 @@
+#!/usr/bin/env bats
+# -----------------------------------------------------------------------------
+# Unit tests for
+# packages/core/platform/images/migrations/migrations/lib/cozystack-version.sh
+#
+# render_cozystack_version_manifest <version> emits the cozystack-version
+# ConfigMap manifest on stdout, always carrying the
+# platform.cozystack.io/no-delete=true label that the
+# cozystack-no-delete-guardrail ValidatingAdmissionPolicy keys on. Migrations
+# apply it under the default kubectl field manager; a label-less apply by that
+# same manager would strip the label, so the manifest is centralized here and
+# every migration sources it instead of copy-pasting a heredoc.
+#
+# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its
+# own line; there is no bats `run` or `$status`. Assertions are expressed as
+# direct shell tests that exit non-zero on failure. Each test runs in its own
+# subshell, so NAMESPACE set/unset in one test does not leak into another.
+#
+# Run with: hack/cozytest.sh hack/cozystack-version-stamp.bats
+# -----------------------------------------------------------------------------
+
+@test "renders parseable YAML for a version" {
+    . packages/core/platform/images/migrations/migrations/lib/cozystack-version.sh
+    render_cozystack_version_manifest 45 | yq . >/dev/null
+}
+
+@test "carries the no-delete label set to true" {
+    . packages/core/platform/images/migrations/migrations/lib/cozystack-version.sh
+    val=$(render_cozystack_version_manifest 45 \
+      | yq -r '.metadata.labels."platform.cozystack.io/no-delete"')
+    [ "$val" = "true" ]
+}
+
+@test "names the ConfigMap cozystack-version" {
+    . packages/core/platform/images/migrations/migrations/lib/cozystack-version.sh
+    name=$(render_cozystack_version_manifest 45 | yq -r '.metadata.name')
+    [ "$name" = "cozystack-version" ]
+}
+
+@test "stamps the requested version as a quoted string" {
+    # The version MUST render as a quoted YAML string; an unquoted numeric
+    # scalar (version: 45) would change the ConfigMap data type the rest of
+    # the platform reads as a string.
+    . packages/core/platform/images/migrations/migrations/lib/cozystack-version.sh
+    out=$(render_cozystack_version_manifest 45)
+    printf '%s\n' "$out" | grep -qx '  version: "45"'
+    v=$(printf '%s\n' "$out" | yq -r '.data.version')
+    [ "$v" = "45" ]
+}
+
+@test "defaults namespace to cozy-system when NAMESPACE is unset" {
+    unset NAMESPACE
+    . packages/core/platform/images/migrations/migrations/lib/cozystack-version.sh
+    ns=$(render_cozystack_version_manifest 45 | yq -r '.metadata.namespace')
+    [ "$ns" = "cozy-system" ]
+}
+
+@test "honors a NAMESPACE override" {
+    . packages/core/platform/images/migrations/migrations/lib/cozystack-version.sh
+    out=$(export NAMESPACE=cozy-other; render_cozystack_version_manifest 45)
+    ns=$(printf '%s\n' "$out" | yq -r '.metadata.namespace')
+    [ "$ns" = "cozy-other" ]
+}
+
+@test "requires a version argument" {
+    . packages/core/platform/images/migrations/migrations/lib/cozystack-version.sh
+    if ( render_cozystack_version_manifest ) 2>/dev/null; then
+        echo "expected non-zero exit when version arg is missing" >&2
+        exit 1
+    fi
+}
+
+@test "stamp_cozystack_version requires a version argument" {
+    # The guard fires before the kubectl pipe, so this needs no cluster: a
+    # missing version must abort rather than apply empty input to kubectl.
+    . packages/core/platform/images/migrations/migrations/lib/cozystack-version.sh
+    if ( stamp_cozystack_version ) 2>/dev/null; then
+        echo "expected non-zero exit when version arg is missing" >&2
+        exit 1
+    fi
+}
+
+@test "render output matches the canonical labeled manifest" {
+    # Golden test: pins the exact bytes migrations 42/43/44 apply today, so the
+    # refactor that routes them through this helper is provably manifest-stable.
+    unset NAMESPACE
+    . packages/core/platform/images/migrations/migrations/lib/cozystack-version.sh
+    expected=$(cat <<'EOF'
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cozystack-version
+  namespace: cozy-system
+  labels:
+    platform.cozystack.io/no-delete: "true"
+data:
+  version: "45"
+EOF
+)
+    actual=$(render_cozystack_version_manifest 45)
+    if [ "$actual" != "$expected" ]; then
+        printf 'expected:\n%s\n---\nactual:\n%s\n' "$expected" "$actual" >&2
+        exit 1
+    fi
+}
+
+@test "no go-forward stamp bypasses the helper (migrations >= 42 and run-migrations.sh)" {
+    # The helper exists so go-forward stamps cannot drift back to a label-less
+    # apply that drops the no-delete label. Migrations 1-41 are frozen and
+    # backfilled by 42, so they are exempt; migration 42 onward and the
+    # bootstrap path must each source the helper, call its API, and never stamp
+    # cozystack-version directly with kubectl. Blocking only the historical
+    # 'kubectl create configmap' shape is too weak — an inline 'kubectl apply'
+    # of a hand-rolled label-less manifest would slip past it.
+    mig_dir=packages/core/platform/images/migrations/migrations
+    files=""
+    for f in "$mig_dir"/*; do
+        n=$(basename "$f")
+        case "$n" in *[!0-9]*) continue ;; esac   # skip lib/ and other non-numeric
+        [ "$n" -ge 42 ] || continue
+        files="$files $f"
+    done
+    files="$files packages/core/platform/images/migrations/run-migrations.sh"
+
+    for f in $files; do
+        if ! grep -Eq 'cozystack-version\.sh' "$f"; then
+            echo "$f does not source the cozystack-version helper" >&2
+            exit 1
+        fi
+        if ! grep -Eq '(render_cozystack_version_manifest|stamp_cozystack_version)' "$f"; then
+            echo "$f does not call the cozystack-version helper API" >&2
+            exit 1
+        fi
+        # Strip full-comment lines first: a comment that explains why a stamp
+        # routes through the helper must not read as a direct stamp itself. A
+        # real bypass command never starts with '#', so the guard keeps its
+        # strength against actual direct stamping.
+        if grep -v '^[[:space:]]*#' "$f" | grep -Eq 'kubectl[[:space:]].*(create[[:space:]]+configmap|apply).*cozystack-version'; then
+            echo "$f stamps cozystack-version directly instead of via the helper" >&2
+            exit 1
+        fi
+    done
+}

hack/cozytest.sh

@@ -95,17 +95,33 @@ TMP_SH=$(mktemp) || { echo "Failed to create temp file" >&2; exit 1; }
 COZY_REPORT_DIR="${COZY_REPORT_DIR:-_out/cozyreport}"
 _cozy_on_exit() {
   _rc=$?
-  if [ "$_rc" -ne 0 ] && command -v crust-gather >/dev/null 2>&1; then
+  if [ "$_rc" -ne 0 ]; then
     _snap="$COZY_REPORT_DIR/snapshots/$(basename "$TEST_FILE" .bats)"
     mkdir -p "$_snap" 2>/dev/null || true
-    echo "» capturing crust-gather snapshot of failed $(basename "$TEST_FILE") -> $_snap"
-    # Bound with a timeout: crust-gather collect has hung indefinitely on a
-    # contended/degraded cluster (e.g. streaming logs from a crashlooping pod),
-    # wedging the whole test step for hours until the job-level cancel. 5 min is
-    # ample for a host snapshot; a partial capture (timeout exits 124, swallowed
-    # by `|| true`) still beats a multi-hour hang. -k 30 hard-kills if a blocked
-    # collect ignores the SIGTERM.
-    timeout -k 30 300 crust-gather collect --exclude-kind Secret -f "$_snap/host" >/dev/null 2>&1 || true
+    if command -v crust-gather >/dev/null 2>&1; then
+      echo "» capturing crust-gather snapshot of failed $(basename "$TEST_FILE") -> $_snap"
+      # Bound with a timeout: crust-gather collect has hung indefinitely on a
+      # contended/degraded cluster (e.g. streaming logs from a crashlooping pod),
+      # wedging the whole test step for hours until the job-level cancel. 5 min is
+      # ample for a host snapshot; a partial capture (timeout exits 124, swallowed
+      # by `|| true`) still beats a multi-hour hang. -k 30 hard-kills if a blocked
+      # collect ignores the SIGTERM.
+      timeout -k 30 300 crust-gather collect --exclude-kind Secret -f "$_snap/host" >/dev/null 2>&1 || true
+    fi
+    # Diagnostic-only: capture the host->pod CNI data-plane state for any
+    # NotReady pod so the recurrent host->local-pod "connection refused"
+    # transient (rooted in our cilium+kube-ovn chaining:
+    # enable-host-legacy-routing + CNI InstallEndpointRoute:false, which
+    # delegates host->local-pod routing to kube-ovn/ovn0) can be root-caused
+    # from the uploaded artifact. crust-gather captures object state but not the
+    # node's L3 forwarding state. This NEVER affects the test outcome: every
+    # capture is time-boxed and `|| true`, and the whole run is wrapped in a
+    # wall-clock backstop so it cannot stall the job. It no-ops when there are no
+    # affected pods or when kubectl/the tooling is absent.
+    if command -v kubectl >/dev/null 2>&1; then
+      echo "» capturing host->pod data-plane for NotReady pods -> $_snap/dataplane"
+      timeout -k 30 600 "$(dirname "$0")/e2e-capture-dataplane.sh" "$_snap/dataplane" 2>&1 || true
+    fi
   fi
   if command -v cozy_cleanup >/dev/null 2>&1; then
     echo "» cozy_cleanup $(basename "$TEST_FILE" .bats)"

hack/e2e-apps/foundationdb.bats

@@ -91,15 +91,27 @@ EOF
   # Validate status.desiredProcessGroups field
   timeout 60 sh -ec "until kubectl -n tenant-test get foundationdbclusters.apps.foundationdb.org foundationdb-$name -o jsonpath='{.status.desiredProcessGroups}' | grep -q '^[0-9][0-9]*$'; do sleep 10; done"
 
-  # Validate status.generations.reconciled field
-  timeout 60 sh -ec "until kubectl -n tenant-test get foundationdbclusters.apps.foundationdb.org foundationdb-$name -o jsonpath='{.status.generations.reconciled}' | grep -q '^[0-9][0-9]*$'; do sleep 10; done"
+  # Validate status.generations.reconciled field.
+  # The operator only stamps generations.reconciled after every process has
+  # held its role for minimumUptimeSecondsForBounce (60s, see
+  # packages/apps/foundationdb/templates/cluster.yaml), so this is structurally
+  # the last status field to converge. It needs at least the same budget as the
+  # earlier health.available gate (300s), not the short 60s ceiling.
+  timeout 300 sh -ec "until kubectl -n tenant-test get foundationdbclusters.apps.foundationdb.org foundationdb-$name -o jsonpath='{.status.generations.reconciled}' | grep -q '^[0-9][0-9]*$'; do sleep 10; done"
 
-  # Validate status.hasListenIPsForAllPods field
-  timeout 60 sh -ec "until kubectl -n tenant-test get foundationdbclusters.apps.foundationdb.org foundationdb-$name -o jsonpath='{.status.hasListenIPsForAllPods}' | grep -q 'true'; do sleep 10; done"
+  # Validate status.hasListenIPsForAllPods field. A bounce restarts fdbserver
+  # in-place, so pods and their listen IPs do not churn and this usually settles
+  # before the reconciled gate. Widened to 300s for a conservative ceiling
+  # consistent with the surrounding gates; the 300s is headroom, not because
+  # this field is as late-converging as generations.reconciled.
+  timeout 300 sh -ec "until kubectl -n tenant-test get foundationdbclusters.apps.foundationdb.org foundationdb-$name -o jsonpath='{.status.hasListenIPsForAllPods}' | grep -q 'true'; do sleep 10; done"
 
-  # Validate comprehensive status.health fields
-  timeout 60 sh -ec "until kubectl -n tenant-test get foundationdbclusters.apps.foundationdb.org foundationdb-$name -o jsonpath='{.status.health.fullReplication}' | grep -q 'true'; do sleep 10; done"
-  timeout 60 sh -ec "until kubectl -n tenant-test get foundationdbclusters.apps.foundationdb.org foundationdb-$name -o jsonpath='{.status.health.healthy}' | grep -q 'true'; do sleep 10; done"
+  # Validate comprehensive status.health fields. fullReplication and healthy
+  # settle only once the cluster is fully reconciled (after the post-bounce
+  # uptime gate), so they share the same 300s budget rather than the short 60s
+  # ceiling.
+  timeout 300 sh -ec "until kubectl -n tenant-test get foundationdbclusters.apps.foundationdb.org foundationdb-$name -o jsonpath='{.status.health.fullReplication}' | grep -q 'true'; do sleep 10; done"
+  timeout 300 sh -ec "until kubectl -n tenant-test get foundationdbclusters.apps.foundationdb.org foundationdb-$name -o jsonpath='{.status.health.healthy}' | grep -q 'true'; do sleep 10; done"
 
   # Verify security context is applied correctly (non-root user)
   storage_pod=$(kubectl -n tenant-test get pods -l foundationdb.org/fdb-cluster-name=foundationdb-$name,foundationdb.org/fdb-process-class=storage --no-headers | head -n1 | awk '{print $1}')

hack/e2e-apps/kafka.bats

@@ -17,13 +17,13 @@ spec:
     replicas: 2
     storageClass: ""
     resources: {}
-    resourcesPreset: "nano"
+    resourcesPreset: "c1.small"
   zookeeper:
     size: 5Gi
     replicas: 2
     storageClass: ""
     resources:
-    resourcesPreset: "nano"
+    resourcesPreset: "c1.small"
   topics:
     - name: testResults
       partitions: 1

hack/e2e-capture-dataplane.sh

@@ -0,0 +1,739 @@
+#!/bin/sh
+# e2e-capture-dataplane.sh - capture host->pod data-plane state for NotReady pods.
+#
+# DIAGNOSTIC ONLY. This script collects extra evidence on an already-failed
+# test; it never mutates the cluster and never changes the test's pass/fail
+# outcome. cozytest.sh invokes it from its on-failure hook, AFTER the
+# crust-gather snapshot and only when a test has already failed, writing into
+# the same snapshot dir so the output lands in the uploaded cozyreport artifact.
+#
+# Why this exists: a recurrent install failure is a CNI host->local-pod
+# data-plane transient -- kubelet on a node reaches a *local* pod's
+# readiness/startup probe with "connection refused" for several minutes while
+# overlay (pod->pod) traffic works, then it self-heals. This is rooted in the
+# cozystack cilium+kube-ovn chaining config (forced enable-host-legacy-routing,
+# CNI InstallEndpointRoute:false -> host->local-pod routing is delegated to
+# kube-ovn/ovn0). The standard crust-gather snapshot captures Kubernetes object
+# state but NOT the L3 forwarding state on the node, so the mechanism cannot be
+# root-caused after the fact. This collects exactly that state from the pod's
+# node so the next recurrence is dispositive.
+#
+# What it captures, per affected pod (NotReady, scheduled, has a podIP):
+#   - cilium-agent on the node:  cilium-dbg endpoint list; bpf ct entries for
+#     the podIP; a short bounded `cilium-dbg monitor --type drop`; hubble
+#     dropped-verdict observations (if the hubble CLI is present in the agent).
+#     NOTE: under enable-host-legacy-routing the host->local-pod path traverses
+#     the KERNEL netfilter stack, not cilium BPF, so the cilium CT/monitor view
+#     is complementary -- the authoritative conntrack table for this path is the
+#     kernel one captured below. Both are kept: cilium's view still covers
+#     pod->pod and confirms the path is NOT in BPF, which is itself evidence.
+#   - host netns on the node (via the kube-ovn cni-server, which is
+#     hostNetwork+NET_ADMIN): `ip route get <podIP>`, `ip neigh`, `ip rule`,
+#     `ip addr show ovn0`, and the KERNEL conntrack entries for the podIP
+#     (`conntrack -L`, falling back to /proc/net/nf_conntrack). These reveal a
+#     kernel-path misforward / missing route / wrong rule / unresolved neighbor
+#     -- the actual mechanism behind host->local-pod "connection refused" on the
+#     legacy-routing path (the crux of the transient).
+#   - OVS/OVN on the node: `ovs-ofctl dump-flows br-int`, and the OVN
+#     Port_Binding / Logical_Switch_Port for the pod (LSP name = <pod>.<ns> in
+#     kube-ovn) plus a bounded `ovn-sbctl lflow-list`.
+#   - OVN flow-programming timing on the node (ovs-ovn pod, app=ovs -- its
+#     openvswitch container also runs ovn-controller): the tail of
+#     /var/log/ovn/ovn-controller.log plus a grep for the decisive lines
+#     (physical_flow_output, if_status_mgr, "took <N>ms", recompute,
+#     "Unreasonably long ... poll interval"); the per-interface ovn-installed
+#     flag + ovn-installed-ts on the pod's OVS interface; and the ovs-ovn
+#     container's cgroup cpu.stat (nr_throttled / throttled_usec). Why: the
+#     host->local-pod transient is consistent with OVN incremental-processing
+#     lag -- ovn-installed=true (the kubelet/CNI "port ready" barrier) is set by
+#     if_status_mgr BEFORE physical_flow_output installs the LSP's local-delivery
+#     OpenFlow under burst, aggravated when ovn-controller is CPU-throttled (it
+#     shares the ovs-ovn CPU limit with vswitchd). A `physical_flow_output ...
+#     took <N>ms` line spanning the failure window, an ovn-installed-ts that
+#     predates that flow, and non-zero cpu.stat throttling together make a
+#     recurrence dispositive.
+#
+# What it ALSO captures (added section) -- an UNREACHABLE LoadBalancer datapath:
+#   The NotReady-pod trigger above misses a second, equally-recurrent flake. A
+#   tenant `Service type=LoadBalancer` IP goes unreachable while its backend
+#   pod/VMI stays Ready+Running -- so there is NO NotReady pod, the capture above
+#   no-ops, and the host->cross-node->bridged-worker-VMI->tenant-nodePort
+#   datapath (the suspected failing path) is never characterised. This section
+#   closes that gap. It is gated by a LIVE reachability probe so it only does the
+#   heavy capture for an LB that is actually broken; reachable LBs are enumerated
+#   and explicitly recorded as "reachable, skipped".
+#     - Enumerate every Service type=LoadBalancer that has a status ingress IP.
+#       For each, derive: the EndpointSlice backend (endpoint IP + node +
+#       targetPort), the Service nodePort + externalTrafficPolicy, and the
+#       ANNOUNCER node from the MetalLB speaker logs (the hostNetwork speaker is
+#       the L2 owner; the node of its most recent serviceAnnounced for the IP is
+#       the announcer).
+#     - Probe the LB IP:port a few times from a host netns (the kube-ovn
+#       cni-server, so the probe traverses the same host-sourced datapath the
+#       flake breaks). Only an LB whose every probe FAILS gets the heavy capture
+#       below, and that same probe is the traffic the tcpdumps observe.
+#     - For a FAILING LB, on BOTH the announcer node and the endpoint node
+#       (reusing pod_on_node), labelled by node + role (ANNOUNCER vs ENDPOINT):
+#         * `cilium-dbg bpf lb list` grepped for the LB IP + nodePort -- did the
+#           HOST cilium program the LB->backend translation? This is the first
+#           fork: host-cilium-not-programming vs kube-ovn-delivery-failure.
+#         * kernel `conntrack -L -d <LB IP>` + `ip neigh` (LB IP, endpoint IP)
+#           via the cni-server host netns.
+#         * `ovs-ofctl dump-flows br-int` filtered to the nodePort / endpoint IP.
+#         * a bounded `tcpdump` on the announcer node's geneve tunnel iface and
+#           the endpoint node's backend OVS/tap iface, run WHILE the probe is
+#           replayed, to show whether the packet crosses announcer->endpoint and
+#           reaches the backend's host-side interface.
+#         * the ovn-controller.log decisive lines for the window.
+#     In-guest capture INSIDE the worker VMI (tenant `cilium-dbg bpf lb list` /
+#     `tcpdump eth0`) is the one hop this cannot reach from the host; it is left
+#     as a documented stretch at the call site (it needs a tenant
+#     kubeconfig/virtctl this EXIT-trap diagnostic does not have). The host-side
+#     ANNOUNCER/ENDPOINT split is the deliverable and already localises the
+#     failing hop to host-cilium vs kube-ovn delivery.
+#
+# Robustness contract (matches docs/agents/e2e-testing.md): pure diagnostics,
+# no retries, no behavior change, no traps. Every live capture is time-boxed
+# and every command is `|| true`, so a missing tool, an absent pod, or a hung
+# exec can never fail or stall the job. A wall-clock backstop wraps the whole
+# run at the call site. It no-ops cleanly when there are no affected pods.
+set -u
+
+# --------------------------------------------------------------------------- #
+# Pure, side-effect-free helpers for the LoadBalancer-datapath section below.  #
+# Each takes text on stdin / in args and emits text -- no kubectl, no globals  #
+# -- so hack/capture-dataplane.bats can source this file (with                 #
+# E2E_CAPTURE_DATAPLANE_LIB set, see the guard below) and unit-test the        #
+# enumeration parsing, announcer-node detection, and capture-or-skip decision  #
+# against mock input without a cluster. Keep them above the guard and free of  #
+# any runtime state.                                                           #
+# --------------------------------------------------------------------------- #
+
+# lb_filter_services: stdin = `ns|name|type|lbip|port|nodePort|extPolicy` rows
+# (one Service per line, as emitted by the kubectl jsonpath in main). Emits only
+# the rows that are type=LoadBalancer AND carry a status ingress IP -- i.e. the
+# Services that actually have an external datapath to characterise.
+lb_filter_services() {
+  awk -F'|' '$3 == "LoadBalancer" && $4 != "" { print }'
+}
+
+# lb_first_ready_endpoint: stdin = `ip|node|targetNs|targetName|ready` rows (one
+# EndpointSlice endpoint per line). Emits `ip|node|targetNs|targetName` for the
+# first endpoint that has an address and is not explicitly NotReady, then stops.
+# A blank `ready` (slice without conditions) counts as ready; only "false" is
+# excluded. This is the backend the LB IP is supposed to reach.
+lb_first_ready_endpoint() {
+  awk -F'|' '$1 != "" && $5 != "false" { print $1 "|" $2 "|" $3 "|" $4; exit }'
+}
+
+# lb_announcer_node <lbip>: stdin = MetalLB speaker logs, each line prefixed with
+# the emitting speaker pod's node and a TAB (`<node>\t<logline>`). The speaker is
+# hostNetwork and only the elected L2 owner currently announces the IP. Reports a
+# node ONLY IF that node's LAST own IP-event for this exact IP is an announce: per
+# node we track its most recent announce/withdraw line for the IP, so a node that
+# announced then withdrew (last event = withdraw) is excluded and a node that is
+# still announcing (last event = announce) qualifies. This is robust to the
+# per-pod (not globally time-sorted) concat order of the speaker logs and to L2
+# failover. If more than one node still qualifies (should not happen for an L2
+# announce), the one whose last announce appears latest in the input wins.
+# The IP is matched as a whole token (not a substring), so a query for
+# `192.0.2.5` does NOT match a `192.0.2.50` line. Emits the announcer node, or
+# nothing when no node currently announces the IP.
+lb_announcer_node() {
+  awk -F'\t' -v ip="$1" '
+    BEGIN {
+      # Match the IP as a maximal IP-literal token: not flanked by another
+      # IP char (digit / dot / hex / colon), so `192.0.2.5` != `192.0.2.50`.
+      # `[.]` escapes each dot portably (no gsub-replacement backslash games).
+      ipre = ip
+      gsub(/\./, "[.]", ipre)
+      re = "(^|[^0-9A-Fa-f:.])" ipre "([^0-9A-Fa-f:.]|$)"
+    }
+    $0 ~ re {
+      if      ($0 ~ /[Ww]ithdraw/) ev = "withdraw"
+      else if ($0 ~ /[Aa]nnounc/)  ev = "announce"
+      else                         next
+      last_ev[$1]   = ev
+      last_line[$1] = NR
+    }
+    END {
+      best = ""; bestline = -1
+      for (n in last_ev) {
+        if (last_ev[n] == "announce" && last_line[n] > bestline) {
+          bestline = last_line[n]
+          best = n
+        }
+      }
+      if (best != "") print best
+    }'
+}
+
+# lb_capture_decision: stdin = one probe outcome token per line ("ok" / "fail").
+# Emits the gate decision for the heavy per-node capture:
+#   - "capture" only when at least one probe ran AND every probe failed (the LB
+#     IP is unreachable -- the symptom we want characterised);
+#   - "skip" when any probe succeeded (LB reachable) OR no probe ran at all (no
+#     HTTP/TCP client in the host netns -> cannot conclude unreachable, so only
+#     the cheap metadata is kept, never the heavy capture).
+lb_capture_decision() {
+  awk '
+    { if ($0 == "") next; n++; if ($0 == "ok") ok++ }
+    END {
+      if (n == 0) { print "skip"; exit }
+      if (ok > 0) { print "skip"; exit }
+      print "capture"
+    }'
+}
+
+# lb_budget_ok <captured-so-far> <max>: gate for the heavy per-LB capture budget.
+# Emits "yes" while fewer than <max> LBs have been CAPTURED, "no" once the cap is
+# reached. The cap bounds LBs actually captured -- the caller increments only on
+# the capture branch, so reachable/skipped LBs never consume it and a broken LB
+# enumerated after many reachable ones is still characterised. The cap bounds
+# WORK, not wall-clock: at >=2 unreachable LBs (each heavy capture takes tens of
+# seconds) the real wall-clock bound is the outer `timeout -k 30 600` backstop at
+# the call site, not this cap.
+lb_budget_ok() {
+  if [ "$1" -lt "$2" ]; then echo yes; else echo no; fi
+}
+
+# Sourcing guard: hack/capture-dataplane.bats sets E2E_CAPTURE_DATAPLANE_LIB and
+# sources this file purely to reach the helpers above; return before touching $1
+# or running any capture so the unit test never needs a cluster. The script's
+# only executing caller (the cozytest.sh EXIT trap) never sets this, so the
+# guard is a no-op there.
+if [ -n "${E2E_CAPTURE_DATAPLANE_LIB:-}" ]; then
+  return 0 2>/dev/null
+fi
+
+OUT="${1:?Usage: e2e-capture-dataplane.sh <output-dir>}"
+
+CILIUM_NS="${COZY_CILIUM_NS:-cozy-cilium}"
+KUBEOVN_NS="${COZY_KUBEOVN_NS:-cozy-kubeovn}"
+# Cap how many pods we inspect so a fully-wedged cluster cannot explode the
+# runtime; the per-command timeouts and the call-site wall-clock wrapper are the
+# other two bounds. Node-global captures are deduped per node, so the effective
+# work is closer to (#affected-nodes) than (#affected-pods).
+MAX_PODS="${COZY_DATAPLANE_MAX_PODS:-12}"
+
+# LoadBalancer-datapath section tunables (see the header block and the
+# capture_lb_datapath function near the end of this file). The speaker selector
+# and geneve iface match the cozystack metallb/cilium+kube-ovn defaults but stay
+# overridable so a renamed component cannot silently blank the capture.
+METALLB_NS="${COZY_METALLB_NS:-cozy-metallb}"
+SPEAKER_SELECTOR="${COZY_METALLB_SPEAKER_SELECTOR:-app.kubernetes.io/component=speaker}"
+GENEVE_IFACE="${COZY_GENEVE_IFACE:-genev_sys_6081}"
+# Cap how many UNREACHABLE LBs get the heavy datapath capture; reachable/skipped
+# LBs never count toward it (see the captured-budget gate in capture_lb_datapath).
+# This bounds WORK, not wall-clock: at >=2 unreachable LBs the real wall-clock
+# bound is the outer `timeout -k 30 600` backstop at the call site (truncate +
+# hard-kill), since each heavy capture itself takes tens of seconds.
+MAX_LBS="${COZY_DATAPLANE_MAX_LBS:-6}"
+
+command -v kubectl >/dev/null 2>&1 || exit 0
+mkdir -p "$OUT" 2>/dev/null || exit 0
+
+log() { echo "[capture-dataplane] $*"; }
+
+# Affected = scheduled (has nodeName), has a podIP (so an endpoint exists to
+# inspect), Ready!=True, and not already terminal. That is the superset of the
+# "readiness/startup probe failing with connection refused" symptom -- a pod
+# that is Running with an IP but whose Ready condition is False is exactly the
+# transient's signature. Succeeded/Failed pods (completed Jobs, hook pods) are
+# Ready!=True too but are not the symptom, so they are excluded to keep the
+# MAX_PODS budget on Running/Pending pods that are actually wedged.
+# jsonpath keeps this dependency-free (no jq / go-template reassignment).
+affected=$(kubectl get pods -A \
+  -o jsonpath='{range .items[*]}{.metadata.namespace}{"|"}{.metadata.name}{"|"}{.status.podIP}{"|"}{.spec.nodeName}{"|"}{.status.conditions[?(@.type=="Ready")].status}{"|"}{.status.phase}{"\n"}{end}' \
+  2>/dev/null | awk -F'|' '$3!="" && $4!="" && $5!="True" && $6!="Succeeded" && $6!="Failed"')
+
+if [ -z "$affected" ]; then
+  log "no NotReady pods with a podIP+node -- nothing to capture"
+  exit 0
+fi
+
+ncount=$(printf '%s\n' "$affected" | wc -l | tr -d ' ')
+log "capturing host->pod data-plane for up to $MAX_PODS of $ncount affected pod(s) -> $OUT"
+
+# OVN southbound logical-flow dump is cluster-global, so capture it once rather
+# than per pod. ovn-central is a Deployment (not per-node); any replica answers.
+# --no-leader-only lets a read land on a raft follower instead of erroring.
+central=$(kubectl get pod -n "$KUBEOVN_NS" -l app=ovn-central \
+  -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
+if [ -n "$central" ]; then
+  {
+    echo "=== ovn-sbctl lflow-list (cluster-global, pod=$central) ==="
+    timeout 30 kubectl exec -n "$KUBEOVN_NS" "$central" -c ovn-central -- \
+      ovn-sbctl --no-leader-only lflow-list 2>&1 || true
+  } > "$OUT/ovn-lflows.txt" 2>&1 || true
+else
+  log "no ovn-central pod in $KUBEOVN_NS -- skipping OVN logical-flow dump"
+fi
+
+# pod_on_node <ns> <label> <node> -> first matching pod name (empty if none).
+pod_on_node() {
+  kubectl get pod -n "$1" -l "$2" --field-selector "spec.nodeName=$3" \
+    -o jsonpath='{.items[0].metadata.name}' 2>/dev/null
+}
+
+# Per-node captures are node-global (every pod on a node shares one cilium-agent
+# / ovs / cni-server), so run them once per node and reuse across that node's
+# pods. POSIX-sh membership test over a space-delimited string.
+_SEEN_NODES=" "
+node_seen() { case "$_SEEN_NODES" in *" $1 "*) return 0 ;; esac; return 1; }
+mark_node() { _SEEN_NODES="$_SEEN_NODES$1 "; }
+
+capture_node() {
+  node=$1
+  node_seen "$node" && return 0
+  mark_node "$node"
+  nf="$OUT/node-$node.txt"
+
+  agent=$(pod_on_node "$CILIUM_NS" k8s-app=cilium "$node")
+  ovs=$(pod_on_node "$KUBEOVN_NS" app=ovs "$node")
+  {
+    echo "################################################################"
+    echo "# NODE $node  (cilium-agent=${agent:-<none>} ovs=${ovs:-<none>})"
+    echo "################################################################"
+
+    if [ -n "$agent" ]; then
+      echo
+      echo "=== cilium-dbg endpoint list ==="
+      timeout 25 kubectl exec -n "$CILIUM_NS" "$agent" -c cilium-agent -- \
+        cilium-dbg endpoint list 2>&1 || true
+
+      echo
+      echo "=== cilium-dbg monitor --type drop (bounded ~8s) ==="
+      # Two nested bounds: an inner `timeout 8` so the capture self-terminates
+      # if the agent ships coreutils, and an outer `timeout 12` on the exec as
+      # the hard backstop if it does not. Either way it cannot hang.
+      timeout 12 kubectl exec -n "$CILIUM_NS" "$agent" -c cilium-agent -- \
+        sh -c 'timeout 8 cilium-dbg monitor --type drop 2>&1 || true' 2>&1 || true
+
+      echo
+      echo "=== hubble observe --verdict DROPPED --last 200 (if hubble present) ==="
+      timeout 25 kubectl exec -n "$CILIUM_NS" "$agent" -c cilium-agent -- \
+        sh -c 'command -v hubble >/dev/null 2>&1 && hubble observe --verdict DROPPED --last 200 2>&1 || echo "hubble CLI not present in agent"' 2>&1 || true
+    else
+      echo
+      echo "(no cilium-agent pod found on node $node)"
+    fi
+
+    if [ -n "$ovs" ]; then
+      echo
+      echo "=== ovs-ofctl dump-flows br-int ==="
+      timeout 25 kubectl exec -n "$KUBEOVN_NS" "$ovs" -c openvswitch -- \
+        ovs-ofctl dump-flows br-int 2>&1 || true
+
+      echo
+      echo "=== ovn-controller.log decisive lines (flow-programming / I-P timing) ==="
+      # ovn-controller runs inside this ovs-ovn (app=ovs) pod's openvswitch
+      # container and logs to /var/log/ovn/ovn-controller.log. Grep the lines
+      # that pin OVN incremental-processing lag: physical_flow_output (installs
+      # the LSP local-delivery OpenFlow), if_status_mgr (sets ovn-installed --
+      # the CNI "port ready" barrier), and "took <N>ms" / recompute /
+      # "Unreasonably long ... poll interval" (ovn-controller stalls). A
+      # physical_flow_output "took <N>ms" spanning the failure window is proof.
+      timeout 25 kubectl exec -n "$KUBEOVN_NS" "$ovs" -c openvswitch -- \
+        sh -c 'grep -E "physical_flow_output|if_status_mgr|took [0-9]+ ?ms|recompute|Unreasonably long" /var/log/ovn/ovn-controller.log 2>/dev/null | tail -n 400 || echo "no matching lines in /var/log/ovn/ovn-controller.log"' 2>&1 || true
+
+      echo
+      echo "=== ovn-controller.log tail (bounded) ==="
+      timeout 20 kubectl exec -n "$KUBEOVN_NS" "$ovs" -c openvswitch -- \
+        sh -c 'tail -n 2000 /var/log/ovn/ovn-controller.log 2>/dev/null || echo "no /var/log/ovn/ovn-controller.log"' 2>&1 || true
+
+      echo
+      echo "=== ovs-ovn cgroup cpu.stat (ovn-controller/vswitchd CPU throttling) ==="
+      # ovs-ovn caps CPU (shared between ovn-controller and vswitchd); non-zero
+      # nr_throttled / throttled_usec means ovn-controller was CPU-starved, which
+      # aggravates the flow-programming lag above. cgroup v2 path first, v1
+      # fallback.
+      timeout 15 kubectl exec -n "$KUBEOVN_NS" "$ovs" -c openvswitch -- \
+        sh -c 'cat /sys/fs/cgroup/cpu.stat 2>/dev/null || cat /sys/fs/cgroup/cpu/cpu.stat 2>/dev/null || echo "no cpu.stat at /sys/fs/cgroup/cpu.stat (v2) or /sys/fs/cgroup/cpu/cpu.stat (v1)"' 2>&1 || true
+    else
+      echo
+      echo "(no ovs pod found on node $node)"
+    fi
+
+    cni=$(pod_on_node "$KUBEOVN_NS" app=kube-ovn-cni "$node")
+    if [ -n "$cni" ]; then
+      echo
+      echo "=== host netns: ip neigh (via kube-ovn cni-server, hostNetwork) ==="
+      timeout 15 kubectl exec -n "$KUBEOVN_NS" "$cni" -c cni-server -- \
+        ip neigh 2>&1 || true
+
+      echo
+      echo "=== host netns: ip rule ==="
+      timeout 15 kubectl exec -n "$KUBEOVN_NS" "$cni" -c cni-server -- \
+        ip rule 2>&1 || true
+
+      echo
+      echo "=== host netns: ip addr show ovn0 ==="
+      timeout 15 kubectl exec -n "$KUBEOVN_NS" "$cni" -c cni-server -- \
+        ip addr show ovn0 2>&1 || true
+    else
+      echo
+      echo "(no kube-ovn-cni pod found on node $node -- host netns capture skipped)"
+    fi
+  } >> "$nf" 2>&1 || true
+}
+
+i=0
+printf '%s\n' "$affected" | {
+  while IFS='|' read -r ns pod podip node _ready _phase; do
+    [ -n "$ns" ] && [ -n "$pod" ] && [ -n "$podip" ] && [ -n "$node" ] || continue
+    i=$((i + 1))
+    if [ "$i" -gt "$MAX_PODS" ]; then
+      log "reached MAX_PODS=$MAX_PODS cap; $((ncount - MAX_PODS)) more affected pod(s) NOT captured"
+      break
+    fi
+
+    # Node-global state (cilium endpoints, monitor, hubble, ovs flows, ip neigh,
+    # ip rule, ovn0 addr) -- captured once per node.
+    capture_node "$node"
+
+    # Pod-specific state.
+    pf="$OUT/pod-$ns-$pod.txt"
+    agent=$(pod_on_node "$CILIUM_NS" k8s-app=cilium "$node")
+    ovs=$(pod_on_node "$KUBEOVN_NS" app=ovs "$node")
+    {
+      echo "################################################################"
+      echo "# POD $ns/$pod  podIP=$podip  node=$node  (Ready=$_ready)"
+      echo "# node-global captures are in node-$node.txt"
+      echo "################################################################"
+
+      echo
+      echo "=== pod Ready conditions + recent probe events ==="
+      kubectl get pod -n "$ns" "$pod" \
+        -o jsonpath='{range .status.conditions[*]}{.type}={.status} reason={.reason}: {.message}{"\n"}{end}' 2>&1 || true
+      kubectl get events -n "$ns" --field-selector "involvedObject.name=$pod" \
+        -o jsonpath='{range .items[*]}{.lastTimestamp}{" "}{.reason}{": "}{.message}{"\n"}{end}' 2>&1 || true
+
+      if [ -n "$agent" ]; then
+        echo
+        echo "=== cilium-dbg bpf ct list global | grep $podip (node=$node agent=$agent) ==="
+        timeout 25 kubectl exec -n "$CILIUM_NS" "$agent" -c cilium-agent -- \
+          sh -c "cilium-dbg bpf ct list global 2>/dev/null | grep -F '$podip' || echo 'no CT entries for $podip'" 2>&1 || true
+      fi
+
+      cni=$(pod_on_node "$KUBEOVN_NS" app=kube-ovn-cni "$node")
+      if [ -n "$cni" ]; then
+        echo
+        echo "=== host netns: ip route get $podip (via kube-ovn cni-server) ==="
+        timeout 15 kubectl exec -n "$KUBEOVN_NS" "$cni" -c cni-server -- \
+          ip route get "$podip" 2>&1 || true
+
+        echo
+        echo "=== host netns: kernel conntrack for $podip ==="
+        # Under enable-host-legacy-routing the host->local-pod path is in the
+        # kernel netfilter conntrack table, NOT cilium BPF -- this is the
+        # authoritative table for the transient. Prefer the conntrack CLI; fall
+        # back to /proc/net/nf_conntrack when it is absent from the image.
+        timeout 15 kubectl exec -n "$KUBEOVN_NS" "$cni" -c cni-server -- \
+          sh -c "if command -v conntrack >/dev/null 2>&1; then conntrack -L 2>/dev/null | grep -F '$podip' || echo 'no conntrack entries for $podip'; else grep -F '$podip' /proc/net/nf_conntrack 2>/dev/null || echo 'no conntrack CLI; no /proc/net/nf_conntrack match for $podip'; fi" 2>&1 || true
+      fi
+
+      if [ -n "$central" ]; then
+        echo
+        echo "=== OVN Port_Binding / Logical_Switch_Port for $pod.$ns ==="
+        # kube-ovn names the OVN logical port <pod>.<namespace>.
+        timeout 20 kubectl exec -n "$KUBEOVN_NS" "$central" -c ovn-central -- \
+          ovn-sbctl --no-leader-only find port_binding "logical_port=$pod.$ns" 2>&1 || true
+        timeout 20 kubectl exec -n "$KUBEOVN_NS" "$central" -c ovn-central -- \
+          ovn-nbctl --no-leader-only find logical_switch_port "name=$pod.$ns" 2>&1 || true
+      fi
+
+      if [ -n "$ovs" ]; then
+        echo
+        echo "=== OVS interface ovn-installed flag + ts for iface-id=$pod.$ns (ovs pod $ovs) ==="
+        # kube-ovn stamps external_ids:ovn-installed (+ -ts) on the pod's OVS
+        # interface once ovn-controller reports the port programmed -- this is
+        # the barrier the CNI waits on. The OVS interface name is opaque, so
+        # look it up by iface-id (<pod>.<ns>), then read the flag + timestamp.
+        # An ovn-installed-ts set early (before physical_flow_output installed
+        # the local-delivery flow, see node-$node.txt) is the I-P-lag signature.
+        ovsif=$(timeout 20 kubectl exec -n "$KUBEOVN_NS" "$ovs" -c openvswitch -- \
+          ovs-vsctl --no-heading --columns=name find interface "external_ids:iface-id=$pod.$ns" 2>/dev/null \
+          | head -n 1 | tr -d '" ')
+        if [ -n "$ovsif" ]; then
+          echo "ovs interface = $ovsif"
+          timeout 20 kubectl exec -n "$KUBEOVN_NS" "$ovs" -c openvswitch -- \
+            ovs-vsctl get interface "$ovsif" external_ids:ovn-installed external_ids:ovn-installed-ts 2>&1 || true
+        else
+          echo "no OVS interface with external_ids:iface-id=$pod.$ns"
+        fi
+      fi
+    } > "$pf" 2>&1 || true
+  done
+  log "host->pod data-plane capture complete"
+}
+
+# ============================================================================ #
+# LoadBalancer-datapath capture (see the header block for the full rationale).  #
+# Independent of the NotReady-pod path above: it fires for a Service            #
+# type=LoadBalancer whose external IP is unreachable even though its backend is #
+# Ready, which the pod path cannot see. Everything below is bounded + best-     #
+# effort and never changes the job outcome.                                     #
+# ============================================================================ #
+
+# host_http_probe <node> <lbip> <port> -- one bounded reachability probe of the
+# LB IP from <node>'s host netns (via the hostNetwork kube-ovn cni-server, so the
+# probe traverses the same host->cross-node->backend datapath the flake breaks).
+# Echoes "ok" / "fail" per the result, or nothing when the cni-server image ships
+# no probe client (the decision helper treats no-attempt as skip). Prefers a pure
+# TCP connect (nc -z) so a non-HTTP backend is not misread as unreachable; the
+# curl/wget fallbacks send HTTP and so fail-toward-capture on a non-HTTP port,
+# the safe direction (capture is cheap; a missed broken LB is not). Doubles as
+# the traffic generator for the tcpdumps.
+host_http_probe() {
+  _hp_node=$1; _hp_ip=$2; _hp_port=$3
+  _hp_cni=$(pod_on_node "$KUBEOVN_NS" app=kube-ovn-cni "$_hp_node")
+  [ -n "$_hp_cni" ] || return 0
+  # The LB IP/port are embedded as inner single-quoted literals (same idiom as
+  # the pod-path captures above) so the inner shell never re-splits them.
+  timeout 12 kubectl exec -n "$KUBEOVN_NS" "$_hp_cni" -c cni-server -- \
+    sh -c "
+      if command -v nc >/dev/null 2>&1; then
+        nc -z -w 5 '$_hp_ip' '$_hp_port' >/dev/null 2>&1 && echo ok || echo fail
+      elif command -v curl >/dev/null 2>&1; then
+        curl -sS -o /dev/null --max-time 6 --connect-timeout 5 'http://$_hp_ip:$_hp_port/' >/dev/null 2>&1 && echo ok || echo fail
+      elif command -v wget >/dev/null 2>&1; then
+        wget -q -T 6 -O /dev/null 'http://$_hp_ip:$_hp_port/' >/dev/null 2>&1 && echo ok || echo fail
+      fi
+    " 2>/dev/null || true
+}
+
+# ovs_iface_for <node> <iface-id> -- the OVS interface name on <node> whose
+# external_ids:iface-id matches (kube-ovn stamps <pod>.<ns>); empty if none. Used
+# to point the endpoint-node tcpdump at the backend's host-side tap.
+ovs_iface_for() {
+  _oi_ovs=$(pod_on_node "$KUBEOVN_NS" app=ovs "$1")
+  [ -n "$_oi_ovs" ] || return 0
+  timeout 20 kubectl exec -n "$KUBEOVN_NS" "$_oi_ovs" -c openvswitch -- \
+    ovs-vsctl --no-heading --columns=name find interface "external_ids:iface-id=$2" 2>/dev/null \
+    | head -n 1 | tr -d '" '
+}
+
+# capture_lb_node <node> <role> <lbip> <nodeport> <endpointip> <outfile>
+# -- the static (non-tcpdump) host-side captures on one node for a failing LB.
+# role is ANNOUNCER or ENDPOINT and is stamped on every block so the two halves
+# of the path are unambiguous in the artifact.
+capture_lb_node() {
+  _n=$1; _role=$2; _lbip=$3; _np=$4; _epip=$5; _of=$6
+  _agent=$(pod_on_node "$CILIUM_NS" k8s-app=cilium "$_n")
+  _ovs=$(pod_on_node "$KUBEOVN_NS" app=ovs "$_n")
+  _cni=$(pod_on_node "$KUBEOVN_NS" app=kube-ovn-cni "$_n")
+  {
+    echo
+    echo "---------------- $_role node=$_n (cilium=${_agent:-<none>} ovs=${_ovs:-<none>} cni=${_cni:-<none>}) ----------------"
+
+    if [ -n "$_agent" ]; then
+      echo
+      echo "=== [$_role $_n] cilium-dbg bpf lb list | grep LB IP / nodePort -- host cilium LB->backend programming ==="
+      # Answers the first fork: if the host cilium has NO LB map entry for the
+      # LB IP or the nodePort, the fault is host-cilium-not-programming; if it
+      # does, suspicion shifts to kube-ovn delivery (the captures below).
+      timeout 25 kubectl exec -n "$CILIUM_NS" "$_agent" -c cilium-agent -- \
+        sh -c "cilium-dbg bpf lb list 2>/dev/null | grep -E '$_lbip|:$_np' || echo 'no bpf lb entry for $_lbip or nodePort $_np'" 2>&1 || true
+    fi
+
+    if [ -n "$_cni" ]; then
+      echo
+      echo "=== [$_role $_n] host netns: kernel conntrack -d $_lbip (via cni-server) ==="
+      timeout 15 kubectl exec -n "$KUBEOVN_NS" "$_cni" -c cni-server -- \
+        sh -c "if command -v conntrack >/dev/null 2>&1; then conntrack -L -d '$_lbip' 2>/dev/null || echo 'no conntrack entries for $_lbip'; else grep -F '$_lbip' /proc/net/nf_conntrack 2>/dev/null || echo 'no conntrack CLI; no /proc/net/nf_conntrack match for $_lbip'; fi" 2>&1 || true
+
+      echo
+      echo "=== [$_role $_n] host netns: ip neigh (LB IP $_lbip, endpoint IP $_epip) ==="
+      timeout 15 kubectl exec -n "$KUBEOVN_NS" "$_cni" -c cni-server -- \
+        sh -c "ip neigh 2>/dev/null | grep -E '$_lbip|$_epip' || echo 'no neigh entry for $_lbip or $_epip'" 2>&1 || true
+    fi
+
+    if [ -n "$_ovs" ]; then
+      echo
+      echo "=== [$_role $_n] ovs-ofctl dump-flows br-int | grep nodePort $_np / endpoint $_epip ==="
+      timeout 25 kubectl exec -n "$KUBEOVN_NS" "$_ovs" -c openvswitch -- \
+        sh -c "ovs-ofctl dump-flows br-int 2>/dev/null | grep -E '$_np|$_epip' || echo 'no br-int flow matching nodePort $_np or endpoint $_epip'" 2>&1 || true
+
+      echo
+      echo "=== [$_role $_n] ovn-controller.log decisive lines ==="
+      timeout 20 kubectl exec -n "$KUBEOVN_NS" "$_ovs" -c openvswitch -- \
+        sh -c 'grep -E "physical_flow_output|if_status_mgr|took [0-9]+ ?ms|recompute|Unreasonably long" /var/log/ovn/ovn-controller.log 2>/dev/null | tail -n 200 || echo "no matching lines in /var/log/ovn/ovn-controller.log"' 2>&1 || true
+    fi
+  } >> "$_of" 2>&1 || true
+}
+
+# capture_lb_datapath: enumerate LBs, gate on a live probe, and characterise the
+# announcer/endpoint datapath for any LB whose every probe fails. Wraps the whole
+# per-LB body in `|| true`-guarded, time-boxed captures so it is always best-
+# effort and self-limiting (MAX_LBS cap + per-command timeouts).
+capture_lb_datapath() {
+  _raw=$(timeout 20 kubectl get svc -A \
+    -o jsonpath='{range .items[*]}{.metadata.namespace}{"|"}{.metadata.name}{"|"}{.spec.type}{"|"}{.status.loadBalancer.ingress[0].ip}{"|"}{.spec.ports[0].port}{"|"}{.spec.ports[0].nodePort}{"|"}{.spec.externalTrafficPolicy}{"\n"}{end}' \
+    2>/dev/null)
+  _lbs=$(printf '%s\n' "$_raw" | lb_filter_services)
+  if [ -z "$_lbs" ]; then
+    log "no Service type=LoadBalancer with an ingress IP -- skipping LB-datapath capture"
+    return 0
+  fi
+
+  _lbcount=$(printf '%s\n' "$_lbs" | wc -l | tr -d ' ')
+  log "probing $_lbcount LoadBalancer service(s); heavy-capturing up to $MAX_LBS unreachable one(s) -> $OUT/lb-*.txt"
+
+  # MetalLB speaker logs once, prefixed with each speaker's node (hostNetwork, so
+  # .spec.nodeName IS the announcing node). Kept as an artifact file that
+  # lb_announcer_node greps per LB. A blank file is fine -- the announcer just
+  # reports <unknown> and the probe falls back to the endpoint node.
+  _speakerlog="$OUT/lb-speaker-logs.txt"
+  : > "$_speakerlog" 2>/dev/null || _speakerlog=""
+  _speakers=$(kubectl get pod -n "$METALLB_NS" -l "$SPEAKER_SELECTOR" \
+    -o jsonpath='{range .items[*]}{.metadata.name}{"|"}{.spec.nodeName}{"\n"}{end}' 2>/dev/null)
+  if [ -n "$_speakers" ] && [ -n "$_speakerlog" ]; then
+    printf '%s\n' "$_speakers" | while IFS='|' read -r _sp _spnode; do
+      [ -n "$_sp" ] && [ -n "$_spnode" ] || continue
+      timeout 15 kubectl logs -n "$METALLB_NS" "$_sp" -c speaker --tail=2000 2>/dev/null \
+        | awk -v n="$_spnode" '{ print n "\t" $0 }' >> "$_speakerlog" 2>/dev/null || true
+    done
+  fi
+
+  # Counts LBs actually CAPTURED (heavy capture performed), not enumerated, so the
+  # MAX_LBS cap only bounds failing LBs -- reachable/skipped ones below never
+  # increment it and a broken LB enumerated after many reachable ones still gets
+  # captured. The gate lives on the capture branch (lb_budget_ok), not here.
+  _captured=0
+  printf '%s\n' "$_lbs" | {
+    while IFS='|' read -r _ns _name _type _lbip _port _np _etp; do
+      [ -n "$_lbip" ] || continue
+      _lbport="${_port:-0}"
+
+      # EndpointSlice backend for this Service (first ready, addressed endpoint).
+      _eps=$(timeout 20 kubectl get endpointslices -n "$_ns" \
+        -l "kubernetes.io/service-name=$_name" \
+        -o jsonpath='{range .items[*]}{range .endpoints[*]}{.addresses[0]}{"|"}{.nodeName}{"|"}{.targetRef.namespace}{"|"}{.targetRef.name}{"|"}{.conditions.ready}{"\n"}{end}{end}' \
+        2>/dev/null)
+      _backend=$(printf '%s\n' "$_eps" | lb_first_ready_endpoint)
+      _epip=$(printf '%s' "$_backend" | cut -d'|' -f1)
+      _epnode=$(printf '%s' "$_backend" | cut -d'|' -f2)
+      _eptns=$(printf '%s' "$_backend" | cut -d'|' -f3)
+      _eptname=$(printf '%s' "$_backend" | cut -d'|' -f4)
+      _eptport=$(timeout 20 kubectl get endpointslices -n "$_ns" \
+        -l "kubernetes.io/service-name=$_name" \
+        -o jsonpath='{.items[0].ports[0].port}' 2>/dev/null)
+
+      # Announcer node = node of the most recent serviceAnnounced for the IP.
+      _annode=$(lb_announcer_node "$_lbip" < "${_speakerlog:-/dev/null}")
+
+      _of="$OUT/lb-$_ns-$_name.txt"
+      {
+        echo "################################################################"
+        echo "# LB $_ns/$_name  ip=$_lbip port=$_lbport nodePort=${_np:-<none>} etp=${_etp:-<default>}"
+        echo "# backend: ip=${_epip:-<none>} node=${_epnode:-<none>} pod=${_eptns:-?}/${_eptname:-?} targetPort=${_eptport:-<none>}"
+        echo "# announcer node: ${_annode:-<unknown>}"
+        echo "################################################################"
+      } > "$_of" 2>&1 || true
+
+      # Gate: probe the LB IP from the announcer node's host netns (fall back to
+      # the endpoint node if the announcer is unknown). Reachable -> record and
+      # skip the heavy capture; unreachable -> characterise both hops.
+      _probenode="${_annode:-$_epnode}"
+      _decision=skip
+      if [ -n "$_probenode" ] && [ "$_lbport" != "0" ]; then
+        _decision=$( { host_http_probe "$_probenode" "$_lbip" "$_lbport"
+                       host_http_probe "$_probenode" "$_lbip" "$_lbport"
+                       host_http_probe "$_probenode" "$_lbip" "$_lbport"; } | lb_capture_decision )
+      fi
+
+      if [ "$_decision" != "capture" ]; then
+        echo "probe: LB reachable or not probeable from ${_probenode:-<none>} -- reachable, skipped (no heavy capture)" >> "$_of" 2>&1 || true
+        log "LB $_ns/$_name ($_lbip) reachable or not probeable -- skipped"
+        continue
+      fi
+
+      # Heavy-capture budget: only failing LBs consume MAX_LBS (reachable ones
+      # skipped above without counting). Stop once the cap's worth of broken LBs
+      # have been captured; remaining ones are left to the outer wall-clock bound.
+      if [ "$(lb_budget_ok "$_captured" "$MAX_LBS")" != "yes" ]; then
+        echo "probe: LB IP unreachable from node $_probenode but MAX_LBS=$MAX_LBS captured cap reached -- NOT characterised" >> "$_of" 2>&1 || true
+        log "reached MAX_LBS=$MAX_LBS captured cap; further unreachable LB(s) NOT characterised"
+        break
+      fi
+      _captured=$((_captured + 1))
+
+      log "LB $_ns/$_name ($_lbip) UNREACHABLE -- capturing announcer/endpoint datapath"
+      {
+        echo
+        echo "probe: LB IP unreachable from node $_probenode (every probe failed) -- heavy capture follows"
+      } >> "$_of" 2>&1 || true
+
+      # Static host-side captures on each hop.
+      if [ -n "$_annode" ]; then
+        capture_lb_node "$_annode" ANNOUNCER "$_lbip" "${_np:-0}" "${_epip:-0.0.0.0}" "$_of"
+      fi
+      if [ -n "$_epnode" ] && [ "$_epnode" != "$_annode" ]; then
+        capture_lb_node "$_epnode" ENDPOINT "$_lbip" "${_np:-0}" "${_epip:-0.0.0.0}" "$_of"
+      elif [ -n "$_epnode" ] && [ "$_epnode" = "$_annode" ]; then
+        echo "(endpoint node == announcer node $_annode -- single-node path, no cross-node hop)" >> "$_of" 2>&1 || true
+      fi
+
+      # Live tcpdump on both hops WHILE the probe is replayed: announcer geneve
+      # tunnel egress + endpoint backend tap ingress. Shows whether the packet
+      # crosses announcer->endpoint and reaches the backend's host-side iface.
+      _an_cni=$(pod_on_node "$KUBEOVN_NS" app=kube-ovn-cni "${_annode:-}")
+      _en_cni=$(pod_on_node "$KUBEOVN_NS" app=kube-ovn-cni "${_epnode:-}")
+      _en_iface="$GENEVE_IFACE"
+      if [ -n "$_eptname" ] && [ -n "$_eptns" ]; then
+        _cand=$(ovs_iface_for "${_epnode:-}" "$_eptname.$_eptns")
+        [ -n "$_cand" ] && _en_iface="$_cand"
+      fi
+
+      _an_pcap="$OUT/lb-$_ns-$_name.tcpdump-announcer.txt"
+      _en_pcap="$OUT/lb-$_ns-$_name.tcpdump-endpoint.txt"
+      _an_td=""
+      _en_td=""
+      if [ -n "$_an_cni" ]; then
+        {
+          echo "# ANNOUNCER tcpdump node=${_annode:-<none>} iface=$GENEVE_IFACE filter='host $_lbip or host ${_epip:-0.0.0.0}'"
+          timeout 15 kubectl exec -n "$KUBEOVN_NS" "$_an_cni" -c cni-server -- \
+            sh -c "timeout 10 tcpdump -n -i '$GENEVE_IFACE' -c 60 'host $_lbip or host ${_epip:-0.0.0.0}' 2>&1 || true"
+        } > "$_an_pcap" 2>&1 &
+        _an_td=$!
+      fi
+      if [ -n "$_en_cni" ]; then
+        {
+          echo "# ENDPOINT tcpdump node=${_epnode:-<none>} iface=$_en_iface filter='host $_lbip or host ${_epip:-0.0.0.0} or port ${_np:-0}'"
+          timeout 15 kubectl exec -n "$KUBEOVN_NS" "$_en_cni" -c cni-server -- \
+            sh -c "timeout 10 tcpdump -n -i '$_en_iface' -c 60 'host $_lbip or host ${_epip:-0.0.0.0} or port ${_np:-0}' 2>&1 || true"
+        } > "$_en_pcap" 2>&1 &
+        _en_td=$!
+      fi
+
+      # Give the captures a moment to attach, then drive the probe traffic and
+      # collect. Bounded by the inner tcpdump timeout/-c and the exec timeout.
+      if [ -n "$_an_td" ] || [ -n "$_en_td" ]; then
+        sleep 2
+        host_http_probe "$_probenode" "$_lbip" "$_lbport" >/dev/null 2>&1 || true
+        host_http_probe "$_probenode" "$_lbip" "$_lbport" >/dev/null 2>&1 || true
+        [ -n "$_an_td" ] && { wait "$_an_td" 2>/dev/null || true; }
+        [ -n "$_en_td" ] && { wait "$_en_td" 2>/dev/null || true; }
+      fi
+
+      # STRETCH / TODO (deferred -- needs tenant-cluster access, not host-side):
+      #   The one hop this cannot reach is INSIDE the worker VMI. To finish the
+      #   chain, exec/console into the tenant cluster (virtctl console / SSH to
+      #   the worker VMI) and capture, in-guest:
+      #     - `cilium-dbg bpf lb list | grep <tenant-nodePort>` -- did the TENANT
+      #       cilium program the nodePort->pod translation? (separates a
+      #       tenant-side miss from a host-side delivery miss);
+      #     - `tcpdump -ni eth0 port <tenant-nodePort>` -- did the packet even
+      #       arrive on the VMI's NIC?
+      #   Deferred because it needs a tenant kubeconfig/virtctl that this EXIT-
+      #   trap diagnostic does not have; the host-side ANNOUNCER/ENDPOINT split
+      #   already localises the failing hop to host-cilium vs kube-ovn delivery.
+    done
+    log "LoadBalancer-datapath capture complete"
+  }
+}
+
+capture_lb_datapath
+
+exit 0

hack/e2e-cilium-endpoint-leak-healer.sh

@@ -23,19 +23,36 @@
 # upstream at cilium/cilium#38313 (closed stale); no released or pre-release
 # Cilium fixes the leak and no flag disables it.
 #
-# This watchdog applies the least-disruptive known remedy: instead of
-# restarting the agent (which drops the whole node's in-memory registry), it
-# evicts only the orphaned endpoint via
-#   cilium-dbg endpoint disconnect ipv4:<X>
-# which maps to DELETE /endpoint/{id} -> RemoveEndpoint -> unexpose ->
-# removeReferencesLocked, clearing the stale IP-keyed entry. The wedged pod's
-# next CNI ADD retry then succeeds. Blast radius is one dead endpoint.
+# This watchdog heals in two tiers, least-disruptive first:
+#
+#   Tier 1 (per-endpoint, default): evict only the orphaned endpoint via
+#     cilium-dbg endpoint disconnect ipv4:<X>
+#   which maps to DELETE /endpoint/{id} -> RemoveEndpoint -> unexpose ->
+#   removeReferencesLocked, clearing the stale IP-keyed entry; or, when a
+#   reserved/infra endpoint holds the IP, delete-reschedule the wedged pod onto
+#   a free address. Blast radius is one dead endpoint or one pod.
+#
+#   Tier 2 (per-node escalation): on some variants the leak lives in agent
+#   in-memory state that `endpoint disconnect` cannot reach — the IP is no longer
+#   in the queryable endpoint registry (so Tier 1 logs "no endpoint holds ip" and
+#   no-ops) yet the agent still rejects the sandbox, and IPAM keeps re-handing the
+#   SAME IP, so a delete-rescheduled pod just re-wedges on it: an inescapable
+#   loop. When the same (node,ip) survives ESCALATE_AFTER heal cycles of Tier 1,
+#   escalate to the only remedy that clears an in-memory leak — restart the node's
+#   cilium-agent (delete its pod; the DaemonSet recreates it, rebuilding from
+#   CRDs/IPAM without the stale entry). This is heavier — it drops the node's
+#   whole in-memory endpoint registry — and a production operator would avoid it,
+#   but for a test-only, install-phase healer it is the effective, acceptable
+#   remedy. It is bounded by a per-node cap (MAX_ESCALATIONS_PER_NODE) so it can
+#   never loop-restart agents, and every action is logged loudly with its reason.
 #
 # It is READ-ONLY until it has positively confirmed an orphan: an endpoint in
 # the owning node's registry holds the disputed IP, that endpoint does not back
 # a live Running pod with that IP, and a different pod is currently wedged
 # requesting the same IP. If anything is ambiguous it logs and does nothing —
-# it never disconnects a live endpoint.
+# it never disconnects a live endpoint. The Tier-2 escalation never fires for a
+# refused (genuine duplicate-IP) case: only repeated, confirmed leak candidates
+# count toward it, so an agent restart can never mask a real product bug.
 #
 # REMOVE this script, hack/e2e-cilium-leak-healer.yaml, and the setup_file/
 # teardown_file hooks in hack/e2e-install-cozystack.bats once a fixed Cilium
@@ -45,6 +62,15 @@ set -u
 CILIUM_NS="${CILIUM_NS:-cozy-cilium}"
 POLL_INTERVAL="${CILIUM_HEALER_POLL_SEC:-15}"
 
+# Tier-2 escalation tuning (see header). ESCALATE_AFTER is how many consecutive
+# heal cycles the SAME (node,ip) must stay wedged under Tier-1 before the
+# cilium-agent on that node is restarted; the per-node cap bounds how many times
+# an agent may be restarted over the cluster lifetime so the watchdog can never
+# loop-restart agents.
+ESCALATE_AFTER="${CILIUM_HEALER_ESCALATE_AFTER:-3}"
+MAX_ESCALATIONS_PER_NODE="${CILIUM_HEALER_MAX_AGENT_RESTARTS:-2}"
+STATE_DIR="${CILIUM_HEALER_STATE_DIR:-/tmp/cilium-leak-healer}"
+
 log() { echo "[cilium-leak-healer $(date -u +%H:%M:%S)] $*"; }
 
 # cilium-agent pod running on a given node (DaemonSet selector k8s-app=cilium).
@@ -54,7 +80,35 @@ agent_on_node() {
     -o jsonpath='{.items[0].metadata.name}' 2>/dev/null
 }
 
-log "started (ns=$CILIUM_NS poll=${POLL_INTERVAL}s, in-cluster, runs for the cluster lifetime)"
+# Per-(node,ip) recurrence and per-node escalation counters, kept on the pod's
+# local filesystem. State is intentionally non-durable: if the watchdog crashes
+# and the Job restarts it, counters reset to zero, which only costs a few extra
+# heal cycles before re-escalating — never an unbounded restart loop, because the
+# per-node cap is re-derived from the (then-empty) state and the leak has either
+# cleared or recurs legibly.
+keyfrag() { printf '%s' "$1" | tr -c 'A-Za-z0-9._-' '_'; }
+node_dir() { printf '%s/%s' "$STATE_DIR" "$(keyfrag "$1")"; }
+read_count() { c=$(cat "$1" 2>/dev/null); case "$c" in ''|*[!0-9]*) printf '0' ;; *) printf '%s' "$c" ;; esac; }
+
+# bump_seen NODE IP -> increments and prints the (node,ip) recurrence count.
+bump_seen() {
+  d=$(node_dir "$1"); mkdir -p "$d" 2>/dev/null || true
+  f="$d/seen-$(keyfrag "$2")"; n=$(read_count "$f"); n=$((n + 1)); echo "$n" > "$f"; printf '%s' "$n"
+}
+# reset_node_seen NODE -> clears every (node,*) recurrence counter. One agent
+# restart clears the whole node's in-memory registry, so all its IPs get a fresh
+# grace window afterwards instead of immediately re-escalating.
+reset_node_seen() { rm -f "$(node_dir "$1")"/seen-* 2>/dev/null || true; }
+# escalations NODE -> prints how many agent restarts already issued for the node.
+escalations() { read_count "$(node_dir "$1")/escalations"; }
+# record_escalation NODE -> increments the node's agent-restart counter.
+record_escalation() {
+  d=$(node_dir "$1"); mkdir -p "$d" 2>/dev/null || true
+  f="$d/escalations"; n=$(read_count "$f"); n=$((n + 1)); echo "$n" > "$f"
+}
+
+mkdir -p "$STATE_DIR" 2>/dev/null || true
+log "started (ns=$CILIUM_NS poll=${POLL_INTERVAL}s, escalate-after=${ESCALATE_AFTER} cycles, max-agent-restarts/node=${MAX_ESCALATIONS_PER_NODE}, in-cluster, runs for the cluster lifetime)"
 
 # Endless loop: the pod is torn down with the ephemeral e2e cluster. Before
 # Cilium is serving (early install) the kubectl/exec calls simply fail and the
@@ -87,44 +141,83 @@ while true; do
     # before exposing — so any endpoint holding the IP is the stale one.)
     ep_json=$(kubectl exec -n "$CILIUM_NS" "$agent" -c cilium-agent -- \
                 cilium-dbg endpoint get "ipv4:$ip" -o json 2>/dev/null)
-    [ -n "$ep_json" ] && [ "$ep_json" != "[]" ] || \
-      { log "no endpoint holds ip=$ip on node=$node (already cleared?) for $ns/$pod"; continue; }
+    epid=""; epns=""; eppod=""
+    if [ -n "$ep_json" ] && [ "$ep_json" != "[]" ]; then
+      epid=$(printf '%s' "$ep_json" | jq -r '.[0].id // empty' 2>/dev/null)
+      epns=$(printf '%s' "$ep_json" | jq -r '.[0].status."external-identifiers"."k8s-namespace" // empty' 2>/dev/null)
+      eppod=$(printf '%s' "$ep_json" | jq -r '.[0].status."external-identifiers"."k8s-pod-name" // empty' 2>/dev/null)
 
-    epid=$(printf '%s' "$ep_json" | jq -r '.[0].id // empty' 2>/dev/null)
-    epns=$(printf '%s' "$ep_json" | jq -r '.[0].status."external-identifiers"."k8s-namespace" // empty' 2>/dev/null)
-    eppod=$(printf '%s' "$ep_json" | jq -r '.[0].status."external-identifiers"."k8s-pod-name" // empty' 2>/dev/null)
-
-    # If that endpoint claims a pod that is alive on the cluster with this IP,
-    # it is a LIVE endpoint -> a genuine duplicate-IP, NOT the leak. Refuse and
-    # shout so the real problem is visible instead of silently broken.
-    #
-    # Refuse for ANY non-terminal owner phase, not just Running: a Pending owner
-    # can already hold a live IPAM allocation (status.podIP is populated once the
-    # sandbox is created, before the pod goes Running), so disconnecting it would
-    # evict a legitimate endpoint mid-startup. Only a Succeeded/Failed owner is
-    # safely past needing its endpoint. (owner_ip == ip already gates on the pod
-    # still existing with that IP, so an empty phase from a vanished pod can't
-    # reach this branch.)
-    if [ -n "$epns" ] && [ -n "$eppod" ]; then
-      owner_ip=$(kubectl get pod -n "$epns" "$eppod" -o jsonpath='{.status.podIP}' 2>/dev/null)
-      owner_phase=$(kubectl get pod -n "$epns" "$eppod" -o jsonpath='{.status.phase}' 2>/dev/null)
-      if [ "$owner_ip" = "$ip" ] && [ "$owner_phase" != "Succeeded" ] && [ "$owner_phase" != "Failed" ]; then
-        log "REFUSE ip=$ip on node=$node: held by non-terminal pod $epns/$eppod (phase=$owner_phase; genuine duplicate IP, not the leak) — investigate"
-        continue
+      # If that endpoint claims a pod that is alive on the cluster with this IP,
+      # it is a LIVE endpoint -> a genuine duplicate-IP, NOT the leak. Refuse and
+      # shout so the real problem is visible instead of silently broken. This
+      # path `continue`s WITHOUT bumping the recurrence counter, so a genuine
+      # duplicate never accrues toward the Tier-2 agent restart.
+      #
+      # Refuse for ANY non-terminal owner phase, not just Running: a Pending owner
+      # can already hold a live IPAM allocation (status.podIP is populated once the
+      # sandbox is created, before the pod goes Running), so disconnecting it would
+      # evict a legitimate endpoint mid-startup. Only a Succeeded/Failed owner is
+      # safely past needing its endpoint. (owner_ip == ip already gates on the pod
+      # still existing with that IP, so an empty phase from a vanished pod can't
+      # reach this branch.)
+      if [ -n "$epns" ] && [ -n "$eppod" ]; then
+        owner_ip=$(kubectl get pod -n "$epns" "$eppod" -o jsonpath='{.status.podIP}' 2>/dev/null)
+        owner_phase=$(kubectl get pod -n "$epns" "$eppod" -o jsonpath='{.status.phase}' 2>/dev/null)
+        if [ "$owner_ip" = "$ip" ] && [ "$owner_phase" != "Succeeded" ] && [ "$owner_phase" != "Failed" ]; then
+          log "REFUSE ip=$ip on node=$node: held by non-terminal pod $epns/$eppod (phase=$owner_phase; genuine duplicate IP, not the leak) — investigate"
+          continue
+        fi
       fi
     fi
 
+    # Past the REFUSE gate this is a genuine leak candidate: no endpoint holds the
+    # IP (the stale entry is outside the queryable registry; disconnect can't reach
+    # it), a reserved/infra endpoint holds it, or a stale pod endpoint holds it.
+    # The Tier-1 remedies below cannot clear a leak that lives purely in the
+    # agent's in-memory state when IPAM keeps re-handing the SAME IP — the
+    # recreated pod re-wedges on it, an inescapable loop. Track recurrence per
+    # (node,ip); once the same IP has survived ESCALATE_AFTER heal cycles,
+    # escalate (Tier 2) to the only remedy that clears an in-memory leak:
+    # restarting the node's cilium-agent. Bounded by a per-node cap so it can
+    # never loop-restart agents.
+    seen=$(bump_seen "$node" "$ip")
+    if [ "$seen" -ge "$ESCALATE_AFTER" ]; then
+      esc=$(escalations "$node")
+      if [ "$esc" -lt "$MAX_ESCALATIONS_PER_NODE" ]; then
+        log "ESCALATE node=$node agent=$agent ip=$ip wedged=$ns/$pod: leaked-IP reuse unhealed after $seen heal cycles of Tier-1 remedies (cilium/cilium#38313) -> restart cilium-agent (restart $((esc + 1))/$MAX_ESCALATIONS_PER_NODE for this node; drops its stale in-memory endpoint registry)"
+        out=$(kubectl delete pod -n "$CILIUM_NS" "$agent" --wait=false 2>&1)
+        log "  result: ${out:-<none>}"
+        record_escalation "$node"
+        # One restart clears every leaked IP on the node, so give the whole node a
+        # fresh grace window rather than immediately re-escalating each IP.
+        reset_node_seen "$node"
+      else
+        log "ESCALATION CAP reached node=$node (max=$MAX_ESCALATIONS_PER_NODE agent restarts): leaked ip=$ip for $ns/$pod still unhealed — surfacing the failure rather than restarting again (cilium/cilium#38313)"
+      fi
+      continue
+    fi
+
+    # Below the escalation threshold: apply the least-disruptive Tier-1 remedy.
+    #
+    # No endpoint holds the IP -> nothing for `endpoint disconnect` to act on (the
+    # leak, if any, is outside the queryable registry). Log and wait; if it keeps
+    # recurring on the same IP the Tier-2 escalation above takes over.
+    if [ -z "$epid" ]; then
+      log "no endpoint holds ip=$ip on node=$node (already cleared, or leak not in endpoint registry) for $ns/$pod [seen $seen/$ESCALATE_AFTER before agent restart]"
+      continue
+    fi
+
     # An endpoint that holds the IP but backs no k8s pod (empty k8s identity) is
     # a reserved/infra endpoint — e.g. reserved:ingress, the per-node Cilium
     # Ingress endpoint — that legitimately owns the IP; the duplicate is an IPAM
     # race that handed the same pod-CIDR IP to a pod. Such endpoints both cannot
     # be disconnected ("endpoint may not be associated reserved labels") and must
-    # not be: evicting infra is wrong. The correct remedy is to reschedule the
-    # wedged pod onto a free IP by deleting it; its controller recreates it and
+    # not be: evicting infra is wrong. The correct Tier-1 remedy is to reschedule
+    # the wedged pod onto a free IP by deleting it; its controller recreates it and
     # the next CNI ADD picks a fresh IP (verified: the new pod lands on a clean
     # address while the ingress endpoint keeps the disputed one).
     if [ -z "$epns" ] || [ -z "$eppod" ]; then
-      log "HEAL node=$node ep=$epid ip=$ip held by reserved/infra endpoint (no k8s pod) -> delete wedged pod $ns/$pod"
+      log "HEAL node=$node ep=$epid ip=$ip held by reserved/infra endpoint (no k8s pod) -> delete wedged pod $ns/$pod [seen $seen/$ESCALATE_AFTER before agent restart]"
       out=$(kubectl delete pod -n "$ns" "$pod" --wait=false 2>&1)
       log "  result: ${out:-<none>}"
       continue
@@ -133,7 +226,7 @@ while true; do
     # Confirmed stale pod endpoint: endpoint $epid holds $ip, backs a pod that is
     # no longer live, and pod $ns/$pod is wedged requesting the same IP. Evict
     # only this endpoint.
-    log "HEAL node=$node agent=$agent ep=$epid ip=$ip wedged=$ns/$pod -> endpoint disconnect ipv4:$ip"
+    log "HEAL node=$node agent=$agent ep=$epid ip=$ip wedged=$ns/$pod -> endpoint disconnect ipv4:$ip [seen $seen/$ESCALATE_AFTER before agent restart]"
     out=$(kubectl exec -n "$CILIUM_NS" "$agent" -c cilium-agent -- \
             cilium-dbg endpoint disconnect "ipv4:$ip" 2>&1)
     log "  result: ${out:-<none>}"

hack/e2e-cilium-leak-healer.yaml

@@ -24,9 +24,12 @@ rules:
   - apiGroups: [""]
     resources: ["events"]
     verbs: ["get", "list", "watch"]
-  # inspect the wedged pod and the endpoint's claimed owner, and delete the
-  # wedged pod to reschedule it onto a free IP when the disputed IP is held by
-  # a reserved/infra endpoint (or a disconnect otherwise fails)
+  # inspect the wedged pod and the endpoint's claimed owner; delete the wedged
+  # pod to reschedule it onto a free IP (Tier 1, when the disputed IP is held by
+  # a reserved/infra endpoint or a disconnect otherwise fails); and delete the
+  # node's cilium-agent pod in cozy-cilium to restart it (Tier 2 escalation,
+  # which clears a stale in-memory endpoint leak the DaemonSet then rebuilds
+  # without). cluster-wide `delete` already covers cozy-cilium, so no extra verb.
   - apiGroups: [""]
     resources: ["pods"]
     verbs: ["get", "list", "delete"]

hack/e2e-install-cozystack.bats

@@ -5,8 +5,11 @@
   # class): a deleted pod's endpoint is orphaned in the agent's registry, IPAM
   # re-hands its IP to a new pod, and the agent then rejects the new sandbox with
   # "IP <X> is already in use" until an agent restart. The watchdog runs as an
-  # in-cluster Job that surgically evicts only the orphaned endpoint, covering
-  # install and (the Job outlives this file) the whole app suite.
+  # in-cluster Job that first surgically evicts only the orphaned endpoint and,
+  # if the leaked IP keeps recurring (the leak is in agent in-memory state that
+  # endpoint-disconnect/delete-reschedule cannot reach), escalates to restarting
+  # that node's cilium-agent — bounded by a per-node cap. It covers install and
+  # (the Job outlives this file) the whole app suite.
   #
   # This is a real @test, NOT a bats setup_file hook: the e2e runner
   # (hack/cozytest.sh) only executes @test functions — it never invokes

hack/e2e-prepare-cluster.bats

@@ -261,8 +261,15 @@ EOF
   # ephemeral runners can take several attempts of ~5s each.
   timeout 60 sh -ec 'until talosctl bootstrap -n 192.168.123.11 -e 192.168.123.11; do sleep 2; done'
 
-  # Wait until etcd is healthy
-  if ! timeout 180 sh -ec 'until talosctl etcd members -n 192.168.123.11,192.168.123.12,192.168.123.13 -e 192.168.123.10 >/dev/null 2>&1; do sleep 1; done'; then
+  # Wait until etcd is healthy.
+  # Budget is 5m, not 3m: a 3-node etcd converges via Talos's serialized
+  # learner promotion (etcd admits one learner at a time), so the third member
+  # is only promoted after the second is fully caught up. On a loaded ephemeral
+  # runner this deterministic path can legitimately approach ~3m wall clock —
+  # a 180s ceiling photo-finishes against it and times out a cluster that is
+  # actually healthy. The wait itself is already event-driven (poll until
+  # members respond); only the ceiling needed widening.
+  if ! timeout 300 sh -ec 'until talosctl etcd members -n 192.168.123.11,192.168.123.12,192.168.123.13 -e 192.168.123.10 >/dev/null 2>&1; do sleep 1; done'; then
     talosctl dmesg -n 192.168.123.11,192.168.123.12,192.168.123.13 -e 192.168.123.10 || true
     exit 1
   fi

hack/select-e2e.sh

@@ -43,8 +43,21 @@ build_owners_index() {
 }
 
 # yq: PackageSource name -> sources that depend on it (reverse of dependsOn)
+#
+# Exclude cozystack.cozystack-engine as a propagation hub. Every *-application
+# source declares `dependsOn: cozystack.cozystack-engine` purely as an INSTALL
+# ORDERING edge — the app's *-rd HelmRelease must wait for the engine to
+# register the ApplicationDefinition CRD before it can reconcile. That is a
+# universal lifecycle dependency, NOT a behavioral one: a change to an engine
+# dependency (postgres-operator, keycloak, cert-manager, ...) does not alter any
+# app's runtime behavior, so it must not fan test selection out to every app.
+# Without this filter, postgres-operator -> keycloak -> cozystack-engine -> EVERY
+# app, defeating test-impact analysis. Dropping the engine reverse edges keeps
+# the engine reachable but stops it propagating selection downstream. A change to
+# the engine itself still triggers the full suite via the no-app-descendants
+# safety-net at the bottom of this script.
 build_reverse_deps() {
-  yq -rN '.metadata.name as $n | .spec.variants[]?.dependsOn[]? | select(. != null and . != "") | . + "\t" + $n' "$SOURCES_DIR"/*.yaml
+  yq -rN '.metadata.name as $n | .spec.variants[]?.dependsOn[]? | select(. != null and . != "" and . != "cozystack.cozystack-engine") | . + "\t" + $n' "$SOURCES_DIR"/*.yaml
 }
 
 OWNERS=$(build_owners_index | sort -u)

hack/select-e2e_test.bats

@@ -37,6 +37,26 @@
     fi
 }
 
+@test "engine-dependency change does not fan out via the ordering edge" {
+    # cert-manager is a dependency of cozystack-engine, and every app declares
+    # dependsOn cozystack-engine purely as an INSTALL-ORDERING edge (the app's
+    # *-rd HelmRelease waits for the ApplicationDefinition CRD). That edge must
+    # not propagate test selection: a cert-manager change selects only its
+    # genuine direct dependents (postgres, harbor, ...), never unrelated apps
+    # like kafka that reach cert-manager solely through the engine.
+    tmp=$(mktemp -d)
+    trap 'rm -rf "$tmp"' EXIT
+    cp -r packages/core/platform/sources "$tmp/sources"
+    echo "packages/system/cert-manager/values.yaml" > "$tmp/diff"
+    output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources")
+    echo "$output" | grep -wq postgres
+    echo "$output" | grep -wq harbor
+    if echo "$output" | grep -wq kafka; then
+        echo "cert-manager change must not fan out via engine; got: $output" >&2
+        exit 1
+    fi
+}
+
 @test "networking change triggers full suite" {
     tmp=$(mktemp -d)
     trap 'rm -rf "$tmp"' EXIT

packages/apps/kubernetes/.gitignore

@@ -0,0 +1 @@
+images/kubevirt-csi-driver/kubevirt-csi-driver

packages/apps/kubernetes/images/cluster-autoscaler/Dockerfile

@@ -1,5 +1,5 @@
 # Source: https://raw.githubusercontent.com/kubernetes/autoscaler/refs/heads/master/cluster-autoscaler/Dockerfile.amd64
-ARG builder_image=docker.io/library/golang:1.23.4
+ARG builder_image=docker.io/library/golang:1.26
 ARG BASEIMAGE=gcr.io/distroless/static:nonroot-${TARGETARCH}
 
 FROM ${builder_image} AS builder

packages/apps/kubernetes/images/kubevirt-cloud-provider/Dockerfile

@@ -1,5 +1,5 @@
 # Source: https://github.com/kubevirt/cloud-provider-kubevirt/blob/main/build/images/kubevirt-cloud-controller-manager/Dockerfile
-FROM golang:1.20.6 AS builder
+FROM golang:1.26 AS builder
 
 ARG TARGETOS
 ARG TARGETARCH

packages/apps/kubernetes/images/kubevirt-csi-driver/go.mod

@@ -1,6 +1,6 @@
 module cozystack.io/kubevirt-csi-driver
 
-go 1.26.2
+go 1.26.4
 
 require (
 	github.com/container-storage-interface/spec v1.10.0
@@ -44,11 +44,11 @@ require (
 	github.com/x448/float16 v0.8.4 // indirect
 	go.yaml.in/yaml/v2 v2.4.3 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
-	golang.org/x/net v0.53.0 // indirect
+	golang.org/x/net v0.56.0 // indirect
 	golang.org/x/oauth2 v0.36.0 // indirect
-	golang.org/x/sys v0.43.0 // indirect
-	golang.org/x/term v0.42.0 // indirect
-	golang.org/x/text v0.36.0 // indirect
+	golang.org/x/sys v0.46.0 // indirect
+	golang.org/x/term v0.44.0 // indirect
+	golang.org/x/text v0.38.0 // indirect
 	golang.org/x/time v0.14.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect
 	google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect

packages/apps/kubernetes/images/kubevirt-csi-driver/go.sum

@@ -282,8 +282,8 @@ golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210=
 golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
 golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
 golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
-golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
-golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
+golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
+golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
 golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
 golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -340,8 +340,8 @@ golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
-golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
-golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
+golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
 golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0=
 golang.org/x/telemetry v0.0.0-20250710130107-8d8967aff50b/go.mod h1:4ZwOYna0/zsOKwuR5X/m0QFOJpSZvAxFfkQT+Erd9D4=
@@ -368,8 +368,8 @@ golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss=
 golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
 golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg=
 golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww=
-golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY=
-golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY=
+golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc=
+golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -391,8 +391,8 @@ golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM=
 golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
 golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
 golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
-golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
-golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
+golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
+golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
 golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
 golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -420,8 +420,8 @@ golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs
 golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=
 golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc=
 golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg=
-golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s=
-golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
+golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8=
+golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0=
 golang.org/x/tools/go/expect v0.1.0-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY=
 golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY=
 golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGNv1ib0J382/DPCRS/BPnsGebyM1Gj5VSDpG8=

packages/apps/mariadb/Makefile

@@ -3,6 +3,9 @@ MARIADB_BACKUP_TAG = $(shell awk '$$1 == "version:" {print $$2}' Chart.yaml)
 include ../../../hack/common-envs.mk
 include ../../../hack/package.mk
 
+test:
+	helm unittest .
+
 generate:
 	cozyvalues-gen -m 'mariadb' -v values.yaml -s values.schema.json -r README.md -g ../../../api/apps/v1alpha1/mariadb/types.go
 	../../../hack/update-crd.sh

packages/apps/mariadb/templates/mariadb.yaml

@@ -43,6 +43,14 @@ spec:
     enabled: true
     exporter:
       image: prom/mysqld-exporter:v0.15.1
+      # The mariadb-operator hardcodes the exporter's liveness/readiness probe
+      # as an httpGet on port 9104, which kubelet dials over the pod's IPv4
+      # address. Bind the exporter to the IPv4 wildcard so the probe connects
+      # deterministically; the default [::] listener intermittently refuses the
+      # IPv4 connect on single-stack-IPv4 data planes, flapping the pod to
+      # NotReady and leaving the metrics Service endpoint unpopulated.
+      args:
+      - --web.listen-address=0.0.0.0:9104
       resources:
         requests:
           cpu: 50m

packages/apps/mariadb/tests/exporter_listen_address_test.yaml

@@ -0,0 +1,32 @@
+suite: MariaDB metrics exporter listen address
+
+# Regression cover: the mariadb-operator hardcodes the mysqld-exporter
+# liveness/readiness probe as an httpGet on port 9104, which kubelet dials
+# over the pod's IPv4 address. The exporter must therefore listen on an
+# IPv4 socket; an IPv6 wildcard ([::]) listener intermittently refuses the
+# IPv4 probe on single-stack-IPv4 data planes, flapping the pod to NotReady
+# and leaving the metrics Service endpoint unpopulated. Pin the explicit
+# IPv4 listen-address flag so the probe is reachable deterministically.
+
+templates:
+  - templates/mariadb.yaml
+
+tests:
+  - it: renders the MariaDB CR
+    release:
+      name: mariadb-test
+      namespace: tenant-test
+    asserts:
+      - hasDocuments:
+          count: 1
+      - isKind:
+          of: MariaDB
+
+  - it: pins the exporter to the IPv4 wildcard so the kubelet probe is reachable
+    release:
+      name: mariadb-test
+      namespace: tenant-test
+    asserts:
+      - contains:
+          path: spec.metrics.exporter.args
+          content: --web.listen-address=0.0.0.0:9104

packages/apps/tenant/templates/cleanup-job.yaml

@@ -21,7 +21,7 @@ metadata:
 rules:
 - apiGroups: ["helm.toolkit.fluxcd.io"]
   resources: ["helmreleases"]
-  verbs: ["get", "list", "delete"]
+  verbs: ["get", "list", "watch", "delete"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
@@ -60,11 +60,20 @@ spec:
     spec:
       serviceAccountName: {{ include "tenant.name" . }}-cleanup
       restartPolicy: OnFailure
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65534
+        seccompProfile:
+          type: RuntimeDefault
       containers:
       - name: cleanup
-        image: bitnami/kubectl:latest
+        image: docker.io/clastix/kubectl:v1.32
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop: ["ALL"]
         command:
-        - /bin/bash
+        - /bin/sh
         - -c
         - |
           set -e

packages/apps/tenant/tests/cleanup_role_watch_test.yaml

@@ -0,0 +1,91 @@
+suite: tenant cleanup pre-delete hook
+
+# The pre-delete cleanup hook deletes the tenant's HelmReleases with
+# `kubectl delete --wait=true`, which sets up a watch on helmreleases to block
+# until finalization completes. Without the watch verb that watch is rejected
+# with HTTP 403, the delete call hangs, the Job never finishes, and tenant
+# deletion stalls on the hook. Pin the verb so it cannot regress.
+#
+# The Job runs kubectl from a maintained, version-pinned image. Pin the image
+# so it cannot drift back to an unpinned / sunset base (e.g. bitnami:latest),
+# and pin the command interpreter to a shell that image actually ships
+# (clastix/kubectl is Alpine — /bin/sh only, no /bin/bash) so the container
+# can exec at all.
+
+# Restrict rendering to the cleanup template so the chart's namespace.yaml
+# `lookup` (which errors under helm-unittest) is not evaluated.
+templates:
+  - templates/cleanup-job.yaml
+
+release:
+  name: tenant-test
+  namespace: tenant-test
+
+tests:
+  - it: cleanup Role grants watch on helmreleases
+    asserts:
+      - template: templates/cleanup-job.yaml
+        documentIndex: 1
+        isKind:
+          of: Role
+      - template: templates/cleanup-job.yaml
+        documentIndex: 1
+        contains:
+          path: rules[0].verbs
+          content: watch
+
+  - it: cleanup Job uses the maintained clastix kubectl image
+    asserts:
+      - template: templates/cleanup-job.yaml
+        documentIndex: 3
+        isKind:
+          of: Job
+      - template: templates/cleanup-job.yaml
+        documentIndex: 3
+        equal:
+          path: spec.template.spec.containers[0].image
+          value: docker.io/clastix/kubectl:v1.32
+
+  - it: cleanup Job invokes a shell the image actually ships
+    asserts:
+      - template: templates/cleanup-job.yaml
+        documentIndex: 3
+        isKind:
+          of: Job
+      - template: templates/cleanup-job.yaml
+        documentIndex: 3
+        equal:
+          path: spec.template.spec.containers[0].command[0]
+          value: /bin/sh
+
+  - it: cleanup Job runs non-root under a restrictive securityContext
+    asserts:
+      - template: templates/cleanup-job.yaml
+        documentIndex: 3
+        isKind:
+          of: Job
+      - template: templates/cleanup-job.yaml
+        documentIndex: 3
+        equal:
+          path: spec.template.spec.securityContext.runAsNonRoot
+          value: true
+      - template: templates/cleanup-job.yaml
+        documentIndex: 3
+        equal:
+          path: spec.template.spec.securityContext.runAsUser
+          value: 65534
+      - template: templates/cleanup-job.yaml
+        documentIndex: 3
+        equal:
+          path: spec.template.spec.securityContext.seccompProfile.type
+          value: RuntimeDefault
+      - template: templates/cleanup-job.yaml
+        documentIndex: 3
+        equal:
+          path: spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation
+          value: false
+      - template: templates/cleanup-job.yaml
+        documentIndex: 3
+        contains:
+          path: spec.template.spec.containers[0].securityContext.capabilities.drop
+          content: ALL

packages/core/installer/templates/cozystack-operator.yaml

@@ -78,6 +78,9 @@ spec:
         {{- if .Values.cozystackOperator.platformSourceRef }}
         - --platform-source-ref={{ .Values.cozystackOperator.platformSourceRef }}
         {{- end }}
+        {{- if .Values.cozystackOperator.platformSourceSecret }}
+        - --platform-source-secret={{ .Values.cozystackOperator.platformSourceSecret }}
+        {{- end }}
         {{- if eq .Values.cozystackOperator.variant "talos" }}
         env:
         # Talos KubePrism endpoint

packages/core/installer/values.yaml

@@ -29,6 +29,16 @@ cozystackOperator:
   # (operator default: 5). Number of release revisions Helm keeps per HR.
   # Use 0 for unlimited; lower values reduce Secret accumulation in test clusters.
   helmReleaseMaxHistory: ""
+  # Name of the Secret in cozy-system namespace containing credentials for the
+  # platform source. When non-empty, sets spec.secretRef on the generated
+  # OCIRepository or GitRepository resource. Secret type depends on source kind:
+  #   OCI (oci://...): kubernetes.io/dockerconfigjson with registry credentials.
+  #   Git over HTTPS:  Opaque secret with `username`+`password`, or `bearerToken`.
+  #   Git over SSH:    Opaque secret with `identity` (PEM private key) and `known_hosts`.
+  # The Secret must already exist in cozy-system before the operator reconciles;
+  # otherwise source-controller will report a failed condition until it is created.
+  # See FluxCD source-controller docs for the exact key layout per scheme.
+  platformSourceSecret: ''
 # Generic variant configuration (only used when cozystackOperator.variant=generic)
 cozystack:
   # Kubernetes API server host (IP only, no protocol/port)

packages/core/platform/images/migrations/migrations/42

@@ -14,29 +14,20 @@
 # the next chart-driven apply sees no diff.
 #
 # 3-way-merge note: this apply records the no-delete label in the kubectl
-# last-applied-configuration annotation under the default field manager.
-# Any future migration that re-stamps cozystack-version via the same field
-# manager MUST include the label inline (matching the chart template at
-# templates/cozystack-version.yaml) — a label-less apply by the same field
-# manager would strip it.
+# last-applied-configuration annotation under the default field manager. A
+# label-less apply by the same field manager would strip it, so the manifest
+# is rendered by the shared helper (lib/cozystack-version.sh) that every
+# migration sources rather than a hand-copied heredoc.
 
 set -euo pipefail
 
+# shellcheck source=lib/cozystack-version.sh
+. "$(dirname "$0")/lib/cozystack-version.sh"
+
 NAMESPACE="${NAMESPACE:-cozy-system}"
 DRY_RUN="${MIGRATION_DRY_RUN:-0}"
 
-MANIFEST=$(cat <<EOF
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: cozystack-version
-  namespace: ${NAMESPACE}
-  labels:
-    platform.cozystack.io/no-delete: "true"
-data:
-  version: "43"
-EOF
-)
+MANIFEST=$(render_cozystack_version_manifest 43)
 
 if [ "$DRY_RUN" = "1" ]; then
   echo "[dry-run] would apply:"

packages/core/platform/images/migrations/migrations/43

@@ -12,6 +12,9 @@
 
 set -euo pipefail
 
+# shellcheck source=lib/cozystack-version.sh
+. "$(dirname "$0")/lib/cozystack-version.sh"
+
 # Iterate every namespace that has a Cluster named "seaweedfs-db".
 for ns in $(kubectl get cluster.postgresql.cnpg.io -A \
               -o jsonpath='{range .items[?(@.metadata.name=="seaweedfs-db")]}{.metadata.namespace}{"\n"}{end}'); do
@@ -27,21 +30,6 @@ for ns in $(kubectl get cluster.postgresql.cnpg.io -A \
   fi
 done
 
-# Stamp version.
-# Mirror migration 42's labeled manifest. A label-less apply by the same field
-# manager (kubectl create | kubectl apply) would strip the
-# platform.cozystack.io/no-delete label migration 42 added, dropping
-# cozystack-version out of the cozystack-no-delete-guardrail
-# ValidatingAdmissionPolicy. templates/cozystack-version.yaml only re-renders the
-# labeled ConfigMap on first install, so the label must be carried inline here.
-kubectl apply --filename - <<EOF
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: cozystack-version
-  namespace: cozy-system
-  labels:
-    platform.cozystack.io/no-delete: "true"
-data:
-  version: "44"
-EOF
+# Stamp version. The labeled manifest lives in the shared helper so it cannot
+# drift back to a label-less apply that strips platform.cozystack.io/no-delete.
+stamp_cozystack_version 44

packages/core/platform/images/migrations/migrations/44

@@ -10,6 +10,9 @@
 
 set -euo pipefail
 
+# shellcheck source=lib/cozystack-version.sh
+. "$(dirname "$0")/lib/cozystack-version.sh"
+
 if kubectl get deployment flux-tenants -n cozy-fluxcd >/dev/null 2>&1; then
   # No early-close (head) in the pipe: under pipefail a large name list would
   # SIGPIPE kubectl and abort the migration.
@@ -23,21 +26,6 @@ if kubectl get deployment flux-tenants -n cozy-fluxcd >/dev/null 2>&1; then
   fi
 fi
 
-# Stamp version.
-# Mirror migration 43's labeled manifest. A label-less apply by the same field
-# manager would strip the platform.cozystack.io/no-delete label, dropping
-# cozystack-version out of the cozystack-no-delete-guardrail
-# ValidatingAdmissionPolicy. templates/cozystack-version.yaml only re-renders
-# the labeled ConfigMap on first install, so the label must be carried inline
-# here.
-kubectl apply --filename - <<EOF
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: cozystack-version
-  namespace: cozy-system
-  labels:
-    platform.cozystack.io/no-delete: "true"
-data:
-  version: "45"
-EOF
+# Stamp version. The labeled manifest lives in the shared helper so it cannot
+# drift back to a label-less apply that strips platform.cozystack.io/no-delete.
+stamp_cozystack_version 45

packages/core/platform/images/migrations/migrations/lib/cozystack-version.sh

@@ -0,0 +1,51 @@
+# shellcheck shell=sh
+# Shared helper for stamping the cozystack-version ConfigMap during migrations.
+#
+# The ConfigMap MUST carry the platform.cozystack.io/no-delete=true label so the
+# cozystack-no-delete-guardrail ValidatingAdmissionPolicy guards it against
+# deletion. Migrations apply it under the default kubectl field manager; a
+# label-less apply by that same manager would strip the label (the failure mode
+# migration 42 first fixed). templates/cozystack-version.yaml renders the
+# labeled ConfigMap only on first install (lookup-if-not), so every upgrade-time
+# stamp must re-emit the label by hand.
+#
+# Centralizing the manifest here means migration 45+ cannot drift back to a
+# label-less apply: every go-forward stamp (migration 42 onward and the
+# bootstrap path in run-migrations.sh) sources this file and calls the helper
+# instead of copy-pasting a heredoc. Migrations 1-41 predate the no-delete
+# label and stamp label-less on purpose; migration 42 backfills the label on
+# any upgrade path through them, so they are left frozen. The
+# cozystack-version-stamp.bats suite pins this — no migration numbered >= 42
+# (nor run-migrations.sh) may stamp cozystack-version without the helper.
+#
+# Sourced, not executed. Each migrations/<N> script (42+) does:
+#   . "$(dirname "$0")/lib/cozystack-version.sh"
+# and run-migrations.sh sources /migrations/lib/cozystack-version.sh.
+
+# render_cozystack_version_manifest <version>
+# Emit the labeled cozystack-version ConfigMap manifest on stdout. Pure (no
+# kubectl), so it is unit-testable in isolation. The namespace defaults to
+# cozy-system and is overridable via $NAMESPACE.
+render_cozystack_version_manifest() {
+  : "${1:?render_cozystack_version_manifest: version argument required}"
+  cat <<EOF
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cozystack-version
+  namespace: ${NAMESPACE:-cozy-system}
+  labels:
+    platform.cozystack.io/no-delete: "true"
+data:
+  version: "$1"
+EOF
+}
+
+# stamp_cozystack_version <version>
+# Render and apply the labeled manifest under the default kubectl field manager.
+stamp_cozystack_version() {
+  # Guard the version here too, not only in render: without pipefail a render
+  # failure would not abort the pipeline and kubectl would apply empty input.
+  : "${1:?stamp_cozystack_version: version argument required}"
+  render_cozystack_version_manifest "$1" | kubectl apply --filename -
+}

packages/core/platform/images/migrations/run-migrations.sh

@@ -1,6 +1,9 @@
 #!/bin/sh
 set -euo pipefail
 
+# shellcheck source=migrations/lib/cozystack-version.sh
+. /migrations/lib/cozystack-version.sh
+
 NAMESPACE="${NAMESPACE:-cozy-system}"
 CURRENT_VERSION="${CURRENT_VERSION:-0}"
 TARGET_VERSION="${TARGET_VERSION:-0}"
@@ -10,9 +13,11 @@ echo "Starting migrations from version $CURRENT_VERSION to $TARGET_VERSION"
 # Check if ConfigMap exists
 if ! kubectl get configmap --namespace "$NAMESPACE" cozystack-version >/dev/null 2>&1; then
   echo "ConfigMap cozystack-version does not exist, creating it with version $TARGET_VERSION"
-  kubectl create configmap --namespace "$NAMESPACE" cozystack-version \
-    --from-literal=version="$TARGET_VERSION" \
-    --dry-run=client --output yaml | kubectl apply --filename -
+  # Stamp via the shared helper so the bootstrap ConfigMap carries the
+  # platform.cozystack.io/no-delete label, matching every go-forward stamp
+  # (migration 42 onward; migrations 1-41 stamp label-less and are backfilled
+  # by migration 42, so they are intentionally left as-is).
+  stamp_cozystack_version "$TARGET_VERSION"
   echo "ConfigMap created with version $TARGET_VERSION"
   exit 0
 fi

packages/core/platform/sources/backupstrategy-controller.yaml

@@ -18,8 +18,16 @@ spec:
     # All three must be installed before the templates render, otherwise
     # the Helm release fails with "no matches for kind". Order is enforced
     # via dependsOn so a fresh-cluster bootstrap cannot race the CRDs.
+    #
+    # The cozy-backups Bucket is also created INTO the tenant-root namespace,
+    # which is provisioned by cozystack-basics (the static tenant-root release).
+    # Without the cozystack-basics edge the Bucket patch can race namespace
+    # creation on a fresh install and fail with `namespaces "tenant-root" not
+    # found`, which then races the install script's wait deadline. Depend on
+    # cozystack-basics so the target namespace exists first.
     dependsOn:
     - cozystack.networking
+    - cozystack.cozystack-basics
     - cozystack.backup-controller
     - cozystack.bucket-application
     - cozystack.objectstorage-controller

packages/core/platform/sources/bootbox-application.yaml

@@ -13,6 +13,8 @@ spec:
     - name: default
       dependsOn:
         - cozystack.networking
+        # cozystack-engine ships the ApplicationDefinition CRD rendered by the *-rd component below.
+        - cozystack.cozystack-engine
       libraries:
         - name: cozy-lib
           path: library/cozy-lib

packages/core/platform/sources/bucket-application.yaml

@@ -14,6 +14,8 @@ spec:
       dependsOn:
         - cozystack.networking
         - cozystack.objectstorage-controller
+        # cozystack-engine ships the ApplicationDefinition CRD rendered by the *-rd component below.
+        - cozystack.cozystack-engine
       libraries:
         - name: cozy-lib
           path: library/cozy-lib

packages/core/platform/sources/clickhouse-application.yaml

@@ -13,6 +13,8 @@ spec:
     - name: default
       dependsOn:
         - cozystack.clickhouse-operator
+        # cozystack-engine ships the ApplicationDefinition CRD rendered by the *-rd component below.
+        - cozystack.cozystack-engine
       libraries:
         - name: cozy-lib
           path: library/cozy-lib

packages/core/platform/sources/cozystack-basics.yaml

@@ -28,8 +28,11 @@ spec:
     # Drift detection is off on operator-generated HelmReleases, so a
     # capability gate in the templates would render the policy out at first
     # install and never add it back; ordering via dependsOn is the reliable
-    # fix. cozystack-basics is a dependency-graph leaf (nothing dependsOn it)
-    # and engine components consume none of its outputs, so this adds no cycle.
+    # fix. The cozystack-engine and gateway-api-crds packages it depends on
+    # consume none of cozystack-basics' outputs and do not depend back on it,
+    # so these edges add no cycle. (Downstream packages such as backupstrategy-
+    # controller dependsOn cozystack-basics, but those do not feed engine or
+    # gateway-api-crds, so they cannot close a loop here.)
     - cozystack.cozystack-engine
     - cozystack.gateway-api-crds
     components:

packages/core/platform/sources/etcd-application.yaml

@@ -13,6 +13,8 @@ spec:
     - name: default
       dependsOn:
         - cozystack.networking
+        # cozystack-engine ships the ApplicationDefinition CRD rendered by the *-rd component below.
+        - cozystack.cozystack-engine
       libraries:
         - name: cozy-lib
           path: library/cozy-lib

packages/core/platform/sources/external-dns-application.yaml

@@ -13,6 +13,8 @@ spec:
     - name: default
       dependsOn:
         - cozystack.networking
+        # cozystack-engine ships the ApplicationDefinition CRD rendered by the *-rd component below.
+        - cozystack.cozystack-engine
       libraries:
         - name: cozy-lib
           path: library/cozy-lib

packages/core/platform/sources/foundationdb-application.yaml

@@ -14,6 +14,8 @@ spec:
       dependsOn:
         - cozystack.networking
         - cozystack.foundationdb-operator
+        # cozystack-engine ships the ApplicationDefinition CRD rendered by the *-rd component below.
+        - cozystack.cozystack-engine
       libraries:
         - name: cozy-lib
           path: library/cozy-lib

packages/core/platform/sources/harbor-application.yaml

@@ -16,6 +16,8 @@ spec:
         - cozystack.postgres-operator
         - cozystack.redis-operator
         - cozystack.objectstorage-controller
+        # cozystack-engine ships the ApplicationDefinition CRD rendered by the *-rd component below.
+        - cozystack.cozystack-engine
       libraries:
         - name: cozy-lib
           path: library/cozy-lib

packages/core/platform/sources/http-cache-application.yaml

@@ -13,6 +13,8 @@ spec:
     - name: default
       dependsOn:
         - cozystack.networking
+        # cozystack-engine ships the ApplicationDefinition CRD rendered by the *-rd component below.
+        - cozystack.cozystack-engine
       libraries:
         - name: cozy-lib
           path: library/cozy-lib

packages/core/platform/sources/info-application.yaml

@@ -13,6 +13,8 @@ spec:
     - name: default
       dependsOn:
         - cozystack.networking
+        # cozystack-engine ships the ApplicationDefinition CRD rendered by the *-rd component below.
+        - cozystack.cozystack-engine
       libraries:
         - name: cozy-lib
           path: library/cozy-lib

packages/core/platform/sources/ingress-application.yaml

@@ -13,6 +13,8 @@ spec:
     - name: default
       dependsOn:
         - cozystack.networking
+        # cozystack-engine ships the ApplicationDefinition CRD rendered by the *-rd component below.
+        - cozystack.cozystack-engine
       libraries:
         - name: cozy-lib
           path: library/cozy-lib

packages/core/platform/sources/kafka-application.yaml

@@ -14,6 +14,8 @@ spec:
       dependsOn:
         - cozystack.networking
         - cozystack.kafka-operator
+        # cozystack-engine ships the ApplicationDefinition CRD rendered by the *-rd component below.
+        - cozystack.cozystack-engine
       libraries:
         - name: cozy-lib
           path: library/cozy-lib

packages/core/platform/sources/kubernetes-application.yaml

@@ -18,6 +18,8 @@ spec:
         - cozystack.capi-provider-core
         - cozystack.capi-provider-cp-kamaji
         - cozystack.capi-provider-infra-kubevirt
+        # cozystack-engine ships the ApplicationDefinition CRD rendered by the *-rd component below.
+        - cozystack.cozystack-engine
       libraries:
         - name: cozy-lib
           path: library/cozy-lib

packages/core/platform/sources/mariadb-application.yaml

@@ -14,6 +14,8 @@ spec:
       dependsOn:
         - cozystack.networking
         - cozystack.mariadb-operator
+        # cozystack-engine ships the ApplicationDefinition CRD rendered by the *-rd component below.
+        - cozystack.cozystack-engine
       libraries:
         - name: cozy-lib
           path: library/cozy-lib

packages/core/platform/sources/mongodb-application.yaml

@@ -13,6 +13,8 @@ spec:
     - name: default
       dependsOn:
         - cozystack.networking
+        # cozystack-engine ships the ApplicationDefinition CRD rendered by the *-rd component below.
+        - cozystack.cozystack-engine
       libraries:
         - name: cozy-lib
           path: library/cozy-lib

packages/core/platform/sources/monitoring-application.yaml

@@ -16,6 +16,8 @@ spec:
         - cozystack.postgres-operator
         - cozystack.grafana-operator
         - cozystack.victoria-metrics-operator
+        # cozystack-engine ships the ApplicationDefinition CRD rendered by the *-rd component below.
+        - cozystack.cozystack-engine
       libraries:
         - name: cozy-lib
           path: library/cozy-lib

packages/core/platform/sources/nats-application.yaml

@@ -13,6 +13,8 @@ spec:
     - name: default
       dependsOn:
         - cozystack.networking
+        # cozystack-engine ships the ApplicationDefinition CRD rendered by the *-rd component below.
+        - cozystack.cozystack-engine
       libraries:
         - name: cozy-lib
           path: library/cozy-lib

packages/core/platform/sources/openbao-application.yaml

@@ -13,6 +13,8 @@ spec:
     - name: default
       dependsOn:
         - cozystack.networking
+        # cozystack-engine ships the ApplicationDefinition CRD rendered by the *-rd component below.
+        - cozystack.cozystack-engine
       libraries:
         - name: cozy-lib
           path: library/cozy-lib

packages/core/platform/sources/opensearch-application.yaml

@@ -14,6 +14,8 @@ spec:
     dependsOn:
     - cozystack.networking
     - cozystack.opensearch-operator
+    # cozystack-engine ships the ApplicationDefinition CRD rendered by the *-rd component below.
+    - cozystack.cozystack-engine
     libraries:
     - name: cozy-lib
       path: library/cozy-lib

packages/core/platform/sources/postgres-application.yaml

@@ -14,6 +14,8 @@ spec:
       dependsOn:
         - cozystack.networking
         - cozystack.postgres-operator
+        # cozystack-engine ships the ApplicationDefinition CRD rendered by the *-rd component below.
+        - cozystack.cozystack-engine
       libraries:
         - name: cozy-lib
           path: library/cozy-lib

packages/core/platform/sources/qdrant-application.yaml

@@ -13,6 +13,8 @@ spec:
     - name: default
       dependsOn:
         - cozystack.networking
+        # cozystack-engine ships the ApplicationDefinition CRD rendered by the *-rd component below.
+        - cozystack.cozystack-engine
       libraries:
         - name: cozy-lib
           path: library/cozy-lib

packages/core/platform/sources/rabbitmq-application.yaml

@@ -14,6 +14,8 @@ spec:
       dependsOn:
         - cozystack.networking
         - cozystack.rabbitmq-operator
+        # cozystack-engine ships the ApplicationDefinition CRD rendered by the *-rd component below.
+        - cozystack.cozystack-engine
       libraries:
         - name: cozy-lib
           path: library/cozy-lib

packages/core/platform/sources/redis-application.yaml

@@ -14,6 +14,8 @@ spec:
       dependsOn:
         - cozystack.networking
         - cozystack.redis-operator
+        # cozystack-engine ships the ApplicationDefinition CRD rendered by the *-rd component below.
+        - cozystack.cozystack-engine
       libraries:
         - name: cozy-lib
           path: library/cozy-lib

packages/core/platform/sources/seaweedfs-application.yaml

@@ -13,6 +13,8 @@ spec:
     - name: default
       dependsOn:
         - cozystack.networking
+        # cozystack-engine ships the ApplicationDefinition CRD rendered by the *-rd component below.
+        - cozystack.cozystack-engine
       libraries:
         - name: cozy-lib
           path: library/cozy-lib

packages/core/platform/sources/tcp-balancer-application.yaml

@@ -13,6 +13,8 @@ spec:
     - name: default
       dependsOn:
         - cozystack.networking
+        # cozystack-engine ships the ApplicationDefinition CRD rendered by the *-rd component below.
+        - cozystack.cozystack-engine
       libraries:
         - name: cozy-lib
           path: library/cozy-lib

packages/core/platform/sources/tenant-application.yaml

@@ -19,6 +19,8 @@ spec:
         - cozystack.info-application
         - cozystack.monitoring-application
         - cozystack.etcd-application
+        # cozystack-engine ships the ApplicationDefinition CRD rendered by the *-rd component below.
+        - cozystack.cozystack-engine
       libraries:
         - name: cozy-lib
           path: library/cozy-lib

packages/core/platform/sources/virtualprivatecloud-application.yaml

@@ -14,6 +14,8 @@ spec:
       dependsOn:
         - cozystack.networking
         - cozystack.multus
+        # cozystack-engine ships the ApplicationDefinition CRD rendered by the *-rd component below.
+        - cozystack.cozystack-engine
       libraries:
         - name: cozy-lib
           path: library/cozy-lib

packages/core/platform/sources/vm-disk-application.yaml

@@ -14,6 +14,8 @@ spec:
       dependsOn:
         - cozystack.networking
         - cozystack.kubevirt-cdi
+        # cozystack-engine ships the ApplicationDefinition CRD rendered by the *-rd component below.
+        - cozystack.cozystack-engine
       libraries:
         - name: cozy-lib
           path: library/cozy-lib

packages/core/platform/sources/vm-instance-application.yaml

@@ -14,6 +14,8 @@ spec:
       dependsOn:
         - cozystack.networking
         - cozystack.kubevirt
+        # cozystack-engine ships the ApplicationDefinition CRD rendered by the *-rd component below.
+        - cozystack.cozystack-engine
       libraries:
         - name: cozy-lib
           path: library/cozy-lib

packages/core/platform/sources/vpn-application.yaml

@@ -13,6 +13,8 @@ spec:
     - name: default
       dependsOn:
         - cozystack.networking
+        # cozystack-engine ships the ApplicationDefinition CRD rendered by the *-rd component below.
+        - cozystack.cozystack-engine
       libraries:
         - name: cozy-lib
           path: library/cozy-lib

packages/core/platform/tests/sources_backupstrategy_dependson_test.yaml

@@ -0,0 +1,34 @@
+suite: backupstrategy-controller waits for the tenant-root namespace and its CRDs
+# Regression guard for a fresh-install ordering race: backupstrategy-controller
+# creates the cozy-backups Bucket INTO the tenant-root namespace, which is
+# provisioned by cozystack-basics. Without the cozystack-basics edge the Bucket
+# patch races namespace creation and fails with `namespaces "tenant-root" not
+# found`, which then races the install script's wait deadline. The CRD edges
+# (backup-controller / bucket-application / objectstorage-controller / velero)
+# guard the `no matches for kind` race. All edges below must stay.
+templates:
+  - templates/sources.yaml
+release:
+  name: cozystack
+  namespace: cozy-system
+tests:
+  - it: backupstrategy-controller dependsOn cozystack-basics and the CRD providers
+    documentSelector:
+      path: metadata.name
+      value: cozystack.backupstrategy-controller
+    asserts:
+      - contains:
+          path: spec.variants[0].dependsOn
+          content: cozystack.cozystack-basics
+      - contains:
+          path: spec.variants[0].dependsOn
+          content: cozystack.backup-controller
+      - contains:
+          path: spec.variants[0].dependsOn
+          content: cozystack.bucket-application
+      - contains:
+          path: spec.variants[0].dependsOn
+          content: cozystack.objectstorage-controller
+      - contains:
+          path: spec.variants[0].dependsOn
+          content: cozystack.velero

packages/system/bucket/images/s3manager/Dockerfile

@@ -1,6 +1,6 @@
 # Source: https://github.com/cloudlena/s3manager/blob/main/Dockerfile
 
-FROM docker.io/library/golang:1 AS builder
+FROM docker.io/library/golang:1.26 AS builder
 
 ARG TARGETOS
 ARG TARGETARCH
@@ -12,7 +12,7 @@ RUN git apply /cozystack.patch
 RUN go mod tidy
 RUN GOOS=$TARGETOS GOARCH=$TARGETARCH CGO_ENABLED=0 go build -ldflags="-s -w" -a -installsuffix cgo -o bin/s3manager
 
-FROM docker.io/library/alpine:latest
+FROM docker.io/library/alpine:3.24
 WORKDIR /usr/src/app
 RUN addgroup -S s3manager && adduser -S s3manager -G s3manager
 RUN apk add --no-cache \

packages/system/dashboard/images/token-proxy/go.mod

@@ -1,23 +1,23 @@
 module token-proxy
 
-go 1.26.2
+go 1.26.4
 
 require (
 	github.com/gorilla/securecookie v1.1.2
-	github.com/lestrrat-go/httprc/v3 v3.0.5
-	github.com/lestrrat-go/jwx/v3 v3.1.0
+	github.com/lestrrat-go/httprc/v3 v3.0.6
+	github.com/lestrrat-go/jwx/v3 v3.1.1
 )
 
 require (
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 // indirect
 	github.com/goccy/go-json v0.10.6 // indirect
 	github.com/lestrrat-go/blackmagic v1.0.4 // indirect
-	github.com/lestrrat-go/dsig v1.2.1 // indirect
+	github.com/lestrrat-go/dsig v1.3.0 // indirect
 	github.com/lestrrat-go/dsig-secp256k1 v1.0.0 // indirect
 	github.com/lestrrat-go/httpcc v1.0.1 // indirect
 	github.com/lestrrat-go/option/v2 v2.0.0 // indirect
 	github.com/segmentio/asm v1.2.1 // indirect
 	github.com/valyala/fastjson v1.6.10 // indirect
-	golang.org/x/crypto v0.50.0 // indirect
-	golang.org/x/sys v0.43.0 // indirect
+	golang.org/x/crypto v0.53.0 // indirect
+	golang.org/x/sys v0.46.0 // indirect
 )

packages/system/dashboard/images/token-proxy/go.sum

@@ -11,16 +11,16 @@ github.com/gorilla/securecookie v1.1.2 h1:YCIWL56dvtr73r6715mJs5ZvhtnY73hBvEF8kX
 github.com/gorilla/securecookie v1.1.2/go.mod h1:NfCASbcHqRSY+3a8tlWJwsQap2VX5pwzwo4h3eOamfo=
 github.com/lestrrat-go/blackmagic v1.0.4 h1:IwQibdnf8l2KoO+qC3uT4OaTWsW7tuRQXy9TRN9QanA=
 github.com/lestrrat-go/blackmagic v1.0.4/go.mod h1:6AWFyKNNj0zEXQYfTMPfZrAXUWUfTIZ5ECEUEJaijtw=
-github.com/lestrrat-go/dsig v1.2.1 h1:MwxzZhE4+4fguHi+uDALKVlC3Cn+O1QU1Q/F8D7hVIc=
-github.com/lestrrat-go/dsig v1.2.1/go.mod h1:RD2eOaidyPvpc7IJQoO3Qq52RWdy8ZcJs8lrOnoa1Kc=
+github.com/lestrrat-go/dsig v1.3.0 h1:phjMOCXvYzhuIgn7Voe2rex8z166vGfxRxmqM25P9/Q=
+github.com/lestrrat-go/dsig v1.3.0/go.mod h1:RD2eOaidyPvpc7IJQoO3Qq52RWdy8ZcJs8lrOnoa1Kc=
 github.com/lestrrat-go/dsig-secp256k1 v1.0.0 h1:JpDe4Aybfl0soBvoVwjqDbp+9S1Y2OM7gcrVVMFPOzY=
 github.com/lestrrat-go/dsig-secp256k1 v1.0.0/go.mod h1:CxUgAhssb8FToqbL8NjSPoGQlnO4w3LG1P0qPWQm/NU=
 github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE=
 github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E=
-github.com/lestrrat-go/httprc/v3 v3.0.5 h1:S+Mb4L2I+bM6JGTibLmxExhyTOqnXjqx+zi9MoXw/TM=
-github.com/lestrrat-go/httprc/v3 v3.0.5/go.mod h1:mSMtkZW92Z98M5YoNNztbRGxbXHql7tSitCvaxvo9l0=
-github.com/lestrrat-go/jwx/v3 v3.1.0 h1:AyyLtxc0QM75F75JroWgt1phwC7X+wOb3XKhH7XBZWw=
-github.com/lestrrat-go/jwx/v3 v3.1.0/go.mod h1:uw/MN2M/Xiu4FhwcIwH11Zsh9JWx9SWzgALl7/uIEkU=
+github.com/lestrrat-go/httprc/v3 v3.0.6 h1:4FpLQ18KK/ypPbVU3NLWJNRvH3kcYiqKqWfKGqNWxxI=
+github.com/lestrrat-go/httprc/v3 v3.0.6/go.mod h1:mSMtkZW92Z98M5YoNNztbRGxbXHql7tSitCvaxvo9l0=
+github.com/lestrrat-go/jwx/v3 v3.1.1 h1:yd9AdPmZ4INnQ7k42IrzXYpnEG803+SrQ6hdMvzHJzw=
+github.com/lestrrat-go/jwx/v3 v3.1.1/go.mod h1:uw/MN2M/Xiu4FhwcIwH11Zsh9JWx9SWzgALl7/uIEkU=
 github.com/lestrrat-go/option/v2 v2.0.0 h1:XxrcaJESE1fokHy3FpaQ/cXW8ZsIdWcdFzzLOcID3Ss=
 github.com/lestrrat-go/option/v2 v2.0.0/go.mod h1:oSySsmzMoR0iRzCDCaUfsCzxQHUEuhOViQObyy7S6Vg=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -33,10 +33,10 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 github.com/valyala/fastjson v1.6.10 h1:/yjJg8jaVQdYR3arGxPE2X5z89xrlhS0eGXdv+ADTh4=
 github.com/valyala/fastjson v1.6.10/go.mod h1:e6FubmQouUNP73jtMLmcbxS6ydWIpOfhz34TSfO3JaE=
-golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
-golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
-golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
-golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=
+golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio=
+golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
+golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

packages/system/kamaji/images/kamaji/Dockerfile

@@ -1,7 +1,7 @@
 # Build the manager binary
 FROM golang:1.26 AS builder
 
-ARG VERSION=26.3.5-edge
+ARG VERSION=26.3.6-edge
 ARG TARGETOS
 ARG TARGETARCH
 
@@ -12,6 +12,13 @@ RUN curl -sSL https://github.com/clastix/kamaji/archive/refs/tags/${VERSION}.tar
 COPY patches /patches
 RUN git apply /patches/*.diff
 
+# Overlay patched Go modules. The kamaji edge tags up to 26.6.4 still pin
+# vulnerable golang.org/x/net (< 0.55.0), golang.org/x/crypto (< 0.52.0) and
+# go.opentelemetry.io/otel/sdk (< 1.43.0). These advisories ship inside the
+# compiled binary, so force the fixed versions before building.
+RUN go get golang.org/x/net@v0.55.0 golang.org/x/crypto@v0.52.0 go.opentelemetry.io/otel/sdk@v1.43.0 && \
+    go mod tidy
+
 RUN GOOS=$TARGETOS GOARCH=$TARGETARCH CGO_ENABLED=0 go build \
     -ldflags "-X github.com/clastix/kamaji/internal.GitRepo=$GIT_REPO -X github.com/clastix/kamaji/internal.GitTag=$GIT_LAST_TAG -X github.com/clastix/kamaji/internal.GitCommit=$GIT_HEAD_COMMIT -X github.com/clastix/kamaji/internal.GitDirty=$GIT_MODIFIED -X github.com/clastix/kamaji/internal.BuildTime=$BUILD_DATE" \
     -a -o kamaji main.go

packages/system/keycloak-operator/images/keycloak-operator/Dockerfile

@@ -11,7 +11,7 @@ RUN curl -sSL https://github.com/epam/edp-keycloak-operator/archive/${COMMIT_REF
 COPY patches /patches
 RUN git apply /patches/*.diff
 
-FROM --platform=$BUILDPLATFORM docker.io/golang:1.24 AS builder
+FROM --platform=$BUILDPLATFORM docker.io/golang:1.26 AS builder
 ARG TARGETOS
 ARG TARGETARCH
 

packages/system/keycloak/templates/sts.yaml

@@ -142,15 +142,23 @@ spec:
             - name: KC_DB
               value: "postgres"
             - name: KC_DB_URL_HOST
+              {{- if .Values.db.host }}
+              value: {{ .Values.db.host | quote }}
+              {{- else }}
               valueFrom:
                 secretKeyRef:
                   name: keycloak-db-app
                   key: "host"
+              {{- end }}
             - name: KC_DB_URL_PORT
+              {{- if .Values.db.port }}
+              value: {{ .Values.db.port | quote }}
+              {{- else }}
               valueFrom:
                 secretKeyRef:
                   name: keycloak-db-app
                   key: "port"
+              {{- end }}
             - name: KC_DB_USERNAME
               valueFrom:
                 secretKeyRef:
@@ -166,6 +174,10 @@ spec:
                 secretKeyRef:
                   name: keycloak-db-app
                   key: "dbname"
+            {{- with .Values.db.urlProperties }}
+            - name: KC_DB_URL_PROPERTIES
+              value: {{ . | quote }}
+            {{- end }}
             - name: KC_FEATURES
               value: "docker"
             - name: KC_HOSTNAME
@@ -175,10 +187,15 @@ spec:
             {{- with .Values.extraEnv }}
             {{- toYaml . | nindent 12 }}
             {{- end }}
-          {{- if .Values.themes }}
+          {{- if or .Values.themes .Values.extraVolumeMounts }}
           volumeMounts:
+            {{- if .Values.themes }}
             - name: themes
               mountPath: /opt/keycloak/themes
+            {{- end }}
+            {{- with .Values.extraVolumeMounts }}
+            {{- toYaml . | nindent 12 }}
+            {{- end }}
           {{- end }}
           ports:
             - name: http
@@ -207,10 +224,15 @@ spec:
             periodSeconds: 10
             timeoutSeconds: 5
             failureThreshold: 3
-      {{- if .Values.themes }}
+      {{- if or .Values.themes .Values.extraVolumes }}
       volumes:
+        {{- if .Values.themes }}
         - name: themes
           emptyDir:
             sizeLimit: 256Mi
+        {{- end }}
+        {{- with .Values.extraVolumes }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
       {{- end }}
       terminationGracePeriodSeconds: 60

packages/system/keycloak/values.yaml

@@ -39,3 +39,29 @@ extraEnv: []
 #     secretKeyRef:
 #       name: my-secret
 #       key: my-key
+
+# Database connection. By default Keycloak connects directly to the chart's
+# CNPG cluster using the keycloak-db-app secret. Override these to route the
+# connection through an external endpoint (e.g. a connection proxy such as
+# PgBouncer) without the chart needing to know about it.
+db:
+  # DB host. Empty = use the keycloak-db-app secret's "host" key.
+  host: ""
+  # DB port. Empty = use the keycloak-db-app secret's "port" key.
+  port: ""
+  # Extra JDBC URL properties, appended verbatim as KC_DB_URL_PROPERTIES, e.g.
+  #   "?sslmode=verify-full&prepareThreshold=0&sslrootcert=/etc/keycloak-db-ca/ca.crt"
+  # Empty = KC_DB_URL_PROPERTIES is not set.
+  urlProperties: ""
+
+# Extra volumes and volume mounts on the Keycloak container (in addition to
+# themes). Useful for mounting a CA so KC_DB_URL_PROPERTIES can reference an
+# sslrootcert when routing the DB connection through a proxy.
+extraVolumes: []
+# - name: keycloak-db-ca
+#   secret:
+#     secretName: keycloak-db-ca
+extraVolumeMounts: []
+# - name: keycloak-db-ca
+#   mountPath: /etc/keycloak-db-ca
+#   readOnly: true

packages/system/kilo/images/kilo/Dockerfile

@@ -1,5 +1,5 @@
 # Build the kg binary
-FROM --platform=$BUILDPLATFORM golang:1.25 AS builder
+FROM --platform=$BUILDPLATFORM golang:1.26 AS builder
 
 ARG VERSION=944dff5745d747c06555a964c2137b9a986ff6ec
 ARG TARGETOS

packages/system/kubeovn-webhook/.gitignore

@@ -0,0 +1,2 @@
+images/kubeovn-webhook/kube-ovn-webhook
+images/kubeovn-webhook/webhook

packages/system/kubeovn-webhook/images/kubeovn-webhook/go.mod

@@ -1,46 +1,58 @@
 module kube-ovn-webhook
 
-go 1.26.2
+go 1.26.4
 
 require (
 	github.com/mattbaird/jsonpatch v0.0.0-20240118010651-0ba75a80ca38
-	k8s.io/api v0.26.0
-	k8s.io/apimachinery v0.26.0
-	k8s.io/client-go v0.26.0
+	k8s.io/api v0.36.2
+	k8s.io/apimachinery v0.36.2
+	k8s.io/client-go v0.36.2
 )
 
 require (
-	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
-	github.com/go-logr/logr v1.2.3 // indirect
-	github.com/go-openapi/jsonpointer v0.19.5 // indirect
-	github.com/go-openapi/jsonreference v0.20.0 // indirect
-	github.com/go-openapi/swag v0.19.14 // indirect
-	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/google/gnostic v0.5.7-v3refs // indirect
-	github.com/google/go-cmp v0.7.0 // indirect
-	github.com/google/gofuzz v1.1.0 // indirect
-	github.com/josharian/intern v1.0.0 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/emicklei/go-restful/v3 v3.13.0 // indirect
+	github.com/evanphx/json-patch v0.5.2 // indirect
+	github.com/fxamacker/cbor/v2 v2.9.2 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-openapi/jsonpointer v0.23.1 // indirect
+	github.com/go-openapi/jsonreference v0.21.6 // indirect
+	github.com/go-openapi/swag v0.26.1 // indirect
+	github.com/go-openapi/swag/cmdutils v0.26.1 // indirect
+	github.com/go-openapi/swag/conv v0.26.1 // indirect
+	github.com/go-openapi/swag/fileutils v0.26.1 // indirect
+	github.com/go-openapi/swag/jsonname v0.26.1 // indirect
+	github.com/go-openapi/swag/jsonutils v0.26.1 // indirect
+	github.com/go-openapi/swag/loading v0.26.1 // indirect
+	github.com/go-openapi/swag/mangling v0.26.1 // indirect
+	github.com/go-openapi/swag/netutils v0.26.1 // indirect
+	github.com/go-openapi/swag/stringutils v0.26.1 // indirect
+	github.com/go-openapi/swag/typeutils v0.26.1 // indirect
+	github.com/go-openapi/swag/yamlutils v0.26.1 // indirect
+	github.com/google/gnostic-models v0.7.1 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/mailru/easyjson v0.7.6 // indirect
+	github.com/kr/text v0.2.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	golang.org/x/net v0.53.0 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
+	go.yaml.in/yaml/v2 v2.4.4 // indirect
+	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	golang.org/x/net v0.56.0 // indirect
 	golang.org/x/oauth2 v0.36.0 // indirect
-	golang.org/x/sys v0.43.0 // indirect
-	golang.org/x/term v0.42.0 // indirect
-	golang.org/x/text v0.36.0 // indirect
-	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
-	google.golang.org/protobuf v1.36.11 // indirect
+	golang.org/x/sys v0.46.0 // indirect
+	golang.org/x/term v0.44.0 // indirect
+	golang.org/x/text v0.38.0 // indirect
+	golang.org/x/time v0.15.0 // indirect
+	google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect
+	gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/klog/v2 v2.80.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect
-	k8s.io/utils v0.0.0-20221107191617-1a15be271d1d // indirect
-	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
-	sigs.k8s.io/yaml v1.3.0 // indirect
+	k8s.io/klog/v2 v2.140.0 // indirect
+	k8s.io/kube-openapi v0.0.0-20260603220949-865597e52e25 // indirect
+	k8s.io/utils v0.0.0-20260507154919-ff6756f316d2 // indirect
+	sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect
+	sigs.k8s.io/randfill v1.0.0 // indirect
+	sigs.k8s.io/structured-merge-diff/v6 v6.4.0 // indirect
+	sigs.k8s.io/yaml v1.6.0 // indirect
 )

packages/system/kubeovn-webhook/images/kubeovn-webhook/go.sum

@@ -1,209 +1,135 @@
-cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
-github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE=
-github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
-github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84=
-github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
-github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
-github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
-github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
-github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
-github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA=
-github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo=
-github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
-github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng=
-github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
-github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
-github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
-github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
-github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
-github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
-github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
-github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
-github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
-github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54=
-github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ=
-github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes=
+github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
+github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k=
+github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
+github.com/fxamacker/cbor/v2 v2.9.2 h1:X4Ksno9+x3cz0TZv69ec1hxP/+tymuR8PXQJyDwfh78=
+github.com/fxamacker/cbor/v2 v2.9.2/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
+github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
+github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-openapi/jsonpointer v0.23.1 h1:1HBACs7XIwR2RcmItfdSFlALhGbe6S92p0ry4d1GWg4=
+github.com/go-openapi/jsonpointer v0.23.1/go.mod h1:iWRmZTrGn7XwYhtPt/fvdSFj1OfNBngqRT2UG3BxSqY=
+github.com/go-openapi/jsonreference v0.21.6 h1:NZ5nGfnaM1n4I43Xjm1e5/M2GjOwQwndQz22uhxwD+Y=
+github.com/go-openapi/jsonreference v0.21.6/go.mod h1:xzbgtQ3ZbWxvET3AxdzCJlJt6vkovbf+IfSPJjD0tUY=
+github.com/go-openapi/swag v0.26.1 h1:l5sVEyVpwj+DDYeZyo7wQI/Ebn/mKYIyGB/pFwAfGoQ=
+github.com/go-openapi/swag v0.26.1/go.mod h1:yNY38BbIVthxbkDtq1UHBCGasBqjakW3lCR6ANzdBEw=
+github.com/go-openapi/swag/cmdutils v0.26.1 h1:f2iE1ijYaJ3nuu5PaEMx3zpEhzhZFgivCJObWEObLIQ=
+github.com/go-openapi/swag/cmdutils v0.26.1/go.mod h1:Sm1MVFMkF6guJJ+pQqHnQA3N0j9qALV3NxzDSv6bETM=
+github.com/go-openapi/swag/conv v0.26.1 h1:slr5FVkg9Wc3Y5zcwenD8Sd/PQ94b2I/QJI7N7KTBpg=
+github.com/go-openapi/swag/conv v0.26.1/go.mod h1:mvQXgPptZk9GTrFgGwWvT4q+dN+zQej9JfmGwnipz1A=
+github.com/go-openapi/swag/fileutils v0.26.1 h1:K1XCM2CGhfNsc6YDt6v7Q5+1e59rftYWdcu/isZhvFw=
+github.com/go-openapi/swag/fileutils v0.26.1/go.mod h1:mYUgxQAKX4ShS3qvvySx+/9yrlUnDhjiD1CalaQl8lQ=
+github.com/go-openapi/swag/jsonname v0.26.1 h1:VReupaV6WxlAsCn0e4DUfgV6bPmINnPpyJDLqSfNPcE=
+github.com/go-openapi/swag/jsonname v0.26.1/go.mod h1:OvdW6BoWoj33pTfi7x9vFrgmT+fk7aw0BRwvCE0YOuc=
+github.com/go-openapi/swag/jsonutils v0.26.1 h1:2hdBfFkHg+7Wrz2VsCbeyR6hzkRDs7AztnMR2u84yOY=
+github.com/go-openapi/swag/jsonutils v0.26.1/go.mod h1:U+RMJH3wa+6BRiphuRtIyI8fW9HPFqFQ4sHk2oRx0UQ=
+github.com/go-openapi/swag/jsonutils/fixtures_test v0.26.1 h1:1CD7NiLLb/TXl3tOnFYU4b+mNfb5rtgHkaA+q7RMYYQ=
+github.com/go-openapi/swag/jsonutils/fixtures_test v0.26.1/go.mod h1:ZWafc8nMdYzTE3uYY6W86f0n46+IF0g4uUyRhJw/kXc=
+github.com/go-openapi/swag/loading v0.26.1 h1:E9K4wqXeROlhjFQ13K9zMz6ojFGXIggGe+ad1odrK9w=
+github.com/go-openapi/swag/loading v0.26.1/go.mod h1:3qvRIlWzWdq1HvmldwmuJ2ohpcAryN6xVt2OTKd0/7E=
+github.com/go-openapi/swag/mangling v0.26.1 h1:gpYI4WuPKFJJVjV5cDLGlDVJhFIxYjQc7yN5eEb4CqM=
+github.com/go-openapi/swag/mangling v0.26.1/go.mod h1:POETDH01hqAdASXfw7ISEd9bCOE6xBHOt8NHmGZRmYM=
+github.com/go-openapi/swag/netutils v0.26.1 h1:BNctoc39WTAUMxyAs355fExOPzMZtPbZ0ZZ1Am2FR5M=
+github.com/go-openapi/swag/netutils v0.26.1/go.mod h1:y02vByhZhQPAVwOX+0KipXFZ/hUbk6G/Enhf5rGaOkQ=
+github.com/go-openapi/swag/stringutils v0.26.1 h1:f88uYyTso7TnHrKM/bUBsQ5e2wKf37cpgo6pvbzd9yU=
+github.com/go-openapi/swag/stringutils v0.26.1/go.mod h1:Sc6d3bU8fgk5AyZR8/8jEQ+Is/Ald+TD/IIggPN8UJk=
+github.com/go-openapi/swag/typeutils v0.26.1 h1:yg42FgMzRR6PVQ3M3qHz1s+Y6/P4HoJ3cBarXa3OVnU=
+github.com/go-openapi/swag/typeutils v0.26.1/go.mod h1:VfnV+oUtSP2vCSCn2aJgnr8OevUYemyIzzS1VOzS10o=
+github.com/go-openapi/swag/yamlutils v0.26.1 h1:0TSLK+lXs9vfIhAWzBeI/lOzEnIoot6WTCO1aAeWFTk=
+github.com/go-openapi/swag/yamlutils v0.26.1/go.mod h1:7W5b7PRX9MxwL7TjeG7H8HkyBGRsIDRObhyMWFgBI2M=
+github.com/go-openapi/testify/enable/yaml/v2 v2.5.1 h1:q9NtHwK4qHF7yZziBPvZyv7zWAIk8ok88Gh2mR6Jpc8=
+github.com/go-openapi/testify/enable/yaml/v2 v2.5.1/go.mod h1:JW0MXIotCYps/XsgJnG3a8Q7rE5xAiBwoOD5OfaIQBk=
+github.com/go-openapi/testify/v2 v2.5.1 h1:TMdhCaw8fUNraVSf3Omoob1dO/AzBfhtFAPW0an6sBo=
+github.com/go-openapi/testify/v2 v2.5.1/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw=
+github.com/google/gnostic-models v0.7.1 h1:SisTfuFKJSKM5CPZkffwi6coztzzeYUhc3v4yxLWH8c=
+github.com/google/gnostic-models v0.7.1/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
-github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
-github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
-github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
-github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
-github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mattbaird/jsonpatch v0.0.0-20240118010651-0ba75a80ca38 h1:hQWBtNqRYrI7CWIaUSXXtNKR90KzcUA5uiuxFVWw7sU=
 github.com/mattbaird/jsonpatch v0.0.0-20240118010651-0ba75a80ca38/go.mod h1:M1qoD/MqPgTZIk0EWKB38wE28ACRfVcn+cU08jyArI0=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8=
+github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
-github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
-github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
-github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs=
-github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo=
-github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys=
-github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
-github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
+github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
+github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
+github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
-github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
-golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
+github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
+go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
+go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
+go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
+golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
 golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
 golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
-golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
-golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY=
-golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
-golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
-golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44=
-golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
-google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
-google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
-google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
-google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
-google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
-google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
-google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
+golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
+golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc=
+golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y=
+golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
+golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
+golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
+golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
+google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI=
+google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
-gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo=
+gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
-gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-k8s.io/api v0.26.0 h1:IpPlZnxBpV1xl7TGk/X6lFtpgjgntCg8PJ+qrPHAC7I=
-k8s.io/api v0.26.0/go.mod h1:k6HDTaIFC8yn1i6pSClSqIwLABIcLV9l5Q4EcngKnQg=
-k8s.io/apimachinery v0.26.0 h1:1feANjElT7MvPqp0JT6F3Ss6TWDwmcjLypwoPpEf7zg=
-k8s.io/apimachinery v0.26.0/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74=
-k8s.io/client-go v0.26.0 h1:lT1D3OfO+wIi9UFolCrifbjUUgu7CpLca0AD8ghRLI8=
-k8s.io/client-go v0.26.0/go.mod h1:I2Sh57A79EQsDmn7F7ASpmru1cceh3ocVT9KlX2jEZg=
-k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4=
-k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
-k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E=
-k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4=
-k8s.io/utils v0.0.0-20221107191617-1a15be271d1d h1:0Smp/HP1OH4Rvhe+4B8nWGERtlqAGSftbSbbmm45oFs=
-k8s.io/utils v0.0.0-20221107191617-1a15be271d1d/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
-sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k=
-sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
-sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE=
-sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E=
-sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
-sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
+k8s.io/api v0.36.2 h1:TF6YDLIzKfccK7cq9YpTcGX8TJmEkHVRv78DM51fRYY=
+k8s.io/api v0.36.2/go.mod h1:F4LbMO4brjZYh7yFkXWhynSvtB7YauxV4c+HHkNRGNg=
+k8s.io/apimachinery v0.36.2 h1:0PE/W/WNy1UX61NLbXY5TMbJ6UwLL6E6lAPkYrKFxbQ=
+k8s.io/apimachinery v0.36.2/go.mod h1:fvf/HOLXq9RId0rnDIbN1OEBvHXdQbLMM8nu0LcBUf4=
+k8s.io/client-go v0.36.2 h1:bfgxmFKc9CgqsgX4xKLAAdmTQlWee7Ob/HlDOrJ5TBI=
+k8s.io/client-go v0.36.2/go.mod h1:1vgO4OAlfPnoLcb+Rze2GF5rAr14w8qjrYMoyXJzQj0=
+k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc=
+k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0=
+k8s.io/kube-openapi v0.0.0-20260603220949-865597e52e25 h1:mPMaPMpBij2V1Wv/fR+HW124vVGXXvOSS9ver/9yjWs=
+k8s.io/kube-openapi v0.0.0-20260603220949-865597e52e25/go.mod h1:V/QaCUYDa+0QpcHhVVc5l99Uz56wEMEXBSj9oCDkNDY=
+k8s.io/utils v0.0.0-20260507154919-ff6756f316d2 h1:wU4tMEhLGgIbLvXQb1cfN+EcM0wf7zC6CPF+C79jroc=
+k8s.io/utils v0.0.0-20260507154919-ff6756f316d2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk=
+sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg=
+sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=
+sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU=
+sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
+sigs.k8s.io/structured-merge-diff/v6 v6.4.0 h1:qmp2e3ZfFi1/jJbDGpD4mt3wyp6PE1NfKHCYLqgNQJo=
+sigs.k8s.io/structured-merge-diff/v6 v6.4.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE=
+sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs=
+sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4=

packages/system/kubeovn/Makefile

@@ -4,6 +4,9 @@ export NAMESPACE=cozy-$(NAME)
 include ../../../hack/common-envs.mk
 include ../../../hack/package.mk
 
+test:
+	helm unittest .
+
 update:
 	rm -rf charts values.yaml Chart.yaml
 	tag=$$(git ls-remote --tags --sort="v:refname" https://github.com/cozystack/kubeovn-chart | awk -F'[/^]' 'END{print $$3}') && \

packages/system/kubeovn/tests/ovs-ovn-resources_test.yaml

@@ -0,0 +1,37 @@
+suite: ovs-ovn container CPU resources
+
+# Pins the CPU headroom this package gives the ovs-ovn DaemonSet container.
+# ovn-controller runs inside that one container (alongside ovsdb-server and
+# ovs-vswitchd) and is CPU-bound while programming OpenFlow for newly created
+# logical ports; under a burst install it CFS-throttles against the container
+# CPU limit, lengthening flow-programming lag. The limit must stay at 4 cores
+# (not the upstream/legacy 2) and the request must keep a guaranteed share so
+# the controller keeps up under contention.
+
+templates:
+  - charts/kube-ovn/templates/ovsovn-ds.yaml
+
+release:
+  name: kubeovn
+  namespace: cozy-kubeovn
+
+tests:
+  - it: ovs-ovn container carries the raised CPU limit and a guaranteed request
+    asserts:
+      - isKind:
+          of: DaemonSet
+      - equal:
+          path: spec.template.spec.containers[0].name
+          value: openvswitch
+      - equal:
+          path: spec.template.spec.containers[0].resources.limits.cpu
+          value: 4
+      - equal:
+          path: spec.template.spec.containers[0].resources.requests.cpu
+          value: 100m
+
+  - it: ovs-ovn CPU limit is not the throttle-prone 2 cores
+    asserts:
+      - notEqual:
+          path: spec.template.spec.containers[0].resources.limits.cpu
+          value: 2

packages/system/kubeovn/values.yaml

@@ -25,11 +25,18 @@ kube-ovn:
       cpu: "3"
       memory: "4Gi"
   ovs-ovn:
+    # ovn-controller runs inside this container (alongside ovsdb-server and
+    # ovs-vswitchd) and is CPU-bound while programming OpenFlow for newly
+    # created logical ports. During a burst install (many pods created at
+    # once) it competes for CPU in this one container; too tight a limit makes
+    # it CFS-throttle, lengthening flow-programming lag that leaves fresh pods
+    # unreachable from their node until the flows land. Give it CPU headroom
+    # (limit) plus a guaranteed share (request) so it keeps up under burst.
     requests:
-      cpu: "10m"
+      cpu: "100m"
       memory: "50Mi"
     limits:
-      cpu: "2"
+      cpu: "4"
       memory: "1000Mi"
   kube-ovn-controller:
     requests:

packages/system/kubevirt-operator/Makefile

@@ -6,8 +6,8 @@ include ../../../hack/package.mk
 update:
 	rm -rf templates
 	mkdir templates
-	# v1.7.0 blocked by https://github.com/kubevirt/kubevirt/issues/16386
-	export RELEASE=v1.8.2 && \
+	# Skipping v1.7.x: live-migration regression, https://github.com/kubevirt/kubevirt/issues/16386
+	export RELEASE=v1.8.4 && \
 	wget https://github.com/kubevirt/kubevirt/releases/download/$${RELEASE}/kubevirt-operator.yaml -O templates/kubevirt-operator.yaml && \
 	sed -i 's/namespace: kubevirt/namespace: $(NAMESPACE)/g' templates/kubevirt-operator.yaml
 	awk -i inplace -v RS="---" '!/kind: Namespace/{printf "%s", $$0 RS}' templates/kubevirt-operator.yaml

packages/system/kubevirt-operator/templates/kubevirt-operator.yaml

@@ -8536,14 +8536,14 @@ spec:
         - virt-operator
         env:
         - name: VIRT_OPERATOR_IMAGE
-          value: quay.io/kubevirt/virt-operator:v1.8.2
+          value: quay.io/kubevirt/virt-operator:v1.8.4
         - name: WATCH_NAMESPACE
           valueFrom:
             fieldRef:
               fieldPath: metadata.annotations['olm.targetNamespaces']
         - name: KUBEVIRT_VERSION
-          value: v1.8.2
-        image: quay.io/kubevirt/virt-operator:v1.8.2
+          value: v1.8.4
+        image: quay.io/kubevirt/virt-operator:v1.8.4
         imagePullPolicy: IfNotPresent
         livenessProbe:
           httpGet:

packages/system/linstor/images/linstor-csi/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.25 AS builder
+FROM golang:1.26 AS builder
 
 ARG VERSION=v1.10.6
 ARG LINSTOR_WAIT_UNTIL_VERSION=v0.3.1

packages/system/linstor/templates/cluster.yaml

@@ -39,6 +39,16 @@ spec:
         containers:
         - name: linstor-controller
           image: {{ .Values.piraeusServer.image.repository }}:{{ .Values.piraeusServer.image.tag }}
+          # Guarantee CPU/memory so the JVM keeps answering its /health probe
+          # under node contention. Requests only — see values.yaml controller.*.
+          resources:
+            {{- toYaml .Values.controller.resources | nindent 12 }}
+          # Partial override: piraeus-operator merges podTemplate as a strategic
+          # merge patch, so only these timing fields change. The operator's
+          # httpGet (GET /health on the api port) is preserved as-is.
+          livenessProbe:
+            timeoutSeconds: {{ .Values.controller.livenessProbe.timeoutSeconds }}
+            failureThreshold: {{ .Values.controller.livenessProbe.failureThreshold }}
         - name: plunger
           image: {{ .Values.piraeusServer.image.repository }}:{{ .Values.piraeusServer.image.tag }}
           command:

packages/system/linstor/tests/cluster_test.yaml

@@ -0,0 +1,75 @@
+suite: LinstorCluster controller resources and liveness hardening
+
+# Pins the controller-pod surface this package adds on top of the
+# piraeus-operator defaults. The operator merges spec.controller.podTemplate
+# into the generated linstor-controller Deployment as a strategic merge patch
+# (containers merged by name, struct fields merged field-by-field), so:
+#   - resources.requests guarantee the JVM gets scheduled CPU/memory so it can
+#     answer GET /health under node CPU contention (no limits: avoid throttling
+#     / OOM-killing the JVM);
+#   - the livenessProbe override carries ONLY the timing fields (timeoutSeconds,
+#     failureThreshold) relaxed from the operator defaults (1s / 3) so a JVM GC
+#     pause or CPU contention does not trip a liveness kill. The httpGet target
+#     (GET /health on the api port) is intentionally NOT overridden — it is
+#     supplied by the operator, and re-stating it risks a wrong path/scheme.
+
+templates:
+  - templates/cluster.yaml
+
+tests:
+  - it: "linstor-controller container carries CPU and memory requests (no limits)"
+    asserts:
+      - isKind:
+          of: LinstorCluster
+      - equal:
+          path: spec.controller.podTemplate.spec.containers[0].name
+          value: linstor-controller
+      - equal:
+          path: spec.controller.podTemplate.spec.containers[0].resources.requests.cpu
+          value: 250m
+      - equal:
+          path: spec.controller.podTemplate.spec.containers[0].resources.requests.memory
+          value: 700Mi
+      - notExists:
+          path: spec.controller.podTemplate.spec.containers[0].resources.limits
+
+  - it: "linstor-controller liveness probe relaxes timing but leaves httpGet to the operator"
+    asserts:
+      - equal:
+          path: spec.controller.podTemplate.spec.containers[0].livenessProbe.timeoutSeconds
+          value: 5
+      - equal:
+          path: spec.controller.podTemplate.spec.containers[0].livenessProbe.failureThreshold
+          value: 6
+      - notExists:
+          path: spec.controller.podTemplate.spec.containers[0].livenessProbe.httpGet
+
+  - it: "resource and liveness knobs are values-driven"
+    set:
+      controller:
+        resources:
+          requests:
+            cpu: 500m
+            memory: 1Gi
+        livenessProbe:
+          timeoutSeconds: 10
+          failureThreshold: 9
+    asserts:
+      - equal:
+          path: spec.controller.podTemplate.spec.containers[0].resources.requests.cpu
+          value: 500m
+      - equal:
+          path: spec.controller.podTemplate.spec.containers[0].resources.requests.memory
+          value: 1Gi
+      - equal:
+          path: spec.controller.podTemplate.spec.containers[0].livenessProbe.timeoutSeconds
+          value: 10
+      - equal:
+          path: spec.controller.podTemplate.spec.containers[0].livenessProbe.failureThreshold
+          value: 9
+
+  - it: "controller image pin is preserved alongside the new fields"
+    asserts:
+      - matchRegex:
+          path: spec.controller.podTemplate.spec.containers[0].image
+          pattern: "^ghcr\\.io/cozystack/cozystack/piraeus-server:"

packages/system/linstor/values.yaml

@@ -14,3 +14,22 @@ linstorCSI:
   image:
     repository: ghcr.io/cozystack/cozystack/linstor-csi
     tag: v1.5.0@sha256:c60519d12aa8d078e4891d6e436c709ff53f2cc96457994390ca9424b6505903
+controller:
+  # Resource requests for the linstor-controller JVM. Requests (not limits) are
+  # set on purpose: a CPU request guarantees CPU shares so the JVM can answer its
+  # GET /health liveness probe under node CPU contention, while omitting limits
+  # avoids throttling the JVM (CPU limit) or OOM-killing it (memory limit). The
+  # memory value is a conservative starting estimate for the JVM heap + overhead;
+  # tune it to observed RSS.
+  resources:
+    requests:
+      cpu: 250m
+      memory: 700Mi
+  # Liveness-probe timing for the linstor-controller container. Only these timing
+  # fields are overridden; the httpGet target (GET /health on the api port) is
+  # supplied by piraeus-operator and intentionally left untouched. Relaxed from
+  # the operator defaults (timeoutSeconds 1, failureThreshold 3) so a JVM GC
+  # pause or CPU contention does not trip a kubelet liveness kill.
+  livenessProbe:
+    timeoutSeconds: 5
+    failureThreshold: 6

packages/system/monitoring-agents/tests/fluent-bit_probes_test.yaml

@@ -0,0 +1,53 @@
+suite: fluent-bit liveness/readiness probes carry load headroom
+
+# Regression guard for the fluent-bit crash loop on a loaded install. The
+# subchart default probes (initialDelaySeconds 0, timeoutSeconds 1,
+# failureThreshold 3) fire into a connection-refused :2020 before fluent-bit's
+# HTTP server binds — that bind happens only AFTER the kubernetes filter's
+# "testing connectivity with API server" step, which the kube-API can gate for
+# ~10s under a concurrent install. The aggressive defaults then liveness-kill
+# fluent-bit into a restart loop, timing out the HelmRelease. The umbrella
+# values relax both probes. Pin the relaxed timing AND the unchanged target
+# (httpGet path / on port http = :2020) — a wrong path/port would make the
+# probe always fail, strictly worse than the bug it fixes.
+release:
+  name: monitoring-agents
+  namespace: cozy-monitoring
+tests:
+  - it: liveness probe keeps :2020 target and gains load headroom
+    template: charts/fluent-bit/templates/daemonset.yaml
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.httpGet.path
+          value: /
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.httpGet.port
+          value: http
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.initialDelaySeconds
+          value: 15
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.periodSeconds
+          value: 10
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.timeoutSeconds
+          value: 5
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.failureThreshold
+          value: 6
+
+  - it: readiness probe keeps :2020 target and gains load headroom
+    template: charts/fluent-bit/templates/daemonset.yaml
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.httpGet.path
+          value: /
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.httpGet.port
+          value: http
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.timeoutSeconds
+          value: 5
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.failureThreshold
+          value: 6

packages/system/monitoring-agents/values.yaml

@@ -287,9 +287,28 @@ vmagent:
 fluent-bit:
   rbac:
     eventsAccess: true
+  # Relax probe timing to survive a loaded install. Fluent-bit's HTTP server on
+  # :2020 (the probe target) does not bind until AFTER the kubernetes filter's
+  # "testing connectivity with API server" step; under concurrent install the
+  # kube-API can be unreachable for ~10s, delaying the :2020 bind, and the
+  # single-threaded HTTP server can miss a 1s timeout while the engine is busy.
+  # The subchart defaults (initialDelaySeconds 0, timeoutSeconds 1,
+  # failureThreshold 3) then liveness-kill fluent-bit into a crash loop. httpGet
+  # stays on path / port http (:2020) — same target as the subchart defaults.
+  livenessProbe:
+    httpGet:
+      path: /
+      port: http
+    initialDelaySeconds: 15
+    periodSeconds: 10
+    timeoutSeconds: 5
+    failureThreshold: 6
   readinessProbe:
     httpGet:
       path: /
+      port: http
+    timeoutSeconds: 5
+    failureThreshold: 6
   daemonSetVolumes:
     - name: varlog
       hostPath:

packages/system/objectstorage-controller/images/objectstorage/Dockerfile

@@ -11,7 +11,7 @@ RUN curl -sSL https://github.com/kubernetes-sigs/container-object-storage-interf
 COPY patches /patches
 RUN git apply /patches/*.diff
 
-FROM --platform=$BUILDPLATFORM docker.io/golang:1.24 AS builder
+FROM --platform=$BUILDPLATFORM docker.io/golang:1.26 AS builder
 ARG TARGETOS
 ARG TARGETARCH
 

packages/system/opensearch-operator/Makefile

@@ -8,3 +8,4 @@ update:
 	helm repo add opensearch-operator https://opensearch-project.github.io/opensearch-k8s-operator/
 	helm repo update opensearch-operator
 	helm pull opensearch-operator/opensearch-operator --version 2.8.0 --untar --untardir charts
+	patch --no-backup-if-mismatch -p1 < patches/leaderElection.diff

packages/system/opensearch-operator/charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml

@@ -5,7 +5,14 @@ metadata:
     control-plane: controller-manager
   name: {{ include "opensearch-operator.fullname" . }}-controller-manager
 spec:
-  replicas: 1
+  replicas: {{ .Values.manager.replicaCount | default 1 }}
+  {{- if le (int (.Values.manager.replicaCount | default 1)) 1 }}
+  # On a single replica a RollingUpdate briefly runs two managers (maxSurge) during
+  # an upgrade; with leader election gated off they would both reconcile without lease
+  # coordination. Recreate closes that window at zero availability cost on one replica.
+  strategy:
+    type: Recreate
+  {{- end }}
   selector:
     matchLabels:
       control-plane: controller-manager
@@ -49,7 +56,9 @@ spec:
       - args:
         - --health-probe-bind-address=:8081
         - --metrics-bind-address=127.0.0.1:8080
+        {{- if gt (int (.Values.manager.replicaCount | default 1)) 1 }}
         - --leader-elect
+        {{- end }}
         {{- if .Values.manager.watchNamespace }}
         - --watch-namespace={{ .Values.manager.watchNamespace }}
         {{- end }}

packages/system/opensearch-operator/patches/leaderElection.diff

@@ -0,0 +1,29 @@
+diff --git a/charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml b/charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml
+--- a/charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml
++++ b/charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml
+@@ -5,7 +5,14 @@
+     control-plane: controller-manager
+   name: {{ include "opensearch-operator.fullname" . }}-controller-manager
+ spec:
+-  replicas: 1
++  replicas: {{ .Values.manager.replicaCount | default 1 }}
++  {{- if le (int (.Values.manager.replicaCount | default 1)) 1 }}
++  # On a single replica a RollingUpdate briefly runs two managers (maxSurge) during
++  # an upgrade; with leader election gated off they would both reconcile without lease
++  # coordination. Recreate closes that window at zero availability cost on one replica.
++  strategy:
++    type: Recreate
++  {{- end }}
+   selector:
+     matchLabels:
+       control-plane: controller-manager
+@@ -49,7 +56,9 @@
+       - args:
+         - --health-probe-bind-address=:8081
+         - --metrics-bind-address=127.0.0.1:8080
++        {{- if gt (int (.Values.manager.replicaCount | default 1)) 1 }}
+         - --leader-elect
++        {{- end }}
+         {{- if .Values.manager.watchNamespace }}
+         - --watch-namespace={{ .Values.manager.watchNamespace }}
+         {{- end }}

packages/system/opensearch-operator/tests/leader_election_test.yaml

@@ -0,0 +1,57 @@
+suite: opensearch-operator leader election is gated on replica count
+
+# The operator Deployment ships a single replica. Leader election on one replica
+# adds no HA value but lets a transient apiserver/etcd lease-renewal blip make the
+# manager self-terminate (controller-runtime os.Exit on "leader election lost"),
+# which surfaces at install time as a crashlooping Deployment marked Failed.
+# Gate --leader-elect on replica count: off for the default single replica, back
+# on automatically when an operator scales to more than one replica for real HA.
+release:
+  name: opensearch-operator
+  namespace: cozy-opensearch-operator
+tests:
+  - it: default single-replica install drops --leader-elect from the manager
+    template: charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml
+    documentSelector:
+      path: kind
+      value: Deployment
+    asserts:
+      - equal:
+          path: spec.replicas
+          value: 1
+      # Guard container ordering: kube-rbac-proxy is [0], manager is [1].
+      - equal:
+          path: spec.template.spec.containers[1].name
+          value: operator-controller-manager
+      - notContains:
+          path: spec.template.spec.containers[1].args
+          content: --leader-elect
+      # Recreate avoids two uncoordinated managers overlapping during an upgrade
+      # while leader election is gated off on the single replica.
+      - equal:
+          path: spec.strategy.type
+          value: Recreate
+
+  - it: scaling past one replica re-enables --leader-elect for real HA
+    template: charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml
+    documentSelector:
+      path: kind
+      value: Deployment
+    set:
+      opensearch-operator:
+        manager:
+          replicaCount: 2
+    asserts:
+      - equal:
+          path: spec.replicas
+          value: 2
+      - equal:
+          path: spec.template.spec.containers[1].name
+          value: operator-controller-manager
+      - contains:
+          path: spec.template.spec.containers[1].args
+          content: --leader-elect
+      # Multi-replica installs keep the default RollingUpdate (no explicit
+      # strategy) so rollouts stay available.
+      - notExists:
+          path: spec.strategy

packages/system/opensearch-operator/values.yaml

@@ -1,5 +1,11 @@
 opensearch-operator:
   manager:
+    # Number of operator replicas. Leader election is enabled automatically only
+    # when this is greater than 1. On a single replica leader election adds no HA
+    # value but lets a transient apiserver/etcd lease-renewal blip self-terminate
+    # the manager (controller-runtime exits on "leader election lost"), which
+    # surfaces at install time as a crashlooping, Failed Deployment.
+    replicaCount: 1
     # Enable cluster-wide mode to watch all namespaces
     watchNamespace: ""
   kubeRbacProxy:

packages/system/redis-operator/images/redis-operator/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.20 AS builder
+FROM golang:1.26 AS builder
 
 ARG VERSION=v1.3.0-rc1
 

packages/system/seaweedfs/tests/master_probe_timeout_test.yaml

@@ -0,0 +1,64 @@
+suite: seaweedfs master probe timeouts
+
+# Pins the master probe timeout overrides that keep master-0 alive during raft
+# bootstrap. The master's /cluster/status handler calls topo.Leader(), which
+# backs off up to ~20s (MaxElapsedTime=20s) while the node has no leader in
+# view — e.g. a master that was briefly the initial raft leader then was demoted.
+# The subchart default 10s probe timeout is shorter than that worst case, so such
+# a master is liveness-killed into a permanent CrashLoop and the master
+# StatefulSet never reaches full readiness. Both probes must allow more than the
+# 20s backoff. The overlay overrides only timeoutSeconds and relies on Helm's
+# deep-merge to keep every other subchart probe field, so this also guards that
+# the merge does not drop the surrounding probe configuration.
+
+templates:
+  - charts/seaweedfs/templates/master/master-statefulset.yaml
+
+release:
+  name: seaweedfs
+  namespace: cozy-seaweedfs
+
+tests:
+  - it: raises both master probe timeouts above the cluster-status backoff
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.timeoutSeconds
+          value: 30
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.timeoutSeconds
+          value: 30
+
+  - it: preserves the other master probe fields through the deep-merge
+    asserts:
+      # liveness: only timeoutSeconds is overridden, the rest survive
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.httpGet.path
+          value: /cluster/status
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.failureThreshold
+          value: 4
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.successThreshold
+          value: 1
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.initialDelaySeconds
+          value: 20
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.periodSeconds
+          value: 30
+      # readiness: only timeoutSeconds is overridden, the rest survive
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.httpGet.path
+          value: /cluster/status
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.failureThreshold
+          value: 100
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.successThreshold
+          value: 2
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.initialDelaySeconds
+          value: 10
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.periodSeconds
+          value: 45

packages/system/seaweedfs/values.yaml

@@ -32,6 +32,19 @@ seaweedfs:
       type: "emptyDir"
     logs:
       type: ""
+    # The master's /cluster/status handler calls topo.Leader(), which backs off
+    # up to ~20s (MaxElapsedTime=20s) while the node has no leader in view — e.g.
+    # a master that was briefly the initial raft leader then was demoted during
+    # bootstrap. The default 10s probe timeout is shorter than that worst case, so
+    # such a master is liveness-killed every ~137s into a permanent CrashLoop and
+    # the master StatefulSet never reaches full readiness, timing out the install.
+    # Both probes must allow more than the 20s backoff (the volume component in
+    # this chart already uses 30s). Override only timeoutSeconds; Helm deep-merges
+    # these onto the subchart defaults, keeping enabled/path/initialDelay/etc.
+    livenessProbe:
+      timeoutSeconds: 30
+    readinessProbe:
+      timeoutSeconds: 30
   volume:
     replicas: 2
     resizeHook:

pkg/registry/core/tenantnamespace/rest.go

@@ -139,6 +139,11 @@ func (r *REST) Get(
 	ns := &corev1.Namespace{}
 	err = r.c.Get(ctx, types.NamespacedName{Namespace: "", Name: name}, ns, &client.GetOptions{Raw: opts})
 	if err != nil {
+		// Repackage NotFound so the Status reports this resource, not the
+		// backing Namespace one.
+		if apierrors.IsNotFound(err) {
+			return nil, apierrors.NewNotFound(r.gvr.GroupResource(), name)
+		}
 		return nil, err
 	}
 
@@ -437,7 +442,7 @@ func (r *REST) filterAccessible(
 ) ([]string, error) {
 	u, ok := request.UserFrom(ctx)
 	if !ok {
-		return []string{}, fmt.Errorf("user missing in context")
+		return nil, apierrors.NewUnauthorized("user missing in context")
 	}
 	groups := make(map[string]struct{})
 	for _, group := range u.GetGroups() {
@@ -456,7 +461,7 @@ func (r *REST) filterAccessible(
 	rbs := &rbacv1.RoleBindingList{}
 	err := r.c.List(ctx, rbs)
 	if err != nil {
-		return []string{}, fmt.Errorf("failed to list rolebindings: %w", err)
+		return nil, apierrors.NewInternalError(fmt.Errorf("failed to list rolebindings: %w", err))
 	}
 	allowedNameSet := make(map[string]struct{})
 	for i := range rbs.Items {
@@ -491,7 +496,7 @@ func (r *REST) hasAccessToNamespace(
 ) (bool, error) {
 	u, ok := request.UserFrom(ctx)
 	if !ok {
-		return false, fmt.Errorf("user missing in context")
+		return false, apierrors.NewUnauthorized("user missing in context")
 	}
 	groups := make(map[string]struct{})
 	for _, group := range u.GetGroups() {
@@ -520,7 +525,7 @@ func (r *REST) hasAccessToNamespaceForUser(
 	rbs := &rbacv1.RoleBindingList{}
 	err := r.c.List(ctx, rbs, client.InNamespace(namespace))
 	if err != nil {
-		return false, fmt.Errorf("failed to list rolebindings in %s: %w", namespace, err)
+		return false, apierrors.NewInternalError(fmt.Errorf("failed to list rolebindings in %s: %w", namespace, err))
 	}
 
 	// Check if user is in any RoleBinding subjects