fix(opensearch-operator): gate leader election on replica count

The operator Deployment ships a single replica but always passes
--leader-elect to the manager. On one replica leader election adds no HA
value: a transient apiserver/etcd lease-renewal blip makes
controller-runtime self-terminate (os.Exit on "leader election lost"),
which crashloops the manager and surfaces at install time as a
Deployment stuck in Failed.

Gate --leader-elect on the replica count via a manager.replicaCount
value (default 1). The single-replica install now rides out
control-plane latency instead of crashlooping, and scaling to more than
one replica re-enables leader election automatically for genuine HA.

Also set the Deployment strategy to Recreate on a single replica: with
leader election off, a default RollingUpdate would briefly run two
managers (maxSurge) during an operator upgrade, both reconciling without
lease coordination. Recreate closes that window at zero availability
cost, while installs scaled past one replica keep RollingUpdate.

The change to the vendored opensearch-operator 2.8.0 chart is carried in
patches/leaderElection.diff and re-applied by 'make update' so it
survives a chart refresh.

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>

packages/system/opensearch-operator/Makefile

@@ -8,3 +8,4 @@ update:
 	helm repo add opensearch-operator https://opensearch-project.github.io/opensearch-k8s-operator/
 	helm repo update opensearch-operator
 	helm pull opensearch-operator/opensearch-operator --version 2.8.0 --untar --untardir charts
+	patch --no-backup-if-mismatch -p1 < patches/leaderElection.diff

packages/system/opensearch-operator/charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml

@@ -5,7 +5,14 @@ metadata:
     control-plane: controller-manager
   name: {{ include "opensearch-operator.fullname" . }}-controller-manager
 spec:
-  replicas: 1
+  replicas: {{ .Values.manager.replicaCount | default 1 }}
+  {{- if le (int (.Values.manager.replicaCount | default 1)) 1 }}
+  # On a single replica a RollingUpdate briefly runs two managers (maxSurge) during
+  # an upgrade; with leader election gated off they would both reconcile without lease
+  # coordination. Recreate closes that window at zero availability cost on one replica.
+  strategy:
+    type: Recreate
+  {{- end }}
   selector:
     matchLabels:
       control-plane: controller-manager
@@ -49,7 +56,9 @@ spec:
       - args:
         - --health-probe-bind-address=:8081
         - --metrics-bind-address=127.0.0.1:8080
+        {{- if gt (int (.Values.manager.replicaCount | default 1)) 1 }}
         - --leader-elect
+        {{- end }}
         {{- if .Values.manager.watchNamespace }}
         - --watch-namespace={{ .Values.manager.watchNamespace }}
         {{- end }}

packages/system/opensearch-operator/patches/leaderElection.diff

@@ -0,0 +1,29 @@
+diff --git a/charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml b/charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml
+--- a/charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml
++++ b/charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml
+@@ -5,7 +5,14 @@
+     control-plane: controller-manager
+   name: {{ include "opensearch-operator.fullname" . }}-controller-manager
+ spec:
+-  replicas: 1
++  replicas: {{ .Values.manager.replicaCount | default 1 }}
++  {{- if le (int (.Values.manager.replicaCount | default 1)) 1 }}
++  # On a single replica a RollingUpdate briefly runs two managers (maxSurge) during
++  # an upgrade; with leader election gated off they would both reconcile without lease
++  # coordination. Recreate closes that window at zero availability cost on one replica.
++  strategy:
++    type: Recreate
++  {{- end }}
+   selector:
+     matchLabels:
+       control-plane: controller-manager
+@@ -49,7 +56,9 @@
+       - args:
+         - --health-probe-bind-address=:8081
+         - --metrics-bind-address=127.0.0.1:8080
++        {{- if gt (int (.Values.manager.replicaCount | default 1)) 1 }}
+         - --leader-elect
++        {{- end }}
+         {{- if .Values.manager.watchNamespace }}
+         - --watch-namespace={{ .Values.manager.watchNamespace }}
+         {{- end }}

packages/system/opensearch-operator/tests/leader_election_test.yaml

@@ -0,0 +1,57 @@
+suite: opensearch-operator leader election is gated on replica count
+
+# The operator Deployment ships a single replica. Leader election on one replica
+# adds no HA value but lets a transient apiserver/etcd lease-renewal blip make the
+# manager self-terminate (controller-runtime os.Exit on "leader election lost"),
+# which surfaces at install time as a crashlooping Deployment marked Failed.
+# Gate --leader-elect on replica count: off for the default single replica, back
+# on automatically when an operator scales to more than one replica for real HA.
+release:
+  name: opensearch-operator
+  namespace: cozy-opensearch-operator
+tests:
+  - it: default single-replica install drops --leader-elect from the manager
+    template: charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml
+    documentSelector:
+      path: kind
+      value: Deployment
+    asserts:
+      - equal:
+          path: spec.replicas
+          value: 1
+      # Guard container ordering: kube-rbac-proxy is [0], manager is [1].
+      - equal:
+          path: spec.template.spec.containers[1].name
+          value: operator-controller-manager
+      - notContains:
+          path: spec.template.spec.containers[1].args
+          content: --leader-elect
+      # Recreate avoids two uncoordinated managers overlapping during an upgrade
+      # while leader election is gated off on the single replica.
+      - equal:
+          path: spec.strategy.type
+          value: Recreate
+
+  - it: scaling past one replica re-enables --leader-elect for real HA
+    template: charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml
+    documentSelector:
+      path: kind
+      value: Deployment
+    set:
+      opensearch-operator:
+        manager:
+          replicaCount: 2
+    asserts:
+      - equal:
+          path: spec.replicas
+          value: 2
+      - equal:
+          path: spec.template.spec.containers[1].name
+          value: operator-controller-manager
+      - contains:
+          path: spec.template.spec.containers[1].args
+          content: --leader-elect
+      # Multi-replica installs keep the default RollingUpdate (no explicit
+      # strategy) so rollouts stay available.
+      - notExists:
+          path: spec.strategy

packages/system/opensearch-operator/values.yaml

@@ -1,5 +1,11 @@
 opensearch-operator:
   manager:
+    # Number of operator replicas. Leader election is enabled automatically only
+    # when this is greater than 1. On a single replica leader election adds no HA
+    # value but lets a transient apiserver/etcd lease-renewal blip self-terminate
+    # the manager (controller-runtime exits on "leader election lost"), which
+    # surfaces at install time as a crashlooping, Failed Deployment.
+    replicaCount: 1
     # Enable cluster-wide mode to watch all namespaces
     watchNamespace: ""
   kubeRbacProxy: