Merge remote-tracking branch 'origin/ray/fix-opensearch-leader-election' into ci/e2e-flake-hub

packages/system/opensearch-operator/Makefile

@@ -8,3 +8,4 @@ update:
 	helm repo add opensearch-operator https://opensearch-project.github.io/opensearch-k8s-operator/
 	helm repo update opensearch-operator
 	helm pull opensearch-operator/opensearch-operator --version 2.8.0 --untar --untardir charts
+	patch --no-backup-if-mismatch -p1 < patches/leaderElection.diff

packages/system/opensearch-operator/charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml

@@ -5,7 +5,14 @@ metadata:
     control-plane: controller-manager
   name: {{ include "opensearch-operator.fullname" . }}-controller-manager
 spec:
-  replicas: 1
+  replicas: {{ .Values.manager.replicaCount | default 1 }}
+  {{- if le (int (.Values.manager.replicaCount | default 1)) 1 }}
+  # On a single replica a RollingUpdate briefly runs two managers (maxSurge) during
+  # an upgrade; with leader election gated off they would both reconcile without lease
+  # coordination. Recreate closes that window at zero availability cost on one replica.
+  strategy:
+    type: Recreate
+  {{- end }}
   selector:
     matchLabels:
       control-plane: controller-manager
@@ -49,7 +56,9 @@ spec:
       - args:
         - --health-probe-bind-address=:8081
         - --metrics-bind-address=127.0.0.1:8080
+        {{- if gt (int (.Values.manager.replicaCount | default 1)) 1 }}
         - --leader-elect
+        {{- end }}
         {{- if .Values.manager.watchNamespace }}
         - --watch-namespace={{ .Values.manager.watchNamespace }}
         {{- end }}

packages/system/opensearch-operator/patches/leaderElection.diff

@@ -0,0 +1,29 @@
+diff --git a/charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml b/charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml
+--- a/charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml
++++ b/charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml
+@@ -5,7 +5,14 @@
+     control-plane: controller-manager
+   name: {{ include "opensearch-operator.fullname" . }}-controller-manager
+ spec:
+-  replicas: 1
++  replicas: {{ .Values.manager.replicaCount | default 1 }}
++  {{- if le (int (.Values.manager.replicaCount | default 1)) 1 }}
++  # On a single replica a RollingUpdate briefly runs two managers (maxSurge) during
++  # an upgrade; with leader election gated off they would both reconcile without lease
++  # coordination. Recreate closes that window at zero availability cost on one replica.
++  strategy:
++    type: Recreate
++  {{- end }}
+   selector:
+     matchLabels:
+       control-plane: controller-manager
+@@ -49,7 +56,9 @@
+       - args:
+         - --health-probe-bind-address=:8081
+         - --metrics-bind-address=127.0.0.1:8080
++        {{- if gt (int (.Values.manager.replicaCount | default 1)) 1 }}
+         - --leader-elect
++        {{- end }}
+         {{- if .Values.manager.watchNamespace }}
+         - --watch-namespace={{ .Values.manager.watchNamespace }}
+         {{- end }}

packages/system/opensearch-operator/tests/leader_election_test.yaml

@@ -0,0 +1,57 @@
+suite: opensearch-operator leader election is gated on replica count
+
+# The operator Deployment ships a single replica. Leader election on one replica
+# adds no HA value but lets a transient apiserver/etcd lease-renewal blip make the
+# manager self-terminate (controller-runtime os.Exit on "leader election lost"),
+# which surfaces at install time as a crashlooping Deployment marked Failed.
+# Gate --leader-elect on replica count: off for the default single replica, back
+# on automatically when an operator scales to more than one replica for real HA.
+release:
+  name: opensearch-operator
+  namespace: cozy-opensearch-operator
+tests:
+  - it: default single-replica install drops --leader-elect from the manager
+    template: charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml
+    documentSelector:
+      path: kind
+      value: Deployment
+    asserts:
+      - equal:
+          path: spec.replicas
+          value: 1
+      # Guard container ordering: kube-rbac-proxy is [0], manager is [1].
+      - equal:
+          path: spec.template.spec.containers[1].name
+          value: operator-controller-manager
+      - notContains:
+          path: spec.template.spec.containers[1].args
+          content: --leader-elect
+      # Recreate avoids two uncoordinated managers overlapping during an upgrade
+      # while leader election is gated off on the single replica.
+      - equal:
+          path: spec.strategy.type
+          value: Recreate
+
+  - it: scaling past one replica re-enables --leader-elect for real HA
+    template: charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml
+    documentSelector:
+      path: kind
+      value: Deployment
+    set:
+      opensearch-operator:
+        manager:
+          replicaCount: 2
+    asserts:
+      - equal:
+          path: spec.replicas
+          value: 2
+      - equal:
+          path: spec.template.spec.containers[1].name
+          value: operator-controller-manager
+      - contains:
+          path: spec.template.spec.containers[1].args
+          content: --leader-elect
+      # Multi-replica installs keep the default RollingUpdate (no explicit
+      # strategy) so rollouts stay available.
+      - notExists:
+          path: spec.strategy

packages/system/opensearch-operator/values.yaml

@@ -1,5 +1,11 @@
 opensearch-operator:
   manager:
+    # Number of operator replicas. Leader election is enabled automatically only
+    # when this is greater than 1. On a single replica leader election adds no HA
+    # value but lets a transient apiserver/etcd lease-renewal blip self-terminate
+    # the manager (controller-runtime exits on "leader election lost"), which
+    # surfaces at install time as a crashlooping, Failed Deployment.
+    replicaCount: 1
     # Enable cluster-wide mode to watch all namespaces
     watchNamespace: ""
   kubeRbacProxy: