From db797b1dd34b92bda71d96ea17fd7d07ddda6c6c Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Wed, 24 Jun 2026 22:27:26 +0300 Subject: [PATCH] fix(opensearch-operator): gate leader election on replica count The operator Deployment ships a single replica but always passes --leader-elect to the manager. On one replica leader election adds no HA value: a transient apiserver/etcd lease-renewal blip makes controller-runtime self-terminate (os.Exit on "leader election lost"), which crashloops the manager and surfaces at install time as a Deployment stuck in Failed. Gate --leader-elect on the replica count via a manager.replicaCount value (default 1). The single-replica install now rides out control-plane latency instead of crashlooping, and scaling to more than one replica re-enables leader election automatically for genuine HA. Also set the Deployment strategy to Recreate on a single replica: with leader election off, a default RollingUpdate would briefly run two managers (maxSurge) during an operator upgrade, both reconciling without lease coordination. Recreate closes that window at zero availability cost, while installs scaled past one replica keep RollingUpdate. The change to the vendored opensearch-operator 2.8.0 chart is carried in patches/leaderElection.diff and re-applied by 'make update' so it survives a chart refresh. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- packages/system/opensearch-operator/Makefile | 1 + ...perator-controller-manager-deployment.yaml | 11 +++- .../patches/leaderElection.diff | 29 ++++++++++ .../tests/leader_election_test.yaml | 57 +++++++++++++++++++ .../system/opensearch-operator/values.yaml | 6 ++ 5 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 packages/system/opensearch-operator/patches/leaderElection.diff create mode 100644 packages/system/opensearch-operator/tests/leader_election_test.yaml diff --git a/packages/system/opensearch-operator/Makefile b/packages/system/opensearch-operator/Makefile index 365224956..ea0e2bf9d 100644 --- a/packages/system/opensearch-operator/Makefile +++ b/packages/system/opensearch-operator/Makefile @@ -8,3 +8,4 @@ update: helm repo add opensearch-operator https://opensearch-project.github.io/opensearch-k8s-operator/ helm repo update opensearch-operator helm pull opensearch-operator/opensearch-operator --version 2.8.0 --untar --untardir charts + patch --no-backup-if-mismatch -p1 < patches/leaderElection.diff diff --git a/packages/system/opensearch-operator/charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml b/packages/system/opensearch-operator/charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml index 114e6b200..fb58d6527 100644 --- a/packages/system/opensearch-operator/charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml +++ b/packages/system/opensearch-operator/charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml @@ -5,7 +5,14 @@ metadata: control-plane: controller-manager name: {{ include "opensearch-operator.fullname" . }}-controller-manager spec: - replicas: 1 + replicas: {{ .Values.manager.replicaCount | default 1 }} + {{- if le (int (.Values.manager.replicaCount | default 1)) 1 }} + # On a single replica a RollingUpdate briefly runs two managers (maxSurge) during + # an upgrade; with leader election gated off they would both reconcile without lease + # coordination. Recreate closes that window at zero availability cost on one replica. + strategy: + type: Recreate + {{- end }} selector: matchLabels: control-plane: controller-manager @@ -49,7 +56,9 @@ spec: - args: - --health-probe-bind-address=:8081 - --metrics-bind-address=127.0.0.1:8080 + {{- if gt (int (.Values.manager.replicaCount | default 1)) 1 }} - --leader-elect + {{- end }} {{- if .Values.manager.watchNamespace }} - --watch-namespace={{ .Values.manager.watchNamespace }} {{- end }} diff --git a/packages/system/opensearch-operator/patches/leaderElection.diff b/packages/system/opensearch-operator/patches/leaderElection.diff new file mode 100644 index 000000000..57dc6cff8 --- /dev/null +++ b/packages/system/opensearch-operator/patches/leaderElection.diff @@ -0,0 +1,29 @@ +diff --git a/charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml b/charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml +--- a/charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml ++++ b/charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml +@@ -5,7 +5,14 @@ + control-plane: controller-manager + name: {{ include "opensearch-operator.fullname" . }}-controller-manager + spec: +- replicas: 1 ++ replicas: {{ .Values.manager.replicaCount | default 1 }} ++ {{- if le (int (.Values.manager.replicaCount | default 1)) 1 }} ++ # On a single replica a RollingUpdate briefly runs two managers (maxSurge) during ++ # an upgrade; with leader election gated off they would both reconcile without lease ++ # coordination. Recreate closes that window at zero availability cost on one replica. ++ strategy: ++ type: Recreate ++ {{- end }} + selector: + matchLabels: + control-plane: controller-manager +@@ -49,7 +56,9 @@ + - args: + - --health-probe-bind-address=:8081 + - --metrics-bind-address=127.0.0.1:8080 ++ {{- if gt (int (.Values.manager.replicaCount | default 1)) 1 }} + - --leader-elect ++ {{- end }} + {{- if .Values.manager.watchNamespace }} + - --watch-namespace={{ .Values.manager.watchNamespace }} + {{- end }} diff --git a/packages/system/opensearch-operator/tests/leader_election_test.yaml b/packages/system/opensearch-operator/tests/leader_election_test.yaml new file mode 100644 index 000000000..663394b75 --- /dev/null +++ b/packages/system/opensearch-operator/tests/leader_election_test.yaml @@ -0,0 +1,57 @@ +suite: opensearch-operator leader election is gated on replica count + +# The operator Deployment ships a single replica. Leader election on one replica +# adds no HA value but lets a transient apiserver/etcd lease-renewal blip make the +# manager self-terminate (controller-runtime os.Exit on "leader election lost"), +# which surfaces at install time as a crashlooping Deployment marked Failed. +# Gate --leader-elect on replica count: off for the default single replica, back +# on automatically when an operator scales to more than one replica for real HA. +release: + name: opensearch-operator + namespace: cozy-opensearch-operator +tests: + - it: default single-replica install drops --leader-elect from the manager + template: charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml + documentSelector: + path: kind + value: Deployment + asserts: + - equal: + path: spec.replicas + value: 1 + # Guard container ordering: kube-rbac-proxy is [0], manager is [1]. + - equal: + path: spec.template.spec.containers[1].name + value: operator-controller-manager + - notContains: + path: spec.template.spec.containers[1].args + content: --leader-elect + # Recreate avoids two uncoordinated managers overlapping during an upgrade + # while leader election is gated off on the single replica. + - equal: + path: spec.strategy.type + value: Recreate + + - it: scaling past one replica re-enables --leader-elect for real HA + template: charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml + documentSelector: + path: kind + value: Deployment + set: + opensearch-operator: + manager: + replicaCount: 2 + asserts: + - equal: + path: spec.replicas + value: 2 + - equal: + path: spec.template.spec.containers[1].name + value: operator-controller-manager + - contains: + path: spec.template.spec.containers[1].args + content: --leader-elect + # Multi-replica installs keep the default RollingUpdate (no explicit + # strategy) so rollouts stay available. + - notExists: + path: spec.strategy diff --git a/packages/system/opensearch-operator/values.yaml b/packages/system/opensearch-operator/values.yaml index 137a7f5b3..956cc6894 100644 --- a/packages/system/opensearch-operator/values.yaml +++ b/packages/system/opensearch-operator/values.yaml @@ -1,5 +1,11 @@ opensearch-operator: manager: + # Number of operator replicas. Leader election is enabled automatically only + # when this is greater than 1. On a single replica leader election adds no HA + # value but lets a transient apiserver/etcd lease-renewal blip self-terminate + # the manager (controller-runtime exits on "leader election lost"), which + # surfaces at install time as a crashlooping, Failed Deployment. + replicaCount: 1 # Enable cluster-wide mode to watch all namespaces watchNamespace: "" kubeRbacProxy: