diff --git a/packages/system/opensearch-operator/Makefile b/packages/system/opensearch-operator/Makefile index 365224956..ea0e2bf9d 100644 --- a/packages/system/opensearch-operator/Makefile +++ b/packages/system/opensearch-operator/Makefile @@ -8,3 +8,4 @@ update: helm repo add opensearch-operator https://opensearch-project.github.io/opensearch-k8s-operator/ helm repo update opensearch-operator helm pull opensearch-operator/opensearch-operator --version 2.8.0 --untar --untardir charts + patch --no-backup-if-mismatch -p1 < patches/leaderElection.diff diff --git a/packages/system/opensearch-operator/charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml b/packages/system/opensearch-operator/charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml index 114e6b200..fb58d6527 100644 --- a/packages/system/opensearch-operator/charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml +++ b/packages/system/opensearch-operator/charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml @@ -5,7 +5,14 @@ metadata: control-plane: controller-manager name: {{ include "opensearch-operator.fullname" . }}-controller-manager spec: - replicas: 1 + replicas: {{ .Values.manager.replicaCount | default 1 }} + {{- if le (int (.Values.manager.replicaCount | default 1)) 1 }} + # On a single replica a RollingUpdate briefly runs two managers (maxSurge) during + # an upgrade; with leader election gated off they would both reconcile without lease + # coordination. Recreate closes that window at zero availability cost on one replica. + strategy: + type: Recreate + {{- end }} selector: matchLabels: control-plane: controller-manager @@ -49,7 +56,9 @@ spec: - args: - --health-probe-bind-address=:8081 - --metrics-bind-address=127.0.0.1:8080 + {{- if gt (int (.Values.manager.replicaCount | default 1)) 1 }} - --leader-elect + {{- end }} {{- if .Values.manager.watchNamespace }} - --watch-namespace={{ .Values.manager.watchNamespace }} {{- end }} diff --git a/packages/system/opensearch-operator/patches/leaderElection.diff b/packages/system/opensearch-operator/patches/leaderElection.diff new file mode 100644 index 000000000..57dc6cff8 --- /dev/null +++ b/packages/system/opensearch-operator/patches/leaderElection.diff @@ -0,0 +1,29 @@ +diff --git a/charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml b/charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml +--- a/charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml ++++ b/charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml +@@ -5,7 +5,14 @@ + control-plane: controller-manager + name: {{ include "opensearch-operator.fullname" . }}-controller-manager + spec: +- replicas: 1 ++ replicas: {{ .Values.manager.replicaCount | default 1 }} ++ {{- if le (int (.Values.manager.replicaCount | default 1)) 1 }} ++ # On a single replica a RollingUpdate briefly runs two managers (maxSurge) during ++ # an upgrade; with leader election gated off they would both reconcile without lease ++ # coordination. Recreate closes that window at zero availability cost on one replica. ++ strategy: ++ type: Recreate ++ {{- end }} + selector: + matchLabels: + control-plane: controller-manager +@@ -49,7 +56,9 @@ + - args: + - --health-probe-bind-address=:8081 + - --metrics-bind-address=127.0.0.1:8080 ++ {{- if gt (int (.Values.manager.replicaCount | default 1)) 1 }} + - --leader-elect ++ {{- end }} + {{- if .Values.manager.watchNamespace }} + - --watch-namespace={{ .Values.manager.watchNamespace }} + {{- end }} diff --git a/packages/system/opensearch-operator/tests/leader_election_test.yaml b/packages/system/opensearch-operator/tests/leader_election_test.yaml new file mode 100644 index 000000000..663394b75 --- /dev/null +++ b/packages/system/opensearch-operator/tests/leader_election_test.yaml @@ -0,0 +1,57 @@ +suite: opensearch-operator leader election is gated on replica count + +# The operator Deployment ships a single replica. Leader election on one replica +# adds no HA value but lets a transient apiserver/etcd lease-renewal blip make the +# manager self-terminate (controller-runtime os.Exit on "leader election lost"), +# which surfaces at install time as a crashlooping Deployment marked Failed. +# Gate --leader-elect on replica count: off for the default single replica, back +# on automatically when an operator scales to more than one replica for real HA. +release: + name: opensearch-operator + namespace: cozy-opensearch-operator +tests: + - it: default single-replica install drops --leader-elect from the manager + template: charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml + documentSelector: + path: kind + value: Deployment + asserts: + - equal: + path: spec.replicas + value: 1 + # Guard container ordering: kube-rbac-proxy is [0], manager is [1]. + - equal: + path: spec.template.spec.containers[1].name + value: operator-controller-manager + - notContains: + path: spec.template.spec.containers[1].args + content: --leader-elect + # Recreate avoids two uncoordinated managers overlapping during an upgrade + # while leader election is gated off on the single replica. + - equal: + path: spec.strategy.type + value: Recreate + + - it: scaling past one replica re-enables --leader-elect for real HA + template: charts/opensearch-operator/templates/opensearch-operator-controller-manager-deployment.yaml + documentSelector: + path: kind + value: Deployment + set: + opensearch-operator: + manager: + replicaCount: 2 + asserts: + - equal: + path: spec.replicas + value: 2 + - equal: + path: spec.template.spec.containers[1].name + value: operator-controller-manager + - contains: + path: spec.template.spec.containers[1].args + content: --leader-elect + # Multi-replica installs keep the default RollingUpdate (no explicit + # strategy) so rollouts stay available. + - notExists: + path: spec.strategy diff --git a/packages/system/opensearch-operator/values.yaml b/packages/system/opensearch-operator/values.yaml index 137a7f5b3..956cc6894 100644 --- a/packages/system/opensearch-operator/values.yaml +++ b/packages/system/opensearch-operator/values.yaml @@ -1,5 +1,11 @@ opensearch-operator: manager: + # Number of operator replicas. Leader election is enabled automatically only + # when this is greater than 1. On a single replica leader election adds no HA + # value but lets a transient apiserver/etcd lease-renewal blip self-terminate + # the manager (controller-runtime exits on "leader election lost"), which + # surfaces at install time as a crashlooping, Failed Deployment. + replicaCount: 1 # Enable cluster-wide mode to watch all namespaces watchNamespace: "" kubeRbacProxy: