From 711f09f8d174d320cd7198342bfc59b05bbc8ca1 Mon Sep 17 00:00:00 2001 From: Ivan Okhotnikov Date: Sat, 4 Jul 2026 15:57:49 +0300 Subject: [PATCH] fix(cert-manager): raise cainjector memory limit to unblock caBundle injection cainjector loads all ValidatingWebhookConfigurations, APIServices and CRDs into its informer caches at startup. On a full cozystack install that working set exceeds the 128Mi limit, so the single leader-elected cainjector Pod is OOMKilled during cache population and CrashLoops before it injects any caBundle. Webhook ValidatingWebhookConfigurations then keep an empty caBundle and every admission call fails with "x509: certificate signed by unknown authority" (failurePolicy: Fail), intermittently blocking Ingress creation and stalling tenant installs. Raise the cainjector memory limit to 512Mi (matching the controller) and the request to 128Mi, and add a helm-unittest that pins the sizing so a blanket resource change or a `make update` cannot silently lower it again. Signed-off-by: Ivan Okhotnikov (cherry picked from commit c3afc63cb3a84c8085e991223ebd3f18a70d5183) (cherry picked from commit 9f312d7a216c34724848142415d2de7c086a711e) --- packages/system/cert-manager/Makefile | 3 ++ .../tests/cainjector_resources_test.yaml | 30 +++++++++++++ packages/system/cert-manager/values.yaml | 42 +++++++++++++++++++ 3 files changed, 75 insertions(+) create mode 100644 packages/system/cert-manager/tests/cainjector_resources_test.yaml diff --git a/packages/system/cert-manager/Makefile b/packages/system/cert-manager/Makefile index c3ff574b7..a0885bf9a 100644 --- a/packages/system/cert-manager/Makefile +++ b/packages/system/cert-manager/Makefile @@ -3,6 +3,9 @@ export NAMESPACE=cozy-$(NAME) include ../../../hack/package.mk +test: + helm unittest . + update: rm -rf charts helm repo add jetstack https://charts.jetstack.io diff --git a/packages/system/cert-manager/tests/cainjector_resources_test.yaml b/packages/system/cert-manager/tests/cainjector_resources_test.yaml new file mode 100644 index 000000000..5dab84b0d --- /dev/null +++ b/packages/system/cert-manager/tests/cainjector_resources_test.yaml @@ -0,0 +1,30 @@ +suite: cert-manager cainjector resources + +# Pins the cainjector memory sizing that keeps CA injection working on a full +# cozystack install. cainjector loads all ValidatingWebhookConfigurations, +# APIServices and CRDs into its informer caches at startup; on a large cluster +# that working set exceeds the old 128Mi limit, so the single leader-elected +# cainjector Pod is OOMKilled during cache population and CrashLoops before it +# ever injects a caBundle. Every webhook then keeps an empty caBundle and +# admission calls fail with "x509: certificate signed by unknown authority" +# (failurePolicy: Fail). This guards the override in values.yaml against a +# blanket resource change or a `make update` silently lowering it again. + +templates: + - charts/cert-manager/templates/cainjector-deployment.yaml + +release: + name: cert-manager + namespace: cozy-cert-manager + +tests: + - it: sizes the cainjector memory limit above the OOM floor + asserts: + - isKind: + of: Deployment + - equal: + path: spec.template.spec.containers[0].resources.limits.memory + value: 512Mi + - equal: + path: spec.template.spec.containers[0].resources.requests.memory + value: 128Mi diff --git a/packages/system/cert-manager/values.yaml b/packages/system/cert-manager/values.yaml index e69de29bb..2ce6734e5 100644 --- a/packages/system/cert-manager/values.yaml +++ b/packages/system/cert-manager/values.yaml @@ -0,0 +1,42 @@ +cert-manager: + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 50m + memory: 128Mi + webhook: + resources: + limits: + cpu: 200m + memory: 128Mi + requests: + cpu: 10m + memory: 32Mi + cainjector: + # cainjector loads all ValidatingWebhookConfigurations, APIServices and CRDs + # into its informer caches at startup. On a full cozystack install that + # working set exceeds 128Mi, so cainjector is OOMKilled during cache + # population and CrashLoops before it ever injects a caBundle. Because it + # runs as a single leader-elected replica, that stalls CA injection for the + # whole cluster: webhook ValidatingWebhookConfigurations keep an empty + # caBundle and every admission call fails with + # "x509: certificate signed by unknown authority" (failurePolicy: Fail). + # Size the limit to the controller's 512Mi and lift the request off the + # OOM floor so injection completes deterministically. + resources: + limits: + cpu: 200m + memory: 512Mi + requests: + cpu: 10m + memory: 128Mi + startupapicheck: + resources: + limits: + cpu: 100m + memory: 64Mi + requests: + cpu: 10m + memory: 16Mi