From 291b1fa3c042ddc90c37b531211ebe23657b1da4 Mon Sep 17 00:00:00 2001 From: Yasar Rahman Date: Tue, 13 Oct 2026 19:20:34 +0500 Subject: [PATCH 1/2] fix(vpa): shorten the admission webhook timeout to 5s The VPA admission controller registers its Pod CREATE webhook in every namespace with failurePolicy Ignore and a 30s timeout. When its pod is down and the Service IP blackholes (a cold start after a full cluster shutdown, before the CNI is back), each Pod creation waits the full 30s here on top of the other webhooks and fails with "context deadline exceeded". That includes the CNI's own DaemonSet pods, so the cluster cannot recover without deleting the webhook configuration by hand. A mutation takes milliseconds; 5s keeps slow responses working and lets Ignore actually ignore. Uses the admission controller's documented --webhook-timeout-seconds flag through the chart's extraArgs. Refs #3125 Assisted-by: LLM Signed-off-by: Yasar Rahman --- packages/system/vertical-pod-autoscaler/values.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/system/vertical-pod-autoscaler/values.yaml b/packages/system/vertical-pod-autoscaler/values.yaml index 46678903e..b870049be 100644 --- a/packages/system/vertical-pod-autoscaler/values.yaml +++ b/packages/system/vertical-pod-autoscaler/values.yaml @@ -37,6 +37,15 @@ vertical-pod-autoscaler: prometheus-cadvisor-job-name: cadvisor admissionController: + extraArgs: + # The admission controller registers its mutating webhook for Pod CREATE in every + # namespace with failurePolicy Ignore and, by default, a 30s timeout. When its pod is + # down and the Service IP blackholes (e.g. a cold start after a full cluster shutdown, + # before the CNI is back), every Pod creation waits the full 30s here on top of the + # other webhooks and fails with "context deadline exceeded" - including the CNI's own + # DaemonSet pods, so the cluster cannot recover by itself (#3125). A mutation takes + # milliseconds; 5s keeps a slow response working and lets Ignore actually ignore. + webhook-timeout-seconds: 5 resources: limits: memory: 256Mi From d1b472319a24e95ad9b844151a5d41aa29222ff7 Mon Sep 17 00:00:00 2001 From: Yasar Rahman Date: Tue, 13 Oct 2026 19:45:30 +0500 Subject: [PATCH 2/2] fix(vpa): explain that the webhook waits add up within one request failurePolicy Ignore applies only after the timeout; the Pod CREATE webhooks are called sequentially within one request, so their waits accumulate and can pass the request deadline. Assisted-by: LLM Signed-off-by: Yasar Rahman --- .../system/vertical-pod-autoscaler/values.yaml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/system/vertical-pod-autoscaler/values.yaml b/packages/system/vertical-pod-autoscaler/values.yaml index b870049be..a828ee95b 100644 --- a/packages/system/vertical-pod-autoscaler/values.yaml +++ b/packages/system/vertical-pod-autoscaler/values.yaml @@ -39,12 +39,14 @@ vertical-pod-autoscaler: admissionController: extraArgs: # The admission controller registers its mutating webhook for Pod CREATE in every - # namespace with failurePolicy Ignore and, by default, a 30s timeout. When its pod is - # down and the Service IP blackholes (e.g. a cold start after a full cluster shutdown, - # before the CNI is back), every Pod creation waits the full 30s here on top of the - # other webhooks and fails with "context deadline exceeded" - including the CNI's own - # DaemonSet pods, so the cluster cannot recover by itself (#3125). A mutation takes - # milliseconds; 5s keeps a slow response working and lets Ignore actually ignore. + # namespace with failurePolicy Ignore and, by default, a 30s timeout. Ignore only + # applies after the timeout: when its pod is down and the Service IP blackholes (e.g. a + # cold start after a full cluster shutdown, before the CNI is back), the API server + # still waits the full 30s. The Pod CREATE webhooks are called one after another within + # one request, so their waits add up; past the request's deadline the creation fails + # with "context deadline exceeded", including for the CNI's own DaemonSet pods (#3125). + # A mutation takes milliseconds; 5s keeps slow responses working and keeps the total + # wait well inside the deadline. webhook-timeout-seconds: 5 resources: limits: