diff --git a/packages/system/ingress-nginx/templates/admission-webhook-egress-policy.yaml b/packages/system/ingress-nginx/templates/admission-webhook-egress-policy.yaml new file mode 100644 index 000000000..a745249e4 --- /dev/null +++ b/packages/system/ingress-nginx/templates/admission-webhook-egress-policy.yaml @@ -0,0 +1,115 @@ +{{- $policy := .Values.admissionWebhookPolicy | default dict -}} +{{- $inc := index .Values "ingress-nginx" | default dict -}} +{{- $ingress := index .Subcharts "ingress-nginx" -}} +{{- $admissionWebhooks := dig "controller" "admissionWebhooks" dict $inc | default dict -}} +{{- if and (dig "enabled" true $policy) (dig "enabled" true $admissionWebhooks) (.Capabilities.APIVersions.Has "cilium.io/v2/CiliumClusterwideNetworkPolicy") }} +{{- $controllerName := include "ingress-nginx.name" $ingress -}} +{{- $controllerNamespace := include "ingress-nginx.namespace" $ingress -}} +{{- $controllerFullname := include "ingress-nginx.controller.fullname" $ingress -}} +{{- $webhookServiceName := printf "%s-admission" $controllerFullname -}} +{{- $webhookPort := dig "port" 8443 $admissionWebhooks -}} +{{- $webhookServicePort := dig "service" "servicePort" 443 $admissionWebhooks -}} +{{- $policyHash := printf "%s/%s" $controllerNamespace .Release.Name | sha256sum | trunc 12 -}} +--- +# The ingress-nginx admission webhook only needs to be reachable by the API +# server. Deny pod egress through its exact Service and directly to its exact +# endpoint identity while leaving every other destination and port unchanged. +# +# This is deliberately source-side. A target-side ingress policy with a +# cluster-wide fromEndpoints selector makes every namespace identity part of +# the webhook endpoint's policy. Namespace deletion can then regenerate that +# endpoint and transiently drop legitimate API-server webhook calls (#3983). +# Keeping the namespace-wide selector on source endpoints removes that +# production failure mode while preserving the same pod-to-webhook boundary. +# +# The first rule covers every workload outside kube-system. The second covers +# kube-system except the konnectivity-agent ServiceAccount: Kamaji-hosted API +# servers reach admission webhooks through that pod. The ServiceAccount label +# is a Cilium-derived identity label, so a workload outside kube-system cannot +# forge its way into the exemption with an ordinary pod label. +# +# The final rule keeps the external-world deny on the webhook endpoint. Its +# caller selector is a static entity rather than a cluster-wide endpoint set, +# so it does not couple target regeneration to namespace lifecycle events. +apiVersion: cilium.io/v2 +kind: CiliumClusterwideNetworkPolicy +metadata: + name: ingress-nginx-admission-webhook-{{ $policyHash }} +specs: +- endpointSelector: + matchExpressions: + - key: k8s:io.kubernetes.pod.namespace + operator: NotIn + values: + - kube-system + enableDefaultDeny: + ingress: false + egress: false + egressDeny: + - toServices: + - k8sService: + namespace: {{ $controllerNamespace | quote }} + serviceName: {{ $webhookServiceName | quote }} + toPorts: + - ports: + - port: {{ $webhookServicePort | quote }} + protocol: TCP + - toEndpoints: + - matchLabels: + "k8s:io.kubernetes.pod.namespace": {{ $controllerNamespace | quote }} + "k8s:app.kubernetes.io/name": {{ $controllerName | quote }} + "k8s:app.kubernetes.io/instance": {{ .Release.Name | quote }} + "k8s:app.kubernetes.io/component": controller + toPorts: + - ports: + - port: {{ $webhookPort | quote }} + protocol: TCP +- endpointSelector: + matchExpressions: + - key: k8s:io.kubernetes.pod.namespace + operator: In + values: + - kube-system + - key: io.cilium.k8s.policy.serviceaccount + operator: NotIn + values: + - konnectivity-agent + enableDefaultDeny: + ingress: false + egress: false + egressDeny: + - toServices: + - k8sService: + namespace: {{ $controllerNamespace | quote }} + serviceName: {{ $webhookServiceName | quote }} + toPorts: + - ports: + - port: {{ $webhookServicePort | quote }} + protocol: TCP + - toEndpoints: + - matchLabels: + "k8s:io.kubernetes.pod.namespace": {{ $controllerNamespace | quote }} + "k8s:app.kubernetes.io/name": {{ $controllerName | quote }} + "k8s:app.kubernetes.io/instance": {{ .Release.Name | quote }} + "k8s:app.kubernetes.io/component": controller + toPorts: + - ports: + - port: {{ $webhookPort | quote }} + protocol: TCP +- endpointSelector: + matchLabels: + "k8s:io.kubernetes.pod.namespace": {{ $controllerNamespace | quote }} + "k8s:app.kubernetes.io/name": {{ $controllerName | quote }} + "k8s:app.kubernetes.io/instance": {{ .Release.Name | quote }} + "k8s:app.kubernetes.io/component": controller + enableDefaultDeny: + ingress: false + egress: false + ingressDeny: + - fromEntities: + - world + toPorts: + - ports: + - port: {{ $webhookPort | quote }} + protocol: TCP +{{- end }} diff --git a/packages/system/ingress-nginx/templates/admission-webhook-networkpolicy.yaml b/packages/system/ingress-nginx/templates/admission-webhook-networkpolicy.yaml deleted file mode 100644 index b2ec50d93..000000000 --- a/packages/system/ingress-nginx/templates/admission-webhook-networkpolicy.yaml +++ /dev/null @@ -1,97 +0,0 @@ -{{- if .Values.admissionWebhookPolicy.enabled }} -{{- $inc := index .Values "ingress-nginx" | default dict }} -{{- $webhookPort := dig "controller" "admissionWebhooks" "port" 8443 $inc }} ---- -# The ingress-nginx admission webhook only ever needs to be reachable by the -# API server. Leaving it reachable from every pod in the cluster is avoidable -# attack surface: any flaw in the webhook then becomes exploitable by any -# workload in any namespace, which is precisely what made CVE-2025-1974 -# (IngressNightmare) critical rather than merely serious. -# -# This policy is deny-only and explicitly disables default-deny. A Cilium rule -# that carries an ingress section normally puts the selected endpoints behind a -# default-deny, which here would mean enumerating every port the controller -# serves (80, 443, metrics, the protobuf exporter, any tcp/udp services) and -# silently breaking traffic on the first one forgotten. With -# enableDefaultDeny.ingress=false the rule only subtracts access to the webhook -# port and leaves everything else exactly as it was. -# -# Two legitimate callers are carved out of the deny: -# -# * host-network sources, which are not Cilium endpoints and so are never -# matched by fromEndpoints. This covers a management cluster, where the -# kube-apiserver runs on the host network. -# -# * the konnectivity agent **in kube-system**. For a Kamaji-hosted tenant -# cluster the API server lives outside the cluster and reaches in-cluster -# webhooks through the konnectivity tunnel: Kamaji defines only the -# "cluster" egress selection -# (internal/resources/konnectivity/egress_selector_configuration_resource.go), -# and webhook calls fall under it. The connection therefore leaves the -# tunnel from the agent pod in kube-system, not from the API server address, -# so a fromEntities[kube-apiserver] allow would not match it and every -# Ingress create/update in the tenant cluster would start failing. Kamaji -# puts nothing but `k8s-app: konnectivity-agent` on that pod template -# (internal/resources/konnectivity/agent.go), so the label is the only -# available discriminator — hence the namespace confinement below rather -# than a more specific label match. -apiVersion: cilium.io/v2 -kind: CiliumNetworkPolicy -metadata: - name: ingress-nginx-admission-webhook - namespace: {{ .Release.Namespace }} -spec: - endpointSelector: - matchLabels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/component: controller - enableDefaultDeny: - ingress: false - egress: false - ingressDeny: - # Every pod in the cluster, with the konnectivity-agent exemption confined to - # kube-system. The two selectors OR together, and each one names the namespace - # key explicitly so the rule spans all namespaces: a plain fromEndpoints in a - # namespaced CiliumNetworkPolicy would only ever select pods sharing this - # policy's namespace. - # - # The split is load-bearing rather than cosmetic. `k8s-app` is an ordinary pod - # label that any workload sets on itself, and because this rule disables - # default-deny, falling out of the deny *is* the entire access decision — so a - # single selector carrying the label exemption would let any pod in any - # namespace reach the webhook simply by labelling itself - # `k8s-app: konnectivity-agent`, which is exactly the access this policy - # exists to remove. Nothing in the platform constrains pod labels by - # admission, and Cilium keeps `k8s-app` as an identity-relevant label (its - # label filter ships no default whitelist), so the forged label really would - # change the endpoint's identity. The namespace key is derived by Cilium from - # the pod's actual namespace and cannot be set by the workload, which is what - # makes it safe to key the exemption on. - - fromEndpoints: - # Outside kube-system: no exemption exists, so no label can buy one. - - matchExpressions: - - key: k8s:io.kubernetes.pod.namespace - operator: NotIn - values: - - kube-system - # Inside kube-system: everything except the konnectivity agent itself. - - matchExpressions: - - key: k8s:io.kubernetes.pod.namespace - operator: In - values: - - kube-system - - key: k8s-app - operator: NotIn - values: - - konnectivity-agent - toPorts: - - ports: - - port: {{ $webhookPort | quote }} - protocol: TCP - - fromEntities: - - world - toPorts: - - ports: - - port: {{ $webhookPort | quote }} - protocol: TCP -{{- end }} diff --git a/packages/system/ingress-nginx/tests/admission_webhook_egress_policy_capability_test.yaml b/packages/system/ingress-nginx/tests/admission_webhook_egress_policy_capability_test.yaml new file mode 100644 index 000000000..6afe8f00c --- /dev/null +++ b/packages/system/ingress-nginx/tests/admission_webhook_egress_policy_capability_test.yaml @@ -0,0 +1,13 @@ +suite: ingress-nginx admission webhook policy capability gate +release: + name: ingress-nginx + namespace: cozy-ingress-nginx +templates: + - templates/admission-webhook-egress-policy.yaml +capabilities: + apiVersions: [] +tests: + - it: renders nothing when Cilium clusterwide policies are unavailable + asserts: + - hasDocuments: + count: 0 diff --git a/packages/system/ingress-nginx/tests/admission_webhook_egress_policy_test.yaml b/packages/system/ingress-nginx/tests/admission_webhook_egress_policy_test.yaml new file mode 100644 index 000000000..4376b483f --- /dev/null +++ b/packages/system/ingress-nginx/tests/admission_webhook_egress_policy_test.yaml @@ -0,0 +1,195 @@ +suite: ingress-nginx admission webhook exposure +release: + name: ingress-nginx + namespace: cozy-ingress-nginx +templates: + - templates/admission-webhook-egress-policy.yaml +capabilities: + apiVersions: + - cilium.io/v2/CiliumClusterwideNetworkPolicy +tests: + - it: denies pod egress through the exact admission Service + asserts: + - isKind: + of: CiliumClusterwideNetworkPolicy + - equal: + path: specs[0].egressDeny[0].toServices[0].k8sService.namespace + value: cozy-ingress-nginx + - equal: + path: specs[0].egressDeny[0].toServices[0].k8sService.serviceName + value: ingress-nginx-controller-admission + - equal: + path: specs[0].egressDeny[0].toPorts[0].ports[0].port + value: "443" + + - it: denies direct pod egress to the exact webhook backend and port + asserts: + - equal: + path: specs[0].egressDeny[1].toEndpoints[0].matchLabels["k8s:io.kubernetes.pod.namespace"] + value: cozy-ingress-nginx + - equal: + path: specs[0].egressDeny[1].toEndpoints[0].matchLabels["k8s:app.kubernetes.io/name"] + value: ingress-nginx + - equal: + path: specs[0].egressDeny[1].toEndpoints[0].matchLabels["k8s:app.kubernetes.io/instance"] + value: ingress-nginx + - equal: + path: specs[0].egressDeny[1].toEndpoints[0].matchLabels["k8s:app.kubernetes.io/component"] + value: controller + - equal: + path: specs[0].egressDeny[1].toPorts[0].ports[0].port + value: "8443" + - equal: + path: specs[0].egressDeny[1].toPorts[0].ports[0].protocol + value: TCP + + - it: applies the same destination deny to both source groups + asserts: + - equal: + path: specs[1].egressDeny + value: + - toServices: + - k8sService: + namespace: cozy-ingress-nginx + serviceName: ingress-nginx-controller-admission + toPorts: + - ports: + - port: "443" + protocol: TCP + - toEndpoints: + - matchLabels: + k8s:io.kubernetes.pod.namespace: cozy-ingress-nginx + k8s:app.kubernetes.io/name: ingress-nginx + k8s:app.kubernetes.io/instance: ingress-nginx + k8s:app.kubernetes.io/component: controller + toPorts: + - ports: + - port: "8443" + protocol: TCP + + - it: covers all non-system pods without a forgeable label exemption + asserts: + - equal: + path: specs[0].endpointSelector.matchExpressions + value: + - key: k8s:io.kubernetes.pod.namespace + operator: NotIn + values: + - kube-system + + - it: exempts only the konnectivity-agent ServiceAccount in kube-system + asserts: + - contains: + path: specs[1].endpointSelector.matchExpressions + content: + key: k8s:io.kubernetes.pod.namespace + operator: In + values: + - kube-system + - contains: + path: specs[1].endpointSelector.matchExpressions + content: + key: io.cilium.k8s.policy.serviceaccount + operator: NotIn + values: + - konnectivity-agent + - lengthEqual: + path: specs[1].endpointSelector.matchExpressions + count: 2 + + - it: denies world ingress without a namespace-dependent caller selector + asserts: + - equal: + path: specs[2].endpointSelector.matchLabels + value: + k8s:io.kubernetes.pod.namespace: cozy-ingress-nginx + k8s:app.kubernetes.io/name: ingress-nginx + k8s:app.kubernetes.io/instance: ingress-nginx + k8s:app.kubernetes.io/component: controller + - contains: + path: specs[2].ingressDeny[0].fromEntities + content: world + - equal: + path: specs[2].ingressDeny[0].toPorts[0].ports[0].port + value: "8443" + - notExists: + path: specs[2].endpointSelector.matchExpressions + + - it: does not enable default deny on source or target endpoints + asserts: + - equal: + path: specs[0].enableDefaultDeny + value: + ingress: false + egress: false + - equal: + path: specs[1].enableDefaultDeny + value: + ingress: false + egress: false + - equal: + path: specs[2].enableDefaultDeny + value: + ingress: false + egress: false + - notExists: + path: specs[0].ingress + - notExists: + path: specs[0].ingressDeny + - notExists: + path: specs[0].egress + + - it: gives each release a cluster-scoped collision-safe name + asserts: + - matchRegex: + path: metadata.name + pattern: ^ingress-nginx-admission-webhook-[a-f0-9]{12}$ + - notExists: + path: metadata.namespace + + - it: follows upstream name namespace and webhook port overrides + set: + ingress-nginx.nameOverride: edge-ingress + ingress-nginx.namespaceOverride: custom-ingress + ingress-nginx.controller.admissionWebhooks.port: 9443 + ingress-nginx.controller.admissionWebhooks.service.servicePort: 9444 + asserts: + - equal: + path: specs[0].egressDeny[0].toServices[0].k8sService.namespace + value: custom-ingress + - equal: + path: specs[0].egressDeny[0].toServices[0].k8sService.serviceName + value: ingress-nginx-edge-ingress-controller-admission + - equal: + path: specs[0].egressDeny[0].toPorts[0].ports[0].port + value: "9444" + - equal: + path: specs[0].egressDeny[1].toEndpoints[0].matchLabels["k8s:io.kubernetes.pod.namespace"] + value: custom-ingress + - equal: + path: specs[0].egressDeny[1].toEndpoints[0].matchLabels["k8s:app.kubernetes.io/name"] + value: edge-ingress + - equal: + path: specs[0].egressDeny[1].toPorts[0].ports[0].port + value: "9443" + + - it: does not render when the upstream admission webhook is disabled + set: + ingress-nginx.controller.admissionWebhooks.enabled: false + asserts: + - hasDocuments: + count: 0 + + - it: defaults to enabled when a backport has no policy values + set: + admissionWebhookPolicy: null + asserts: + - hasDocuments: + count: 1 + + - it: can be explicitly disabled + set: + admissionWebhookPolicy.enabled: false + asserts: + - hasDocuments: + count: 0 diff --git a/packages/system/ingress-nginx/tests/admission_webhook_networkpolicy_test.yaml b/packages/system/ingress-nginx/tests/admission_webhook_networkpolicy_test.yaml deleted file mode 100644 index 8441868da..000000000 --- a/packages/system/ingress-nginx/tests/admission_webhook_networkpolicy_test.yaml +++ /dev/null @@ -1,117 +0,0 @@ -suite: ingress-nginx admission webhook exposure -release: - name: ingress-nginx - namespace: cozy-ingress-nginx -templates: - - templates/admission-webhook-networkpolicy.yaml -tests: - - it: denies pod traffic to the admission webhook port - # :8443 is the admission webhook. Reachability from arbitrary pods is what - # turned CVE-2025-1974 from a webhook bug into unauthenticated in-cluster - # RCE, so the deny on this port is the whole point of the policy. - asserts: - - equal: - path: spec.ingressDeny[0].toPorts[0].ports[0].port - value: "8443" - - equal: - path: spec.ingressDeny[0].toPorts[0].ports[0].protocol - value: TCP - - - it: spans every namespace rather than only the release namespace - # A bare fromEndpoints in a namespaced CiliumNetworkPolicy selects only - # pods in the policy's own namespace, which would leave the webhook open - # to every other namespace — the exact gap being closed here. Both - # selectors therefore have to name the namespace key. - asserts: - - contains: - path: spec.ingressDeny[0].fromEndpoints[0].matchExpressions - content: - key: k8s:io.kubernetes.pod.namespace - operator: NotIn - values: - - kube-system - - contains: - path: spec.ingressDeny[0].fromEndpoints[1].matchExpressions - content: - key: k8s:io.kubernetes.pod.namespace - operator: In - values: - - kube-system - - - it: does not let a pod outside kube-system buy an exemption with a label - # The whole reason the deny is split in two. `k8s-app` is an ordinary pod - # label any workload can set on itself, and with default-deny disabled - # escaping the deny IS being allowed — so a selector that pairs - # "every namespace" with the label exemption would hand webhook access to - # any pod anywhere that labels itself konnectivity-agent. The selector - # covering everything outside kube-system must therefore carry no label - # condition whatsoever. - asserts: - - lengthEqual: - path: spec.ingressDeny[0].fromEndpoints[0].matchExpressions - count: 1 - - notContains: - path: spec.ingressDeny[0].fromEndpoints[0].matchExpressions - content: - key: k8s-app - operator: NotIn - values: - - konnectivity-agent - - - it: exempts the konnectivity agent only inside kube-system - # On a Kamaji-hosted tenant cluster the API server is outside the cluster - # and reaches webhooks through the konnectivity tunnel, so the connection - # originates from the agent pod in kube-system. Denying it would make every - # Ingress create/update in the tenant cluster fail on the webhook call — - # but the exemption has to be confined to that namespace, and a pod in - # kube-system without the label stays denied. - asserts: - - contains: - path: spec.ingressDeny[0].fromEndpoints[1].matchExpressions - content: - key: k8s-app - operator: NotIn - values: - - konnectivity-agent - - lengthEqual: - path: spec.ingressDeny[0].fromEndpoints[1].matchExpressions - count: 2 - - - it: denies world traffic to the admission webhook port - asserts: - - contains: - path: spec.ingressDeny[1].fromEntities - content: world - - equal: - path: spec.ingressDeny[1].toPorts[0].ports[0].port - value: "8443" - - - it: does not put the controller behind a default deny - # The controller also serves 80/443, metrics and the protobuf exporter. - # Enabling default-deny here would require enumerating all of them and - # would break traffic on the first one omitted. - asserts: - - equal: - path: spec.enableDefaultDeny.ingress - value: false - - equal: - path: spec.enableDefaultDeny.egress - value: false - - isNull: - path: spec.ingress - - - it: selects only the controller pods - asserts: - - equal: - path: spec.endpointSelector.matchLabels["app.kubernetes.io/component"] - value: controller - - equal: - path: spec.endpointSelector.matchLabels["app.kubernetes.io/name"] - value: ingress-nginx - - - it: can be disabled - set: - admissionWebhookPolicy.enabled: false - asserts: - - hasDocuments: - count: 0 diff --git a/packages/system/kubeovn-webhook/templates/_helpers.tpl b/packages/system/kubeovn-webhook/templates/_helpers.tpl index d131a3414..86899b2e0 100644 --- a/packages/system/kubeovn-webhook/templates/_helpers.tpl +++ b/packages/system/kubeovn-webhook/templates/_helpers.tpl @@ -7,8 +7,16 @@ kube-ovn-webhook {{- end }} {{/* -The port the webhook serves on. Defined once and consumed by the container -port, the Service targetPort and the CiliumNetworkPolicy, because those three +The port exposed by the Service. Keep the Service and source-side Cilium policy +on the same value so pod traffic cannot bypass the deny through the ClusterIP. +*/}} +{{- define "namespace-annotation-webhook.servicePort" -}} +443 +{{- end }} + +{{/* +The port the webhook serves on. Defined once and consumed by the container, +the Service targetPort and the Cilium clusterwide policy, because those three must agree: the policy has default-deny disabled, so a policy pointed at a port nothing serves denies nothing while the real port stays open -- a drift that removes the control silently rather than breaking anything visibly. diff --git a/packages/system/kubeovn-webhook/templates/networkpolicy.yaml b/packages/system/kubeovn-webhook/templates/networkpolicy.yaml index a4fa3d87b..f91309cc2 100644 --- a/packages/system/kubeovn-webhook/templates/networkpolicy.yaml +++ b/packages/system/kubeovn-webhook/templates/networkpolicy.yaml @@ -1,71 +1,62 @@ -{{- if .Values.webhookPolicy.enabled }} +{{- $policy := .Values.webhookPolicy | default dict -}} +{{- if and (dig "enabled" true $policy) (.Capabilities.APIVersions.Has "cilium.io/v2/CiliumClusterwideNetworkPolicy") }} +{{- $policyHash := printf "%s/%s" .Release.Namespace .Release.Name | sha256sum | trunc 12 -}} --- {{/* -The /mutate-pods handler answers whoever asks it with namespace-derived -annotations, and only the API server has any business calling it -(GHSA-g883-q79m-8225). Client-certificate verification cannot close this on -its own: enabling --client-ca-file requires the API server to *present* a -certificate, which it does only when its AdmissionConfiguration names a -kubeConfigFile for the webhook plugins. A stock Cozystack control plane ships -a PodSecurity plugin block and nothing else, so no certificate is offered and -RequireAndVerifyClientCert would reject the API server itself -- with -failurePolicy: Fail that stops pod creation cluster-wide. Until that changes, -the network is the only place the caller can actually be constrained. +The /mutate-pods handler returns namespace-derived annotations to any caller, +and only the API server has any business calling it (GHSA-g883-q79m-8225). +Client-certificate verification cannot enforce that on the stock control plane +because the API server presents no client certificate to this webhook. -This is a CiliumNetworkPolicy, not a core NetworkPolicy, and the distinction -is the whole reason it works: packages/system/kubeovn/values.yaml sets -ENABLE_NP: false, so kube-ovn programs no ACLs and a core NetworkPolicy would -be created and never enforced. Cilium enforces its own policies regardless, -which is why the platform already relies on them elsewhere. +This Cilium policy is source-side and deny-only. Selecting callers avoids +coupling the webhook endpoint's ingress policy to every namespace identity: +namespace deletion can otherwise regenerate that endpoint and transiently +break Pod admission (#3983). enableDefaultDeny remains false, so the policy +subtracts only the webhook destination at its frontend and backend ports from +pod egress. -Deny-only, with default-deny explicitly disabled. A Cilium rule carrying an -ingress section puts the selected endpoints behind a default-deny, which here -would mean enumerating every port these pods serve and silently breaking -traffic on the first one forgotten. Subtracting one port leaves everything -else exactly as it was. - -The API server needs no carve-out, and this was confirmed on a live kube-ovn -cluster rather than reasoned about: cilium monitor on the webhook's endpoint -shows the API server's connection arriving from the node InternalIP with -identity "host" when it originates on the same node and "kube-apiserver" when -it originates on another, never from the kube-ovn join subnet and never as -"world". Neither identity is a Cilium endpoint, so fromEndpoints cannot match -it and the world deny does not apply to it. Unlike the ingress-nginx admission -webhook, this component exists only on the management cluster, so there is no -konnectivity-tunnelled caller to exempt either. - -Note the scope this actually has: it removes in-cluster pods and external -traffic, not every possible caller. Any host-network workload on the -management cluster carries the same host / remote-node identity that exempts -the API server and keeps its reach. +The API server and host-network workloads are not selected as pod endpoints. +This component exists only on the management cluster, so unlike ingress-nginx +there is no konnectivity-tunnelled caller to exempt. A second, target-side rule +keeps the external-world deny; its static entity selector does not depend on +namespace identities and therefore does not recreate the churn failure mode. */}} apiVersion: cilium.io/v2 -kind: CiliumNetworkPolicy +kind: CiliumClusterwideNetworkPolicy metadata: - name: {{ include "namespace-annotation-webhook.name" . }} - namespace: {{ .Release.Namespace }} -spec: - endpointSelector: - matchLabels: - app.kubernetes.io/name: {{ include "namespace-annotation-webhook.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} + name: kube-ovn-webhook-{{ $policyHash }} +specs: +- endpointSelector: {} enableDefaultDeny: ingress: false - ingressDeny: -{{- /* -Every pod in the cluster. The namespace key is matched with Exists so the rule -spans all namespaces: a plain fromEndpoints in a namespaced -CiliumNetworkPolicy would only ever select pods sharing this policy's -namespace, which would leave the webhook open to every other one. -*/}} - - fromEndpoints: - - matchExpressions: - - key: k8s:io.kubernetes.pod.namespace - operator: Exists + egress: false + egressDeny: + - toServices: + - k8sService: + namespace: {{ .Release.Namespace | quote }} + serviceName: {{ include "namespace-annotation-webhook.fullname" . | quote }} + toPorts: + - ports: + - port: {{ include "namespace-annotation-webhook.servicePort" . | quote }} + protocol: TCP + - toEndpoints: + - matchLabels: + "k8s:io.kubernetes.pod.namespace": {{ .Release.Namespace | quote }} + "k8s:app.kubernetes.io/name": {{ include "namespace-annotation-webhook.name" . | quote }} + "k8s:app.kubernetes.io/instance": {{ .Release.Name | quote }} toPorts: - ports: - port: {{ include "namespace-annotation-webhook.port" . | quote }} protocol: TCP +- endpointSelector: + matchLabels: + "k8s:io.kubernetes.pod.namespace": {{ .Release.Namespace | quote }} + "k8s:app.kubernetes.io/name": {{ include "namespace-annotation-webhook.name" . | quote }} + "k8s:app.kubernetes.io/instance": {{ .Release.Name | quote }} + enableDefaultDeny: + ingress: false + egress: false + ingressDeny: - fromEntities: - world toPorts: diff --git a/packages/system/kubeovn-webhook/templates/service.yaml b/packages/system/kubeovn-webhook/templates/service.yaml index 06d9d6455..bf0bff162 100644 --- a/packages/system/kubeovn-webhook/templates/service.yaml +++ b/packages/system/kubeovn-webhook/templates/service.yaml @@ -8,7 +8,7 @@ metadata: spec: type: ClusterIP ports: - - port: 443 + - port: {{ include "namespace-annotation-webhook.servicePort" . }} targetPort: {{ include "namespace-annotation-webhook.port" . }} protocol: TCP name: https diff --git a/packages/system/kubeovn-webhook/tests/networkpolicy_capability_test.yaml b/packages/system/kubeovn-webhook/tests/networkpolicy_capability_test.yaml new file mode 100644 index 000000000..f722ab447 --- /dev/null +++ b/packages/system/kubeovn-webhook/tests/networkpolicy_capability_test.yaml @@ -0,0 +1,13 @@ +suite: kubeovn-webhook policy capability gate +release: + name: kubeovn-webhook + namespace: cozy-kubeovn +templates: + - templates/networkpolicy.yaml +capabilities: + apiVersions: [] +tests: + - it: renders nothing when Cilium clusterwide policies are unavailable + asserts: + - hasDocuments: + count: 0 diff --git a/packages/system/kubeovn-webhook/tests/networkpolicy_test.yaml b/packages/system/kubeovn-webhook/tests/networkpolicy_test.yaml index 5ec403c16..e0a120a83 100644 --- a/packages/system/kubeovn-webhook/tests/networkpolicy_test.yaml +++ b/packages/system/kubeovn-webhook/tests/networkpolicy_test.yaml @@ -4,78 +4,88 @@ release: namespace: cozy-kubeovn templates: - templates/networkpolicy.yaml +capabilities: + apiVersions: + - cilium.io/v2/CiliumClusterwideNetworkPolicy tests: - - it: denies pod traffic to the webhook port - # The /mutate-pods handler returns namespace-derived annotations to any - # caller that reaches it (GHSA-g883-q79m-8225), so closing the port to pods - # is the whole point of the object. - asserts: - - equal: - path: spec.ingressDeny[0].toPorts[0].ports[0].port - value: "8443" - - equal: - path: spec.ingressDeny[0].toPorts[0].ports[0].protocol - value: TCP - - - it: spans every namespace rather than only the release namespace - # A bare fromEndpoints in a namespaced CiliumNetworkPolicy selects only pods - # in the policy's own namespace, which would leave the webhook reachable - # from every other one — the exact gap being closed. - asserts: - - contains: - path: spec.ingressDeny[0].fromEndpoints[0].matchExpressions - content: - key: k8s:io.kubernetes.pod.namespace - operator: Exists - - - it: carries no label-based exemption - # There is no caller to carve out: the API server runs on the host network - # and is therefore not a Cilium endpoint, so fromEndpoints never matches it. - # A label exemption would be both unnecessary and forgeable — any workload - # can set a pod label on itself, and with default-deny disabled, falling out - # of the deny is the entire access decision. - asserts: - - lengthEqual: - path: spec.ingressDeny[0].fromEndpoints[0].matchExpressions - count: 1 - - lengthEqual: - path: spec.ingressDeny[0].fromEndpoints - count: 1 - - - it: denies world traffic to the webhook port - asserts: - - contains: - path: spec.ingressDeny[1].fromEntities - content: world - - equal: - path: spec.ingressDeny[1].toPorts[0].ports[0].port - value: "8443" - - - it: does not put the webhook behind a default deny - # Enabling default-deny would require enumerating every port these pods - # serve and would break traffic on the first one omitted. - asserts: - - equal: - path: spec.enableDefaultDeny.ingress - value: false - - isNull: - path: spec.ingress - - - it: is a CiliumNetworkPolicy rather than a core NetworkPolicy - # kube-ovn runs with ENABLE_NP: false, so a core NetworkPolicy would be - # created and never enforced. Cilium enforces its own policies regardless. + - it: denies pod egress through the exact webhook Service asserts: - isKind: - of: CiliumNetworkPolicy + of: CiliumClusterwideNetworkPolicy - equal: - path: apiVersion - value: cilium.io/v2 + path: specs[0].endpointSelector + value: {} + - equal: + path: specs[0].egressDeny[0].toServices[0].k8sService.namespace + value: cozy-kubeovn + - equal: + path: specs[0].egressDeny[0].toServices[0].k8sService.serviceName + value: kube-ovn-webhook + - equal: + path: specs[0].egressDeny[0].toPorts[0].ports[0].port + value: "443" - - it: selects only the webhook pods + - it: denies direct pod egress to the exact webhook backend and port asserts: - equal: - path: spec.endpointSelector.matchLabels["app.kubernetes.io/instance"] + path: specs[0].egressDeny[1].toEndpoints[0].matchLabels["k8s:io.kubernetes.pod.namespace"] + value: cozy-kubeovn + - equal: + path: specs[0].egressDeny[1].toEndpoints[0].matchLabels["k8s:app.kubernetes.io/name"] + value: kube-ovn-webhook + - equal: + path: specs[0].egressDeny[1].toEndpoints[0].matchLabels["k8s:app.kubernetes.io/instance"] value: kubeovn-webhook + - equal: + path: specs[0].egressDeny[1].toPorts[0].ports[0].port + value: "8443" + - equal: + path: specs[0].egressDeny[1].toPorts[0].ports[0].protocol + value: TCP + + - it: denies world ingress without a namespace-dependent caller selector + asserts: + - equal: + path: specs[1].endpointSelector.matchLabels + value: + k8s:io.kubernetes.pod.namespace: cozy-kubeovn + k8s:app.kubernetes.io/name: kube-ovn-webhook + k8s:app.kubernetes.io/instance: kubeovn-webhook + - contains: + path: specs[1].ingressDeny[0].fromEntities + content: world + - equal: + path: specs[1].ingressDeny[0].toPorts[0].ports[0].port + value: "8443" + - notExists: + path: specs[1].endpointSelector.matchExpressions + + - it: does not enable default deny on source or target endpoints + asserts: + - equal: + path: specs[0].enableDefaultDeny + value: + ingress: false + egress: false + - equal: + path: specs[1].enableDefaultDeny + value: + ingress: false + egress: false + - notExists: + path: specs[0].ingress + - notExists: + path: specs[0].ingressDeny + - notExists: + path: specs[0].egress + + - it: gives each release a cluster-scoped collision-safe name + asserts: + - matchRegex: + path: metadata.name + pattern: ^kube-ovn-webhook-[a-f0-9]{12}$ + - notExists: + path: metadata.namespace - it: can be disabled where no Cilium is present to enforce it set: diff --git a/packages/system/kubeovn-webhook/tests/port_agreement_test.yaml b/packages/system/kubeovn-webhook/tests/port_agreement_test.yaml index 8dd64866b..19560728e 100644 --- a/packages/system/kubeovn-webhook/tests/port_agreement_test.yaml +++ b/packages/system/kubeovn-webhook/tests/port_agreement_test.yaml @@ -6,20 +6,23 @@ templates: - templates/networkpolicy.yaml - templates/deployment.yaml - templates/service.yaml +capabilities: + apiVersions: + - cilium.io/v2/CiliumClusterwideNetworkPolicy tests: - it: protects exactly the port the webhook serves and the Service routes to # The regression guard for the one drift that fails open. The policy runs # with default-deny disabled, so a policy port that disagrees with the # container port denies a port nothing serves and leaves the real one # reachable -- the control disappears without anything breaking visibly. - # All three render from namespace-annotation-webhook.port for that reason. + # All four render from namespace-annotation-webhook.port for that reason. asserts: - equal: - path: spec.ingressDeny[0].toPorts[0].ports[0].port + path: specs[0].egressDeny[1].toPorts[0].ports[0].port value: "8443" template: templates/networkpolicy.yaml - equal: - path: spec.ingressDeny[1].toPorts[0].ports[0].port + path: specs[1].ingressDeny[0].toPorts[0].ports[0].port value: "8443" template: templates/networkpolicy.yaml - equal: @@ -30,3 +33,14 @@ tests: path: spec.ports[0].targetPort value: 8443 template: templates/service.yaml + + - it: protects exactly the frontend port exposed by the Service + asserts: + - equal: + path: specs[0].egressDeny[0].toPorts[0].ports[0].port + value: "443" + template: templates/networkpolicy.yaml + - equal: + path: spec.ports[0].port + value: 443 + template: templates/service.yaml