mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-09 05:25:14 +09:00
fix(networking): isolate webhook policies from namespace churn
Move pod denials to source-side Cilium policies so namespace churn cannot regenerate fail-closed webhook endpoints. Keep the external world deny as a static target rule. Assisted-By: GPT-5 <noreply@openai.com> Signed-off-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
This commit is contained in:
11 files changed
+484
-339
No files matched your search
@@ -0,0 +1,115 @@
|
||||
{{- $policy := .Values.admissionWebhookPolicy | default dict -}}
|
||||
{{- $inc := index .Values "ingress-nginx" | default dict -}}
|
||||
{{- $ingress := index .Subcharts "ingress-nginx" -}}
|
||||
{{- $admissionWebhooks := dig "controller" "admissionWebhooks" dict $inc | default dict -}}
|
||||
{{- if and (dig "enabled" true $policy) (dig "enabled" true $admissionWebhooks) (.Capabilities.APIVersions.Has "cilium.io/v2/CiliumClusterwideNetworkPolicy") }}
|
||||
{{- $controllerName := include "ingress-nginx.name" $ingress -}}
|
||||
{{- $controllerNamespace := include "ingress-nginx.namespace" $ingress -}}
|
||||
{{- $controllerFullname := include "ingress-nginx.controller.fullname" $ingress -}}
|
||||
{{- $webhookServiceName := printf "%s-admission" $controllerFullname -}}
|
||||
{{- $webhookPort := dig "port" 8443 $admissionWebhooks -}}
|
||||
{{- $webhookServicePort := dig "service" "servicePort" 443 $admissionWebhooks -}}
|
||||
{{- $policyHash := printf "%s/%s" $controllerNamespace .Release.Name | sha256sum | trunc 12 -}}
|
||||
---
|
||||
# The ingress-nginx admission webhook only needs to be reachable by the API
|
||||
# server. Deny pod egress through its exact Service and directly to its exact
|
||||
# endpoint identity while leaving every other destination and port unchanged.
|
||||
#
|
||||
# This is deliberately source-side. A target-side ingress policy with a
|
||||
# cluster-wide fromEndpoints selector makes every namespace identity part of
|
||||
# the webhook endpoint's policy. Namespace deletion can then regenerate that
|
||||
# endpoint and transiently drop legitimate API-server webhook calls (#3983).
|
||||
# Keeping the namespace-wide selector on source endpoints removes that
|
||||
# production failure mode while preserving the same pod-to-webhook boundary.
|
||||
#
|
||||
# The first rule covers every workload outside kube-system. The second covers
|
||||
# kube-system except the konnectivity-agent ServiceAccount: Kamaji-hosted API
|
||||
# servers reach admission webhooks through that pod. The ServiceAccount label
|
||||
# is a Cilium-derived identity label, so a workload outside kube-system cannot
|
||||
# forge its way into the exemption with an ordinary pod label.
|
||||
#
|
||||
# The final rule keeps the external-world deny on the webhook endpoint. Its
|
||||
# caller selector is a static entity rather than a cluster-wide endpoint set,
|
||||
# so it does not couple target regeneration to namespace lifecycle events.
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumClusterwideNetworkPolicy
|
||||
metadata:
|
||||
name: ingress-nginx-admission-webhook-{{ $policyHash }}
|
||||
specs:
|
||||
- endpointSelector:
|
||||
matchExpressions:
|
||||
- key: k8s:io.kubernetes.pod.namespace
|
||||
operator: NotIn
|
||||
values:
|
||||
- kube-system
|
||||
enableDefaultDeny:
|
||||
ingress: false
|
||||
egress: false
|
||||
egressDeny:
|
||||
- toServices:
|
||||
- k8sService:
|
||||
namespace: {{ $controllerNamespace | quote }}
|
||||
serviceName: {{ $webhookServiceName | quote }}
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: {{ $webhookServicePort | quote }}
|
||||
protocol: TCP
|
||||
- toEndpoints:
|
||||
- matchLabels:
|
||||
"k8s:io.kubernetes.pod.namespace": {{ $controllerNamespace | quote }}
|
||||
"k8s:app.kubernetes.io/name": {{ $controllerName | quote }}
|
||||
"k8s:app.kubernetes.io/instance": {{ .Release.Name | quote }}
|
||||
"k8s:app.kubernetes.io/component": controller
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: {{ $webhookPort | quote }}
|
||||
protocol: TCP
|
||||
- endpointSelector:
|
||||
matchExpressions:
|
||||
- key: k8s:io.kubernetes.pod.namespace
|
||||
operator: In
|
||||
values:
|
||||
- kube-system
|
||||
- key: io.cilium.k8s.policy.serviceaccount
|
||||
operator: NotIn
|
||||
values:
|
||||
- konnectivity-agent
|
||||
enableDefaultDeny:
|
||||
ingress: false
|
||||
egress: false
|
||||
egressDeny:
|
||||
- toServices:
|
||||
- k8sService:
|
||||
namespace: {{ $controllerNamespace | quote }}
|
||||
serviceName: {{ $webhookServiceName | quote }}
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: {{ $webhookServicePort | quote }}
|
||||
protocol: TCP
|
||||
- toEndpoints:
|
||||
- matchLabels:
|
||||
"k8s:io.kubernetes.pod.namespace": {{ $controllerNamespace | quote }}
|
||||
"k8s:app.kubernetes.io/name": {{ $controllerName | quote }}
|
||||
"k8s:app.kubernetes.io/instance": {{ .Release.Name | quote }}
|
||||
"k8s:app.kubernetes.io/component": controller
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: {{ $webhookPort | quote }}
|
||||
protocol: TCP
|
||||
- endpointSelector:
|
||||
matchLabels:
|
||||
"k8s:io.kubernetes.pod.namespace": {{ $controllerNamespace | quote }}
|
||||
"k8s:app.kubernetes.io/name": {{ $controllerName | quote }}
|
||||
"k8s:app.kubernetes.io/instance": {{ .Release.Name | quote }}
|
||||
"k8s:app.kubernetes.io/component": controller
|
||||
enableDefaultDeny:
|
||||
ingress: false
|
||||
egress: false
|
||||
ingressDeny:
|
||||
- fromEntities:
|
||||
- world
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: {{ $webhookPort | quote }}
|
||||
protocol: TCP
|
||||
{{- end }}
|
||||
@@ -1,97 +0,0 @@
|
||||
{{- if .Values.admissionWebhookPolicy.enabled }}
|
||||
{{- $inc := index .Values "ingress-nginx" | default dict }}
|
||||
{{- $webhookPort := dig "controller" "admissionWebhooks" "port" 8443 $inc }}
|
||||
---
|
||||
# The ingress-nginx admission webhook only ever needs to be reachable by the
|
||||
# API server. Leaving it reachable from every pod in the cluster is avoidable
|
||||
# attack surface: any flaw in the webhook then becomes exploitable by any
|
||||
# workload in any namespace, which is precisely what made CVE-2025-1974
|
||||
# (IngressNightmare) critical rather than merely serious.
|
||||
#
|
||||
# This policy is deny-only and explicitly disables default-deny. A Cilium rule
|
||||
# that carries an ingress section normally puts the selected endpoints behind a
|
||||
# default-deny, which here would mean enumerating every port the controller
|
||||
# serves (80, 443, metrics, the protobuf exporter, any tcp/udp services) and
|
||||
# silently breaking traffic on the first one forgotten. With
|
||||
# enableDefaultDeny.ingress=false the rule only subtracts access to the webhook
|
||||
# port and leaves everything else exactly as it was.
|
||||
#
|
||||
# Two legitimate callers are carved out of the deny:
|
||||
#
|
||||
# * host-network sources, which are not Cilium endpoints and so are never
|
||||
# matched by fromEndpoints. This covers a management cluster, where the
|
||||
# kube-apiserver runs on the host network.
|
||||
#
|
||||
# * the konnectivity agent **in kube-system**. For a Kamaji-hosted tenant
|
||||
# cluster the API server lives outside the cluster and reaches in-cluster
|
||||
# webhooks through the konnectivity tunnel: Kamaji defines only the
|
||||
# "cluster" egress selection
|
||||
# (internal/resources/konnectivity/egress_selector_configuration_resource.go),
|
||||
# and webhook calls fall under it. The connection therefore leaves the
|
||||
# tunnel from the agent pod in kube-system, not from the API server address,
|
||||
# so a fromEntities[kube-apiserver] allow would not match it and every
|
||||
# Ingress create/update in the tenant cluster would start failing. Kamaji
|
||||
# puts nothing but `k8s-app: konnectivity-agent` on that pod template
|
||||
# (internal/resources/konnectivity/agent.go), so the label is the only
|
||||
# available discriminator — hence the namespace confinement below rather
|
||||
# than a more specific label match.
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: ingress-nginx-admission-webhook
|
||||
namespace: {{ .Release.Namespace }}
|
||||
spec:
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: ingress-nginx
|
||||
app.kubernetes.io/component: controller
|
||||
enableDefaultDeny:
|
||||
ingress: false
|
||||
egress: false
|
||||
ingressDeny:
|
||||
# Every pod in the cluster, with the konnectivity-agent exemption confined to
|
||||
# kube-system. The two selectors OR together, and each one names the namespace
|
||||
# key explicitly so the rule spans all namespaces: a plain fromEndpoints in a
|
||||
# namespaced CiliumNetworkPolicy would only ever select pods sharing this
|
||||
# policy's namespace.
|
||||
#
|
||||
# The split is load-bearing rather than cosmetic. `k8s-app` is an ordinary pod
|
||||
# label that any workload sets on itself, and because this rule disables
|
||||
# default-deny, falling out of the deny *is* the entire access decision — so a
|
||||
# single selector carrying the label exemption would let any pod in any
|
||||
# namespace reach the webhook simply by labelling itself
|
||||
# `k8s-app: konnectivity-agent`, which is exactly the access this policy
|
||||
# exists to remove. Nothing in the platform constrains pod labels by
|
||||
# admission, and Cilium keeps `k8s-app` as an identity-relevant label (its
|
||||
# label filter ships no default whitelist), so the forged label really would
|
||||
# change the endpoint's identity. The namespace key is derived by Cilium from
|
||||
# the pod's actual namespace and cannot be set by the workload, which is what
|
||||
# makes it safe to key the exemption on.
|
||||
- fromEndpoints:
|
||||
# Outside kube-system: no exemption exists, so no label can buy one.
|
||||
- matchExpressions:
|
||||
- key: k8s:io.kubernetes.pod.namespace
|
||||
operator: NotIn
|
||||
values:
|
||||
- kube-system
|
||||
# Inside kube-system: everything except the konnectivity agent itself.
|
||||
- matchExpressions:
|
||||
- key: k8s:io.kubernetes.pod.namespace
|
||||
operator: In
|
||||
values:
|
||||
- kube-system
|
||||
- key: k8s-app
|
||||
operator: NotIn
|
||||
values:
|
||||
- konnectivity-agent
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: {{ $webhookPort | quote }}
|
||||
protocol: TCP
|
||||
- fromEntities:
|
||||
- world
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: {{ $webhookPort | quote }}
|
||||
protocol: TCP
|
||||
{{- end }}
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
suite: ingress-nginx admission webhook policy capability gate
|
||||
release:
|
||||
name: ingress-nginx
|
||||
namespace: cozy-ingress-nginx
|
||||
templates:
|
||||
- templates/admission-webhook-egress-policy.yaml
|
||||
capabilities:
|
||||
apiVersions: []
|
||||
tests:
|
||||
- it: renders nothing when Cilium clusterwide policies are unavailable
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
@@ -0,0 +1,195 @@
|
||||
suite: ingress-nginx admission webhook exposure
|
||||
release:
|
||||
name: ingress-nginx
|
||||
namespace: cozy-ingress-nginx
|
||||
templates:
|
||||
- templates/admission-webhook-egress-policy.yaml
|
||||
capabilities:
|
||||
apiVersions:
|
||||
- cilium.io/v2/CiliumClusterwideNetworkPolicy
|
||||
tests:
|
||||
- it: denies pod egress through the exact admission Service
|
||||
asserts:
|
||||
- isKind:
|
||||
of: CiliumClusterwideNetworkPolicy
|
||||
- equal:
|
||||
path: specs[0].egressDeny[0].toServices[0].k8sService.namespace
|
||||
value: cozy-ingress-nginx
|
||||
- equal:
|
||||
path: specs[0].egressDeny[0].toServices[0].k8sService.serviceName
|
||||
value: ingress-nginx-controller-admission
|
||||
- equal:
|
||||
path: specs[0].egressDeny[0].toPorts[0].ports[0].port
|
||||
value: "443"
|
||||
|
||||
- it: denies direct pod egress to the exact webhook backend and port
|
||||
asserts:
|
||||
- equal:
|
||||
path: specs[0].egressDeny[1].toEndpoints[0].matchLabels["k8s:io.kubernetes.pod.namespace"]
|
||||
value: cozy-ingress-nginx
|
||||
- equal:
|
||||
path: specs[0].egressDeny[1].toEndpoints[0].matchLabels["k8s:app.kubernetes.io/name"]
|
||||
value: ingress-nginx
|
||||
- equal:
|
||||
path: specs[0].egressDeny[1].toEndpoints[0].matchLabels["k8s:app.kubernetes.io/instance"]
|
||||
value: ingress-nginx
|
||||
- equal:
|
||||
path: specs[0].egressDeny[1].toEndpoints[0].matchLabels["k8s:app.kubernetes.io/component"]
|
||||
value: controller
|
||||
- equal:
|
||||
path: specs[0].egressDeny[1].toPorts[0].ports[0].port
|
||||
value: "8443"
|
||||
- equal:
|
||||
path: specs[0].egressDeny[1].toPorts[0].ports[0].protocol
|
||||
value: TCP
|
||||
|
||||
- it: applies the same destination deny to both source groups
|
||||
asserts:
|
||||
- equal:
|
||||
path: specs[1].egressDeny
|
||||
value:
|
||||
- toServices:
|
||||
- k8sService:
|
||||
namespace: cozy-ingress-nginx
|
||||
serviceName: ingress-nginx-controller-admission
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: "443"
|
||||
protocol: TCP
|
||||
- toEndpoints:
|
||||
- matchLabels:
|
||||
k8s:io.kubernetes.pod.namespace: cozy-ingress-nginx
|
||||
k8s:app.kubernetes.io/name: ingress-nginx
|
||||
k8s:app.kubernetes.io/instance: ingress-nginx
|
||||
k8s:app.kubernetes.io/component: controller
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: "8443"
|
||||
protocol: TCP
|
||||
|
||||
- it: covers all non-system pods without a forgeable label exemption
|
||||
asserts:
|
||||
- equal:
|
||||
path: specs[0].endpointSelector.matchExpressions
|
||||
value:
|
||||
- key: k8s:io.kubernetes.pod.namespace
|
||||
operator: NotIn
|
||||
values:
|
||||
- kube-system
|
||||
|
||||
- it: exempts only the konnectivity-agent ServiceAccount in kube-system
|
||||
asserts:
|
||||
- contains:
|
||||
path: specs[1].endpointSelector.matchExpressions
|
||||
content:
|
||||
key: k8s:io.kubernetes.pod.namespace
|
||||
operator: In
|
||||
values:
|
||||
- kube-system
|
||||
- contains:
|
||||
path: specs[1].endpointSelector.matchExpressions
|
||||
content:
|
||||
key: io.cilium.k8s.policy.serviceaccount
|
||||
operator: NotIn
|
||||
values:
|
||||
- konnectivity-agent
|
||||
- lengthEqual:
|
||||
path: specs[1].endpointSelector.matchExpressions
|
||||
count: 2
|
||||
|
||||
- it: denies world ingress without a namespace-dependent caller selector
|
||||
asserts:
|
||||
- equal:
|
||||
path: specs[2].endpointSelector.matchLabels
|
||||
value:
|
||||
k8s:io.kubernetes.pod.namespace: cozy-ingress-nginx
|
||||
k8s:app.kubernetes.io/name: ingress-nginx
|
||||
k8s:app.kubernetes.io/instance: ingress-nginx
|
||||
k8s:app.kubernetes.io/component: controller
|
||||
- contains:
|
||||
path: specs[2].ingressDeny[0].fromEntities
|
||||
content: world
|
||||
- equal:
|
||||
path: specs[2].ingressDeny[0].toPorts[0].ports[0].port
|
||||
value: "8443"
|
||||
- notExists:
|
||||
path: specs[2].endpointSelector.matchExpressions
|
||||
|
||||
- it: does not enable default deny on source or target endpoints
|
||||
asserts:
|
||||
- equal:
|
||||
path: specs[0].enableDefaultDeny
|
||||
value:
|
||||
ingress: false
|
||||
egress: false
|
||||
- equal:
|
||||
path: specs[1].enableDefaultDeny
|
||||
value:
|
||||
ingress: false
|
||||
egress: false
|
||||
- equal:
|
||||
path: specs[2].enableDefaultDeny
|
||||
value:
|
||||
ingress: false
|
||||
egress: false
|
||||
- notExists:
|
||||
path: specs[0].ingress
|
||||
- notExists:
|
||||
path: specs[0].ingressDeny
|
||||
- notExists:
|
||||
path: specs[0].egress
|
||||
|
||||
- it: gives each release a cluster-scoped collision-safe name
|
||||
asserts:
|
||||
- matchRegex:
|
||||
path: metadata.name
|
||||
pattern: ^ingress-nginx-admission-webhook-[a-f0-9]{12}$
|
||||
- notExists:
|
||||
path: metadata.namespace
|
||||
|
||||
- it: follows upstream name namespace and webhook port overrides
|
||||
set:
|
||||
ingress-nginx.nameOverride: edge-ingress
|
||||
ingress-nginx.namespaceOverride: custom-ingress
|
||||
ingress-nginx.controller.admissionWebhooks.port: 9443
|
||||
ingress-nginx.controller.admissionWebhooks.service.servicePort: 9444
|
||||
asserts:
|
||||
- equal:
|
||||
path: specs[0].egressDeny[0].toServices[0].k8sService.namespace
|
||||
value: custom-ingress
|
||||
- equal:
|
||||
path: specs[0].egressDeny[0].toServices[0].k8sService.serviceName
|
||||
value: ingress-nginx-edge-ingress-controller-admission
|
||||
- equal:
|
||||
path: specs[0].egressDeny[0].toPorts[0].ports[0].port
|
||||
value: "9444"
|
||||
- equal:
|
||||
path: specs[0].egressDeny[1].toEndpoints[0].matchLabels["k8s:io.kubernetes.pod.namespace"]
|
||||
value: custom-ingress
|
||||
- equal:
|
||||
path: specs[0].egressDeny[1].toEndpoints[0].matchLabels["k8s:app.kubernetes.io/name"]
|
||||
value: edge-ingress
|
||||
- equal:
|
||||
path: specs[0].egressDeny[1].toPorts[0].ports[0].port
|
||||
value: "9443"
|
||||
|
||||
- it: does not render when the upstream admission webhook is disabled
|
||||
set:
|
||||
ingress-nginx.controller.admissionWebhooks.enabled: false
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
|
||||
- it: defaults to enabled when a backport has no policy values
|
||||
set:
|
||||
admissionWebhookPolicy: null
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 1
|
||||
|
||||
- it: can be explicitly disabled
|
||||
set:
|
||||
admissionWebhookPolicy.enabled: false
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
@@ -1,117 +0,0 @@
|
||||
suite: ingress-nginx admission webhook exposure
|
||||
release:
|
||||
name: ingress-nginx
|
||||
namespace: cozy-ingress-nginx
|
||||
templates:
|
||||
- templates/admission-webhook-networkpolicy.yaml
|
||||
tests:
|
||||
- it: denies pod traffic to the admission webhook port
|
||||
# :8443 is the admission webhook. Reachability from arbitrary pods is what
|
||||
# turned CVE-2025-1974 from a webhook bug into unauthenticated in-cluster
|
||||
# RCE, so the deny on this port is the whole point of the policy.
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.ingressDeny[0].toPorts[0].ports[0].port
|
||||
value: "8443"
|
||||
- equal:
|
||||
path: spec.ingressDeny[0].toPorts[0].ports[0].protocol
|
||||
value: TCP
|
||||
|
||||
- it: spans every namespace rather than only the release namespace
|
||||
# A bare fromEndpoints in a namespaced CiliumNetworkPolicy selects only
|
||||
# pods in the policy's own namespace, which would leave the webhook open
|
||||
# to every other namespace — the exact gap being closed here. Both
|
||||
# selectors therefore have to name the namespace key.
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.ingressDeny[0].fromEndpoints[0].matchExpressions
|
||||
content:
|
||||
key: k8s:io.kubernetes.pod.namespace
|
||||
operator: NotIn
|
||||
values:
|
||||
- kube-system
|
||||
- contains:
|
||||
path: spec.ingressDeny[0].fromEndpoints[1].matchExpressions
|
||||
content:
|
||||
key: k8s:io.kubernetes.pod.namespace
|
||||
operator: In
|
||||
values:
|
||||
- kube-system
|
||||
|
||||
- it: does not let a pod outside kube-system buy an exemption with a label
|
||||
# The whole reason the deny is split in two. `k8s-app` is an ordinary pod
|
||||
# label any workload can set on itself, and with default-deny disabled
|
||||
# escaping the deny IS being allowed — so a selector that pairs
|
||||
# "every namespace" with the label exemption would hand webhook access to
|
||||
# any pod anywhere that labels itself konnectivity-agent. The selector
|
||||
# covering everything outside kube-system must therefore carry no label
|
||||
# condition whatsoever.
|
||||
asserts:
|
||||
- lengthEqual:
|
||||
path: spec.ingressDeny[0].fromEndpoints[0].matchExpressions
|
||||
count: 1
|
||||
- notContains:
|
||||
path: spec.ingressDeny[0].fromEndpoints[0].matchExpressions
|
||||
content:
|
||||
key: k8s-app
|
||||
operator: NotIn
|
||||
values:
|
||||
- konnectivity-agent
|
||||
|
||||
- it: exempts the konnectivity agent only inside kube-system
|
||||
# On a Kamaji-hosted tenant cluster the API server is outside the cluster
|
||||
# and reaches webhooks through the konnectivity tunnel, so the connection
|
||||
# originates from the agent pod in kube-system. Denying it would make every
|
||||
# Ingress create/update in the tenant cluster fail on the webhook call —
|
||||
# but the exemption has to be confined to that namespace, and a pod in
|
||||
# kube-system without the label stays denied.
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.ingressDeny[0].fromEndpoints[1].matchExpressions
|
||||
content:
|
||||
key: k8s-app
|
||||
operator: NotIn
|
||||
values:
|
||||
- konnectivity-agent
|
||||
- lengthEqual:
|
||||
path: spec.ingressDeny[0].fromEndpoints[1].matchExpressions
|
||||
count: 2
|
||||
|
||||
- it: denies world traffic to the admission webhook port
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.ingressDeny[1].fromEntities
|
||||
content: world
|
||||
- equal:
|
||||
path: spec.ingressDeny[1].toPorts[0].ports[0].port
|
||||
value: "8443"
|
||||
|
||||
- it: does not put the controller behind a default deny
|
||||
# The controller also serves 80/443, metrics and the protobuf exporter.
|
||||
# Enabling default-deny here would require enumerating all of them and
|
||||
# would break traffic on the first one omitted.
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.enableDefaultDeny.ingress
|
||||
value: false
|
||||
- equal:
|
||||
path: spec.enableDefaultDeny.egress
|
||||
value: false
|
||||
- isNull:
|
||||
path: spec.ingress
|
||||
|
||||
- it: selects only the controller pods
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.endpointSelector.matchLabels["app.kubernetes.io/component"]
|
||||
value: controller
|
||||
- equal:
|
||||
path: spec.endpointSelector.matchLabels["app.kubernetes.io/name"]
|
||||
value: ingress-nginx
|
||||
|
||||
- it: can be disabled
|
||||
set:
|
||||
admissionWebhookPolicy.enabled: false
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
@@ -7,8 +7,16 @@ kube-ovn-webhook
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
The port the webhook serves on. Defined once and consumed by the container
|
||||
port, the Service targetPort and the CiliumNetworkPolicy, because those three
|
||||
The port exposed by the Service. Keep the Service and source-side Cilium policy
|
||||
on the same value so pod traffic cannot bypass the deny through the ClusterIP.
|
||||
*/}}
|
||||
{{- define "namespace-annotation-webhook.servicePort" -}}
|
||||
443
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
The port the webhook serves on. Defined once and consumed by the container,
|
||||
the Service targetPort and the Cilium clusterwide policy, because those three
|
||||
must agree: the policy has default-deny disabled, so a policy pointed at a port
|
||||
nothing serves denies nothing while the real port stays open -- a drift that
|
||||
removes the control silently rather than breaking anything visibly.
|
||||
|
||||
@@ -1,71 +1,62 @@
|
||||
{{- if .Values.webhookPolicy.enabled }}
|
||||
{{- $policy := .Values.webhookPolicy | default dict -}}
|
||||
{{- if and (dig "enabled" true $policy) (.Capabilities.APIVersions.Has "cilium.io/v2/CiliumClusterwideNetworkPolicy") }}
|
||||
{{- $policyHash := printf "%s/%s" .Release.Namespace .Release.Name | sha256sum | trunc 12 -}}
|
||||
---
|
||||
{{/*
|
||||
The /mutate-pods handler answers whoever asks it with namespace-derived
|
||||
annotations, and only the API server has any business calling it
|
||||
(GHSA-g883-q79m-8225). Client-certificate verification cannot close this on
|
||||
its own: enabling --client-ca-file requires the API server to *present* a
|
||||
certificate, which it does only when its AdmissionConfiguration names a
|
||||
kubeConfigFile for the webhook plugins. A stock Cozystack control plane ships
|
||||
a PodSecurity plugin block and nothing else, so no certificate is offered and
|
||||
RequireAndVerifyClientCert would reject the API server itself -- with
|
||||
failurePolicy: Fail that stops pod creation cluster-wide. Until that changes,
|
||||
the network is the only place the caller can actually be constrained.
|
||||
The /mutate-pods handler returns namespace-derived annotations to any caller,
|
||||
and only the API server has any business calling it (GHSA-g883-q79m-8225).
|
||||
Client-certificate verification cannot enforce that on the stock control plane
|
||||
because the API server presents no client certificate to this webhook.
|
||||
|
||||
This is a CiliumNetworkPolicy, not a core NetworkPolicy, and the distinction
|
||||
is the whole reason it works: packages/system/kubeovn/values.yaml sets
|
||||
ENABLE_NP: false, so kube-ovn programs no ACLs and a core NetworkPolicy would
|
||||
be created and never enforced. Cilium enforces its own policies regardless,
|
||||
which is why the platform already relies on them elsewhere.
|
||||
This Cilium policy is source-side and deny-only. Selecting callers avoids
|
||||
coupling the webhook endpoint's ingress policy to every namespace identity:
|
||||
namespace deletion can otherwise regenerate that endpoint and transiently
|
||||
break Pod admission (#3983). enableDefaultDeny remains false, so the policy
|
||||
subtracts only the webhook destination at its frontend and backend ports from
|
||||
pod egress.
|
||||
|
||||
Deny-only, with default-deny explicitly disabled. A Cilium rule carrying an
|
||||
ingress section puts the selected endpoints behind a default-deny, which here
|
||||
would mean enumerating every port these pods serve and silently breaking
|
||||
traffic on the first one forgotten. Subtracting one port leaves everything
|
||||
else exactly as it was.
|
||||
|
||||
The API server needs no carve-out, and this was confirmed on a live kube-ovn
|
||||
cluster rather than reasoned about: cilium monitor on the webhook's endpoint
|
||||
shows the API server's connection arriving from the node InternalIP with
|
||||
identity "host" when it originates on the same node and "kube-apiserver" when
|
||||
it originates on another, never from the kube-ovn join subnet and never as
|
||||
"world". Neither identity is a Cilium endpoint, so fromEndpoints cannot match
|
||||
it and the world deny does not apply to it. Unlike the ingress-nginx admission
|
||||
webhook, this component exists only on the management cluster, so there is no
|
||||
konnectivity-tunnelled caller to exempt either.
|
||||
|
||||
Note the scope this actually has: it removes in-cluster pods and external
|
||||
traffic, not every possible caller. Any host-network workload on the
|
||||
management cluster carries the same host / remote-node identity that exempts
|
||||
the API server and keeps its reach.
|
||||
The API server and host-network workloads are not selected as pod endpoints.
|
||||
This component exists only on the management cluster, so unlike ingress-nginx
|
||||
there is no konnectivity-tunnelled caller to exempt. A second, target-side rule
|
||||
keeps the external-world deny; its static entity selector does not depend on
|
||||
namespace identities and therefore does not recreate the churn failure mode.
|
||||
*/}}
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
kind: CiliumClusterwideNetworkPolicy
|
||||
metadata:
|
||||
name: {{ include "namespace-annotation-webhook.name" . }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
spec:
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: {{ include "namespace-annotation-webhook.name" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
name: kube-ovn-webhook-{{ $policyHash }}
|
||||
specs:
|
||||
- endpointSelector: {}
|
||||
enableDefaultDeny:
|
||||
ingress: false
|
||||
ingressDeny:
|
||||
{{- /*
|
||||
Every pod in the cluster. The namespace key is matched with Exists so the rule
|
||||
spans all namespaces: a plain fromEndpoints in a namespaced
|
||||
CiliumNetworkPolicy would only ever select pods sharing this policy's
|
||||
namespace, which would leave the webhook open to every other one.
|
||||
*/}}
|
||||
- fromEndpoints:
|
||||
- matchExpressions:
|
||||
- key: k8s:io.kubernetes.pod.namespace
|
||||
operator: Exists
|
||||
egress: false
|
||||
egressDeny:
|
||||
- toServices:
|
||||
- k8sService:
|
||||
namespace: {{ .Release.Namespace | quote }}
|
||||
serviceName: {{ include "namespace-annotation-webhook.fullname" . | quote }}
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: {{ include "namespace-annotation-webhook.servicePort" . | quote }}
|
||||
protocol: TCP
|
||||
- toEndpoints:
|
||||
- matchLabels:
|
||||
"k8s:io.kubernetes.pod.namespace": {{ .Release.Namespace | quote }}
|
||||
"k8s:app.kubernetes.io/name": {{ include "namespace-annotation-webhook.name" . | quote }}
|
||||
"k8s:app.kubernetes.io/instance": {{ .Release.Name | quote }}
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: {{ include "namespace-annotation-webhook.port" . | quote }}
|
||||
protocol: TCP
|
||||
- endpointSelector:
|
||||
matchLabels:
|
||||
"k8s:io.kubernetes.pod.namespace": {{ .Release.Namespace | quote }}
|
||||
"k8s:app.kubernetes.io/name": {{ include "namespace-annotation-webhook.name" . | quote }}
|
||||
"k8s:app.kubernetes.io/instance": {{ .Release.Name | quote }}
|
||||
enableDefaultDeny:
|
||||
ingress: false
|
||||
egress: false
|
||||
ingressDeny:
|
||||
- fromEntities:
|
||||
- world
|
||||
toPorts:
|
||||
|
||||
@@ -8,7 +8,7 @@ metadata:
|
||||
spec:
|
||||
type: ClusterIP
|
||||
ports:
|
||||
- port: 443
|
||||
- port: {{ include "namespace-annotation-webhook.servicePort" . }}
|
||||
targetPort: {{ include "namespace-annotation-webhook.port" . }}
|
||||
protocol: TCP
|
||||
name: https
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
suite: kubeovn-webhook policy capability gate
|
||||
release:
|
||||
name: kubeovn-webhook
|
||||
namespace: cozy-kubeovn
|
||||
templates:
|
||||
- templates/networkpolicy.yaml
|
||||
capabilities:
|
||||
apiVersions: []
|
||||
tests:
|
||||
- it: renders nothing when Cilium clusterwide policies are unavailable
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
@@ -4,78 +4,88 @@ release:
|
||||
namespace: cozy-kubeovn
|
||||
templates:
|
||||
- templates/networkpolicy.yaml
|
||||
capabilities:
|
||||
apiVersions:
|
||||
- cilium.io/v2/CiliumClusterwideNetworkPolicy
|
||||
tests:
|
||||
- it: denies pod traffic to the webhook port
|
||||
# The /mutate-pods handler returns namespace-derived annotations to any
|
||||
# caller that reaches it (GHSA-g883-q79m-8225), so closing the port to pods
|
||||
# is the whole point of the object.
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.ingressDeny[0].toPorts[0].ports[0].port
|
||||
value: "8443"
|
||||
- equal:
|
||||
path: spec.ingressDeny[0].toPorts[0].ports[0].protocol
|
||||
value: TCP
|
||||
|
||||
- it: spans every namespace rather than only the release namespace
|
||||
# A bare fromEndpoints in a namespaced CiliumNetworkPolicy selects only pods
|
||||
# in the policy's own namespace, which would leave the webhook reachable
|
||||
# from every other one — the exact gap being closed.
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.ingressDeny[0].fromEndpoints[0].matchExpressions
|
||||
content:
|
||||
key: k8s:io.kubernetes.pod.namespace
|
||||
operator: Exists
|
||||
|
||||
- it: carries no label-based exemption
|
||||
# There is no caller to carve out: the API server runs on the host network
|
||||
# and is therefore not a Cilium endpoint, so fromEndpoints never matches it.
|
||||
# A label exemption would be both unnecessary and forgeable — any workload
|
||||
# can set a pod label on itself, and with default-deny disabled, falling out
|
||||
# of the deny is the entire access decision.
|
||||
asserts:
|
||||
- lengthEqual:
|
||||
path: spec.ingressDeny[0].fromEndpoints[0].matchExpressions
|
||||
count: 1
|
||||
- lengthEqual:
|
||||
path: spec.ingressDeny[0].fromEndpoints
|
||||
count: 1
|
||||
|
||||
- it: denies world traffic to the webhook port
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.ingressDeny[1].fromEntities
|
||||
content: world
|
||||
- equal:
|
||||
path: spec.ingressDeny[1].toPorts[0].ports[0].port
|
||||
value: "8443"
|
||||
|
||||
- it: does not put the webhook behind a default deny
|
||||
# Enabling default-deny would require enumerating every port these pods
|
||||
# serve and would break traffic on the first one omitted.
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.enableDefaultDeny.ingress
|
||||
value: false
|
||||
- isNull:
|
||||
path: spec.ingress
|
||||
|
||||
- it: is a CiliumNetworkPolicy rather than a core NetworkPolicy
|
||||
# kube-ovn runs with ENABLE_NP: false, so a core NetworkPolicy would be
|
||||
# created and never enforced. Cilium enforces its own policies regardless.
|
||||
- it: denies pod egress through the exact webhook Service
|
||||
asserts:
|
||||
- isKind:
|
||||
of: CiliumNetworkPolicy
|
||||
of: CiliumClusterwideNetworkPolicy
|
||||
- equal:
|
||||
path: apiVersion
|
||||
value: cilium.io/v2
|
||||
path: specs[0].endpointSelector
|
||||
value: {}
|
||||
- equal:
|
||||
path: specs[0].egressDeny[0].toServices[0].k8sService.namespace
|
||||
value: cozy-kubeovn
|
||||
- equal:
|
||||
path: specs[0].egressDeny[0].toServices[0].k8sService.serviceName
|
||||
value: kube-ovn-webhook
|
||||
- equal:
|
||||
path: specs[0].egressDeny[0].toPorts[0].ports[0].port
|
||||
value: "443"
|
||||
|
||||
- it: selects only the webhook pods
|
||||
- it: denies direct pod egress to the exact webhook backend and port
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.endpointSelector.matchLabels["app.kubernetes.io/instance"]
|
||||
path: specs[0].egressDeny[1].toEndpoints[0].matchLabels["k8s:io.kubernetes.pod.namespace"]
|
||||
value: cozy-kubeovn
|
||||
- equal:
|
||||
path: specs[0].egressDeny[1].toEndpoints[0].matchLabels["k8s:app.kubernetes.io/name"]
|
||||
value: kube-ovn-webhook
|
||||
- equal:
|
||||
path: specs[0].egressDeny[1].toEndpoints[0].matchLabels["k8s:app.kubernetes.io/instance"]
|
||||
value: kubeovn-webhook
|
||||
- equal:
|
||||
path: specs[0].egressDeny[1].toPorts[0].ports[0].port
|
||||
value: "8443"
|
||||
- equal:
|
||||
path: specs[0].egressDeny[1].toPorts[0].ports[0].protocol
|
||||
value: TCP
|
||||
|
||||
- it: denies world ingress without a namespace-dependent caller selector
|
||||
asserts:
|
||||
- equal:
|
||||
path: specs[1].endpointSelector.matchLabels
|
||||
value:
|
||||
k8s:io.kubernetes.pod.namespace: cozy-kubeovn
|
||||
k8s:app.kubernetes.io/name: kube-ovn-webhook
|
||||
k8s:app.kubernetes.io/instance: kubeovn-webhook
|
||||
- contains:
|
||||
path: specs[1].ingressDeny[0].fromEntities
|
||||
content: world
|
||||
- equal:
|
||||
path: specs[1].ingressDeny[0].toPorts[0].ports[0].port
|
||||
value: "8443"
|
||||
- notExists:
|
||||
path: specs[1].endpointSelector.matchExpressions
|
||||
|
||||
- it: does not enable default deny on source or target endpoints
|
||||
asserts:
|
||||
- equal:
|
||||
path: specs[0].enableDefaultDeny
|
||||
value:
|
||||
ingress: false
|
||||
egress: false
|
||||
- equal:
|
||||
path: specs[1].enableDefaultDeny
|
||||
value:
|
||||
ingress: false
|
||||
egress: false
|
||||
- notExists:
|
||||
path: specs[0].ingress
|
||||
- notExists:
|
||||
path: specs[0].ingressDeny
|
||||
- notExists:
|
||||
path: specs[0].egress
|
||||
|
||||
- it: gives each release a cluster-scoped collision-safe name
|
||||
asserts:
|
||||
- matchRegex:
|
||||
path: metadata.name
|
||||
pattern: ^kube-ovn-webhook-[a-f0-9]{12}$
|
||||
- notExists:
|
||||
path: metadata.namespace
|
||||
|
||||
- it: can be disabled where no Cilium is present to enforce it
|
||||
set:
|
||||
|
||||
@@ -6,20 +6,23 @@ templates:
|
||||
- templates/networkpolicy.yaml
|
||||
- templates/deployment.yaml
|
||||
- templates/service.yaml
|
||||
capabilities:
|
||||
apiVersions:
|
||||
- cilium.io/v2/CiliumClusterwideNetworkPolicy
|
||||
tests:
|
||||
- it: protects exactly the port the webhook serves and the Service routes to
|
||||
# The regression guard for the one drift that fails open. The policy runs
|
||||
# with default-deny disabled, so a policy port that disagrees with the
|
||||
# container port denies a port nothing serves and leaves the real one
|
||||
# reachable -- the control disappears without anything breaking visibly.
|
||||
# All three render from namespace-annotation-webhook.port for that reason.
|
||||
# All four render from namespace-annotation-webhook.port for that reason.
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.ingressDeny[0].toPorts[0].ports[0].port
|
||||
path: specs[0].egressDeny[1].toPorts[0].ports[0].port
|
||||
value: "8443"
|
||||
template: templates/networkpolicy.yaml
|
||||
- equal:
|
||||
path: spec.ingressDeny[1].toPorts[0].ports[0].port
|
||||
path: specs[1].ingressDeny[0].toPorts[0].ports[0].port
|
||||
value: "8443"
|
||||
template: templates/networkpolicy.yaml
|
||||
- equal:
|
||||
@@ -30,3 +33,14 @@ tests:
|
||||
path: spec.ports[0].targetPort
|
||||
value: 8443
|
||||
template: templates/service.yaml
|
||||
|
||||
- it: protects exactly the frontend port exposed by the Service
|
||||
asserts:
|
||||
- equal:
|
||||
path: specs[0].egressDeny[0].toPorts[0].ports[0].port
|
||||
value: "443"
|
||||
template: templates/networkpolicy.yaml
|
||||
- equal:
|
||||
path: spec.ports[0].port
|
||||
value: 443
|
||||
template: templates/service.yaml
|
||||
Reference in new issue
Block a user