Files
cozystack/hack/e2e-chainsaw/gateway/chainsaw-test.yaml
T
Aleksei Sviridkin ad3f15d4f3 feat(gateway)!: reserve a passthrough listener's name by the spec alone
The TenantGateway controller withheld the HTTPS-terminate listener for a
name a passthrough listener answers only once a TLSRoute was attached
there and resolved a backend, mirroring how Cilium builds its filter
chains. That kept a served endpoint up on Cilium 1.19.5, but it tied
every route's status to one implementation's translator and needed
Service and ReferenceGrant watches to land the deferred decision.

The shipped operator accepts a wildcard terminate listener beside a
narrower passthrough listener on 443 and keeps conflicting every other
overlap. A terminate listener for the passthrough name itself is not
narrower, so on 443 the rule now protects the dataplane outright; on a
native port Gateway API would serve the pair, and the rule holds as a
policy that keeps a platform name with its passthrough listener. Reading
it off the spec makes the outcome the same on every implementation.

Under http01 an HTTPRoute claiming such a name gets no listener and no
certificate whether or not a TLSRoute is attached. A route whose every
hostname is reserved reads Accepted=False with
NoMatchingListenerHostname; one that also claims a free hostname stays
Accepted and is served there, as Gateway API accepts a route whose
hostnames intersect at least one listener. dns01 and existingSecret are
unchanged: their wildcard listener still covers such a name, and the
passthrough listener answers it by its exact server name.

The chart writes the passthrough services on the publishing tenant
alone, so the names reserved by default are the ones the platform
publishes there through its own TLSRoutes. The shape an upgrade
changes is an HTTPRoute on such a name whose platform TLSRoute forwards
nowhere, because it never attached (the package owning the service is
not installed) or because none of its backends resolve: that route kept
a terminate listener and now gets none.

The backend resolution, its watches and the RBAC rules that served them
go away.

Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin <f@lex.la>
2026-10-10 06:32:42 +03:00

2389 lines
95 KiB
YAML

# Chainsaw port of hack/e2e-apps/gateway.bats (29 cases).
#
# Declarative cases (CRD/GatewayClass/Gateway/HTTPRoute/TenantGateway status,
# VAP existence) use asserts. Admission-rejection cases need kubectl
# impersonation or error-message greps that Chainsaw `apply` can't express, so
# they stay scripts. The tenant apex is read at runtime from the tenant-test
# namespace's namespace.cozystack.io/host label, so these tests pass both in CI
# (test.example.org) and on a dev cluster with a different host.
apiVersion: chainsaw.kyverno.io/v1alpha1
kind: Test
metadata:
name: gateway-api-crds
spec:
steps:
- name: crds-established-and-gatewayclass-accepted
try:
- assert:
timeout: 1m
resource:
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: gatewayclasses.gateway.networking.k8s.io
status:
(conditions[?type == 'Established']):
- status: "True"
- assert:
timeout: 1m
resource:
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: gateways.gateway.networking.k8s.io
status:
(conditions[?type == 'Established']):
- status: "True"
- assert:
timeout: 1m
resource:
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: httproutes.gateway.networking.k8s.io
status:
(conditions[?type == 'Established']):
- status: "True"
- assert:
timeout: 3m
resource:
apiVersion: gateway.networking.k8s.io/v1
kind: GatewayClass
metadata:
name: cilium
status:
(conditions[?type == 'Accepted']):
- status: "True"
---
apiVersion: chainsaw.kyverno.io/v1alpha1
kind: Test
metadata:
name: gateway-minimal-programmed
spec:
catch:
- events: {}
- describe:
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
name: gateway-e2e-probe
steps:
- name: create-minimal-gateway
try:
- apply:
file: gw-minimal.yaml
- name: assert-accepted-programmed
try:
- assert:
timeout: 2m
resource:
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
name: gateway-e2e-probe
status:
(conditions[?type == 'Accepted']):
- status: "True"
(conditions[?type == 'Programmed']):
- status: "True"
# Cilium materialises a LoadBalancer Service per programmed Gateway.
- assert:
timeout: 1m
resource:
apiVersion: v1
kind: Service
metadata:
name: cilium-gateway-gateway-e2e-probe
---
apiVersion: chainsaw.kyverno.io/v1alpha1
kind: Test
metadata:
name: gateway-hostname-policy-installed
spec:
steps:
- name: vap-and-binding-and-host-label
try:
- assert:
resource:
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingAdmissionPolicy
metadata:
name: cozystack-gateway-hostname-policy
- assert:
resource:
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingAdmissionPolicyBinding
metadata:
name: cozystack-gateway-hostname-policy
# Binding must Deny — [Audit]/[] would let requests through silently.
(contains(spec.validationActions, 'Deny')): true
# The whole hostname policy keys off this label; assert it is set
# (script, not an assert: numeric/length compares on labels are unreliable).
- script:
content: |
set -eu
kubectl get namespace "$NAMESPACE" \
-o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}' | grep -q .
check:
($error == null): true
---
apiVersion: chainsaw.kyverno.io/v1alpha1
kind: Test
metadata:
name: gateway-foreign-hostname-rejected
spec:
steps:
# No mixed-case-apex counterpart here, unlike the Route and Ingress suites.
# Deliberate: this policy applies the same rule as the Route one — own apex
# or a dot-bounded subdomain of it, with no outside-root branch — so a
# Gateway probe would re-exercise the same two comparisons that
# route-hostname-policy already drives against a mixed-case apex at runtime,
# in both directions. The two differ only in how the apex reaches the
# comparison: a `variables.tenantHost` binding here, the label inlined there.
# Because that binding is what this suite does NOT cover, both of its
# comparisons are pinned operand-by-operand in
# tests/gateway-hostname-policy_test.yaml instead.
- name: foreign-listener-hostname-denied
try:
- script:
content: |
set -eu
out=$(kubectl apply -n "$NAMESPACE" -f - 2>&1 <<'EOF'
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
name: hostname-hijack-probe
spec:
gatewayClassName: cilium
listeners:
- name: https
protocol: HTTPS
port: 443
hostname: "dashboard.example.org"
tls:
mode: Terminate
certificateRefs:
- name: noop
allowedRoutes:
namespaces:
from: Same
EOF
) && rc=0 || rc=$?
echo "rc=$rc out=$out"
if [ "$rc" -eq 0 ]; then
kubectl -n "$NAMESPACE" delete gateway hostname-hijack-probe --ignore-not-found
echo "BUG: admission accepted cross-tenant hostname" >&2
exit 1
fi
echo "$out" | grep -qi "ValidatingAdmissionPolicy"
echo "$out" | grep -q "must equal"
check:
($error == null): true
---
apiVersion: chainsaw.kyverno.io/v1alpha1
kind: Test
metadata:
name: package-attachednamespaces-accept
spec:
steps:
# Admission must accept a tenant-* entry in gateway.attachedNamespaces; a
# successful apply (and Chainsaw auto-cleanup of the Package) is the proof.
- name: apply-package-probe
try:
- apply:
file: package-attach-probe.yaml
---
apiVersion: chainsaw.kyverno.io/v1alpha1
kind: Test
metadata:
name: tenant-host-policy
spec:
steps:
- name: untrusted-sa-cannot-set-tenant-host
try:
- script:
content: |
set -eu
ns="$NAMESPACE"
# Authorization runs before admission; grant the SA RBAC to create
# Tenants so the apply reaches the VAP rather than failing on RBAC.
kubectl apply -f - <<EOF
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: vap-probe-tenant-create
namespace: ${ns}
rules:
- apiGroups: ["apps.cozystack.io"]
resources: ["tenants"]
verbs: ["create","get","list","watch","update","patch","delete"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: vap-probe-tenant-create
namespace: ${ns}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: vap-probe-tenant-create
subjects:
- kind: ServiceAccount
name: default
namespace: ${ns}
EOF
cleanup() {
# || true each delete: the trap runs under set -eu, and one
# transient delete failure must not skip the remaining cleanup.
kubectl -n "$ns" delete rolebinding vap-probe-tenant-create --ignore-not-found || true
kubectl -n "$ns" delete role vap-probe-tenant-create --ignore-not-found || true
kubectl -n "$ns" delete tenants.apps.cozystack.io vaphostprobe --ignore-not-found || true
}
trap cleanup EXIT
out=$(kubectl --as="system:serviceaccount:${ns}:default" \
--as-group=system:serviceaccounts \
--as-group="system:serviceaccounts:${ns}" \
apply -n "$ns" -f - 2>&1 <<EOF
apiVersion: apps.cozystack.io/v1alpha1
kind: Tenant
metadata:
name: vaphostprobe
namespace: ${ns}
spec:
host: foreign.example.org
EOF
) && rc=0 || rc=$?
echo "rc=$rc out=$out"
[ "$rc" -ne 0 ]
echo "$out" | grep -qi "ValidatingAdmissionPolicy"
echo "$out" | grep -q "spec.host can only be set"
check:
($error == null): true
---
apiVersion: chainsaw.kyverno.io/v1alpha1
kind: Test
metadata:
name: namespace-host-label-policy
spec:
steps:
- name: untrusted-sa-cannot-change-host-label
try:
- script:
content: |
set -eu
ns="$NAMESPACE"
host=$(kubectl get namespace "$ns" -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}')
kubectl apply -f - <<EOF
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: vap-probe-namespace-patch
rules:
- apiGroups: [""]
resources: ["namespaces"]
verbs: ["get","list","watch","update","patch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: vap-probe-namespace-patch
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: vap-probe-namespace-patch
subjects:
- kind: ServiceAccount
name: default
namespace: ${ns}
EOF
cleanup() {
# Restore the original host label in case admission let it through.
# || true each command: the trap runs under set -eu, and one
# transient failure must not skip the remaining cleanup.
kubectl label namespace "$ns" "namespace.cozystack.io/host=${host}" --overwrite >/dev/null 2>&1 || true
kubectl delete clusterrolebinding vap-probe-namespace-patch --ignore-not-found || true
kubectl delete clusterrole vap-probe-namespace-patch --ignore-not-found || true
}
trap cleanup EXIT
out=$(kubectl --as="system:serviceaccount:${ns}:default" \
--as-group=system:serviceaccounts \
--as-group="system:serviceaccounts:${ns}" \
label namespace "$ns" namespace.cozystack.io/host=foreign.example.org --overwrite 2>&1) && rc=0 || rc=$?
echo "rc=$rc out=$out"
[ "$rc" -ne 0 ]
echo "$out" | grep -qi "ValidatingAdmissionPolicy"
echo "$out" | grep -q "immutable"
check:
($error == null): true
- name: untrusted-sa-cannot-set-host-label-at-create
try:
- script:
content: |
set -eu
ns="$NAMESPACE"
kubectl apply -f - <<EOF
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: vap-probe-namespace-create
rules:
- apiGroups: [""]
resources: ["namespaces"]
verbs: ["create","get","list","delete"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: vap-probe-namespace-create
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: vap-probe-namespace-create
subjects:
- kind: ServiceAccount
name: default
namespace: ${ns}
EOF
cleanup() {
# || true each delete: the trap runs under set -eu, and one
# transient delete failure must not skip the remaining cleanup.
kubectl delete namespace vap-host-label-probe --ignore-not-found --wait=false || true
kubectl delete clusterrolebinding vap-probe-namespace-create --ignore-not-found || true
kubectl delete clusterrole vap-probe-namespace-create --ignore-not-found || true
}
trap cleanup EXIT
out=$(kubectl --as="system:serviceaccount:${ns}:default" \
--as-group=system:serviceaccounts \
--as-group="system:serviceaccounts:${ns}" \
apply -f - 2>&1 <<EOF
apiVersion: v1
kind: Namespace
metadata:
name: vap-host-label-probe
labels:
namespace.cozystack.io/host: foreign.example.org
EOF
) && rc=0 || rc=$?
echo "rc=$rc out=$out"
[ "$rc" -ne 0 ]
echo "$out" | grep -qi "ValidatingAdmissionPolicy"
echo "$out" | grep -q "immutable"
check:
($error == null): true
---
apiVersion: chainsaw.kyverno.io/v1alpha1
kind: Test
metadata:
name: tenantgateway-crd
spec:
steps:
- name: crd-established
try:
- assert:
timeout: 1m
resource:
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: tenantgateways.gateway.cozystack.io
status:
(conditions[?type == 'Established']):
- status: "True"
---
apiVersion: chainsaw.kyverno.io/v1alpha1
kind: Test
metadata:
name: tenantgateway-reconciler
spec:
catch:
- events: {}
- describe:
apiVersion: gateway.cozystack.io/v1alpha1
kind: TenantGateway
name: tg-e2e-probe
steps:
- name: create-tenantgateway
try:
# apex is read from the namespace host label so the CR is valid on any host.
- script:
content: |
set -eu
ns="$NAMESPACE"
apex=$(kubectl get namespace "$ns" -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}')
kubectl apply -n "$ns" -f - <<EOF
apiVersion: gateway.cozystack.io/v1alpha1
kind: TenantGateway
metadata:
name: tg-e2e-probe
spec:
apex: ${apex}
certMode: http01
gatewayClassName: cilium
EOF
check:
($error == null): true
- name: assert-gateway-issuer-status
try:
- assert:
timeout: 2m
resource:
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
name: tg-e2e-probe
- assert:
timeout: 2m
resource:
apiVersion: cert-manager.io/v1
kind: Issuer
metadata:
name: tg-e2e-probe-gateway
- assert:
timeout: 2m
resource:
apiVersion: gateway.cozystack.io/v1alpha1
kind: TenantGateway
metadata:
name: tg-e2e-probe
status:
(conditions[?type == 'Ready']):
- status: "True"
finally:
- delete:
ref:
apiVersion: gateway.cozystack.io/v1alpha1
kind: TenantGateway
name: tg-e2e-probe
---
apiVersion: chainsaw.kyverno.io/v1alpha1
kind: Test
metadata:
name: route-hostname-policy
spec:
steps:
- name: foreign-apex-route-denied
try:
- script:
content: |
set -eu
out=$(kubectl apply -n "$NAMESPACE" -f - 2>&1 <<'EOF'
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: route-hostname-hijack-probe
spec:
hostnames:
- "attacker.com"
rules:
- backendRefs:
- name: kubernetes
namespace: default
port: 443
EOF
) && rc=0 || rc=$?
echo "rc=$rc out=$out"
if [ "$rc" -eq 0 ]; then
kubectl -n "$NAMESPACE" delete httproute route-hostname-hijack-probe --ignore-not-found
echo "BUG: admission accepted cross-apex HTTPRoute hostname" >&2
exit 1
fi
echo "$out" | grep -qi "ValidatingAdmissionPolicy"
# Matches the apex-comparison half of the message, which is common to
# every denial this policy issues. Pinning the leading words instead
# would break whenever the message grows a clause for another shape.
echo "$out" | grep -qi "must equal the namespace"
check:
($error == null): true
# A route that declares no hostname at all is the one shape the expression
# used to skip: it opened with `!has(object.spec.hostnames) ||`, so such a
# route satisfied the first operand and was admitted without a single
# hostname being compared. That route is not inert — under Gateway API it
# inherits the hostnames of the listener it attaches to, and parentRefs may
# name a Gateway in another namespace, so it ends up covering whatever that
# listener covers.
#
# Both empty shapes are probed because they reach the hole by different
# routes and one guard does not imply the other: the field being absent is
# caught by has(), while an explicitly empty list satisfies has() and then
# passes because all() over an empty list is vacuously true. Dropping either
# term from the CEL re-admits the whole shape, and the unit suite can only
# pin that the terms are present in the expression — this step is what proves
# the apiserver acts on them.
#
# HTTPRoute carries both cases. The TLSRoute probe is applied at v1alpha2
# because v1 and v1alpha3 mark spec.hostnames required with minItems: 1,
# leaving v1alpha2 the only version where a hostname-less TLSRoute reaches
# admission at all. It and the GRPCRoute probe carry the absent case only,
# and that is a coverage choice rather than a reachability fact: v1alpha2
# sets neither constraint, and the grpcroutes schema sets neither at any
# served version, so an explicitly empty list reaches admission on both
# just as an absent field does. All three policies render one
# byte-identical CEL string, so the HTTPRoute pair above already proves
# the empty-list term end to end.
# Scripted rather than an apply/expect pair so the grep can prove the VAP
# did the denying: a bare "the apply failed" assertion would be satisfied
# by a typo in the manifest.
- name: hostname-less-route-denied
try:
- script:
content: |
set -eu
# deny <case> <kind> <name> reads a manifest on stdin and expects the
# VAP to reject it, mirroring the deny() helper in the Ingress suite.
deny() {
kubectl -n "$NAMESPACE" delete "$2" "$3" --ignore-not-found >/dev/null
out=$(kubectl apply -n "$NAMESPACE" -f - 2>&1) && rc=0 || rc=$?
echo "case=$1 rc=$rc out=$out"
if [ "$rc" -eq 0 ]; then
kubectl -n "$NAMESPACE" delete "$2" "$3" --ignore-not-found
echo "BUG: admission accepted $1" >&2
exit 1
fi
echo "$out" | grep -qi "ValidatingAdmissionPolicy"
echo "$out" | grep -qi "must declare spec.hostnames"
}
deny httproute-hostnames-absent httproute route-nohost-probe <<'EOF'
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: route-nohost-probe
spec:
rules:
- backendRefs:
- name: kubernetes
namespace: default
port: 443
EOF
deny httproute-hostnames-empty httproute route-emptyhost-probe <<'EOF'
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: route-emptyhost-probe
spec:
hostnames: []
rules:
- backendRefs:
- name: kubernetes
namespace: default
port: 443
EOF
deny tlsroute-hostnames-absent tlsroute route-tls-nohost-probe <<'EOF'
apiVersion: gateway.networking.k8s.io/v1alpha2
kind: TLSRoute
metadata:
name: route-tls-nohost-probe
spec:
rules:
- backendRefs:
- name: kubernetes
namespace: default
port: 443
EOF
deny grpcroute-hostnames-absent grpcroute route-grpc-nohost-probe <<'EOF'
apiVersion: gateway.networking.k8s.io/v1
kind: GRPCRoute
metadata:
name: route-grpc-nohost-probe
spec:
rules:
- backendRefs:
- name: kubernetes
namespace: default
port: 443
EOF
check:
($error == null): true
- name: own-apex-route-allowed
try:
- script:
content: |
set -eu
ns="$NAMESPACE"
apex=$(kubectl get namespace "$ns" -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}')
kubectl apply -n "$ns" -f - <<EOF
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: route-hostname-allow-probe
spec:
hostnames:
- "harbor.${apex}"
rules:
- backendRefs:
- name: kubernetes
namespace: default
port: 443
EOF
kubectl -n "$ns" delete httproute route-hostname-allow-probe --ignore-not-found
check:
($error == null): true
# Case-insensitivity. The route hostname is always lowercase — the CRD's
# hostname pattern permits only lowercase, so an uppercase hostname is rejected
# before the VAP runs — but the apex operand is the namespace.cozystack.io/host
# label VALUE, and Kubernetes permits uppercase in a label value. Under a
# mixed-case apex a lowercase in-apex route must still be admitted, so the VAP
# normalizes both operands with .lowerAscii(). Pre-fix it compares
# "probe.case-probe.route.invalid" against "Case-Probe.route.invalid", misses,
# and denies a legitimate route — this step is red pre-fix, green post-fix.
# A mixed-case apex is the only runtime shape that exercises the
# normalization.
#
# The apex here does not need to sit under the platform root: unlike the
# Ingress policy this VAP compares against the tenant apex only and has no
# outside-root branch, so a reserved .invalid apex (RFC 2606) keeps the case
# host-independent and unable to collide with a dev cluster's real apex.
#
# The mixed-case apex goes on a throwaway tenant-* namespace rather than on
# tenant-test: relabelling the shared namespace would leave it mixed-case for
# every later test if a restore were missed. The tenant-* prefix is what makes
# the VAP's matchCondition fire, and the runner is a trusted caller for the
# host-label immutability VAP, so the labelled CREATE is admitted (see
# untrusted-sa-cannot-set-host-label-at-create above). Applied declaratively:
# an admission denial fails the apply, and Chainsaw auto-cleanup deletes the
# route and the namespace.
- name: mixed-case-apex-route-allowed
try:
# Pre-clean a namespace left by an interrupted prior run: a leftover
# tenant-case-probe stuck in Terminating fails the apply below, and because
# the label here is a fixed literal a surviving namespace would otherwise be
# silently reused rather than rebuilt.
- script:
content: |
set -eu
kubectl delete namespace tenant-case-probe --ignore-not-found --wait=true --timeout=60s
check:
($error == null): true
- apply:
resource:
apiVersion: v1
kind: Namespace
metadata:
name: tenant-case-probe
labels:
namespace.cozystack.io/host: Case-Probe.route.invalid
# Guard against a vacuous pass: re-read the persisted label. Had anything
# normalized it to lowercase, the route below would be admitted with or
# without the fix and the step would stop proving case-insensitivity.
- assert:
resource:
apiVersion: v1
kind: Namespace
metadata:
name: tenant-case-probe
labels:
namespace.cozystack.io/host: Case-Probe.route.invalid
- apply:
resource:
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: route-hostname-case-probe
namespace: tenant-case-probe
spec:
hostnames:
- "probe.case-probe.route.invalid"
rules:
- backendRefs:
- name: kubernetes
namespace: default
port: 443
# The deny half of the mixed-case apex, pairing with the step above the way
# foreign-apex-route-denied pairs with own-apex-route-allowed. The allow half
# alone only proves the normalized match does not UNDER-admit; over-admission
# is the direction that hijacks a hostname, and normalizing both operands is
# exactly the edit that can widen a match by accident. Reuses the namespace
# the previous step created — Chainsaw cleans up at test scope, not step
# scope. Scripted rather than an apply/expect pair because the grep for
# "ValidatingAdmissionPolicy" is what proves the VAP did the denying: a bare
# "the apply failed" assertion would be satisfied by a typo in the manifest.
- name: mixed-case-apex-route-denied
try:
- script:
content: |
set -eu
# deny <case> <name> reads a manifest on stdin and expects the VAP to
# reject it, mirroring the deny() helper in the Ingress suite below.
deny() {
kubectl -n tenant-case-probe delete httproute "$2" --ignore-not-found >/dev/null
out=$(kubectl apply -n tenant-case-probe -f - 2>&1) && rc=0 || rc=$?
echo "case=$1 rc=$rc"
if [ "$rc" -eq 0 ]; then
kubectl -n tenant-case-probe delete httproute "$2" --ignore-not-found
echo "BUG: admission accepted $1: $out" >&2
exit 1
fi
echo "$out" | grep -qi "ValidatingAdmissionPolicy"
}
# A wholly foreign apex stays denied once both operands are lowercased.
deny foreign-apex route-case-foreign-probe <<'EOF'
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: route-case-foreign-probe
spec:
hostnames:
- "evil.other.invalid"
rules:
- backendRefs:
- name: kubernetes
namespace: default
port: 443
EOF
# Dot-boundary guard under the mixed-case apex: "evilcase-probe.route.invalid"
# ends with the lowercased apex "case-probe.route.invalid" but is NOT a
# subdomain of it. The CEL only denies it because the suffix test is
# endsWith("." + apex); drop that leading dot and this hostname is
# admitted and hijacks the apex. This is the only case in this suite
# that would notice.
deny prefix-adjacent route-case-adjacent-probe <<'EOF'
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: route-case-adjacent-probe
spec:
hostnames:
- "evilcase-probe.route.invalid"
rules:
- backendRefs:
- name: kubernetes
namespace: default
port: 443
EOF
check:
($error == null): true
---
apiVersion: chainsaw.kyverno.io/v1alpha1
kind: Test
metadata:
name: httproute-parentref-accepted
spec:
catch:
- events: {}
- describe:
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
name: httproute-probe
steps:
- name: create-gateway-and-route
try:
- apply:
file: gw-route-probe.yaml
# The route is scripted rather than declared in gw-route-probe.yaml
# because it needs the tenant apex interpolated. This suite runs in
# tenant-test, which cozystack-route-hostname-policy matches, so a
# route naming no hostname is denied; the apex is read from the
# namespace label rather than hardcoded so the suite still passes on
# a dev cluster whose host differs from CI's. Same reason and same
# shape as own-apex-route-allowed above.
- script:
content: |
set -eu
ns="$NAMESPACE"
apex=$(kubectl get namespace "$ns" -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}')
[ -n "$apex" ] || { echo "SETUP FAILURE: $ns lacks the namespace.cozystack.io/host label" >&2; exit 1; }
kubectl apply -n "$ns" -f - <<EOF
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: httproute-probe
spec:
hostnames:
- "parentref-probe.${apex}"
parentRefs:
- name: gateway-route-probe
sectionName: http
rules:
- matches:
- path:
type: PathPrefix
value: /
backendRefs:
- name: kubernetes
namespace: default
port: 443
EOF
check:
($error == null): true
- assert:
timeout: 3m
resource:
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
name: gateway-route-probe
status:
(conditions[?type == 'Programmed']):
- status: "True"
- assert:
timeout: 2m
resource:
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: httproute-probe
(status.parents[0].conditions[?type == 'Accepted']):
- status: "True"
# Creation and assertions share one step so this reclaim covers both.
# It is needed because chainsaw reclaims what it applied from a file
# but not what a script created, and the suite runs in one long-lived
# namespace. finally runs at the end of its own step, so splitting
# the assertions into a later step would delete the route before they
# ran, and hanging the reclaim off that later step would leak the
# route whenever this one failed after the apply.
finally:
- delete:
ref:
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
name: httproute-probe
---
apiVersion: chainsaw.kyverno.io/v1alpha1
kind: Test
metadata:
name: tenant-gateway-label-lockstep
spec:
catch:
- events: {}
steps:
- name: create-child-tenant-with-gateway
try:
- apply:
resource:
apiVersion: apps.cozystack.io/v1alpha1
kind: Tenant
metadata:
name: gwprop
spec:
gateway: true
- name: assert-label-and-values
try:
- assert:
timeout: 3m
resource:
apiVersion: v1
kind: Namespace
metadata:
name: tenant-test-gwprop
labels:
namespace.cozystack.io/gateway: tenant-test-gwprop
- script:
timeout: 2m
content: |
set -eu
deadline=$(( $(date +%s) + 60 ))
until kubectl -n tenant-test-gwprop get secret cozystack-values >/dev/null 2>&1; do
[ "$(date +%s)" -ge "$deadline" ] && { echo "cozystack-values not created" >&2; exit 1; }
sleep 2
done
v=$(kubectl -n tenant-test-gwprop get secret cozystack-values -o jsonpath='{.data.values\.yaml}' | base64 -d)
echo "$v" | grep -E '^\s*gateway:\s*"tenant-test-gwprop"\s*$'
check:
($error == null): true
finally:
- delete:
ref:
apiVersion: apps.cozystack.io/v1alpha1
kind: Tenant
name: gwprop
---
apiVersion: chainsaw.kyverno.io/v1alpha1
kind: Test
metadata:
name: tenant-gateway-inheritance
spec:
catch:
- events: {}
steps:
- name: create-parent-tenant
try:
- apply:
resource:
apiVersion: apps.cozystack.io/v1alpha1
kind: Tenant
metadata:
name: gwparent
spec:
gateway: true
- assert:
timeout: 3m
resource:
apiVersion: v1
kind: Namespace
metadata:
name: tenant-test-gwparent
labels:
namespace.cozystack.io/gateway: tenant-test-gwparent
- name: create-child-tenant
try:
- apply:
resource:
apiVersion: apps.cozystack.io/v1alpha1
kind: Tenant
metadata:
name: gwchild
namespace: tenant-test-gwparent
spec: {}
- assert:
timeout: 3m
resource:
apiVersion: v1
kind: Namespace
metadata:
name: tenant-test-gwparent-gwchild
labels:
namespace.cozystack.io/gateway: tenant-test-gwparent
- name: assert-inheritance-values
try:
- script:
timeout: 2m
content: |
set -eu
deadline=$(( $(date +%s) + 60 ))
until kubectl -n tenant-test-gwparent-gwchild get secret cozystack-values >/dev/null 2>&1; do
[ "$(date +%s)" -ge "$deadline" ] && { echo "child cozystack-values not created" >&2; exit 1; }
sleep 2
done
v=$(kubectl -n tenant-test-gwparent-gwchild get secret cozystack-values -o jsonpath='{.data.values\.yaml}' | base64 -d)
echo "$v" | grep -E '^\s*gateway:\s*"tenant-test-gwparent"\s*$'
check:
($error == null): true
# The child must NOT own a gateway HelmRelease — inheritance, not its own.
- error:
resource:
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: gateway
namespace: tenant-test-gwparent-gwchild
finally:
- delete:
ref:
apiVersion: apps.cozystack.io/v1alpha1
kind: Tenant
name: gwchild
namespace: tenant-test-gwparent
- delete:
ref:
apiVersion: apps.cozystack.io/v1alpha1
kind: Tenant
name: gwparent
---
apiVersion: chainsaw.kyverno.io/v1alpha1
kind: Test
metadata:
name: tenant-gateway-route-listener
spec:
catch:
- events: {}
steps:
- name: parent-child-and-route-drive-listener-set
try:
# Above the 660s of polls plus the 675s the trap's deletes may take with
# their kill grace, so a poll or a delete is what fires and the trap still
# runs: a kill at the op ceiling would skip cleanup and leave both tenants
# behind for every later suite. hack/chainsaw-op-budgets.bats holds the op
# to that sum.
- script:
timeout: 24m
content: |
set -eu
ns="$NAMESPACE"
# Each delete waits, child before parent, because a parent deleted
# while its child is still uninstalling wedges the parent's cleanup.
# Each delete is bounded as a whole by an outer timeout, because the
# trap runs inside the op ceiling and kubectl's --timeout covers only
# the wait after the request, not discovery or a stalled request. A
# tenant delete gets the 5m that .chainsaw.yaml gives a delete op,
# because its child namespaces take minutes to drain. A child still
# uninstalling at its bound leaves the parent in place: a leaked
# tenant rather than a wedged one. A failed tenant delete fails the
# step with kubectl's own error and exit status (124 is the bound),
# so the leak is reported here rather than in whichever suite trips
# on it.
cleanup() {
timeout -k 5 60 kubectl -n tenant-test-rparent-rchild delete httproute inherit-probe --ignore-not-found >/dev/null 2>&1 || true
if timeout -k 5 300 kubectl -n tenant-test-rparent delete tenant rchild --ignore-not-found >/dev/null; then
timeout -k 5 300 kubectl -n "$ns" delete tenant rparent --ignore-not-found >/dev/null || {
echo "the tenant rparent delete exited $?, so tenant rparent is left in $ns" >&2
exit 1
}
else
echo "the tenant rchild delete exited $?, so tenant rparent is left in $ns" >&2
exit 1
fi
}
trap cleanup EXIT
# Parent tenant owns the Gateway.
kubectl apply -n "$ns" -f - <<EOF
apiVersion: apps.cozystack.io/v1alpha1
kind: Tenant
metadata:
name: rparent
spec:
gateway: true
EOF
timeout 180 sh -ec 'until kubectl get ns tenant-test-rparent >/dev/null 2>&1; do sleep 3; done'
timeout 120 sh -ec 'until kubectl -n tenant-test-rparent get gateway cozystack >/dev/null 2>&1; do sleep 3; done'
# Child tenant inherits the parent Gateway.
kubectl apply -n tenant-test-rparent -f - <<EOF
apiVersion: apps.cozystack.io/v1alpha1
kind: Tenant
metadata:
name: rchild
spec: {}
EOF
timeout 180 sh -ec 'until kubectl get ns tenant-test-rparent-rchild >/dev/null 2>&1; do sleep 3; done'
child_apex=$(kubectl get ns tenant-test-rparent-rchild -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}')
[ -n "$child_apex" ]
route_host="harbor.${child_apex}"
# An HTTPRoute in the child, attached to the parent Gateway.
kubectl apply -n tenant-test-rparent-rchild -f - <<EOF
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: inherit-probe
spec:
parentRefs:
- group: gateway.networking.k8s.io
kind: Gateway
name: cozystack
namespace: tenant-test-rparent
hostnames:
- "${route_host}"
rules:
- matches:
- path:
type: PathPrefix
value: /
backendRefs:
- name: kubernetes
namespace: default
port: 443
EOF
# cozystack-controller collects the route across the inheritance label
# and grows a per-listener HTTPS entry on the parent Gateway.
timeout 120 sh -ec '
until kubectl -n tenant-test-rparent get gateway cozystack \
-o jsonpath="{range .spec.listeners[?(@.protocol==\"HTTPS\")]}{.hostname}{\"\n\"}{end}" 2>/dev/null \
| grep -qx "'"$route_host"'"; do sleep 3; done'
# ...and a matching per-listener Certificate.
timeout 60 sh -ec '
until kubectl -n tenant-test-rparent get certificate \
-l cozystack.io/per-listener-cert=true \
-o jsonpath="{range .items[*]}{.spec.dnsNames[0]}{\"\n\"}{end}" 2>/dev/null \
| grep -qx "'"$route_host"'"; do sleep 3; done'
check:
($error == null): true
---
# What a passthrough listener costs an HTTPRoute that publishes its name,
# measured on the objects rather than on the renderer. The chart renders
# passthrough listeners only on the publishing tenant and only with
# gateway.enabled, which CI leaves off, so the test declares tls-api on 443
# itself through the TenantGateway spec. The name is reserved by the
# declaration alone, so the route claiming it gets no terminate listener
# and no certificate with no TLSRoute anywhere.
#
# Readiness is measured before the test declares any HTTPRoute. The
# reservation is an http01 behaviour, and an http01 terminate listener
# waits for a Let's Encrypt certificate that is never issued for a test
# hostname, so once the control route below has its listener the tenant is
# not Ready, correctly.
#
# Absence proves nothing on its own, since a reconcile that has not run yet
# looks the same. A second route on a free name is the control: the
# controller renders the Gateway and the Certificates before it writes route
# status in the same pass, it never runs two passes over one TenantGateway
# at once, and a route its cache has seen stays there until it is deleted.
# So once both routes carry the verdicts asserted below, one read of the
# Gateway reflects a pass that saw both, whichever route came first.
apiVersion: chainsaw.kyverno.io/v1alpha1
kind: Test
metadata:
name: tenant-gateway-passthrough-reservation
spec:
catch:
- events: {}
steps:
- name: create-tenant
try:
- apply:
resource:
apiVersion: apps.cozystack.io/v1alpha1
kind: Tenant
metadata:
name: pw
spec:
gateway: true
- assert:
timeout: 3m
resource:
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
name: cozystack
namespace: tenant-test-pw
spec:
(listeners[?name == 'http']):
- protocol: HTTP
# The premise of the merge patches below, not a measurement of the
# chart's scoping: with gateway.enabled off no tenant renders these, so
# in CI this holds whether or not the chart keeps them to the publishing
# tenant. packages/extra/gateway/tests/tenantgateway_test.yaml pins that.
# One render carries every listener of the spec, so the Gateway that
# answered the assert above is the whole render of the chart's spec.
- script:
timeout: 1m
content: |
set -eu
tls_names=$(kubectl -n tenant-test-pw get gateway cozystack -o 'jsonpath={range .spec.listeners[?(@.protocol=="TLS")]}{.name}{"\n"}{end}')
if [ -n "$tls_names" ]; then
echo "child tenant Gateway renders passthrough listeners: $tls_names" >&2
exit 1
fi
check:
($error == null): true
# Two listeners with no TLSRoute behind them, one on 443 and one on a
# native port. The package README rests a claim on that shape: Cilium
# judges a listener on its protocol, its allowed kinds and its
# certificate refs and never on its route set, so a routeless passthrough
# listener is Accepted and Programmed and tenant readiness is untouched.
# The unit tests run against a fake client and cannot measure that; only
# a cluster can.
- name: routeless-listeners-keep-the-tenant-ready
try:
- script:
timeout: 2m
content: |
set -eu
tenant_ns=tenant-test-pw
apex=$(kubectl get ns "$tenant_ns" -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}')
[ -n "$apex" ]
# Merged rather than applied: the chart renders neither field
# here, and a key it renders in neither manifest is left alone
# by the next helm upgrade.
kubectl -n "$tenant_ns" patch tenantgateway cozystack --type=merge \
-p "$(printf '{"spec":{"tlsPassthroughServices":["api"],"tlsPassthroughListeners":[{"name":"pgprobe","port":5432,"hostname":"pgprobe.%s"}]}}' "$apex")"
check:
($error == null): true
# The per-listener entries are the measurement, asserted together with
# the aggregate because the TenantGateway may already be Ready when
# this step starts: Ready alone would be answered by the state that
# preceded the patch. The entries do not exist until the controller has
# judged the new listeners, each flag folds its listener's Accepted and
# Programmed, and one status write carries them with Ready.
- assert:
timeout: 2m
resource:
apiVersion: gateway.cozystack.io/v1alpha1
kind: TenantGateway
metadata:
name: cozystack
namespace: tenant-test-pw
status:
(listeners[?name == 'tls-api']):
- ready: true
(listeners[?name == 'tls-pgprobe']):
- ready: true
(conditions[?type == 'Ready']):
- status: "True"
# The hostnames are built from the apex, which is known only once a script
# reads it off the tenant namespace's label, so the routes are written
# here. They live in the tenant namespace and go with the Tenant Chainsaw
# applied above.
- name: declare-the-routes
try:
- script:
timeout: 2m
content: |
set -eu
tenant_ns=tenant-test-pw
apex=$(kubectl get ns "$tenant_ns" -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}')
[ -n "$apex" ]
kubectl apply -n "$tenant_ns" -f - <<EOF
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: app-probe
spec:
parentRefs:
- group: gateway.networking.k8s.io
kind: Gateway
name: cozystack
namespace: ${tenant_ns}
hostnames:
- "app-probe.${apex}"
rules:
- backendRefs:
- name: kubernetes
namespace: default
port: 443
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: api-probe
spec:
parentRefs:
- group: gateway.networking.k8s.io
kind: Gateway
name: cozystack
namespace: ${tenant_ns}
hostnames:
- "api.${apex}"
rules:
- backendRefs:
- name: kubernetes
namespace: default
port: 443
EOF
check:
($error == null): true
- name: reserved-name-gets-no-terminate-listener
try:
- assert:
timeout: 2m
resource:
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: app-probe
namespace: tenant-test-pw
status:
(parents[?controllerName == 'gateway.cozystack.io/tenantgateway-controller']):
- (conditions[?type == 'Accepted']):
- status: "True"
- assert:
timeout: 2m
resource:
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: api-probe
namespace: tenant-test-pw
status:
(parents[?controllerName == 'gateway.cozystack.io/tenantgateway-controller']):
- (conditions[?type == 'Accepted']):
- status: "False"
reason: NoMatchingListenerHostname
# Single reads, not polls: the verdicts above were written after the
# Gateway and the Certificates of the pass that reached them.
- script:
timeout: 1m
content: |
set -eu
tenant_ns=tenant-test-pw
apex=$(kubectl get ns "$tenant_ns" -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}')
[ -n "$apex" ]
https_hosts=$(kubectl -n "$tenant_ns" get gateway cozystack -o 'jsonpath={range .spec.listeners[?(@.protocol=="HTTPS")]}{.hostname}{"\n"}{end}')
tls_names=$(kubectl -n "$tenant_ns" get gateway cozystack -o 'jsonpath={range .spec.listeners[?(@.protocol=="TLS")]}{.name}{"\n"}{end}')
cert_hosts=$(kubectl -n "$tenant_ns" get certificate -l cozystack.io/per-listener-cert=true -o 'jsonpath={range .items[*]}{.spec.dnsNames[0]}{"\n"}{end}')
printf '%s\n' "$https_hosts" | grep -qxF "app-probe.${apex}"
printf '%s\n' "$cert_hosts" | grep -qxF "app-probe.${apex}"
printf '%s\n' "$tls_names" | grep -qxF tls-api
if printf '%s\n' "$https_hosts" | grep -qxF "api.${apex}"; then
echo "terminate listener rendered for reserved api.${apex}" >&2
exit 1
fi
if printf '%s\n' "$cert_hosts" | grep -qxF "api.${apex}"; then
echo "Certificate ordered for reserved api.${apex}" >&2
exit 1
fi
check:
($error == null): true
---
# Legacy-Ingress counterpart of the gateway/route hostname VAPs (bats: the 12
# cozystack-ingress-hostname-policy cases). Gateway API is off by default, so
# tenant apps publish through a legacy networking.k8s.io/Ingress; the VAP
# constrains tenant Ingress hostnames on the shared ingress-nginx. The VAP
# matches on the tenant-* namespace, not the caller, so plain applies as the
# runner's identity exercise it — no impersonation needed. Hostnames are
# derived at runtime: the tenant apex from this namespace's
# namespace.cozystack.io/host label (as elsewhere in this suite) and the
# platform ROOT apex from the same label on tenant-root, so the
# under-root-but-outside-own-apex cases stay host-independent where the bats
# hardcoded test.example.org / example.org.
apiVersion: chainsaw.kyverno.io/v1alpha1
kind: Test
metadata:
name: ingress-hostname-policy
spec:
catch:
- events: {}
steps:
# If these resources are missing, admission cannot constrain tenant Ingress
# hostnames — make that loud instead of letting it fall through as a silent
# pass.
- name: vap-and-binding-installed
try:
- assert:
resource:
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingAdmissionPolicy
metadata:
name: cozystack-ingress-hostname-policy
- assert:
resource:
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingAdmissionPolicyBinding
metadata:
name: cozystack-ingress-hostname-policy
spec:
(contains(validationActions, 'Deny')): true
# Apex-derived rejects:
# foreign-apex-host dashboard.<root> — under the platform root apex but
# outside the tenant's own apex
# prefix-adjacent evil<apex> — dot-boundary guard: a naive suffix
# check without the leading dot would match "<apex>"
# inside "evil<apex>"; the CEL uses endsWith("." + apex)
# foreign-tls-host in-apex rule host, but a spec.tls[].hosts[] entry
# under the root apex — the tls branch denies on its own
# wildcard-matches-apex *.<root-parent> is "outside" the root apex by a
# plain string test, but at ingress-nginx it also
# matches the bare root apex; the outside-root path is
# gated on the host not being a wildcard
- name: foreign-and-adjacent-hosts-denied
try:
- script:
content: |
set -eu
ns="$NAMESPACE"
apex=$(kubectl get namespace "$ns" -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}')
root=$(kubectl get namespace tenant-root -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}')
[ -n "$apex" ] || { echo "SETUP FAILURE: $ns lacks the namespace.cozystack.io/host label" >&2; exit 1; }
[ -n "$root" ] || { echo "SETUP FAILURE: tenant-root lacks the namespace.cozystack.io/host label" >&2; exit 1; }
[ "$apex" != "$root" ] || { echo "SETUP FAILURE: tenant apex equals root apex; under-root-outside-own cases are undefined" >&2; exit 1; }
# deny <case> <ingress-name> <extra-grep|""> reads a manifest on
# stdin and expects admission to reject it with a VAP message.
deny() {
kubectl -n "$ns" delete ingress "$2" --ignore-not-found >/dev/null
out=$(kubectl apply -n "$ns" -f - 2>&1) && rc=0 || rc=$?
echo "case=$1 rc=$rc"
if [ "$rc" -eq 0 ]; then
kubectl -n "$ns" delete ingress "$2" --ignore-not-found
echo "BUG: admission accepted $1: $out" >&2
exit 1
fi
echo "$out" | grep -qi "ValidatingAdmissionPolicy"
[ -z "$3" ] || echo "$out" | grep -qi "$3"
}
deny foreign-apex-host ingress-out-of-apex-host-probe "must be within" <<EOF
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: ingress-out-of-apex-host-probe
spec:
rules:
- host: "dashboard.${root}"
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: noop
port:
number: 80
EOF
deny prefix-adjacent ingress-prefix-adjacent-probe "" <<EOF
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: ingress-prefix-adjacent-probe
spec:
rules:
- host: "evil${apex}"
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: noop
port:
number: 80
EOF
deny foreign-tls-host ingress-tls-out-of-apex-probe "must be within" <<EOF
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: ingress-tls-out-of-apex-probe
spec:
tls:
- hosts:
- "harbor.${root}"
secretName: noop-tls
rules:
- host: "harbor.${apex}"
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: noop
port:
number: 80
EOF
case "$root" in
*.*) w="*.${root#*.}" ;;
*) w="*.${root}" ;;
esac
deny wildcard-matches-apex ingress-wildcard-host-probe "" <<EOF
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: ingress-wildcard-host-probe
spec:
rules:
- host: "${w}"
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: noop
port:
number: 80
EOF
check:
($error == null): true
# Static rejects — catch-all shapes on the shared ingress-nginx:
# hostless a rule with no .host matches all inbound HTTP
# empty-host host: "" is the same catch-all; the CEL denies it via
# r.host != "" (or has(r.host) if the apiserver drops it)
# defaultBackend catch-all for otherwise-unmatched traffic
- name: catch-all-shapes-denied
try:
- script:
content: |
set -eu
ns="$NAMESPACE"
deny() {
kubectl -n "$ns" delete ingress "$2" --ignore-not-found >/dev/null
out=$(kubectl apply -n "$ns" -f - 2>&1) && rc=0 || rc=$?
echo "case=$1 rc=$rc"
if [ "$rc" -eq 0 ]; then
kubectl -n "$ns" delete ingress "$2" --ignore-not-found
echo "BUG: admission accepted $1: $out" >&2
exit 1
fi
echo "$out" | grep -qi "ValidatingAdmissionPolicy"
}
deny hostless-rule ingress-hostless-probe <<'EOF'
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: ingress-hostless-probe
spec:
rules:
- http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: noop
port:
number: 80
EOF
deny empty-host ingress-empty-host-probe <<'EOF'
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: ingress-empty-host-probe
spec:
rules:
- host: ""
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: noop
port:
number: 80
EOF
deny default-backend ingress-defaultbackend-probe <<'EOF'
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: ingress-defaultbackend-probe
spec:
defaultBackend:
service:
name: noop
port:
number: 80
EOF
check:
($error == null): true
# Inverse sanity checks under the tenant's own apex: an in-apex rule+tls
# Ingress and a wildcard under the own apex (*.<apex>) must NOT be rejected
# — the anti-wildcard gate applies only to the outside-root path. Created
# inside the script (they need ${apex} interpolation), so also deleted here.
- name: own-apex-allowed
try:
- script:
content: |
set -eu
ns="$NAMESPACE"
apex=$(kubectl get namespace "$ns" -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}')
[ -n "$apex" ] || { echo "SETUP FAILURE: $ns lacks the namespace.cozystack.io/host label" >&2; exit 1; }
kubectl apply -n "$ns" -f - <<EOF
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: ingress-hostname-allow-probe
spec:
tls:
- hosts:
- "harbor.${apex}"
secretName: noop-tls
rules:
- host: "harbor.${apex}"
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: noop
port:
number: 80
EOF
kubectl -n "$ns" delete ingress ingress-hostname-allow-probe --ignore-not-found
kubectl apply -n "$ns" -f - <<EOF
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: ingress-ownapex-wildcard-probe
spec:
rules:
- host: "*.${apex}"
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: noop
port:
number: 80
EOF
kubectl -n "$ns" delete ingress ingress-ownapex-wildcard-probe --ignore-not-found
check:
($error == null): true
# Case-insensitivity. spec.rules[].host and spec.tls[].hosts[] are always
# lowercase — the apiserver's host validation permits only lowercase, so an
# uppercase host is rejected before the VAP runs — but the apex operand is the
# namespace.cozystack.io/host label VALUE, and Kubernetes permits uppercase in
# a label value. Under a mixed-case apex a lowercase in-apex Ingress must
# still be admitted, so the VAP normalizes both operands with .lowerAscii().
# Ingress is the default publish path and carries the most complex CEL (a
# rule-host and a tls-host branch), so it gets its own runtime guard beside
# the route one; both branches are exercised at once because the tls branch
# denies on its own.
#
# The probe apex is derived as Case-Probe.<root> rather than hardcoded, and it
# has to sit UNDER the platform root to mean anything: the outside-root branch
# admits any non-wildcard host outside the root apex whatever the tenant apex
# says, so a probe on an unrelated domain would be admitted pre-fix and prove
# nothing. Under the root that escape is closed — pre-fix the own-apex branch
# misses on case and the outside-root branch is excluded, so admission denies
# and this step is red; post-fix the own-apex branch matches and it is green.
#
# The mixed-case apex goes on a throwaway tenant-* namespace rather than on
# tenant-test, for the same reason as the route case. Built in a script since
# it needs ${root} interpolation, so the namespace is torn down in finally
# rather than by Chainsaw auto-cleanup. That teardown is also why the allow and
# deny halves share ONE step here while the route case splits them in two:
# finally runs at step scope, so a second step would find the namespace gone.
# The route namespace is applied declaratively and reclaimed at test scope, so
# it survives into its own deny step.
- name: mixed-case-apex-allowed-and-denied
try:
- script:
content: |
set -eu
root=$(kubectl get namespace tenant-root -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}')
[ -n "$root" ] || { echo "SETUP FAILURE: tenant-root lacks the namespace.cozystack.io/host label" >&2; exit 1; }
apex="Case-Probe.${root}"
host="probe.case-probe.${root}"
# Guard against a vacuous pass: the probe apex must really be
# mixed-case. An all-lowercase apex would admit the Ingress with or
# without the fix — this catches a later edit that lowercases the
# literal above, which the persisted-label check below cannot see.
[ "$apex" != "$(echo "$apex" | tr '[:upper:]' '[:lower:]')" ] || { echo "SETUP FAILURE: probe apex '$apex' is not mixed-case" >&2; exit 1; }
# Pre-clean a namespace left behind by an interrupted prior run.
kubectl delete namespace tenant-ingress-case-probe --ignore-not-found --wait=true --timeout=60s
kubectl apply -f - <<EOF
apiVersion: v1
kind: Namespace
metadata:
name: tenant-ingress-case-probe
labels:
namespace.cozystack.io/host: ${apex}
EOF
# Second vacuous-pass guard: re-read the persisted label, in case
# anything normalized the value on the way in.
got=$(kubectl get namespace tenant-ingress-case-probe -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}')
[ "$got" = "$apex" ] || { echo "SETUP FAILURE: apex label persisted as '$got', want '$apex'" >&2; exit 1; }
kubectl apply -n tenant-ingress-case-probe -f - <<EOF
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: ingress-hostname-case-probe
spec:
tls:
- hosts:
- "${host}"
secretName: noop-tls
rules:
- host: "${host}"
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: noop
port:
number: 80
EOF
# The deny half, pairing with the allow above the way
# foreign-and-adjacent-hosts-denied pairs with own-apex-allowed. The
# allow half alone only proves the normalized match does not
# UNDER-admit; over-admission is what hijacks a hostname, and
# normalizing both operands is exactly the edit that can widen a match
# by accident.
deny() {
kubectl -n tenant-ingress-case-probe delete ingress "$2" --ignore-not-found >/dev/null
out=$(kubectl apply -n tenant-ingress-case-probe -f - 2>&1) && rc=0 || rc=$?
echo "case=$1 rc=$rc"
if [ "$rc" -eq 0 ]; then
kubectl -n tenant-ingress-case-probe delete ingress "$2" --ignore-not-found
echo "BUG: admission accepted $1: $out" >&2
exit 1
fi
echo "$out" | grep -qi "ValidatingAdmissionPolicy"
}
# Under the platform root but outside the probe's own apex: the
# own-apex branch must miss on more than case, and the outside-root
# branch must stay closed under the root.
deny foreign-apex-host ingress-case-foreign-probe <<EOF
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: ingress-case-foreign-probe
spec:
rules:
- host: "dashboard.${root}"
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: noop
port:
number: 80
EOF
# Dot-boundary guard on the normalized path: "evilcase-probe.<root>"
# ends with the lowercased apex "case-probe.<root>" without being a
# subdomain of it. Only endsWith("." + apex) denies it; drop that
# leading dot and the apex is hijackable.
deny prefix-adjacent ingress-case-adjacent-probe <<EOF
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: ingress-case-adjacent-probe
spec:
rules:
- host: "evilcase-probe.${root}"
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: noop
port:
number: 80
EOF
check:
($error == null): true
finally:
- delete:
ref:
apiVersion: v1
kind: Namespace
name: tenant-ingress-case-probe
# Static accepts — concrete external domains outside the platform root apex
# on both spec.rules[].host and spec.tls[].hosts. Declarative apply; Chainsaw
# auto-cleanup deletes the probes.
- name: external-domain-allowed
try:
- apply:
file: ingress-external-allow.yaml
---
# Regression for #3983, which reported fail-closed admission webhooks becoming
# unreachable from the API server while an unrelated namespace was deleted. The
# mechanism is NOT confirmed: the errno in that report cannot come from a policy
# denial, and namespace deletion churns service backends as well as Cilium
# identities. So this case asserts the property the issue is about -- both
# webhooks still admit across a namespace deletion -- rather than claiming to
# reproduce a cause. Both affected paths are exercised immediately after the
# deletion, with no retry to hide a window.
#
# Every apply in this test passes through both fail-closed webhooks, so a
# cluster whose webhooks were ALREADY down fails here in a way that reads
# exactly like the outage under test. The first run of this case did precisely
# that: the ingress-nginx controller was crash-looping, the admission Service
# had no ready backend, and the setup apply died with "dial tcp ...: no route to
# host" before any churn happened. So availability is asserted before the churn
# as a precondition with its own message, and the churn verdict is only reached
# once both webhooks are known to have been up to begin with.
apiVersion: chainsaw.kyverno.io/v1alpha1
kind: Test
metadata:
name: admission-webhooks-survive-namespace-churn
spec:
catch:
- events: {}
steps:
- name: ingress-and-pod-admission-remain-available-after-deletion
try:
- script:
# Above the sum of the inner budgets, per the rule in
# hack/e2e-chainsaw/.chainsaw.yaml: 2x90s of webhook preconditions, 60s
# of pre-clean delete, 90s waiting for the churn pod, 180s for the
# namespace delete = 510s, and timeouts.exec would otherwise cap the
# whole script at 5m and SIGKILL it with no verdict.
timeout: 12m
content: |
set -eu
ns="$NAMESPACE"
churn_ns=tenant-admission-churn-probe
apex=$(kubectl get namespace "$ns" -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}')
[ -n "$apex" ] || { echo "SETUP FAILURE: $ns lacks the namespace.cozystack.io/host label" >&2; exit 1; }
# Coordinates come from the webhook registrations themselves, which is
# what the API server dials, rather than from a guessed release name or
# namespace. A webhook that is not registered at all is a setup
# failure: nothing below could then prove anything about it.
webhook_services() {
kubectl get "$1" \
-o jsonpath='{range .items[*]}{range .webhooks[*]}{.name}{" "}{.clientConfig.service.namespace}{" "}{.clientConfig.service.name}{"\n"}{end}{end}'
}
# Ready endpoint addresses behind a Service. Counted from
# EndpointSlice .endpoints[].conditions.ready rather than from pod
# readiness, because that is the state the API server routes on, and
# because a `kubectl wait --for=condition=Ready` is satisfied
# transiently by a pod that is flapping.
ready_backends() {
n=$(kubectl -n "$1" get endpointslice -l "kubernetes.io/service-name=$2" \
-o jsonpath='{range .items[*]}{range .endpoints[*]}{.conditions.ready}{"\n"}{end}{end}' \
2>/dev/null | grep -c '^true$' || true)
[ -n "$n" ] || n=0
echo "$n"
}
# Distinct variable names from require_webhook_up below, deliberately:
# POSIX shell functions share one scope, so reusing `label` here
# appended each iteration's suffix to the caller's own copy and the
# second registration reported itself as "(1/2) (2/2)".
require_service_up() {
svclabel="$1"; svcns="$2"; svcname="$3"; svcbudget="$4"
svcdeadline=$(( $(date +%s) + svcbudget ))
while :; do
n=$(ready_backends "$svcns" "$svcname")
if [ "$n" -gt 0 ]; then
echo "PRECONDITION OK $svclabel ($svcns/$svcname) has $n ready backend(s)"
return 0
fi
[ "$(date +%s)" -lt "$svcdeadline" ] || break
sleep 5
done
echo "SETUP FAILURE: $svclabel ($svcns/$svcname) has no ready backend after ${svcbudget}s." >&2
echo " This test cannot tell a churn-induced outage from a webhook that was" >&2
echo " already down, and every apply below goes through it, so it stops here" >&2
echo " rather than reporting the wrong cause." >&2
kubectl -n "$svcns" get endpointslice -l "kubernetes.io/service-name=$svcname" -o yaml >&2 2>&1 || true
kubectl -n "$svcns" get pods -o wide >&2 2>&1 || true
return 1
}
# EVERY registration of the named webhook, not just the first. All of
# them fire on the applies below, so one being down aborts the apply
# and the failure would be reported as a churn-induced outage. The
# loop indexes with sed rather than piping into `while read`, because
# a pipeline's body runs in a subshell under dash and a `return` from
# it would not reach this function.
require_webhook_up() {
label="$1"; kind="$2"; hook="$3"; budget="$4"
coords=$(webhook_services "$kind" | awk -v h="$hook" '$1 == h { print $2, $3 }')
[ -n "$coords" ] || {
echo "SETUP FAILURE: no $kind registers webhook $hook, so its availability cannot be asserted" >&2
return 1
}
total=$(printf '%s\n' "$coords" | grep -c . || true)
[ -n "$total" ] || total=0
i=0
while [ "$i" -lt "$total" ]; do
i=$(( i + 1 ))
line=$(printf '%s\n' "$coords" | sed -n "${i}p")
require_service_up "$label ($i/$total)" "${line% *}" "${line#* }" "$budget" || return 1
done
return 0
}
require_webhook_up "ingress admission" validatingwebhookconfigurations \
validate.nginx.ingress.kubernetes.io 90
require_webhook_up "pod admission" mutatingwebhookconfigurations \
kube-ovn-webhook.cozystack.io 90
# Reproduce the incident shape: an ordinary tenant namespace carrying
# an admitted Ingress AND a running pod, deleted synchronously, with
# the fail-closed webhooks invoked while Cilium processes the removal.
#
# The pod is what makes this a churn test rather than a smoke test. A
# Cilium identity is the set of an endpoint's identity-relevant
# labels, and k8s:io.kubernetes.pod.namespace is one of them, so a pod
# in a brand-new namespace allocates a NEW identity, and deleting the
# namespace releases it. An empty namespace has no endpoint, allocates
# no identity, and its deletion drives no identity churn at all, so
# without the pod this case would exercise nothing it names.
kubectl delete namespace "$churn_ns" --ignore-not-found --wait=true --timeout=60s
kubectl apply -f - <<EOF
apiVersion: v1
kind: Namespace
metadata:
name: ${churn_ns}
labels:
namespace.cozystack.io/host: churn.${apex}
EOF
kubectl apply -n "$churn_ns" -f - <<EOF
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: admission-churn-source
spec:
rules:
- host: probe.churn.${apex}
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: noop
port:
number: 80
EOF
# busybox:musl is staged onto every node by hack/e2e-prepull-images.sh,
# so this adds no image pull. The securityContext satisfies the
# restricted PodSecurity profile, so it runs whatever level the tenant
# namespace enforces.
kubectl apply -n "$churn_ns" -f - <<'EOF'
apiVersion: v1
kind: Pod
metadata:
name: admission-churn-identity
spec:
restartPolicy: Never
automountServiceAccountToken: false
securityContext:
runAsNonRoot: true
runAsUser: 65534
runAsGroup: 65534
seccompProfile:
type: RuntimeDefault
containers:
- name: hold
image: busybox:musl
command: ["sleep", "600"]
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
EOF
kubectl -n "$churn_ns" wait --for=condition=Ready pod/admission-churn-identity --timeout=90s || {
echo "SETUP FAILURE: the churn pod never became Ready, so deleting $churn_ns releases no Cilium identity and this test would assert nothing" >&2
kubectl -n "$churn_ns" describe pod admission-churn-identity >&2 2>&1 || true
exit 1
}
kubectl delete namespace "$churn_ns" --wait=true --timeout=180s
# No retry on either call: a retry would paper over exactly the window
# this test exists to catch.
#
# ingress-nginx validating admission
kubectl apply --dry-run=server -n "$ns" -o name -f - <<EOF
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: admission-after-churn
spec:
rules:
- host: admission-after-churn.${apex}
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: noop
port:
number: 80
EOF
# kubeovn-webhook mutating admission. The securityContext is carried
# even though a server dry-run Pod is never scheduled: if the tenant
# namespace ever enforces restricted PodSecurity, a non-compliant Pod
# is rejected in the validating phase and `set -eu` would abort this
# check, which reads as the webhook being down.
kubectl apply --dry-run=server -n "$ns" -o name -f - <<'EOF'
apiVersion: v1
kind: Pod
metadata:
name: admission-after-churn
spec:
restartPolicy: Never
automountServiceAccountToken: false
securityContext:
runAsNonRoot: true
runAsUser: 65534
runAsGroup: 65534
seccompProfile:
type: RuntimeDefault
containers:
- name: pause
image: registry.k8s.io/pause:3.10
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
EOF
echo "OK both fail-closed webhooks admitted after $churn_ns was deleted"
check:
($error == null): true
finally:
- delete:
ref:
apiVersion: v1
kind: Namespace
name: tenant-admission-churn-probe
---
# Companion to admission-webhooks-survive-namespace-churn above. That test
# proves the webhooks stay AVAILABLE; this one proves they are still
# UNREACHABLE from pods, which is the property the policies exist for, and that
# the deny is scoped to the webhook port alone.
#
# The third assertion is the one that earns this test. The policies previously
# expressed the Service-frontend deny with `toServices`, which Cilium resolves
# only inside ALLOW rules -- leaving the deny rule with an empty L3 selector
# that Cilium reads as a wildcard, so "deny this Service on 443" became "deny
# EVERY destination on 443" for every selected pod. Every rendered-YAML
# assertion in the package suites passed on that policy. Only a probe that
# checks an unrelated 443 destination still works can tell the two apart, and
# the ingress controller's own HTTPS port is exactly that: the same pods, the
# same tenant egress allowances, a different port.
#
# Runs in tenant-root because that is where the host cluster's ingress-nginx
# lives and where pods are permitted to reach it (allow-internal-communication),
# so the deny is the only thing that can block the webhook port. From the
# isolated tenant-test the probe would pass for the wrong reason.
#
# And "for the wrong reason" is the whole difficulty, because a refused connect
# does not say why it was refused. "Denied by policy" and "no backend to route
# to" are the same observation from inside the probe, so both deny assertions
# would report success against a cluster with no policy installed at all, as
# long as the backend was dead. That is what the first run of this case did:
# the controller was crash-looping on a ~100s cycle, `kubectl wait` caught it
# during a Ready window, the probe then measured a corpse, and the port-scoping
# assertion reported a policy bug that did not exist.
#
# So the deny verdicts here are gated on the admission Service having ready
# backends before the probe, on every sample while it runs, and after it
# finishes, and on the controller pods not having restarted or been replaced
# across that window. The probe itself only reports reachable/unreachable per
# destination and always exits 0; this script attributes the cause, because it
# is the half that holds the endpoint evidence. Anything that leaves the
# question undecidable fails as a setup failure rather than as a verdict.
apiVersion: chainsaw.kyverno.io/v1alpha1
kind: Test
metadata:
name: admission-webhook-deny-still-enforced
spec:
namespace: tenant-root
catch:
- events: {}
steps:
- name: pods-cannot-reach-the-webhook-but-keep-unrelated-443-egress
try:
- script:
# Above the sum of the inner budgets, per the rule in
# hack/e2e-chainsaw/.chainsaw.yaml: 120s of endpoint precondition, 30s
# of stability sampling, 60s to clear a leftover probe and 180s of
# phase polling = 390s. The previous 6m sat BELOW its own sum, so a
# slow-but-healthy cluster was killed before the diagnostics ran.
timeout: 10m
content: |
set -eu
ns="$NAMESPACE"
probe=admission-deny-probe
sel=app.kubernetes.io/name=ingress-nginx,app.kubernetes.io/component=controller
svcs=$(kubectl -n "$ns" get svc -l "$sel" \
-o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
adm_candidates=$(printf '%s\n' "$svcs" | grep -- '-admission$' || true)
# -metrics is excluded explicitly rather than relying on
# ingress-nginx-controller sorting before a metrics Service: the
# positive control has to be the one unambiguous controller Service.
ctl_candidates=$(printf '%s\n' "$svcs" | grep -v -- '-admission$' | grep -v -- '-metrics$' | grep . || true)
adm_count=$(printf '%s\n' "$adm_candidates" | grep -c . || true)
ctl_count=$(printf '%s\n' "$ctl_candidates" | grep -c . || true)
[ "$adm_count" -eq 1 ] || {
echo "SETUP FAILURE: expected one ingress-nginx admission Service in $ns, found $adm_count" >&2
printf ' candidates: %s\n' "${adm_candidates:-<none>}" >&2
exit 1
}
[ "$ctl_count" -eq 1 ] || {
echo "SETUP FAILURE: expected one ingress-nginx controller Service in $ns, found $ctl_count" >&2
printf ' candidates: %s\n' "${ctl_candidates:-<none>}" >&2
exit 1
}
adm=$adm_candidates
ctl=$ctl_candidates
service_port() {
kubectl -n "$ns" get svc "$1" -o json | jq -r --arg port_name "$2" '
[.spec.ports[]?
| select(.name == $port_name)
| select((.protocol // "TCP") == "TCP")
| .port
| select(type == "number")]
| if length == 1 then .[0] else empty end
'
}
adm_port=$(service_port "$adm" https-webhook)
ctl_port=$(service_port "$ctl" https)
adm_cluster_ip=$(kubectl -n "$ns" get svc "$adm" -o jsonpath='{.spec.clusterIP}')
[ -n "$adm_cluster_ip" ] && [ "$adm_cluster_ip" != None ] || {
echo "SETUP FAILURE: $adm must have a real ClusterIP, found ${adm_cluster_ip:-<empty>}" >&2
kubectl -n "$ns" get svc "$adm" -o yaml >&2 2>&1 || true
exit 1
}
[ -n "$adm_port" ] || {
echo "SETUP FAILURE: $adm must expose exactly one TCP https-webhook port" >&2
kubectl -n "$ns" get svc "$adm" -o yaml >&2 2>&1 || true
exit 1
}
[ -n "$ctl_port" ] || {
echo "SETUP FAILURE: $ctl must expose exactly one TCP https port" >&2
kubectl -n "$ns" get svc "$ctl" -o yaml >&2 2>&1 || true
exit 1
}
# Ready endpoint addresses behind a Service, from EndpointSlice rather
# than from pod readiness: this is the state the API server and the
# service load balancer route on.
ready_backends() {
n=$(kubectl -n "$ns" get endpointslice -l "kubernetes.io/service-name=$1" \
-o jsonpath='{range .items[*]}{range .endpoints[*]}{.conditions.ready}{"\n"}{end}{end}' \
2>/dev/null | grep -c '^true$' || true)
[ -n "$n" ] || n=0
echo "$n"
}
# Exact ready admission coordinates. The Pod identity, IP and backend
# port come from the same EndpointSlice as the ready condition;
# selecting them independently from .items[0] could probe an unready
# replica while a different replica kept aggregate readiness above
# zero.
ready_admission_backends() {
kubectl -n "$ns" get endpointslice -l "kubernetes.io/service-name=$adm" -o json | jq -r '
.items[]? as $slice
| $slice.ports[]? as $port
| select($port.name == "https-webhook")
| select(($port.protocol // "TCP") == "TCP")
| select(($port.port | type) == "number")
| $slice.endpoints[]?
| select(.conditions.ready == true)
| select(.targetRef.kind == "Pod")
| select((.targetRef.name // "") != "")
| select((.targetRef.uid // "") != "")
| .targetRef.name as $pod
| .targetRef.uid as $uid
| .addresses[]?
| select(type == "string" and length > 0)
| [$pod, $uid, ., ($port.port | tostring)]
| @tsv
' | sort -u
}
# Pod identity, IP and every container restart count, as one string.
# Compared for equality across the probe window, so a restart AND a
# replacement both show up -- a crash-looping controller can satisfy
# a readiness check at both ends of a window it spent dying in the
# middle of.
controller_generation() {
generation=$(kubectl -n "$ns" get pod -l "$sel" \
-o jsonpath='{range .items[*]}{.metadata.name}/{.metadata.uid}/{.status.podIP}={range .status.containerStatuses[*]}{.name}:{.restartCount},{end}{"\n"}{end}') || {
echo "SETUP FAILURE: unable to read ingress-nginx controller generation" >&2
return 1
}
[ -n "$generation" ] || {
echo "SETUP FAILURE: ingress-nginx controller generation is empty" >&2
return 1
}
printf '%s\n' "$generation" | sort
}
# Precondition. Both Services are backed by the same controller pods,
# so requiring the admission Service to be live also establishes that
# the controller Service -- the positive control below -- has a
# backend to answer.
deadline=$(( $(date +%s) + 120 ))
while :; do
adm_ready=$(ready_admission_backends | grep -c . || true)
ctl_ready=$(ready_backends "$ctl")
if [ "$adm_ready" -gt 0 ] && [ "$ctl_ready" -gt 0 ]; then break; fi
[ "$(date +%s)" -lt "$deadline" ] || {
echo "SETUP FAILURE: no ready backend for $adm ($adm_ready) or $ctl ($ctl_ready) after 120s." >&2
echo " A refused connect cannot be attributed to policy while the destination" >&2
echo " has nothing listening, so this test stops instead of reporting a deny" >&2
echo " it did not observe." >&2
kubectl -n "$ns" get endpointslice -l "kubernetes.io/service-name=$adm" -o yaml >&2 2>&1 || true
kubectl -n "$ns" get pod -l "$sel" -o wide >&2 2>&1 || true
exit 1
}
sleep 5
done
backend=$(ready_admission_backends | head -1)
[ -n "$backend" ] || { echo "SETUP FAILURE: $adm has no complete ready Pod/UID/IP/port coordinate" >&2; exit 1; }
tab=$(printf '\t')
backend_fields=$(printf '%s\n' "$backend" | awk -F "$tab" '{ print NF }')
[ "$backend_fields" -eq 4 ] || { echo "SETUP FAILURE: malformed ready backend coordinate: $backend" >&2; exit 1; }
pod_name=$(printf '%s\n' "$backend" | cut -f1)
pod_uid=$(printf '%s\n' "$backend" | cut -f2)
pod_ip=$(printf '%s\n' "$backend" | cut -f3)
hook_port=$(printf '%s\n' "$backend" | cut -f4)
[ -n "$pod_name" ] && [ -n "$pod_uid" ] && [ -n "$pod_ip" ] && [ -n "$hook_port" ] || {
echo "SETUP FAILURE: incomplete ready backend coordinate: $backend" >&2
exit 1
}
selected_backend_ready() {
ready_admission_backends | awk -F "$tab" \
-v pod="$pod_name" -v uid="$pod_uid" -v ip="$pod_ip" -v port="$hook_port" '
$1 == pod && $2 == uid && $3 == ip && $4 == port { found = 1 }
END { exit !found }
'
}
require_selected_backend() {
selected_backend_ready && return 0
echo "SETUP FAILURE: selected admission backend $pod_name/$pod_uid ($pod_ip:$hook_port) was not ready $1" >&2
kubectl -n "$ns" get endpointslice -l "kubernetes.io/service-name=$adm" -o yaml >&2 2>&1 || true
kubectl -n "$ns" get pod "$pod_name" -o wide >&2 2>&1 || true
exit 1
}
require_selected_backend "before the stability window"
# Stability: the same evidence held across a window, not at one
# instant. Six samples over 30s, all of which must be live and show no
# restart or replacement.
gen_before=$(controller_generation)
printf '%s\n' "$gen_before" | awk \
-v prefix="${pod_name}/${pod_uid}/${pod_ip}=" '
index($0, prefix) == 1 { found = 1 }
END { exit !found }
' || {
echo "SETUP FAILURE: selected admission backend is not a current controller Pod: $backend" >&2
exit 1
}
i=0
while [ "$i" -lt 6 ]; do
sleep 5
i=$(( i + 1 ))
require_selected_backend "during the stability window (sample $i)"
n=$(ready_backends "$ctl")
[ "$n" -gt 0 ] || {
echo "SETUP FAILURE: $ctl lost its ready backend during the stability window (sample $i)" >&2
kubectl -n "$ns" get pod -l "$sel" -o wide >&2 2>&1 || true
exit 1
}
done
gen_stable=$(controller_generation)
[ "$gen_stable" = "$gen_before" ] || {
echo "SETUP FAILURE: the ingress-nginx controller restarted or was replaced during the stability window" >&2
echo " before: $gen_before" >&2
echo " after: $gen_stable" >&2
kubectl -n "$ns" get pod -l "$sel" -o wide >&2 2>&1 || true
exit 1
}
echo "probing from $ns: expect reachable ${ctl}:${ctl_port}, denied ${adm}@${adm_cluster_ip}:${adm_port}, denied ${pod_name}/${pod_uid}@${pod_ip}:${hook_port}"
kubectl -n "$ns" delete pod "$probe" --ignore-not-found --wait=true --timeout=60s
# busybox:musl is staged onto every node by hack/e2e-prepull-images.sh,
# so this adds no image pull. The securityContext satisfies the
# restricted PodSecurity profile, so the probe runs in a tenant
# namespace whatever level is enforced there.
kubectl -n "$ns" apply -f - <<EOF
apiVersion: v1
kind: Pod
metadata:
name: ${probe}
spec:
restartPolicy: Never
automountServiceAccountToken: false
securityContext:
runAsNonRoot: true
runAsUser: 65534
runAsGroup: 65534
seccompProfile:
type: RuntimeDefault
containers:
- name: probe
image: busybox:musl
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
env:
- name: CTL_HOST
value: "${ctl}.${ns}.svc"
- name: CTL_PORT
value: "${ctl_port}"
- name: ADM_HOST
value: "${adm_cluster_ip}"
- name: ADM_PORT
value: "${adm_port}"
- name: POD_IP
value: "${pod_ip}"
- name: HOOK_PORT
value: "${hook_port}"
command: ["sh", "-c"]
args:
# Measurement only, and always exit 0. The pod reports what it
# observed per destination; the script that started it holds the
# endpoint evidence and is the only place that can say why a
# connect was refused. Keeping the verdict out of here is also
# what stops "the probe could not run" from being read as "the
# assertion failed".
- |
set -u
reach() { nc -w 5 "\$1" "\$2" </dev/null >/dev/null 2>&1; }
report() {
if reach "\$2" "\$3"; then
echo "\$1 reachable \$2:\$3"
else
echo "\$1 unreachable \$2:\$3"
fi
}
report CTL "\$CTL_HOST" "\$CTL_PORT"
report ADM "\$ADM_HOST" "\$ADM_PORT"
report HOOK "\$POD_IP" "\$HOOK_PORT"
EOF
# Wait for a terminal phase rather than a condition, and re-check the
# backend on every poll: a controller that dies mid-probe invalidates
# the deny verdicts, and this is the window it would die in.
deadline=$(( $(date +%s) + 180 ))
phase=""
while [ "$(date +%s)" -lt "$deadline" ]; do
require_selected_backend "while the probe was running"
n=$(ready_backends "$ctl")
[ "$n" -gt 0 ] || {
echo "SETUP FAILURE: $ctl lost its ready backend while the probe was running, so an unreachable positive control proves nothing" >&2
kubectl -n "$ns" get pod -l "$sel" -o wide >&2 2>&1 || true
kubectl -n "$ns" logs "$probe" >&2 2>&1 || true
exit 1
}
phase=$(kubectl -n "$ns" get pod "$probe" -o jsonpath='{.status.phase}' 2>/dev/null || true)
case "$phase" in Succeeded|Failed) break ;; esac
sleep 3
done
log=$(kubectl -n "$ns" logs "$probe" 2>&1 || true)
printf '%s\n' "$log"
# Postcondition, before the log is interpreted at all.
[ "$phase" = "Succeeded" ] || {
echo "SETUP FAILURE: the probe did not run to completion (phase=${phase:-unknown})" >&2
kubectl -n "$ns" describe pod "$probe" >&2 2>&1 || true
exit 1
}
gen_after=$(controller_generation)
[ "$gen_after" = "$gen_before" ] || {
echo "SETUP FAILURE: the ingress-nginx controller restarted or was replaced during the probe" >&2
echo " before: $gen_before" >&2
echo " after: $gen_after" >&2
kubectl -n "$ns" get pod -l "$sel" -o wide >&2 2>&1 || true
exit 1
}
require_selected_backend "after the probe"
n=$(ready_backends "$ctl")
[ "$n" -gt 0 ] || {
echo "SETUP FAILURE: $ctl has no ready backend after the probe, so its unreachability proves nothing" >&2
exit 1
}
verdict() { printf '%s\n' "$log" | awk -v k="$1" '$1 == k { print $2; exit }'; }
ctl_seen=$(verdict CTL)
adm_seen=$(verdict ADM)
hook_seen=$(verdict HOOK)
for v in "$ctl_seen" "$adm_seen" "$hook_seen"; do
[ -n "$v" ] || {
echo "SETUP FAILURE: the probe did not report all three destinations" >&2
exit 1
}
done
# From here every branch is a real verdict: the exact direct
# destination stayed ready before, throughout and after, and the
# controller pods neither restarted nor were replaced.
rc=0
if [ "$ctl_seen" = reachable ]; then
echo "OK unrelated 443 egress works (${ctl}:${ctl_port})"
else
echo "BUG unrelated 443 egress DENIED to a live backend (${ctl}:${ctl_port}) -- the webhook deny is not scoped to the webhook port" >&2
rc=1
fi
if [ "$adm_seen" = unreachable ]; then
echo "OK webhook denied through its Service ClusterIP"
else
echo "BUG webhook reachable through its Service ClusterIP (${adm}:${adm_port})" >&2
rc=1
fi
if [ "$hook_seen" = unreachable ]; then
echo "OK webhook denied on its backend port"
else
echo "BUG webhook reachable directly on its backend port (${pod_ip}:${hook_port})" >&2
rc=1
fi
exit "$rc"
check:
($error == null): true
finally:
- delete:
ref:
apiVersion: v1
kind: Pod
name: admission-deny-probe