mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 20:23:39 +09:00
The TenantGateway controller withheld the HTTPS-terminate listener for a name a passthrough listener answers only once a TLSRoute was attached there and resolved a backend, mirroring how Cilium builds its filter chains. That kept a served endpoint up on Cilium 1.19.5, but it tied every route's status to one implementation's translator and needed Service and ReferenceGrant watches to land the deferred decision. The shipped operator accepts a wildcard terminate listener beside a narrower passthrough listener on 443 and keeps conflicting every other overlap. A terminate listener for the passthrough name itself is not narrower, so on 443 the rule now protects the dataplane outright; on a native port Gateway API would serve the pair, and the rule holds as a policy that keeps a platform name with its passthrough listener. Reading it off the spec makes the outcome the same on every implementation. Under http01 an HTTPRoute claiming such a name gets no listener and no certificate whether or not a TLSRoute is attached. A route whose every hostname is reserved reads Accepted=False with NoMatchingListenerHostname; one that also claims a free hostname stays Accepted and is served there, as Gateway API accepts a route whose hostnames intersect at least one listener. dns01 and existingSecret are unchanged: their wildcard listener still covers such a name, and the passthrough listener answers it by its exact server name. The chart writes the passthrough services on the publishing tenant alone, so the names reserved by default are the ones the platform publishes there through its own TLSRoutes. The shape an upgrade changes is an HTTPRoute on such a name whose platform TLSRoute forwards nowhere, because it never attached (the package owning the service is not installed) or because none of its backends resolve: that route kept a terminate listener and now gets none. The backend resolution, its watches and the RBAC rules that served them go away. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <f@lex.la>
2389 lines
95 KiB
YAML
2389 lines
95 KiB
YAML
# Chainsaw port of hack/e2e-apps/gateway.bats (29 cases).
|
|
#
|
|
# Declarative cases (CRD/GatewayClass/Gateway/HTTPRoute/TenantGateway status,
|
|
# VAP existence) use asserts. Admission-rejection cases need kubectl
|
|
# impersonation or error-message greps that Chainsaw `apply` can't express, so
|
|
# they stay scripts. The tenant apex is read at runtime from the tenant-test
|
|
# namespace's namespace.cozystack.io/host label, so these tests pass both in CI
|
|
# (test.example.org) and on a dev cluster with a different host.
|
|
apiVersion: chainsaw.kyverno.io/v1alpha1
|
|
kind: Test
|
|
metadata:
|
|
name: gateway-api-crds
|
|
spec:
|
|
steps:
|
|
- name: crds-established-and-gatewayclass-accepted
|
|
try:
|
|
- assert:
|
|
timeout: 1m
|
|
resource:
|
|
apiVersion: apiextensions.k8s.io/v1
|
|
kind: CustomResourceDefinition
|
|
metadata:
|
|
name: gatewayclasses.gateway.networking.k8s.io
|
|
status:
|
|
(conditions[?type == 'Established']):
|
|
- status: "True"
|
|
- assert:
|
|
timeout: 1m
|
|
resource:
|
|
apiVersion: apiextensions.k8s.io/v1
|
|
kind: CustomResourceDefinition
|
|
metadata:
|
|
name: gateways.gateway.networking.k8s.io
|
|
status:
|
|
(conditions[?type == 'Established']):
|
|
- status: "True"
|
|
- assert:
|
|
timeout: 1m
|
|
resource:
|
|
apiVersion: apiextensions.k8s.io/v1
|
|
kind: CustomResourceDefinition
|
|
metadata:
|
|
name: httproutes.gateway.networking.k8s.io
|
|
status:
|
|
(conditions[?type == 'Established']):
|
|
- status: "True"
|
|
- assert:
|
|
timeout: 3m
|
|
resource:
|
|
apiVersion: gateway.networking.k8s.io/v1
|
|
kind: GatewayClass
|
|
metadata:
|
|
name: cilium
|
|
status:
|
|
(conditions[?type == 'Accepted']):
|
|
- status: "True"
|
|
---
|
|
apiVersion: chainsaw.kyverno.io/v1alpha1
|
|
kind: Test
|
|
metadata:
|
|
name: gateway-minimal-programmed
|
|
spec:
|
|
catch:
|
|
- events: {}
|
|
- describe:
|
|
apiVersion: gateway.networking.k8s.io/v1
|
|
kind: Gateway
|
|
name: gateway-e2e-probe
|
|
steps:
|
|
- name: create-minimal-gateway
|
|
try:
|
|
- apply:
|
|
file: gw-minimal.yaml
|
|
- name: assert-accepted-programmed
|
|
try:
|
|
- assert:
|
|
timeout: 2m
|
|
resource:
|
|
apiVersion: gateway.networking.k8s.io/v1
|
|
kind: Gateway
|
|
metadata:
|
|
name: gateway-e2e-probe
|
|
status:
|
|
(conditions[?type == 'Accepted']):
|
|
- status: "True"
|
|
(conditions[?type == 'Programmed']):
|
|
- status: "True"
|
|
# Cilium materialises a LoadBalancer Service per programmed Gateway.
|
|
- assert:
|
|
timeout: 1m
|
|
resource:
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: cilium-gateway-gateway-e2e-probe
|
|
---
|
|
apiVersion: chainsaw.kyverno.io/v1alpha1
|
|
kind: Test
|
|
metadata:
|
|
name: gateway-hostname-policy-installed
|
|
spec:
|
|
steps:
|
|
- name: vap-and-binding-and-host-label
|
|
try:
|
|
- assert:
|
|
resource:
|
|
apiVersion: admissionregistration.k8s.io/v1
|
|
kind: ValidatingAdmissionPolicy
|
|
metadata:
|
|
name: cozystack-gateway-hostname-policy
|
|
- assert:
|
|
resource:
|
|
apiVersion: admissionregistration.k8s.io/v1
|
|
kind: ValidatingAdmissionPolicyBinding
|
|
metadata:
|
|
name: cozystack-gateway-hostname-policy
|
|
# Binding must Deny — [Audit]/[] would let requests through silently.
|
|
(contains(spec.validationActions, 'Deny')): true
|
|
# The whole hostname policy keys off this label; assert it is set
|
|
# (script, not an assert: numeric/length compares on labels are unreliable).
|
|
- script:
|
|
content: |
|
|
set -eu
|
|
kubectl get namespace "$NAMESPACE" \
|
|
-o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}' | grep -q .
|
|
check:
|
|
($error == null): true
|
|
---
|
|
apiVersion: chainsaw.kyverno.io/v1alpha1
|
|
kind: Test
|
|
metadata:
|
|
name: gateway-foreign-hostname-rejected
|
|
spec:
|
|
steps:
|
|
# No mixed-case-apex counterpart here, unlike the Route and Ingress suites.
|
|
# Deliberate: this policy applies the same rule as the Route one — own apex
|
|
# or a dot-bounded subdomain of it, with no outside-root branch — so a
|
|
# Gateway probe would re-exercise the same two comparisons that
|
|
# route-hostname-policy already drives against a mixed-case apex at runtime,
|
|
# in both directions. The two differ only in how the apex reaches the
|
|
# comparison: a `variables.tenantHost` binding here, the label inlined there.
|
|
# Because that binding is what this suite does NOT cover, both of its
|
|
# comparisons are pinned operand-by-operand in
|
|
# tests/gateway-hostname-policy_test.yaml instead.
|
|
- name: foreign-listener-hostname-denied
|
|
try:
|
|
- script:
|
|
content: |
|
|
set -eu
|
|
out=$(kubectl apply -n "$NAMESPACE" -f - 2>&1 <<'EOF'
|
|
apiVersion: gateway.networking.k8s.io/v1
|
|
kind: Gateway
|
|
metadata:
|
|
name: hostname-hijack-probe
|
|
spec:
|
|
gatewayClassName: cilium
|
|
listeners:
|
|
- name: https
|
|
protocol: HTTPS
|
|
port: 443
|
|
hostname: "dashboard.example.org"
|
|
tls:
|
|
mode: Terminate
|
|
certificateRefs:
|
|
- name: noop
|
|
allowedRoutes:
|
|
namespaces:
|
|
from: Same
|
|
EOF
|
|
) && rc=0 || rc=$?
|
|
echo "rc=$rc out=$out"
|
|
if [ "$rc" -eq 0 ]; then
|
|
kubectl -n "$NAMESPACE" delete gateway hostname-hijack-probe --ignore-not-found
|
|
echo "BUG: admission accepted cross-tenant hostname" >&2
|
|
exit 1
|
|
fi
|
|
echo "$out" | grep -qi "ValidatingAdmissionPolicy"
|
|
echo "$out" | grep -q "must equal"
|
|
check:
|
|
($error == null): true
|
|
---
|
|
apiVersion: chainsaw.kyverno.io/v1alpha1
|
|
kind: Test
|
|
metadata:
|
|
name: package-attachednamespaces-accept
|
|
spec:
|
|
steps:
|
|
# Admission must accept a tenant-* entry in gateway.attachedNamespaces; a
|
|
# successful apply (and Chainsaw auto-cleanup of the Package) is the proof.
|
|
- name: apply-package-probe
|
|
try:
|
|
- apply:
|
|
file: package-attach-probe.yaml
|
|
---
|
|
apiVersion: chainsaw.kyverno.io/v1alpha1
|
|
kind: Test
|
|
metadata:
|
|
name: tenant-host-policy
|
|
spec:
|
|
steps:
|
|
- name: untrusted-sa-cannot-set-tenant-host
|
|
try:
|
|
- script:
|
|
content: |
|
|
set -eu
|
|
ns="$NAMESPACE"
|
|
# Authorization runs before admission; grant the SA RBAC to create
|
|
# Tenants so the apply reaches the VAP rather than failing on RBAC.
|
|
kubectl apply -f - <<EOF
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: Role
|
|
metadata:
|
|
name: vap-probe-tenant-create
|
|
namespace: ${ns}
|
|
rules:
|
|
- apiGroups: ["apps.cozystack.io"]
|
|
resources: ["tenants"]
|
|
verbs: ["create","get","list","watch","update","patch","delete"]
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: RoleBinding
|
|
metadata:
|
|
name: vap-probe-tenant-create
|
|
namespace: ${ns}
|
|
roleRef:
|
|
apiGroup: rbac.authorization.k8s.io
|
|
kind: Role
|
|
name: vap-probe-tenant-create
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: default
|
|
namespace: ${ns}
|
|
EOF
|
|
cleanup() {
|
|
# || true each delete: the trap runs under set -eu, and one
|
|
# transient delete failure must not skip the remaining cleanup.
|
|
kubectl -n "$ns" delete rolebinding vap-probe-tenant-create --ignore-not-found || true
|
|
kubectl -n "$ns" delete role vap-probe-tenant-create --ignore-not-found || true
|
|
kubectl -n "$ns" delete tenants.apps.cozystack.io vaphostprobe --ignore-not-found || true
|
|
}
|
|
trap cleanup EXIT
|
|
out=$(kubectl --as="system:serviceaccount:${ns}:default" \
|
|
--as-group=system:serviceaccounts \
|
|
--as-group="system:serviceaccounts:${ns}" \
|
|
apply -n "$ns" -f - 2>&1 <<EOF
|
|
apiVersion: apps.cozystack.io/v1alpha1
|
|
kind: Tenant
|
|
metadata:
|
|
name: vaphostprobe
|
|
namespace: ${ns}
|
|
spec:
|
|
host: foreign.example.org
|
|
EOF
|
|
) && rc=0 || rc=$?
|
|
echo "rc=$rc out=$out"
|
|
[ "$rc" -ne 0 ]
|
|
echo "$out" | grep -qi "ValidatingAdmissionPolicy"
|
|
echo "$out" | grep -q "spec.host can only be set"
|
|
check:
|
|
($error == null): true
|
|
---
|
|
apiVersion: chainsaw.kyverno.io/v1alpha1
|
|
kind: Test
|
|
metadata:
|
|
name: namespace-host-label-policy
|
|
spec:
|
|
steps:
|
|
- name: untrusted-sa-cannot-change-host-label
|
|
try:
|
|
- script:
|
|
content: |
|
|
set -eu
|
|
ns="$NAMESPACE"
|
|
host=$(kubectl get namespace "$ns" -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}')
|
|
kubectl apply -f - <<EOF
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: ClusterRole
|
|
metadata:
|
|
name: vap-probe-namespace-patch
|
|
rules:
|
|
- apiGroups: [""]
|
|
resources: ["namespaces"]
|
|
verbs: ["get","list","watch","update","patch"]
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: ClusterRoleBinding
|
|
metadata:
|
|
name: vap-probe-namespace-patch
|
|
roleRef:
|
|
apiGroup: rbac.authorization.k8s.io
|
|
kind: ClusterRole
|
|
name: vap-probe-namespace-patch
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: default
|
|
namespace: ${ns}
|
|
EOF
|
|
cleanup() {
|
|
# Restore the original host label in case admission let it through.
|
|
# || true each command: the trap runs under set -eu, and one
|
|
# transient failure must not skip the remaining cleanup.
|
|
kubectl label namespace "$ns" "namespace.cozystack.io/host=${host}" --overwrite >/dev/null 2>&1 || true
|
|
kubectl delete clusterrolebinding vap-probe-namespace-patch --ignore-not-found || true
|
|
kubectl delete clusterrole vap-probe-namespace-patch --ignore-not-found || true
|
|
}
|
|
trap cleanup EXIT
|
|
out=$(kubectl --as="system:serviceaccount:${ns}:default" \
|
|
--as-group=system:serviceaccounts \
|
|
--as-group="system:serviceaccounts:${ns}" \
|
|
label namespace "$ns" namespace.cozystack.io/host=foreign.example.org --overwrite 2>&1) && rc=0 || rc=$?
|
|
echo "rc=$rc out=$out"
|
|
[ "$rc" -ne 0 ]
|
|
echo "$out" | grep -qi "ValidatingAdmissionPolicy"
|
|
echo "$out" | grep -q "immutable"
|
|
check:
|
|
($error == null): true
|
|
- name: untrusted-sa-cannot-set-host-label-at-create
|
|
try:
|
|
- script:
|
|
content: |
|
|
set -eu
|
|
ns="$NAMESPACE"
|
|
kubectl apply -f - <<EOF
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: ClusterRole
|
|
metadata:
|
|
name: vap-probe-namespace-create
|
|
rules:
|
|
- apiGroups: [""]
|
|
resources: ["namespaces"]
|
|
verbs: ["create","get","list","delete"]
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: ClusterRoleBinding
|
|
metadata:
|
|
name: vap-probe-namespace-create
|
|
roleRef:
|
|
apiGroup: rbac.authorization.k8s.io
|
|
kind: ClusterRole
|
|
name: vap-probe-namespace-create
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: default
|
|
namespace: ${ns}
|
|
EOF
|
|
cleanup() {
|
|
# || true each delete: the trap runs under set -eu, and one
|
|
# transient delete failure must not skip the remaining cleanup.
|
|
kubectl delete namespace vap-host-label-probe --ignore-not-found --wait=false || true
|
|
kubectl delete clusterrolebinding vap-probe-namespace-create --ignore-not-found || true
|
|
kubectl delete clusterrole vap-probe-namespace-create --ignore-not-found || true
|
|
}
|
|
trap cleanup EXIT
|
|
out=$(kubectl --as="system:serviceaccount:${ns}:default" \
|
|
--as-group=system:serviceaccounts \
|
|
--as-group="system:serviceaccounts:${ns}" \
|
|
apply -f - 2>&1 <<EOF
|
|
apiVersion: v1
|
|
kind: Namespace
|
|
metadata:
|
|
name: vap-host-label-probe
|
|
labels:
|
|
namespace.cozystack.io/host: foreign.example.org
|
|
EOF
|
|
) && rc=0 || rc=$?
|
|
echo "rc=$rc out=$out"
|
|
[ "$rc" -ne 0 ]
|
|
echo "$out" | grep -qi "ValidatingAdmissionPolicy"
|
|
echo "$out" | grep -q "immutable"
|
|
check:
|
|
($error == null): true
|
|
---
|
|
apiVersion: chainsaw.kyverno.io/v1alpha1
|
|
kind: Test
|
|
metadata:
|
|
name: tenantgateway-crd
|
|
spec:
|
|
steps:
|
|
- name: crd-established
|
|
try:
|
|
- assert:
|
|
timeout: 1m
|
|
resource:
|
|
apiVersion: apiextensions.k8s.io/v1
|
|
kind: CustomResourceDefinition
|
|
metadata:
|
|
name: tenantgateways.gateway.cozystack.io
|
|
status:
|
|
(conditions[?type == 'Established']):
|
|
- status: "True"
|
|
---
|
|
apiVersion: chainsaw.kyverno.io/v1alpha1
|
|
kind: Test
|
|
metadata:
|
|
name: tenantgateway-reconciler
|
|
spec:
|
|
catch:
|
|
- events: {}
|
|
- describe:
|
|
apiVersion: gateway.cozystack.io/v1alpha1
|
|
kind: TenantGateway
|
|
name: tg-e2e-probe
|
|
steps:
|
|
- name: create-tenantgateway
|
|
try:
|
|
# apex is read from the namespace host label so the CR is valid on any host.
|
|
- script:
|
|
content: |
|
|
set -eu
|
|
ns="$NAMESPACE"
|
|
apex=$(kubectl get namespace "$ns" -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}')
|
|
kubectl apply -n "$ns" -f - <<EOF
|
|
apiVersion: gateway.cozystack.io/v1alpha1
|
|
kind: TenantGateway
|
|
metadata:
|
|
name: tg-e2e-probe
|
|
spec:
|
|
apex: ${apex}
|
|
certMode: http01
|
|
gatewayClassName: cilium
|
|
EOF
|
|
check:
|
|
($error == null): true
|
|
- name: assert-gateway-issuer-status
|
|
try:
|
|
- assert:
|
|
timeout: 2m
|
|
resource:
|
|
apiVersion: gateway.networking.k8s.io/v1
|
|
kind: Gateway
|
|
metadata:
|
|
name: tg-e2e-probe
|
|
- assert:
|
|
timeout: 2m
|
|
resource:
|
|
apiVersion: cert-manager.io/v1
|
|
kind: Issuer
|
|
metadata:
|
|
name: tg-e2e-probe-gateway
|
|
- assert:
|
|
timeout: 2m
|
|
resource:
|
|
apiVersion: gateway.cozystack.io/v1alpha1
|
|
kind: TenantGateway
|
|
metadata:
|
|
name: tg-e2e-probe
|
|
status:
|
|
(conditions[?type == 'Ready']):
|
|
- status: "True"
|
|
finally:
|
|
- delete:
|
|
ref:
|
|
apiVersion: gateway.cozystack.io/v1alpha1
|
|
kind: TenantGateway
|
|
name: tg-e2e-probe
|
|
---
|
|
apiVersion: chainsaw.kyverno.io/v1alpha1
|
|
kind: Test
|
|
metadata:
|
|
name: route-hostname-policy
|
|
spec:
|
|
steps:
|
|
- name: foreign-apex-route-denied
|
|
try:
|
|
- script:
|
|
content: |
|
|
set -eu
|
|
out=$(kubectl apply -n "$NAMESPACE" -f - 2>&1 <<'EOF'
|
|
apiVersion: gateway.networking.k8s.io/v1
|
|
kind: HTTPRoute
|
|
metadata:
|
|
name: route-hostname-hijack-probe
|
|
spec:
|
|
hostnames:
|
|
- "attacker.com"
|
|
rules:
|
|
- backendRefs:
|
|
- name: kubernetes
|
|
namespace: default
|
|
port: 443
|
|
EOF
|
|
) && rc=0 || rc=$?
|
|
echo "rc=$rc out=$out"
|
|
if [ "$rc" -eq 0 ]; then
|
|
kubectl -n "$NAMESPACE" delete httproute route-hostname-hijack-probe --ignore-not-found
|
|
echo "BUG: admission accepted cross-apex HTTPRoute hostname" >&2
|
|
exit 1
|
|
fi
|
|
echo "$out" | grep -qi "ValidatingAdmissionPolicy"
|
|
# Matches the apex-comparison half of the message, which is common to
|
|
# every denial this policy issues. Pinning the leading words instead
|
|
# would break whenever the message grows a clause for another shape.
|
|
echo "$out" | grep -qi "must equal the namespace"
|
|
check:
|
|
($error == null): true
|
|
# A route that declares no hostname at all is the one shape the expression
|
|
# used to skip: it opened with `!has(object.spec.hostnames) ||`, so such a
|
|
# route satisfied the first operand and was admitted without a single
|
|
# hostname being compared. That route is not inert — under Gateway API it
|
|
# inherits the hostnames of the listener it attaches to, and parentRefs may
|
|
# name a Gateway in another namespace, so it ends up covering whatever that
|
|
# listener covers.
|
|
#
|
|
# Both empty shapes are probed because they reach the hole by different
|
|
# routes and one guard does not imply the other: the field being absent is
|
|
# caught by has(), while an explicitly empty list satisfies has() and then
|
|
# passes because all() over an empty list is vacuously true. Dropping either
|
|
# term from the CEL re-admits the whole shape, and the unit suite can only
|
|
# pin that the terms are present in the expression — this step is what proves
|
|
# the apiserver acts on them.
|
|
#
|
|
# HTTPRoute carries both cases. The TLSRoute probe is applied at v1alpha2
|
|
# because v1 and v1alpha3 mark spec.hostnames required with minItems: 1,
|
|
# leaving v1alpha2 the only version where a hostname-less TLSRoute reaches
|
|
# admission at all. It and the GRPCRoute probe carry the absent case only,
|
|
# and that is a coverage choice rather than a reachability fact: v1alpha2
|
|
# sets neither constraint, and the grpcroutes schema sets neither at any
|
|
# served version, so an explicitly empty list reaches admission on both
|
|
# just as an absent field does. All three policies render one
|
|
# byte-identical CEL string, so the HTTPRoute pair above already proves
|
|
# the empty-list term end to end.
|
|
# Scripted rather than an apply/expect pair so the grep can prove the VAP
|
|
# did the denying: a bare "the apply failed" assertion would be satisfied
|
|
# by a typo in the manifest.
|
|
- name: hostname-less-route-denied
|
|
try:
|
|
- script:
|
|
content: |
|
|
set -eu
|
|
# deny <case> <kind> <name> reads a manifest on stdin and expects the
|
|
# VAP to reject it, mirroring the deny() helper in the Ingress suite.
|
|
deny() {
|
|
kubectl -n "$NAMESPACE" delete "$2" "$3" --ignore-not-found >/dev/null
|
|
out=$(kubectl apply -n "$NAMESPACE" -f - 2>&1) && rc=0 || rc=$?
|
|
echo "case=$1 rc=$rc out=$out"
|
|
if [ "$rc" -eq 0 ]; then
|
|
kubectl -n "$NAMESPACE" delete "$2" "$3" --ignore-not-found
|
|
echo "BUG: admission accepted $1" >&2
|
|
exit 1
|
|
fi
|
|
echo "$out" | grep -qi "ValidatingAdmissionPolicy"
|
|
echo "$out" | grep -qi "must declare spec.hostnames"
|
|
}
|
|
deny httproute-hostnames-absent httproute route-nohost-probe <<'EOF'
|
|
apiVersion: gateway.networking.k8s.io/v1
|
|
kind: HTTPRoute
|
|
metadata:
|
|
name: route-nohost-probe
|
|
spec:
|
|
rules:
|
|
- backendRefs:
|
|
- name: kubernetes
|
|
namespace: default
|
|
port: 443
|
|
EOF
|
|
deny httproute-hostnames-empty httproute route-emptyhost-probe <<'EOF'
|
|
apiVersion: gateway.networking.k8s.io/v1
|
|
kind: HTTPRoute
|
|
metadata:
|
|
name: route-emptyhost-probe
|
|
spec:
|
|
hostnames: []
|
|
rules:
|
|
- backendRefs:
|
|
- name: kubernetes
|
|
namespace: default
|
|
port: 443
|
|
EOF
|
|
deny tlsroute-hostnames-absent tlsroute route-tls-nohost-probe <<'EOF'
|
|
apiVersion: gateway.networking.k8s.io/v1alpha2
|
|
kind: TLSRoute
|
|
metadata:
|
|
name: route-tls-nohost-probe
|
|
spec:
|
|
rules:
|
|
- backendRefs:
|
|
- name: kubernetes
|
|
namespace: default
|
|
port: 443
|
|
EOF
|
|
deny grpcroute-hostnames-absent grpcroute route-grpc-nohost-probe <<'EOF'
|
|
apiVersion: gateway.networking.k8s.io/v1
|
|
kind: GRPCRoute
|
|
metadata:
|
|
name: route-grpc-nohost-probe
|
|
spec:
|
|
rules:
|
|
- backendRefs:
|
|
- name: kubernetes
|
|
namespace: default
|
|
port: 443
|
|
EOF
|
|
check:
|
|
($error == null): true
|
|
- name: own-apex-route-allowed
|
|
try:
|
|
- script:
|
|
content: |
|
|
set -eu
|
|
ns="$NAMESPACE"
|
|
apex=$(kubectl get namespace "$ns" -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}')
|
|
kubectl apply -n "$ns" -f - <<EOF
|
|
apiVersion: gateway.networking.k8s.io/v1
|
|
kind: HTTPRoute
|
|
metadata:
|
|
name: route-hostname-allow-probe
|
|
spec:
|
|
hostnames:
|
|
- "harbor.${apex}"
|
|
rules:
|
|
- backendRefs:
|
|
- name: kubernetes
|
|
namespace: default
|
|
port: 443
|
|
EOF
|
|
kubectl -n "$ns" delete httproute route-hostname-allow-probe --ignore-not-found
|
|
check:
|
|
($error == null): true
|
|
# Case-insensitivity. The route hostname is always lowercase — the CRD's
|
|
# hostname pattern permits only lowercase, so an uppercase hostname is rejected
|
|
# before the VAP runs — but the apex operand is the namespace.cozystack.io/host
|
|
# label VALUE, and Kubernetes permits uppercase in a label value. Under a
|
|
# mixed-case apex a lowercase in-apex route must still be admitted, so the VAP
|
|
# normalizes both operands with .lowerAscii(). Pre-fix it compares
|
|
# "probe.case-probe.route.invalid" against "Case-Probe.route.invalid", misses,
|
|
# and denies a legitimate route — this step is red pre-fix, green post-fix.
|
|
# A mixed-case apex is the only runtime shape that exercises the
|
|
# normalization.
|
|
#
|
|
# The apex here does not need to sit under the platform root: unlike the
|
|
# Ingress policy this VAP compares against the tenant apex only and has no
|
|
# outside-root branch, so a reserved .invalid apex (RFC 2606) keeps the case
|
|
# host-independent and unable to collide with a dev cluster's real apex.
|
|
#
|
|
# The mixed-case apex goes on a throwaway tenant-* namespace rather than on
|
|
# tenant-test: relabelling the shared namespace would leave it mixed-case for
|
|
# every later test if a restore were missed. The tenant-* prefix is what makes
|
|
# the VAP's matchCondition fire, and the runner is a trusted caller for the
|
|
# host-label immutability VAP, so the labelled CREATE is admitted (see
|
|
# untrusted-sa-cannot-set-host-label-at-create above). Applied declaratively:
|
|
# an admission denial fails the apply, and Chainsaw auto-cleanup deletes the
|
|
# route and the namespace.
|
|
- name: mixed-case-apex-route-allowed
|
|
try:
|
|
# Pre-clean a namespace left by an interrupted prior run: a leftover
|
|
# tenant-case-probe stuck in Terminating fails the apply below, and because
|
|
# the label here is a fixed literal a surviving namespace would otherwise be
|
|
# silently reused rather than rebuilt.
|
|
- script:
|
|
content: |
|
|
set -eu
|
|
kubectl delete namespace tenant-case-probe --ignore-not-found --wait=true --timeout=60s
|
|
check:
|
|
($error == null): true
|
|
- apply:
|
|
resource:
|
|
apiVersion: v1
|
|
kind: Namespace
|
|
metadata:
|
|
name: tenant-case-probe
|
|
labels:
|
|
namespace.cozystack.io/host: Case-Probe.route.invalid
|
|
# Guard against a vacuous pass: re-read the persisted label. Had anything
|
|
# normalized it to lowercase, the route below would be admitted with or
|
|
# without the fix and the step would stop proving case-insensitivity.
|
|
- assert:
|
|
resource:
|
|
apiVersion: v1
|
|
kind: Namespace
|
|
metadata:
|
|
name: tenant-case-probe
|
|
labels:
|
|
namespace.cozystack.io/host: Case-Probe.route.invalid
|
|
- apply:
|
|
resource:
|
|
apiVersion: gateway.networking.k8s.io/v1
|
|
kind: HTTPRoute
|
|
metadata:
|
|
name: route-hostname-case-probe
|
|
namespace: tenant-case-probe
|
|
spec:
|
|
hostnames:
|
|
- "probe.case-probe.route.invalid"
|
|
rules:
|
|
- backendRefs:
|
|
- name: kubernetes
|
|
namespace: default
|
|
port: 443
|
|
# The deny half of the mixed-case apex, pairing with the step above the way
|
|
# foreign-apex-route-denied pairs with own-apex-route-allowed. The allow half
|
|
# alone only proves the normalized match does not UNDER-admit; over-admission
|
|
# is the direction that hijacks a hostname, and normalizing both operands is
|
|
# exactly the edit that can widen a match by accident. Reuses the namespace
|
|
# the previous step created — Chainsaw cleans up at test scope, not step
|
|
# scope. Scripted rather than an apply/expect pair because the grep for
|
|
# "ValidatingAdmissionPolicy" is what proves the VAP did the denying: a bare
|
|
# "the apply failed" assertion would be satisfied by a typo in the manifest.
|
|
- name: mixed-case-apex-route-denied
|
|
try:
|
|
- script:
|
|
content: |
|
|
set -eu
|
|
# deny <case> <name> reads a manifest on stdin and expects the VAP to
|
|
# reject it, mirroring the deny() helper in the Ingress suite below.
|
|
deny() {
|
|
kubectl -n tenant-case-probe delete httproute "$2" --ignore-not-found >/dev/null
|
|
out=$(kubectl apply -n tenant-case-probe -f - 2>&1) && rc=0 || rc=$?
|
|
echo "case=$1 rc=$rc"
|
|
if [ "$rc" -eq 0 ]; then
|
|
kubectl -n tenant-case-probe delete httproute "$2" --ignore-not-found
|
|
echo "BUG: admission accepted $1: $out" >&2
|
|
exit 1
|
|
fi
|
|
echo "$out" | grep -qi "ValidatingAdmissionPolicy"
|
|
}
|
|
# A wholly foreign apex stays denied once both operands are lowercased.
|
|
deny foreign-apex route-case-foreign-probe <<'EOF'
|
|
apiVersion: gateway.networking.k8s.io/v1
|
|
kind: HTTPRoute
|
|
metadata:
|
|
name: route-case-foreign-probe
|
|
spec:
|
|
hostnames:
|
|
- "evil.other.invalid"
|
|
rules:
|
|
- backendRefs:
|
|
- name: kubernetes
|
|
namespace: default
|
|
port: 443
|
|
EOF
|
|
# Dot-boundary guard under the mixed-case apex: "evilcase-probe.route.invalid"
|
|
# ends with the lowercased apex "case-probe.route.invalid" but is NOT a
|
|
# subdomain of it. The CEL only denies it because the suffix test is
|
|
# endsWith("." + apex); drop that leading dot and this hostname is
|
|
# admitted and hijacks the apex. This is the only case in this suite
|
|
# that would notice.
|
|
deny prefix-adjacent route-case-adjacent-probe <<'EOF'
|
|
apiVersion: gateway.networking.k8s.io/v1
|
|
kind: HTTPRoute
|
|
metadata:
|
|
name: route-case-adjacent-probe
|
|
spec:
|
|
hostnames:
|
|
- "evilcase-probe.route.invalid"
|
|
rules:
|
|
- backendRefs:
|
|
- name: kubernetes
|
|
namespace: default
|
|
port: 443
|
|
EOF
|
|
check:
|
|
($error == null): true
|
|
---
|
|
apiVersion: chainsaw.kyverno.io/v1alpha1
|
|
kind: Test
|
|
metadata:
|
|
name: httproute-parentref-accepted
|
|
spec:
|
|
catch:
|
|
- events: {}
|
|
- describe:
|
|
apiVersion: gateway.networking.k8s.io/v1
|
|
kind: HTTPRoute
|
|
name: httproute-probe
|
|
steps:
|
|
- name: create-gateway-and-route
|
|
try:
|
|
- apply:
|
|
file: gw-route-probe.yaml
|
|
# The route is scripted rather than declared in gw-route-probe.yaml
|
|
# because it needs the tenant apex interpolated. This suite runs in
|
|
# tenant-test, which cozystack-route-hostname-policy matches, so a
|
|
# route naming no hostname is denied; the apex is read from the
|
|
# namespace label rather than hardcoded so the suite still passes on
|
|
# a dev cluster whose host differs from CI's. Same reason and same
|
|
# shape as own-apex-route-allowed above.
|
|
- script:
|
|
content: |
|
|
set -eu
|
|
ns="$NAMESPACE"
|
|
apex=$(kubectl get namespace "$ns" -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}')
|
|
[ -n "$apex" ] || { echo "SETUP FAILURE: $ns lacks the namespace.cozystack.io/host label" >&2; exit 1; }
|
|
kubectl apply -n "$ns" -f - <<EOF
|
|
apiVersion: gateway.networking.k8s.io/v1
|
|
kind: HTTPRoute
|
|
metadata:
|
|
name: httproute-probe
|
|
spec:
|
|
hostnames:
|
|
- "parentref-probe.${apex}"
|
|
parentRefs:
|
|
- name: gateway-route-probe
|
|
sectionName: http
|
|
rules:
|
|
- matches:
|
|
- path:
|
|
type: PathPrefix
|
|
value: /
|
|
backendRefs:
|
|
- name: kubernetes
|
|
namespace: default
|
|
port: 443
|
|
EOF
|
|
check:
|
|
($error == null): true
|
|
- assert:
|
|
timeout: 3m
|
|
resource:
|
|
apiVersion: gateway.networking.k8s.io/v1
|
|
kind: Gateway
|
|
metadata:
|
|
name: gateway-route-probe
|
|
status:
|
|
(conditions[?type == 'Programmed']):
|
|
- status: "True"
|
|
- assert:
|
|
timeout: 2m
|
|
resource:
|
|
apiVersion: gateway.networking.k8s.io/v1
|
|
kind: HTTPRoute
|
|
metadata:
|
|
name: httproute-probe
|
|
(status.parents[0].conditions[?type == 'Accepted']):
|
|
- status: "True"
|
|
# Creation and assertions share one step so this reclaim covers both.
|
|
# It is needed because chainsaw reclaims what it applied from a file
|
|
# but not what a script created, and the suite runs in one long-lived
|
|
# namespace. finally runs at the end of its own step, so splitting
|
|
# the assertions into a later step would delete the route before they
|
|
# ran, and hanging the reclaim off that later step would leak the
|
|
# route whenever this one failed after the apply.
|
|
finally:
|
|
- delete:
|
|
ref:
|
|
apiVersion: gateway.networking.k8s.io/v1
|
|
kind: HTTPRoute
|
|
name: httproute-probe
|
|
---
|
|
apiVersion: chainsaw.kyverno.io/v1alpha1
|
|
kind: Test
|
|
metadata:
|
|
name: tenant-gateway-label-lockstep
|
|
spec:
|
|
catch:
|
|
- events: {}
|
|
steps:
|
|
- name: create-child-tenant-with-gateway
|
|
try:
|
|
- apply:
|
|
resource:
|
|
apiVersion: apps.cozystack.io/v1alpha1
|
|
kind: Tenant
|
|
metadata:
|
|
name: gwprop
|
|
spec:
|
|
gateway: true
|
|
- name: assert-label-and-values
|
|
try:
|
|
- assert:
|
|
timeout: 3m
|
|
resource:
|
|
apiVersion: v1
|
|
kind: Namespace
|
|
metadata:
|
|
name: tenant-test-gwprop
|
|
labels:
|
|
namespace.cozystack.io/gateway: tenant-test-gwprop
|
|
- script:
|
|
timeout: 2m
|
|
content: |
|
|
set -eu
|
|
deadline=$(( $(date +%s) + 60 ))
|
|
until kubectl -n tenant-test-gwprop get secret cozystack-values >/dev/null 2>&1; do
|
|
[ "$(date +%s)" -ge "$deadline" ] && { echo "cozystack-values not created" >&2; exit 1; }
|
|
sleep 2
|
|
done
|
|
v=$(kubectl -n tenant-test-gwprop get secret cozystack-values -o jsonpath='{.data.values\.yaml}' | base64 -d)
|
|
echo "$v" | grep -E '^\s*gateway:\s*"tenant-test-gwprop"\s*$'
|
|
check:
|
|
($error == null): true
|
|
finally:
|
|
- delete:
|
|
ref:
|
|
apiVersion: apps.cozystack.io/v1alpha1
|
|
kind: Tenant
|
|
name: gwprop
|
|
---
|
|
apiVersion: chainsaw.kyverno.io/v1alpha1
|
|
kind: Test
|
|
metadata:
|
|
name: tenant-gateway-inheritance
|
|
spec:
|
|
catch:
|
|
- events: {}
|
|
steps:
|
|
- name: create-parent-tenant
|
|
try:
|
|
- apply:
|
|
resource:
|
|
apiVersion: apps.cozystack.io/v1alpha1
|
|
kind: Tenant
|
|
metadata:
|
|
name: gwparent
|
|
spec:
|
|
gateway: true
|
|
- assert:
|
|
timeout: 3m
|
|
resource:
|
|
apiVersion: v1
|
|
kind: Namespace
|
|
metadata:
|
|
name: tenant-test-gwparent
|
|
labels:
|
|
namespace.cozystack.io/gateway: tenant-test-gwparent
|
|
- name: create-child-tenant
|
|
try:
|
|
- apply:
|
|
resource:
|
|
apiVersion: apps.cozystack.io/v1alpha1
|
|
kind: Tenant
|
|
metadata:
|
|
name: gwchild
|
|
namespace: tenant-test-gwparent
|
|
spec: {}
|
|
- assert:
|
|
timeout: 3m
|
|
resource:
|
|
apiVersion: v1
|
|
kind: Namespace
|
|
metadata:
|
|
name: tenant-test-gwparent-gwchild
|
|
labels:
|
|
namespace.cozystack.io/gateway: tenant-test-gwparent
|
|
- name: assert-inheritance-values
|
|
try:
|
|
- script:
|
|
timeout: 2m
|
|
content: |
|
|
set -eu
|
|
deadline=$(( $(date +%s) + 60 ))
|
|
until kubectl -n tenant-test-gwparent-gwchild get secret cozystack-values >/dev/null 2>&1; do
|
|
[ "$(date +%s)" -ge "$deadline" ] && { echo "child cozystack-values not created" >&2; exit 1; }
|
|
sleep 2
|
|
done
|
|
v=$(kubectl -n tenant-test-gwparent-gwchild get secret cozystack-values -o jsonpath='{.data.values\.yaml}' | base64 -d)
|
|
echo "$v" | grep -E '^\s*gateway:\s*"tenant-test-gwparent"\s*$'
|
|
check:
|
|
($error == null): true
|
|
# The child must NOT own a gateway HelmRelease — inheritance, not its own.
|
|
- error:
|
|
resource:
|
|
apiVersion: helm.toolkit.fluxcd.io/v2
|
|
kind: HelmRelease
|
|
metadata:
|
|
name: gateway
|
|
namespace: tenant-test-gwparent-gwchild
|
|
finally:
|
|
- delete:
|
|
ref:
|
|
apiVersion: apps.cozystack.io/v1alpha1
|
|
kind: Tenant
|
|
name: gwchild
|
|
namespace: tenant-test-gwparent
|
|
- delete:
|
|
ref:
|
|
apiVersion: apps.cozystack.io/v1alpha1
|
|
kind: Tenant
|
|
name: gwparent
|
|
---
|
|
apiVersion: chainsaw.kyverno.io/v1alpha1
|
|
kind: Test
|
|
metadata:
|
|
name: tenant-gateway-route-listener
|
|
spec:
|
|
catch:
|
|
- events: {}
|
|
steps:
|
|
- name: parent-child-and-route-drive-listener-set
|
|
try:
|
|
# Above the 660s of polls plus the 675s the trap's deletes may take with
|
|
# their kill grace, so a poll or a delete is what fires and the trap still
|
|
# runs: a kill at the op ceiling would skip cleanup and leave both tenants
|
|
# behind for every later suite. hack/chainsaw-op-budgets.bats holds the op
|
|
# to that sum.
|
|
- script:
|
|
timeout: 24m
|
|
content: |
|
|
set -eu
|
|
ns="$NAMESPACE"
|
|
|
|
# Each delete waits, child before parent, because a parent deleted
|
|
# while its child is still uninstalling wedges the parent's cleanup.
|
|
# Each delete is bounded as a whole by an outer timeout, because the
|
|
# trap runs inside the op ceiling and kubectl's --timeout covers only
|
|
# the wait after the request, not discovery or a stalled request. A
|
|
# tenant delete gets the 5m that .chainsaw.yaml gives a delete op,
|
|
# because its child namespaces take minutes to drain. A child still
|
|
# uninstalling at its bound leaves the parent in place: a leaked
|
|
# tenant rather than a wedged one. A failed tenant delete fails the
|
|
# step with kubectl's own error and exit status (124 is the bound),
|
|
# so the leak is reported here rather than in whichever suite trips
|
|
# on it.
|
|
cleanup() {
|
|
timeout -k 5 60 kubectl -n tenant-test-rparent-rchild delete httproute inherit-probe --ignore-not-found >/dev/null 2>&1 || true
|
|
if timeout -k 5 300 kubectl -n tenant-test-rparent delete tenant rchild --ignore-not-found >/dev/null; then
|
|
timeout -k 5 300 kubectl -n "$ns" delete tenant rparent --ignore-not-found >/dev/null || {
|
|
echo "the tenant rparent delete exited $?, so tenant rparent is left in $ns" >&2
|
|
exit 1
|
|
}
|
|
else
|
|
echo "the tenant rchild delete exited $?, so tenant rparent is left in $ns" >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
# Parent tenant owns the Gateway.
|
|
kubectl apply -n "$ns" -f - <<EOF
|
|
apiVersion: apps.cozystack.io/v1alpha1
|
|
kind: Tenant
|
|
metadata:
|
|
name: rparent
|
|
spec:
|
|
gateway: true
|
|
EOF
|
|
timeout 180 sh -ec 'until kubectl get ns tenant-test-rparent >/dev/null 2>&1; do sleep 3; done'
|
|
timeout 120 sh -ec 'until kubectl -n tenant-test-rparent get gateway cozystack >/dev/null 2>&1; do sleep 3; done'
|
|
|
|
# Child tenant inherits the parent Gateway.
|
|
kubectl apply -n tenant-test-rparent -f - <<EOF
|
|
apiVersion: apps.cozystack.io/v1alpha1
|
|
kind: Tenant
|
|
metadata:
|
|
name: rchild
|
|
spec: {}
|
|
EOF
|
|
timeout 180 sh -ec 'until kubectl get ns tenant-test-rparent-rchild >/dev/null 2>&1; do sleep 3; done'
|
|
|
|
child_apex=$(kubectl get ns tenant-test-rparent-rchild -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}')
|
|
[ -n "$child_apex" ]
|
|
route_host="harbor.${child_apex}"
|
|
|
|
# An HTTPRoute in the child, attached to the parent Gateway.
|
|
kubectl apply -n tenant-test-rparent-rchild -f - <<EOF
|
|
apiVersion: gateway.networking.k8s.io/v1
|
|
kind: HTTPRoute
|
|
metadata:
|
|
name: inherit-probe
|
|
spec:
|
|
parentRefs:
|
|
- group: gateway.networking.k8s.io
|
|
kind: Gateway
|
|
name: cozystack
|
|
namespace: tenant-test-rparent
|
|
hostnames:
|
|
- "${route_host}"
|
|
rules:
|
|
- matches:
|
|
- path:
|
|
type: PathPrefix
|
|
value: /
|
|
backendRefs:
|
|
- name: kubernetes
|
|
namespace: default
|
|
port: 443
|
|
EOF
|
|
|
|
# cozystack-controller collects the route across the inheritance label
|
|
# and grows a per-listener HTTPS entry on the parent Gateway.
|
|
timeout 120 sh -ec '
|
|
until kubectl -n tenant-test-rparent get gateway cozystack \
|
|
-o jsonpath="{range .spec.listeners[?(@.protocol==\"HTTPS\")]}{.hostname}{\"\n\"}{end}" 2>/dev/null \
|
|
| grep -qx "'"$route_host"'"; do sleep 3; done'
|
|
|
|
# ...and a matching per-listener Certificate.
|
|
timeout 60 sh -ec '
|
|
until kubectl -n tenant-test-rparent get certificate \
|
|
-l cozystack.io/per-listener-cert=true \
|
|
-o jsonpath="{range .items[*]}{.spec.dnsNames[0]}{\"\n\"}{end}" 2>/dev/null \
|
|
| grep -qx "'"$route_host"'"; do sleep 3; done'
|
|
check:
|
|
($error == null): true
|
|
---
|
|
# What a passthrough listener costs an HTTPRoute that publishes its name,
|
|
# measured on the objects rather than on the renderer. The chart renders
|
|
# passthrough listeners only on the publishing tenant and only with
|
|
# gateway.enabled, which CI leaves off, so the test declares tls-api on 443
|
|
# itself through the TenantGateway spec. The name is reserved by the
|
|
# declaration alone, so the route claiming it gets no terminate listener
|
|
# and no certificate with no TLSRoute anywhere.
|
|
#
|
|
# Readiness is measured before the test declares any HTTPRoute. The
|
|
# reservation is an http01 behaviour, and an http01 terminate listener
|
|
# waits for a Let's Encrypt certificate that is never issued for a test
|
|
# hostname, so once the control route below has its listener the tenant is
|
|
# not Ready, correctly.
|
|
#
|
|
# Absence proves nothing on its own, since a reconcile that has not run yet
|
|
# looks the same. A second route on a free name is the control: the
|
|
# controller renders the Gateway and the Certificates before it writes route
|
|
# status in the same pass, it never runs two passes over one TenantGateway
|
|
# at once, and a route its cache has seen stays there until it is deleted.
|
|
# So once both routes carry the verdicts asserted below, one read of the
|
|
# Gateway reflects a pass that saw both, whichever route came first.
|
|
apiVersion: chainsaw.kyverno.io/v1alpha1
|
|
kind: Test
|
|
metadata:
|
|
name: tenant-gateway-passthrough-reservation
|
|
spec:
|
|
catch:
|
|
- events: {}
|
|
steps:
|
|
- name: create-tenant
|
|
try:
|
|
- apply:
|
|
resource:
|
|
apiVersion: apps.cozystack.io/v1alpha1
|
|
kind: Tenant
|
|
metadata:
|
|
name: pw
|
|
spec:
|
|
gateway: true
|
|
- assert:
|
|
timeout: 3m
|
|
resource:
|
|
apiVersion: gateway.networking.k8s.io/v1
|
|
kind: Gateway
|
|
metadata:
|
|
name: cozystack
|
|
namespace: tenant-test-pw
|
|
spec:
|
|
(listeners[?name == 'http']):
|
|
- protocol: HTTP
|
|
# The premise of the merge patches below, not a measurement of the
|
|
# chart's scoping: with gateway.enabled off no tenant renders these, so
|
|
# in CI this holds whether or not the chart keeps them to the publishing
|
|
# tenant. packages/extra/gateway/tests/tenantgateway_test.yaml pins that.
|
|
# One render carries every listener of the spec, so the Gateway that
|
|
# answered the assert above is the whole render of the chart's spec.
|
|
- script:
|
|
timeout: 1m
|
|
content: |
|
|
set -eu
|
|
tls_names=$(kubectl -n tenant-test-pw get gateway cozystack -o 'jsonpath={range .spec.listeners[?(@.protocol=="TLS")]}{.name}{"\n"}{end}')
|
|
if [ -n "$tls_names" ]; then
|
|
echo "child tenant Gateway renders passthrough listeners: $tls_names" >&2
|
|
exit 1
|
|
fi
|
|
check:
|
|
($error == null): true
|
|
# Two listeners with no TLSRoute behind them, one on 443 and one on a
|
|
# native port. The package README rests a claim on that shape: Cilium
|
|
# judges a listener on its protocol, its allowed kinds and its
|
|
# certificate refs and never on its route set, so a routeless passthrough
|
|
# listener is Accepted and Programmed and tenant readiness is untouched.
|
|
# The unit tests run against a fake client and cannot measure that; only
|
|
# a cluster can.
|
|
- name: routeless-listeners-keep-the-tenant-ready
|
|
try:
|
|
- script:
|
|
timeout: 2m
|
|
content: |
|
|
set -eu
|
|
tenant_ns=tenant-test-pw
|
|
apex=$(kubectl get ns "$tenant_ns" -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}')
|
|
[ -n "$apex" ]
|
|
|
|
# Merged rather than applied: the chart renders neither field
|
|
# here, and a key it renders in neither manifest is left alone
|
|
# by the next helm upgrade.
|
|
kubectl -n "$tenant_ns" patch tenantgateway cozystack --type=merge \
|
|
-p "$(printf '{"spec":{"tlsPassthroughServices":["api"],"tlsPassthroughListeners":[{"name":"pgprobe","port":5432,"hostname":"pgprobe.%s"}]}}' "$apex")"
|
|
check:
|
|
($error == null): true
|
|
# The per-listener entries are the measurement, asserted together with
|
|
# the aggregate because the TenantGateway may already be Ready when
|
|
# this step starts: Ready alone would be answered by the state that
|
|
# preceded the patch. The entries do not exist until the controller has
|
|
# judged the new listeners, each flag folds its listener's Accepted and
|
|
# Programmed, and one status write carries them with Ready.
|
|
- assert:
|
|
timeout: 2m
|
|
resource:
|
|
apiVersion: gateway.cozystack.io/v1alpha1
|
|
kind: TenantGateway
|
|
metadata:
|
|
name: cozystack
|
|
namespace: tenant-test-pw
|
|
status:
|
|
(listeners[?name == 'tls-api']):
|
|
- ready: true
|
|
(listeners[?name == 'tls-pgprobe']):
|
|
- ready: true
|
|
(conditions[?type == 'Ready']):
|
|
- status: "True"
|
|
# The hostnames are built from the apex, which is known only once a script
|
|
# reads it off the tenant namespace's label, so the routes are written
|
|
# here. They live in the tenant namespace and go with the Tenant Chainsaw
|
|
# applied above.
|
|
- name: declare-the-routes
|
|
try:
|
|
- script:
|
|
timeout: 2m
|
|
content: |
|
|
set -eu
|
|
tenant_ns=tenant-test-pw
|
|
apex=$(kubectl get ns "$tenant_ns" -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}')
|
|
[ -n "$apex" ]
|
|
kubectl apply -n "$tenant_ns" -f - <<EOF
|
|
apiVersion: gateway.networking.k8s.io/v1
|
|
kind: HTTPRoute
|
|
metadata:
|
|
name: app-probe
|
|
spec:
|
|
parentRefs:
|
|
- group: gateway.networking.k8s.io
|
|
kind: Gateway
|
|
name: cozystack
|
|
namespace: ${tenant_ns}
|
|
hostnames:
|
|
- "app-probe.${apex}"
|
|
rules:
|
|
- backendRefs:
|
|
- name: kubernetes
|
|
namespace: default
|
|
port: 443
|
|
---
|
|
apiVersion: gateway.networking.k8s.io/v1
|
|
kind: HTTPRoute
|
|
metadata:
|
|
name: api-probe
|
|
spec:
|
|
parentRefs:
|
|
- group: gateway.networking.k8s.io
|
|
kind: Gateway
|
|
name: cozystack
|
|
namespace: ${tenant_ns}
|
|
hostnames:
|
|
- "api.${apex}"
|
|
rules:
|
|
- backendRefs:
|
|
- name: kubernetes
|
|
namespace: default
|
|
port: 443
|
|
EOF
|
|
check:
|
|
($error == null): true
|
|
- name: reserved-name-gets-no-terminate-listener
|
|
try:
|
|
- assert:
|
|
timeout: 2m
|
|
resource:
|
|
apiVersion: gateway.networking.k8s.io/v1
|
|
kind: HTTPRoute
|
|
metadata:
|
|
name: app-probe
|
|
namespace: tenant-test-pw
|
|
status:
|
|
(parents[?controllerName == 'gateway.cozystack.io/tenantgateway-controller']):
|
|
- (conditions[?type == 'Accepted']):
|
|
- status: "True"
|
|
- assert:
|
|
timeout: 2m
|
|
resource:
|
|
apiVersion: gateway.networking.k8s.io/v1
|
|
kind: HTTPRoute
|
|
metadata:
|
|
name: api-probe
|
|
namespace: tenant-test-pw
|
|
status:
|
|
(parents[?controllerName == 'gateway.cozystack.io/tenantgateway-controller']):
|
|
- (conditions[?type == 'Accepted']):
|
|
- status: "False"
|
|
reason: NoMatchingListenerHostname
|
|
# Single reads, not polls: the verdicts above were written after the
|
|
# Gateway and the Certificates of the pass that reached them.
|
|
- script:
|
|
timeout: 1m
|
|
content: |
|
|
set -eu
|
|
tenant_ns=tenant-test-pw
|
|
apex=$(kubectl get ns "$tenant_ns" -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}')
|
|
[ -n "$apex" ]
|
|
https_hosts=$(kubectl -n "$tenant_ns" get gateway cozystack -o 'jsonpath={range .spec.listeners[?(@.protocol=="HTTPS")]}{.hostname}{"\n"}{end}')
|
|
tls_names=$(kubectl -n "$tenant_ns" get gateway cozystack -o 'jsonpath={range .spec.listeners[?(@.protocol=="TLS")]}{.name}{"\n"}{end}')
|
|
cert_hosts=$(kubectl -n "$tenant_ns" get certificate -l cozystack.io/per-listener-cert=true -o 'jsonpath={range .items[*]}{.spec.dnsNames[0]}{"\n"}{end}')
|
|
printf '%s\n' "$https_hosts" | grep -qxF "app-probe.${apex}"
|
|
printf '%s\n' "$cert_hosts" | grep -qxF "app-probe.${apex}"
|
|
printf '%s\n' "$tls_names" | grep -qxF tls-api
|
|
if printf '%s\n' "$https_hosts" | grep -qxF "api.${apex}"; then
|
|
echo "terminate listener rendered for reserved api.${apex}" >&2
|
|
exit 1
|
|
fi
|
|
if printf '%s\n' "$cert_hosts" | grep -qxF "api.${apex}"; then
|
|
echo "Certificate ordered for reserved api.${apex}" >&2
|
|
exit 1
|
|
fi
|
|
check:
|
|
($error == null): true
|
|
---
|
|
# Legacy-Ingress counterpart of the gateway/route hostname VAPs (bats: the 12
|
|
# cozystack-ingress-hostname-policy cases). Gateway API is off by default, so
|
|
# tenant apps publish through a legacy networking.k8s.io/Ingress; the VAP
|
|
# constrains tenant Ingress hostnames on the shared ingress-nginx. The VAP
|
|
# matches on the tenant-* namespace, not the caller, so plain applies as the
|
|
# runner's identity exercise it — no impersonation needed. Hostnames are
|
|
# derived at runtime: the tenant apex from this namespace's
|
|
# namespace.cozystack.io/host label (as elsewhere in this suite) and the
|
|
# platform ROOT apex from the same label on tenant-root, so the
|
|
# under-root-but-outside-own-apex cases stay host-independent where the bats
|
|
# hardcoded test.example.org / example.org.
|
|
apiVersion: chainsaw.kyverno.io/v1alpha1
|
|
kind: Test
|
|
metadata:
|
|
name: ingress-hostname-policy
|
|
spec:
|
|
catch:
|
|
- events: {}
|
|
steps:
|
|
# If these resources are missing, admission cannot constrain tenant Ingress
|
|
# hostnames — make that loud instead of letting it fall through as a silent
|
|
# pass.
|
|
- name: vap-and-binding-installed
|
|
try:
|
|
- assert:
|
|
resource:
|
|
apiVersion: admissionregistration.k8s.io/v1
|
|
kind: ValidatingAdmissionPolicy
|
|
metadata:
|
|
name: cozystack-ingress-hostname-policy
|
|
- assert:
|
|
resource:
|
|
apiVersion: admissionregistration.k8s.io/v1
|
|
kind: ValidatingAdmissionPolicyBinding
|
|
metadata:
|
|
name: cozystack-ingress-hostname-policy
|
|
spec:
|
|
(contains(validationActions, 'Deny')): true
|
|
# Apex-derived rejects:
|
|
# foreign-apex-host dashboard.<root> — under the platform root apex but
|
|
# outside the tenant's own apex
|
|
# prefix-adjacent evil<apex> — dot-boundary guard: a naive suffix
|
|
# check without the leading dot would match "<apex>"
|
|
# inside "evil<apex>"; the CEL uses endsWith("." + apex)
|
|
# foreign-tls-host in-apex rule host, but a spec.tls[].hosts[] entry
|
|
# under the root apex — the tls branch denies on its own
|
|
# wildcard-matches-apex *.<root-parent> is "outside" the root apex by a
|
|
# plain string test, but at ingress-nginx it also
|
|
# matches the bare root apex; the outside-root path is
|
|
# gated on the host not being a wildcard
|
|
- name: foreign-and-adjacent-hosts-denied
|
|
try:
|
|
- script:
|
|
content: |
|
|
set -eu
|
|
ns="$NAMESPACE"
|
|
apex=$(kubectl get namespace "$ns" -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}')
|
|
root=$(kubectl get namespace tenant-root -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}')
|
|
[ -n "$apex" ] || { echo "SETUP FAILURE: $ns lacks the namespace.cozystack.io/host label" >&2; exit 1; }
|
|
[ -n "$root" ] || { echo "SETUP FAILURE: tenant-root lacks the namespace.cozystack.io/host label" >&2; exit 1; }
|
|
[ "$apex" != "$root" ] || { echo "SETUP FAILURE: tenant apex equals root apex; under-root-outside-own cases are undefined" >&2; exit 1; }
|
|
# deny <case> <ingress-name> <extra-grep|""> reads a manifest on
|
|
# stdin and expects admission to reject it with a VAP message.
|
|
deny() {
|
|
kubectl -n "$ns" delete ingress "$2" --ignore-not-found >/dev/null
|
|
out=$(kubectl apply -n "$ns" -f - 2>&1) && rc=0 || rc=$?
|
|
echo "case=$1 rc=$rc"
|
|
if [ "$rc" -eq 0 ]; then
|
|
kubectl -n "$ns" delete ingress "$2" --ignore-not-found
|
|
echo "BUG: admission accepted $1: $out" >&2
|
|
exit 1
|
|
fi
|
|
echo "$out" | grep -qi "ValidatingAdmissionPolicy"
|
|
[ -z "$3" ] || echo "$out" | grep -qi "$3"
|
|
}
|
|
deny foreign-apex-host ingress-out-of-apex-host-probe "must be within" <<EOF
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: ingress-out-of-apex-host-probe
|
|
spec:
|
|
rules:
|
|
- host: "dashboard.${root}"
|
|
http:
|
|
paths:
|
|
- path: /
|
|
pathType: Prefix
|
|
backend:
|
|
service:
|
|
name: noop
|
|
port:
|
|
number: 80
|
|
EOF
|
|
deny prefix-adjacent ingress-prefix-adjacent-probe "" <<EOF
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: ingress-prefix-adjacent-probe
|
|
spec:
|
|
rules:
|
|
- host: "evil${apex}"
|
|
http:
|
|
paths:
|
|
- path: /
|
|
pathType: Prefix
|
|
backend:
|
|
service:
|
|
name: noop
|
|
port:
|
|
number: 80
|
|
EOF
|
|
deny foreign-tls-host ingress-tls-out-of-apex-probe "must be within" <<EOF
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: ingress-tls-out-of-apex-probe
|
|
spec:
|
|
tls:
|
|
- hosts:
|
|
- "harbor.${root}"
|
|
secretName: noop-tls
|
|
rules:
|
|
- host: "harbor.${apex}"
|
|
http:
|
|
paths:
|
|
- path: /
|
|
pathType: Prefix
|
|
backend:
|
|
service:
|
|
name: noop
|
|
port:
|
|
number: 80
|
|
EOF
|
|
case "$root" in
|
|
*.*) w="*.${root#*.}" ;;
|
|
*) w="*.${root}" ;;
|
|
esac
|
|
deny wildcard-matches-apex ingress-wildcard-host-probe "" <<EOF
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: ingress-wildcard-host-probe
|
|
spec:
|
|
rules:
|
|
- host: "${w}"
|
|
http:
|
|
paths:
|
|
- path: /
|
|
pathType: Prefix
|
|
backend:
|
|
service:
|
|
name: noop
|
|
port:
|
|
number: 80
|
|
EOF
|
|
check:
|
|
($error == null): true
|
|
# Static rejects — catch-all shapes on the shared ingress-nginx:
|
|
# hostless a rule with no .host matches all inbound HTTP
|
|
# empty-host host: "" is the same catch-all; the CEL denies it via
|
|
# r.host != "" (or has(r.host) if the apiserver drops it)
|
|
# defaultBackend catch-all for otherwise-unmatched traffic
|
|
- name: catch-all-shapes-denied
|
|
try:
|
|
- script:
|
|
content: |
|
|
set -eu
|
|
ns="$NAMESPACE"
|
|
deny() {
|
|
kubectl -n "$ns" delete ingress "$2" --ignore-not-found >/dev/null
|
|
out=$(kubectl apply -n "$ns" -f - 2>&1) && rc=0 || rc=$?
|
|
echo "case=$1 rc=$rc"
|
|
if [ "$rc" -eq 0 ]; then
|
|
kubectl -n "$ns" delete ingress "$2" --ignore-not-found
|
|
echo "BUG: admission accepted $1: $out" >&2
|
|
exit 1
|
|
fi
|
|
echo "$out" | grep -qi "ValidatingAdmissionPolicy"
|
|
}
|
|
deny hostless-rule ingress-hostless-probe <<'EOF'
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: ingress-hostless-probe
|
|
spec:
|
|
rules:
|
|
- http:
|
|
paths:
|
|
- path: /
|
|
pathType: Prefix
|
|
backend:
|
|
service:
|
|
name: noop
|
|
port:
|
|
number: 80
|
|
EOF
|
|
deny empty-host ingress-empty-host-probe <<'EOF'
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: ingress-empty-host-probe
|
|
spec:
|
|
rules:
|
|
- host: ""
|
|
http:
|
|
paths:
|
|
- path: /
|
|
pathType: Prefix
|
|
backend:
|
|
service:
|
|
name: noop
|
|
port:
|
|
number: 80
|
|
EOF
|
|
deny default-backend ingress-defaultbackend-probe <<'EOF'
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: ingress-defaultbackend-probe
|
|
spec:
|
|
defaultBackend:
|
|
service:
|
|
name: noop
|
|
port:
|
|
number: 80
|
|
EOF
|
|
check:
|
|
($error == null): true
|
|
# Inverse sanity checks under the tenant's own apex: an in-apex rule+tls
|
|
# Ingress and a wildcard under the own apex (*.<apex>) must NOT be rejected
|
|
# — the anti-wildcard gate applies only to the outside-root path. Created
|
|
# inside the script (they need ${apex} interpolation), so also deleted here.
|
|
- name: own-apex-allowed
|
|
try:
|
|
- script:
|
|
content: |
|
|
set -eu
|
|
ns="$NAMESPACE"
|
|
apex=$(kubectl get namespace "$ns" -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}')
|
|
[ -n "$apex" ] || { echo "SETUP FAILURE: $ns lacks the namespace.cozystack.io/host label" >&2; exit 1; }
|
|
kubectl apply -n "$ns" -f - <<EOF
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: ingress-hostname-allow-probe
|
|
spec:
|
|
tls:
|
|
- hosts:
|
|
- "harbor.${apex}"
|
|
secretName: noop-tls
|
|
rules:
|
|
- host: "harbor.${apex}"
|
|
http:
|
|
paths:
|
|
- path: /
|
|
pathType: Prefix
|
|
backend:
|
|
service:
|
|
name: noop
|
|
port:
|
|
number: 80
|
|
EOF
|
|
kubectl -n "$ns" delete ingress ingress-hostname-allow-probe --ignore-not-found
|
|
kubectl apply -n "$ns" -f - <<EOF
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: ingress-ownapex-wildcard-probe
|
|
spec:
|
|
rules:
|
|
- host: "*.${apex}"
|
|
http:
|
|
paths:
|
|
- path: /
|
|
pathType: Prefix
|
|
backend:
|
|
service:
|
|
name: noop
|
|
port:
|
|
number: 80
|
|
EOF
|
|
kubectl -n "$ns" delete ingress ingress-ownapex-wildcard-probe --ignore-not-found
|
|
check:
|
|
($error == null): true
|
|
# Case-insensitivity. spec.rules[].host and spec.tls[].hosts[] are always
|
|
# lowercase — the apiserver's host validation permits only lowercase, so an
|
|
# uppercase host is rejected before the VAP runs — but the apex operand is the
|
|
# namespace.cozystack.io/host label VALUE, and Kubernetes permits uppercase in
|
|
# a label value. Under a mixed-case apex a lowercase in-apex Ingress must
|
|
# still be admitted, so the VAP normalizes both operands with .lowerAscii().
|
|
# Ingress is the default publish path and carries the most complex CEL (a
|
|
# rule-host and a tls-host branch), so it gets its own runtime guard beside
|
|
# the route one; both branches are exercised at once because the tls branch
|
|
# denies on its own.
|
|
#
|
|
# The probe apex is derived as Case-Probe.<root> rather than hardcoded, and it
|
|
# has to sit UNDER the platform root to mean anything: the outside-root branch
|
|
# admits any non-wildcard host outside the root apex whatever the tenant apex
|
|
# says, so a probe on an unrelated domain would be admitted pre-fix and prove
|
|
# nothing. Under the root that escape is closed — pre-fix the own-apex branch
|
|
# misses on case and the outside-root branch is excluded, so admission denies
|
|
# and this step is red; post-fix the own-apex branch matches and it is green.
|
|
#
|
|
# The mixed-case apex goes on a throwaway tenant-* namespace rather than on
|
|
# tenant-test, for the same reason as the route case. Built in a script since
|
|
# it needs ${root} interpolation, so the namespace is torn down in finally
|
|
# rather than by Chainsaw auto-cleanup. That teardown is also why the allow and
|
|
# deny halves share ONE step here while the route case splits them in two:
|
|
# finally runs at step scope, so a second step would find the namespace gone.
|
|
# The route namespace is applied declaratively and reclaimed at test scope, so
|
|
# it survives into its own deny step.
|
|
- name: mixed-case-apex-allowed-and-denied
|
|
try:
|
|
- script:
|
|
content: |
|
|
set -eu
|
|
root=$(kubectl get namespace tenant-root -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}')
|
|
[ -n "$root" ] || { echo "SETUP FAILURE: tenant-root lacks the namespace.cozystack.io/host label" >&2; exit 1; }
|
|
apex="Case-Probe.${root}"
|
|
host="probe.case-probe.${root}"
|
|
# Guard against a vacuous pass: the probe apex must really be
|
|
# mixed-case. An all-lowercase apex would admit the Ingress with or
|
|
# without the fix — this catches a later edit that lowercases the
|
|
# literal above, which the persisted-label check below cannot see.
|
|
[ "$apex" != "$(echo "$apex" | tr '[:upper:]' '[:lower:]')" ] || { echo "SETUP FAILURE: probe apex '$apex' is not mixed-case" >&2; exit 1; }
|
|
# Pre-clean a namespace left behind by an interrupted prior run.
|
|
kubectl delete namespace tenant-ingress-case-probe --ignore-not-found --wait=true --timeout=60s
|
|
kubectl apply -f - <<EOF
|
|
apiVersion: v1
|
|
kind: Namespace
|
|
metadata:
|
|
name: tenant-ingress-case-probe
|
|
labels:
|
|
namespace.cozystack.io/host: ${apex}
|
|
EOF
|
|
# Second vacuous-pass guard: re-read the persisted label, in case
|
|
# anything normalized the value on the way in.
|
|
got=$(kubectl get namespace tenant-ingress-case-probe -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}')
|
|
[ "$got" = "$apex" ] || { echo "SETUP FAILURE: apex label persisted as '$got', want '$apex'" >&2; exit 1; }
|
|
kubectl apply -n tenant-ingress-case-probe -f - <<EOF
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: ingress-hostname-case-probe
|
|
spec:
|
|
tls:
|
|
- hosts:
|
|
- "${host}"
|
|
secretName: noop-tls
|
|
rules:
|
|
- host: "${host}"
|
|
http:
|
|
paths:
|
|
- path: /
|
|
pathType: Prefix
|
|
backend:
|
|
service:
|
|
name: noop
|
|
port:
|
|
number: 80
|
|
EOF
|
|
# The deny half, pairing with the allow above the way
|
|
# foreign-and-adjacent-hosts-denied pairs with own-apex-allowed. The
|
|
# allow half alone only proves the normalized match does not
|
|
# UNDER-admit; over-admission is what hijacks a hostname, and
|
|
# normalizing both operands is exactly the edit that can widen a match
|
|
# by accident.
|
|
deny() {
|
|
kubectl -n tenant-ingress-case-probe delete ingress "$2" --ignore-not-found >/dev/null
|
|
out=$(kubectl apply -n tenant-ingress-case-probe -f - 2>&1) && rc=0 || rc=$?
|
|
echo "case=$1 rc=$rc"
|
|
if [ "$rc" -eq 0 ]; then
|
|
kubectl -n tenant-ingress-case-probe delete ingress "$2" --ignore-not-found
|
|
echo "BUG: admission accepted $1: $out" >&2
|
|
exit 1
|
|
fi
|
|
echo "$out" | grep -qi "ValidatingAdmissionPolicy"
|
|
}
|
|
# Under the platform root but outside the probe's own apex: the
|
|
# own-apex branch must miss on more than case, and the outside-root
|
|
# branch must stay closed under the root.
|
|
deny foreign-apex-host ingress-case-foreign-probe <<EOF
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: ingress-case-foreign-probe
|
|
spec:
|
|
rules:
|
|
- host: "dashboard.${root}"
|
|
http:
|
|
paths:
|
|
- path: /
|
|
pathType: Prefix
|
|
backend:
|
|
service:
|
|
name: noop
|
|
port:
|
|
number: 80
|
|
EOF
|
|
# Dot-boundary guard on the normalized path: "evilcase-probe.<root>"
|
|
# ends with the lowercased apex "case-probe.<root>" without being a
|
|
# subdomain of it. Only endsWith("." + apex) denies it; drop that
|
|
# leading dot and the apex is hijackable.
|
|
deny prefix-adjacent ingress-case-adjacent-probe <<EOF
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: ingress-case-adjacent-probe
|
|
spec:
|
|
rules:
|
|
- host: "evilcase-probe.${root}"
|
|
http:
|
|
paths:
|
|
- path: /
|
|
pathType: Prefix
|
|
backend:
|
|
service:
|
|
name: noop
|
|
port:
|
|
number: 80
|
|
EOF
|
|
check:
|
|
($error == null): true
|
|
finally:
|
|
- delete:
|
|
ref:
|
|
apiVersion: v1
|
|
kind: Namespace
|
|
name: tenant-ingress-case-probe
|
|
# Static accepts — concrete external domains outside the platform root apex
|
|
# on both spec.rules[].host and spec.tls[].hosts. Declarative apply; Chainsaw
|
|
# auto-cleanup deletes the probes.
|
|
- name: external-domain-allowed
|
|
try:
|
|
- apply:
|
|
file: ingress-external-allow.yaml
|
|
---
|
|
# Regression for #3983, which reported fail-closed admission webhooks becoming
|
|
# unreachable from the API server while an unrelated namespace was deleted. The
|
|
# mechanism is NOT confirmed: the errno in that report cannot come from a policy
|
|
# denial, and namespace deletion churns service backends as well as Cilium
|
|
# identities. So this case asserts the property the issue is about -- both
|
|
# webhooks still admit across a namespace deletion -- rather than claiming to
|
|
# reproduce a cause. Both affected paths are exercised immediately after the
|
|
# deletion, with no retry to hide a window.
|
|
#
|
|
# Every apply in this test passes through both fail-closed webhooks, so a
|
|
# cluster whose webhooks were ALREADY down fails here in a way that reads
|
|
# exactly like the outage under test. The first run of this case did precisely
|
|
# that: the ingress-nginx controller was crash-looping, the admission Service
|
|
# had no ready backend, and the setup apply died with "dial tcp ...: no route to
|
|
# host" before any churn happened. So availability is asserted before the churn
|
|
# as a precondition with its own message, and the churn verdict is only reached
|
|
# once both webhooks are known to have been up to begin with.
|
|
apiVersion: chainsaw.kyverno.io/v1alpha1
|
|
kind: Test
|
|
metadata:
|
|
name: admission-webhooks-survive-namespace-churn
|
|
spec:
|
|
catch:
|
|
- events: {}
|
|
steps:
|
|
- name: ingress-and-pod-admission-remain-available-after-deletion
|
|
try:
|
|
- script:
|
|
# Above the sum of the inner budgets, per the rule in
|
|
# hack/e2e-chainsaw/.chainsaw.yaml: 2x90s of webhook preconditions, 60s
|
|
# of pre-clean delete, 90s waiting for the churn pod, 180s for the
|
|
# namespace delete = 510s, and timeouts.exec would otherwise cap the
|
|
# whole script at 5m and SIGKILL it with no verdict.
|
|
timeout: 12m
|
|
content: |
|
|
set -eu
|
|
ns="$NAMESPACE"
|
|
churn_ns=tenant-admission-churn-probe
|
|
apex=$(kubectl get namespace "$ns" -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}')
|
|
[ -n "$apex" ] || { echo "SETUP FAILURE: $ns lacks the namespace.cozystack.io/host label" >&2; exit 1; }
|
|
|
|
# Coordinates come from the webhook registrations themselves, which is
|
|
# what the API server dials, rather than from a guessed release name or
|
|
# namespace. A webhook that is not registered at all is a setup
|
|
# failure: nothing below could then prove anything about it.
|
|
webhook_services() {
|
|
kubectl get "$1" \
|
|
-o jsonpath='{range .items[*]}{range .webhooks[*]}{.name}{" "}{.clientConfig.service.namespace}{" "}{.clientConfig.service.name}{"\n"}{end}{end}'
|
|
}
|
|
|
|
# Ready endpoint addresses behind a Service. Counted from
|
|
# EndpointSlice .endpoints[].conditions.ready rather than from pod
|
|
# readiness, because that is the state the API server routes on, and
|
|
# because a `kubectl wait --for=condition=Ready` is satisfied
|
|
# transiently by a pod that is flapping.
|
|
ready_backends() {
|
|
n=$(kubectl -n "$1" get endpointslice -l "kubernetes.io/service-name=$2" \
|
|
-o jsonpath='{range .items[*]}{range .endpoints[*]}{.conditions.ready}{"\n"}{end}{end}' \
|
|
2>/dev/null | grep -c '^true$' || true)
|
|
[ -n "$n" ] || n=0
|
|
echo "$n"
|
|
}
|
|
|
|
# Distinct variable names from require_webhook_up below, deliberately:
|
|
# POSIX shell functions share one scope, so reusing `label` here
|
|
# appended each iteration's suffix to the caller's own copy and the
|
|
# second registration reported itself as "(1/2) (2/2)".
|
|
require_service_up() {
|
|
svclabel="$1"; svcns="$2"; svcname="$3"; svcbudget="$4"
|
|
svcdeadline=$(( $(date +%s) + svcbudget ))
|
|
while :; do
|
|
n=$(ready_backends "$svcns" "$svcname")
|
|
if [ "$n" -gt 0 ]; then
|
|
echo "PRECONDITION OK $svclabel ($svcns/$svcname) has $n ready backend(s)"
|
|
return 0
|
|
fi
|
|
[ "$(date +%s)" -lt "$svcdeadline" ] || break
|
|
sleep 5
|
|
done
|
|
echo "SETUP FAILURE: $svclabel ($svcns/$svcname) has no ready backend after ${svcbudget}s." >&2
|
|
echo " This test cannot tell a churn-induced outage from a webhook that was" >&2
|
|
echo " already down, and every apply below goes through it, so it stops here" >&2
|
|
echo " rather than reporting the wrong cause." >&2
|
|
kubectl -n "$svcns" get endpointslice -l "kubernetes.io/service-name=$svcname" -o yaml >&2 2>&1 || true
|
|
kubectl -n "$svcns" get pods -o wide >&2 2>&1 || true
|
|
return 1
|
|
}
|
|
|
|
# EVERY registration of the named webhook, not just the first. All of
|
|
# them fire on the applies below, so one being down aborts the apply
|
|
# and the failure would be reported as a churn-induced outage. The
|
|
# loop indexes with sed rather than piping into `while read`, because
|
|
# a pipeline's body runs in a subshell under dash and a `return` from
|
|
# it would not reach this function.
|
|
require_webhook_up() {
|
|
label="$1"; kind="$2"; hook="$3"; budget="$4"
|
|
coords=$(webhook_services "$kind" | awk -v h="$hook" '$1 == h { print $2, $3 }')
|
|
[ -n "$coords" ] || {
|
|
echo "SETUP FAILURE: no $kind registers webhook $hook, so its availability cannot be asserted" >&2
|
|
return 1
|
|
}
|
|
total=$(printf '%s\n' "$coords" | grep -c . || true)
|
|
[ -n "$total" ] || total=0
|
|
i=0
|
|
while [ "$i" -lt "$total" ]; do
|
|
i=$(( i + 1 ))
|
|
line=$(printf '%s\n' "$coords" | sed -n "${i}p")
|
|
require_service_up "$label ($i/$total)" "${line% *}" "${line#* }" "$budget" || return 1
|
|
done
|
|
return 0
|
|
}
|
|
|
|
require_webhook_up "ingress admission" validatingwebhookconfigurations \
|
|
validate.nginx.ingress.kubernetes.io 90
|
|
require_webhook_up "pod admission" mutatingwebhookconfigurations \
|
|
kube-ovn-webhook.cozystack.io 90
|
|
|
|
# Reproduce the incident shape: an ordinary tenant namespace carrying
|
|
# an admitted Ingress AND a running pod, deleted synchronously, with
|
|
# the fail-closed webhooks invoked while Cilium processes the removal.
|
|
#
|
|
# The pod is what makes this a churn test rather than a smoke test. A
|
|
# Cilium identity is the set of an endpoint's identity-relevant
|
|
# labels, and k8s:io.kubernetes.pod.namespace is one of them, so a pod
|
|
# in a brand-new namespace allocates a NEW identity, and deleting the
|
|
# namespace releases it. An empty namespace has no endpoint, allocates
|
|
# no identity, and its deletion drives no identity churn at all, so
|
|
# without the pod this case would exercise nothing it names.
|
|
kubectl delete namespace "$churn_ns" --ignore-not-found --wait=true --timeout=60s
|
|
kubectl apply -f - <<EOF
|
|
apiVersion: v1
|
|
kind: Namespace
|
|
metadata:
|
|
name: ${churn_ns}
|
|
labels:
|
|
namespace.cozystack.io/host: churn.${apex}
|
|
EOF
|
|
kubectl apply -n "$churn_ns" -f - <<EOF
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: admission-churn-source
|
|
spec:
|
|
rules:
|
|
- host: probe.churn.${apex}
|
|
http:
|
|
paths:
|
|
- path: /
|
|
pathType: Prefix
|
|
backend:
|
|
service:
|
|
name: noop
|
|
port:
|
|
number: 80
|
|
EOF
|
|
# busybox:musl is staged onto every node by hack/e2e-prepull-images.sh,
|
|
# so this adds no image pull. The securityContext satisfies the
|
|
# restricted PodSecurity profile, so it runs whatever level the tenant
|
|
# namespace enforces.
|
|
kubectl apply -n "$churn_ns" -f - <<'EOF'
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata:
|
|
name: admission-churn-identity
|
|
spec:
|
|
restartPolicy: Never
|
|
automountServiceAccountToken: false
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 65534
|
|
runAsGroup: 65534
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
containers:
|
|
- name: hold
|
|
image: busybox:musl
|
|
command: ["sleep", "600"]
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop: ["ALL"]
|
|
EOF
|
|
kubectl -n "$churn_ns" wait --for=condition=Ready pod/admission-churn-identity --timeout=90s || {
|
|
echo "SETUP FAILURE: the churn pod never became Ready, so deleting $churn_ns releases no Cilium identity and this test would assert nothing" >&2
|
|
kubectl -n "$churn_ns" describe pod admission-churn-identity >&2 2>&1 || true
|
|
exit 1
|
|
}
|
|
|
|
kubectl delete namespace "$churn_ns" --wait=true --timeout=180s
|
|
|
|
# No retry on either call: a retry would paper over exactly the window
|
|
# this test exists to catch.
|
|
#
|
|
# ingress-nginx validating admission
|
|
kubectl apply --dry-run=server -n "$ns" -o name -f - <<EOF
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: admission-after-churn
|
|
spec:
|
|
rules:
|
|
- host: admission-after-churn.${apex}
|
|
http:
|
|
paths:
|
|
- path: /
|
|
pathType: Prefix
|
|
backend:
|
|
service:
|
|
name: noop
|
|
port:
|
|
number: 80
|
|
EOF
|
|
|
|
# kubeovn-webhook mutating admission. The securityContext is carried
|
|
# even though a server dry-run Pod is never scheduled: if the tenant
|
|
# namespace ever enforces restricted PodSecurity, a non-compliant Pod
|
|
# is rejected in the validating phase and `set -eu` would abort this
|
|
# check, which reads as the webhook being down.
|
|
kubectl apply --dry-run=server -n "$ns" -o name -f - <<'EOF'
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata:
|
|
name: admission-after-churn
|
|
spec:
|
|
restartPolicy: Never
|
|
automountServiceAccountToken: false
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 65534
|
|
runAsGroup: 65534
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
containers:
|
|
- name: pause
|
|
image: registry.k8s.io/pause:3.10
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop: ["ALL"]
|
|
EOF
|
|
echo "OK both fail-closed webhooks admitted after $churn_ns was deleted"
|
|
check:
|
|
($error == null): true
|
|
finally:
|
|
- delete:
|
|
ref:
|
|
apiVersion: v1
|
|
kind: Namespace
|
|
name: tenant-admission-churn-probe
|
|
---
|
|
# Companion to admission-webhooks-survive-namespace-churn above. That test
|
|
# proves the webhooks stay AVAILABLE; this one proves they are still
|
|
# UNREACHABLE from pods, which is the property the policies exist for, and that
|
|
# the deny is scoped to the webhook port alone.
|
|
#
|
|
# The third assertion is the one that earns this test. The policies previously
|
|
# expressed the Service-frontend deny with `toServices`, which Cilium resolves
|
|
# only inside ALLOW rules -- leaving the deny rule with an empty L3 selector
|
|
# that Cilium reads as a wildcard, so "deny this Service on 443" became "deny
|
|
# EVERY destination on 443" for every selected pod. Every rendered-YAML
|
|
# assertion in the package suites passed on that policy. Only a probe that
|
|
# checks an unrelated 443 destination still works can tell the two apart, and
|
|
# the ingress controller's own HTTPS port is exactly that: the same pods, the
|
|
# same tenant egress allowances, a different port.
|
|
#
|
|
# Runs in tenant-root because that is where the host cluster's ingress-nginx
|
|
# lives and where pods are permitted to reach it (allow-internal-communication),
|
|
# so the deny is the only thing that can block the webhook port. From the
|
|
# isolated tenant-test the probe would pass for the wrong reason.
|
|
#
|
|
# And "for the wrong reason" is the whole difficulty, because a refused connect
|
|
# does not say why it was refused. "Denied by policy" and "no backend to route
|
|
# to" are the same observation from inside the probe, so both deny assertions
|
|
# would report success against a cluster with no policy installed at all, as
|
|
# long as the backend was dead. That is what the first run of this case did:
|
|
# the controller was crash-looping on a ~100s cycle, `kubectl wait` caught it
|
|
# during a Ready window, the probe then measured a corpse, and the port-scoping
|
|
# assertion reported a policy bug that did not exist.
|
|
#
|
|
# So the deny verdicts here are gated on the admission Service having ready
|
|
# backends before the probe, on every sample while it runs, and after it
|
|
# finishes, and on the controller pods not having restarted or been replaced
|
|
# across that window. The probe itself only reports reachable/unreachable per
|
|
# destination and always exits 0; this script attributes the cause, because it
|
|
# is the half that holds the endpoint evidence. Anything that leaves the
|
|
# question undecidable fails as a setup failure rather than as a verdict.
|
|
apiVersion: chainsaw.kyverno.io/v1alpha1
|
|
kind: Test
|
|
metadata:
|
|
name: admission-webhook-deny-still-enforced
|
|
spec:
|
|
namespace: tenant-root
|
|
catch:
|
|
- events: {}
|
|
steps:
|
|
- name: pods-cannot-reach-the-webhook-but-keep-unrelated-443-egress
|
|
try:
|
|
- script:
|
|
# Above the sum of the inner budgets, per the rule in
|
|
# hack/e2e-chainsaw/.chainsaw.yaml: 120s of endpoint precondition, 30s
|
|
# of stability sampling, 60s to clear a leftover probe and 180s of
|
|
# phase polling = 390s. The previous 6m sat BELOW its own sum, so a
|
|
# slow-but-healthy cluster was killed before the diagnostics ran.
|
|
timeout: 10m
|
|
content: |
|
|
set -eu
|
|
ns="$NAMESPACE"
|
|
probe=admission-deny-probe
|
|
sel=app.kubernetes.io/name=ingress-nginx,app.kubernetes.io/component=controller
|
|
|
|
svcs=$(kubectl -n "$ns" get svc -l "$sel" \
|
|
-o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
|
|
adm_candidates=$(printf '%s\n' "$svcs" | grep -- '-admission$' || true)
|
|
# -metrics is excluded explicitly rather than relying on
|
|
# ingress-nginx-controller sorting before a metrics Service: the
|
|
# positive control has to be the one unambiguous controller Service.
|
|
ctl_candidates=$(printf '%s\n' "$svcs" | grep -v -- '-admission$' | grep -v -- '-metrics$' | grep . || true)
|
|
adm_count=$(printf '%s\n' "$adm_candidates" | grep -c . || true)
|
|
ctl_count=$(printf '%s\n' "$ctl_candidates" | grep -c . || true)
|
|
[ "$adm_count" -eq 1 ] || {
|
|
echo "SETUP FAILURE: expected one ingress-nginx admission Service in $ns, found $adm_count" >&2
|
|
printf ' candidates: %s\n' "${adm_candidates:-<none>}" >&2
|
|
exit 1
|
|
}
|
|
[ "$ctl_count" -eq 1 ] || {
|
|
echo "SETUP FAILURE: expected one ingress-nginx controller Service in $ns, found $ctl_count" >&2
|
|
printf ' candidates: %s\n' "${ctl_candidates:-<none>}" >&2
|
|
exit 1
|
|
}
|
|
adm=$adm_candidates
|
|
ctl=$ctl_candidates
|
|
|
|
service_port() {
|
|
kubectl -n "$ns" get svc "$1" -o json | jq -r --arg port_name "$2" '
|
|
[.spec.ports[]?
|
|
| select(.name == $port_name)
|
|
| select((.protocol // "TCP") == "TCP")
|
|
| .port
|
|
| select(type == "number")]
|
|
| if length == 1 then .[0] else empty end
|
|
'
|
|
}
|
|
adm_port=$(service_port "$adm" https-webhook)
|
|
ctl_port=$(service_port "$ctl" https)
|
|
adm_cluster_ip=$(kubectl -n "$ns" get svc "$adm" -o jsonpath='{.spec.clusterIP}')
|
|
[ -n "$adm_cluster_ip" ] && [ "$adm_cluster_ip" != None ] || {
|
|
echo "SETUP FAILURE: $adm must have a real ClusterIP, found ${adm_cluster_ip:-<empty>}" >&2
|
|
kubectl -n "$ns" get svc "$adm" -o yaml >&2 2>&1 || true
|
|
exit 1
|
|
}
|
|
[ -n "$adm_port" ] || {
|
|
echo "SETUP FAILURE: $adm must expose exactly one TCP https-webhook port" >&2
|
|
kubectl -n "$ns" get svc "$adm" -o yaml >&2 2>&1 || true
|
|
exit 1
|
|
}
|
|
[ -n "$ctl_port" ] || {
|
|
echo "SETUP FAILURE: $ctl must expose exactly one TCP https port" >&2
|
|
kubectl -n "$ns" get svc "$ctl" -o yaml >&2 2>&1 || true
|
|
exit 1
|
|
}
|
|
|
|
# Ready endpoint addresses behind a Service, from EndpointSlice rather
|
|
# than from pod readiness: this is the state the API server and the
|
|
# service load balancer route on.
|
|
ready_backends() {
|
|
n=$(kubectl -n "$ns" get endpointslice -l "kubernetes.io/service-name=$1" \
|
|
-o jsonpath='{range .items[*]}{range .endpoints[*]}{.conditions.ready}{"\n"}{end}{end}' \
|
|
2>/dev/null | grep -c '^true$' || true)
|
|
[ -n "$n" ] || n=0
|
|
echo "$n"
|
|
}
|
|
|
|
# Exact ready admission coordinates. The Pod identity, IP and backend
|
|
# port come from the same EndpointSlice as the ready condition;
|
|
# selecting them independently from .items[0] could probe an unready
|
|
# replica while a different replica kept aggregate readiness above
|
|
# zero.
|
|
ready_admission_backends() {
|
|
kubectl -n "$ns" get endpointslice -l "kubernetes.io/service-name=$adm" -o json | jq -r '
|
|
.items[]? as $slice
|
|
| $slice.ports[]? as $port
|
|
| select($port.name == "https-webhook")
|
|
| select(($port.protocol // "TCP") == "TCP")
|
|
| select(($port.port | type) == "number")
|
|
| $slice.endpoints[]?
|
|
| select(.conditions.ready == true)
|
|
| select(.targetRef.kind == "Pod")
|
|
| select((.targetRef.name // "") != "")
|
|
| select((.targetRef.uid // "") != "")
|
|
| .targetRef.name as $pod
|
|
| .targetRef.uid as $uid
|
|
| .addresses[]?
|
|
| select(type == "string" and length > 0)
|
|
| [$pod, $uid, ., ($port.port | tostring)]
|
|
| @tsv
|
|
' | sort -u
|
|
}
|
|
|
|
# Pod identity, IP and every container restart count, as one string.
|
|
# Compared for equality across the probe window, so a restart AND a
|
|
# replacement both show up -- a crash-looping controller can satisfy
|
|
# a readiness check at both ends of a window it spent dying in the
|
|
# middle of.
|
|
controller_generation() {
|
|
generation=$(kubectl -n "$ns" get pod -l "$sel" \
|
|
-o jsonpath='{range .items[*]}{.metadata.name}/{.metadata.uid}/{.status.podIP}={range .status.containerStatuses[*]}{.name}:{.restartCount},{end}{"\n"}{end}') || {
|
|
echo "SETUP FAILURE: unable to read ingress-nginx controller generation" >&2
|
|
return 1
|
|
}
|
|
[ -n "$generation" ] || {
|
|
echo "SETUP FAILURE: ingress-nginx controller generation is empty" >&2
|
|
return 1
|
|
}
|
|
printf '%s\n' "$generation" | sort
|
|
}
|
|
|
|
# Precondition. Both Services are backed by the same controller pods,
|
|
# so requiring the admission Service to be live also establishes that
|
|
# the controller Service -- the positive control below -- has a
|
|
# backend to answer.
|
|
deadline=$(( $(date +%s) + 120 ))
|
|
while :; do
|
|
adm_ready=$(ready_admission_backends | grep -c . || true)
|
|
ctl_ready=$(ready_backends "$ctl")
|
|
if [ "$adm_ready" -gt 0 ] && [ "$ctl_ready" -gt 0 ]; then break; fi
|
|
[ "$(date +%s)" -lt "$deadline" ] || {
|
|
echo "SETUP FAILURE: no ready backend for $adm ($adm_ready) or $ctl ($ctl_ready) after 120s." >&2
|
|
echo " A refused connect cannot be attributed to policy while the destination" >&2
|
|
echo " has nothing listening, so this test stops instead of reporting a deny" >&2
|
|
echo " it did not observe." >&2
|
|
kubectl -n "$ns" get endpointslice -l "kubernetes.io/service-name=$adm" -o yaml >&2 2>&1 || true
|
|
kubectl -n "$ns" get pod -l "$sel" -o wide >&2 2>&1 || true
|
|
exit 1
|
|
}
|
|
sleep 5
|
|
done
|
|
|
|
backend=$(ready_admission_backends | head -1)
|
|
[ -n "$backend" ] || { echo "SETUP FAILURE: $adm has no complete ready Pod/UID/IP/port coordinate" >&2; exit 1; }
|
|
tab=$(printf '\t')
|
|
backend_fields=$(printf '%s\n' "$backend" | awk -F "$tab" '{ print NF }')
|
|
[ "$backend_fields" -eq 4 ] || { echo "SETUP FAILURE: malformed ready backend coordinate: $backend" >&2; exit 1; }
|
|
pod_name=$(printf '%s\n' "$backend" | cut -f1)
|
|
pod_uid=$(printf '%s\n' "$backend" | cut -f2)
|
|
pod_ip=$(printf '%s\n' "$backend" | cut -f3)
|
|
hook_port=$(printf '%s\n' "$backend" | cut -f4)
|
|
[ -n "$pod_name" ] && [ -n "$pod_uid" ] && [ -n "$pod_ip" ] && [ -n "$hook_port" ] || {
|
|
echo "SETUP FAILURE: incomplete ready backend coordinate: $backend" >&2
|
|
exit 1
|
|
}
|
|
|
|
selected_backend_ready() {
|
|
ready_admission_backends | awk -F "$tab" \
|
|
-v pod="$pod_name" -v uid="$pod_uid" -v ip="$pod_ip" -v port="$hook_port" '
|
|
$1 == pod && $2 == uid && $3 == ip && $4 == port { found = 1 }
|
|
END { exit !found }
|
|
'
|
|
}
|
|
|
|
require_selected_backend() {
|
|
selected_backend_ready && return 0
|
|
echo "SETUP FAILURE: selected admission backend $pod_name/$pod_uid ($pod_ip:$hook_port) was not ready $1" >&2
|
|
kubectl -n "$ns" get endpointslice -l "kubernetes.io/service-name=$adm" -o yaml >&2 2>&1 || true
|
|
kubectl -n "$ns" get pod "$pod_name" -o wide >&2 2>&1 || true
|
|
exit 1
|
|
}
|
|
|
|
require_selected_backend "before the stability window"
|
|
|
|
# Stability: the same evidence held across a window, not at one
|
|
# instant. Six samples over 30s, all of which must be live and show no
|
|
# restart or replacement.
|
|
gen_before=$(controller_generation)
|
|
printf '%s\n' "$gen_before" | awk \
|
|
-v prefix="${pod_name}/${pod_uid}/${pod_ip}=" '
|
|
index($0, prefix) == 1 { found = 1 }
|
|
END { exit !found }
|
|
' || {
|
|
echo "SETUP FAILURE: selected admission backend is not a current controller Pod: $backend" >&2
|
|
exit 1
|
|
}
|
|
i=0
|
|
while [ "$i" -lt 6 ]; do
|
|
sleep 5
|
|
i=$(( i + 1 ))
|
|
require_selected_backend "during the stability window (sample $i)"
|
|
n=$(ready_backends "$ctl")
|
|
[ "$n" -gt 0 ] || {
|
|
echo "SETUP FAILURE: $ctl lost its ready backend during the stability window (sample $i)" >&2
|
|
kubectl -n "$ns" get pod -l "$sel" -o wide >&2 2>&1 || true
|
|
exit 1
|
|
}
|
|
done
|
|
gen_stable=$(controller_generation)
|
|
[ "$gen_stable" = "$gen_before" ] || {
|
|
echo "SETUP FAILURE: the ingress-nginx controller restarted or was replaced during the stability window" >&2
|
|
echo " before: $gen_before" >&2
|
|
echo " after: $gen_stable" >&2
|
|
kubectl -n "$ns" get pod -l "$sel" -o wide >&2 2>&1 || true
|
|
exit 1
|
|
}
|
|
|
|
echo "probing from $ns: expect reachable ${ctl}:${ctl_port}, denied ${adm}@${adm_cluster_ip}:${adm_port}, denied ${pod_name}/${pod_uid}@${pod_ip}:${hook_port}"
|
|
|
|
kubectl -n "$ns" delete pod "$probe" --ignore-not-found --wait=true --timeout=60s
|
|
# busybox:musl is staged onto every node by hack/e2e-prepull-images.sh,
|
|
# so this adds no image pull. The securityContext satisfies the
|
|
# restricted PodSecurity profile, so the probe runs in a tenant
|
|
# namespace whatever level is enforced there.
|
|
kubectl -n "$ns" apply -f - <<EOF
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata:
|
|
name: ${probe}
|
|
spec:
|
|
restartPolicy: Never
|
|
automountServiceAccountToken: false
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 65534
|
|
runAsGroup: 65534
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
containers:
|
|
- name: probe
|
|
image: busybox:musl
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop: ["ALL"]
|
|
env:
|
|
- name: CTL_HOST
|
|
value: "${ctl}.${ns}.svc"
|
|
- name: CTL_PORT
|
|
value: "${ctl_port}"
|
|
- name: ADM_HOST
|
|
value: "${adm_cluster_ip}"
|
|
- name: ADM_PORT
|
|
value: "${adm_port}"
|
|
- name: POD_IP
|
|
value: "${pod_ip}"
|
|
- name: HOOK_PORT
|
|
value: "${hook_port}"
|
|
command: ["sh", "-c"]
|
|
args:
|
|
# Measurement only, and always exit 0. The pod reports what it
|
|
# observed per destination; the script that started it holds the
|
|
# endpoint evidence and is the only place that can say why a
|
|
# connect was refused. Keeping the verdict out of here is also
|
|
# what stops "the probe could not run" from being read as "the
|
|
# assertion failed".
|
|
- |
|
|
set -u
|
|
reach() { nc -w 5 "\$1" "\$2" </dev/null >/dev/null 2>&1; }
|
|
report() {
|
|
if reach "\$2" "\$3"; then
|
|
echo "\$1 reachable \$2:\$3"
|
|
else
|
|
echo "\$1 unreachable \$2:\$3"
|
|
fi
|
|
}
|
|
report CTL "\$CTL_HOST" "\$CTL_PORT"
|
|
report ADM "\$ADM_HOST" "\$ADM_PORT"
|
|
report HOOK "\$POD_IP" "\$HOOK_PORT"
|
|
EOF
|
|
|
|
# Wait for a terminal phase rather than a condition, and re-check the
|
|
# backend on every poll: a controller that dies mid-probe invalidates
|
|
# the deny verdicts, and this is the window it would die in.
|
|
deadline=$(( $(date +%s) + 180 ))
|
|
phase=""
|
|
while [ "$(date +%s)" -lt "$deadline" ]; do
|
|
require_selected_backend "while the probe was running"
|
|
n=$(ready_backends "$ctl")
|
|
[ "$n" -gt 0 ] || {
|
|
echo "SETUP FAILURE: $ctl lost its ready backend while the probe was running, so an unreachable positive control proves nothing" >&2
|
|
kubectl -n "$ns" get pod -l "$sel" -o wide >&2 2>&1 || true
|
|
kubectl -n "$ns" logs "$probe" >&2 2>&1 || true
|
|
exit 1
|
|
}
|
|
phase=$(kubectl -n "$ns" get pod "$probe" -o jsonpath='{.status.phase}' 2>/dev/null || true)
|
|
case "$phase" in Succeeded|Failed) break ;; esac
|
|
sleep 3
|
|
done
|
|
|
|
log=$(kubectl -n "$ns" logs "$probe" 2>&1 || true)
|
|
printf '%s\n' "$log"
|
|
|
|
# Postcondition, before the log is interpreted at all.
|
|
[ "$phase" = "Succeeded" ] || {
|
|
echo "SETUP FAILURE: the probe did not run to completion (phase=${phase:-unknown})" >&2
|
|
kubectl -n "$ns" describe pod "$probe" >&2 2>&1 || true
|
|
exit 1
|
|
}
|
|
gen_after=$(controller_generation)
|
|
[ "$gen_after" = "$gen_before" ] || {
|
|
echo "SETUP FAILURE: the ingress-nginx controller restarted or was replaced during the probe" >&2
|
|
echo " before: $gen_before" >&2
|
|
echo " after: $gen_after" >&2
|
|
kubectl -n "$ns" get pod -l "$sel" -o wide >&2 2>&1 || true
|
|
exit 1
|
|
}
|
|
require_selected_backend "after the probe"
|
|
n=$(ready_backends "$ctl")
|
|
[ "$n" -gt 0 ] || {
|
|
echo "SETUP FAILURE: $ctl has no ready backend after the probe, so its unreachability proves nothing" >&2
|
|
exit 1
|
|
}
|
|
|
|
verdict() { printf '%s\n' "$log" | awk -v k="$1" '$1 == k { print $2; exit }'; }
|
|
ctl_seen=$(verdict CTL)
|
|
adm_seen=$(verdict ADM)
|
|
hook_seen=$(verdict HOOK)
|
|
for v in "$ctl_seen" "$adm_seen" "$hook_seen"; do
|
|
[ -n "$v" ] || {
|
|
echo "SETUP FAILURE: the probe did not report all three destinations" >&2
|
|
exit 1
|
|
}
|
|
done
|
|
|
|
# From here every branch is a real verdict: the exact direct
|
|
# destination stayed ready before, throughout and after, and the
|
|
# controller pods neither restarted nor were replaced.
|
|
rc=0
|
|
if [ "$ctl_seen" = reachable ]; then
|
|
echo "OK unrelated 443 egress works (${ctl}:${ctl_port})"
|
|
else
|
|
echo "BUG unrelated 443 egress DENIED to a live backend (${ctl}:${ctl_port}) -- the webhook deny is not scoped to the webhook port" >&2
|
|
rc=1
|
|
fi
|
|
if [ "$adm_seen" = unreachable ]; then
|
|
echo "OK webhook denied through its Service ClusterIP"
|
|
else
|
|
echo "BUG webhook reachable through its Service ClusterIP (${adm}:${adm_port})" >&2
|
|
rc=1
|
|
fi
|
|
if [ "$hook_seen" = unreachable ]; then
|
|
echo "OK webhook denied on its backend port"
|
|
else
|
|
echo "BUG webhook reachable directly on its backend port (${pod_ip}:${hook_port})" >&2
|
|
rc=1
|
|
fi
|
|
exit "$rc"
|
|
check:
|
|
($error == null): true
|
|
finally:
|
|
- delete:
|
|
ref:
|
|
apiVersion: v1
|
|
kind: Pod
|
|
name: admission-deny-probe
|