mirror of
https://github.com/cozystack/cozystack.git
synced 2026-10-30 16:47:20 +09:00
fix(cozystack-basics): validate legacy Ingress hostnames against the tenant apex
The Gateway (cozystack-gateway-hostname-policy) and Route (cozystack-route-hostname-policy) admission policies constrain tenant hostnames on the Gateway API path. Gateway API is opt-in and off by default, so in the default configuration tenant applications publish through a legacy networking.k8s.io/Ingress on the shared ingress-nginx, which had no equivalent policy. Add a fail-closed ValidatingAdmissionPolicy + Binding for Ingress in tenant-* namespaces. A hostname on spec.rules[].host or spec.tls[].hosts[] is allowed when it is within the namespace's own apex (namespace.cozystack.io/host), or when it is a concrete (non-wildcard) name entirely outside the platform root apex (_cluster.root-host); a hostname under the platform root apex but outside the namespace's own apex, and a wildcard taken through the outside-root path (which could match the platform apex itself), are rejected. The outside-root branch keeps the kubernetes app's Proxied exposeMethod working, since it routes a user-supplied external domain to a nested cluster. A rule with no host (absent or empty, an unbounded catch-all) and spec.defaultBackend are also rejected. The policy templates the platform root apex into its CEL and renders only when _cluster.root-host is set, so it is never shipped in a state that would admit every hostname. A tenant-* namespace missing its own apex label is denied. Cover the policy with helm-unittest (render, including the root-host, empty-host, and wildcard guards) and behavioral admission tests against the live apiserver (in-apex allow, external-domain allow, own-apex-wildcard allow, external-tls allow, under-root-outside-apex deny, prefix-adjacent-under-root deny, wildcard-matching-apex deny, hostless deny, empty-string-host deny, defaultBackend deny), and document it as an additional layer in the gateway security model. Assisted-By: Claude <noreply@anthropic.com> Signed-off-by: Aleksei Sviridkin <f@lex.la>
This commit is contained in:
4 files changed
+716
-1
No files matched your search
@@ -401,6 +401,402 @@ EOF
|
||||
kubectl -n tenant-test delete httproute route-hostname-allow-probe --ignore-not-found
|
||||
}
|
||||
|
||||
@test "cozystack-ingress-hostname-policy VAP and binding are installed" {
|
||||
# Legacy-Ingress counterpart of the gateway/route hostname VAPs. Gateway
|
||||
# API is off by default, so tenant apps publish through a legacy
|
||||
# networking.k8s.io/Ingress; if these resources are missing, admission
|
||||
# cannot constrain tenant Ingress hostnames on the shared ingress-nginx.
|
||||
# Make that loud instead of letting it fall through as a silent pass.
|
||||
kubectl get validatingadmissionpolicy cozystack-ingress-hostname-policy -o yaml
|
||||
kubectl get validatingadmissionpolicybinding cozystack-ingress-hostname-policy -o yaml
|
||||
actions=$(kubectl get validatingadmissionpolicybinding cozystack-ingress-hostname-policy -o jsonpath='{.spec.validationActions}')
|
||||
echo "binding.validationActions=$actions"
|
||||
case "$actions" in
|
||||
*Deny*) ;;
|
||||
*) echo "SETUP FAILURE: binding.validationActions lacks Deny (got '$actions')" >&2; return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
@test "cozystack-ingress-hostname-policy VAP rejects an Ingress with a foreign-apex host" {
|
||||
# tenant-test's own apex (namespace.cozystack.io/host) is
|
||||
# test.example.org; the platform root apex is example.org.
|
||||
# dashboard.example.org sits under the platform root apex but outside
|
||||
# the tenant's own apex, so it must be denied. As with the route tests,
|
||||
# use `if !` so the dash `set -e` runner captures the expected-rejection
|
||||
# exit status.
|
||||
# Pre-clean a stale probe left by an interrupted prior run (e2e convention).
|
||||
kubectl -n tenant-test delete ingress ingress-out-of-apex-host-probe --ignore-not-found
|
||||
if ! output=$(kubectl apply -f - 2>&1 <<'EOF'
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: ingress-out-of-apex-host-probe
|
||||
namespace: tenant-test
|
||||
spec:
|
||||
rules:
|
||||
- host: "dashboard.example.org"
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: noop
|
||||
port:
|
||||
number: 80
|
||||
EOF
|
||||
); then
|
||||
echo "$output" | grep -qi "ValidatingAdmissionPolicy"
|
||||
echo "$output" | grep -qi "must be within"
|
||||
else
|
||||
kubectl -n tenant-test delete ingress ingress-out-of-apex-host-probe --ignore-not-found
|
||||
echo "BUG: admission accepted an out-of-apex Ingress host — ingress was created in tenant-test" >&2
|
||||
echo "$output" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
@test "cozystack-ingress-hostname-policy VAP rejects a prefix-adjacent host under the root apex" {
|
||||
# Dot-boundary guard: eviltest.example.org is under the platform root
|
||||
# apex example.org but is NOT a subdomain of the tenant's own apex
|
||||
# test.example.org. A naive suffix check without the leading dot would
|
||||
# match "test.example.org" inside "eviltest.example.org" and wrongly
|
||||
# admit it; the CEL uses endsWith("." + apex), so it is correctly denied
|
||||
# (under root, outside own apex).
|
||||
# Pre-clean a stale probe left by an interrupted prior run (e2e convention).
|
||||
kubectl -n tenant-test delete ingress ingress-prefix-adjacent-probe --ignore-not-found
|
||||
if ! output=$(kubectl apply -f - 2>&1 <<'EOF'
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: ingress-prefix-adjacent-probe
|
||||
namespace: tenant-test
|
||||
spec:
|
||||
rules:
|
||||
- host: "eviltest.example.org"
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: noop
|
||||
port:
|
||||
number: 80
|
||||
EOF
|
||||
); then
|
||||
echo "$output" | grep -qi "ValidatingAdmissionPolicy"
|
||||
else
|
||||
kubectl -n tenant-test delete ingress ingress-prefix-adjacent-probe --ignore-not-found
|
||||
echo "BUG: admission accepted a prefix-adjacent host under the root apex in tenant-test" >&2
|
||||
echo "$output" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
@test "cozystack-ingress-hostname-policy VAP rejects an Ingress with a foreign-apex tls host" {
|
||||
# The rule host is in-apex, but a spec.tls[].hosts[] entry
|
||||
# (harbor.example.org) sits under the platform root apex outside the
|
||||
# tenant's own apex. The tls branch must deny it on its own — an Ingress
|
||||
# carries a hostname in spec.tls[].hosts[] as well as spec.rules[].host,
|
||||
# so both are constrained the same way.
|
||||
# Pre-clean a stale probe left by an interrupted prior run (e2e convention).
|
||||
kubectl -n tenant-test delete ingress ingress-tls-out-of-apex-probe --ignore-not-found
|
||||
if ! output=$(kubectl apply -f - 2>&1 <<'EOF'
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: ingress-tls-out-of-apex-probe
|
||||
namespace: tenant-test
|
||||
spec:
|
||||
tls:
|
||||
- hosts:
|
||||
- "harbor.example.org"
|
||||
secretName: noop-tls
|
||||
rules:
|
||||
- host: "harbor.test.example.org"
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: noop
|
||||
port:
|
||||
number: 80
|
||||
EOF
|
||||
); then
|
||||
echo "$output" | grep -qi "ValidatingAdmissionPolicy"
|
||||
echo "$output" | grep -qi "must be within"
|
||||
else
|
||||
kubectl -n tenant-test delete ingress ingress-tls-out-of-apex-probe --ignore-not-found
|
||||
echo "BUG: admission accepted an out-of-apex Ingress tls host in tenant-test" >&2
|
||||
echo "$output" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
@test "cozystack-ingress-hostname-policy VAP rejects a hostless catch-all Ingress rule" {
|
||||
# A rule with no .host matches all inbound HTTP on the shared
|
||||
# ingress-nginx — an unbounded catch-all. Every rule must carry an
|
||||
# in-apex host, so a hostless rule is denied.
|
||||
# Pre-clean a stale probe left by an interrupted prior run (e2e convention).
|
||||
kubectl -n tenant-test delete ingress ingress-hostless-probe --ignore-not-found
|
||||
if ! output=$(kubectl apply -f - 2>&1 <<'EOF'
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: ingress-hostless-probe
|
||||
namespace: tenant-test
|
||||
spec:
|
||||
rules:
|
||||
- http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: noop
|
||||
port:
|
||||
number: 80
|
||||
EOF
|
||||
); then
|
||||
echo "$output" | grep -qi "ValidatingAdmissionPolicy"
|
||||
else
|
||||
kubectl -n tenant-test delete ingress ingress-hostless-probe --ignore-not-found
|
||||
echo "BUG: admission accepted a hostless catch-all Ingress rule in tenant-test" >&2
|
||||
echo "$output" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
@test "cozystack-ingress-hostname-policy VAP rejects an empty-string rule host" {
|
||||
# host: "" is a valid Ingress rule and a catch-all on the shared
|
||||
# ingress-nginx, the same as an absent host. Without an explicit empty
|
||||
# guard it would satisfy the "outside the root apex" branch and be
|
||||
# admitted, so the CEL denies it via r.host != "" (and via has(r.host)
|
||||
# if the apiserver drops the empty value). Either way, expect denial.
|
||||
# Pre-clean a stale probe left by an interrupted prior run (e2e convention).
|
||||
kubectl -n tenant-test delete ingress ingress-empty-host-probe --ignore-not-found
|
||||
if ! output=$(kubectl apply -f - 2>&1 <<'EOF'
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: ingress-empty-host-probe
|
||||
namespace: tenant-test
|
||||
spec:
|
||||
rules:
|
||||
- host: ""
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: noop
|
||||
port:
|
||||
number: 80
|
||||
EOF
|
||||
); then
|
||||
echo "$output" | grep -qi "ValidatingAdmissionPolicy"
|
||||
else
|
||||
kubectl -n tenant-test delete ingress ingress-empty-host-probe --ignore-not-found
|
||||
echo "BUG: admission accepted an empty-string rule host in tenant-test" >&2
|
||||
echo "$output" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
@test "cozystack-ingress-hostname-policy VAP rejects an Ingress default backend" {
|
||||
# spec.defaultBackend is a catch-all for otherwise-unmatched traffic on
|
||||
# the shared ingress-nginx; a tenant may not declare one.
|
||||
# Pre-clean a stale probe left by an interrupted prior run (e2e convention).
|
||||
kubectl -n tenant-test delete ingress ingress-defaultbackend-probe --ignore-not-found
|
||||
if ! output=$(kubectl apply -f - 2>&1 <<'EOF'
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: ingress-defaultbackend-probe
|
||||
namespace: tenant-test
|
||||
spec:
|
||||
defaultBackend:
|
||||
service:
|
||||
name: noop
|
||||
port:
|
||||
number: 80
|
||||
EOF
|
||||
); then
|
||||
echo "$output" | grep -qi "ValidatingAdmissionPolicy"
|
||||
else
|
||||
kubectl -n tenant-test delete ingress ingress-defaultbackend-probe --ignore-not-found
|
||||
echo "BUG: admission accepted an Ingress with spec.defaultBackend in tenant-test" >&2
|
||||
echo "$output" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
@test "cozystack-ingress-hostname-policy VAP allows an in-apex Ingress" {
|
||||
# Inverse sanity check: an Ingress whose rule host and tls hosts are all
|
||||
# under test.example.org must NOT be rejected by the VAP.
|
||||
# Pre-clean a stale probe left by an interrupted prior run (e2e convention).
|
||||
kubectl -n tenant-test delete ingress ingress-hostname-allow-probe --ignore-not-found
|
||||
kubectl apply -f - <<'EOF'
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: ingress-hostname-allow-probe
|
||||
namespace: tenant-test
|
||||
spec:
|
||||
tls:
|
||||
- hosts:
|
||||
- "harbor.test.example.org"
|
||||
secretName: noop-tls
|
||||
rules:
|
||||
- host: "harbor.test.example.org"
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: noop
|
||||
port:
|
||||
number: 80
|
||||
EOF
|
||||
|
||||
kubectl -n tenant-test delete ingress ingress-hostname-allow-probe --ignore-not-found
|
||||
}
|
||||
|
||||
@test "cozystack-ingress-hostname-policy VAP allows an Ingress host outside the platform root apex" {
|
||||
# Narrow rule: a tenant may route its own domain that lies entirely
|
||||
# outside the platform root apex (example.org). This is the path the
|
||||
# kubernetes app's Proxied exposeMethod uses — addons.ingressNginx.hosts
|
||||
# routes a user-supplied external domain to a nested cluster — and it
|
||||
# must be admitted. shop.example.net is not under example.org.
|
||||
# Pre-clean a stale probe left by an interrupted prior run (e2e convention).
|
||||
kubectl -n tenant-test delete ingress ingress-external-host-probe --ignore-not-found
|
||||
kubectl apply -f - <<'EOF'
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: ingress-external-host-probe
|
||||
namespace: tenant-test
|
||||
spec:
|
||||
rules:
|
||||
- host: "shop.example.net"
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: noop
|
||||
port:
|
||||
number: 80
|
||||
EOF
|
||||
|
||||
kubectl -n tenant-test delete ingress ingress-external-host-probe --ignore-not-found
|
||||
}
|
||||
|
||||
@test "cozystack-ingress-hostname-policy VAP rejects a wildcard host that matches the platform apex" {
|
||||
# *.org is "outside" the platform root apex example.org by a plain string
|
||||
# test, but at ingress-nginx *.org also matches the bare root apex
|
||||
# example.org. The outside-root path is gated on the host not being a
|
||||
# wildcard, so this must be denied. (A wildcard under the tenant's own
|
||||
# apex, e.g. *.test.example.org, is still allowed via the own-apex path.)
|
||||
# Pre-clean a stale probe left by an interrupted prior run (e2e convention).
|
||||
kubectl -n tenant-test delete ingress ingress-wildcard-host-probe --ignore-not-found
|
||||
if ! output=$(kubectl apply -f - 2>&1 <<'EOF'
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: ingress-wildcard-host-probe
|
||||
namespace: tenant-test
|
||||
spec:
|
||||
rules:
|
||||
- host: "*.org"
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: noop
|
||||
port:
|
||||
number: 80
|
||||
EOF
|
||||
); then
|
||||
echo "$output" | grep -qi "ValidatingAdmissionPolicy"
|
||||
else
|
||||
kubectl -n tenant-test delete ingress ingress-wildcard-host-probe --ignore-not-found
|
||||
echo "BUG: admission accepted a wildcard host matching the platform apex in tenant-test" >&2
|
||||
echo "$output" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
@test "cozystack-ingress-hostname-policy VAP allows a wildcard host under the own apex" {
|
||||
# A wildcard under the tenant's own apex (*.test.example.org) is admitted
|
||||
# via the own-apex path — the anti-wildcard gate applies only to the
|
||||
# outside-root path. Locks the behavior the CEL comment describes.
|
||||
# Pre-clean a stale probe left by an interrupted prior run (e2e convention).
|
||||
kubectl -n tenant-test delete ingress ingress-ownapex-wildcard-probe --ignore-not-found
|
||||
kubectl apply -f - <<'EOF'
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: ingress-ownapex-wildcard-probe
|
||||
namespace: tenant-test
|
||||
spec:
|
||||
rules:
|
||||
- host: "*.test.example.org"
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: noop
|
||||
port:
|
||||
number: 80
|
||||
EOF
|
||||
|
||||
kubectl -n tenant-test delete ingress ingress-ownapex-wildcard-probe --ignore-not-found
|
||||
}
|
||||
|
||||
@test "cozystack-ingress-hostname-policy VAP allows an external domain on spec.tls[].hosts" {
|
||||
# The outside-root allowance covers spec.tls[].hosts[] as well as
|
||||
# spec.rules[].host: a concrete external domain not under the platform
|
||||
# root apex must be admitted on the tls block too.
|
||||
# Pre-clean a stale probe left by an interrupted prior run (e2e convention).
|
||||
kubectl -n tenant-test delete ingress ingress-external-tls-probe --ignore-not-found
|
||||
kubectl apply -f - <<'EOF'
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: ingress-external-tls-probe
|
||||
namespace: tenant-test
|
||||
spec:
|
||||
tls:
|
||||
- hosts:
|
||||
- "shop.example.net"
|
||||
secretName: noop-tls
|
||||
rules:
|
||||
- host: "shop.example.net"
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: noop
|
||||
port:
|
||||
number: 80
|
||||
EOF
|
||||
|
||||
kubectl -n tenant-test delete ingress ingress-external-tls-probe --ignore-not-found
|
||||
}
|
||||
|
||||
@test "HTTPRoute with a matching parentRef reaches Accepted status" {
|
||||
# Put a Gateway and a route in the same namespace so allowedRoutes: Same accepts them.
|
||||
kubectl apply -f - <<'EOF'
|
||||
|
||||
@@ -81,7 +81,7 @@ Tenants in Cozystack interact with the platform exclusively through `apps.cozyst
|
||||
The protections below split into three groups by who they defend against:
|
||||
|
||||
- **Tenant-user-input gates** — Layer 4 (`cozystack-tenant-host-policy`). `Tenant.spec.host` is the user-supplied field that surfaces as a security boundary at the hostname layer; it is gated on every Create / Update via `cozystack-api`'s admission chain (`pkg/registry/apps/application/rest.go`).
|
||||
- **Defense-in-depth** — Layers 1, 2, 5, 6, 7. These do not protect against tenant-user input (tenants don't hold the relevant RBAC). They guard against bugs in cozystack-controller / Flux, supply-chain compromise of an app chart that emits Gateway API resources, and confused-deputy mistakes by a cluster admin. Fail-closed via `failurePolicy: Fail` + `validationActions: [Deny]`.
|
||||
- **Defense-in-depth** — Layers 1, 2, 5, 6, 7, 8. These do not protect against tenant-user input (tenants don't hold the relevant RBAC). They guard against bugs in cozystack-controller / Flux, supply-chain compromise of an app chart that emits Gateway API or Ingress resources, and confused-deputy mistakes by a cluster admin. Fail-closed via `failurePolicy: Fail` + `validationActions: [Deny]`. Layers 1-7 cover the opt-in Gateway API dataplane; Layer 8 covers the legacy Ingress dataplane, which is the default (Gateway API is off by default).
|
||||
- **Admin-against-themselves** — Layer 3 (`cozystack-gateway-attached-namespaces-policy`). Rejects a `kubectl edit packages.cozystack.io` that would slip a `tenant-*` entry into the platform Package's `gateway.attachedNamespaces`. Layer 6 catches the same misconfiguration at helm render time.
|
||||
|
||||
1. **Namespace whitelist on listeners.** Every listener carries an `allowedRoutes.namespaces.from: Selector` matching the built-in `kubernetes.io/metadata.name` label (written by kube-apiserver, unspoofable). HTTPS / TLS-passthrough listeners accept routes from the publishing tenant's namespace plus `gateway.attachedNamespaces` in the platform chart (default includes the `cozy-*` namespaces for platform services and `default` for the Kubernetes API TLSRoute). A namespace outside the list literally cannot attach any `HTTPRoute` or `TLSRoute` to those listeners. The plain-HTTP listener (port 80) carries a strictly narrower selector — only the tenant namespace itself (where the controller-owned http→https redirect HTTPRoute lives) and `cozy-cert-manager` (HTTP-01 ACME challenge HTTPRoutes) — so app HTTPRoutes attaching by hostname cannot bind to port 80 and serve plaintext. HTTPS listeners additionally restrict `allowedRoutes.kinds` to `HTTPRoute` (and TLS-passthrough listeners to `TLSRoute`), preventing GRPCRoute / TCPRoute / UDPRoute from attaching outside the route-hostname VAP's coverage.
|
||||
@@ -91,6 +91,7 @@ The protections below split into three groups by who they defend against:
|
||||
5. **`cozystack-namespace-host-label-policy`** — VAP on core `v1 Namespace` CREATE/UPDATE. Rejects any set or change of the `namespace.cozystack.io/host` label, except by the same trusted-caller whitelist as layer 4. This closes both first-time label writes on CREATE and first-time adds on UPDATE — only cozystack/Flux service accounts (which apply the tenant chart) can stamp the label.
|
||||
6. **Render-time `fail` in cozystack-basics.** The cozystack-basics chart fails the helm render if `_cluster.gateway-attached-namespaces` contains any `tenant-*` entry. Triggers on the helm-install path before the cluster ever sees the values — complements layer 3 which triggers at `kubectl apply` time.
|
||||
7. **`cozystack-route-hostname-policy`** — VAP on `gateway.networking.k8s.io/v1 HTTPRoute` and `v1alpha2 TLSRoute` CREATE/UPDATE. Scoped to `tenant-*` namespaces (cozy-* are cluster-admin-managed and trusted to publish under any apex). Rejects any `spec.hostnames` entry that is not equal to the namespace's `namespace.cozystack.io/host` label or a subdomain of it. Defense-in-depth against an app chart bug or supply-chain compromise that emits Gateway API resources outside the tenant's apex — tenants in Cozystack do not hold `gateway.networking.k8s.io/*` RBAC by design, so this is not a tenant-user defense. The within-apex cross-namespace case (a tenant chart claiming a hostname that is published by a `cozy-*` app) is handled by the controller at reconciliation time: when two routes from different namespaces claim the same hostname, the `cozy-*` namespace wins and the loser receives a `HostnameConflict` condition under the controller's name in `Status.Parents`.
|
||||
8. **`cozystack-ingress-hostname-policy`** — VAP on core `networking.k8s.io/v1 Ingress` CREATE/UPDATE. Gateway API is opt-in and **off by default**, so in the default configuration tenant applications publish through a legacy Ingress on the shared ingress-nginx; layers 1-7 constrain tenant hostnames on the opt-in Gateway path, and this VAP adds a hostname constraint on the default Ingress path. Scoped to `tenant-*` namespaces (cozy-* are cluster-admin-managed and trusted). A hostname on `spec.rules[].host` or `spec.tls[].hosts[]` is allowed when it is within the namespace's own `namespace.cozystack.io/host` apex, OR when it lies entirely outside the platform root apex (`_cluster.root-host`) — the second case lets a tenant route its own external custom domain (for example the `kubernetes` app's Proxied `addons.ingressNginx.hosts`, which routes a user-supplied domain to a nested cluster). A hostname that falls under the platform root apex but outside the namespace's own apex is rejected; a rule with no host (an unbounded catch-all) and `spec.defaultBackend` (a catch-all for unmatched traffic) are also rejected. Fail-closed: a `tenant-*` namespace missing its host label is denied, and the policy renders only when `_cluster.root-host` is set. This bounds which apexes a tenant Ingress may claim under the platform domain; it does not attempt to resolve every possible hostname collision.
|
||||
|
||||
For `tenant-root` the allowed host suffix is `publishing.host`; for any `tenant-<name>` that inherits from its parent the suffix is `<name>.<parent apex>`. A child tenant with an independent apex (`customer1.io` instead of a subdomain) is handled correctly because the VAP reads the per-namespace label rather than assuming a subdomain hierarchy.
|
||||
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
{{/*
|
||||
cozystack-ingress-hostname-policy
|
||||
|
||||
Legacy-Ingress counterpart of the Gateway and Route hostname VAPs in
|
||||
this package. Gateway API is opt-in and off by default; in the default
|
||||
configuration tenant applications publish through a legacy
|
||||
networking.k8s.io/Ingress (see packages/apps/harbor). This VAP gives
|
||||
the default Ingress path a tenant-scoped hostname constraint, in the
|
||||
same family as the Gateway and Route policies.
|
||||
|
||||
Scope: only Ingresses in tenant-* namespaces are gated. cozy-*
|
||||
namespaces are cluster-admin-managed and trusted to publish under any
|
||||
apex (they back platform services like dashboard, harbor, bucket).
|
||||
|
||||
Rule: a tenant Ingress hostname is allowed when it is within the
|
||||
namespace's own apex (the namespace.cozystack.io/host label), OR when
|
||||
it is a concrete (non-wildcard) name that lies entirely outside the
|
||||
platform root apex (_cluster.root-host) — the latter lets a tenant
|
||||
route its own external custom domain (for example via the kubernetes
|
||||
app's Proxied exposeMethod, which routes a user-supplied domain to a
|
||||
nested cluster). A hostname that falls under the platform root apex but
|
||||
outside the namespace's own apex is rejected; so is a wildcard taken
|
||||
through the outside-root path, because a wildcard such as *.<root-parent>
|
||||
would also match the platform root apex itself. Both hostname carriers
|
||||
are checked:
|
||||
- spec.rules[].host — routing host. A rule without a host (absent or
|
||||
empty) matches all inbound HTTP on the shared ingress-nginx, an
|
||||
unbounded catch-all, so every rule must carry a non-empty host — a
|
||||
missing or empty host is rejected.
|
||||
- spec.tls[].hosts[] — TLS SNI hosts.
|
||||
spec.defaultBackend (a catch-all for otherwise-unmatched traffic) is
|
||||
also rejected.
|
||||
|
||||
The policy renders only when _cluster.root-host is set; without it the
|
||||
"outside the root apex" test is undefined and the policy could not
|
||||
constrain anything, so it is omitted rather than shipped in a state
|
||||
that admits everything.
|
||||
*/}}
|
||||
{{- /*
|
||||
CEL safety note: has() only works on field access expressions, not on
|
||||
top-level variables. namespaceObject is a top-level var supplied by
|
||||
the apiserver, so the nil check uses `== null` first; the nested
|
||||
has() then walks into metadata.labels safely. See the feedback memory
|
||||
cel-has-field-only.md.
|
||||
|
||||
Case handling: kube-apiserver validates spec.rules[].host and
|
||||
spec.tls[].hosts[] as DNS-1123 subdomains (lowercase only), so the
|
||||
apex/subdomain string comparisons below need no case normalization — a
|
||||
mixed-case host is rejected by built-in validation and never reaches a
|
||||
created state, so casing cannot be used to sidestep the comparisons.
|
||||
|
||||
Defense-in-depth contract: this VAP is fail-closed. A tenant-*
|
||||
namespace without `namespace.cozystack.io/host` is rejected (not
|
||||
silently allowed) so a missing or scrubbed own-apex label cannot open
|
||||
the Ingress path by itself. The only legitimate state where the label
|
||||
is absent is an in-flight namespace creation that hasn't finished
|
||||
labelling yet — in which case the caller should retry.
|
||||
*/}}
|
||||
{{- $rootHost := index .Values._cluster "root-host" }}
|
||||
{{- if $rootHost }}
|
||||
{{- $A := `namespaceObject.metadata.labels["namespace.cozystack.io/host"]` }}
|
||||
{{- $celValidator := printf `(namespaceObject == null || !has(namespaceObject.metadata.labels) || !("namespace.cozystack.io/host" in namespaceObject.metadata.labels)) ? false : (!has(object.spec.defaultBackend) && (!has(object.spec.rules) || object.spec.rules.all(r, has(r.host) && r.host != "" && (r.host == %s || r.host.endsWith("." + %s) || (!r.host.startsWith("*.") && !(r.host == %q || r.host.endsWith(%q)))))) && (!has(object.spec.tls) || object.spec.tls.all(t, !has(t.hosts) || t.hosts.all(h, h != "" && (h == %s || h.endsWith("." + %s) || (!h.startsWith("*.") && !(h == %q || h.endsWith(%q))))))))` $A $A $rootHost (printf ".%s" $rootHost) $A $A $rootHost (printf ".%s" $rootHost) -}}
|
||||
---
|
||||
apiVersion: admissionregistration.k8s.io/v1
|
||||
kind: ValidatingAdmissionPolicy
|
||||
metadata:
|
||||
name: cozystack-ingress-hostname-policy
|
||||
labels:
|
||||
internal.cozystack.io/managed-by-cozystack: ""
|
||||
spec:
|
||||
failurePolicy: Fail
|
||||
matchConstraints:
|
||||
resourceRules:
|
||||
- apiGroups: ["networking.k8s.io"]
|
||||
apiVersions: ["v1"]
|
||||
operations: ["CREATE", "UPDATE"]
|
||||
resources: ["ingresses"]
|
||||
matchConditions:
|
||||
- name: tenant-namespace
|
||||
expression: object.metadata.namespace.startsWith("tenant-")
|
||||
validations:
|
||||
- expression: >-
|
||||
{{ $celValidator }}
|
||||
messageExpression: >-
|
||||
"Ingress hostnames must be within the namespace's namespace.cozystack.io/host apex, or be a concrete name outside the platform apex {{ $rootHost }}; a hostname under {{ $rootHost }} but outside the namespace apex, a wildcard taken through the outside-apex path, a rule without a host, and spec.defaultBackend are not allowed (namespace " + object.metadata.namespace + ")"
|
||||
reason: Forbidden
|
||||
---
|
||||
apiVersion: admissionregistration.k8s.io/v1
|
||||
kind: ValidatingAdmissionPolicyBinding
|
||||
metadata:
|
||||
name: cozystack-ingress-hostname-policy
|
||||
labels:
|
||||
internal.cozystack.io/managed-by-cozystack: ""
|
||||
spec:
|
||||
policyName: cozystack-ingress-hostname-policy
|
||||
validationActions: [Deny]
|
||||
{{- end }}
|
||||
@@ -0,0 +1,221 @@
|
||||
suite: cozystack-ingress-hostname-policy VAP
|
||||
templates:
|
||||
- templates/ingress-hostname-policy.yaml
|
||||
|
||||
release:
|
||||
name: cozystack-basics
|
||||
namespace: cozy-system
|
||||
|
||||
# The policy templates the platform root apex (_cluster.root-host) into
|
||||
# its CEL and renders only when that value is set, so every rendering
|
||||
# test supplies it. RFC 2606 example.org stands in for the platform apex.
|
||||
set:
|
||||
_cluster:
|
||||
root-host: example.org
|
||||
|
||||
tests:
|
||||
- it: renders ValidatingAdmissionPolicy + Binding for legacy Ingress
|
||||
# Legacy Ingress (networking.k8s.io/v1) is the default ingress path
|
||||
# when Gateway API is off. This template gates that path in the same
|
||||
# family as the Gateway/route VAPs: 1 VAP + 1 Binding = 2 docs total.
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 2
|
||||
- documentIndex: 0
|
||||
equal:
|
||||
path: kind
|
||||
value: ValidatingAdmissionPolicy
|
||||
- documentIndex: 0
|
||||
equal:
|
||||
path: metadata.name
|
||||
value: cozystack-ingress-hostname-policy
|
||||
- documentIndex: 1
|
||||
equal:
|
||||
path: kind
|
||||
value: ValidatingAdmissionPolicyBinding
|
||||
- documentIndex: 1
|
||||
equal:
|
||||
path: metadata.name
|
||||
value: cozystack-ingress-hostname-policy
|
||||
|
||||
- it: gates on tenant-* namespace prefix via matchConditions
|
||||
asserts:
|
||||
- documentIndex: 0
|
||||
equal:
|
||||
path: spec.matchConditions[0].name
|
||||
value: tenant-namespace
|
||||
- documentIndex: 0
|
||||
equal:
|
||||
path: spec.matchConditions[0].expression
|
||||
value: object.metadata.namespace.startsWith("tenant-")
|
||||
|
||||
- it: targets networking.k8s.io v1 Ingress on CREATE and UPDATE
|
||||
asserts:
|
||||
- documentIndex: 0
|
||||
equal:
|
||||
path: spec.matchConstraints.resourceRules[0].apiGroups
|
||||
value: ["networking.k8s.io"]
|
||||
- documentIndex: 0
|
||||
equal:
|
||||
path: spec.matchConstraints.resourceRules[0].apiVersions
|
||||
value: ["v1"]
|
||||
- documentIndex: 0
|
||||
equal:
|
||||
path: spec.matchConstraints.resourceRules[0].resources
|
||||
value: ["ingresses"]
|
||||
- documentIndex: 0
|
||||
equal:
|
||||
path: spec.matchConstraints.resourceRules[0].operations
|
||||
value: ["CREATE", "UPDATE"]
|
||||
|
||||
- it: validates both rules[].host and tls[].hosts against namespace.cozystack.io/host label
|
||||
asserts:
|
||||
- documentIndex: 0
|
||||
matchRegex:
|
||||
path: spec.validations[0].expression
|
||||
pattern: 'object\.spec\.rules\.all\(r,'
|
||||
- documentIndex: 0
|
||||
matchRegex:
|
||||
path: spec.validations[0].expression
|
||||
pattern: 'object\.spec\.tls\.all\(t,'
|
||||
- documentIndex: 0
|
||||
matchRegex:
|
||||
path: spec.validations[0].expression
|
||||
pattern: 't\.hosts\.all\(h,'
|
||||
- documentIndex: 0
|
||||
matchRegex:
|
||||
path: spec.validations[0].expression
|
||||
pattern: 'namespace\.cozystack\.io/host'
|
||||
|
||||
- it: allows a hostname outside the platform root apex (external custom domain)
|
||||
# Narrow rule: a tenant may route its own domain that lies entirely
|
||||
# outside the platform root apex (templated from _cluster.root-host).
|
||||
# Pin the not-under-root clause for both rules and tls so this
|
||||
# allowance is not silently dropped — it is the path the kubernetes
|
||||
# app's Proxied exposeMethod relies on to route a user-supplied domain.
|
||||
asserts:
|
||||
- documentIndex: 0
|
||||
matchRegex:
|
||||
path: spec.validations[0].expression
|
||||
pattern: '!\(r\.host == "example\.org" \|\| r\.host\.endsWith\("\.example\.org"\)\)'
|
||||
- documentIndex: 0
|
||||
matchRegex:
|
||||
path: spec.validations[0].expression
|
||||
pattern: '!\(h == "example\.org" \|\| h\.endsWith\("\.example\.org"\)\)'
|
||||
|
||||
- it: does not let a wildcard use the outside-root path (would match the platform apex)
|
||||
# A wildcard on the root's parent label (e.g. *.org when root is
|
||||
# example.org) is "outside root" by the plain string test but matches
|
||||
# the bare platform apex at ingress-nginx. The outside-root branch is
|
||||
# therefore gated on the host not being a wildcard, so such a host is
|
||||
# denied (a wildcard is only admitted when it is under the own apex).
|
||||
asserts:
|
||||
- documentIndex: 0
|
||||
matchRegex:
|
||||
path: spec.validations[0].expression
|
||||
pattern: '!r\.host\.startsWith\("\*\."\) && !\(r\.host == "example\.org"'
|
||||
- documentIndex: 0
|
||||
matchRegex:
|
||||
path: spec.validations[0].expression
|
||||
pattern: '!h\.startsWith\("\*\."\) && !\(h == "example\.org"'
|
||||
|
||||
- it: allows a wildcard under the own apex (own-apex path is not wildcard-gated)
|
||||
# The !startsWith("*.") gate applies ONLY to the outside-root branch.
|
||||
# A wildcard under the tenant's own apex (e.g. *.test.example.org)
|
||||
# reaches the allow via the own-apex endsWith clause, which is a
|
||||
# standalone disjunct BEFORE the wildcard-gated group. Pin that
|
||||
# ordering so a future "gate all wildcards" change cannot silently
|
||||
# deny an own-apex wildcard cert.
|
||||
asserts:
|
||||
- documentIndex: 0
|
||||
matchRegex:
|
||||
path: spec.validations[0].expression
|
||||
pattern: 'r\.host\.endsWith\("\." \+ namespaceObject\.metadata\.labels\["namespace\.cozystack\.io/host"\]\) \|\| \(!r\.host\.startsWith\("\*\."\)'
|
||||
- documentIndex: 0
|
||||
matchRegex:
|
||||
path: spec.validations[0].expression
|
||||
pattern: 'h\.endsWith\("\." \+ namespaceObject\.metadata\.labels\["namespace\.cozystack\.io/host"\]\) \|\| \(!h\.startsWith\("\*\."\)'
|
||||
|
||||
- it: rejects a hostless or empty-host catch-all rule
|
||||
# A legacy Ingress rule with no .host (absent OR empty string) matches
|
||||
# ALL inbound HTTP on the shared ingress-nginx — an unbounded catch-all.
|
||||
# Unlike a Gateway route (bounded by its parent Gateway's
|
||||
# already-validated listeners), such a rule has no bound, so every rule
|
||||
# MUST carry a non-empty host. Pin both has(r.host) and r.host != "" —
|
||||
# the empty-string guard matters because an empty host would otherwise
|
||||
# satisfy the "outside the root apex" branch and be admitted.
|
||||
asserts:
|
||||
- documentIndex: 0
|
||||
matchRegex:
|
||||
path: spec.validations[0].expression
|
||||
pattern: 'object\.spec\.rules\.all\(r,\s*has\(r\.host\) && r\.host != ""'
|
||||
- documentIndex: 0
|
||||
matchRegex:
|
||||
path: spec.validations[0].expression
|
||||
pattern: 't\.hosts\.all\(h,\s*h != ""'
|
||||
|
||||
- it: rejects a tenant Ingress default backend (a catch-all for unmatched traffic)
|
||||
asserts:
|
||||
- documentIndex: 0
|
||||
matchRegex:
|
||||
path: spec.validations[0].expression
|
||||
pattern: '!has\(object\.spec\.defaultBackend\)'
|
||||
|
||||
- it: validation actions are Deny so requests fail closed
|
||||
asserts:
|
||||
- documentIndex: 1
|
||||
equal:
|
||||
path: spec.validationActions
|
||||
value: [Deny]
|
||||
|
||||
- it: failurePolicy is Fail so admission errors deny instead of allow
|
||||
asserts:
|
||||
- documentIndex: 0
|
||||
equal:
|
||||
path: spec.failurePolicy
|
||||
value: Fail
|
||||
|
||||
- it: CEL is fail-closed when namespace has no host label (returns false, not true)
|
||||
# Regression guard mirroring the route policy: a tenant-* namespace
|
||||
# missing its own-apex namespace.cozystack.io/host label must be
|
||||
# DENIED, not silently allowed. Pin the false branch and the exact
|
||||
# nil-guard structure so nobody flips it back without a test failure.
|
||||
asserts:
|
||||
- documentIndex: 0
|
||||
matchRegex:
|
||||
path: spec.validations[0].expression
|
||||
pattern: '\)\s*\?\s*false\s*:'
|
||||
- documentIndex: 0
|
||||
matchRegex:
|
||||
path: spec.validations[0].expression
|
||||
pattern: 'namespaceObject\s*==\s*null'
|
||||
- documentIndex: 0
|
||||
matchRegex:
|
||||
path: spec.validations[0].expression
|
||||
pattern: '!has\(namespaceObject\.metadata\.labels\)'
|
||||
- documentIndex: 0
|
||||
matchRegex:
|
||||
path: spec.validations[0].expression
|
||||
pattern: '!\("namespace\.cozystack\.io/host" in namespaceObject\.metadata\.labels\)'
|
||||
|
||||
- it: reason is Forbidden and message references the host label
|
||||
asserts:
|
||||
- documentIndex: 0
|
||||
equal:
|
||||
path: spec.validations[0].reason
|
||||
value: Forbidden
|
||||
- documentIndex: 0
|
||||
matchRegex:
|
||||
path: spec.validations[0].messageExpression
|
||||
pattern: 'namespace\.cozystack\.io/host'
|
||||
|
||||
- it: omits the policy when the platform root apex is unset (no fail-open)
|
||||
# Without _cluster.root-host the "outside the root apex" test is
|
||||
# undefined, so the policy could not constrain anything. Render nothing
|
||||
# rather than ship a policy that would admit every hostname.
|
||||
set:
|
||||
_cluster:
|
||||
root-host: ""
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
Reference in new issue
Block a user