From f5ec885d8d2c98c591a3e9ba4b4c09c6b1226196 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 9 Jul 2026 13:33:37 +0300 Subject: [PATCH 1/2] test(e2e): gate the talos image cache on real tenant reachability The in-sandbox Talos image mirror was pointed at by tenant Kubernetes CRs whenever its Deployment was Available, but "Available" only proves the seed finished and the server answers on localhost. It does not prove a worker's CDI importer can reach the Service. Tenant namespaces run under a default-deny Cilium egress that permits the outside world and kube-dns but not an arbitrary kube-system Service, so the importer resolves the name yet its TCP connect to the ClusterIP is silently dropped. The disk import then never starts, no tenant node joins within the 12-minute deadline, and the kubernetes-* tests fail deterministically -- worse than the intermittent public-factory flake the mirror was meant to relieve. Replace the localhost range-probe with one that faces the exact same network path as a worker importer: a throwaway Pod in the tenant namespace, labelled cdi.kubevirt.io=importer, that fetches the seeded image from the Service ClusterIP with a Range request and expects a 206. Point tenants at the mirror only when that end-to-end check passes; otherwise fall back to the public factory, restoring the guarantee that the mirror can only help, never make CI worse. The resolver also installs the importer egress allow at point-of-use, because Cilium's CRDs do not exist yet when the mirror manifest is first applied before install. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- hack/e2e-apps/talos-image-cache.sh | 171 +++++++++++++++++++++++------ 1 file changed, 140 insertions(+), 31 deletions(-) diff --git a/hack/e2e-apps/talos-image-cache.sh b/hack/e2e-apps/talos-image-cache.sh index 38d8e21ff..70760a735 100644 --- a/hack/e2e-apps/talos-image-cache.sh +++ b/hack/e2e-apps/talos-image-cache.sh @@ -2,33 +2,142 @@ # e2e CRs. # # Prefer the in-sandbox caching mirror (hack/e2e-talos-image-cache.yaml, deployed -# by hack/e2e-install-cozystack.bats) when it is Available, otherwise fall back to -# the chart default (the public factory.talos.dev) by emitting nothing. The -# mirror rides out the public factory's flaky/range-less egress that otherwise -# stalls worker DataVolume imports past the 12-minute node-join deadline — see -# cozystack/cozystack#3231. Falling back to the default means the mirror can only -# help, never make CI worse. +# by hack/e2e-install-cozystack.bats) when a worker's CDI importer can actually +# reach it, otherwise fall back to the chart default (the public factory.talos.dev) +# by emitting nothing. The mirror rides out the public factory's flaky/range-less +# egress that otherwise stalls worker DataVolume imports past the 12-minute +# node-join deadline — see cozystack/cozystack#3231. Falling back to the default +# means the mirror can only help, never make CI worse. +# +# Reachability is not assumed. Tenant namespaces run under a default-deny Cilium +# egress installed by the tenant chart: a worker's importer may reach the `world` +# entity (hence the public factory) and the tenant's own tree, but not an +# arbitrary kube-system Service. Without an explicit allow, the importer resolves +# talos-image-cache.kube-system.svc yet its TCP connect to the ClusterIP is +# silently dropped and the disk import never starts. This helper installs that +# allow (the CiliumClusterwideNetworkPolicy shipped in the mirror manifest) and +# then verifies the path end to end from a Pod that faces the exact same egress +# rules as a real importer, so the mirror is used only when it genuinely works. TALOS_IMAGE_CACHE_SVC_URL="${TALOS_IMAGE_CACHE_SVC_URL:-http://talos-image-cache.kube-system.svc}" _TALOS_IMAGE_FACTORY_DECISION_FILE="${_TALOS_IMAGE_FACTORY_DECISION_FILE:-/tmp/e2e-talos-image-factory-url}" +TALOS_IMAGE_CACHE_MANIFEST="${TALOS_IMAGE_CACHE_MANIFEST:-hack/e2e-talos-image-cache.yaml}" +# Tenant namespace the kubernetes-* CRs are created in (run-kubernetes.sh and the +# oidc bats all target tenant-test); the reachability probe runs here so it inherits +# that namespace's default-deny egress, exactly as a worker importer does. +TALOS_IMAGE_CACHE_PROBE_NS="${TALOS_IMAGE_CACHE_PROBE_NS:-tenant-test}" +TALOS_IMAGE_CACHE_PROBE_POD="${TALOS_IMAGE_CACHE_PROBE_POD:-talos-image-cache-probe}" -# _talos_image_cache_serves_ranges: succeed iff the mirror answers a byte-range -# request with 206 Partial Content. Runs curl inside the serve pod against -# localhost, range-probing the seeded image file it finds under /data, so the -# check works without the sandbox being able to reach the ClusterIP. -_talos_image_cache_serves_ranges() { - local pod code +# _apply_talos_image_cache_egress_policy: install the CiliumClusterwideNetworkPolicy +# that lets tenant CDI importer Pods egress to the mirror. It lives in the mirror +# manifest but is applied here, not in hack/e2e-install-cozystack.bats, because that +# manifest is applied before Cozystack (and thus before Cilium's CRDs) exists. +# Idempotent; best-effort — a failure just leaves the probe below to fall back. +_apply_talos_image_cache_egress_policy() { + command -v yq >/dev/null 2>&1 || return 1 + yq 'select(.kind == "CiliumClusterwideNetworkPolicy")' "$TALOS_IMAGE_CACHE_MANIFEST" 2>/dev/null \ + | kubectl apply -f - >/dev/null 2>&1 +} + +# _talos_image_cache_probe_overrides: emit the kubectl --overrides JSON for the +# probe Pod. $1 is the container image, $2 the shell command it runs. The single +# container replaces the one `kubectl run` generates (merge patch semantics), and +# the security context satisfies the tenant namespace's restricted PSA. +# +# Pure string builder, kept separate so it can be unit-tested: an edit that +# unbalances a quote here would emit invalid JSON, every probe would fail, and CI +# would silently fall back to the public factory instead of erroring. +_talos_image_cache_probe_overrides() { + printf '{"spec":{"securityContext":{"runAsNonRoot":true,"runAsUser":1000,"seccompProfile":{"type":"RuntimeDefault"}},"containers":[{"name":"c","image":"%s","command":["sh","-ec","%s"],"securityContext":{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]}}}]}}' \ + "$1" "$2" +} + +# _talos_image_cache_probe_succeeded: succeed iff the probe output ($1) carries the +# strict 206 marker. +# +# Gate strictly on 206. The old localhost check kept a reachable-but-range-less +# mirror and only warned; here any non-206 — unreachable (code=000) or a plain 200 — +# falls back to the public factory instead. serve.py always answers a valid range +# with 206, so the 200 case is unreachable in practice; the strict gate keeps the +# "use the mirror only when it is fully working" rule simple. +# +# Pure predicate, kept separate so it can be unit-tested: this single comparison is +# what decides whether tenant workers are pointed at the mirror at all. +_talos_image_cache_probe_succeeded() { + printf '%s' "$1" | grep -q 'code=206' +} + +# _talos_image_cache_reachable_from_tenant: succeed iff a Pod that faces the exact +# same tenant egress restrictions and network path as a worker's CDI importer can +# reach the mirror Service ClusterIP with a byte-range (206) response. +# +# This is the load-bearing gate: a green result guarantees a real importer will +# reach the mirror. It runs a throwaway Pod in the tenant namespace, labelled +# cdi.kubevirt.io=importer (so the tenant egress default-deny plus the allow +# policy above apply to it just as they do to a real importer), and has it curl +# the ClusterIP Service for the seeded image with a Range header — exercising DNS, +# the egress policy, ClusterIP translation, and range support together, not merely +# that the server answers on localhost. +# +# cdi.kubevirt.io=importer is CDI's own importer-pod label. The probe and the +# egress policy's endpointSelector must both use it; if a future CDI renames it, +# they would still agree with each other but no longer with reality, and the probe +# would pass while real importers stayed blocked. It is correct as of the CDI +# version this platform ships (packages/system/kubevirt-cdi-operator). +_talos_image_cache_reachable_from_tenant() { + local pod rel img curl_cmd overrides out pod=$(kubectl -n kube-system get pod \ -l app.kubernetes.io/name=talos-image-cache \ --field-selector=status.phase=Running \ -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) || return 1 [ -n "$pod" ] || return 1 - code=$(kubectl -n kube-system exec "$pod" -c serve -- sh -ec ' - f=$(find /data -name "*.raw.xz" 2>/dev/null | head -n1) - [ -n "$f" ] || exit 1 - curl -s -o /dev/null -w "%{http_code}" -r 0-0 "http://127.0.0.1:8080/${f#/data/}" - ' 2>/dev/null) || return 1 - [ "$code" = "206" ] + # Discover the seeded file (its presence also confirms the seed finished) and + # probe that exact path, so the probe requests what a worker actually requests. + rel=$(kubectl -n kube-system exec "$pod" -c serve -- sh -ec \ + 'f=$(find /data -name "*.raw.xz" 2>/dev/null | head -n1); [ -n "$f" ] || exit 1; printf "%s" "${f#/data/}"' \ + 2>/dev/null) || return 1 + [ -n "$rel" ] || return 1 + # Reuse the mirror's own digest-pinned image for the probe Pod. It is the same + # image many CI hooks share, so by this late stage of the suite it is usually + # already cached on whatever node the probe lands on and no pull is needed; if + # the probe does land on a node without it, the pull is bounded by the Pod's + # --pod-running-timeout below and a timeout just falls back to the public factory. + img=$(kubectl -n kube-system get deploy talos-image-cache \ + -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null) || return 1 + [ -n "$img" ] || return 1 + + # --retry rides out the seconds Cilium needs to program the freshly-applied allow + # policy; a persistently unreachable ClusterIP still fails fast per attempt + # (connect-timeout) and returns a non-206, falling back to the public factory. + # The total budget is deliberately generous: this decision is cached once for the + # whole suite, so a single slow policy-programming window must not permanently + # disable the mirror. --max-time still bounds a hung attempt (a genuinely + # unreachable ClusterIP resolves well within it and falls back). + curl_cmd="curl -s -o /dev/null -w code=%{http_code} --retry 8 --retry-all-errors --retry-delay 5 --connect-timeout 5 --max-time 45 -r 0-0 ${TALOS_IMAGE_CACHE_SVC_URL}/${rel}" + overrides=$(_talos_image_cache_probe_overrides "$img" "$curl_cmd") + + kubectl -n "$TALOS_IMAGE_CACHE_PROBE_NS" delete pod "$TALOS_IMAGE_CACHE_PROBE_POD" \ + --ignore-not-found >/dev/null 2>&1 + # No --rm: with --rm the Pod is gone the instant it exits, and --attach can lose + # the tail of a fast-completing Pod's stream, leaving $out empty — a false negative + # that would fall back to the flaky public factory this mirror exists to avoid. + # Keep the Pod so its logs can be re-read if the attach stream was lost, then + # delete it explicitly below. + out=$(kubectl -n "$TALOS_IMAGE_CACHE_PROBE_NS" run "$TALOS_IMAGE_CACHE_PROBE_POD" \ + --restart=Never --attach --pod-running-timeout=90s \ + --image="$img" --labels='cdi.kubevirt.io=importer' \ + --overrides="$overrides" 2>&1) || true + # --attach returns only after the container exits, so its logs are ready now; + # fall back to them when the attach stream carried no result marker. + if ! printf '%s' "$out" | grep -q 'code='; then + out=$(kubectl -n "$TALOS_IMAGE_CACHE_PROBE_NS" logs "$TALOS_IMAGE_CACHE_PROBE_POD" 2>/dev/null || true) + fi + # Fire-and-forget: the result is already captured in $out, so nothing depends on + # the Pod actually being gone. The pre-run delete above must NOT do this — there, + # `kubectl run` would race a still-terminating Pod of the same name. + kubectl -n "$TALOS_IMAGE_CACHE_PROBE_NS" delete pod "$TALOS_IMAGE_CACHE_PROBE_POD" \ + --ignore-not-found --wait=false >/dev/null 2>&1 + _talos_image_cache_probe_succeeded "$out" } # resolve_talos_image_factory_url: print the imageFactoryURL to use, or an empty @@ -44,21 +153,21 @@ resolve_talos_image_factory_url() { local url="" if kubectl -n kube-system get deploy talos-image-cache >/dev/null 2>&1; then # Available only after the seed initContainer has fully fetched the image, so - # this is the signal that the mirror is warm and safe to point tenants at. + # this is the signal that the mirror is warm. if kubectl -n kube-system rollout status deploy/talos-image-cache --timeout=12m >/dev/null 2>&1; then - url="$TALOS_IMAGE_CACHE_SVC_URL" - # Available only proves the seed finished and the port is open. Verify the - # server actually answers a byte-range request with 206 (not a range-less - # 200): range support is what lets CDI skip the slow copy-to-scratch path, - # and a future swap to a range-incapable server would otherwise regress it - # silently. Probe from inside the serve pod so this does not depend on the - # sandbox reaching a ClusterIP. A non-206 still imports correctly (just - # without the fast path), so warn loudly rather than discard the mirror and - # reintroduce the public factory's egress flakiness. - if _talos_image_cache_serves_ranges; then - echo "talos-image-cache mirror ready (byte-range verified) — tenant workers import from ${url}" >&2 + # Available proves only that the seed finished and the server answers on + # localhost — NOT that a worker's importer, penned in by the tenant egress + # default-deny, can reach the ClusterIP. Install the allow policy (Cilium is + # up by now, unlike at seed-deploy time) and verify the whole importer path + # end to end. Point tenants at the mirror only when that check passes; + # otherwise fall back to the public factory so the mirror can only help, + # never point workers at an unreachable ClusterIP and make CI worse. + _apply_talos_image_cache_egress_policy || true + if _talos_image_cache_reachable_from_tenant; then + url="$TALOS_IMAGE_CACHE_SVC_URL" + echo "talos-image-cache mirror reachable from tenant namespace (byte-range 206 verified) — tenant workers import from ${url}" >&2 else - echo "WARNING: talos-image-cache is Available but did not answer a range request with 206 — tenant workers still import from ${url}, but CDI will fall back to a full copy" >&2 + echo "WARNING: talos-image-cache is Available but a tenant-scoped probe could not reach its ClusterIP with a 206 — tenant workers fall back to public factory.talos.dev" >&2 fi else echo "WARNING: talos-image-cache mirror not Available in time — tenant workers fall back to public factory.talos.dev" >&2 From 1a0680b77821a3abf44c46cd69d7d83247309ccf Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 9 Jul 2026 13:33:45 +0300 Subject: [PATCH 2/2] test(e2e): allow tenant CDI importers to reach the image cache Tenant namespaces run under a default-deny Cilium egress installed by the tenant chart, which lets a worker's CDI importer reach the outside world and kube-dns but nothing in kube-system. The in-sandbox image mirror is a kube-system Service, so without an explicit allow the importer's connect to its ClusterIP is dropped and the worker disk import never starts. Ship a tightly-scoped CiliumClusterwideNetworkPolicy in the mirror manifest that lets importer Pods in tenant-test egress to the mirror Pod in kube-system. Cilium unions allow rules, so this only adds a hole; scoping it to tenant-test (already under default-deny egress) means it never flips another namespace's posture, and the mirror Pod needs no ingress rule because it is selected by no ingress policy. The policy is kept out of the pre-install apply because that runs before Cilium's CRDs exist; the resolver in hack/e2e-apps/talos-image-cache.sh applies it at point-of-use once Cilium is up. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- hack/e2e-install-cozystack.bats | 16 ++-- hack/e2e-talos-image-cache.yaml | 59 ++++++++++++- hack/talos-image-cache_test.bats | 144 +++++++++++++++++++++++++++++++ 3 files changed, 212 insertions(+), 7 deletions(-) create mode 100644 hack/talos-image-cache_test.bats diff --git a/hack/e2e-install-cozystack.bats b/hack/e2e-install-cozystack.bats index 627c20c14..9e488536a 100644 --- a/hack/e2e-install-cozystack.bats +++ b/hack/e2e-install-cozystack.bats @@ -42,11 +42,12 @@ # (see cozystack/cozystack#3231). This Deployment fetches the image ONCE with a # hard curl retry loop and serves it locally; tenant CRs then point # spec.talos.imageFactoryURL at its Service. Deployed here (before the long - # install) so the seed download overlaps the install churn — readiness is gated - # at point-of-use in hack/e2e-apps/run-kubernetes.sh, which falls back to the - # public factory if the mirror never becomes Available, so this can only help. - # Best-effort: never fail the suite on the band-aid. Remove once tenant workers - # no longer bulk-pull the OS image from the public internet in CI. + # install) so the seed download overlaps the install churn — reachability is + # gated at point-of-use in hack/e2e-apps/run-kubernetes.sh, which falls back to + # the public factory unless a tenant-scoped probe confirms a worker can actually + # reach the ClusterIP, so this can only help. Best-effort: never fail the suite + # on the band-aid. Remove once tenant workers no longer bulk-pull the OS image + # from the public internet in CI. local sid ver sid=$(yq '.talos.schematicID' packages/apps/kubernetes/values.yaml 2>/dev/null) ver=$(yq '.talos.version' packages/apps/kubernetes/values.yaml 2>/dev/null) @@ -54,7 +55,12 @@ echo "WARNING: could not read talos.schematicID/version from values.yaml — skipping mirror (fallback to public factory)" return 0 fi + # The CiliumClusterwideNetworkPolicy in the manifest is intentionally NOT applied + # here: Cilium's CRDs do not exist until Cozystack is installed (below), so + # applying it now would error. It is applied at point-of-use by + # hack/e2e-apps/talos-image-cache.sh once Cilium is up. sed -e "s|__SCHEMATIC_ID__|${sid}|g" -e "s|__TALOS_VERSION__|${ver}|g" hack/e2e-talos-image-cache.yaml \ + | yq 'select(.kind != "CiliumClusterwideNetworkPolicy")' \ | kubectl apply -f - || echo "WARNING: failed to apply talos-image-cache (fallback to public factory)" if kubectl -n kube-system get deploy talos-image-cache >/dev/null 2>&1; then echo "talos-image-cache Deployment created (seeding ${sid}/${ver} in background)" diff --git a/hack/e2e-talos-image-cache.yaml b/hack/e2e-talos-image-cache.yaml index 0435a6902..8df38b582 100644 --- a/hack/e2e-talos-image-cache.yaml +++ b/hack/e2e-talos-image-cache.yaml @@ -16,8 +16,15 @@ # per-worker imports), then serves it locally over plain HTTP with range support. # Tenant `Kubernetes` CRs point `spec.talos.imageFactoryURL` at this Service # (see hack/e2e-apps/run-kubernetes.sh) so every worker imports from the warm -# local copy instead of the public internet. If this Deployment never becomes -# Available the e2e harness falls back to the public factory (status quo), so the +# local copy instead of the public internet. Tenant namespaces run under a +# default-deny Cilium egress installed by the tenant chart, whose allow rules cover +# same-namespace endpoints, the tenant tree and the `world` entity (which is why the +# public factory works) but NOT an arbitrary kube-system Service, so a worker's CDI +# importer would resolve this Service yet have its TCP connect to the ClusterIP +# silently dropped. The CiliumClusterwideNetworkPolicy at the end of this file punches a +# single tightly-scoped hole for that path. The e2e harness only points tenants +# here after a tenant-scoped probe confirms the importer path actually reaches the +# ClusterIP with a 206; otherwise it falls back to the public factory, so the # mirror can only help, never make CI worse. # # The __SCHEMATIC_ID__ / __TALOS_VERSION__ placeholders are substituted at apply @@ -221,3 +228,51 @@ data: with http.server.ThreadingHTTPServer(("0.0.0.0", PORT), RangeHandler) as httpd: print("serving %s on :%d" % (ROOT, PORT), flush=True) httpd.serve_forever() +--- +# CI-only: allow the tenant CDI importer Pods to reach the mirror. +# +# Tenant namespaces run under a default-deny Cilium egress installed by the tenant +# chart (allow-internal-communication, allow-external-communication and the +# `-egress` CiliumClusterwideNetworkPolicy). Their union allows egress to +# same-namespace endpoints, the tenant tree and the `world` entity (so the public +# factory works), but nothing permits egress to an arbitrary kube-system Service — +# so a worker's CDI importer resolves talos-image-cache.kube-system.svc yet its TCP +# connect to the Service ClusterIP is silently dropped (dial ... i/o timeout), the +# disk import never starts, and the tenant node never joins. Cilium unions allow +# rules across policies, so this adds one tightly-scoped allow: importer Pods in +# tenant-test may egress to the mirror Pod in kube-system. +# +# Scoped to tenant-test (which the tenant chart already puts under default-deny +# egress) so it only ever ADDS an allow — it never flips another namespace from +# default-allow to default-deny. The mirror Pod is selected by no ingress policy +# (default-allow ingress), so no ingress rule is needed; replies return on the +# established conntrack entry. +# +# Applied at point-of-use by hack/e2e-apps/talos-image-cache.sh, not with the rest +# of this manifest in hack/e2e-install-cozystack.bats: that runs before Cozystack +# (and thus before Cilium's CRDs) is installed, so this kind does not yet exist +# when the manifest is first applied. +apiVersion: cilium.io/v2 +kind: CiliumClusterwideNetworkPolicy +metadata: + name: e2e-talos-image-cache-importer-egress + labels: + app.kubernetes.io/name: talos-image-cache +spec: + endpointSelector: + matchLabels: + "k8s:cdi.kubevirt.io": importer + "k8s:io.kubernetes.pod.namespace": tenant-test + # No toPorts: the allow is already scoped to a single kube-system pod identity + # (CI-only, torn down with the sandbox), so an L4 port restriction adds no + # meaningful blast-radius reduction — only risk. With kube-proxy-less Cilium the + # importer's connect to the Service ClusterIP is socket-LB translated to the + # backend pod port BEFORE egress policy is evaluated, so a port literal here + # would have to track the container port (8080), not the Service port (80); a + # wrong guess silently drops the traffic and reintroduces the exact public-factory + # fallback this policy exists to prevent. + egress: + - toEndpoints: + - matchLabels: + "k8s:io.kubernetes.pod.namespace": kube-system + "k8s:app.kubernetes.io/name": talos-image-cache diff --git a/hack/talos-image-cache_test.bats b/hack/talos-image-cache_test.bats new file mode 100644 index 000000000..9a3225d35 --- /dev/null +++ b/hack/talos-image-cache_test.bats @@ -0,0 +1,144 @@ +#!/usr/bin/env bats +# ----------------------------------------------------------------------------- +# Unit tests for the in-sandbox Talos image cache manifest split and the +# importer-reachability contract it relies on. +# +# hack/e2e-talos-image-cache.yaml bundles four documents but they are applied in +# two phases: hack/e2e-install-cozystack.bats applies everything EXCEPT the +# CiliumClusterwideNetworkPolicy (its CRD does not exist before Cozystack is +# installed), and hack/e2e-apps/talos-image-cache.sh applies ONLY that policy +# later, once Cilium is up. If a future document is added to the manifest and +# silently dropped from the pre-Cilium apply, or the Cilium document leaks into +# it (and errors on the missing CRD), the mirror breaks and e2e falls back to +# the flaky public factory. These tests pin that split. +# +# They also pin the load-bearing reachability invariant: the throwaway probe pod +# is a faithful proxy for a real CDI importer only if it carries the exact label +# and namespace the egress policy selects. If either side drifts, the probe can +# pass while real importers stay blocked (a false positive that makes CI worse). +# +# Finally they cover the two pure fragments of the decision itself, sourced from +# hack/e2e-apps/talos-image-cache.sh (sourcing it has no cluster side effects — it +# only sets parameter defaults and defines functions): the strict 206 gate that +# decides whether tenants are pointed at the mirror at all, and the --overrides JSON +# builder whose silent breakage would drop every probe to the public factory. The +# kubectl orchestration around them needs a live cluster and is covered by the e2e +# run, matching how the other hack/ helpers are tested. +# +# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its +# own line; there is no bats `run` or `$status`, and setup()/teardown() are not +# honored. Each test runs under `set -eu -x`; assertions are direct shell tests +# that exit non-zero on failure. mikefarah yq prints `---` between matched +# documents, so document streams are compared with those separators stripped. +# +# Run with: hack/cozytest.sh hack/talos-image-cache_test.bats +# ----------------------------------------------------------------------------- + +@test "manifest documents partition into pre-Cilium apply plus the Cilium policy" { + manifest=hack/e2e-talos-image-cache.yaml + total=$(yq '.kind' "$manifest" | grep -vc '^---$') + excluded=$(yq 'select(.kind != "CiliumClusterwideNetworkPolicy") | .kind' "$manifest" | grep -vc '^---$') + selected=$(yq 'select(.kind == "CiliumClusterwideNetworkPolicy") | .kind' "$manifest" | grep -vc '^---$') + [ "$total" -eq 4 ] + [ "$excluded" -eq 3 ] + [ "$selected" -eq 1 ] + [ $((excluded + selected)) -eq "$total" ] +} + +@test "pre-Cilium apply keeps Service, Deployment, ConfigMap and drops the Cilium policy" { + manifest=hack/e2e-talos-image-cache.yaml + kinds=$(yq 'select(.kind != "CiliumClusterwideNetworkPolicy") | .kind' "$manifest" | grep -v '^---$') + for want in Service Deployment ConfigMap; do + printf '%s\n' "$kinds" | grep -qx "$want" || { echo "pre-Cilium apply is missing $want" >&2; exit 1; } + done + if printf '%s\n' "$kinds" | grep -qx CiliumClusterwideNetworkPolicy; then + echo "CiliumClusterwideNetworkPolicy leaked into the pre-Cilium apply" >&2 + exit 1 + fi +} + +@test "point-of-use apply selects exactly the Cilium policy" { + manifest=hack/e2e-talos-image-cache.yaml + kinds=$(yq 'select(.kind == "CiliumClusterwideNetworkPolicy") | .kind' "$manifest" | grep -v '^---$') + [ "$kinds" = "CiliumClusterwideNetworkPolicy" ] +} + +@test "egress policy selects the importer label and namespace the probe pod uses" { + manifest=hack/e2e-talos-image-cache.yaml + helper=hack/e2e-apps/talos-image-cache.sh + label=$(yq 'select(.kind == "CiliumClusterwideNetworkPolicy") | .spec.endpointSelector.matchLabels["k8s:cdi.kubevirt.io"]' "$manifest") + ns=$(yq 'select(.kind == "CiliumClusterwideNetworkPolicy") | .spec.endpointSelector.matchLabels["k8s:io.kubernetes.pod.namespace"]' "$manifest") + [ "$label" = "importer" ] + [ "$ns" = "tenant-test" ] + # The probe pod must carry the same label the policy selects, else it faces a + # different egress than a real importer and the guarantee breaks. + grep -q "cdi.kubevirt.io=importer" "$helper" || { echo "probe pod label drifted from the egress selector" >&2; exit 1; } + grep -q "TALOS_IMAGE_CACHE_PROBE_NS:-tenant-test" "$helper" || { echo "probe namespace drifted from the egress selector" >&2; exit 1; } +} + +@test "egress rule targets the mirror pod's own identity" { + manifest=hack/e2e-talos-image-cache.yaml + target=$(yq 'select(.kind == "CiliumClusterwideNetworkPolicy") | .spec.egress[0].toEndpoints[0].matchLabels["k8s:app.kubernetes.io/name"]' "$manifest") + target_ns=$(yq 'select(.kind == "CiliumClusterwideNetworkPolicy") | .spec.egress[0].toEndpoints[0].matchLabels["k8s:io.kubernetes.pod.namespace"]' "$manifest") + mirror=$(yq 'select(.kind == "Deployment") | .spec.template.metadata.labels["app.kubernetes.io/name"]' "$manifest") + mirror_ns=$(yq 'select(.kind == "Deployment") | .metadata.namespace' "$manifest") + [ "$target" = "talos-image-cache" ] + [ "$target_ns" = "kube-system" ] + # The allow's destination must equal the mirror Deployment's own pod label and + # namespace, otherwise the hole is punched toward nothing: moving the mirror + # would leave these tests green while importers could no longer reach it. + [ "$target" = "$mirror" ] + [ "$target_ns" = "$mirror_ns" ] +} + +@test "strict 206 gate selects the mirror on a byte-range success" { + . hack/e2e-apps/talos-image-cache.sh + if ! _talos_image_cache_probe_succeeded "code=206"; then + echo "expected a 206 probe result to select the mirror" >&2 + exit 1 + fi +} + +@test "strict 206 gate finds the marker inside surrounding probe output" { + . hack/e2e-apps/talos-image-cache.sh + out=$(printf 'pod/talos-image-cache-probe created\ncode=206\npod "talos-image-cache-probe" deleted\n') + if ! _talos_image_cache_probe_succeeded "$out"; then + echo "expected 206 to be found in multi-line attach output" >&2 + exit 1 + fi +} + +@test "strict 206 gate falls back on range-less, unreachable, empty and unrelated output" { + . hack/e2e-apps/talos-image-cache.sh + # A plain 200 means the mirror answered without range support; 000 means curl + # never connected. Both fall back to the public factory, as does output that + # carried no result marker at all (an attach stream lost with no logs to recover). + for bad in "code=200" "code=000" "code=404" "" "error: pods not found"; do + if _talos_image_cache_probe_succeeded "$bad"; then + echo "expected non-206 output to fall back to the public factory: [$bad]" >&2 + exit 1 + fi + done +} + +@test "probe overrides builder emits valid JSON with the image, command and restricted PSA" { + . hack/e2e-apps/talos-image-cache.sh + img=example.invalid/img:tag + cmd='curl -s -o /dev/null -w code=%{http_code} -r 0-0 http://talos-image-cache.kube-system.svc/x.raw.xz' + json=$(_talos_image_cache_probe_overrides "$img" "$cmd") + # Invalid JSON would make every probe fail and silently drop CI back to the + # public factory, so parse it rather than pattern-match it. + printf '%s' "$json" | yq -p=json -o=json '.' >/dev/null + [ "$(printf '%s' "$json" | yq -p=json '.spec.containers[0].name')" = "c" ] + [ "$(printf '%s' "$json" | yq -p=json '.spec.containers[0].image')" = "$img" ] + [ "$(printf '%s' "$json" | yq -p=json '.spec.containers[0].command[2]')" = "$cmd" ] + # Every field the tenant namespace's restricted Pod Security Admission requires: + # drop any one of them and the probe Pod is rejected at admission, the probe + # fails, and CI silently falls back to the public factory. + [ "$(printf '%s' "$json" | yq -p=json '.spec.securityContext.runAsNonRoot')" = "true" ] + [ "$(printf '%s' "$json" | yq -p=json '.spec.securityContext.runAsUser')" = "1000" ] + [ "$(printf '%s' "$json" | yq -p=json '.spec.securityContext.seccompProfile.type')" = "RuntimeDefault" ] + [ "$(printf '%s' "$json" | yq -p=json '.spec.containers[0].securityContext.allowPrivilegeEscalation')" = "false" ] + [ "$(printf '%s' "$json" | yq -p=json '.spec.containers[0].securityContext.capabilities.drop | length')" = "1" ] + [ "$(printf '%s' "$json" | yq -p=json '.spec.containers[0].securityContext.capabilities.drop[0]')" = "ALL" ] +}