# shellcheck shell=bash # Shared helper: choose the Talos Image Factory base URL for tenant Kubernetes # e2e CRs. # # Prefer the in-sandbox caching mirror (hack/e2e-talos-image-cache.yaml, deployed # by hack/e2e-install-cozystack.bats) when a worker's CDI importer can actually # reach it, otherwise fall back to the chart default (the public factory.talos.dev) # by emitting nothing. The mirror rides out the public factory's flaky/range-less # egress that otherwise stalls worker DataVolume imports past the 12-minute # node-join deadline — see cozystack/cozystack#3231. Falling back to the default # means the mirror can only help, never make CI worse. # # Reachability is not assumed. Tenant namespaces run under a default-deny Cilium # egress installed by the tenant chart: a worker's importer may reach the `world` # entity (hence the public factory) and the tenant's own tree, but not an # arbitrary kube-system Service. Without an explicit allow, the importer resolves # talos-image-cache.kube-system.svc yet its TCP connect to the ClusterIP is # silently dropped and the disk import never starts. This helper installs that # allow (the CiliumClusterwideNetworkPolicy shipped in the mirror manifest) and # then verifies the path end to end from a Pod that faces the exact same egress # rules as a real importer, so the mirror is used only when it genuinely works. # # Root cause + fix are from cozystack/cozystack#3254 (@lexfrei), ported here to # the Chainsaw layout (hack/e2e-chainsaw/_lib/). TALOS_IMAGE_CACHE_SVC_URL="${TALOS_IMAGE_CACHE_SVC_URL:-http://talos-image-cache.kube-system.svc}" _TALOS_IMAGE_FACTORY_DECISION_FILE="${_TALOS_IMAGE_FACTORY_DECISION_FILE:-/tmp/e2e-talos-image-factory-url}" TALOS_IMAGE_CACHE_MANIFEST="${TALOS_IMAGE_CACHE_MANIFEST:-hack/e2e-talos-image-cache.yaml}" # Tenant namespace the kubernetes-* CRs are created in (run-kubernetes.sh and the # oidc suites all target tenant-test); the reachability probe runs here so it # inherits that namespace's default-deny egress, exactly as a worker importer does. TALOS_IMAGE_CACHE_PROBE_NS="${TALOS_IMAGE_CACHE_PROBE_NS:-tenant-test}" TALOS_IMAGE_CACHE_PROBE_POD="${TALOS_IMAGE_CACHE_PROBE_POD:-talos-image-cache-probe}" # _apply_talos_image_cache_egress_policy: install the CiliumClusterwideNetworkPolicy # that lets tenant CDI importer Pods egress to the mirror. It lives in the mirror # manifest but is applied here, not in hack/e2e-install-cozystack.bats, because that # manifest is applied before Cozystack (and thus before Cilium's CRDs) exists. # Idempotent; best-effort — a failure just leaves the probe below to fall back. _apply_talos_image_cache_egress_policy() { command -v yq >/dev/null 2>&1 || return 1 yq 'select(.kind == "CiliumClusterwideNetworkPolicy")' "$TALOS_IMAGE_CACHE_MANIFEST" 2>/dev/null \ | kubectl apply -f - >/dev/null 2>&1 } # _talos_image_cache_probe_overrides: emit the kubectl --overrides JSON for the # probe Pod. $1 is the container image, $2 the shell command it runs. The single # container replaces the one `kubectl run` generates (merge patch semantics), and # the security context satisfies the tenant namespace's restricted PSA. # # Pure string builder, kept separate so it can be unit-tested: an edit that # unbalances a quote here would emit invalid JSON, every probe would fail, and CI # would silently fall back to the public factory instead of erroring. _talos_image_cache_probe_overrides() { printf '{"spec":{"securityContext":{"runAsNonRoot":true,"runAsUser":1000,"seccompProfile":{"type":"RuntimeDefault"}},"containers":[{"name":"c","image":"%s","command":["sh","-ec","%s"],"securityContext":{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]}}}]}}' \ "$1" "$2" } # _talos_image_cache_probe_succeeded: succeed iff the probe output ($1) carries the # strict 206 marker. # # Gate strictly on 206. The old localhost check kept a reachable-but-range-less # mirror and only warned; here any non-206 — unreachable (code=000) or a plain 200 — # falls back to the public factory instead. serve.py always answers a valid range # with 206, so the 200 case is unreachable in practice; the strict gate keeps the # "use the mirror only when it is fully working" rule simple. # # Pure predicate, kept separate so it can be unit-tested: this single comparison is # what decides whether tenant workers are pointed at the mirror at all. _talos_image_cache_probe_succeeded() { printf '%s' "$1" | grep -q 'code=206' } # _talos_image_cache_reachable_from_tenant: succeed iff a Pod that faces the exact # same tenant egress restrictions and network path as a worker's CDI importer can # reach the mirror Service ClusterIP with a byte-range (206) response. # # This is the load-bearing gate: a green result guarantees a real importer will # reach the mirror. It runs a throwaway Pod in the tenant namespace, labelled # cdi.kubevirt.io=importer (so the tenant egress default-deny plus the allow # policy above apply to it just as they do to a real importer), and has it curl # the ClusterIP Service for the seeded image with a Range header — exercising DNS, # the egress policy, ClusterIP translation, and range support together, not merely # that the server answers on localhost. # # cdi.kubevirt.io=importer is CDI's own importer-pod label. The probe and the # egress policy's endpointSelector must both use it; if a future CDI renames it, # they would still agree with each other but no longer with reality, and the probe # would pass while real importers stayed blocked. It is correct as of the CDI # version this platform ships (packages/system/kubevirt-cdi-operator). _talos_image_cache_reachable_from_tenant() { local pod rel img curl_cmd overrides out pod=$(kubectl -n kube-system get pod \ -l app.kubernetes.io/name=talos-image-cache \ --field-selector=status.phase=Running \ -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) || return 1 [ -n "$pod" ] || return 1 # Discover the seeded file (its presence also confirms the seed finished) and # probe that exact path, so the probe requests what a worker actually requests. rel=$(kubectl -n kube-system exec "$pod" -c serve -- sh -ec \ 'f=$(find /data -name "*.raw.xz" 2>/dev/null | head -n1); [ -n "$f" ] || exit 1; printf "%s" "${f#/data/}"' \ 2>/dev/null) || return 1 [ -n "$rel" ] || return 1 # Reuse the mirror's own digest-pinned image for the probe Pod. It is the same # image many CI hooks share, so by this late stage of the suite it is usually # already cached on whatever node the probe lands on and no pull is needed; if # the probe does land on a node without it, the pull is bounded by the Pod's # --pod-running-timeout below and a timeout just falls back to the public factory. img=$(kubectl -n kube-system get deploy talos-image-cache \ -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null) || return 1 [ -n "$img" ] || return 1 # --retry rides out the seconds Cilium needs to program the freshly-applied allow # policy; a persistently unreachable ClusterIP still fails fast per attempt # (connect-timeout) and returns a non-206, falling back to the public factory. # The total budget is deliberately generous: this decision is cached once for the # whole suite, so a single slow policy-programming window must not permanently # disable the mirror. --max-time still bounds a hung attempt (a genuinely # unreachable ClusterIP resolves well within it and falls back). curl_cmd="curl -s -o /dev/null -w code=%{http_code} --retry 8 --retry-all-errors --retry-delay 5 --connect-timeout 5 --max-time 45 -r 0-0 ${TALOS_IMAGE_CACHE_SVC_URL}/${rel}" overrides=$(_talos_image_cache_probe_overrides "$img" "$curl_cmd") kubectl -n "$TALOS_IMAGE_CACHE_PROBE_NS" delete pod "$TALOS_IMAGE_CACHE_PROBE_POD" \ --ignore-not-found >/dev/null 2>&1 # No --rm: with --rm the Pod is gone the instant it exits, and --attach can lose # the tail of a fast-completing Pod's stream, leaving $out empty — a false negative # that would fall back to the flaky public factory this mirror exists to avoid. # Keep the Pod so its logs can be re-read if the attach stream was lost, then # delete it explicitly below. out=$(kubectl -n "$TALOS_IMAGE_CACHE_PROBE_NS" run "$TALOS_IMAGE_CACHE_PROBE_POD" \ --restart=Never --attach --pod-running-timeout=90s \ --image="$img" --labels='cdi.kubevirt.io=importer' \ --overrides="$overrides" 2>&1) || true # --attach returns only after the container exits, so its logs are ready now; # fall back to them when the attach stream carried no result marker. if ! printf '%s' "$out" | grep -q 'code='; then out=$(kubectl -n "$TALOS_IMAGE_CACHE_PROBE_NS" logs "$TALOS_IMAGE_CACHE_PROBE_POD" 2>/dev/null || true) fi # Fire-and-forget: the result is already captured in $out, so nothing depends on # the Pod actually being gone. The pre-run delete above must NOT do this — there, # `kubectl run` would race a still-terminating Pod of the same name. kubectl -n "$TALOS_IMAGE_CACHE_PROBE_NS" delete pod "$TALOS_IMAGE_CACHE_PROBE_POD" \ --ignore-not-found --wait=false >/dev/null 2>&1 _talos_image_cache_probe_succeeded "$out" } # resolve_talos_image_factory_url: print the imageFactoryURL to use, or an empty # string to signal "use the chart default". The decision is resolved once (with a # bounded wait for the mirror to finish seeding) and cached in a /tmp file that # persists across bats files (they all exec in the same sandbox container), so # only the first tenant Kubernetes test pays the readiness wait. resolve_talos_image_factory_url() { if [ -f "$_TALOS_IMAGE_FACTORY_DECISION_FILE" ]; then cat "$_TALOS_IMAGE_FACTORY_DECISION_FILE" return 0 fi local url="" if kubectl -n kube-system get deploy talos-image-cache >/dev/null 2>&1; then # Available only after the seed initContainer has fully fetched the image, so # this is the signal that the mirror is warm. if kubectl -n kube-system rollout status deploy/talos-image-cache --timeout=12m >/dev/null 2>&1; then # Available proves only that the seed finished and the server answers on # localhost — NOT that a worker's importer, penned in by the tenant egress # default-deny, can reach the ClusterIP. Install the allow policy (Cilium is # up by now, unlike at seed-deploy time) and verify the whole importer path # end to end. Point tenants at the mirror only when that check passes; # otherwise fall back to the public factory so the mirror can only help, # never point workers at an unreachable ClusterIP and make CI worse. _apply_talos_image_cache_egress_policy || true if _talos_image_cache_reachable_from_tenant; then url="$TALOS_IMAGE_CACHE_SVC_URL" echo "talos-image-cache mirror reachable from tenant namespace (byte-range 206 verified) — tenant workers import from ${url}" >&2 else echo "WARNING: talos-image-cache is Available but a tenant-scoped probe could not reach its ClusterIP with a 206 — tenant workers fall back to public factory.talos.dev" >&2 fi else echo "WARNING: talos-image-cache mirror not Available in time — tenant workers fall back to public factory.talos.dev" >&2 fi else echo "talos-image-cache mirror not deployed — tenant workers use public factory.talos.dev" >&2 fi printf '%s' "$url" > "$_TALOS_IMAGE_FACTORY_DECISION_FILE" printf '%s' "$url" } # talos_image_cache_diagnose: on-demand diagnostic for the node-join failure path # in run-kubernetes.sh (Chainsaw-branch addition, not in #3254). Re-probes the # mirror from the tenant namespace — the same path a real importer faces — and # dumps the cacher + egress-policy state, so an operator can tell "the egress # allow / ClusterIP path is broken" apart from "the upstream factory was slow". # All output to stderr; never fails the caller. talos_image_cache_diagnose() { if ! kubectl -n kube-system get deploy talos-image-cache >/dev/null 2>&1; then echo "talos-image-cache not deployed — tenant workers used the public factory; nothing to diagnose" >&2 return 0 fi { if _talos_image_cache_reachable_from_tenant; then echo "talos-image-cache reachable from ${TALOS_IMAGE_CACHE_PROBE_NS} at diagnose time (byte-range 206)" else echo "talos-image-cache NOT reachable from ${TALOS_IMAGE_CACHE_PROBE_NS} at diagnose time (tenant egress allow missing/unprogrammed, or cache down)" fi echo "--- cache deploy/pod/svc/endpointslice ---" kubectl -n kube-system get deploy,pod,svc,endpointslice \ -l app.kubernetes.io/name=talos-image-cache -o wide 2>&1 || true echo "--- importer egress allow policy ---" kubectl get ciliumclusterwidenetworkpolicy e2e-talos-image-cache-importer-egress -o wide 2>&1 || true echo "--- serve container log (tail) ---" kubectl -n kube-system logs -l app.kubernetes.io/name=talos-image-cache \ -c serve --tail=50 --prefix 2>&1 || true } >&2 } # talos_image_factory_spec_block: emit a two-line YAML block # talos: # imageFactoryURL: # indented for insertion directly under a tenant Kubernetes CR `spec:`, or # nothing when the chart default should apply. Ends with a trailing newline when # non-empty so it can prefix the next `spec` key in a heredoc. talos_image_factory_spec_block() { local url url=$(resolve_talos_image_factory_url) if [ -n "$url" ]; then printf ' talos:\n imageFactoryURL: %s\n' "$url" fi }