cozystack/hack/e2e-talos-image-cache.yaml

# In-sandbox caching mirror for the Talos Image Factory worker OS disk image.
#
# Why this exists: tenant Kubernetes worker VMs boot from a Talos raw disk image
# that CDI streams over HTTP from the PUBLIC Talos Image Factory
# (https://factory.talos.dev/image/<schematicID>/<version>/openstack-amd64.raw.xz).
# That endpoint does not support byte-range requests (so CDI falls back to a full
# sequential copy-to-scratch) and its egress is intermittently flaky from the CI
# runner — a mid-transfer HTTP/2 stream reset or a stalled download makes the
# worker DataVolume import never finish, the worker VM never boots, no tenant
# node joins within the chart's 12-minute node-join deadline, and the
# `kubernetes-*` e2e tests fail (and, even when they pass, take 18-22 minutes
# each dominated by these downloads). See cozystack/cozystack#3231.
#
# This Deployment fetches the image ONCE at e2e-install time with an aggressive
# curl retry loop (decoupling the one flaky external fetch from the many
# per-worker imports), then serves it locally over plain HTTP with range support.
# Tenant `Kubernetes` CRs point `spec.talos.imageFactoryURL` at this Service
# (see hack/e2e-chainsaw/_lib/run-kubernetes.sh) so every worker imports from the
# warm local copy instead of the public internet. Tenant namespaces run under a
# default-deny Cilium egress installed by the tenant chart, whose allow rules cover
# same-namespace endpoints, the tenant tree and the `world` entity (which is why the
# public factory works) but NOT an arbitrary kube-system Service, so a worker's CDI
# importer would resolve this Service yet have its TCP connect to the ClusterIP
# silently dropped. The CiliumClusterwideNetworkPolicy at the end of this file punches a
# single tightly-scoped hole for that path. The e2e harness only points tenants
# here after a tenant-scoped probe confirms the importer path actually reaches the
# ClusterIP with a 206; otherwise it falls back to the public factory, so the
# mirror can only help, never make CI worse.
#
# The __SCHEMATIC_ID__ / __TALOS_VERSION__ placeholders are substituted at apply
# time by hack/e2e-install-cozystack.bats from packages/apps/kubernetes/values.yaml.
#
# CI band-aid, not a product artifact. The durable fix is the overridable
# `talos.imageFactoryURL` chart value this manifest exercises; a self-hosted
# Talos Image Factory or an operator's own mirror is the production path.
apiVersion: v1
kind: Service
metadata:
  name: talos-image-cache
  namespace: kube-system
  labels:
    app.kubernetes.io/name: talos-image-cache
spec:
  selector:
    app.kubernetes.io/name: talos-image-cache
  ports:
    - name: http
      port: 80
      targetPort: http
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: talos-image-cache
  namespace: kube-system
  labels:
    app.kubernetes.io/name: talos-image-cache
spec:
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: talos-image-cache
  strategy:
    type: Recreate
  template:
    metadata:
      labels:
        app.kubernetes.io/name: talos-image-cache
    spec:
      # The seed fetch and static serving use ordinary pod networking — the same
      # egress path the CDI importer pods already use to reach factory.talos.dev.
      initContainers:
        # Fetch the compressed raw disk image ONCE, retrying hard against the
        # flaky upstream. The Deployment only becomes Available after this
        # succeeds, so "Available" is the e2e harness's signal that the mirror is
        # warm and safe to point tenants at.
        - name: seed
          # Reuse the in-tree, digest-pinned image already shipped by the
          # installer and pre-pulled for the cilium-leak-healer watchdog, so this
          # pod does not add a fresh external image pull of its own.
          image: docker.io/alpine/k8s:1.36.2@sha256:44ef4942e171939b9c665a4a84beb80e2dcdb9a24330d4651cfdfd2e9deecc47
          command: [/bin/sh, -ec]
          args:
            - |
              dest="/data/image/__SCHEMATIC_ID__/__TALOS_VERSION__/openstack-amd64.raw.xz"
              url="https://factory.talos.dev/image/__SCHEMATIC_ID__/__TALOS_VERSION__/openstack-amd64.raw.xz"
              mkdir -p "$(dirname "$dest")"
              echo "seeding $dest from $url"
              # --retry-all-errors + a long retry budget rides out the mid-transfer
              # stream resets that break the CDI importer's single attempt.
              # --speed-time/--speed-limit additionally abort a transfer that
              # stalls below 1 KiB/s for 60s — a frozen (not reset) download is
              # not an "error", so --retry-all-errors alone would let it hang out
              # the whole rollout budget; the abort is then retried like any other
              # failure.
              curl --fail --location --show-error \
                   --retry 30 --retry-all-errors --retry-delay 15 \
                   --connect-timeout 30 --speed-time 60 --speed-limit 1024 \
                   --output "$dest" \
                   "$url"
              ls -la "$dest"
          volumeMounts:
            - name: data
              mountPath: /data
      containers:
        - name: serve
          image: docker.io/alpine/k8s:1.36.2@sha256:44ef4942e171939b9c665a4a84beb80e2dcdb9a24330d4651cfdfd2e9deecc47
          # Serve the seeded image over HTTP with byte-range (206) support so CDI
          # can use its fast range import instead of the copy-to-scratch fallback
          # the range-less public factory forces. The pinned image ships neither a
          # `busybox httpd` applet nor a standalone httpd/nginx, and stock
          # `python3 -m http.server` ignores Range headers (answers 200, not 206),
          # so a tiny range-capable server (serve.py ConfigMap below) runs on the
          # image's bundled python3 — no extra package pull that would reintroduce
          # a flaky external dependency.
          command: [python3, /serve/serve.py, /data, "8080"]
          ports:
            - name: http
              containerPort: 8080
          readinessProbe:
            # httpGet (not tcpSocket) so readiness reflects the server actually
            # answering HTTP, not merely the port being open.
            httpGet:
              path: /
              port: http
            initialDelaySeconds: 2
            periodSeconds: 5
          resources:
            requests:
              cpu: 20m
              memory: 32Mi
            limits:
              memory: 128Mi
          volumeMounts:
            - name: data
              mountPath: /data
            - name: serve-script
              mountPath: /serve
      volumes:
        - name: data
          emptyDir: {}
        - name: serve-script
          configMap:
            name: talos-image-cache-serve
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: talos-image-cache-serve
  namespace: kube-system
  labels:
    app.kubernetes.io/name: talos-image-cache
data:
  # Minimal static file server with HTTP byte-range support. Stock
  # `python3 -m http.server` answers Range requests with 200 + the full body;
  # this adds the single-range 206 path (Content-Range + Accept-Ranges) CDI
  # needs to skip its slow copy-to-scratch import.
  serve.py: |
    import http.server
    import os
    import sys
    import time

    ROOT = sys.argv[1] if len(sys.argv) > 1 else "/data"
    PORT = int(sys.argv[2]) if len(sys.argv) > 2 else 8080


    class RangeHandler(http.server.SimpleHTTPRequestHandler):
        # HTTP/1.1 keep-alive lets CDI pull many ranges over one connection.
        protocol_version = "HTTP/1.1"

        def __init__(self, *args, **kwargs):
            super().__init__(*args, directory=ROOT, **kwargs)

        def send_head(self):
            path = self.translate_path(self.path)
            if os.path.isdir(path):
                return super().send_head()
            try:
                f = open(path, "rb")
            except OSError:
                self.send_error(404)
                return None
            size = os.fstat(f.fileno()).st_size
            ctype = self.guess_type(path)
            rng = self.headers.get("Range", "")
            if not rng.startswith("bytes="):
                self._remaining = size
                self._transfer_range = "full"
                self.send_response(200)
                self.send_header("Content-Type", ctype)
                self.send_header("Content-Length", str(size))
                self.send_header("Accept-Ranges", "bytes")
                self.end_headers()
                return f
            start_s, _, end_s = rng[6:].split(",")[0].strip().partition("-")
            start = int(start_s) if start_s else max(0, size - int(end_s))
            end = int(end_s) if start_s and end_s else size - 1
            if start >= size or start > end:
                self.send_response(416)
                self.send_header("Content-Range", "bytes */%d" % size)
                self.send_header("Content-Length", "0")
                self.end_headers()
                f.close()
                return None
            end = min(end, size - 1)
            f.seek(start)
            self._remaining = end - start + 1
            self._transfer_range = "%d-%d/%d" % (start, end, size)
            self.send_response(206)
            self.send_header("Content-Type", ctype)
            self.send_header("Content-Range", "bytes %d-%d/%d" % (start, end, size))
            self.send_header("Content-Length", str(self._remaining))
            self.send_header("Accept-Ranges", "bytes")
            self.end_headers()
            return f

        def copyfile(self, source, outputfile):
            remaining = getattr(self, "_remaining", None)
            if remaining is None:
                super().copyfile(source, outputfile)
                return
            self._remaining = None
            requested = remaining
            sent = 0
            started = time.monotonic()
            try:
                while remaining > 0:
                    chunk = source.read(min(65536, remaining))
                    if not chunk:
                        break
                    outputfile.write(chunk)
                    sent += len(chunk)
                    remaining -= len(chunk)
            finally:
                status = "complete" if sent == requested else "interrupted"
                self.log_message(
                    "transfer-finished status=%s path=%s range=%s requested=%d sent=%d duration=%.3fs",
                    status,
                    self.path,
                    getattr(self, "_transfer_range", "unknown"),
                    requested,
                    sent,
                    time.monotonic() - started,
                )


    with http.server.ThreadingHTTPServer(("0.0.0.0", PORT), RangeHandler) as httpd:
        print("serving %s on :%d" % (ROOT, PORT), flush=True)
        httpd.serve_forever()
---
# CI-only: allow the tenant CDI importer Pods to reach the mirror.
#
# Tenant namespaces run under a default-deny Cilium egress installed by the tenant
# chart (allow-internal-communication, allow-external-communication and the
# `<tenant>-egress` CiliumClusterwideNetworkPolicy). Their union allows egress to
# same-namespace endpoints, the tenant tree and the `world` entity (so the public
# factory works), but nothing permits egress to an arbitrary kube-system Service —
# so a worker's CDI importer resolves talos-image-cache.kube-system.svc yet its TCP
# connect to the Service ClusterIP is silently dropped (dial ... i/o timeout), the
# disk import never starts, and the tenant node never joins. Cilium unions allow
# rules across policies, so this adds one tightly-scoped allow: importer Pods in
# tenant-test may egress to the mirror Pod in kube-system.
#
# Scoped to tenant-test (which the tenant chart already puts under default-deny
# egress) so it only ever ADDS an allow — it never flips another namespace from
# default-allow to default-deny. The mirror Pod is selected by no ingress policy
# (default-allow ingress), so no ingress rule is needed; replies return on the
# established conntrack entry.
#
# Applied at point-of-use by hack/e2e-chainsaw/_lib/talos-image-cache.sh, not with
# the rest of this manifest in hack/e2e-install-cozystack.bats: that runs before
# Cozystack (and thus before Cilium's CRDs) is installed, so this kind does not yet
# exist when the manifest is first applied.
apiVersion: cilium.io/v2
kind: CiliumClusterwideNetworkPolicy
metadata:
  name: e2e-talos-image-cache-importer-egress
  labels:
    app.kubernetes.io/name: talos-image-cache
spec:
  endpointSelector:
    matchLabels:
      "k8s:cdi.kubevirt.io": importer
      "k8s:io.kubernetes.pod.namespace": tenant-test
  # No toPorts: the allow is already scoped to a single kube-system pod identity
  # (CI-only, torn down with the sandbox), so an L4 port restriction adds no
  # meaningful blast-radius reduction — only risk. With kube-proxy-less Cilium the
  # importer's connect to the Service ClusterIP is socket-LB translated to the
  # backend pod port BEFORE egress policy is evaluated, so a port literal here
  # would have to track the container port (8080), not the Service port (80); a
  # wrong guess silently drops the traffic and reintroduces the exact public-factory
  # fallback this policy exists to prevent.
  egress:
    - toEndpoints:
        - matchLabels:
            "k8s:io.kubernetes.pod.namespace": kube-system
            "k8s:app.kubernetes.io/name": talos-image-cache