From f3223c33d3d37b351ea5bbb086fcdb0e8cd5673e Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Fri, 9 Oct 2026 16:03:02 +0300 Subject: [PATCH] ci(e2e): run nightly and release-tag e2e on the container lane Nightly and the rc gate still booted the management nodes as QEMU guests, the substrate the merge-gating lanes left behind. On it the tenant Kubernetes suites have missed their node-join deadline on most nights since 2026-10-02, each failure spends the whole deadline, and the job reaches its timeout before the alphabetically later suites run. Over the same days those suites passed on every PR run that selected them, where the nodes are Talos containers and a tenant worker sits one virtualization level shallower. Both lanes now use the PR lanes' container substrate. Nightly's e2e no longer waits for the nocloud disk, which it does not boot; the disk is still built and published. A manual e2e-tag dispatch against a tag that predates the container lane fails early and names the ref to dispatch from instead. DRBD, the replicated StorageClass and the Cozystack Talos node image lose their last e2e coverage with this. The docs and comments that pointed at nightly and the rc gate as that coverage now say so. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin (cherry picked from commit da28ea7ee8b1bcb007c542690d9dfeef76994f26) Signed-off-by: Myasnikov Daniil --- .github/workflows/e2e-tag.yaml | 87 +++++++++++++++++------- .github/workflows/nightly.yaml | 70 +++++++++++++------ .github/workflows/pull-requests.yaml | 15 ++-- docs/agents/e2e-testing.md | 4 +- docs/release.md | 6 +- hack/e2e-chainsaw/_lib/run-kubernetes.sh | 9 +-- hack/platform-packages_test.bats | 35 +++++++--- hack/promote-gate-contract.bats | 19 ++++++ 8 files changed, 170 insertions(+), 75 deletions(-) diff --git a/.github/workflows/e2e-tag.yaml b/.github/workflows/e2e-tag.yaml index d70b62be5..9e5785b79 100644 --- a/.github/workflows/e2e-tag.yaml +++ b/.github/workflows/e2e-tag.yaml @@ -11,8 +11,10 @@ name: E2E Release Tag # This lane validates published tags only; draft and promote-time validation are # explicitly out of scope because release checks run at RC publication time. # Prerelease tags use the matching digest-pinned staging branch, while stable -# tags use their pinned merge tree directly. In both cases the nocloud disk -# comes from the published release identified by the same tag. +# tags use their pinned merge tree directly. The nodes run as Talos containers +# booted from the upstream image, so the release's nocloud disk is required to +# exist but is not what the suite runs on. It is still required because users +# install from it, and promote-rc.yaml checks it for an rc only. on: workflow_call: @@ -37,8 +39,8 @@ concurrency: group: e2e-tag-${{ inputs.tag }} cancel-in-progress: false -# Listing published releases, reading Git refs, downloading a release asset, and -# checking out the pinned tree require only repository contents read access. +# Listing published releases, reading Git refs and checking out the pinned tree +# require only repository contents read access. # Draft releases are not part of this lane's contract. The public GHCR images # referenced by that tree need no package token. # @@ -57,7 +59,6 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 outputs: - disk_id: ${{ steps.resolve.outputs.disk_id }} tree_ref: ${{ steps.resolve.outputs.tree_ref }} steps: - name: Validate tag and resolve release @@ -123,16 +124,15 @@ jobs: throw error; } - core.setOutput('disk_id', String(disk.id)); core.setOutput('tree_ref', treeRef); core.info(`Resolved ${tag}: tree '${treeRef}', disk asset ${disk.id}`); e2e: name: E2E ${{ inputs.tag }} (full suite) needs: resolve - # The sandbox boots three 8-vCPU / 24-GiB QEMU guests. This runner leaves - # enough host headroom for QEMU, networking, containerd, and the runner - # agent while the full platform installs across all three guests. + # The sandbox runs three 8-vCPU / 24-GiB Talos containers. This runner + # leaves enough host headroom for networking, containerd, and the runner + # agent while the full platform installs across all three nodes. runs-on: oracle-vm-32cpu-128gb-x86-64 timeout-minutes: 180 permissions: @@ -148,45 +148,74 @@ jobs: fetch-depth: 1 persist-credentials: false - - name: Download disk asset from release - env: - DISK_ID: ${{ needs.resolve.outputs.disk_id }} - GH_TOKEN: ${{ github.token }} - run: | - mkdir -p _out/assets - curl --fail-with-body --silent --show-error --location \ - -H "Authorization: Bearer ${GH_TOKEN}" \ - -H "Accept: application/octet-stream" \ - -o _out/assets/nocloud-amd64.raw.xz \ - "https://api.github.com/repos/${GITHUB_REPOSITORY}/releases/assets/${DISK_ID}" - - name: Set sandbox ID env: TAG: ${{ inputs.tag }} run: echo "SANDBOX_NAME=cozy-e2e-sandbox-$(echo "${GITHUB_REPOSITORY}:${GITHUB_WORKFLOW}:${TAG}" | sha256sum | cut -c1-10)" >> "$GITHUB_ENV" + # tags.yaml calls the e2e-tag.yaml from the tag's own commit, so this can + # only fire on a dispatch from a newer ref against an older tag. + - name: Require a container-lane tree + env: + TAG: ${{ inputs.tag }} + run: | + if [ ! -f hack/e2e-compose.yaml ]; then + echo "::error title=e2e-tag::${TAG} predates the container lane; dispatch e2e-tag.yaml with --ref ${TAG} to run it on the lane it shipped with" + exit 1 + fi + - name: Prepare workspace run: | rm -rf "/tmp/$SANDBOX_NAME" cp -r "${{ github.workspace }}" "/tmp/$SANDBOX_NAME" - # This is the only retried stage: Talos image unpacking, QEMU startup, and - # sandbox networking are pure runner infrastructure, not product logic. + # Talos cannot load kernel modules in container mode, so the nodes get + # exactly what this host has loaded. Why each module, and why failing + # here is better than an hour later: the same step in pull-requests.yaml. + - name: Prepare container-lane host + run: | + set -eu + for m in openvswitch zfs; do + grep -q "^${m} " /proc/modules && continue + if [ "${m}" = zfs ]; then + sudo apt-get update -qq + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq zfsutils-linux + fi + sudo modprobe "${m}" || echo "${m}: modprobe failed" + done + missing='' + for m in openvswitch zfs; do + grep -q "^${m} " /proc/modules || missing="${missing} ${m}" + done + if [ -n "${missing}" ]; then + echo "::error title=container-lane host::kernel module(s) could not be loaded:${missing}" + exit 1 + fi + + # This is the only retried stage: Talos container startup and sandbox + # networking are pure runner infrastructure, not product logic. - name: Prepare environment run: | cd "/tmp/$SANDBOX_NAME" attempt=0 - until make SANDBOX_NAME=$SANDBOX_NAME prepare-env; do + until make SANDBOX_NAME="$SANDBOX_NAME" prepare-env-container; do attempt=$((attempt + 1)) if [ $attempt -ge 3 ]; then echo "Attempt $attempt failed, exiting..." exit 1 fi echo "Attempt $attempt failed, retrying..." + make -C packages/core/testing SANDBOX_NAME="$SANDBOX_NAME" delete-cluster-container || true done echo "Prepare environment completed after $((attempt + 1)) attempts" - name: Install Cozystack into sandbox + env: + # No ARP VIP in container mode: .11 is a plain control-plane node. + COZY_APISERVER_ENDPOINT: https://192.168.123.11:6443 + # The nodes share the runner kernel and cannot load DRBD. + COZY_LINSTOR_DRBD_ENABLED: "false" + COZY_E2E_STORAGE_CLASS: local run: | cd "/tmp/$SANDBOX_NAME" if ! make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME install-cozystack; then @@ -216,6 +245,8 @@ jobs: # CHAINSAW_SUITES is the testing Makefile's full-suite mode. - name: Run E2E tests (full suite) id: e2e_tests + env: + COZY_E2E_STORAGE_CLASS: local run: | cd "/tmp/$SANDBOX_NAME" if ! make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-chainsaw CHAINSAW_SUITES=""; then @@ -358,13 +389,17 @@ jobs: Resume from inside: `breakpoint resume`. Otherwise the breakpoint exits 10 minutes after the last SSH session disconnects. + # The compose project owns the nodes and the zpools outlive it, so both + # halves go, in this order. - name: Tear down sandbox if: always() - run: make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME delete || true + run: | + make -C packages/core/testing SANDBOX_NAME="$SANDBOX_NAME" delete-cluster-container || true + make -C packages/core/testing SANDBOX_NAME="$SANDBOX_NAME" delete || true - name: Remove workspace if: always() - run: rm -rf "/tmp/$SANDBOX_NAME" + run: rm -rf "/tmp/${SANDBOX_NAME:?}" - name: Summarize outcome if: always() diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml index 315485eaf..c126a123f 100644 --- a/.github/workflows/nightly.yaml +++ b/.github/workflows/nightly.yaml @@ -10,7 +10,8 @@ name: Nightly # cozy-installer chart are re-published on GHCR so the whole install closure is # self-contained there. The only thing genuinely built is the Talos nocloud disk # (assembled from upstream profiles, independent of the registry), published as a -# GHCR OCI artifact so a nightly is installable on real hardware, not just e2e. +# GHCR OCI artifact so a nightly is installable on real hardware. The nightly +# e2e does not boot it. # # No GitHub release and no api/apps/v1alpha1 Go-module tag are created — a # nightly is an installable artifact set on GHCR, nothing more. retention.yaml @@ -216,13 +217,18 @@ jobs: oras tag "${DST_REGISTRY}/cozystack-nocloud:${VERSION}" nightly echo "Published cozystack-nocloud:${VERSION} (+ :nightly) to GHCR." - # ── Full e2e against the published GHCR closure (chart + images + disk). + # ── Full e2e against the published GHCR closure (chart + images), on the + # same Talos-container substrate as the PR lanes. The nodes boot the + # upstream Talos image, so the nocloud disk built above is published but + # not exercised here, and e2e does not wait for it. e2e: name: E2E (full suite) - needs: [mirror, build-disk] - # Same sandbox as the other e2e lanes: three nodes at 8 vCPU each commit - # 24 vCPU for the whole run, so the runner class has to be strictly larger - # to leave the host its own cycles. pull-requests.yaml has the full note. + needs: [mirror] + # Same sandbox as the other e2e lanes: three nodes at 8 vCPU / 24 GiB each + # (hack/e2e-compose.yaml) commit 24 vCPU for the whole run, so the runner + # class has to be strictly larger, leaving the host its own cycles for + # containerd and the runner agent. hack/e2e-container-up.sh refuses a + # runner that is not. runs-on: oracle-vm-32cpu-128gb-x86-64 timeout-minutes: 180 permissions: @@ -235,17 +241,15 @@ jobs: ref: ${{ needs.mirror.outputs.sha }} fetch-depth: 1 persist-credentials: false - - name: Set up toolchain (flux, oras) + - name: Set up toolchain (flux) run: | command -v flux >/dev/null || curl -fsSL https://fluxcd.io/install.sh | sudo bash - command -v oras >/dev/null || { curl -fsSL https://github.com/oras-project/oras/releases/download/v1.2.0/oras_1.2.0_linux_amd64.tar.gz | sudo tar -xz -C /usr/local/bin oras; } - name: Login to GHCR env: GHCR_USER: ${{ github.actor }} GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | echo "$GHCR_TOKEN" | docker login "${DST_REGISTRY%%/*}" -u "$GHCR_USER" --password-stdin - echo "$GHCR_TOKEN" | oras login "${DST_REGISTRY%%/*}" -u "$GHCR_USER" --password-stdin # Overlay the GHCR-rewritten tree (operator image, .e2e.image and every # component now point at GHCR), then pin the installer chart to the GHCR @@ -264,17 +268,32 @@ jobs: yq -i ".cozystackOperator.platformSourceUrl = \"oci://${DST_REGISTRY}/cozystack-packages\"" packages/core/installer/values.yaml yq -i ".cozystackOperator.platformSourceRef = \"digest=${PACKAGES_DIGEST}\"" packages/core/installer/values.yaml - - name: Fetch nocloud disk from GHCR - env: - VERSION: ${{ needs.mirror.outputs.version }} - run: | - set -eu - mkdir -p _out/assets - oras pull "${DST_REGISTRY}/cozystack-nocloud:${VERSION}" --output _out/assets - - name: Set sandbox ID run: echo "SANDBOX_NAME=cozy-e2e-sandbox-$(echo "${GITHUB_REPOSITORY}:${GITHUB_WORKFLOW}:${GITHUB_REF}" | sha256sum | cut -c1-10)" >> "$GITHUB_ENV" + # Talos cannot load kernel modules in container mode, so the nodes get + # exactly what this host has loaded. Why each module, and why failing + # here is better than an hour later: the same step in pull-requests.yaml. + - name: Prepare container-lane host + run: | + set -eu + for m in openvswitch zfs; do + grep -q "^${m} " /proc/modules && continue + if [ "${m}" = zfs ]; then + sudo apt-get update -qq + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq zfsutils-linux + fi + sudo modprobe "${m}" || echo "${m}: modprobe failed" + done + missing='' + for m in openvswitch zfs; do + grep -q "^${m} " /proc/modules || missing="${missing} ${m}" + done + if [ -n "${missing}" ]; then + echo "::error title=container-lane host::kernel module(s) could not be loaded:${missing}" + exit 1 + fi + - name: Prepare workspace run: | rm -rf "/tmp/$SANDBOX_NAME" @@ -284,13 +303,20 @@ jobs: run: | cd "/tmp/$SANDBOX_NAME" attempt=0 - until make SANDBOX_NAME=$SANDBOX_NAME prepare-env; do + until make SANDBOX_NAME="$SANDBOX_NAME" prepare-env-container; do attempt=$((attempt + 1)) if [ $attempt -ge 3 ]; then echo "Attempt $attempt failed, exiting..."; exit 1; fi echo "Attempt $attempt failed, retrying..." + make -C packages/core/testing SANDBOX_NAME="$SANDBOX_NAME" delete-cluster-container || true done - name: Install Cozystack into sandbox + env: + # No ARP VIP in container mode: .11 is a plain control-plane node. + COZY_APISERVER_ENDPOINT: https://192.168.123.11:6443 + # The nodes share the runner kernel and cannot load DRBD. + COZY_LINSTOR_DRBD_ENABLED: "false" + COZY_E2E_STORAGE_CLASS: local run: | cd "/tmp/$SANDBOX_NAME" if ! make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME install-cozystack; then @@ -316,6 +342,8 @@ jobs: # Always the FULL app suite — a nightly has no diff to scope against, so # Test Impact Analysis does not apply (empty CHAINSAW_SUITES = whole suite). - name: Run E2E tests (full suite) + env: + COZY_E2E_STORAGE_CLASS: local run: | cd "/tmp/$SANDBOX_NAME" if ! make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-chainsaw CHAINSAW_SUITES=""; then @@ -360,9 +388,13 @@ jobs: path: /tmp/${{ env.SANDBOX_NAME }}/_out/cozyreport.tgz if-no-files-found: warn + # The compose project owns the nodes and the zpools outlive it, so both + # halves go, in this order. - name: Tear down sandbox if: always() - run: make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME delete || true + run: | + make -C packages/core/testing SANDBOX_NAME="$SANDBOX_NAME" delete-cluster-container || true + make -C packages/core/testing SANDBOX_NAME="$SANDBOX_NAME" delete || true - name: Remove workspace if: always() diff --git a/.github/workflows/pull-requests.yaml b/.github/workflows/pull-requests.yaml index 81e6ad119..613641b6e 100644 --- a/.github/workflows/pull-requests.yaml +++ b/.github/workflows/pull-requests.yaml @@ -611,7 +611,7 @@ jobs: # nocloud disk asset and export its id; the container lane boots no disk, # so nothing in this workflow reads that id any more and resolving it only # added a way to fail a release PR over an asset it does not use. The - # asset is still resolved where it is still consumed, in e2e-tag.yaml. + # asset's presence is still checked in e2e-tag.yaml and promote-rc.yaml. - name: Verify draft release exists (release PR) if: contains(github.event.pull_request.labels.*.name, 'release') uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 @@ -631,15 +631,10 @@ jobs: } - # In-tree e2e, on Talos nodes running as CONTAINERS. The QEMU substrate this - # lane used to boot is off the per-PR path only: it is still wired into - # e2e-tag.yaml, which runs `make prepare-env` -> `make -C packages/core/testing - # prepare-cluster`. That target and hack/e2e-prepare-cluster.bats are - # therefore live code, not leftovers, and deleting them takes every - # release-candidate run with them. They carry the coverage containers cannot: - # DRBD, and with it the `replicated` StorageClass, its Immediate binding mode - # and the tenant StorageClass propagation that only applies to - # remotely-accessible classes. + # In-tree e2e, on Talos nodes running as CONTAINERS, like every other e2e + # lane. What containers cannot carry has no lane at all: DRBD, and with it + # the `replicated` StorageClass, its Immediate binding mode and the tenant + # StorageClass propagation that only applies to remotely-accessible classes. e2e: name: "E2E Tests" # hack/e2e-compose.yaml gives each of the three nodes `cpus: 8` and diff --git a/docs/agents/e2e-testing.md b/docs/agents/e2e-testing.md index 22eaf64eb..ef63fd480 100644 --- a/docs/agents/e2e-testing.md +++ b/docs/agents/e2e-testing.md @@ -132,7 +132,7 @@ What that buys, measured in CI on commits where both substrates ran: the two ten Three properties of container mode drive the whole design, and each one fails quietly rather than loudly if ignored. `machine.kernel.modules` is a **silent no-op** — `kernel_module_spec.go` returns early on `ModeContainer` with no error and no event — so the host must load `openvswitch` and `zfs` before the nodes start, and both the host script and the BATS file assert it rather than trusting it. Docker gives a privileged container its **own tmpfs `/dev`, seeded once at creation**, so `/dev/kvm` works but ZFS zvols created later never appear and linstor-csi fails `ControllerPublishVolume`; the compose file binds the host devtmpfs instead, which is also why `talosctl cluster create docker` can never carry LINSTOR. And container mode has **no ARP VIP**, so `192.168.123.11` stands in for the QEMU lane's `192.168.123.10` and `hack/e2e-platform-packages.sh` takes the endpoint from `COZY_APISERVER_ENDPOINT`, defaulting to the unchanged QEMU value. -In CI this is the `e2e` job ("E2E Tests") in `.github/workflows/pull-requests.yaml`, the lane that gates a merge. The QEMU substrate is not wired into it any more, and the PR path no longer builds or downloads a nocloud disk at all. `make -C packages/core/testing prepare-cluster` and `hack/e2e-prepare-cluster.bats` stay in the tree because `e2e-tag.yaml` still runs the suite on QEMU: the release-candidate run is where DRBD, the `replicated` StorageClass and the Cozystack Talos node image keep being exercised, since none of the three can run on containers. +In CI this is every lane that runs the suite: the `e2e` job ("E2E Tests") in `.github/workflows/pull-requests.yaml`, which gates a merge, and the `e2e` jobs in `nightly.yaml` and `e2e-tag.yaml`. No workflow boots the QEMU substrate or downloads a nocloud disk any more. `make -C packages/core/testing prepare-cluster` and `hack/e2e-prepare-cluster.bats` stay in the tree because `hack/sandbox-cidr-disjoint.bats` still reads the cluster's pod subnets from that file, but nothing in CI runs them, so DRBD, the `replicated` StorageClass, the Cozystack Talos node image and CDI's stock 600M worker memory ceiling have no e2e coverage until a lane with a kernel per node exists (cozystack/cozystack#4642). Root entry points are `make prepare-env-container` (host-side compose plus the in-sandbox BATS file) and `make -C packages/core/testing delete-cluster-container`, which is also the retry cleanup: a half-started compose project the next attempt adopts is not a fresh try. @@ -140,7 +140,7 @@ The nodes run as **siblings on the runner's docker daemon**, not nested inside t What the lane gives up is **DRBD, and with it more than the `replicated` StorageClass**. DRBD's resource registry is global per kernel, so three containers sharing one kernel cannot represent three DRBD nodes — a second `new-resource` for the same name returns success while changing nothing, which is worse than an error because LINSTOR then records a replica it does not have. It applies to kind equally; it is a property of one kernel, not of Talos. The linstor chart always adds a `drbd-logger` sidecar to the satellites, which exits at once without DRBD and keeps every satellite unready, so no PVC binds at all. The chart has no value that drops it, so `hack/e2e-post-install-prep.sh` applies a `LinstorSatelliteConfiguration` of its own on this lane, `e2e-no-drbd`, that deletes the sidecar from the satellite DaemonSet, and reads the live DaemonSets back once the nodes are Online. The DRBD-only initContainers are already removed by the chart's Talos satellite configuration. Because linstor-csi couples RWX to DRBD, tenant worker disks become RWO and **live migration is unavailable on this lane**, which nothing in the tree tests today either way. Everything else survives: kube-ovn runs the real OVS dataplane, and LINSTOR keeps real ZFS pools and the production `local` StorageClass. -Two further things move off the per-PR path with the substrate, and both keep their release-candidate coverage rather than losing it. `replicated` is `volumeBindingMode: Immediate` while `local` is `WaitForFirstConsumer`, so the Immediate binding path is not exercised per PR; and tenant StorageClass propagation only applies to classes that are not `allowRemoteVolumeAccess: "false"`, which on this lane is none of them. The management nodes also boot upstream `ghcr.io/siderolabs/talos` rather than the Cozystack image, so the extensions that image carries — `drbd`, `zfs`, `openvswitch` — and the LVM filter are validated by the release-candidate run rather than on every pull request. Tenant worker nodes still import a Talos disk, so that half of the Talos path is unaffected. +Two further things go with the substrate, and no lane covers them. `replicated` is `volumeBindingMode: Immediate` while `local` is `WaitForFirstConsumer`, so the Immediate binding path is not exercised; and tenant StorageClass propagation only applies to classes that are not `allowRemoteVolumeAccess: "false"`, which on this lane is none of them. The management nodes also boot upstream `ghcr.io/siderolabs/talos` rather than the Cozystack image, so the extensions that image carries — `drbd`, `zfs`, `openvswitch` — and the LVM filter are not validated by e2e. Tenant worker nodes still import a Talos disk, so that half of the Talos path is unaffected. The local-only mode has two mandatory install-time adaptations. CDI otherwise ranks `ReadWriteMany`/`Block` first for any LINSTOR StorageClass, so post-install prep waits for `StorageProfile/local` and pins it to `ReadWriteOnce`/`Block` before any tenant DataVolume is created; without that override linstor-csi rejects the first worker disk because RWX requires DRBD. Root VictoriaLogs is the only install workload that explicitly defaults to `replicated` instead of inheriting the default class, so the E2E installer adds its `logsStorages` override to `tenant-root/cozystack-values` before enabling monitoring; patching the generated monitoring HelmRelease afterward is unsafe because it can create an immutable replicated PVC first. diff --git a/docs/release.md b/docs/release.md index a946b4471..f24a6ff56 100644 --- a/docs/release.md +++ b/docs/release.md @@ -256,11 +256,11 @@ So a commit on a supported `release-X.Y` line ships when a maintainer promotes t A nightly is an **installable copy of `main` on GHCR — not a rebuild and not a release**. [`build-main.yaml`](../.github/workflows/build-main.yaml) already builds every push to `main` into the CI registry (OCIR); the nightly promotes that build to the public release registry (GHCR) and proves it installs. -[`nightly.yaml`](../.github/workflows/nightly.yaml) runs daily (gated by the `NIGHTLY_ENABLED` repo variable) in three stages: +[`nightly.yaml`](../.github/workflows/nightly.yaml) runs daily (gated by the `NIGHTLY_ENABLED` repo variable) as three jobs; build-disk and e2e both start once mirror has finished: 1. **mirror** — resolve the exact commit `cozystack-packages:main` was built from, then [`hack/nightly-mirror.sh`](../hack/nightly-mirror.sh) copies every cozystack-owned component image OCIR→GHCR **by digest** (bit-for-bit, no rebuild) and re-publishes the rewritten `cozystack-packages` artifact and the `cozy-installer` chart to GHCR, tagged `0.0.0-nightly...` plus a floating `nightly`. -2. **build-disk** — assemble the Talos `nocloud` disk from the upstream siderolabs imager (the profile references only `ghcr.io/siderolabs/*`, so this needs no cozystack image and no rebuild) and publish it as a GHCR OCI artifact (`cozystack-nocloud`) so a nightly is installable on real hardware, not just in e2e. -3. **e2e** — stage the published GHCR closure (rewritten tree + the `cozy-installer` chart pinned to the GHCR packages artifact + the disk) and run the **full** app suite. A nightly has no diff, so Test Impact Analysis does not apply. +2. **build-disk** — assemble the Talos `nocloud` disk from the upstream siderolabs imager (the profile references only `ghcr.io/siderolabs/*`, so this needs no cozystack image and no rebuild) and publish it as a GHCR OCI artifact (`cozystack-nocloud`) so a nightly is installable on real hardware. The nightly e2e does not boot it. +3. **e2e** — stage the published GHCR closure (rewritten tree + the `cozy-installer` chart pinned to the GHCR packages artifact) and run the **full** app suite on Talos containers, which do not boot the disk. A nightly has no diff, so Test Impact Analysis does not apply. No GitHub release and no `api/apps/v1alpha1/*` Go-module tag are created for a nightly. [`retention.yaml`](../.github/workflows/retention.yaml) prunes old GHCR nightly versions (keeps the newest 14 per package; the floating `nightly`, release tags and untagged versions are never touched). diff --git a/hack/e2e-chainsaw/_lib/run-kubernetes.sh b/hack/e2e-chainsaw/_lib/run-kubernetes.sh index 2ce638fca..69921623b 100644 --- a/hack/e2e-chainsaw/_lib/run-kubernetes.sh +++ b/hack/e2e-chainsaw/_lib/run-kubernetes.sh @@ -3,10 +3,11 @@ . hack/e2e-chainsaw/_lib/remediation-guard.sh . hack/e2e-chainsaw/_lib/talos-image-cache.sh -# The QEMU lane exercises replicated DRBD storage. The container lane cannot -# represent three DRBD nodes on one shared kernel, so its otherwise-identical -# suites use the production local class instead. Keep the accepted values -# narrow because this result is interpolated into Kubernetes YAML below. +# The QEMU sandbox, which no CI lane boots, exercises replicated DRBD storage. +# The container lane cannot represent three DRBD nodes on one shared kernel, so +# its otherwise-identical suites use the production local class instead. Keep +# the accepted values narrow because this result is interpolated into +# Kubernetes YAML below. cozy_e2e_storage_class() { case "${COZY_E2E_STORAGE_CLASS:-replicated}" in local | replicated) printf '%s\n' "${COZY_E2E_STORAGE_CLASS:-replicated}" ;; diff --git a/hack/platform-packages_test.bats b/hack/platform-packages_test.bats index 1d47d1ead..8d2c6bf90 100644 --- a/hack/platform-packages_test.bats +++ b/hack/platform-packages_test.bats @@ -38,19 +38,9 @@ fi } -@test "the gating workflow forwards the container-lane settings into the sandbox" { - endpoint_value=$(yq '.jobs.e2e.steps[] | select(.name == "Install Cozystack into sandbox") | .env.COZY_APISERVER_ENDPOINT' .github/workflows/pull-requests.yaml) - drbd_value=$(yq '.jobs.e2e.steps[] | select(.name == "Install Cozystack into sandbox") | .env.COZY_LINSTOR_DRBD_ENABLED' .github/workflows/pull-requests.yaml) +@test "every e2e lane forwards the container-lane settings into the sandbox" { make_command=$(make -n -C packages/core/testing SANDBOX_NAME=test COZY_APISERVER_ENDPOINT=https://192.168.123.11:6443 COZY_LINSTOR_DRBD_ENABLED=false install-cozystack) - if [ "$endpoint_value" != "https://192.168.123.11:6443" ]; then - echo "container workflow does not set the node API endpoint: $endpoint_value" >&2 - return 1 - fi - if [ "$drbd_value" != "false" ]; then - echo "container workflow does not set COZY_LINSTOR_DRBD_ENABLED=false" >&2 - return 1 - fi if ! printf '%s\n' "$make_command" | grep -Fq -- '-e COZY_APISERVER_ENDPOINT="https://192.168.123.11:6443"'; then echo "testing Makefile does not forward COZY_APISERVER_ENDPOINT" >&2 return 1 @@ -59,4 +49,27 @@ echo "testing Makefile does not forward COZY_LINSTOR_DRBD_ENABLED" >&2 return 1 fi + + # Install and run must agree on the class: a mismatch fails deep inside a suite. + for wf in .github/workflows/pull-requests.yaml .github/workflows/nightly.yaml .github/workflows/e2e-tag.yaml; do + install='.jobs.e2e.steps[] | select(.name == "Install Cozystack into sandbox")' + tests='.jobs.e2e.steps[] | select(.name | test("^Run E2E tests"))' + endpoint_value=$(yq "$install | .env.COZY_APISERVER_ENDPOINT" "$wf") + if [ "$endpoint_value" != "https://192.168.123.11:6443" ]; then + echo "$wf: install does not set the node API endpoint: $endpoint_value" >&2 + return 1 + fi + if [ "$(yq "$install | .env.COZY_LINSTOR_DRBD_ENABLED" "$wf")" != "false" ]; then + echo "$wf: install does not set COZY_LINSTOR_DRBD_ENABLED=false" >&2 + return 1 + fi + if [ "$(yq "$install | .env.COZY_E2E_STORAGE_CLASS" "$wf")" != "local" ]; then + echo "$wf: install does not set COZY_E2E_STORAGE_CLASS=local" >&2 + return 1 + fi + if [ "$(yq "$tests | .env.COZY_E2E_STORAGE_CLASS" "$wf")" != "local" ]; then + echo "$wf: the suite run does not set COZY_E2E_STORAGE_CLASS=local" >&2 + return 1 + fi + done } diff --git a/hack/promote-gate-contract.bats b/hack/promote-gate-contract.bats index c7e8b4745..d5c5d9837 100644 --- a/hack/promote-gate-contract.bats +++ b/hack/promote-gate-contract.bats @@ -140,6 +140,25 @@ step_line() { grep -qF 'CHAINSAW_VERSION' "$REPO_ROOT/packages/core/testing/images/e2e-sandbox/Dockerfile" } +@test "e2e-tag refuses a tree that predates the container lane and names the ref to dispatch from" { + # tags.yaml calls the e2e-tag.yaml from the tag's own commit, so the guard can + # only fire on a dispatch from a newer ref against an older tag, v1.6.3 and + # earlier on this line. + step="$(yq -r '.jobs.e2e.steps[] | select(.name == "Require a container-lane tree") | .run' "$E2E_TAG")" + [ -n "$step" ] + tmp="$(mktemp -d)" + mkdir -p "$tmp/old" "$tmp/new/hack" + touch "$tmp/new/hack/e2e-compose.yaml" + printf '%s\n' "$step" >"$tmp/step.sh" + (cd "$tmp/new" && TAG=v1.6.4 bash -e "$tmp/step.sh") + rc=0 + (cd "$tmp/old" && TAG=v1.6.3 bash -e "$tmp/step.sh") >"$tmp/out" 2>&1 || rc=$? + [ "$rc" -ne 0 ] + grep -qF '::error title=e2e-tag::v1.6.3 predates the container lane' "$tmp/out" + grep -qF -- '--ref v1.6.3' "$tmp/out" + rm -rf "$tmp" +} + # ── the candidate-aware promotion pipeline this base must carry ────────────── @test "the base carries every file promote-rc.yaml's preflight demands" {