name: Nightly # Publish an installable nightly of `main` WITHOUT rebuilding it, then run the # full e2e suite against the published bits. # # A nightly is a renamed copy of what `build-main` already built: every # component image is content-addressed, so `nightly-mirror.sh` copies each one # from the CI build registry (OCIR) to the public release registry (GHCR) BY # DIGEST — bit-for-bit, no rebuild. The cozystack-packages artifact and the # cozy-installer chart are re-published on GHCR so the whole install closure is # self-contained there. The only thing genuinely built is the Talos nocloud disk # (assembled from upstream profiles, independent of the registry), published as a # GHCR OCI artifact so a nightly is installable on real hardware. The nightly # e2e does not boot it. # # No GitHub release and no api/apps/v1alpha1 Go-module tag are created — a # nightly is an installable artifact set on GHCR, nothing more. retention.yaml # prunes the GHCR nightly tags. # # Install a nightly: # helm upgrade --install cozystack \ # oci://ghcr.io/cozystack/cozystack/cozy-installer --version nightly # (the floating `nightly` tag tracks the latest; each build is also published # under an immutable 0.0.0-nightly... tag for pinning) on: schedule: - cron: '0 2 * * *' # daily 02:00 UTC, after build-main has settled workflow_dispatch: {} env: SRC_REGISTRY: iad.ocir.io/idyksih5sir9/cozystack DST_REGISTRY: ghcr.io/cozystack/cozystack concurrency: group: nightly cancel-in-progress: false jobs: # ── Mirror the latest main build OCIR -> GHCR by digest, re-publish the # packages artifact + installer chart on GHCR. No rebuild. mirror: name: Mirror images to GHCR if: vars.NIGHTLY_ENABLED == 'true' || github.event_name == 'workflow_dispatch' runs-on: oracle-vm-4cpu-16gb-x86-64 timeout-minutes: 60 permissions: contents: read packages: write outputs: version: ${{ steps.resolve.outputs.version }} sha: ${{ steps.resolve.outputs.sha }} packages_ref: ${{ steps.resolve.outputs.packages_ref }} packages_digest: ${{ steps.publish.outputs.packages_digest }} steps: - name: Set up flux uses: fluxcd/flux2/action@04acaec6161ac4fb1a82ffafa88901c03271d34f # v2.8.6 with: version: "2.8.6" - name: Set up toolchain (skopeo, yq, helm) env: YQ_VERSION: "4.53.3" YQ_SHA256: "fa52a4e758c63d38299163fbdd1edfb4c4963247918bf9c1c5d31d84789eded4" run: | command -v skopeo >/dev/null || { sudo apt-get update -qq && sudo apt-get install -y -qq skopeo; } if ! yq --version 2>/dev/null | grep -q "mikefarah.* version v${YQ_VERSION}\$"; then yq_bin="$(mktemp)" curl -fsSL -o "$yq_bin" \ "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_amd64" printf '%s %s\n' "$YQ_SHA256" "$yq_bin" | sha256sum -c - sudo install -m 0755 "$yq_bin" /usr/local/bin/yq rm -f "$yq_bin" fi command -v helm >/dev/null || curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash - name: Login to registries env: OCIR_USER: ${{ secrets.OCIR_USER }} OCIR_TOKEN: ${{ secrets.OCIR_TOKEN }} GHCR_USER: ${{ github.actor }} GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | # skopeo (mirror copies), flux + helm (artifact/chart push) all read # their own auth; log each into both registries it touches. Token via # stdin, never argv (a -p token leaks into the process list / logs). echo "$OCIR_TOKEN" | skopeo login "${SRC_REGISTRY%%/*}" -u "$OCIR_USER" --password-stdin echo "$GHCR_TOKEN" | skopeo login "${DST_REGISTRY%%/*}" -u "$GHCR_USER" --password-stdin echo "$OCIR_TOKEN" | docker login "${SRC_REGISTRY%%/*}" -u "$OCIR_USER" --password-stdin echo "$GHCR_TOKEN" | docker login "${DST_REGISTRY%%/*}" -u "$GHCR_USER" --password-stdin echo "$GHCR_TOKEN" | helm registry login "${DST_REGISTRY%%/*}" -u "$GHCR_USER" --password-stdin # The source of truth is the cozystack-packages:main artifact build-main # pushed. Resolve the exact commit it was built from (its revision # annotation) so the disk and the install tree are checked out from the # SAME commit the images were built at — never main's drifting HEAD. - name: Resolve source commit and version id: resolve run: | set -eu # Fetch the :main manifest ONCE and derive BOTH its digest and the # revision annotation from those exact bytes, then pull the baked tree # from that SAME digest below, so a concurrent build-main push of :main # cannot make us check out commit A while mirroring tree B. # # --raw is mandatory here: cozystack-packages is a Flux OCI artifact # (config mediaType application/vnd.cncf.flux.config.v1+json), not a # container image, so a plain `skopeo inspect` attempts an # image-specific operation and fatals on it. `manifest-digest` hashes # the manifest bytes, which is exactly what the registry addresses it # by — no image-specific parsing involved. manifest="${RUNNER_TEMP}/cozystack-packages-main.json" skopeo inspect --raw "docker://${SRC_REGISTRY}/cozystack-packages:main" > "$manifest" packages_ref="$(skopeo manifest-digest "$manifest")" rev="$(yq -r '.annotations["org.opencontainers.image.revision"] // ""' -p json "$manifest" 2>/dev/null || true)" rm -f "$manifest" # revision is ":"; take the sha. sha="${rev##*:}" # The sha flows into `actions/checkout` ref: — validate it is a bare # commit hash before trusting it, so a malformed/forged annotation # cannot inject a ref. echo "$sha" | grep -qE '^[0-9a-f]{40}$' \ || { echo "::error::source revision '$sha' is not a 40-hex commit sha"; exit 1; } # Make every build's tag immutable: the UTC date keeps it human- # readable, and run id + attempt make a same-day re-dispatch or re-run # publish a distinct tag instead of overwriting the day's nightly. version="0.0.0-nightly.$(date -u +%Y%m%d).${GITHUB_RUN_ID}.${GITHUB_RUN_ATTEMPT}" echo "sha=$sha" >> "$GITHUB_OUTPUT" echo "version=$version" >> "$GITHUB_OUTPUT" echo "packages_ref=$packages_ref" >> "$GITHUB_OUTPUT" echo "Resolved main build: sha=$sha version=$version packages_ref=$packages_ref" - name: Checkout source commit uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: ref: ${{ steps.resolve.outputs.sha }} fetch-depth: 1 # Nothing after checkout needs the git token — the mirror steps # authenticate to the registries directly — so don't leave it in # .git/config. persist-credentials: false # Pull the digest-vendored tree build-main baked (but never committed). # Pinned to the digest resolved above, not the mutable :main tag. - name: Pull baked package tree run: | set -eu # `flux pull artifact` rejects an --output path that does not already # exist ("invalid output path"), so create it first. rm -rf _baked && mkdir -p _baked flux pull artifact "oci://${SRC_REGISTRY}/cozystack-packages@${{ steps.resolve.outputs.packages_ref }}" --output _baked # Copy every cozystack-owned component image OCIR -> GHCR by digest and # rewrite the baked tree's image hosts to GHCR (digests untouched). - name: Mirror images and rewrite tree run: ./hack/nightly-mirror.sh "${{ steps.resolve.outputs.version }}" _baked # Re-publish the rewritten tree as the GHCR packages artifact, capture its # digest, point the installer chart at it, package + push the chart. - name: Publish packages artifact and installer chart to GHCR id: publish env: VERSION: ${{ steps.resolve.outputs.version }} SHA: ${{ steps.resolve.outputs.sha }} run: | set -eu flux push artifact "oci://${DST_REGISTRY}/cozystack-packages:${VERSION}" \ --path=_baked \ --source=https://github.com/cozystack/cozystack \ --revision="${VERSION}:${SHA}" 2>&1 | tee _push.log DIGEST="$(awk -F@ '/artifact successfully pushed/ {print $2}' _push.log)" [ -n "$DIGEST" ] || { echo "::error::failed to capture packages artifact digest"; exit 1; } # Floating tag for the latest nightly packages artifact. skopeo copy --multi-arch all \ "docker://${DST_REGISTRY}/cozystack-packages:${VERSION}" \ "docker://${DST_REGISTRY}/cozystack-packages:nightly" # Point the installer chart at the GHCR artifact (the artifact cannot # carry its own digest, so set it on the chart we package now). yq -i ".cozystackOperator.platformSourceUrl = \"oci://${DST_REGISTRY}/cozystack-packages\"" _baked/core/installer/values.yaml yq -i ".cozystackOperator.platformSourceRef = \"digest=${DIGEST}\"" _baked/core/installer/values.yaml PKG="$(helm package _baked/core/installer --version "${VERSION}" --app-version "${VERSION}" | awk '{print $NF}')" helm push "$PKG" "oci://${DST_REGISTRY}" skopeo copy --multi-arch all \ "docker://${DST_REGISTRY}/cozy-installer:${VERSION}" \ "docker://${DST_REGISTRY}/cozy-installer:nightly" echo "packages_digest=${DIGEST}" >> "$GITHUB_OUTPUT" echo "Published cozystack-packages@${DIGEST} and cozy-installer:${VERSION} to GHCR." # ── Assemble the Talos nocloud disk from the upstream imager (no cozystack # image dependency) and publish it as a GHCR OCI artifact. build-disk: name: Build & publish nocloud disk needs: [mirror] runs-on: oracle-vm-4cpu-16gb-x86-64 timeout-minutes: 40 permissions: contents: read packages: write steps: - name: Checkout source commit uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: ref: ${{ needs.mirror.outputs.sha }} fetch-depth: 1 persist-credentials: false - name: Set up oras run: | command -v oras >/dev/null || { curl -fsSL https://github.com/oras-project/oras/releases/download/v1.2.0/oras_1.2.0_linux_amd64.tar.gz | sudo tar -xz -C /usr/local/bin oras; } - name: Login to GHCR env: GHCR_USER: ${{ github.actor }} GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: echo "$GHCR_TOKEN" | oras login "${DST_REGISTRY%%/*}" -u "$GHCR_USER" --password-stdin # Pure asset assembly: the nocloud profile references only upstream # ghcr.io/siderolabs/* images, so this needs no cozystack registry and no # rebuild of any cozystack image. - name: Build nocloud disk run: make -C packages/core/talos talos-nocloud env: # The e2e sandbox boots amd64 only. PLATFORM: linux/amd64 - name: Publish disk to GHCR env: VERSION: ${{ needs.mirror.outputs.version }} run: | set -eu cd _out/assets # image.source links the package to this repo. Without it the first- # ever push creates an unlinked package, and the follow-up `oras tag` # can 403 permission_denied: write_package until it is linked by hand. oras push "${DST_REGISTRY}/cozystack-nocloud:${VERSION}" \ --artifact-type application/vnd.cozystack.talos.nocloud \ --annotation "org.opencontainers.image.source=https://github.com/cozystack/cozystack" \ nocloud-amd64.raw.xz:application/vnd.cozystack.talos.disk.v1+xz oras tag "${DST_REGISTRY}/cozystack-nocloud:${VERSION}" nightly echo "Published cozystack-nocloud:${VERSION} (+ :nightly) to GHCR." # ── Full e2e against the published GHCR closure (chart + images), on the # same Talos-container substrate as the PR lanes. The nodes boot the # upstream Talos image, so the nocloud disk built above is published but # not exercised here, and e2e does not wait for it. e2e: name: E2E (full suite) needs: [mirror] # Same sandbox as the other e2e lanes: three nodes at 8 vCPU / 24 GiB each # (hack/e2e-compose.yaml) commit 24 vCPU for the whole run, so the runner # class has to be strictly larger, leaving the host its own cycles for # containerd and the runner agent. Enforced by # hack/sandbox-runner-headroom.bats. runs-on: oracle-vm-32cpu-128gb-x86-64 # Covers a green full suite. It does not cover a tenant-Kubernetes suite # running into its node-join deadline, so a red there can end as a # cancelled job. Held to one figure across every lane that runs the suite. timeout-minutes: 215 permissions: contents: read packages: read steps: - name: Checkout source commit uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: ref: ${{ needs.mirror.outputs.sha }} fetch-depth: 1 persist-credentials: false - name: Set up flux uses: fluxcd/flux2/action@04acaec6161ac4fb1a82ffafa88901c03271d34f # v2.8.6 with: version: "2.8.6" - name: Login to GHCR env: GHCR_USER: ${{ github.actor }} GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | echo "$GHCR_TOKEN" | docker login "${DST_REGISTRY%%/*}" -u "$GHCR_USER" --password-stdin # Overlay the GHCR-rewritten tree (operator image, .e2e.image and every # component now point at GHCR), then pin the installer chart to the GHCR # packages artifact — exactly the bits a user would install. - name: Stage GHCR install tree env: VERSION: ${{ needs.mirror.outputs.version }} PACKAGES_DIGEST: ${{ needs.mirror.outputs.packages_digest }} run: | set -eu # `flux pull artifact` rejects an --output path that does not already # exist ("invalid output path"), so create it first. rm -rf _rw && mkdir -p _rw flux pull artifact "oci://${DST_REGISTRY}/cozystack-packages:${VERSION}" --output _rw cp -r _rw/. packages/ yq -i ".cozystackOperator.platformSourceUrl = \"oci://${DST_REGISTRY}/cozystack-packages\"" packages/core/installer/values.yaml yq -i ".cozystackOperator.platformSourceRef = \"digest=${PACKAGES_DIGEST}\"" packages/core/installer/values.yaml - name: Set sandbox ID run: echo "SANDBOX_NAME=cozy-e2e-sandbox-$(echo "${GITHUB_REPOSITORY}:${GITHUB_WORKFLOW}:${GITHUB_REF}" | sha256sum | cut -c1-10)" >> "$GITHUB_ENV" # Talos cannot load kernel modules in container mode, so the nodes get # exactly what this host has loaded. Why each module, and why failing # here is better than an hour later: the same step in pull-requests.yaml. - name: Prepare container-lane host run: | set -eu for m in openvswitch zfs; do grep -q "^${m} " /proc/modules && continue if [ "${m}" = zfs ]; then sudo apt-get update -qq sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq zfsutils-linux fi sudo modprobe "${m}" || echo "${m}: modprobe failed" done missing='' for m in openvswitch zfs; do grep -q "^${m} " /proc/modules || missing="${missing} ${m}" done if [ -n "${missing}" ]; then echo "::error title=container-lane host::kernel module(s) could not be loaded:${missing}" exit 1 fi - name: Prepare environment run: | attempt=0 until make SANDBOX_NAME="$SANDBOX_NAME" prepare-env-container; do attempt=$((attempt + 1)) if [ $attempt -ge 3 ]; then echo "Attempt $attempt failed, exiting..."; exit 1; fi echo "Attempt $attempt failed, retrying..." make -C packages/core/testing SANDBOX_NAME="$SANDBOX_NAME" delete-cluster-container || true done - name: Install Cozystack into sandbox env: # No ARP VIP in container mode: .11 is a plain control-plane node. COZY_APISERVER_ENDPOINT: https://192.168.123.11:6443 # The nodes share the runner kernel and cannot load DRBD. COZY_LINSTOR_DRBD_ENABLED: "false" COZY_E2E_STORAGE_CLASS: local run: | if ! make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME install-cozystack; then echo "❌ Install failed (no retry — diagnostics below)" echo "::group::HelmRelease status" docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A -o wide 2>&1 | tail -100' || true echo "::endgroup::" echo "::group::not-Ready HelmReleases" docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A --no-headers 2>/dev/null | awk "\$4 != \"True\" {print \$1\" \"\$2}"' \ | while read -r ns name; do echo "--- $ns/$name ---" docker exec "$SANDBOX_NAME" sh -c "kubectl describe hr '$name' -n '$ns' 2>&1 | tail -50" || true done echo "::endgroup::" exit 1 fi - name: Run OpenAPI tests run: | make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-openapi # Always the FULL app suite — a nightly has no diff to scope against, so # Test Impact Analysis does not apply (empty CHAINSAW_SUITES = whole suite). - name: Run E2E tests (full suite) env: COZY_E2E_STORAGE_CLASS: local run: | if ! make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-chainsaw CHAINSAW_SUITES=""; then echo "❌ Chainsaw E2E failed (see the assertion diffs or LEAKED-PODS lines above and diagnostics below)" echo "::group::Diagnostics: HelmRelease status" docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A -o wide 2>&1 | tail -50' || true echo "::endgroup::" echo "::group::Diagnostics: recent events" docker exec "$SANDBOX_NAME" sh -c 'kubectl get events -A --sort-by=.lastTimestamp 2>&1 | tail -30' || true echo "::endgroup::" exit 1 fi echo "✅ All E2E tests passed" - name: Collect chainsaw report if: always() run: | mkdir -p _out docker cp "$SANDBOX_NAME":/workspace/hack/e2e-chainsaw/chainsaw-report.xml \ _out/chainsaw-report.xml || true - name: Upload chainsaw report if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: chainsaw-report path: _out/chainsaw-report.xml if-no-files-found: ignore # A ceiling of its own, not the job's 215 minutes. `Upload cozyreport.tgz` # is the next step and the collector writes its tarball at the very end of # its run, so a read that never returns spends what is left of the job # budget, the job is cancelled, and the upload never runs -- the artifact # is lost rather than truncated. 30 minutes is measured against the step: # it takes 3-42s across sampled runs, failed e2e jobs included. # `continue-on-error` because the ceiling must not change what the job # reports: `|| true` says collecting diagnostics is never fatal, and a step # killed at its ceiling never reaches it. - name: Collect report id: collect_report if: always() timeout-minutes: 30 continue-on-error: true run: | make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME collect-report || true # `continue-on-error` keeps the overrun out of the job's result, which also # keeps it out of everyone's view: a step that ended at its ceiling renders # green and the only trace is in the raw log. This branch's own standard is # that a bound which fires is recorded, and that has to hold for the outer # bound too. # # The wording says what `outcome` establishes and no more: a step can end # non-zero without any ceiling firing, so naming the timeout as the cause # would be the false-mechanism claim the collector refuses to make for a 137. - name: Report collection overran its ceiling if: always() && steps.collect_report.outcome == 'failure' run: | echo "::warning title=cozyreport::Collect report ended non-zero -- its timeout-minutes fired, or the step failed before the collector ran; cozyreport.tgz is missing or partial for this run" - name: Upload cozyreport.tgz if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: cozyreport path: _out/cozyreport.tgz if-no-files-found: warn # The compose project owns the nodes and the zpools outlive it, so both # halves go, in this order. - name: Tear down sandbox if: always() run: | make -C packages/core/testing SANDBOX_NAME="$SANDBOX_NAME" delete-cluster-container || true make -C packages/core/testing SANDBOX_NAME="$SANDBOX_NAME" delete || true