Files
cozystack/.github/workflows/nightly.yaml
T
Myasnikov Daniil c63e83466a ci(e2e): raise the E2E job limit from 215 to 265 minutes
A green run of the full Chainsaw suite no longer fits in 215 minutes:
the last four nightly runs were cancelled at the limit, and pull
requests that select the full suite end the same way while every
suite that got to run passed. All lanes that run the suite keep one
figure, so a change that passes on a pull request is not cut off on
nightly or on a tag.

Assisted-by: LLM
Signed-off-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
2026-11-12 15:42:25 +05:00

434 lines
21 KiB
YAML

name: Nightly
# Publish an installable nightly of `main` WITHOUT rebuilding it, then run the
# full e2e suite against the published bits.
#
# A nightly is a renamed copy of what `build-main` already built: every
# component image is content-addressed, so `nightly-mirror.sh` copies each one
# from the CI build registry (OCIR) to the public release registry (GHCR) BY
# DIGEST — bit-for-bit, no rebuild. The cozystack-packages artifact and the
# cozy-installer chart are re-published on GHCR so the whole install closure is
# self-contained there. The only thing genuinely built is the Talos nocloud disk
# (assembled from upstream profiles, independent of the registry), published as a
# GHCR OCI artifact so a nightly is installable on real hardware. The nightly
# e2e does not boot it.
#
# No GitHub release and no api/apps/v1alpha1 Go-module tag are created — a
# nightly is an installable artifact set on GHCR, nothing more. retention.yaml
# prunes the GHCR nightly tags.
#
# Install a nightly:
# helm upgrade --install cozystack \
# oci://ghcr.io/cozystack/cozystack/cozy-installer --version nightly
# (the floating `nightly` tag tracks the latest; each build is also published
# under an immutable 0.0.0-nightly.<date>.<run-id>.<attempt> tag for pinning)
on:
schedule:
- cron: '0 2 * * *' # daily 02:00 UTC, after build-main has settled
workflow_dispatch: {}
env:
SRC_REGISTRY: iad.ocir.io/idyksih5sir9/cozystack
DST_REGISTRY: ghcr.io/cozystack/cozystack
concurrency:
group: nightly
cancel-in-progress: false
jobs:
# ── Mirror the latest main build OCIR -> GHCR by digest, re-publish the
# packages artifact + installer chart on GHCR. No rebuild.
mirror:
name: Mirror images to GHCR
if: vars.NIGHTLY_ENABLED == 'true' || github.event_name == 'workflow_dispatch'
runs-on: oracle-vm-4cpu-16gb-x86-64
timeout-minutes: 60
permissions:
contents: read
packages: write
outputs:
version: ${{ steps.resolve.outputs.version }}
sha: ${{ steps.resolve.outputs.sha }}
packages_ref: ${{ steps.resolve.outputs.packages_ref }}
packages_digest: ${{ steps.publish.outputs.packages_digest }}
steps:
- name: Set up flux
uses: fluxcd/flux2/action@04acaec6161ac4fb1a82ffafa88901c03271d34f # v2.8.6
with:
version: "2.8.6"
- name: Set up toolchain (skopeo, yq, helm)
env:
YQ_VERSION: "4.53.3"
YQ_SHA256: "fa52a4e758c63d38299163fbdd1edfb4c4963247918bf9c1c5d31d84789eded4"
run: |
command -v skopeo >/dev/null || { sudo apt-get update -qq && sudo apt-get install -y -qq skopeo; }
if ! yq --version 2>/dev/null | grep -q "mikefarah.* version v${YQ_VERSION}\$"; then
yq_bin="$(mktemp)"
curl -fsSL -o "$yq_bin" \
"https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_amd64"
printf '%s %s\n' "$YQ_SHA256" "$yq_bin" | sha256sum -c -
sudo install -m 0755 "$yq_bin" /usr/local/bin/yq
rm -f "$yq_bin"
fi
command -v helm >/dev/null || curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
- name: Login to registries
env:
OCIR_USER: ${{ secrets.OCIR_USER }}
OCIR_TOKEN: ${{ secrets.OCIR_TOKEN }}
GHCR_USER: ${{ github.actor }}
GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
# skopeo (mirror copies), flux + helm (artifact/chart push) all read
# their own auth; log each into both registries it touches. Token via
# stdin, never argv (a -p token leaks into the process list / logs).
echo "$OCIR_TOKEN" | skopeo login "${SRC_REGISTRY%%/*}" -u "$OCIR_USER" --password-stdin
echo "$GHCR_TOKEN" | skopeo login "${DST_REGISTRY%%/*}" -u "$GHCR_USER" --password-stdin
echo "$OCIR_TOKEN" | docker login "${SRC_REGISTRY%%/*}" -u "$OCIR_USER" --password-stdin
echo "$GHCR_TOKEN" | docker login "${DST_REGISTRY%%/*}" -u "$GHCR_USER" --password-stdin
echo "$GHCR_TOKEN" | helm registry login "${DST_REGISTRY%%/*}" -u "$GHCR_USER" --password-stdin
# The source of truth is the cozystack-packages:main artifact build-main
# pushed. Resolve the exact commit it was built from (its revision
# annotation) so the disk and the install tree are checked out from the
# SAME commit the images were built at — never main's drifting HEAD.
- name: Resolve source commit and version
id: resolve
run: |
set -eu
# Fetch the :main manifest ONCE and derive BOTH its digest and the
# revision annotation from those exact bytes, then pull the baked tree
# from that SAME digest below, so a concurrent build-main push of :main
# cannot make us check out commit A while mirroring tree B.
#
# --raw is mandatory here: cozystack-packages is a Flux OCI artifact
# (config mediaType application/vnd.cncf.flux.config.v1+json), not a
# container image, so a plain `skopeo inspect` attempts an
# image-specific operation and fatals on it. `manifest-digest` hashes
# the manifest bytes, which is exactly what the registry addresses it
# by — no image-specific parsing involved.
manifest="${RUNNER_TEMP}/cozystack-packages-main.json"
skopeo inspect --raw "docker://${SRC_REGISTRY}/cozystack-packages:main" > "$manifest"
packages_ref="$(skopeo manifest-digest "$manifest")"
rev="$(yq -r '.annotations["org.opencontainers.image.revision"] // ""' -p json "$manifest" 2>/dev/null || true)"
rm -f "$manifest"
# revision is "<git-describe>:<sha>"; take the sha.
sha="${rev##*:}"
# The sha flows into `actions/checkout` ref: — validate it is a bare
# commit hash before trusting it, so a malformed/forged annotation
# cannot inject a ref.
echo "$sha" | grep -qE '^[0-9a-f]{40}$' \
|| { echo "::error::source revision '$sha' is not a 40-hex commit sha"; exit 1; }
# Make every build's tag immutable: the UTC date keeps it human-
# readable, and run id + attempt make a same-day re-dispatch or re-run
# publish a distinct tag instead of overwriting the day's nightly.
version="0.0.0-nightly.$(date -u +%Y%m%d).${GITHUB_RUN_ID}.${GITHUB_RUN_ATTEMPT}"
echo "sha=$sha" >> "$GITHUB_OUTPUT"
echo "version=$version" >> "$GITHUB_OUTPUT"
echo "packages_ref=$packages_ref" >> "$GITHUB_OUTPUT"
echo "Resolved main build: sha=$sha version=$version packages_ref=$packages_ref"
- name: Checkout source commit
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
with:
ref: ${{ steps.resolve.outputs.sha }}
fetch-depth: 1
# Nothing after checkout needs the git token — the mirror steps
# authenticate to the registries directly — so don't leave it in
# .git/config.
persist-credentials: false
# Pull the digest-vendored tree build-main baked (but never committed).
# Pinned to the digest resolved above, not the mutable :main tag.
- name: Pull baked package tree
run: |
set -eu
# `flux pull artifact` rejects an --output path that does not already
# exist ("invalid output path"), so create it first.
rm -rf _baked && mkdir -p _baked
flux pull artifact "oci://${SRC_REGISTRY}/cozystack-packages@${{ steps.resolve.outputs.packages_ref }}" --output _baked
# Copy every cozystack-owned component image OCIR -> GHCR by digest and
# rewrite the baked tree's image hosts to GHCR (digests untouched).
- name: Mirror images and rewrite tree
run: ./hack/nightly-mirror.sh "${{ steps.resolve.outputs.version }}" _baked
# Re-publish the rewritten tree as the GHCR packages artifact, capture its
# digest, point the installer chart at it, package + push the chart.
- name: Publish packages artifact and installer chart to GHCR
id: publish
env:
VERSION: ${{ steps.resolve.outputs.version }}
SHA: ${{ steps.resolve.outputs.sha }}
run: |
set -eu
flux push artifact "oci://${DST_REGISTRY}/cozystack-packages:${VERSION}" \
--path=_baked \
--source=https://github.com/cozystack/cozystack \
--revision="${VERSION}:${SHA}" 2>&1 | tee _push.log
DIGEST="$(awk -F@ '/artifact successfully pushed/ {print $2}' _push.log)"
[ -n "$DIGEST" ] || { echo "::error::failed to capture packages artifact digest"; exit 1; }
# Floating tag for the latest nightly packages artifact.
skopeo copy --multi-arch all \
"docker://${DST_REGISTRY}/cozystack-packages:${VERSION}" \
"docker://${DST_REGISTRY}/cozystack-packages:nightly"
# Point the installer chart at the GHCR artifact (the artifact cannot
# carry its own digest, so set it on the chart we package now).
yq -i ".cozystackOperator.platformSourceUrl = \"oci://${DST_REGISTRY}/cozystack-packages\"" _baked/core/installer/values.yaml
yq -i ".cozystackOperator.platformSourceRef = \"digest=${DIGEST}\"" _baked/core/installer/values.yaml
PKG="$(helm package _baked/core/installer --version "${VERSION}" --app-version "${VERSION}" | awk '{print $NF}')"
helm push "$PKG" "oci://${DST_REGISTRY}"
skopeo copy --multi-arch all \
"docker://${DST_REGISTRY}/cozy-installer:${VERSION}" \
"docker://${DST_REGISTRY}/cozy-installer:nightly"
echo "packages_digest=${DIGEST}" >> "$GITHUB_OUTPUT"
echo "Published cozystack-packages@${DIGEST} and cozy-installer:${VERSION} to GHCR."
# ── Assemble the Talos nocloud disk from the upstream imager (no cozystack
# image dependency) and publish it as a GHCR OCI artifact.
build-disk:
name: Build & publish nocloud disk
needs: [mirror]
runs-on: oracle-vm-4cpu-16gb-x86-64
timeout-minutes: 40
permissions:
contents: read
packages: write
steps:
- name: Checkout source commit
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
with:
ref: ${{ needs.mirror.outputs.sha }}
fetch-depth: 1
persist-credentials: false
- name: Set up oras
run: |
command -v oras >/dev/null || { curl -fsSL https://github.com/oras-project/oras/releases/download/v1.2.0/oras_1.2.0_linux_amd64.tar.gz | sudo tar -xz -C /usr/local/bin oras; }
- name: Login to GHCR
env:
GHCR_USER: ${{ github.actor }}
GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: echo "$GHCR_TOKEN" | oras login "${DST_REGISTRY%%/*}" -u "$GHCR_USER" --password-stdin
# Pure asset assembly: the nocloud profile references only upstream
# ghcr.io/siderolabs/* images, so this needs no cozystack registry and no
# rebuild of any cozystack image.
- name: Build nocloud disk
run: make -C packages/core/talos talos-nocloud
env:
# The e2e sandbox boots amd64 only.
PLATFORM: linux/amd64
- name: Publish disk to GHCR
env:
VERSION: ${{ needs.mirror.outputs.version }}
run: |
set -eu
cd _out/assets
# image.source links the package to this repo. Without it the first-
# ever push creates an unlinked package, and the follow-up `oras tag`
# can 403 permission_denied: write_package until it is linked by hand.
oras push "${DST_REGISTRY}/cozystack-nocloud:${VERSION}" \
--artifact-type application/vnd.cozystack.talos.nocloud \
--annotation "org.opencontainers.image.source=https://github.com/cozystack/cozystack" \
nocloud-amd64.raw.xz:application/vnd.cozystack.talos.disk.v1+xz
oras tag "${DST_REGISTRY}/cozystack-nocloud:${VERSION}" nightly
echo "Published cozystack-nocloud:${VERSION} (+ :nightly) to GHCR."
# ── Full e2e against the published GHCR closure (chart + images), on the
# same Talos-container substrate as the PR lanes. The nodes boot the
# upstream Talos image, so the nocloud disk built above is published but
# not exercised here, and e2e does not wait for it.
e2e:
name: E2E (full suite)
needs: [mirror]
# Same sandbox as the other e2e lanes: three nodes at 8 vCPU / 24 GiB each
# (hack/e2e-compose.yaml) commit 24 vCPU for the whole run, so the runner
# class has to be strictly larger, leaving the host its own cycles for
# containerd and the runner agent. Enforced by
# hack/sandbox-runner-headroom.bats.
runs-on: oracle-vm-32cpu-128gb-x86-64
# Covers a green full suite. It does not cover a tenant-Kubernetes suite
# running into its node-join deadline, so a red there can end as a
# cancelled job. Held to one figure across every lane that runs the suite.
timeout-minutes: 265
permissions:
contents: read
packages: read
steps:
- name: Checkout source commit
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
with:
ref: ${{ needs.mirror.outputs.sha }}
fetch-depth: 1
persist-credentials: false
- name: Set up flux
uses: fluxcd/flux2/action@04acaec6161ac4fb1a82ffafa88901c03271d34f # v2.8.6
with:
version: "2.8.6"
- name: Login to GHCR
env:
GHCR_USER: ${{ github.actor }}
GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
echo "$GHCR_TOKEN" | docker login "${DST_REGISTRY%%/*}" -u "$GHCR_USER" --password-stdin
# Overlay the GHCR-rewritten tree (operator image, .e2e.image and every
# component now point at GHCR), then pin the installer chart to the GHCR
# packages artifact — exactly the bits a user would install.
- name: Stage GHCR install tree
env:
VERSION: ${{ needs.mirror.outputs.version }}
PACKAGES_DIGEST: ${{ needs.mirror.outputs.packages_digest }}
run: |
set -eu
# `flux pull artifact` rejects an --output path that does not already
# exist ("invalid output path"), so create it first.
rm -rf _rw && mkdir -p _rw
flux pull artifact "oci://${DST_REGISTRY}/cozystack-packages:${VERSION}" --output _rw
cp -r _rw/. packages/
yq -i ".cozystackOperator.platformSourceUrl = \"oci://${DST_REGISTRY}/cozystack-packages\"" packages/core/installer/values.yaml
yq -i ".cozystackOperator.platformSourceRef = \"digest=${PACKAGES_DIGEST}\"" packages/core/installer/values.yaml
- name: Set sandbox ID
run: echo "SANDBOX_NAME=cozy-e2e-sandbox-$(echo "${GITHUB_REPOSITORY}:${GITHUB_WORKFLOW}:${GITHUB_REF}" | sha256sum | cut -c1-10)" >> "$GITHUB_ENV"
# Talos cannot load kernel modules in container mode, so the nodes get
# exactly what this host has loaded. Why each module, and why failing
# here is better than an hour later: the same step in pull-requests.yaml.
- name: Prepare container-lane host
run: |
set -eu
for m in openvswitch zfs; do
grep -q "^${m} " /proc/modules && continue
if [ "${m}" = zfs ]; then
sudo apt-get update -qq
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq zfsutils-linux
fi
sudo modprobe "${m}" || echo "${m}: modprobe failed"
done
missing=''
for m in openvswitch zfs; do
grep -q "^${m} " /proc/modules || missing="${missing} ${m}"
done
if [ -n "${missing}" ]; then
echo "::error title=container-lane host::kernel module(s) could not be loaded:${missing}"
exit 1
fi
- name: Prepare environment
run: |
attempt=0
until make SANDBOX_NAME="$SANDBOX_NAME" prepare-env-container; do
attempt=$((attempt + 1))
if [ $attempt -ge 3 ]; then echo "Attempt $attempt failed, exiting..."; exit 1; fi
echo "Attempt $attempt failed, retrying..."
make -C packages/core/testing SANDBOX_NAME="$SANDBOX_NAME" delete-cluster-container || true
done
- name: Install Cozystack into sandbox
env:
# No ARP VIP in container mode: .11 is a plain control-plane node.
COZY_APISERVER_ENDPOINT: https://192.168.123.11:6443
# The nodes share the runner kernel and cannot load DRBD.
COZY_LINSTOR_DRBD_ENABLED: "false"
COZY_E2E_STORAGE_CLASS: local
run: |
if ! make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME install-cozystack; then
echo "❌ Install failed (no retry — diagnostics below)"
echo "::group::HelmRelease status"
docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A -o wide 2>&1 | tail -100' || true
echo "::endgroup::"
echo "::group::not-Ready HelmReleases"
docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A --no-headers 2>/dev/null | awk "\$4 != \"True\" {print \$1\" \"\$2}"' \
| while read -r ns name; do
echo "--- $ns/$name ---"
docker exec "$SANDBOX_NAME" sh -c "kubectl describe hr '$name' -n '$ns' 2>&1 | tail -50" || true
done
echo "::endgroup::"
exit 1
fi
- name: Run OpenAPI tests
run: |
make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-openapi
# Always the FULL app suite — a nightly has no diff to scope against, so
# Test Impact Analysis does not apply (empty CHAINSAW_SUITES = whole suite).
- name: Run E2E tests (full suite)
env:
COZY_E2E_STORAGE_CLASS: local
run: |
if ! make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-chainsaw CHAINSAW_SUITES=""; then
echo "❌ Chainsaw E2E failed (see the assertion diffs or LEAKED-PODS lines above and diagnostics below)"
echo "::group::Diagnostics: HelmRelease status"
docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A -o wide 2>&1 | tail -50' || true
echo "::endgroup::"
echo "::group::Diagnostics: recent events"
docker exec "$SANDBOX_NAME" sh -c 'kubectl get events -A --sort-by=.lastTimestamp 2>&1 | tail -30' || true
echo "::endgroup::"
exit 1
fi
echo "✅ All E2E tests passed"
- name: Collect chainsaw report
if: always()
run: |
mkdir -p _out
docker cp "$SANDBOX_NAME":/workspace/hack/e2e-chainsaw/chainsaw-report.xml \
_out/chainsaw-report.xml || true
- name: Upload chainsaw report
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: chainsaw-report
path: _out/chainsaw-report.xml
if-no-files-found: ignore
# A ceiling of its own, not the job's 265 minutes. `Upload cozyreport.tgz`
# is the next step and the collector writes its tarball at the very end of
# its run, so a read that never returns spends what is left of the job
# budget, the job is cancelled, and the upload never runs -- the artifact
# is lost rather than truncated. 30 minutes is measured against the step:
# it takes 3-42s across sampled runs, failed e2e jobs included.
# `continue-on-error` because the ceiling must not change what the job
# reports: `|| true` says collecting diagnostics is never fatal, and a step
# killed at its ceiling never reaches it.
- name: Collect report
id: collect_report
if: always()
timeout-minutes: 30
continue-on-error: true
run: |
make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME collect-report || true
# `continue-on-error` keeps the overrun out of the job's result, which also
# keeps it out of everyone's view: a step that ended at its ceiling renders
# green and the only trace is in the raw log. This branch's own standard is
# that a bound which fires is recorded, and that has to hold for the outer
# bound too.
#
# The wording says what `outcome` establishes and no more: a step can end
# non-zero without any ceiling firing, so naming the timeout as the cause
# would be the false-mechanism claim the collector refuses to make for a 137.
- name: Report collection overran its ceiling
if: always() && steps.collect_report.outcome == 'failure'
run: |
echo "::warning title=cozyreport::Collect report ended non-zero -- its timeout-minutes fired, or the step failed before the collector ran; cozyreport.tgz is missing or partial for this run"
- name: Upload cozyreport.tgz
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: cozyreport
path: _out/cozyreport.tgz
if-no-files-found: warn
# The compose project owns the nodes and the zpools outlive it, so both
# halves go, in this order.
- name: Tear down sandbox
if: always()
run: |
make -C packages/core/testing SANDBOX_NAME="$SANDBOX_NAME" delete-cluster-container || true
make -C packages/core/testing SANDBOX_NAME="$SANDBOX_NAME" delete || true