diff --git a/.github/workflows/e2e-proxmox.yaml b/.github/workflows/e2e-proxmox.yaml new file mode 100644 index 000000000..beb6e56fd --- /dev/null +++ b/.github/workflows/e2e-proxmox.yaml @@ -0,0 +1,122 @@ +name: E2E (proxmox) + +# The Proxmox substrate's own lane, separate from every other e2e workflow for +# one reason that cannot be engineered away: it needs a Proxmox hypervisor, and +# the oracle-vm-* runners the other lanes use will never have one. +# +# Deliberately NOT triggered by pull_request. A self-hosted runner executes +# whatever the workflow checks out, and this one holds credentials to a +# hypervisor and a live cluster; letting an arbitrary fork PR start it would +# hand both to its author. Manual dispatch and a schedule are the whole trigger +# surface, and both come from someone who already has write access. +on: + workflow_dispatch: + inputs: + keep_tenant: + description: "Leave the tenant in place after the run, for debugging" + type: boolean + default: false + schedule: + # Nightly, offset from the upstream lanes so a shared hypervisor is not + # asked to do two things at once. + - cron: "40 2 * * *" + +permissions: + contents: read + +concurrency: + # One hypervisor, one tenant namespace, fixed VM ids — two runs would fight + # over all three. Never cancel in progress: a killed run leaks VMs. + group: e2e-proxmox + cancel-in-progress: false + +jobs: + e2e: + name: Proxmox substrate + # The job needs a self-hosted runner carrying the `proxmox` label. On a + # repository without one a scheduled run would sit in the queue until GitHub + # cancels it, so the schedule fires only where the stand is declared; + # manual dispatch always runs. + if: github.event_name == 'workflow_dispatch' || vars.PROXMOX_NIGHTLY == 'true' + runs-on: [self-hosted, proxmox] + timeout-minutes: 90 + env: + COZY_PROXMOX_NS: tenant-e2eproxmox + COZY_PROXMOX_CLUSTER: k8s-e2e + # No fallbacks: an unset variable fails the first step below instead of + # sending a key to a guessed address or probing a storage the stand lacks. + COZY_PROXMOX_STORAGE: ${{ vars.PROXMOX_STORAGE }} + COZY_PVE_SSH: ${{ vars.PROXMOX_SSH }} + COZY_PVE_SSH_KEY: ${{ github.workspace }}/.ssh-pve + COZY_KEEP_TENANT: ${{ inputs.keep_tenant == true }} + steps: + - name: Checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false + + - name: Require the stand's variables + env: + PVE_URL: ${{ vars.PROXMOX_URL }} + PVE_REGION: ${{ vars.PROXMOX_REGION }} + run: | + set -eu + for v in COZY_PROXMOX_STORAGE COZY_PVE_SSH PVE_URL PVE_REGION; do + eval "val=\${$v:-}" + [ -n "$val" ] || { echo "$v is unset: set the PROXMOX_* repository variables for this stand" >&2; exit 1; } + done + + - name: Stage credentials + env: + KUBECONFIG_B64: ${{ secrets.COZY_KUBECONFIG }} + PVE_SSH_KEY: ${{ secrets.PROXMOX_SSH_KEY }} + PVE_TOKEN_ID: ${{ secrets.PROXMOX_TOKEN_ID }} + PVE_TOKEN_SECRET: ${{ secrets.PROXMOX_TOKEN_SECRET }} + PVE_URL: ${{ vars.PROXMOX_URL }} + PVE_REGION: ${{ vars.PROXMOX_REGION }} + run: | + set -eu + umask 077 + printf '%s' "$KUBECONFIG_B64" | base64 -d > "$GITHUB_WORKSPACE/.kubeconfig" + printf '%s' "$PVE_SSH_KEY" > "$COZY_PVE_SSH_KEY" + echo "KUBECONFIG=$GITHUB_WORKSPACE/.kubeconfig" >> "$GITHUB_ENV" + + # The credentials the management-side CSI controller and CCM read. A + # Secret rather than values so no token is ever written into a + # rendered HelmRelease. + kubectl --kubeconfig "$GITHUB_WORKSPACE/.kubeconfig" create namespace "$COZY_PROXMOX_NS" \ + --dry-run=client -o yaml | kubectl --kubeconfig "$GITHUB_WORKSPACE/.kubeconfig" apply -f - + kubectl --kubeconfig "$GITHUB_WORKSPACE/.kubeconfig" -n "$COZY_PROXMOX_NS" \ + create secret generic proxmox-credentials \ + --from-literal=url="$PVE_URL" \ + --from-literal=token_id="$PVE_TOKEN_ID" \ + --from-literal=token_secret="$PVE_TOKEN_SECRET" \ + --from-literal=region="$PVE_REGION" \ + --dry-run=client -o yaml | kubectl --kubeconfig "$GITHUB_WORKSPACE/.kubeconfig" apply -f - + + - name: Enable the suite + # It ships disabled so the normal lanes cannot select it; this is the + # one place that is allowed to turn it on. + run: mv hack/e2e-chainsaw/kubernetes-proxmox/chainsaw-test.yaml.disabled + hack/e2e-chainsaw/kubernetes-proxmox/chainsaw-test.yaml + + - name: Run the Proxmox suite + run: | + chainsaw test \ + --config hack/e2e-chainsaw/.chainsaw.yaml \ + --test-dir hack/e2e-chainsaw/kubernetes-proxmox + + - name: Tear down + # always(), because the whole point of the teardown is the failing run: + # a leaked tenant leaves VMs on the hypervisor and the next run inherits + # them. + if: always() && !inputs.keep_tenant + run: | + set -eu + kubectl delete kubernetesnodes.apps.cozystack.io -n "$COZY_PROXMOX_NS" --all --ignore-not-found + kubectl delete kubernetes.apps.cozystack.io -n "$COZY_PROXMOX_NS" --all --ignore-not-found + kubectl delete tenant.apps.cozystack.io e2eproxmox -n tenant-root --ignore-not-found + + - name: Remove staged credentials + if: always() + run: rm -f "$GITHUB_WORKSPACE/.kubeconfig" "$COZY_PVE_SSH_KEY" diff --git a/docs/agents/e2e-testing.md b/docs/agents/e2e-testing.md index fcd0719e1..33276852d 100644 --- a/docs/agents/e2e-testing.md +++ b/docs/agents/e2e-testing.md @@ -10,6 +10,16 @@ The app suite is **Kyverno Chainsaw** (`hack/e2e-chainsaw/`, one directory per a Every convention below follows from that finding: **fail fast, fail loud, make the failure legible.** A test that flakes is a test (or a product) with a real race to fix, not a test to wrap in a retry. Chainsaw's per-assertion polling is the structural expression of this: each `assert` waits on its own condition to its own timeout, and a failure is reported as a structured diff plus auto-captured events/describe/logs — not a generic non-zero exit. +## The Proxmox lane + +`hack/e2e-chainsaw/kubernetes-proxmox/` covers the Proxmox substrate and is the one suite in this tree that the normal lanes cannot run. It needs a Proxmox hypervisor, and the `oracle-vm-*` runners have none, so it ships as `chainsaw-test.yaml.disabled`: suite discovery is `find hack/e2e-chainsaw -mindepth 2 -maxdepth 2 -name chainsaw-test.yaml`, which registers it nowhere and makes it unselectable rather than merely unselected. `.github/workflows/e2e-proxmox.yaml` drops the suffix on a self-hosted runner that has one. + +Two properties of that workflow are security boundaries rather than preferences. It is never triggered by `pull_request`, because a self-hosted runner executes what the workflow checks out and this one stages a hypervisor token and a live kubeconfig — a fork PR trigger would hand both to its author. And its concurrency group never cancels in progress, because a killed run leaks VMs on the hypervisor rather than merely losing a result. + +It does not install Cozystack. Every other lane builds a sandbox per run; this one asserts against an existing cluster and an existing hypervisor, which is a weaker guarantee and is stated in the suite header rather than left to be discovered. + +The imperative half lives in `_lib/run-proxmox.sh` and is deliberately not a port of `_lib/run-kubernetes.sh`: that script's diagnostics are virt-launcher Pods, VirtualMachineInstances and guest consoles, none of which exist on an external hypervisor. + ## Conventions ### 1. No retries on deterministic steps; retry only pure infrastructure diff --git a/hack/e2e-chainsaw/_lib/run-proxmox.sh b/hack/e2e-chainsaw/_lib/run-proxmox.sh new file mode 100755 index 000000000..742b69570 --- /dev/null +++ b/hack/e2e-chainsaw/_lib/run-proxmox.sh @@ -0,0 +1,238 @@ +#!/bin/sh +# Imperative half of the kubernetes-proxmox suite. +# +# Chainsaw's declarative asserts cover everything that is a Kubernetes object in +# THIS cluster, and the suite uses them for exactly that. What is left needs a +# shell: the hypervisor is reached over SSH and answers in `qm` output, and the +# tenant cluster is reached with a kubeconfig that only exists once its control +# plane is up. Those are the two things below. +# +# Deliberately not a port of _lib/run-kubernetes.sh. That script's diagnostic +# half is virt-launcher Pods, VirtualMachineInstances and guest consoles — +# objects with no counterpart on an external hypervisor — so sharing it would +# mean carrying a thousand lines that can never fire here. +# +# Every subcommand exits non-zero on failure and says which check failed. There +# is no `|| echo` anywhere in this file on purpose: the pattern turns a failed +# command into a green run, and this repository has been bitten by it. +set -eu + +NS="${COZY_PROXMOX_NS:-tenant-e2eproxmox}" +CLUSTER="${COZY_PROXMOX_CLUSTER:-k8s-e2e}" +PVE_SSH="${COZY_PVE_SSH:?COZY_PVE_SSH is required: the ssh destination of the Proxmox host, user@host}" +PVE_SSH_KEY="${COZY_PVE_SSH_KEY:-$HOME/.ssh/id_ed25519}" +STORAGE="${COZY_PROXMOX_STORAGE:?COZY_PROXMOX_STORAGE is required, the PVE storage id for the PVC probe}" +TENANT_KUBECONFIG="${TMPDIR:-/tmp}/proxmox-e2e-tenant.kubeconfig" +FIXTURE="$(cd "$(dirname "$0")/.." && pwd)/kubernetes-proxmox/tenant.yaml" + +log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; } +die() { printf '[%s] FAILED: %s\n' "$(date -u +%H:%M:%S)" "$*" >&2; exit 1; } + +pve() { ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new -i "$PVE_SSH_KEY" "$PVE_SSH" "$@"; } + +# The Machine carries the VM id in the ProxmoxMachine it references. Reading it +# from the cluster rather than guessing keeps this working when a replacement +# lands on a different id. +vmid_of_machine() { + kubectl -n "$NS" get proxmoxmachine -o jsonpath='{.items[0].spec.virtualMachineID}' +} + +# --- checks ----------------------------------------------------------------- + +# A full clone is a copy of the template's disk; a linked clone references it and +# stores only the increment. Nothing fails when this is wrong — the pool comes +# up either way — so the only place it can be caught is here, by the shape of the +# volume name: a linked clone points at `base-