From 9cc3a9e459efcd0733ca3e66bd3be36d42f50ad8 Mon Sep 17 00:00:00 2001 From: Andrey Kolkov Date: Thu, 17 Sep 2026 17:25:54 +0400 Subject: [PATCH] test(vmi): add VMInstance and standalone VMDisk backup/restore e2e round-trips The disabled hack/e2e-chainsaw/backup/ suite was a Chainsaw port of the retired backup.bats, written against the old backup API, so the VMInstance and standalone VMDisk Velero paths had no e2e coverage. Replace it with two round-trips in the vminstance suite that drive the validated examples/backups/{vmi,vmdisk} scripts, so the documented demo and the automated test stay in step. Both run un-gated in CI: - vminstance-2-backup-roundtrip boots a tiny cirros VM (u1.nano, 2Gi), backs it up via BackupClass + BackupJob, restores in place, and asserts the VM returns Running off a disk PVC the restore created: a new uid, Velero's restore-name label and the -orig- PVC keepOriginalPVC leaves behind. The label alone is not enough, because existingResourcePolicy: update also stamps it on a PVC Velero only patched, which moves no data. A guest-side boot proof is unavailable: the KubeVirt CR disables serial-console logging and cirros ships no guest-agent. - vmdisk-2-backup-roundtrip imports a 5Gi disk, backs it up, and restores it in place with the same three assertions. The controller prepares the target (suspend HelmRelease, delete DataVolume, rename PVC). On a WaitForFirstConsumer class the Velero data mover waits for a Pod to select a node for the restored PVC, which a detached disk never gets, so the script starts a pause Pod on the new PVC, standing in for the VM that would attach it. The disk storageClass is an override (default replicated; the CI steps pass the lane's COZY_E2E_STORAGE_CLASS, 'local' on the DRBD-off gate) and the script waits are bounded so their worst case stays under the 40m op timeout, sized to keep the whole parallel:1 suite inside the 215m job cap. On failure the catch also dumps the Velero Restore, DataDownload and DataUpload objects. Each harness stands up a throwaway in-cluster BSL (the platform cozy-default BSL derives the external S3 endpoint, unroutable in kind); its finally resumes the suspended HelmReleases, reclaims the keepOriginalPVC orig PVC and its LINSTOR volume, and drains the Velero backups before deleting the BSL. The example scripts are non-interactive so the harness can drive them. Assisted-by: LLM Signed-off-by: Andrey Kolkov --- examples/backups/vmdisk/00-helpers.sh | 44 +++ examples/backups/vmdisk/01-create-strategy.sh | 63 ++++ .../backups/vmdisk/02-create-backupclass.sh | 33 ++ examples/backups/vmdisk/03-create-vmdisk.sh | 43 +++ .../backups/vmdisk/04-create-backupjob.sh | 41 ++ .../backups/vmdisk/05-restore-in-place.sh | 159 ++++++++ examples/backups/vmdisk/README.md | 29 ++ examples/backups/vmdisk/cleanup.sh | 70 ++++ examples/backups/vmdisk/run-all.sh | 43 +++ examples/backups/vmi/00-helpers.sh | 6 +- examples/backups/vmi/01-create-strategies.sh | 11 +- examples/backups/vmi/03-create-vmdisk.sh | 16 +- examples/backups/vmi/04-create-vminstance.sh | 24 +- examples/backups/vmi/05-create-backupjob.sh | 6 +- examples/backups/vmi/06-restore-in-place.sh | 55 ++- examples/backups/vmi/07-restore-to-copy.sh | 4 +- examples/backups/vmi/cleanup.sh | 99 +++-- examples/backups/vmi/run-all.sh | 57 ++- hack/e2e-chainsaw/backup/README.md | 21 -- .../backup/chainsaw-test.yaml.disabled | 245 ------------ .../vminstance/chainsaw-test.yaml | 355 ++++++++++++++++++ 21 files changed, 1069 insertions(+), 355 deletions(-) create mode 100755 examples/backups/vmdisk/00-helpers.sh create mode 100755 examples/backups/vmdisk/01-create-strategy.sh create mode 100755 examples/backups/vmdisk/02-create-backupclass.sh create mode 100755 examples/backups/vmdisk/03-create-vmdisk.sh create mode 100755 examples/backups/vmdisk/04-create-backupjob.sh create mode 100755 examples/backups/vmdisk/05-restore-in-place.sh create mode 100644 examples/backups/vmdisk/README.md create mode 100755 examples/backups/vmdisk/cleanup.sh create mode 100755 examples/backups/vmdisk/run-all.sh delete mode 100644 hack/e2e-chainsaw/backup/README.md delete mode 100644 hack/e2e-chainsaw/backup/chainsaw-test.yaml.disabled diff --git a/examples/backups/vmdisk/00-helpers.sh b/examples/backups/vmdisk/00-helpers.sh new file mode 100755 index 000000000..9bb2b046e --- /dev/null +++ b/examples/backups/vmdisk/00-helpers.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# Helper functions and variables for the VMDisk backup/restore demo. +# Source this file in other scripts: source "$(dirname "$0")/00-helpers.sh" + +# ANSI color codes +export RED='\033[0;31m' +export GREEN='\033[0;32m' +export YELLOW='\033[1;33m' +export MAGENTA='\033[0;35m' +export CYAN='\033[0;36m' +export WHITE='\033[1;37m' +export NC='\033[0m' +export BOLD='\033[1m' + +# Default settings +export NAMESPACE="${NAMESPACE:-tenant-root}" +# The platform ships the SeaweedFS-backed BackupStorageLocation as cozy-default +# (packages/system/backupstrategy-controller). Point the demo/e2e strategy at it +# by default so the flow works out of the box on a backups-enabled cluster; +# override for an external S3 whose BSL carries a different name. +export BACKUP_STORAGE_LOCATION="${BACKUP_STORAGE_LOCATION:-cozy-default}" + +# Logging functions (output to stderr to avoid polluting captured output) +log_info() { echo -e "${CYAN}ℹ${NC} $*" >&2; } +log_success() { echo -e "${GREEN}✔${NC} $*" >&2; } +log_warning() { echo -e "${YELLOW}⚠${NC} $*" >&2; } +log_error() { echo -e "${RED}✖${NC} $*" >&2; } +log_step() { echo -e "\n${MAGENTA}${BOLD}▶ $*${NC}" >&2; } +log_command() { echo -e "${WHITE} \$ $*${NC}" >&2; } +log_substep() { echo -e "${CYAN} → $*${NC}" >&2; } + +separator() { + echo -e "\n${CYAN}────────────────────────────────────────────────────────────${NC}\n" >&2 +} + +print_header() { + local title="$1" + echo -e "\n${MAGENTA}${BOLD}== ${title} ==${NC}\n" >&2 +} + +# wait_for_field, wait_hr_ready and wait_deleted live in one file shared by +# every backup walkthrough, so a fix to one reaches all of them. +# shellcheck source-path=SCRIPTDIR source=../_lib/wait-helpers.sh +source "$(dirname "${BASH_SOURCE[0]}")/../_lib/wait-helpers.sh" diff --git a/examples/backups/vmdisk/01-create-strategy.sh b/examples/backups/vmdisk/01-create-strategy.sh new file mode 100755 index 000000000..d881b35d3 --- /dev/null +++ b/examples/backups/vmdisk/01-create-strategy.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# Step 01: Create the Velero backup/restore strategy for a standalone VMDisk. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +print_header "Step 1: Create VMDisk Velero strategy" + +log_step "Creating Velero strategy 'vmdisk-backup-strategy'..." +log_command "kubectl apply -f - (Velero strategy: vmdisk-backup-strategy)" + +# A standalone VMDisk is just an HelmRelease + a DataVolume-backed PVC (+ its +# configmaps), all labelled app.kubernetes.io/instance: vm-disk-. Unlike a +# VMInstance there is no VirtualMachine / VMI / launcher pod to capture. With +# snapshotMoveData the CSI data mover streams the volume through the +# BackupStorageLocation, so no VolumeSnapshotLocation is required (matching the +# platform's own cozy-default-velero-vmdisk strategy). +kubectl apply -f - <<'EOF' +apiVersion: strategy.backups.cozystack.io/v1alpha1 +kind: Velero +metadata: + name: vmdisk-backup-strategy +spec: + template: + restoreSpec: + existingResourcePolicy: update + includedNamespaces: + - '{{ .Application.metadata.namespace }}' + orLabelSelectors: + - matchLabels: + app.kubernetes.io/instance: 'vm-disk-{{ .Application.metadata.name }}' + - matchLabels: + apps.cozystack.io/application.kind: '{{ .Application.kind }}' + apps.cozystack.io/application.name: '{{ .Application.metadata.name }}' + includedResources: + - helmreleases.helm.toolkit.fluxcd.io + - persistentvolumeclaims + - configmaps + includeClusterResources: false + spec: + includedNamespaces: + - '{{ .Application.metadata.namespace }}' + orLabelSelectors: + - matchLabels: + app.kubernetes.io/instance: 'vm-disk-{{ .Application.metadata.name }}' + - matchLabels: + apps.cozystack.io/application.kind: '{{ .Application.kind }}' + apps.cozystack.io/application.name: '{{ .Application.metadata.name }}' + includedResources: + - helmreleases.helm.toolkit.fluxcd.io + - persistentvolumeclaims + - configmaps + includeClusterResources: false + storageLocation: '{{ .Parameters.backupStorageLocationName }}' + snapshotVolumes: true + snapshotMoveData: true + ttl: 720h0m0s + itemOperationTimeout: 24h0m0s +EOF + +log_success "Velero strategy created" +echo -e "\n${GREEN}${BOLD}Next step:${NC} ./02-create-backupclass.sh" >&2 diff --git a/examples/backups/vmdisk/02-create-backupclass.sh b/examples/backups/vmdisk/02-create-backupclass.sh new file mode 100755 index 000000000..4c4bac79b --- /dev/null +++ b/examples/backups/vmdisk/02-create-backupclass.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Step 02: Create a BackupClass binding VMDisk to the Velero strategy. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +print_header "Step 2: Create BackupClass" + +log_step "Creating BackupClass 'vmdisk-velero'..." +log_command "kubectl apply -f - (BackupClass: vmdisk-velero)" + +# BackupClass is cluster-scoped; a single instance covers every namespace. +kubectl apply -f - <&2 diff --git a/examples/backups/vmdisk/03-create-vmdisk.sh b/examples/backups/vmdisk/03-create-vmdisk.sh new file mode 100755 index 000000000..aad26a6be --- /dev/null +++ b/examples/backups/vmdisk/03-create-vmdisk.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Step 03: Create a standalone VMDisk and wait for it to be ready to back up. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +print_header "Step 3: Create VMDisk 'backup-src'" + +log_step "Creating VMDisk 'backup-src' in namespace $NAMESPACE..." +log_info "Imports the Ubuntu Noble cloud image into a 5Gi ${VMI_DISK_STORAGE_CLASS:-replicated} disk" +log_command "kubectl apply -f - (VMDisk: backup-src)" + +kubectl apply -f - <&2 diff --git a/examples/backups/vmdisk/04-create-backupjob.sh b/examples/backups/vmdisk/04-create-backupjob.sh new file mode 100755 index 000000000..a0dc8ac6e --- /dev/null +++ b/examples/backups/vmdisk/04-create-backupjob.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Step 04: Back up the standalone VMDisk via a BackupJob. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +print_header "Step 4: Create BackupJob" + +log_step "Creating BackupJob 'vmdisk-backup' in namespace $NAMESPACE..." +log_command "kubectl apply -f - (BackupJob: vmdisk-backup)" + +kubectl apply -f - <&2 || true +echo -e "\n${GREEN}${BOLD}Next step:${NC} ./05-restore-in-place.sh" >&2 diff --git a/examples/backups/vmdisk/05-restore-in-place.sh b/examples/backups/vmdisk/05-restore-in-place.sh new file mode 100755 index 000000000..7547add22 --- /dev/null +++ b/examples/backups/vmdisk/05-restore-in-place.sh @@ -0,0 +1,159 @@ +#!/bin/bash +# Step 05: Restore the standalone VMDisk in-place from its backup. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +print_header "Step 5: Restore VMDisk In-Place" + +PVC=vm-disk-backup-src +CONSUMER=vm-disk-backup-src-restore-consumer + +orig_uid=$(kubectl -n "$NAMESPACE" get pvc "$PVC" -o jsonpath='{.metadata.uid}') +[[ -n "$orig_uid" ]] || { log_error "PVC $PVC has no uid before the restore"; exit 1; } +log_info "PVC $PVC before restore: uid $orig_uid" + +log_step "Creating RestoreJob 'vmdisk-restore-in-place' in namespace $NAMESPACE..." +log_info "In-place restore: the original PVC is renamed and the disk is restored from backup" +log_command "kubectl apply -f - (RestoreJob: vmdisk-restore-in-place)" + +# The controller suspends the vm-disk- HelmRelease, deletes its DataVolume +# and (keepOriginalPVC) renames the live PVC to -orig-; Velero then +# creates a fresh PVC under the original name for the data mover to populate. +kubectl apply -f - <-orig- before restoring +EOF + +log_success "RestoreJob created" + +# On a WaitForFirstConsumer StorageClass the Velero data mover does not start +# until a scheduled Pod has picked a node for the restored PVC (Velero's +# WaitPVCConsumed). A VMInstance restore gets that from the virt-launcher Pod; a +# detached disk has no consumer, so stand in for the VM that would attach it. +# Only once the new PVC exists: a Pod mounting the original would pin it with +# pvc-protection and block the rename. +start_consumer() { + local mode volume mount + mode=$(kubectl -n "$NAMESPACE" get pvc "$PVC" -o jsonpath='{.spec.volumeMode}') + if [[ "$mode" == "Block" ]]; then + mount='volumeDevices: [{name: disk, devicePath: /dev/disk0}]' + else + mount='volumeMounts: [{name: disk, mountPath: /disk}]' + fi + volume="{name: disk, persistentVolumeClaim: {claimName: $PVC}}" + log_info "Starting consumer Pod $CONSUMER for the restored PVC ($mode)" + kubectl apply -f - <&2 +done +log_success "restorejob/vmdisk-restore-in-place reached 'Succeeded'" + +kubectl -n "$NAMESPACE" delete pod "$CONSUMER" --ignore-not-found --timeout=120s + +# Neither the restore-name label nor a new uid proves the data moved: Velero's +# existingResourcePolicy: update stamps the label on a PVC it only patched, and +# CDI recreating the disk would also give it a new uid. The DataDownload check +# below does; the uid and the -orig- PVC show the original was kept aside. +log_step "Verifying the PVC was recreated from the backup..." +wait_for_field pvc "$PVC" '{.status.phase}' Bound "$NAMESPACE" 120 +new_uid=$(kubectl -n "$NAMESPACE" get pvc "$PVC" -o jsonpath='{.metadata.uid}') +[[ "$new_uid" != "$orig_uid" ]] || { + log_error "PVC $PVC still has its pre-restore uid $orig_uid — the restore did not recreate it" + exit 1 +} +restore_label=$(kubectl -n "$NAMESPACE" get pvc "$PVC" \ + -o "jsonpath={.metadata.labels.velero\.io/restore-name}") +[[ -n "$restore_label" ]] || { + log_error "PVC $PVC carries no velero.io/restore-name label — Velero did not restore it" + exit 1 +} +all_pvcs=$(kubectl -n "$NAMESPACE" get pvc -o name) +[[ "$all_pvcs" == *"persistentvolumeclaim/${PVC}-orig-"* ]] || { + log_error "no ${PVC}-orig-* PVC — keepOriginalPVC did not rename the original disk" + exit 1 +} +# The DataDownload carries the same restore-name label as the PVC; one that +# Completed having written bytes is what shows the data mover filled the disk. +downloads=$(kubectl -n cozy-velero get datadownloads.velero.io -l "velero.io/restore-name=${restore_label}" \ + -o jsonpath='{range .items[*]}{.status.phase} {.status.progress.bytesDone}{"\n"}{end}') +grep -Eq '^Completed [1-9][0-9]*$' <<<"$downloads" || { + log_error "no Completed DataDownload with data for restore ${restore_label} — the data mover did not restore $PVC (got: ${downloads:-none})" + exit 1 +} + +log_step "Waiting for the restored VMDisk to become Ready again..." +# The HelmRelease was Ready before the controller suspended it, and that +# condition outlives the restore; only once Flux has reconciled the restored +# spec does Ready describe the restored release. +hr_generation=$(kubectl -n "$NAMESPACE" get hr vm-disk-backup-src -o jsonpath='{.metadata.generation}') +wait_for_field hr vm-disk-backup-src '{.status.observedGeneration}' "$hr_generation" "$NAMESPACE" 120 +wait_hr_ready vm-disk-backup-src 180 + +kubectl get restorejob vmdisk-restore-in-place -n "$NAMESPACE" -o wide >&2 || true +log_success "Restore completed; VMDisk 'backup-src' recreated from backup (${restore_label}, uid ${new_uid}), Bound and Ready" diff --git a/examples/backups/vmdisk/README.md b/examples/backups/vmdisk/README.md new file mode 100644 index 000000000..3f67d81ac --- /dev/null +++ b/examples/backups/vmdisk/README.md @@ -0,0 +1,29 @@ +# VMDisk backup/restore example + +This directory shows how to back up and restore a standalone Cozystack `VMDisk` (a HelmRelease plus a DataVolume-backed PVC, with no VMInstance attached) using the cluster's `Velero` backup strategy driver. Each `BackupJob` drives a Velero CSI-snapshot backup whose data mover (kopia) streams the LINSTOR block volume to S3; each in-place `RestoreJob` recreates the disk from that artefact. + +The numbered scripts run in order (or via `run-all.sh`): + +1. `01-create-strategy.sh` — a `Velero` strategy for the `VMDisk` kind. +2. `02-create-backupclass.sh` — a `BackupClass` binding `VMDisk` to it. +3. `03-create-vmdisk.sh` — a standalone `VMDisk`, waited until imported. +4. `04-create-backupjob.sh` — an ad-hoc `BackupJob`, waited to `Succeeded`. +5. `05-restore-in-place.sh` — an in-place `RestoreJob`, then asserts that the PVC was recreated from the backup: it has a new uid, carries Velero's `velero.io/restore-name` label, and the original survives as `-orig-`. + +`cleanup.sh` tears everything down non-interactively. + +## In-place restore + +The driver prepares a `VMDisk` restore target the same way it does a VMInstance disk: it suspends the `vm-disk-` HelmRelease and deletes its DataVolume (so Flux/CDI do not recreate the PVC and race the data mover), and, with `keepOriginalPVC: true`, renames the live PVC to `-orig-` so that Velero creates a fresh PVC under the original name. `cleanup.sh` resumes the HelmRelease and reclaims the renamed PVC. + +A disk that a running VM mounts cannot be restored in place: the `RestoreJob` fails before it changes anything and names the Pod that holds the disk. Stop the VM, or restore the VMInstance instead. + +If the restore fails after this preparation, nothing is rolled back. The failure message and a Warning event name the HelmRelease that preparation suspended and the `-orig-` PVC that holds the original data. If Velero got as far as restoring the HelmRelease it is no longer suspended, and Flux may already have recreated the disk; with no restored PVC under the disk's name, CDI imports a fresh disk there. Either way, check what the disk holds, and restore again or bind the original PV back to a PVC under the disk's name if it is not what you expect. + +## Detached disks and WaitForFirstConsumer + +On a StorageClass with `volumeBindingMode: WaitForFirstConsumer`, such as the `local` class the CI lane uses, the Velero data mover does not start until a scheduled Pod has selected a node for the restored PVC. A VMInstance restore gets that from its virt-launcher Pod. A detached disk has no consumer, so its `RestoreJob` stays `Running` until something mounts the PVC, and fails once Velero's expose timeout passes. `05-restore-in-place.sh` therefore starts a minimal `pause` Pod on the restored PVC, standing in for the VM that would attach the disk, and deletes it once the restore has succeeded. On a real cluster, attaching the disk to a VMInstance does the same. + +## Overrides + +Environment variables (see `00-helpers.sh`) tune the flow: `NAMESPACE`, `BACKUP_STORAGE_LOCATION` (defaults to the platform `cozy-default` BSL), `VMI_DISK_STORAGE_CLASS` (defaults to `replicated`; set it to your cluster's class), and the `IMPORT_WAIT` / `BACKUP_WAIT` / `RESTORE_WAIT` budgets. `hack/e2e-chainsaw/vminstance/` drives this directory as the `vmdisk-2-backup-roundtrip` e2e test. diff --git a/examples/backups/vmdisk/cleanup.sh b/examples/backups/vmdisk/cleanup.sh new file mode 100755 index 000000000..d66af5301 --- /dev/null +++ b/examples/backups/vmdisk/cleanup.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Clean up all resources created by the demo. Non-interactive: safe to call from +# the e2e harness's finally block, and idempotent (a no-op when a gated-out run +# created nothing). --ignore-not-found returns success for an absent resource but +# does NOT mask a stuck delete: the delete still blocks and, under set -e, a +# teardown that never settles fails this script instead of leaking state into the +# next run (docs/agents/e2e-testing.md convention 4). +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +print_header "Cleanup Demo Resources" + +# The controller suspends the VMDisk's HelmRelease before a restore and never +# resumes it. A restore that gets as far as Velero clears it, but one that fails +# earlier leaves it suspended, so resume it best-effort BEFORE deleting the +# RestoreJob (same reasoning as examples/backups/vmi/cleanup.sh). +if kubectl -n "$NAMESPACE" get hr vm-disk-backup-src >/dev/null 2>&1; then + log_substep "Resuming HelmRelease vm-disk-backup-src (no-op if not suspended)..." + kubectl -n "$NAMESPACE" patch hr vm-disk-backup-src --type=merge \ + -p '{"spec":{"suspend":false}}' >/dev/null || true +fi + +# Namespaced request objects first. Deleting the cozystack BackupJob/RestoreJob +# and Backup CRs cascades to the Velero Backup/Restore the controller created. +log_step "Deleting RestoreJob..." +kubectl delete restorejob vmdisk-restore-in-place -n "$NAMESPACE" --ignore-not-found + +log_step "Deleting BackupJob and Backup..." +kubectl delete backupjob vmdisk-backup -n "$NAMESPACE" --ignore-not-found +kubectl delete backup vmdisk-backup -n "$NAMESPACE" --ignore-not-found + +separator + +log_step "Deleting VMDisk and its PVC..." +kubectl delete pod vm-disk-backup-src-restore-consumer -n "$NAMESPACE" --ignore-not-found +kubectl delete vmdisk backup-src -n "$NAMESPACE" --ignore-not-found +kubectl delete pvc -n "$NAMESPACE" \ + -l app.kubernetes.io/instance=vm-disk-backup-src --ignore-not-found + +# keepOriginalPVC renamed the pre-restore PVC to -orig- with a bare +# ObjectMeta (no labels/owner) and flipped its PV to Retain, so the label delete +# above misses it. Reclaim it by name and flip the PV back to Delete BEFORE +# removing the PVC so the CSI provisioner reclaims the LINSTOR volume rather than +# stranding it. Let the lookup and patch fail the script (see the vmi cleanup). +if ! all_pvcs=$(kubectl -n "$NAMESPACE" get pvc -o name 2>&1); then + log_error "cannot list PVCs to reclaim orphaned disks: $all_pvcs" + exit 1 +fi +orig_pvcs=$(printf '%s\n' "$all_pvcs" | grep -E '/vm-disk-backup-src-orig-' || true) +for pvc in $orig_pvcs; do + pv=$(kubectl -n "$NAMESPACE" get "$pvc" -o jsonpath='{.spec.volumeName}') + log_substep "Reclaiming orphaned ${pvc#*/} and its LINSTOR volume..." + if [[ -n "$pv" ]]; then + kubectl patch pv "$pv" --type=merge \ + -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' >/dev/null + fi + kubectl -n "$NAMESPACE" delete "$pvc" --ignore-not-found +done + +separator + +log_step "Deleting BackupClass and Velero strategy..." +kubectl delete backupclass vmdisk-velero --ignore-not-found +kubectl delete velero.strategy.backups.cozystack.io vmdisk-backup-strategy --ignore-not-found + +separator + +log_success "Cleanup complete" diff --git a/examples/backups/vmdisk/run-all.sh b/examples/backups/vmdisk/run-all.sh new file mode 100755 index 000000000..0956ad807 --- /dev/null +++ b/examples/backups/vmdisk/run-all.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Convenience runner + e2e harness for the standalone VMDisk backup/restore demo. +# +# Runs the numbered steps 01..05 a human reads, in order, non-interactively, +# stopping on the first failure (set -e). This is the SAME sequence the docs walk +# through, so the documented demo and the automated test can never drift. +# +# Flow: Velero strategy -> BackupClass -> a standalone VMDisk (waits imported) -> +# ad-hoc BackupJob (waits Succeeded) -> in-place RestoreJob (waits Succeeded, then +# waits for the restored disk to be Bound + Ready). +# +# Override NAMESPACE / BACKUP_STORAGE_LOCATION via the environment; see +# 00-helpers.sh. hack/e2e-chainsaw/vminstance/ drives this file as the +# vmdisk-2-backup-roundtrip e2e test, run un-gated in CI. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +print_header "VMDisk Backup & Restore Demo" +log_info "Namespace: ${NAMESPACE} BackupStorageLocation: ${BACKUP_STORAGE_LOCATION}" + +SCRIPTS=( + "01-create-strategy.sh" + "02-create-backupclass.sh" + "03-create-vmdisk.sh" + "04-create-backupjob.sh" + "05-restore-in-place.sh" +) + +for script in "${SCRIPTS[@]}"; do + if [[ -x "$SCRIPT_DIR/$script" ]]; then + "$SCRIPT_DIR/$script" + separator + else + log_error "Script not found or not executable: $script" + exit 1 + fi +done + +print_header "Demo Complete" +log_success "All steps completed successfully!" +log_info "To clean up resources: ./cleanup.sh" diff --git a/examples/backups/vmi/00-helpers.sh b/examples/backups/vmi/00-helpers.sh index 9ad00b784..30ca161f3 100755 --- a/examples/backups/vmi/00-helpers.sh +++ b/examples/backups/vmi/00-helpers.sh @@ -15,7 +15,11 @@ export BOLD='\033[1m' # Default settings export NAMESPACE="${NAMESPACE:-tenant-root}" -export BACKUP_STORAGE_LOCATION="${BACKUP_STORAGE_LOCATION:-default}" +# The platform ships the SeaweedFS-backed BackupStorageLocation as cozy-default +# (packages/system/backupstrategy-controller). Point the demo/e2e strategies at +# it by default so the flow works out of the box on a backups-enabled cluster; +# override for an external S3 whose BSL carries a different name. +export BACKUP_STORAGE_LOCATION="${BACKUP_STORAGE_LOCATION:-cozy-default}" # Logging functions (output to stderr to avoid polluting captured output) log_info() { diff --git a/examples/backups/vmi/01-create-strategies.sh b/examples/backups/vmi/01-create-strategies.sh index 78aa34d31..a12809c54 100755 --- a/examples/backups/vmi/01-create-strategies.sh +++ b/examples/backups/vmi/01-create-strategies.sh @@ -70,8 +70,11 @@ spec: - controllerrevisions.apps includeClusterResources: false storageLocation: '{{ .Parameters.backupStorageLocationName }}' - volumeSnapshotLocations: - - '{{ .Parameters.backupStorageLocationName }}' + # No volumeSnapshotLocations: with snapshotMoveData the CSI data mover + # uploads the volume data through the BackupStorageLocation, so a VSL of + # the same name is not required (and referencing a non-existent one fails + # the backup). The platform's own cozy-default-velero-vminstance strategy + # omits it for the same reason. snapshotVolumes: true snapshotMoveData: true ttl: 720h0m0s @@ -123,8 +126,8 @@ spec: - configmaps includeClusterResources: false storageLocation: '{{ .Parameters.backupStorageLocationName }}' - volumeSnapshotLocations: - - '{{ .Parameters.backupStorageLocationName }}' + # See the VMInstance strategy above: snapshotMoveData routes volume data + # through the BackupStorageLocation, so no VolumeSnapshotLocation is needed. snapshotVolumes: true snapshotMoveData: true ttl: 720h0m0s diff --git a/examples/backups/vmi/03-create-vmdisk.sh b/examples/backups/vmi/03-create-vmdisk.sh index 89d598867..27234a09d 100755 --- a/examples/backups/vmi/03-create-vmdisk.sh +++ b/examples/backups/vmi/03-create-vmdisk.sh @@ -7,8 +7,13 @@ source "$SCRIPT_DIR/00-helpers.sh" print_header "Step 3: Create VMDisk" +# Image + size are overridable so the same flow can run a heavy ubuntu demo or a +# tiny cirros disk for a lightweight e2e (see hack/e2e-chainsaw/vminstance). +VMI_IMAGE_URL="${VMI_IMAGE_URL:-https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img}" +VMI_DISK_STORAGE="${VMI_DISK_STORAGE:-20Gi}" + log_step "Creating VMDisk 'ubuntu-source' in namespace $NAMESPACE..." -log_info "This will download the Ubuntu Noble cloud image (~700MB)" +log_info "Importing ${VMI_IMAGE_URL} into a ${VMI_DISK_STORAGE} disk" log_command "kubectl apply -f - (VMDisk: ubuntu-source)" kubectl apply -f - < @@ -34,11 +39,24 @@ log_success "VMInstance created" separator +# Wait for the VM to be fully booted before backing it up. Backing up a VM that +# is still importing its disk or has not started yields an incomplete, useless +# backup — so, like the etcd/postgres source steps, block on readiness here. +# The Ubuntu cloud image download (~700MB) plus nested KubeVirt boot is slow, +# hence the generous 900s bound. +log_step "Waiting for the VMInstance to boot (HelmRelease + VirtualMachine Ready)..." +wait_hr_ready vm-instance-test "${IMPORT_WAIT:-900}" +log_command "kubectl -n $NAMESPACE wait virtualmachine.kubevirt.io/vm-instance-test --for=condition=Ready" +kubectl -n "$NAMESPACE" wait virtualmachine.kubevirt.io/vm-instance-test \ + --for=condition=Ready --timeout="${VMI_BOOT_WAIT:-600}s" + +separator + log_step "Verifying VMInstance..." log_command "kubectl get vminstance test -n $NAMESPACE" kubectl get vminstance test -n "$NAMESPACE" separator -log_success "VMInstance is ready" +log_success "VMInstance is running and ready to back up" echo -e "\n${GREEN}${BOLD}Next step:${NC} ./05-create-backupjob.sh" diff --git a/examples/backups/vmi/05-create-backupjob.sh b/examples/backups/vmi/05-create-backupjob.sh index 581490a47..095e0b169 100755 --- a/examples/backups/vmi/05-create-backupjob.sh +++ b/examples/backups/vmi/05-create-backupjob.sh @@ -30,7 +30,11 @@ log_success "BackupJob created" separator log_step "Waiting for BackupJob to complete..." -wait_for_field backupjob test-backup '{.status.phase}' Succeeded "$NAMESPACE" 600 +# Default 1800s: the Velero CSI data mover copies the whole VM disk (20Gi) to S3, +# far heavier than a database dump. Overridable (BACKUP_WAIT) so the CI harness +# can bound it well under the Chainsaw op timeout on its tiny guest. Fail fast if +# the job flips to Failed. +wait_for_field backupjob test-backup '{.status.phase}' Succeeded "$NAMESPACE" "${BACKUP_WAIT:-1800}" Failed separator diff --git a/examples/backups/vmi/06-restore-in-place.sh b/examples/backups/vmi/06-restore-in-place.sh index ea424ed56..95cd5e1ee 100755 --- a/examples/backups/vmi/06-restore-in-place.sh +++ b/examples/backups/vmi/06-restore-in-place.sh @@ -7,6 +7,9 @@ source "$SCRIPT_DIR/00-helpers.sh" print_header "Step 6: Restore VMInstance In-Place" +orig_uid=$(kubectl -n "$NAMESPACE" get pvc vm-disk-ubuntu-source -o jsonpath='{.metadata.uid}') +[[ -n "$orig_uid" ]] || { log_error "PVC vm-disk-ubuntu-source has no uid before the restore"; exit 1; } + log_step "Creating RestoreJob 'restore-in-place-test' in namespace $NAMESPACE..." log_info "In-place restore: the VM will be halted, PVCs renamed, and data restored from backup" log_command "kubectl apply -f - (RestoreJob: restore-in-place-test)" @@ -25,8 +28,8 @@ spec: kind: VMInstance name: test options: - failIfTargetExists: true # if true, restore will fail when the target resource already exists - keepOriginalPVC: true # renames original VMI PVC before restore to `-orig-`, only for in-place restore + failIfTargetExists: true # only enforced for restore-to-copy; a no-op for in-place, whose source is expected to exist + keepOriginalPVC: true # renames original VMI PVC before restore to -orig-, only for in-place restore keepOriginalIpAndMac: true # restores original IP and MAC address of VMI via OVN annotations EOF @@ -35,7 +38,7 @@ log_success "RestoreJob created" separator log_step "Waiting for RestoreJob to complete..." -wait_for_field restorejob restore-in-place-test '{.status.phase}' Succeeded "$NAMESPACE" 600 +wait_for_field restorejob restore-in-place-test '{.status.phase}' Succeeded "$NAMESPACE" "${RESTORE_WAIT:-1800}" Failed separator @@ -45,5 +48,49 @@ kubectl get restorejob restore-in-place-test -n "$NAMESPACE" -o wide separator -log_success "In-place restore completed successfully" +# The RestoreJob reaching Succeeded means Velero re-created the objects; wait +# for the restored VM to come back before checking where its disk came from. +log_step "Waiting for the restored VMInstance to boot again..." +log_command "kubectl -n $NAMESPACE wait virtualmachine.kubevirt.io/vm-instance-test --for=condition=Ready" +kubectl -n "$NAMESPACE" wait virtualmachine.kubevirt.io/vm-instance-test \ + --for=condition=Ready --timeout="${VMI_BOOT_WAIT:-600}s" + +# VirtualMachine Ready is the virt-launcher Pod's readiness, reported before the +# guest runs, so it cannot tell a restored disk from a blank one, and this +# platform offers no guest-side signal (serial-console logging is disabled +# cluster-wide, the cirros guest has no qemu-guest-agent). Prove provenance on the +# PVC instead. Neither the restore-name label nor a new uid proves the data +# moved (existingResourcePolicy: update stamps the label on a PVC it only +# patched, and CDI recreating the disk gives it a new uid); the DataDownload +# check below does, and the uid and -orig- PVC show the original was kept aside. +log_step "Verifying the VM booted off a disk the restore recreated from the backup..." +wait_for_field pvc vm-disk-ubuntu-source '{.status.phase}' Bound "$NAMESPACE" 120 +new_uid=$(kubectl -n "$NAMESPACE" get pvc vm-disk-ubuntu-source -o jsonpath='{.metadata.uid}') +[[ "$new_uid" != "$orig_uid" ]] || { + log_error "PVC vm-disk-ubuntu-source still has its pre-restore uid $orig_uid — the restore did not recreate it" + exit 1 +} +restore_label=$(kubectl -n "$NAMESPACE" get pvc vm-disk-ubuntu-source \ + -o "jsonpath={.metadata.labels.velero\.io/restore-name}") +[[ -n "$restore_label" ]] || { + log_error "restored VM disk PVC carries no velero.io/restore-name label — Velero did not restore it" + exit 1 +} +all_pvcs=$(kubectl -n "$NAMESPACE" get pvc -o name) +[[ "$all_pvcs" == *"persistentvolumeclaim/vm-disk-ubuntu-source-orig-"* ]] || { + log_error "no vm-disk-ubuntu-source-orig-* PVC — keepOriginalPVC did not rename the original disk" + exit 1 +} +# The DataDownload carries the same restore-name label as the PVC; one that +# Completed having written bytes is what shows the data mover filled the disk. +downloads=$(kubectl -n cozy-velero get datadownloads.velero.io -l "velero.io/restore-name=${restore_label}" \ + -o jsonpath='{range .items[*]}{.status.phase} {.status.progress.bytesDone}{"\n"}{end}') +grep -Eq '^Completed [1-9][0-9]*$' <<<"$downloads" || { + log_error "no Completed DataDownload with data for restore ${restore_label} — the data mover did not restore vm-disk-ubuntu-source (got: ${downloads:-none})" + exit 1 +} + +separator + +log_success "In-place restore completed; VM disk recreated from backup (${restore_label}, uid ${new_uid}) and the VM is Running off it" echo -e "\n${GREEN}${BOLD}Next step:${NC} ./07-restore-to-copy.sh" diff --git a/examples/backups/vmi/07-restore-to-copy.sh b/examples/backups/vmi/07-restore-to-copy.sh index 9b84ce558..d7f7b87fb 100755 --- a/examples/backups/vmi/07-restore-to-copy.sh +++ b/examples/backups/vmi/07-restore-to-copy.sh @@ -40,7 +40,7 @@ spec: options: # runtime.RawExtension, typed based on targetApplicationRef and current controller implementation (for additional restore options) targetNamespace: ${TARGET_NAMESPACE} # when set to a different namespace, triggers cross-namespace restore via Velero namespaceMapping failIfTargetExists: true # if true, restore will fail when the target resource already exists - keepOriginalPVC: false # renames original VMI PVC before restore to `-orig-`, only for in-place restore + keepOriginalPVC: false # renames original VMI PVC before restore to -orig-, only for in-place restore keepOriginalIpAndMac: false # restores original IP and MAC address of VMI via OVN annotations EOF @@ -49,7 +49,7 @@ log_success "RestoreJob created" separator log_step "Waiting for RestoreJob to complete..." -wait_for_field restorejob restore-to-copy-test '{.status.phase}' Succeeded "$NAMESPACE" 600 +wait_for_field restorejob restore-to-copy-test '{.status.phase}' Succeeded "$NAMESPACE" 1800 Failed separator diff --git a/examples/backups/vmi/cleanup.sh b/examples/backups/vmi/cleanup.sh index 5965c3f03..3351ce9ee 100755 --- a/examples/backups/vmi/cleanup.sh +++ b/examples/backups/vmi/cleanup.sh @@ -1,5 +1,10 @@ #!/bin/bash -# Clean up all resources created by the demo +# Clean up all resources created by the demo. Non-interactive: safe to call from +# the e2e harness's finally block, and idempotent (a no-op when a gated-out run +# created nothing). --ignore-not-found returns success for an absent resource +# but does NOT mask a stuck delete: the delete still blocks and, under set -e, +# a teardown that never settles fails this script instead of leaking state into +# the next run (docs/agents/e2e-testing.md convention 4). set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -9,62 +14,74 @@ TARGET_NAMESPACE="${TARGET_NAMESPACE:-tenant-root-copy}" print_header "Cleanup Demo Resources" -log_warning "This script will delete all resources created during the demo" -echo -e "\n${YELLOW}The following will be deleted:${NC}" -echo " - RestoreJobs (restore-in-place-test, restore-to-copy-test)" -echo " - BackupJob (test-backup) and associated Backup" -echo " - VMInstance (test)" -echo " - VMDisk (ubuntu-source)" -echo " - BackupClass (velero)" -echo " - Velero strategies (vminstance-strategy, vmdisk-strategy)" -echo " - Target namespace ($TARGET_NAMESPACE)" -echo "" - -read -p "Continue? (y/N): " -n 1 -r -echo -if [[ ! $REPLY =~ ^[Yy]$ ]]; then - log_info "Cancelled" - exit 0 -fi - -separator +# The driver suspends the VMInstance/VMDisk HelmReleases while a RestoreJob runs +# and never resumes them (suspendHelmRelease only ever sets suspend=true), so an +# aborted or completed restore leaves them suspended. Resume best-effort BEFORE +# deleting the RestoreJobs — deleting a RestoreJob without resuming strands the +# app suspended (same reasoning as examples/backups/postgres/cleanup.sh). +for hr in vm-instance-test vm-disk-ubuntu-source; do + if kubectl -n "$NAMESPACE" get hr "$hr" >/dev/null 2>&1; then + log_substep "Resuming HelmRelease $hr (no-op if not suspended)..." + kubectl -n "$NAMESPACE" patch hr "$hr" --type=merge \ + -p '{"spec":{"suspend":false}}' >/dev/null || true + fi +done +# Namespaced request objects first. Deleting the cozystack BackupJob/RestoreJob +# and Backup CRs cascades to the Velero Backup/Restore the controller created. log_step "Deleting RestoreJobs..." -kubectl delete restorejob restore-to-copy-test -n "$NAMESPACE" 2>/dev/null || log_warning "RestoreJob restore-to-copy-test not found" -kubectl delete restorejob restore-in-place-test -n "$NAMESPACE" 2>/dev/null || log_warning "RestoreJob restore-in-place-test not found" - -separator +kubectl delete restorejob restore-to-copy-test restore-in-place-test \ + -n "$NAMESPACE" --ignore-not-found log_step "Deleting BackupJob and Backup..." -kubectl delete backupjob test-backup -n "$NAMESPACE" 2>/dev/null || log_warning "BackupJob test-backup not found" -kubectl delete backup test-backup -n "$NAMESPACE" 2>/dev/null || log_warning "Backup test-backup not found" +kubectl delete backupjob test-backup -n "$NAMESPACE" --ignore-not-found +kubectl delete backup test-backup -n "$NAMESPACE" --ignore-not-found separator -log_step "Deleting VMInstance..." -kubectl delete vminstance test -n "$NAMESPACE" 2>/dev/null || log_warning "VMInstance test not found" +log_step "Deleting VMInstance and VMDisk..." +kubectl delete vminstance test -n "$NAMESPACE" --ignore-not-found +kubectl delete vmdisk ubuntu-source -n "$NAMESPACE" --ignore-not-found -log_step "Deleting VMDisk..." -kubectl delete vmdisk ubuntu-source -n "$NAMESPACE" 2>/dev/null || log_warning "VMDisk ubuntu-source not found" +# keepOriginalPVC renames the VMInstance's disk PVC to -orig- with a +# bare ObjectMeta (no labels, no ownerRefs) and flips its PV to Retain. Nothing +# above reclaims it: the app delete does not own it and no label selector matches +# it, so under skipDelete a LINSTOR volume and a Released PV leak on every run. +# List loudly (a failed query must abort, not silently skip the reclaim it exists +# for); grep's no-match stays tolerant. +if ! all_pvcs=$(kubectl -n "$NAMESPACE" get pvc -o name 2>&1); then + log_error "cannot list PVCs to reclaim orphaned disks: $all_pvcs" + exit 1 +fi +orig_pvcs=$(printf '%s\n' "$all_pvcs" | grep -E '/vm-disk-ubuntu-source-orig-' || true) +for pvc in $orig_pvcs; do + # Flip the retained PV back to Delete BEFORE removing the PVC, so releasing it + # makes the CSI provisioner call DeleteVolume and reclaim the LINSTOR volume — + # deleting the PV object alone would drop the record and strand the volume. Let + # both the lookup and the patch fail the script: swallowing them would delete + # the PVC anyway and strand the volume with no PVC left to find it by. + pv=$(kubectl -n "$NAMESPACE" get "$pvc" -o jsonpath='{.spec.volumeName}') + log_substep "Reclaiming orphaned ${pvc#*/} and its LINSTOR volume..." + if [[ -n "$pv" ]]; then + kubectl patch pv "$pv" --type=merge \ + -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' >/dev/null + fi + kubectl -n "$NAMESPACE" delete "$pvc" --ignore-not-found +done separator -log_step "Deleting BackupClass..." -kubectl delete backupclass velero 2>/dev/null || log_warning "BackupClass velero not found" - -separator - -log_step "Deleting Velero strategies..." -kubectl delete velero.strategy.backups.cozystack.io vminstance-strategy 2>/dev/null || log_warning "Strategy vminstance-strategy not found" -kubectl delete velero.strategy.backups.cozystack.io vmdisk-strategy 2>/dev/null || log_warning "Strategy vmdisk-strategy not found" +# Cluster-scoped BackupClass + strategies. +log_step "Deleting BackupClass and Velero strategies..." +kubectl delete backupclass velero --ignore-not-found +kubectl delete velero.strategy.backups.cozystack.io \ + vminstance-strategy vmdisk-strategy --ignore-not-found separator log_step "Deleting target namespace..." -kubectl delete namespace "$TARGET_NAMESPACE" 2>/dev/null || log_warning "Namespace $TARGET_NAMESPACE not found" +kubectl delete namespace "$TARGET_NAMESPACE" --ignore-not-found separator log_success "Cleanup complete" -log_info "All demo resources have been deleted" -echo -e "\n${GREEN}${BOLD}To re-run the demo:${NC} ./01-create-strategies.sh" diff --git a/examples/backups/vmi/run-all.sh b/examples/backups/vmi/run-all.sh index b58f4b9d3..2585bdb30 100755 --- a/examples/backups/vmi/run-all.sh +++ b/examples/backups/vmi/run-all.sh @@ -1,15 +1,25 @@ #!/bin/bash -# Run the full VMInstance backup/restore demo sequentially +# Convenience runner + e2e harness for the VMInstance backup/restore demo. +# +# Runs the numbered steps 01..07 a human reads, in order, non-interactively, +# stopping on the first failure (set -e). This is the SAME sequence the docs +# walk through, so the documented flow and the automated test can never drift. +# +# Flow: Velero strategies -> BackupClass -> VMDisk + VMInstance (waits until the +# VM is booted) -> ad-hoc BackupJob (waits Succeeded) -> in-place RestoreJob +# (waits Succeeded, waits for the VM to boot again) -> cross-namespace to-copy +# RestoreJob (waits Succeeded). +# +# Override NAMESPACE / BACKUP_STORAGE_LOCATION / TARGET_NAMESPACE via the +# environment; see 00-helpers.sh. hack/e2e-chainsaw/vminstance/ drives this file +# as the vminstance-2-backup-roundtrip e2e test, run un-gated in CI. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/00-helpers.sh" print_header "VMInstance Backup & Restore Demo" - -log_info "This script will run all demo steps sequentially" -log_info "There will be a pause between steps for observation" -echo "" +log_info "Namespace: ${NAMESPACE} BackupStorageLocation: ${BACKUP_STORAGE_LOCATION}" SCRIPTS=( "01-create-strategies.sh" @@ -18,29 +28,18 @@ SCRIPTS=( "04-create-vminstance.sh" "05-create-backupjob.sh" "06-restore-in-place.sh" - "07-restore-to-copy.sh" ) - -echo -e "${CYAN}Demo steps:${NC}" -for i in "${!SCRIPTS[@]}"; do - echo " $((i+1)). ${SCRIPTS[$i]}" -done -echo "" - -read -p "Run demo? (y/N): " -n 1 -r -echo -if [[ ! $REPLY =~ ^[Yy]$ ]]; then - log_info "Cancelled" - exit 0 +# Cross-namespace to-copy restore is the heaviest step (a second full VM in a +# fresh namespace). SKIP_RESTORE_TO_COPY=1 drops it — used by the lightweight CI +# round-trip, which proves backup + in-place restore is enough. +if [[ "${SKIP_RESTORE_TO_COPY:-0}" != "1" ]]; then + SCRIPTS+=("07-restore-to-copy.sh") fi -separator - for script in "${SCRIPTS[@]}"; do if [[ -x "$SCRIPT_DIR/$script" ]]; then "$SCRIPT_DIR/$script" separator - wait_for_enter else log_error "Script not found or not executable: $script" exit 1 @@ -50,13 +49,13 @@ done print_header "Demo Complete" log_success "All steps completed successfully!" -echo "" log_info "What was demonstrated:" -echo " 1. Created Velero backup strategies for VMInstance and VMDisk" -echo " 2. Created BackupClass binding strategies to application types" -echo " 3. Provisioned a VMDisk and VMInstance" -echo " 4. Created a backup of the VMInstance via BackupJob" -echo " 5. Restored the VMInstance in-place" -echo " 6. Restored the VMInstance to a copy in a different namespace" -echo "" +echo " 1. Created Velero backup strategies for VMInstance and VMDisk" >&2 +echo " 2. Created BackupClass binding strategies to application types" >&2 +echo " 3. Provisioned a VMDisk and VMInstance" >&2 +echo " 4. Created a backup of the VMInstance via BackupJob" >&2 +echo " 5. Restored the VMInstance in-place" >&2 +if [[ "${SKIP_RESTORE_TO_COPY:-0}" != "1" ]]; then + echo " 6. Restored the VMInstance to a copy in a different namespace" >&2 +fi log_info "To clean up resources: ./cleanup.sh" diff --git a/hack/e2e-chainsaw/backup/README.md b/hack/e2e-chainsaw/backup/README.md deleted file mode 100644 index 4a7fe2a13..000000000 --- a/hack/e2e-chainsaw/backup/README.md +++ /dev/null @@ -1,21 +0,0 @@ -# Backup E2E (disabled / opt-in) - -`chainsaw-test.yaml.disabled` is a Chainsaw port of the former `hack/e2e-apps/backup.bats.disabled`. It boots a VirtualMachine, drives a `BackupJob` through the default Velero strategy, and verifies the resulting Velero backup landed in S3. - -It is **not** part of the automated suite — the `.disabled` suffix keeps it out of chainsaw's default `chainsaw-test.yaml` discovery, and `hack/select-e2e.sh` does not enumerate it as a selectable suite. This preserves the previous `backup.bats.disabled` behavior: the test is heavy (a full VM plus a Velero backup and S3 round-trip) and depends on backup infrastructure the standard e2e sandbox install does not guarantee. - -## Preconditions - -Run it only against a cozystack cluster that has backups enabled, i.e. with the `backup-controller`, `backupstrategy-controller`, and Velero installed, and a `velero-strategy-default` Velero strategy present (the install provides it). The test uses the `tenant-test` namespace, as configured in `hack/e2e-chainsaw/.chainsaw.yaml`. - -## Running - -```sh -chainsaw test --test-file chainsaw-test.yaml.disabled hack/e2e-chainsaw/backup -``` - -To enable it permanently, rename it to `chainsaw-test.yaml` — it then joins the full suite and becomes selectable by Test Impact Analysis. - -## Cleanup - -Chainsaw deletes the Bucket / VirtualMachine / BackupJob it applied. The Velero `Backup`, `BackupStorageLocation`, and credentials secret in the `cozy-velero` namespace are created by the backup-controller (not chainsaw), so prune them by hand after an opt-in run. diff --git a/hack/e2e-chainsaw/backup/chainsaw-test.yaml.disabled b/hack/e2e-chainsaw/backup/chainsaw-test.yaml.disabled deleted file mode 100644 index ee8b13c29..000000000 --- a/hack/e2e-chainsaw/backup/chainsaw-test.yaml.disabled +++ /dev/null @@ -1,245 +0,0 @@ -# Chainsaw port of hack/e2e-apps/backup.bats ("Create Backup for Virtual Machine"). -# -# DISABLED ON PURPOSE. The filename ends in `.disabled` so chainsaw's default -# test-file discovery (chainsaw-test.yaml) skips it and hack/select-e2e.sh does -# not enumerate `backup` as a selectable suite — exactly matching the previous -# `backup.bats.disabled` state. It is heavy (boots a VM, drives a Velero backup, -# verifies S3) and depends on backup infrastructure that the standard e2e -# sandbox install does not guarantee, so it stays opt-in. -# -# Run it by hand against a cozystack cluster that has the backup-controller, -# backupstrategy-controller and Velero installed (see README.md): -# chainsaw test --test-file chainsaw-test.yaml.disabled hack/e2e-chainsaw/backup -# -# Deviation from the bats: the bats re-applied the BackupJob/Velero CRDs and the -# default Velero strategy defensively (its Step 0). That belongs to the install, -# not the test — this port assumes `velero-strategy-default` already exists, as -# it does on any cluster with backups enabled. -apiVersion: chainsaw.kyverno.io/v1alpha1 -kind: Test -metadata: - name: backup -spec: - catch: - - events: {} - - describe: - apiVersion: backups.cozystack.io/v1alpha1 - kind: BackupJob - name: test-backup-job - - describe: - apiVersion: helm.toolkit.fluxcd.io/v2 - kind: HelmRelease - name: virtual-machine-test-backup-vm - - describe: - apiVersion: objectstorage.k8s.io/v1alpha1 - kind: BucketClaim - name: bucket-test-backup-bucket - steps: - # Step 1 — backing object store for the backup target. - - name: create-bucket - try: - - apply: - resource: - apiVersion: apps.cozystack.io/v1alpha1 - kind: Bucket - metadata: - name: test-backup-bucket - spec: {} - - assert: - timeout: 6m - resource: - apiVersion: helm.toolkit.fluxcd.io/v2 - kind: HelmRelease - metadata: - name: bucket-test-backup-bucket - status: - (conditions[?type == 'Ready']): - - status: "True" - - assert: - resource: - apiVersion: objectstorage.k8s.io/v1alpha1 - kind: BucketClaim - metadata: - name: bucket-test-backup-bucket - status: - bucketReady: true - - assert: - resource: - apiVersion: objectstorage.k8s.io/v1alpha1 - kind: BucketAccess - metadata: - name: bucket-test-backup-bucket - status: - accessGranted: true - - # Step 2 — the application to back up: a small Ubuntu VM. Mirrors the - # readiness chain of the vminstance suite (HR -> DataVolume -> PVC -> VM). - - name: create-vm - try: - - apply: - resource: - apiVersion: apps.cozystack.io/v1alpha1 - kind: VirtualMachine - metadata: - name: test-backup-vm - spec: - external: false - externalMethod: PortList - externalPorts: - - 22 - instanceType: "u1.medium" - instanceProfile: ubuntu - systemDisk: - image: ubuntu - storage: 5Gi - storageClass: replicated - gpus: [] - resources: {} - sshKeys: - - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPht0dPk5qQ+54g1hSX7A6AUxXJW5T6n/3d7Ga2F8gTF test@test - cloudInit: | - #cloud-config - users: - - name: test - shell: /bin/bash - sudo: ['ALL=(ALL) NOPASSWD: ALL'] - groups: sudo - ssh_authorized_keys: - - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPht0dPk5qQ+54g1hSX7A6AUxXJW5T6n/3d7Ga2F8gTF test@test - cloudInitSeed: "" - - assert: - timeout: 6m - resource: - apiVersion: helm.toolkit.fluxcd.io/v2 - kind: HelmRelease - metadata: - name: virtual-machine-test-backup-vm - status: - (conditions[?type == 'Ready']): - - status: "True" - - assert: - timeout: 5m - resource: - apiVersion: cdi.kubevirt.io/v1beta1 - kind: DataVolume - metadata: - name: virtual-machine-test-backup-vm - status: - (conditions[?type == 'Ready']): - - status: "True" - - assert: - timeout: 4m - resource: - apiVersion: v1 - kind: PersistentVolumeClaim - metadata: - name: virtual-machine-test-backup-vm - status: - phase: Bound - - assert: - timeout: 5m - resource: - apiVersion: kubevirt.io/v1 - kind: VirtualMachine - metadata: - name: virtual-machine-test-backup-vm - status: - (conditions[?type == 'Ready']): - - status: "True" - - # Step 3 — trigger the backup and assert it reaches Succeeded with a backupRef. - - name: create-backupjob - try: - - apply: - resource: - apiVersion: backups.cozystack.io/v1alpha1 - kind: BackupJob - metadata: - name: test-backup-job - labels: - backups.cozystack.io/triggered-by: e2e-test - spec: - applicationRef: - apiGroup: apps.cozystack.io - kind: VirtualMachine - name: test-backup-vm - storageRef: - apiGroup: apps.cozystack.io - kind: Bucket - name: test-backup-bucket - strategyRef: - apiGroup: strategy.backups.cozystack.io - kind: Velero - name: velero-strategy-default - - assert: - timeout: 6m - resource: - apiVersion: backups.cozystack.io/v1alpha1 - kind: BackupJob - metadata: - name: test-backup-job - status: - phase: Succeeded - (backupRef.name != null): true - - # Step 4 — confirm the underlying Velero Backup completed and that the - # backup objects actually landed in S3. The port-forward + mc verification is - # imperative, so it stays a single script step (same shape as the bucket - # suite); $NAMESPACE is the chainsaw-managed test namespace (tenant-test). - - name: verify-velero-backup-in-s3 - try: - - script: - timeout: 8m - content: | - set -eu - workdir=$(mktemp -d) - trap '[ -n "${pf_pid:-}" ] && kill "$pf_pid" 2>/dev/null || true; rm -rf "$workdir"' EXIT - - # Bucket S3 credentials for the verification client. - kubectl -n "$NAMESPACE" get secret bucket-test-backup-bucket \ - -o jsonpath='{.data.BucketInfo}' | base64 -d > "$workdir/bucket.json" - ACCESS_KEY=$(jq -r '.spec.secretS3.accessKeyID' "$workdir/bucket.json") - SECRET_KEY=$(jq -r '.spec.secretS3.accessSecretKey' "$workdir/bucket.json") - BUCKET_NAME=$(jq -r '.spec.bucketName' "$workdir/bucket.json") - - # Find the Velero backup created for this job. Velero names it - # "--"; wait for it to appear. - echo "Waiting for the Velero backup to appear..." - VELERO_BACKUP_NAME=$(timeout 120 sh -ec ' - while true; do - for b in $(kubectl -n cozy-velero get backups.velero.io \ - -o jsonpath="{.items[*].metadata.name}" 2>/dev/null); do - case "$b" in "'"$NAMESPACE"'-test-backup-job-"*) echo "$b"; exit 0;; esac - done - sleep 5 - done') - [ -n "$VELERO_BACKUP_NAME" ] - echo "Velero backup: $VELERO_BACKUP_NAME" - - # Wait for the Velero Backup to reach a terminal phase, then assert Completed. - timeout 300 sh -ec " - until kubectl -n cozy-velero get backups.velero.io '$VELERO_BACKUP_NAME' \ - -o jsonpath='{.status.phase}' 2>/dev/null | grep -qE 'Completed|Failed|PartiallyFailed'; do - sleep 5 - done" - PHASE=$(kubectl -n cozy-velero get backups.velero.io "$VELERO_BACKUP_NAME" \ - -o jsonpath='{.status.phase}') - [ "$PHASE" = "Completed" ] - - # Port-forward to the S3 endpoint; cleaned up by the trap. - kubectl port-forward service/seaweedfs-s3 -n tenant-root 8333:8333 >/dev/null 2>&1 & - pf_pid=$! - timeout 30 sh -ec 'until nc -z localhost 8333; do sleep 1; done' - - mc alias set backup-s3 https://localhost:8333 "$ACCESS_KEY" "$SECRET_KEY" --insecure - - # The backup directory must exist and hold at least the metadata objects. - BACKUP_PATH="$BUCKET_NAME/backups/$VELERO_BACKUP_NAME" - mc ls --insecure "backup-s3/$BACKUP_PATH/" - [ "$(mc ls --insecure "backup-s3/$BACKUP_PATH/" | wc -l)" -gt 0 ] - check: - ($error == null): true -# Chainsaw deletes the Bucket / VirtualMachine / BackupJob CRs it applied during -# cleanup. The Velero Backup + BackupStorageLocation + credentials secret in -# cozy-velero are created by the backup-controller, not chainsaw, so an opt-in -# run should prune them by hand afterwards (the bats teardown did this). diff --git a/hack/e2e-chainsaw/vminstance/chainsaw-test.yaml b/hack/e2e-chainsaw/vminstance/chainsaw-test.yaml index 9d1372775..691d45302 100644 --- a/hack/e2e-chainsaw/vminstance/chainsaw-test.yaml +++ b/hack/e2e-chainsaw/vminstance/chainsaw-test.yaml @@ -386,3 +386,358 @@ spec: echo "FAIL: a dry-run eviction recorded an evacuation on VMI $vmi" >&2 exit 1 fi +--- +# vminstance-2-backup-roundtrip — REAL Velero backup/restore round-trip for a +# booted VMInstance, run un-gated in CI on a deliberately tiny VM. +# +# This is the in-CI proof of the cozy-default-velero-vminstance path: a running +# VM's disk is CSI-snapshotted, moved to S3 by Velero's data mover (kopia), and +# restored in place — after which the VM boots again. Booting does not tell a +# restored disk from a blank one; the proof is on the disk PVC: a new uid, +# Velero's restore-name label and the -orig- PVC the rename leaves behind. +# It drives the validated example (examples/backups/vmi/run-all.sh) so the demo +# and the test can never drift. +# +# Kept light enough for kind: a cirros guest (~20MB image, instanceProfile +# cirros) on the smallest instance type (u1.nano = 1 vCPU / 512Mi) with a 2Gi +# disk, and SKIP_RESTORE_TO_COPY=1 so only the in-place restore runs (the +# cross-namespace to-copy restore stands up a second full VM — too heavy for CI +# and redundant for proving the round-trip). cirros has no qemu-guest-agent, so +# Velero takes a crash-consistent snapshot without a filesystem freeze, which is +# fine for exercising the mechanics. +# +# S3 target: like vmdisk-2-backup-roundtrip, the harness stands up a throwaway +# BSL cozy-e2e-vmi at the IN-CLUSTER seaweedfs endpoint (the platform +# cozy-default BSL derives the external ingress endpoint, unroutable in kind) +# with insecureSkipTLSVerify + checksumAlgorithm="" (the cozystack#3236 fixes), +# reusing the platform bucket + cozy-backups-creds. Runs in tenant-root so the +# data-mover DataUpload Pods reach seaweedfs over same-tenant egress. +# +# Teardown is explicit (skipDelete: true): cleanup.sh prunes the cluster-scoped +# BackupClass/strategies and the cozy-velero Velero Backup/Restore, and the +# harness deletes the throwaway BSL. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: vminstance-2-backup-roundtrip +spec: + namespace: tenant-root + skipDelete: true + catch: + - events: {} + - describe: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: vm-instance-test + - describe: + apiVersion: kubevirt.io/v1 + kind: VirtualMachineInstance + name: vm-instance-test + - describe: + apiVersion: backups.cozystack.io/v1alpha1 + kind: BackupJob + - describe: + apiVersion: backups.cozystack.io/v1alpha1 + kind: RestoreJob + - describe: + apiVersion: backups.cozystack.io/v1alpha1 + kind: Backup + - describe: + apiVersion: velero.io/v1 + kind: Restore + namespace: cozy-velero + - describe: + apiVersion: velero.io/v2alpha1 + kind: DataDownload + namespace: cozy-velero + - describe: + apiVersion: velero.io/v2alpha1 + kind: DataUpload + namespace: cozy-velero + # Velero server + node-agent (the CSI data mover) run in cozy-velero; a failed + # upload or a TLS handshake against seaweedfs leaves its error only there. + - podLogs: + namespace: cozy-velero + selector: app.kubernetes.io/name=velero + tail: 200 + steps: + - name: run + try: + # 40m budget: build the in-cluster BSL, import the tiny cirros image + boot + # the VM, a CSI-snapshot data-mover backup to S3, and an in-place restore + # with a second boot — all fast on a 2Gi cirros guest (realistic ~15m). BSL + # setup is in this step (not a separate one) so the finally below always tears + # the BSL down, even if setup fails. run-all.sh is `set -e` and fails fast on + # a terminal BackupJob/RestoreJob phase. The script waits are bounded (the + # IMPORT/BOOT/BACKUP/RESTORE_WAIT values below plus fixed ones in the example + # scripts) so their worst-case sum stays under this op timeout, and a hung + # wait trips the script's own named "Timeout waiting for ..." rather than a + # bare `signal: killed` (docs/agents/e2e-testing.md). Sized small on purpose: + # this suite is execution.parallel:1 and shares the job's 215m cap with every + # other Test in it and their finally blocks — an oversized op timeout lets the + # job cap fire first and lose the report before any op timeout can. + - script: + timeout: 40m + content: | + set -eu + # Build the in-cluster BSL the example will target (reuse the platform + # bucket read off cozy-default + cozy-backups-creds). + echo "Waiting for the platform cozy-default BSL to expose its bucket name..." + timeout 180 sh -ec 'until kubectl -n cozy-velero get bsl cozy-default -o jsonpath="{.spec.objectStorage.bucket}" 2>/dev/null | grep -q .; do sleep 5; done' + BUCKET=$(kubectl -n cozy-velero get bsl cozy-default -o jsonpath='{.spec.objectStorage.bucket}') + REGION=$(kubectl -n cozy-velero get bsl cozy-default -o jsonpath='{.spec.config.region}') + [ -n "$REGION" ] || REGION=us-east-1 + echo "Platform bucket: $BUCKET (region $REGION)" + kubectl apply -f - </dev/null)" = "Available" ]; do sleep 5; done' + kubectl -n cozy-velero get bsl cozy-e2e-vmi + + cd ../../.. + [ -x examples/backups/vmi/run-all.sh ] || { echo "vmi backup example scripts not found at examples/backups/vmi" >&2; exit 1; } + # Tiny VM: cirros guest on u1.nano with a 2Gi disk; in-place restore only. + # storageClass tracks the lane (COZY_E2E_STORAGE_CLASS, 'local' on the + # DRBD-off PR gate); 'replicated' is unrenderable there and never binds. + # Waits bounded well under the 40m op timeout above so a hung wait trips + # the script's named fail-fast, not Chainsaw's killer. + NAMESPACE="$NAMESPACE" BACKUP_STORAGE_LOCATION=cozy-e2e-vmi \ + VMI_DISK_STORAGE_CLASS="${COZY_E2E_STORAGE_CLASS:-replicated}" \ + VMI_IMAGE_URL=https://download.cirros-cloud.net/0.6.2/cirros-0.6.2-x86_64-disk.img \ + VMI_DISK_STORAGE=2Gi VMI_INSTANCE_PROFILE=cirros VMI_INSTANCE_TYPE=u1.nano \ + SKIP_RESTORE_TO_COPY=1 \ + IMPORT_WAIT=240 VMI_BOOT_WAIT=180 BACKUP_WAIT=300 RESTORE_WAIT=240 \ + examples/backups/vmi/run-all.sh + check: + ($error == null): true + finally: + # Prune everything the harness created — the example cleanup (cluster-scoped + # BackupClass/strategies, namespaced BackupJob/RestoreJob/Backup which cascade + # to the Velero Backup/Restore in cozy-velero, the VMInstance/VMDisk), then + # the throwaway BSL. Has teeth — a teardown that does not settle fails the step. + - description: vmi backup-flow cleanup + script: + timeout: 15m + content: | + set -eu + cd ../../.. + # Always tear down the harness-owned BSL, but keep a cleanup.sh failure + # fatal (convention 4): capture its rc, then tear down, then re-raise. + rc=0 + NAMESPACE="$NAMESPACE" examples/backups/vmi/cleanup.sh || rc=$? + # cleanup.sh deletes the cozystack Backup, whose finalizer asks Velero to + # delete the Velero Backup asynchronously. Deleting the BSL before that + # finishes makes Velero fail the DeleteBackupRequest ("backup storage + # location not found") and orphan the Velero Backup and its S3 objects. + # Drain the Velero Backups on this BSL first, then delete it. Branch on + # kubectl's exit status, not on empty stdout: a 403/timeout also yields + # no lines and must not read as "drained" (that would delete the BSL + # under the finalizer and orphan the objects this exists to protect). + timeout 180 sh -ec ' + while out=$(kubectl -n cozy-velero get backups.velero.io \ + -o custom-columns=BSL:.spec.storageLocation --no-headers); do + printf "%s\n" "$out" | grep -qx cozy-e2e-vmi || exit 0 + sleep 5 + done + exit 3' && drain_rc=0 || drain_rc=$? + if [ "$drain_rc" != 0 ]; then + echo "error: Velero backups on cozy-e2e-vmi not confirmed drained (rc=$drain_rc); keeping the BSL so they are not orphaned" >&2 + exit 1 + fi + kubectl -n cozy-velero delete bsl cozy-e2e-vmi --ignore-not-found + exit "$rc" +--- +# vmdisk-2-backup-roundtrip — REAL Velero backup/restore round-trip for a +# standalone VMDisk, run un-gated in CI. This is the in-CI proof that the +# cozy-default-velero-vmdisk path works end to end: a CSI VolumeSnapshot of the +# LINSTOR block disk (the lane's storage class, 'local' on the DRBD-off PR gate), +# moved to S3 by Velero's data mover (kopia), then restored back to a Bound disk. +# +# This is the disk-only companion to vminstance-2-backup-roundtrip above: same +# Velero + CSI-data-mover + S3 machinery, but it captures only HelmRelease + PVC +# (+ configmaps) with no VM at all — no nested boot, no guest-agent freeze — so +# it isolates the block-volume backup path from anything VM-specific. Both run +# un-gated in CI now. +# +# S3 target: the platform cozy-default BSL derives the EXTERNAL S3 ingress +# endpoint, which is an unroutable placeholder inside kind (the same reason the +# postgres round-trip targets the in-cluster endpoint). So the harness stands up +# a throwaway BSL `cozy-e2e-vmdisk` pointing at the IN-CLUSTER seaweedfs endpoint +# with insecureSkipTLSVerify (kind cannot trust the self-signed SeaweedFS CA) and +# checksumAlgorithm="" (the velero-plugin-for-aws v1.14/AWS-SDK-v2 vs SeaweedFS +# incompatibility) — i.e. the cozystack/cozystack#3236 fixes applied at the BSL. +# It reuses the platform bucket + cozy-backups-creds (a distinct prefix isolates +# the objects). The example scripts stay generic — they just consume +# BACKUP_STORAGE_LOCATION. +# +# Runs in tenant-root: Velero's data-mover DataUpload Pods run in the source +# namespace and reach tenant-root's seaweedfs over same-tenant egress. Teardown +# is explicit (skipDelete: true) — cleanup.sh prunes the cluster-scoped +# BackupClass/strategy and the cozy-velero Velero Backup/Restore, and the harness +# deletes the throwaway BSL. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: vmdisk-2-backup-roundtrip +spec: + namespace: tenant-root + skipDelete: true + catch: + - events: {} + - describe: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: vm-disk-backup-src + - describe: + apiVersion: backups.cozystack.io/v1alpha1 + kind: BackupJob + - describe: + apiVersion: backups.cozystack.io/v1alpha1 + kind: RestoreJob + - describe: + apiVersion: backups.cozystack.io/v1alpha1 + kind: Backup + - describe: + apiVersion: velero.io/v1 + kind: Restore + namespace: cozy-velero + - describe: + apiVersion: velero.io/v2alpha1 + kind: DataDownload + namespace: cozy-velero + - describe: + apiVersion: velero.io/v2alpha1 + kind: DataUpload + namespace: cozy-velero + # The CSI snapshot + data-mover work lands in these Velero CRs and the + # node-agent/velero Pod logs; dump them on failure. + - podLogs: + namespace: cozy-velero + selector: app.kubernetes.io/name=velero + tail: 200 + steps: + - name: run + try: + # 40m budget: build the in-cluster BSL, import the Ubuntu image into a 5Gi + # disk (~150s proven by the vmdisk test above), a CSI-snapshot data-mover + # backup to S3, and a restore. run-all.sh is `set -e` and fails fast on a + # terminal BackupJob/RestoreJob phase rather than polling the budget out. As + # with the vminstance step above, the worst-case sum of the script waits + # (the BSL waits, IMPORT/BACKUP/RESTORE_WAIT below and the fixed ones in the + # example scripts, the consumer Pod delete included) is 38m, under this op + # timeout, so a hung wait trips the script's own named fail-fast, not + # Chainsaw's bare `signal: killed`; and + # it is sized small on purpose, since this suite (execution.parallel:1) shares + # the 215m job cap with the other Tests and the finally blocks — an oversized + # op timeout lets the cap fire first and lose the report. BSL setup is in this + # step (not a separate one) so the finally below always tears the BSL down. + - script: + timeout: 40m + content: | + set -eu + # Build the in-cluster BSL the example will target. Reuse the platform + # bucket (read its COSI-assigned name off cozy-default, created once + # the platform BucketClaim reconciles) + cozy-backups-creds. + echo "Waiting for the platform cozy-default BSL to expose its bucket name..." + timeout 180 sh -ec 'until kubectl -n cozy-velero get bsl cozy-default -o jsonpath="{.spec.objectStorage.bucket}" 2>/dev/null | grep -q .; do sleep 5; done' + BUCKET=$(kubectl -n cozy-velero get bsl cozy-default -o jsonpath='{.spec.objectStorage.bucket}') + REGION=$(kubectl -n cozy-velero get bsl cozy-default -o jsonpath='{.spec.config.region}') + [ -n "$REGION" ] || REGION=us-east-1 + echo "Platform bucket: $BUCKET (region $REGION)" + kubectl apply -f - </dev/null)" = "Available" ]; do + sleep 5 + done' + kubectl -n cozy-velero get bsl cozy-e2e-vmdisk + + cd ../../.. + [ -x examples/backups/vmdisk/run-all.sh ] || { echo "vmdisk backup example scripts not found at examples/backups/vmdisk" >&2; exit 1; } + # storageClass tracks the lane (COZY_E2E_STORAGE_CLASS, 'local' on the + # DRBD-off PR gate); waits bounded under the 40m op timeout above. + NAMESPACE="$NAMESPACE" BACKUP_STORAGE_LOCATION=cozy-e2e-vmdisk \ + VMI_DISK_STORAGE_CLASS="${COZY_E2E_STORAGE_CLASS:-replicated}" \ + IMPORT_WAIT=240 BACKUP_WAIT=300 RESTORE_WAIT=300 \ + examples/backups/vmdisk/run-all.sh + check: + ($error == null): true + finally: + # Prune the harness-created resources: the example cleanup (cluster-scoped + # BackupClass/strategy, namespaced BackupJob/RestoreJob/Backup which cascade + # to the Velero Backup/Restore in cozy-velero, the VMDisk and any renamed + # original PVC), then the throwaway BSL. Has teeth — a teardown that does not + # settle fails the step. + - description: vmdisk backup-flow cleanup + script: + timeout: 15m + content: | + set -eu + cd ../../.. + # Always tear down the harness-owned BSL, but keep a cleanup.sh failure + # fatal (convention 4): capture its rc, then tear down, then re-raise. + rc=0 + NAMESPACE="$NAMESPACE" examples/backups/vmdisk/cleanup.sh || rc=$? + # Drain the Velero Backups on this BSL before deleting it, or Velero + # fails the finalizer's DeleteBackupRequest ("backup storage location not + # found") and orphans the Velero Backup and its S3 objects. Branch on + # kubectl's exit status, not empty stdout (a 403/timeout must not read as + # "drained"). + timeout 180 sh -ec ' + while out=$(kubectl -n cozy-velero get backups.velero.io \ + -o custom-columns=BSL:.spec.storageLocation --no-headers); do + printf "%s\n" "$out" | grep -qx cozy-e2e-vmdisk || exit 0 + sleep 5 + done + exit 3' && drain_rc=0 || drain_rc=$? + if [ "$drain_rc" != 0 ]; then + echo "error: Velero backups on cozy-e2e-vmdisk not confirmed drained (rc=$drain_rc); keeping the BSL so they are not orphaned" >&2 + exit 1 + fi + kubectl -n cozy-velero delete bsl cozy-e2e-vmdisk --ignore-not-found + exit "$rc"