Files
cozystack/hack/e2e-chainsaw/_lib/run-proxmox.sh
T
Marian Koreniuk 3d97e912b4 fix(e2e): drop the job-level runner.temp and pin the tenant kubeconfig handling
jobs.e2e.env set COZY_PROXMOX_WORKDIR from runner.temp, but the runner
context is not available in job-level env, so GitHub would refuse the
whole workflow at load time and the Proxmox lane would stop starting.
actionlint reports it as `context "runner" is not allowed here`.

Drop the variable. run-proxmox.sh already falls back to RUNNER_TEMP,
which the runner exports into every step, and the cleanup step removes
the file from there.

Pin the behaviour the previous commit added, none of which a test
covered: the fetched kubeconfig carries no group or other bits under
umask 022, apply-tenant removes a file an earlier run left, and the
suite's finally removes the file on the keep_tenant path, so diagnostics
after it prints no tenant section. Each case goes red with its line
removed.

Signed-off-by: Marian Koreniuk <moriarti@cp.if.ua>
Assisted-by: LLM
2026-10-28 18:49:12 +01:00

260 lines
12 KiB
Bash
Executable File

#!/bin/sh
# Imperative half of the kubernetes-proxmox suite.
#
# Chainsaw's declarative asserts cover everything that is a Kubernetes object in
# THIS cluster, and the suite uses them for exactly that. What is left needs a
# shell: the hypervisor is reached over SSH and answers in `qm` output, and the
# tenant cluster is reached with a kubeconfig that only exists once its control
# plane is up. Those are the two things below.
#
# Deliberately not a port of _lib/run-kubernetes.sh. That script's diagnostic
# half is virt-launcher Pods, VirtualMachineInstances and guest consoles —
# objects with no counterpart on an external hypervisor — so sharing it would
# mean carrying a thousand lines that can never fire here.
#
# Every subcommand exits non-zero on failure and says which check failed. There
# is no `|| echo` anywhere in this file on purpose: the pattern turns a failed
# command into a green run, and this repository has been bitten by it.
set -eu
NS="${COZY_PROXMOX_NS:-tenant-e2eproxmox}"
CLUSTER="${COZY_PROXMOX_CLUSTER:-k8s-e2e}"
PVE_SSH="${COZY_PVE_SSH:?COZY_PVE_SSH is required: the ssh destination of the Proxmox host, user@host}"
PVE_SSH_KEY="${COZY_PVE_SSH_KEY:-$HOME/.ssh/id_ed25519}"
STORAGE="${COZY_PROXMOX_STORAGE:?COZY_PROXMOX_STORAGE is required, the PVE storage id for the PVC probe}"
# Where this run keeps the tenant's super-admin kubeconfig. Per run rather than
# a fixed path under /tmp: the file is a credential for the whole tenant, so it
# must not outlive the run on a shared runner, and dump_diagnostics reads
# whatever is at this path, so a file left by a previous run would be reported
# as this run's tenant. On the workflow's runner this is RUNNER_TEMP, which the
# runner exports into every step and empties between jobs; apply-tenant removes
# a leftover before it does anything else, and the suite's finally removes the
# file when the run ends.
WORKDIR="${COZY_PROXMOX_WORKDIR:-${RUNNER_TEMP:-${TMPDIR:-/tmp}}}"
TENANT_KUBECONFIG="$WORKDIR/proxmox-e2e-tenant.kubeconfig"
FIXTURE="$(cd "$(dirname "$0")/.." && pwd)/kubernetes-proxmox/tenant.yaml"
log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; }
die() { printf '[%s] FAILED: %s\n' "$(date -u +%H:%M:%S)" "$*" >&2; exit 1; }
pve() { ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new -i "$PVE_SSH_KEY" "$PVE_SSH" "$@"; }
# The Machine carries the VM id in the ProxmoxMachine it references. Reading it
# from the cluster rather than guessing keeps this working when a replacement
# lands on a different id.
vmid_of_machine() {
kubectl -n "$NS" get proxmoxmachine -o jsonpath='{.items[0].spec.virtualMachineID}'
}
# --- checks -----------------------------------------------------------------
# A full clone is a copy of the template's disk; a linked clone references it and
# stores only the increment. Nothing fails when this is wrong — the pool comes
# up either way — so the only place it can be caught is here, by the shape of the
# volume name: a linked clone points at `base-<template>-disk-N/`.
check_linked_clone() {
vmid=$(vmid_of_machine)
[ -n "$vmid" ] || die "no ProxmoxMachine with a virtualMachineID in $NS"
log "VM $vmid: checking the boot disk is a linked clone"
bootdisk=$(pve "qm config $vmid" | sed -n 's/^scsi0: *//p')
[ -n "$bootdisk" ] || die "VM $vmid has no scsi0"
case "$bootdisk" in
*:base-*-disk-*/*) log "linked clone confirmed: $bootdisk" ;;
*) die "VM $vmid boot disk is a full copy, not a linked clone: $bootdisk" ;;
esac
}
# The fixture is stand-agnostic through ${COZY_PVE_*} variables. GNU envsubst
# expands ${VAR} and nothing else — a ${VAR:-default} stays in the text as it
# is, set or not — so the defaults live here, the variable list is explicit so
# nothing else in the file is touched, and a fixture with a variable left in it
# is refused rather than applied.
render_tenant() {
: "${COZY_PVE_ADDRESSES:=10.0.0.180-10.0.0.185}"
: "${COZY_PVE_GATEWAY:=10.0.0.1}"
: "${COZY_PVE_PREFIX:=24}"
: "${COZY_PVE_BRIDGE:=vmbr50}"
: "${COZY_PVE_POOL:=}"
export COZY_PVE_ADDRESSES COZY_PVE_GATEWAY COZY_PVE_PREFIX COZY_PVE_BRIDGE COZY_PVE_POOL COZY_PROXMOX_STORAGE
rendered=$(envsubst '$COZY_PVE_ADDRESSES $COZY_PVE_GATEWAY $COZY_PVE_PREFIX $COZY_PVE_BRIDGE $COZY_PVE_POOL $COZY_PROXMOX_STORAGE' < "$FIXTURE")
case "$rendered" in
*'${'*) die "tenant.yaml still carries an unexpanded variable after envsubst" ;;
esac
printf '%s\n' "$rendered"
}
apply_tenant() {
# First step of the suite, so this is where a kubeconfig left by an earlier
# run on this runner is removed: from here on the path holds this run's file
# or nothing, and dump_diagnostics cannot pick up another tenant.
rm -f "$TENANT_KUBECONFIG"
render_tenant | kubectl apply -f -
}
# The tenant kubeconfig exists only after Kamaji has issued it. Fetching it is
# also the first proof that the control plane came up at all. The .conf key
# carries the address a runner outside the management cluster can reach; .svc
# is the in-cluster Service the management-side controllers use.
fetch_tenant_kubeconfig() {
# Written under umask 077 in a subshell of its own: the workflow sets that
# umask only in its staging step, and a chainsaw script step runs with the
# runner's default, under which the file would be readable by every local
# user. No pipefail in POSIX sh, so a failed read leaves an empty file, which
# the size check below turns into a failure rather than an empty kubeconfig.
(
umask 077
kubectl -n "$NS" get secret "kubernetes-${CLUSTER}-admin-kubeconfig" \
-o jsonpath='{.data.super-admin\.conf}' | base64 -d > "$TENANT_KUBECONFIG"
)
[ -s "$TENANT_KUBECONFIG" ] || die "tenant kubeconfig is empty — did the control plane come up?"
}
# providerID and the topology labels are applied by the cloud-controller-manager,
# and the CSI node plugin refuses to start without the labels. Asserting them
# here is what separates "the node joined" from "the node is usable".
check_tenant_node() {
fetch_tenant_kubeconfig
log "checking the tenant node is Ready and initialised by the CCM"
node=$(KUBECONFIG="$TENANT_KUBECONFIG" kubectl get nodes -o jsonpath='{.items[0].metadata.name}')
[ -n "$node" ] || die "tenant cluster has no nodes"
provider=$(KUBECONFIG="$TENANT_KUBECONFIG" kubectl get node "$node" -o jsonpath='{.spec.providerID}')
region=$(KUBECONFIG="$TENANT_KUBECONFIG" kubectl get node "$node" \
-o jsonpath='{.metadata.labels.topology\.kubernetes\.io/region}')
[ -n "$provider" ] || die "node $node has no providerID — the CCM did not initialise it"
[ -n "$region" ] || die "node $node has no topology region label — the CSI node plugin cannot start"
log "node $node: providerID=$provider region=$region"
}
# Bound is not proof: a StorageClass can bind a volume that never becomes a disk.
# The disk on the hypervisor is the proof, so the check reads `qm config`, then
# removes what it created and reads the storage again: a deleted claim whose disk
# is still there is the leak the pre-delete hook's Step 1b exists to prevent.
check_pvc_round_trip() {
fetch_tenant_kubeconfig
vmid=$(vmid_of_machine)
log "binding a PVC and looking for its disk on VM $vmid"
KUBECONFIG="$TENANT_KUBECONFIG" kubectl apply -f - <<'EOF'
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: e2e-csi-probe
spec:
accessModes: [ReadWriteOnce]
storageClassName: proxmox
resources:
requests:
storage: 1Gi
---
apiVersion: v1
kind: Pod
metadata:
name: e2e-csi-probe
spec:
containers:
- name: c
image: busybox:1.36
command: ["sh", "-c", "sleep 600"]
volumeMounts:
- name: v
mountPath: /data
volumes:
- name: v
persistentVolumeClaim:
claimName: e2e-csi-probe
EOF
KUBECONFIG="$TENANT_KUBECONFIG" kubectl wait --for=condition=Ready pod/e2e-csi-probe --timeout=5m \
|| die "the probe Pod never became Ready — the volume did not attach"
disk=$(pve "qm config $vmid" | sed -n "s/^scsi[1-9]: *\($STORAGE:[^,]*\),.*/\1/p" | head -1)
if [ -z "$disk" ]; then
pve "qm config $vmid" | sed 's/^/ qm-config: /' >&2
die "PVC bound but no disk is attached to VM $vmid on storage $STORAGE"
fi
log "disk attached on the hypervisor: $disk"
KUBECONFIG="$TENANT_KUBECONFIG" kubectl delete pod e2e-csi-probe --wait=true
KUBECONFIG="$TENANT_KUBECONFIG" kubectl delete pvc e2e-csi-probe --wait=true
wait_disk_freed "$disk"
}
# The disk counts as freed only when a listing that succeeded no longer shows
# it. A listing that fails — ssh down, pvesm erroring — proves nothing, so it
# fails the check instead of reading as an empty storage.
wait_disk_freed() {
disk=$1
volid=${disk#*:}
deadline=$(( $(date +%s) + 180 ))
while :; do
listing=$(pve "pvesm list $STORAGE") \
|| die "cannot list $STORAGE on the hypervisor, so cannot tell whether $disk was freed"
printf '%s\n' "$listing" | grep -qF "$volid" || break
[ "$(date +%s)" -lt "$deadline" ] || die "PVC deleted but $disk is still on $STORAGE: the driver did not free it"
sleep 5
done
log "disk $disk is gone from $STORAGE"
}
# A replaced Machine has to converge without an operator: the departed Node must
# be removed by the cloud-controller-manager and the replacement must register
# in its place. "Without intervention" is the whole assertion, so nothing here
# touches the cluster beyond the initial delete.
check_replacement_converges() {
deadline=$(( $(date +%s) + 900 ))
old=$(kubectl -n "$NS" get machine -o jsonpath='{.items[0].metadata.name}')
[ -n "$old" ] || die "no Machine to replace in $NS"
log "deleting Machine $old and waiting for an unattended replacement"
kubectl -n "$NS" delete machine "$old" --wait=false
while [ "$(date +%s)" -lt "$deadline" ]; do
name=$(kubectl -n "$NS" get machine -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
phase=$(kubectl -n "$NS" get machine -o jsonpath='{.items[0].status.phase}' 2>/dev/null || true)
noderef=$(kubectl -n "$NS" get machine -o jsonpath='{.items[0].status.nodeRef.name}' 2>/dev/null || true)
if [ -n "$name" ] && [ "$name" != "$old" ] && [ "$phase" = "Running" ] && [ -n "$noderef" ]; then
log "replacement $name reached Running with nodeRef $noderef"
return 0
fi
sleep 15
done
kubectl -n "$NS" get machine,machinedeployment -o wide >&2 || true
die "no replacement reached Running with a nodeRef within 15m: the departed Node was not removed, or the replacement never registered"
}
# Runs from the suite's catch. Cheap reads first, so a truncated dump still
# carries the ones that discriminate between failure modes.
dump_diagnostics() {
log "=== Machines and their infrastructure ==="
kubectl -n "$NS" get machine,proxmoxmachine,machinedeployment,proxmoxcluster -o wide || true
log "=== capmox controller ==="
kubectl -n cozy-cluster-api logs -l control-plane=controller-manager --tail=80 \
--selector=cluster.x-k8s.io/provider=infrastructure-proxmox || true
log "=== VMs on the hypervisor ==="
pve "qm list" || true
if [ -s "$TENANT_KUBECONFIG" ]; then
log "=== tenant nodes and pods ==="
KUBECONFIG="$TENANT_KUBECONFIG" kubectl get nodes -o wide || true
KUBECONFIG="$TENANT_KUBECONFIG" kubectl get pods -A -o wide || true
fi
}
case "${1:-}" in
render-tenant) render_tenant ;;
apply-tenant) apply_tenant ;;
linked-clone) check_linked_clone ;;
tenant-node) check_tenant_node ;;
pvc-round-trip) check_pvc_round_trip ;;
disk-freed) wait_disk_freed "${2:?disk-freed needs <storage>:<volume>}" ;;
replacement) check_replacement_converges ;;
diagnostics) dump_diagnostics ;;
*) die "usage: $0 {render-tenant|apply-tenant|linked-clone|tenant-node|pvc-round-trip|disk-freed <storage>:<volume>|replacement|diagnostics}" ;;
esac