mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
jobs.e2e.env set COZY_PROXMOX_WORKDIR from runner.temp, but the runner context is not available in job-level env, so GitHub would refuse the whole workflow at load time and the Proxmox lane would stop starting. actionlint reports it as `context "runner" is not allowed here`. Drop the variable. run-proxmox.sh already falls back to RUNNER_TEMP, which the runner exports into every step, and the cleanup step removes the file from there. Pin the behaviour the previous commit added, none of which a test covered: the fetched kubeconfig carries no group or other bits under umask 022, apply-tenant removes a file an earlier run left, and the suite's finally removes the file on the keep_tenant path, so diagnostics after it prints no tenant section. Each case goes red with its line removed. Signed-off-by: Marian Koreniuk <moriarti@cp.if.ua> Assisted-by: LLM
260 lines
12 KiB
Bash
Executable File
260 lines
12 KiB
Bash
Executable File
#!/bin/sh
|
|
# Imperative half of the kubernetes-proxmox suite.
|
|
#
|
|
# Chainsaw's declarative asserts cover everything that is a Kubernetes object in
|
|
# THIS cluster, and the suite uses them for exactly that. What is left needs a
|
|
# shell: the hypervisor is reached over SSH and answers in `qm` output, and the
|
|
# tenant cluster is reached with a kubeconfig that only exists once its control
|
|
# plane is up. Those are the two things below.
|
|
#
|
|
# Deliberately not a port of _lib/run-kubernetes.sh. That script's diagnostic
|
|
# half is virt-launcher Pods, VirtualMachineInstances and guest consoles —
|
|
# objects with no counterpart on an external hypervisor — so sharing it would
|
|
# mean carrying a thousand lines that can never fire here.
|
|
#
|
|
# Every subcommand exits non-zero on failure and says which check failed. There
|
|
# is no `|| echo` anywhere in this file on purpose: the pattern turns a failed
|
|
# command into a green run, and this repository has been bitten by it.
|
|
set -eu
|
|
|
|
NS="${COZY_PROXMOX_NS:-tenant-e2eproxmox}"
|
|
CLUSTER="${COZY_PROXMOX_CLUSTER:-k8s-e2e}"
|
|
PVE_SSH="${COZY_PVE_SSH:?COZY_PVE_SSH is required: the ssh destination of the Proxmox host, user@host}"
|
|
PVE_SSH_KEY="${COZY_PVE_SSH_KEY:-$HOME/.ssh/id_ed25519}"
|
|
STORAGE="${COZY_PROXMOX_STORAGE:?COZY_PROXMOX_STORAGE is required, the PVE storage id for the PVC probe}"
|
|
# Where this run keeps the tenant's super-admin kubeconfig. Per run rather than
|
|
# a fixed path under /tmp: the file is a credential for the whole tenant, so it
|
|
# must not outlive the run on a shared runner, and dump_diagnostics reads
|
|
# whatever is at this path, so a file left by a previous run would be reported
|
|
# as this run's tenant. On the workflow's runner this is RUNNER_TEMP, which the
|
|
# runner exports into every step and empties between jobs; apply-tenant removes
|
|
# a leftover before it does anything else, and the suite's finally removes the
|
|
# file when the run ends.
|
|
WORKDIR="${COZY_PROXMOX_WORKDIR:-${RUNNER_TEMP:-${TMPDIR:-/tmp}}}"
|
|
TENANT_KUBECONFIG="$WORKDIR/proxmox-e2e-tenant.kubeconfig"
|
|
FIXTURE="$(cd "$(dirname "$0")/.." && pwd)/kubernetes-proxmox/tenant.yaml"
|
|
|
|
log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; }
|
|
die() { printf '[%s] FAILED: %s\n' "$(date -u +%H:%M:%S)" "$*" >&2; exit 1; }
|
|
|
|
pve() { ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new -i "$PVE_SSH_KEY" "$PVE_SSH" "$@"; }
|
|
|
|
# The Machine carries the VM id in the ProxmoxMachine it references. Reading it
|
|
# from the cluster rather than guessing keeps this working when a replacement
|
|
# lands on a different id.
|
|
vmid_of_machine() {
|
|
kubectl -n "$NS" get proxmoxmachine -o jsonpath='{.items[0].spec.virtualMachineID}'
|
|
}
|
|
|
|
# --- checks -----------------------------------------------------------------
|
|
|
|
# A full clone is a copy of the template's disk; a linked clone references it and
|
|
# stores only the increment. Nothing fails when this is wrong — the pool comes
|
|
# up either way — so the only place it can be caught is here, by the shape of the
|
|
# volume name: a linked clone points at `base-<template>-disk-N/`.
|
|
check_linked_clone() {
|
|
vmid=$(vmid_of_machine)
|
|
[ -n "$vmid" ] || die "no ProxmoxMachine with a virtualMachineID in $NS"
|
|
log "VM $vmid: checking the boot disk is a linked clone"
|
|
|
|
bootdisk=$(pve "qm config $vmid" | sed -n 's/^scsi0: *//p')
|
|
[ -n "$bootdisk" ] || die "VM $vmid has no scsi0"
|
|
|
|
case "$bootdisk" in
|
|
*:base-*-disk-*/*) log "linked clone confirmed: $bootdisk" ;;
|
|
*) die "VM $vmid boot disk is a full copy, not a linked clone: $bootdisk" ;;
|
|
esac
|
|
}
|
|
|
|
# The fixture is stand-agnostic through ${COZY_PVE_*} variables. GNU envsubst
|
|
# expands ${VAR} and nothing else — a ${VAR:-default} stays in the text as it
|
|
# is, set or not — so the defaults live here, the variable list is explicit so
|
|
# nothing else in the file is touched, and a fixture with a variable left in it
|
|
# is refused rather than applied.
|
|
render_tenant() {
|
|
: "${COZY_PVE_ADDRESSES:=10.0.0.180-10.0.0.185}"
|
|
: "${COZY_PVE_GATEWAY:=10.0.0.1}"
|
|
: "${COZY_PVE_PREFIX:=24}"
|
|
: "${COZY_PVE_BRIDGE:=vmbr50}"
|
|
: "${COZY_PVE_POOL:=}"
|
|
export COZY_PVE_ADDRESSES COZY_PVE_GATEWAY COZY_PVE_PREFIX COZY_PVE_BRIDGE COZY_PVE_POOL COZY_PROXMOX_STORAGE
|
|
rendered=$(envsubst '$COZY_PVE_ADDRESSES $COZY_PVE_GATEWAY $COZY_PVE_PREFIX $COZY_PVE_BRIDGE $COZY_PVE_POOL $COZY_PROXMOX_STORAGE' < "$FIXTURE")
|
|
case "$rendered" in
|
|
*'${'*) die "tenant.yaml still carries an unexpanded variable after envsubst" ;;
|
|
esac
|
|
printf '%s\n' "$rendered"
|
|
}
|
|
|
|
apply_tenant() {
|
|
# First step of the suite, so this is where a kubeconfig left by an earlier
|
|
# run on this runner is removed: from here on the path holds this run's file
|
|
# or nothing, and dump_diagnostics cannot pick up another tenant.
|
|
rm -f "$TENANT_KUBECONFIG"
|
|
render_tenant | kubectl apply -f -
|
|
}
|
|
|
|
# The tenant kubeconfig exists only after Kamaji has issued it. Fetching it is
|
|
# also the first proof that the control plane came up at all. The .conf key
|
|
# carries the address a runner outside the management cluster can reach; .svc
|
|
# is the in-cluster Service the management-side controllers use.
|
|
fetch_tenant_kubeconfig() {
|
|
# Written under umask 077 in a subshell of its own: the workflow sets that
|
|
# umask only in its staging step, and a chainsaw script step runs with the
|
|
# runner's default, under which the file would be readable by every local
|
|
# user. No pipefail in POSIX sh, so a failed read leaves an empty file, which
|
|
# the size check below turns into a failure rather than an empty kubeconfig.
|
|
(
|
|
umask 077
|
|
kubectl -n "$NS" get secret "kubernetes-${CLUSTER}-admin-kubeconfig" \
|
|
-o jsonpath='{.data.super-admin\.conf}' | base64 -d > "$TENANT_KUBECONFIG"
|
|
)
|
|
[ -s "$TENANT_KUBECONFIG" ] || die "tenant kubeconfig is empty — did the control plane come up?"
|
|
}
|
|
|
|
# providerID and the topology labels are applied by the cloud-controller-manager,
|
|
# and the CSI node plugin refuses to start without the labels. Asserting them
|
|
# here is what separates "the node joined" from "the node is usable".
|
|
check_tenant_node() {
|
|
fetch_tenant_kubeconfig
|
|
log "checking the tenant node is Ready and initialised by the CCM"
|
|
|
|
node=$(KUBECONFIG="$TENANT_KUBECONFIG" kubectl get nodes -o jsonpath='{.items[0].metadata.name}')
|
|
[ -n "$node" ] || die "tenant cluster has no nodes"
|
|
|
|
provider=$(KUBECONFIG="$TENANT_KUBECONFIG" kubectl get node "$node" -o jsonpath='{.spec.providerID}')
|
|
region=$(KUBECONFIG="$TENANT_KUBECONFIG" kubectl get node "$node" \
|
|
-o jsonpath='{.metadata.labels.topology\.kubernetes\.io/region}')
|
|
|
|
[ -n "$provider" ] || die "node $node has no providerID — the CCM did not initialise it"
|
|
[ -n "$region" ] || die "node $node has no topology region label — the CSI node plugin cannot start"
|
|
log "node $node: providerID=$provider region=$region"
|
|
}
|
|
|
|
# Bound is not proof: a StorageClass can bind a volume that never becomes a disk.
|
|
# The disk on the hypervisor is the proof, so the check reads `qm config`, then
|
|
# removes what it created and reads the storage again: a deleted claim whose disk
|
|
# is still there is the leak the pre-delete hook's Step 1b exists to prevent.
|
|
check_pvc_round_trip() {
|
|
fetch_tenant_kubeconfig
|
|
vmid=$(vmid_of_machine)
|
|
log "binding a PVC and looking for its disk on VM $vmid"
|
|
|
|
KUBECONFIG="$TENANT_KUBECONFIG" kubectl apply -f - <<'EOF'
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata:
|
|
name: e2e-csi-probe
|
|
spec:
|
|
accessModes: [ReadWriteOnce]
|
|
storageClassName: proxmox
|
|
resources:
|
|
requests:
|
|
storage: 1Gi
|
|
---
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata:
|
|
name: e2e-csi-probe
|
|
spec:
|
|
containers:
|
|
- name: c
|
|
image: busybox:1.36
|
|
command: ["sh", "-c", "sleep 600"]
|
|
volumeMounts:
|
|
- name: v
|
|
mountPath: /data
|
|
volumes:
|
|
- name: v
|
|
persistentVolumeClaim:
|
|
claimName: e2e-csi-probe
|
|
EOF
|
|
|
|
KUBECONFIG="$TENANT_KUBECONFIG" kubectl wait --for=condition=Ready pod/e2e-csi-probe --timeout=5m \
|
|
|| die "the probe Pod never became Ready — the volume did not attach"
|
|
|
|
disk=$(pve "qm config $vmid" | sed -n "s/^scsi[1-9]: *\($STORAGE:[^,]*\),.*/\1/p" | head -1)
|
|
if [ -z "$disk" ]; then
|
|
pve "qm config $vmid" | sed 's/^/ qm-config: /' >&2
|
|
die "PVC bound but no disk is attached to VM $vmid on storage $STORAGE"
|
|
fi
|
|
log "disk attached on the hypervisor: $disk"
|
|
|
|
KUBECONFIG="$TENANT_KUBECONFIG" kubectl delete pod e2e-csi-probe --wait=true
|
|
KUBECONFIG="$TENANT_KUBECONFIG" kubectl delete pvc e2e-csi-probe --wait=true
|
|
|
|
wait_disk_freed "$disk"
|
|
}
|
|
|
|
# The disk counts as freed only when a listing that succeeded no longer shows
|
|
# it. A listing that fails — ssh down, pvesm erroring — proves nothing, so it
|
|
# fails the check instead of reading as an empty storage.
|
|
wait_disk_freed() {
|
|
disk=$1
|
|
volid=${disk#*:}
|
|
deadline=$(( $(date +%s) + 180 ))
|
|
while :; do
|
|
listing=$(pve "pvesm list $STORAGE") \
|
|
|| die "cannot list $STORAGE on the hypervisor, so cannot tell whether $disk was freed"
|
|
printf '%s\n' "$listing" | grep -qF "$volid" || break
|
|
[ "$(date +%s)" -lt "$deadline" ] || die "PVC deleted but $disk is still on $STORAGE: the driver did not free it"
|
|
sleep 5
|
|
done
|
|
log "disk $disk is gone from $STORAGE"
|
|
}
|
|
|
|
# A replaced Machine has to converge without an operator: the departed Node must
|
|
# be removed by the cloud-controller-manager and the replacement must register
|
|
# in its place. "Without intervention" is the whole assertion, so nothing here
|
|
# touches the cluster beyond the initial delete.
|
|
check_replacement_converges() {
|
|
deadline=$(( $(date +%s) + 900 ))
|
|
old=$(kubectl -n "$NS" get machine -o jsonpath='{.items[0].metadata.name}')
|
|
[ -n "$old" ] || die "no Machine to replace in $NS"
|
|
log "deleting Machine $old and waiting for an unattended replacement"
|
|
|
|
kubectl -n "$NS" delete machine "$old" --wait=false
|
|
|
|
while [ "$(date +%s)" -lt "$deadline" ]; do
|
|
name=$(kubectl -n "$NS" get machine -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
|
|
phase=$(kubectl -n "$NS" get machine -o jsonpath='{.items[0].status.phase}' 2>/dev/null || true)
|
|
noderef=$(kubectl -n "$NS" get machine -o jsonpath='{.items[0].status.nodeRef.name}' 2>/dev/null || true)
|
|
if [ -n "$name" ] && [ "$name" != "$old" ] && [ "$phase" = "Running" ] && [ -n "$noderef" ]; then
|
|
log "replacement $name reached Running with nodeRef $noderef"
|
|
return 0
|
|
fi
|
|
sleep 15
|
|
done
|
|
|
|
kubectl -n "$NS" get machine,machinedeployment -o wide >&2 || true
|
|
die "no replacement reached Running with a nodeRef within 15m: the departed Node was not removed, or the replacement never registered"
|
|
}
|
|
|
|
# Runs from the suite's catch. Cheap reads first, so a truncated dump still
|
|
# carries the ones that discriminate between failure modes.
|
|
dump_diagnostics() {
|
|
log "=== Machines and their infrastructure ==="
|
|
kubectl -n "$NS" get machine,proxmoxmachine,machinedeployment,proxmoxcluster -o wide || true
|
|
log "=== capmox controller ==="
|
|
kubectl -n cozy-cluster-api logs -l control-plane=controller-manager --tail=80 \
|
|
--selector=cluster.x-k8s.io/provider=infrastructure-proxmox || true
|
|
log "=== VMs on the hypervisor ==="
|
|
pve "qm list" || true
|
|
if [ -s "$TENANT_KUBECONFIG" ]; then
|
|
log "=== tenant nodes and pods ==="
|
|
KUBECONFIG="$TENANT_KUBECONFIG" kubectl get nodes -o wide || true
|
|
KUBECONFIG="$TENANT_KUBECONFIG" kubectl get pods -A -o wide || true
|
|
fi
|
|
}
|
|
|
|
case "${1:-}" in
|
|
render-tenant) render_tenant ;;
|
|
apply-tenant) apply_tenant ;;
|
|
linked-clone) check_linked_clone ;;
|
|
tenant-node) check_tenant_node ;;
|
|
pvc-round-trip) check_pvc_round_trip ;;
|
|
disk-freed) wait_disk_freed "${2:?disk-freed needs <storage>:<volume>}" ;;
|
|
replacement) check_replacement_converges ;;
|
|
diagnostics) dump_diagnostics ;;
|
|
*) die "usage: $0 {render-tenant|apply-tenant|linked-clone|tenant-node|pvc-round-trip|disk-freed <storage>:<volume>|replacement|diagnostics}" ;;
|
|
esac
|