Files
cozystack/hack/proxmox-e2e-fixture_test.bats
T
Aleksei Sviridkin 048c029e29 fix(e2e): keep the Proxmox tenant kubeconfig per run and remove it (#4971)
`hack/e2e-chainsaw/_lib/run-proxmox.sh` wrote the tenant's super-admin
kubeconfig to `$TMPDIR/proxmox-e2e-tenant.kubeconfig` with the process
umask and never removed it, so it outlived the run on the self-hosted
runner, world-readable under the usual umask. `dump_diagnostics` then
read whatever sat at that path, so a run that failed before
`fetch_tenant_kubeconfig` reported the previous run's tenant as its own.

- The file now lives under `RUNNER_TEMP`, which the runner exports into
every step and empties between jobs; `COZY_PROXMOX_WORKDIR` overrides it
for a local run, and `TMPDIR` is the last fallback. The workflow sets no
path of its own: the `runner` context is not available in job-level
`env`.
- `fetch_tenant_kubeconfig` writes it under `umask 077` in a subshell,
since the workflow's `umask 077` covers only the staging step.
- `apply-tenant`, the suite's first step, removes a leftover before
doing anything else, so diagnostics in this run can only see this run's
file.
- The suite's `finally` removes it, before the `keep_tenant` exit, and
the workflow's "Remove staged credentials" step removes it again for a
run killed before `finally`.

Verification: `sh -n` and shellcheck on the script; actionlint on
`e2e-proxmox.yaml` (only the custom `proxmox` runner label is reported,
as on main). `hack/proxmox-e2e-fixture_test.bats` has three new cases
with a stub `kubectl`: the fetched file has no group or other bits under
`umask 022`, `apply-tenant` removes a planted leftover, and the suite's
`finally` (run out of the Chainsaw file on the `keep_tenant` path)
removes the file, so `diagnostics` after it prints no tenant section.
Each goes red with its line (`umask 077`, the leftover `rm`, the
`finally` `rm`) removed. All `hack/proxmox-*.bats` pass (fixture 9,
teardown contract 8, step1b 4, talos template 12), also under
`hack/cozytest.sh`. Not run on a Proxmox stand: the only stand I have
cannot dispatch the workflow at the moment, so the workflow steps
themselves are unexercised beyond actionlint.

Closes #4806


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
<!-- coderabbit-ai-generated:v1 -->

## Summary by CodeRabbit

* **Chores**
* Proxmox end-to-end test runs store tenant credentials in a temporary
work directory and create kubeconfig files with restricted permissions.
* Stale tenant kubeconfig files are cleared before a new tenant fixture
is applied. Cleanup removes the file even when tenant data is retained
or a run is interrupted, preventing credentials from appearing in later
diagnostics.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
2026-10-29 11:31:31 +03:00

139 lines
7.1 KiB
Bash
Executable File

#!/usr/bin/env bats
# The Proxmox e2e fixture is stand-agnostic through ${COZY_PVE_*} variables that
# hack/e2e-chainsaw/_lib/run-proxmox.sh expands with envsubst. GNU envsubst
# expands ${VAR} and nothing else: a ${VAR:-default} stays in the text as it is,
# set or not, and reaches the apiserver as a string where the schema wants an
# integer. So the defaults live in the script, the fixture carries bare
# variables, and the render refuses to hand out a fixture with one left in it.
load test_helper
REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)"
FIXTURE="$REPO_ROOT/hack/e2e-chainsaw/kubernetes-proxmox/tenant.yaml"
RUNNER="$REPO_ROOT/hack/e2e-chainsaw/_lib/run-proxmox.sh"
@test "the fixture carries no shell-style default that envsubst would leave in place" {
if grep -qE '\$\{[A-Za-z_]+:[-=?+]' "$FIXTURE"; then echo "FAIL: the fixture must not carry a shell-style default envsubst leaves in place"; false; fi
}
# kubernetes-nodes names its release kubernetes-nodes-<CR name> and refuses one
# that does not start with kubernetes-nodes-<spec.cluster>- (_helpers.tpl), so a
# pool named anything but <cluster>-<pool> fails at helm install and the suite
# only learns it from a Machine that never appears.
@test "the fixture's KubernetesNodes is named <cluster>-<pool>, as kubernetes-nodes requires" {
doc="$(sed -n '/^kind: KubernetesNodes$/,/^---$/p' "$FIXTURE")"
name="$(printf '%s\n' "$doc" | sed -n 's/^ name: *//p' | head -n 1 | tr -d '"')"
cluster="$(printf '%s\n' "$doc" | sed -n 's/^ cluster: *//p' | head -n 1 | tr -d '"')"
if [ -z "$name" ] || [ -z "$cluster" ]; then echo "FAIL: no KubernetesNodes name or spec.cluster in the fixture"; false; fi
case "$name" in
"$cluster"-?*) ;;
*) echo "FAIL: KubernetesNodes $name must be named $cluster-<pool>"; false ;;
esac
}
@test "render-tenant expands every variable with the runner's defaults" {
out="$(COZY_PVE_SSH=x COZY_PROXMOX_STORAGE=teststore "$RUNNER" render-tenant)"
if printf '%s' "$out" | grep -q '\${'; then echo "FAIL: the rendered tenant must not keep an unexpanded variable"; false; fi
printf '%s' "$out" | grep -qE '^ +prefix: 24$'
printf '%s' "$out" | grep -qE '^ +storage: "teststore"$'
printf '%s' "$out" | grep -qE '^ +bridge: "vmbr50"$'
}
@test "render-tenant honours a stand's own values and keeps prefix an integer" {
out="$(COZY_PVE_SSH=x COZY_PROXMOX_STORAGE=s COZY_PVE_PREFIX=16 COZY_PVE_BRIDGE=vmbr1 "$RUNNER" render-tenant)"
printf '%s' "$out" | grep -qE '^ +prefix: 16$'
printf '%s' "$out" | grep -qE '^ +bridge: "vmbr1"$'
}
@test "render-tenant passes the stand's resource pool through, and none by default" {
out="$(COZY_PVE_SSH=x COZY_PROXMOX_STORAGE=s COZY_PVE_POOL=tenant-e2e "$RUNNER" render-tenant)"
printf '%s' "$out" | grep -qE '^ +pool: "tenant-e2e"$'
out="$(COZY_PVE_SSH=x COZY_PROXMOX_STORAGE=s "$RUNNER" render-tenant)"
printf '%s' "$out" | grep -qE '^ +pool: ""$'
}
# The PVC probe waits for the disk to leave the storage. A listing that fails
# must fail the check: read as empty, it would report a leaked disk as freed.
# (The bodies stay POSIX sh, so the legacy hack/cozytest.sh translator can still
# run them: no `run`, no [[ ]].)
@test "disk-freed fails when the storage cannot be listed, instead of calling the disk gone" {
stub="$(mktemp -d)"
printf '#!/bin/sh\necho "ssh: connect to host pve: Connection refused" >&2\nexit 255\n' > "$stub/ssh"
chmod +x "$stub/ssh"
rc=0
out="$(PATH="$stub:$PATH" COZY_PVE_SSH=root@pve COZY_PROXMOX_STORAGE=main-pool "$RUNNER" disk-freed main-pool:vm-1-pvc-x 2>&1)" || rc=$?
rm -rf "$stub"
[ "$rc" -ne 0 ]
case "$out" in *"cannot list main-pool"*) ;; *) echo "$out" >&2; exit 1 ;; esac
case "$out" in *"is gone"*) echo "$out" >&2; exit 1 ;; esac
}
@test "disk-freed reports the disk gone once a successful listing no longer shows it" {
stub="$(mktemp -d)"
printf '#!/bin/sh\necho "Volid Format Type Size VMID"\necho "main-pool:vm-2-pvc-y raw images 1073741824 2"\n' > "$stub/ssh"
chmod +x "$stub/ssh"
out="$(PATH="$stub:$PATH" COZY_PVE_SSH=root@pve COZY_PROXMOX_STORAGE=main-pool "$RUNNER" disk-freed main-pool:vm-1-pvc-x 2>&1)"
rm -rf "$stub"
case "$out" in *"is gone from main-pool"*) ;; *) echo "$out" >&2; exit 1 ;; esac
}
# The tenant's super-admin kubeconfig is a credential for the whole tenant, kept
# under RUNNER_TEMP for one run. The workflow sets no path of its own, so these
# cases go through the same RUNNER_TEMP fallback the runner provides, with the
# override cleared.
KUBECONFIG_NAME=proxmox-e2e-tenant.kubeconfig
SUITE="$REPO_ROOT/hack/e2e-chainsaw/kubernetes-proxmox/chainsaw-test.yaml.disabled"
@test "tenant-node writes the tenant kubeconfig without group or other bits under umask 022" {
work="$(mktemp -d)"
stub="$(mktemp -d)"
# The secret read answers a kubeconfig and the first tenant read fails, which
# ends the check right after the fetch with the file in place.
cat > "$stub/kubectl" <<'STUB'
#!/bin/sh
case "$*" in *"get secret"*) printf '%s' YXBpVmVyc2lvbjogdjEK; exit 0 ;; esac
exit 1
STUB
chmod +x "$stub/kubectl"
rc=0
(umask 022; PATH="$stub:$PATH" RUNNER_TEMP="$work" COZY_PROXMOX_WORKDIR= COZY_PVE_SSH=x COZY_PROXMOX_STORAGE=s "$RUNNER" tenant-node) >/dev/null 2>&1 || rc=$?
mode="$(ls -l "$work/$KUBECONFIG_NAME" | cut -c1-10)"
rm -rf "$work" "$stub"
[ "$rc" -ne 0 ]
[ "$mode" = "-rw-------" ] || { echo "FAIL: the tenant kubeconfig was written $mode" >&2; exit 1; }
}
@test "apply-tenant removes a tenant kubeconfig an earlier run left behind" {
work="$(mktemp -d)"
stub="$(mktemp -d)"
printf 'an earlier run\n' > "$work/$KUBECONFIG_NAME"
printf '#!/bin/sh\ncat >/dev/null\n' > "$stub/kubectl"
chmod +x "$stub/kubectl"
PATH="$stub:$PATH" RUNNER_TEMP="$work" COZY_PROXMOX_WORKDIR= COZY_PVE_SSH=x COZY_PROXMOX_STORAGE=s "$RUNNER" apply-tenant
left=no
[ ! -e "$work/$KUBECONFIG_NAME" ] || left=yes
rm -rf "$work" "$stub"
[ "$left" = no ] || { echo "FAIL: apply-tenant left the earlier run's kubeconfig in place" >&2; exit 1; }
}
@test "the suite's finally removes the tenant kubeconfig, so diagnostics after it prints no tenant section" {
# keep_tenant exits the finally before its deletes, and the kubeconfig has to
# be gone on that path as well. Diagnostics reads whatever is at the path, so
# a file left there would be reported as the next run's tenant.
work="$(mktemp -d)"
stub="$(mktemp -d)"
printf 'this run\n' > "$work/$KUBECONFIG_NAME"
printf '#!/bin/sh\necho "stub $*"\n' > "$stub/kubectl"
printf '#!/bin/sh\nexit 0\n' > "$stub/ssh"
chmod +x "$stub/kubectl" "$stub/ssh"
finally="$(yq -r '.spec.steps[] | select(.name == "replacement-converges-unattended") | .finally[0].script.content' "$SUITE")"
[ -n "$finally" ]
PATH="$stub:$PATH" RUNNER_TEMP="$work" COZY_PROXMOX_WORKDIR= COZY_KEEP_TENANT=true sh -c "$finally" >/dev/null
out="$(PATH="$stub:$PATH" RUNNER_TEMP="$work" COZY_PROXMOX_WORKDIR= COZY_PVE_SSH=x COZY_PROXMOX_STORAGE=s "$RUNNER" diagnostics 2>&1)"
left=no
[ ! -e "$work/$KUBECONFIG_NAME" ] || left=yes
rm -rf "$work" "$stub"
[ "$left" = no ] || { echo "FAIL: the suite's finally left the tenant kubeconfig in place" >&2; exit 1; }
case "$out" in *"tenant nodes and pods"*) echo "$out" >&2; exit 1 ;; esac
}