Merge branch 'main' into fix/tenant-namespace-direct-delete-guard

Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin <f@lex.la>
This commit is contained in:
Aleksei Sviridkin committed 2026-11-13 14:20:44 +03:00
commit d089decab9
593 files changed
+57268 -35769

No files matched your search

+531
View File
@@ -0,0 +1,531 @@
#!/usr/bin/env bats
# Behavioural tests for the shipped ClickHouse (Altinity) strategy script,
# extracted from the rendered cozy-default-altinity Strategy CR and EXECUTED
# against a stubbed clickhouse-backup sidecar API.
#
# The contract pinned here is the start gate: the sidecar serialises its
# operations and answers any new one with HTTP 423 while another is "in
# progress", so a strategy Pod that starts its operation without waiting for
# an idle sidecar burns a Job attempt in seconds. The gate has to fail the Pod
# rather than proceed when the sidecar never becomes idle. Once started, an
# operation the sidecar no longer lists (its action log lives in memory and a
# restart empties it) has to fail the Pod too, since the Job has no deadline.
# A Job retry must not start its operation twice: it takes over the
# restore_remote, or the create_remote of the archive the controller named,
# that the attempt before it left running, and a few lost polls do not end an
# attempt in the first place.
#
# Which archive a restore reads back is decided by the script's own jq
# selection over list/remote, so it is exercised against several archives
# rather than string-matched: exactly the archive the Backup recorded, or for
# a Backup that records none, the newest one stamped no later than its takenAt.
#
# Run from the repo root with make bats-unit-tests. The rendered script is run
# with `bash`, not `sh`: it opens `set -euo pipefail`, which the production
# image's busybox ash supports but the CI runner's /bin/sh (dash) rejects.
load test_helper
CHART=packages/system/backupstrategy-controller
# Render the shipped strategy's in-Pod script into $1 and substitute the
# controller-render placeholders the way the driver fills them: $2 mode, $3
# app (release and source application), $4 the archive name the controller
# chose or the Backup recorded (empty, the default, for none), $5 the Backup
# name, $6 its takenAt in the archive-name layout (default: later than every
# archive the stub lists). An empty render fails early, as does a placeholder
# left unsubstituted: these tests EXECUTE it.
render_script() {
out=$1; mode=$2; app=$3; archive=${4-}; backup=${5-bkp}; takenat=${6-99991231T235959Z}
helm template t "$CHART" -n cozy-backup-controller \
--set backupStorage.bucketNameOverride=test-bucket 2>/dev/null \
| yq eval 'select(.kind == "Altinity") | .spec.template.spec.containers[0].args[0]' - \
| sed -e "s#{{ .Mode }}#${mode}#g" \
-e "s#{{ .Release.Name }}#${app}#g" \
-e "s#{{ .Release.Namespace }}#tenant-x#g" \
-e "s#{{ .Backup.ApplicationRef.Name }}#${app}#g" \
-e "s#{{ .Backup.Name }}#${backup}#g" \
-e "s#{{ .ArchiveName | default \"\" }}#${archive}#g" \
-e "s#{{ .Backup.TakenAt | default \"\" }}#${takenat}#g" > "$out"
[ -s "$out" ] || return 1
if grep -q '{{' "$out"; then
echo "rendered script still holds a controller placeholder" >&2
return 1
fi
}
# A curl stub standing in for the sidecar API, put first on PATH next to a
# no-op sleep. State lives in $1:
# unreachable present: every request fails as a refused connection (7)
# down a number: that many next requests fail the same way, as
# before the sidecar has started
# busy a number: that many GET /backup/actions answers still list
# an operation of someone else "in progress"
# op the command line of the sidecar's own operation, written
# by an accepted POST the way the sidecar logs it ("--resume"
# before the name of a create_remote asking for it), or
# seeded by a test as the operation
# an earlier attempt of the same Job started
# op_left how many more GET /backup/actions answers list op as
# "in progress" before it ends with op_result
# op_duration op_left given to an operation a POST starts (default 0)
# op_result the status op ends with (default success)
# prior extra action-log rows, verbatim, older than op
# remote present: the list/remote answer, verbatim, in place of the
# one archive listed by default
# restarted present: the action log no longer lists op, as after the
# sidecar restarted and lost its in-memory log
# fail_on_post a POST that is accepted makes that many following
# GET /backup/actions requests fail (curl exit 7)
# idle created by an answer that lists nothing in progress; a
# POST is refused (423, curl -f exit 22) unless one was seen
# since the last accepted POST, and while anything runs
# accepted one line per POST the sidecar accepted
# requests one line per request: method and URL
# runaway created when the script exceeds 200 requests; that request
# and every later one fails, so a polling loop that never ends
# stops the test instead of hanging it
make_stubs() {
bin=$1; state=$2
mkdir -p "$bin" "$state"
cat > "$bin/curl" <<STUB
#!/bin/bash
state="$state"
method=GET
url=
prev=
for a in "\$@"; do
if [ "\$prev" = "-X" ]; then method=\$a; fi
case "\$a" in http://*) url=\$a ;; esac
prev=\$a
done
echo "\$method \$url" >> "\$state/requests"
if [ "\$(wc -l < "\$state/requests")" -gt 200 ]; then touch "\$state/runaway"; exit 7; fi
if [ -e "\$state/unreachable" ]; then exit 7; fi
num() { cat "\$state/\$1" 2>/dev/null || echo 0; }
down=\$(num down)
if [ "\$down" -gt 0 ]; then echo \$((down - 1)) > "\$state/down"; exit 7; fi
running=0
if [ -s "\$state/op" ] && [ ! -e "\$state/restarted" ] && [ "\$(num op_left)" -gt 0 ]; then running=1; fi
case "\$method \$url" in
"POST "*)
if [ "\$(num busy)" -gt 0 ] || [ "\$running" = 1 ] || [ ! -e "\$state/idle" ]; then exit 22; fi
case "\$url" in
*/create_remote\?name=*)
name=\${url#*name=}; name=\${name%%&*}
case "\$url" in *[?\&]resume=true*) cmd="create_remote --resume \$name" ;; *) cmd="create_remote \$name" ;; esac
;;
*/restore_remote/*) cmd="restore_remote \${url##*/}" ;;
*) exit 22 ;;
esac
echo "\$cmd" > "\$state/op"
echo "\$cmd" >> "\$state/accepted"
num op_duration > "\$state/op_left"
num fail_on_post > "\$state/fail_polls"
rm -f "\$state/idle"
;;
"GET "*/backup/actions)
fail=\$(num fail_polls)
if [ "\$fail" -gt 0 ]; then echo \$((fail - 1)) > "\$state/fail_polls"; exit 7; fi
echo '{"command":"create_remote clickhouse-myapp-20260101T000000Z","status":"error","start":"2026-01-01 00:00:00","error":"old"}'
cat "\$state/prior" 2>/dev/null || true
inprog=0
busy=\$(num busy)
if [ "\$busy" -gt 0 ]; then
echo \$((busy - 1)) > "\$state/busy"
echo '{"command":"upload --resumable=1 clickhouse-myapp-20260101T000000Z","status":"in progress","start":"2026-01-01 00:01:00"}'
inprog=1
fi
if [ -s "\$state/op" ] && [ ! -e "\$state/restarted" ]; then
left=\$(num op_left)
if [ "\$left" -gt 0 ]; then
echo \$((left - 1)) > "\$state/op_left"
status="in progress"
inprog=1
else
status=\$(cat "\$state/op_result" 2>/dev/null || echo success)
fi
echo "{\"command\":\"\$(cat "\$state/op")\",\"status\":\"\$status\",\"start\":\"2026-01-01 00:02:00\",\"error\":\"boom\"}"
fi
if [ "\$inprog" = 0 ]; then touch "\$state/idle"; fi
;;
"GET "*/backup/list/remote)
if [ -e "\$state/remote" ]; then cat "\$state/remote"; else
echo '{"name":"clickhouse-myapp-20260101T000000Z","created":"2026-01-01 00:00:00"}'
fi
;;
esac
exit 0
STUB
cat > "$bin/sleep" <<'STUB'
#!/bin/sh
exit 0
STUB
chmod 0755 "$bin/curl" "$bin/sleep"
}
# One list/remote row per argument, "name" or "name|desc"; created mirrors
# the name's timestamp.
remote_rows() {
for row in "$@"; do
name=${row%%|*}; desc="tar, regular"
if [ "$row" != "$name" ]; then desc=${row#*|}; fi
ts=${name##*-}
printf '{"name":"%s","created":"%s","location":"remote","required":"","desc":"%s"}\n' "$name" "$ts" "$desc"
done
}
accepted() {
if [ -e "$1/state/accepted" ]; then wc -l < "$1/state/accepted" | tr -d ' '; else echo 0; fi
}
run_script() {
tmp=$1
API_USERNAME=u API_PASSWORD=p PATH="$tmp/bin:$PATH" bash "$tmp/s.sh"
}
@test "an empty render fails the fixture instead of passing as a no-op" {
tmp=$(mktemp -d)
CHART=$tmp/no-such-chart
if render_script "$tmp/s.sh" backup myapp 2>/dev/null; then
echo "FAIL: render_script reported success with no chart"
false
fi
}
@test "backup on an idle sidecar starts create_remote and waits for its success" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
render_script "$tmp/s.sh" backup myapp
run_script "$tmp"
grep -q '^POST .*/backup/create_remote?name=clickhouse-myapp-' "$tmp/state/requests"
}
@test "backup waits until the running operation finishes before create_remote" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
echo 3 > "$tmp/state/busy"
render_script "$tmp/s.sh" backup myapp
run_script "$tmp"
grep -q '^create_remote --resume clickhouse-myapp-' "$tmp/state/accepted"
[ "$(cat "$tmp/state/busy")" = 0 ]
}
@test "restore on an idle sidecar starts restore_remote of the listed archive" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
render_script "$tmp/s.sh" restore myapp
run_script "$tmp"
grep -q '^POST .*/backup/restore_remote/clickhouse-myapp-20260101T000000Z$' "$tmp/state/requests"
}
@test "backup fails without starting create_remote when the sidecar stays busy" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
echo 1000 > "$tmp/state/busy"
render_script "$tmp/s.sh" backup myapp
if run_script "$tmp"; then echo "FAIL: a busy sidecar must fail the Pod"; false; fi
if grep -q '^POST ' "$tmp/state/requests"; then echo "FAIL: no operation may start on a busy sidecar"; false; fi
}
@test "backup fails without starting create_remote when the sidecar never answers" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
touch "$tmp/state/unreachable"
render_script "$tmp/s.sh" backup myapp
if run_script "$tmp"; then echo "FAIL: an unreachable sidecar must fail the Pod"; false; fi
if grep -q '^POST ' "$tmp/state/requests"; then echo "FAIL: no operation may start on an unreachable sidecar"; false; fi
}
@test "restore fails without starting restore_remote when the sidecar never answers" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
touch "$tmp/state/unreachable"
render_script "$tmp/s.sh" restore myapp
if run_script "$tmp" 2> "$tmp/err"; then echo "FAIL: an unreachable sidecar must fail the Pod"; false; fi
if grep -q '^POST ' "$tmp/state/requests"; then echo "FAIL: no restore may start on an unreachable sidecar"; false; fi
grep -q 'list/remote failed 60 times in a row' "$tmp/err"
[ "$(grep -c '^GET .*/backup/list/remote$' "$tmp/state/requests")" = 60 ]
}
@test "restore fails without starting restore_remote when the sidecar stays busy" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
echo 1000 > "$tmp/state/busy"
render_script "$tmp/s.sh" restore myapp
if run_script "$tmp"; then echo "FAIL: a busy sidecar must fail the Pod"; false; fi
if grep -q '^POST ' "$tmp/state/requests"; then echo "FAIL: no restore may start on a busy sidecar"; false; fi
}
@test "backup fails instead of polling forever when the sidecar restarts after create_remote" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
touch "$tmp/state/restarted"
render_script "$tmp/s.sh" backup myapp
if run_script "$tmp" 2> "$tmp/err"; then echo "FAIL: an operation lost from the action log must fail the Pod"; false; fi
grep -q 'no longer in the sidecar action log' "$tmp/err"
grep -q '^POST .*/backup/create_remote?name=clickhouse-myapp-' "$tmp/state/requests"
if [ -e "$tmp/state/runaway" ]; then echo "FAIL: the script kept polling for an operation the sidecar no longer lists"; false; fi
}
@test "restore waits for a sidecar that has not started yet" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
echo 10 > "$tmp/state/down"
render_script "$tmp/s.sh" restore myapp
run_script "$tmp"
[ "$(accepted "$tmp")" = 1 ]
}
@test "restore waits until another operation finishes, then starts its own" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
echo 3 > "$tmp/state/busy"
render_script "$tmp/s.sh" restore myapp
run_script "$tmp"
[ "$(accepted "$tmp")" = 1 ]
grep -qx 'restore_remote clickhouse-myapp-20260101T000000Z' "$tmp/state/accepted"
}
@test "failed polls after restore_remote, short of six in a row, do not fail the attempt" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
echo 3 > "$tmp/state/op_duration"
echo 5 > "$tmp/state/fail_on_post"
render_script "$tmp/s.sh" restore myapp
run_script "$tmp"
[ "$(accepted "$tmp")" = 1 ]
}
@test "a retry after an attempt lost its polls joins the running restore instead of starting another" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
echo 20 > "$tmp/state/op_duration"
echo 6 > "$tmp/state/fail_on_post"
render_script "$tmp/s.sh" restore myapp
if run_script "$tmp"; then echo "FAIL: six failed polls in a row must fail the attempt"; false; fi
[ "$(accepted "$tmp")" = 1 ]
echo 0 > "$tmp/state/fail_polls"
run_script "$tmp"
if [ "$(accepted "$tmp")" != 1 ]; then echo "FAIL: the retry started restore_remote a second time"; false; fi
[ "$(cat "$tmp/state/op_left")" = 0 ]
}
@test "restore does not join an operation running on another archive" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
echo 'restore_remote clickhouse-myapp-20251231T000000Z' > "$tmp/state/op"
echo 3 > "$tmp/state/op_left"
render_script "$tmp/s.sh" restore myapp
run_script "$tmp"
[ "$(accepted "$tmp")" = 1 ]
grep -qx 'restore_remote clickhouse-myapp-20260101T000000Z' "$tmp/state/accepted"
}
# The sidecar never trims its action log, so a finished restore_remote row of
# the same archive may belong to an earlier RestoreJob; only a running one may
# be taken over.
@test "restore starts restore_remote again when the log holds a finished one of the same archive" {
for status in success error cancel; do
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
echo "{\"command\":\"restore_remote clickhouse-myapp-20260101T000000Z\",\"status\":\"$status\",\"start\":\"2026-01-01 00:01:30\"}" > "$tmp/state/prior"
render_script "$tmp/s.sh" restore myapp
run_script "$tmp"
if [ "$(accepted "$tmp")" != 1 ]; then echo "FAIL: a $status row of an earlier restore must not stand in for this one"; false; fi
done
}
@test "a restore that ends in error fails the Pod" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
echo error > "$tmp/state/op_result"
render_script "$tmp/s.sh" restore myapp
if run_script "$tmp"; then echo "FAIL: a failed restore must fail the Pod"; false; fi
[ "$(accepted "$tmp")" = 1 ]
}
@test "a cancelled backup fails the Pod instead of polling forever" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
echo cancel > "$tmp/state/op_result"
render_script "$tmp/s.sh" backup myapp
if run_script "$tmp"; then echo "FAIL: a cancelled operation must fail the Pod"; false; fi
if [ -e "$tmp/state/runaway" ]; then echo "FAIL: the script kept polling a cancelled operation"; false; fi
}
@test "restore reads back the archive the Backup recorded, not the newest one" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
render_script "$tmp/s.sh" restore myapp clickhouse-myapp-20260304T050607Z bkp-a 20260304T050622Z
remote_rows clickhouse-myapp-20260304T040000Z clickhouse-myapp-20260304T050607Z \
clickhouse-myapp-20260304T060000Z clickhouse-myapp-20260304T070000Z > "$tmp/state/remote"
run_script "$tmp"
grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/restore_remote/clickhouse-myapp-20260304T050607Z' "$tmp/state/requests"
[ "$(accepted "$tmp")" = 1 ]
}
@test "restore fails naming the recorded archive when remote storage lacks it" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
render_script "$tmp/s.sh" restore myapp clickhouse-myapp-20260304T050607Z bkp-a 20260304T050622Z
remote_rows clickhouse-myapp-20260304T040000Z clickhouse-myapp-20260304T060000Z > "$tmp/state/remote"
out=""
if out=$(run_script "$tmp" 2>&1); then
echo "FAIL: restore succeeded without the recorded archive: $out"
false
fi
printf '%s\n' "$out" | grep -qF 'archive clickhouse-myapp-20260304T050607Z recorded on Backup bkp-a is not in remote storage'
if grep -qF '/backup/restore_remote/' "$tmp/state/requests"; then
echo "FAIL: restored another archive: $(cat "$tmp/state/requests")"
false
fi
}
@test "restore of a Backup without a recorded archive takes the newest one stamped by its takenAt, and warns" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
render_script "$tmp/s.sh" restore myapp "" bkp-a 20260304T050800Z
# Each decoy sorts after the expected archive and is ruled out by one
# filter: broken, not a bare timestamp after the prefix, or stamped after
# takenAt.
remote_rows clickhouse-myapp-20260304T040000Z clickhouse-myapp-20260304T050607Z \
"clickhouse-myapp-20260304T050700Z|broken (can't stat metadata.json)" \
clickhouse-myapp-20260304T050710Z-copy \
myapp-20260304050720-0f3c9a1e5b7d \
clickhouse-myapp-20260304T060000Z > "$tmp/state/remote"
out=$(run_script "$tmp" 2>&1)
grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/restore_remote/clickhouse-myapp-20260304T050607Z' "$tmp/state/requests"
[ "$(accepted "$tmp")" = 1 ]
printf '%s\n' "$out" | grep -qF 'WARNING: Backup bkp-a records no archive name'
}
@test "restore of a Backup without a recorded archive fails when no archive is stamped by its takenAt" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
render_script "$tmp/s.sh" restore myapp "" bkp-a 20260304T050800Z
remote_rows clickhouse-myapp-20260304T060000Z clickhouse-myapp-20260304T070000Z > "$tmp/state/remote"
out=""
if out=$(run_script "$tmp" 2>&1); then
echo "FAIL: restore succeeded with every archive newer than takenAt: $out"
false
fi
printf '%s\n' "$out" | grep -qF 'no archive of myapp stamped no later than 20260304T050800Z'
if grep -qF '/backup/restore_remote/' "$tmp/state/requests"; then
echo "FAIL: restored an archive newer than takenAt: $(cat "$tmp/state/requests")"
false
fi
}
@test "backup writes the archive under the name the controller chose" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z
run_script "$tmp"
grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/create_remote?name=clickhouse-myapp-20260304T050607Z&resume=true' "$tmp/state/requests"
}
@test "backup falls back to a timestamped name when the controller supplies none" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
render_script "$tmp/s.sh" backup myapp ""
run_script "$tmp"
grep -qE '^POST http://[^ ]*/backup/create_remote\?name=clickhouse-myapp-[0-9]{8}T[0-9]{6}Z&resume=true$' "$tmp/state/requests"
}
# The sidecar answers a second operation with 423 while one runs, so a retry
# that POSTed over its predecessor's upload would burn every attempt left.
@test "a retried backup takes over the upload of its archive the sidecar is still running" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
echo 'create_remote --resume clickhouse-myapp-20260304T050607Z' > "$tmp/state/op"
echo 3 > "$tmp/state/op_left"
render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z
run_script "$tmp"
if grep -q '^POST ' "$tmp/state/requests"; then
echo "FAIL: started another operation over the running one: $(cat "$tmp/state/requests")"
false
fi
[ "$(cat "$tmp/state/op_left")" = 0 ]
}
@test "a taken-over backup that ends in error fails the Pod" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
echo 'create_remote --resume clickhouse-myapp-20260304T050607Z' > "$tmp/state/op"
echo 2 > "$tmp/state/op_left"
echo error > "$tmp/state/op_result"
render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z
if run_script "$tmp"; then echo "FAIL: a taken-over upload that failed must fail the Pod"; false; fi
if grep -q '^POST ' "$tmp/state/requests"; then
echo "FAIL: started another operation over the running one: $(cat "$tmp/state/requests")"
false
fi
}
@test "a retried backup after a failed attempt resumes the same archive" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
echo '{"command":"create_remote clickhouse-myapp-20260304T050607Z","status":"error","start":"2026-01-01 00:01:30","error":"S3 PutObject 403"}' > "$tmp/state/prior"
render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z
run_script "$tmp"
grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/create_remote?name=clickhouse-myapp-20260304T050607Z&resume=true' "$tmp/state/requests"
}
@test "backup does not take over an upload running for another archive" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
echo 'create_remote --resume clickhouse-myapp-20260304T040000Z' > "$tmp/state/op"
echo 3 > "$tmp/state/op_left"
render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z
run_script "$tmp"
[ "$(accepted "$tmp")" = 1 ]
grep -qx 'create_remote --resume clickhouse-myapp-20260304T050607Z' "$tmp/state/accepted"
}
# The archive is named per BackupJob, so a success row of exactly this
# upload is an earlier attempt of the same BackupJob that finished while it
# could not see it. Uploading again with resume would re-upload over the
# finished archive and break it if that second upload dies halfway.
@test "a retried backup takes the upload of its archive an earlier attempt finished as its result" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
echo '{"command":"create_remote --resume clickhouse-myapp-20260304T050607Z","status":"success","start":"2026-01-01 00:01:30"}' > "$tmp/state/prior"
render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z
run_script "$tmp"
if grep -q '^POST ' "$tmp/state/requests"; then
echo "FAIL: uploaded again over a finished archive: $(cat "$tmp/state/requests")"
false
fi
}
@test "a retried backup resumes an upload of its archive that ended in error or was cancelled" {
for status in error cancel; do
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
echo "{\"command\":\"create_remote --resume clickhouse-myapp-20260304T050607Z\",\"status\":\"$status\",\"start\":\"2026-01-01 00:01:30\"}" > "$tmp/state/prior"
render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z
run_script "$tmp"
if [ "$(accepted "$tmp")" != 1 ]; then echo "FAIL: a $status row must lead to a resumed upload"; false; fi
done
}
@test "a retried backup resumes when its latest upload failed after an earlier one succeeded" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
printf '%s\n' \
'{"command":"create_remote --resume clickhouse-myapp-20260304T050607Z","status":"success","start":"2026-01-01 00:01:00"}' \
'{"command":"create_remote --resume clickhouse-myapp-20260304T050607Z","status":"error","start":"2026-01-01 00:01:30"}' > "$tmp/state/prior"
render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z
run_script "$tmp"
[ "$(accepted "$tmp")" = 1 ]
}
@test "backup does not take another archive's finished upload as its own" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
echo '{"command":"create_remote --resume clickhouse-myapp-20260304T040000Z","status":"success","start":"2026-01-01 00:01:30"}' > "$tmp/state/prior"
render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z
run_script "$tmp"
[ "$(accepted "$tmp")" = 1 ]
}
+186
View File
@@ -0,0 +1,186 @@
#!/usr/bin/env bats
# Tests for hack/backport-batch-scheduled.sh, the scheduled backport batch, and
# for the workflow that runs it, .github/workflows/backport-batch.yaml.
#
# The backport-batch binary is stubbed through BACKPORT_BATCH: every call is
# recorded with the tokens gh would see, `lines` answers with $STUB_LINES, and
# a subcommand fails for the line named in $STUB_FAIL_<subcommand>.
#
# Run with: bats hack/backport-batch-scheduled.bats
load test_helper
SCRIPT="${BATS_TEST_DIRNAME}/backport-batch-scheduled.sh"
WF="${BATS_TEST_DIRNAME}/../.github/workflows/backport-batch.yaml"
setup() {
strict_setup
export TMPDIR="$BATS_TEST_TMPDIR"
export CALLS="${BATS_TEST_TMPDIR}/calls"
export BACKPORT_BATCH="${BATS_TEST_TMPDIR}/backport-batch"
cat > "$BACKPORT_BATCH" <<'EOF'
#!/usr/bin/env bash
printf 'GH_TOKEN=%s GITHUB_TOKEN=%s %s\n' "${GH_TOKEN-unset}" "${GITHUB_TOKEN-unset}" "$*" >> "$CALLS"
line=${3:-}
case "$1" in
lines) printf '%s\n' $STUB_LINES ;;
run) [ "${STUB_FAIL_run:-}" != "$line" ] || exit 2 ;;
check) [ "${STUB_FAIL_check:-}" != "$line" ] || exit 1 ;;
notify)
[ "${STUB_FAIL_notify:-}" != "$line" ] || exit 2
echo "$line: something waits"
;;
esac
EOF
chmod +x "$BACKPORT_BATCH"
export STUB_LINES="release-1.7 release-1.6"
unset STUB_FAIL_run STUB_FAIL_check STUB_FAIL_notify
unset BACKPORT_ALERT_TEAM GITHUB_STEP_SUMMARY GH_TOKEN
export GITHUB_TOKEN=job-token
export BACKPORT_BATCH_TOKEN=batch-token
: > "$CALLS"
}
# The calls recorded, without the token fields.
calls() {
sed 's/^GH_TOKEN=[^ ]* GITHUB_TOKEN=[^ ]* //' "$CALLS"
}
@test "runs, checks and reports each line, newest first, pushing with the batch's token alone" {
run "$SCRIPT"
[ "$status" -eq 0 ]
[ "$(calls)" = "lines
run --line release-1.7 --continue --push
check --line release-1.7
notify --line release-1.7 --push
run --line release-1.6 --continue --push
check --line release-1.6
notify --line release-1.6 --push" ]
# Every call sees the batch's token, and the job's token is gone.
[ "$(grep -c '^GH_TOKEN=batch-token GITHUB_TOKEN=unset ' "$CALLS")" -eq 7 ]
}
@test "a dry run pushes, opens and comments nothing, and needs no token" {
unset BACKPORT_BATCH_TOKEN
run "$SCRIPT" --dry-run
[ "$status" -eq 0 ]
if grep -q -- '--push' "$CALLS"; then
echo "a dry run passed --push:"; cat "$CALLS"
false
fi
[ "$(calls | grep -c '^run --line release-.* --continue$')" -eq 2 ]
[ "$(calls | grep -c '^notify --line release-.*[0-9]$')" -eq 2 ]
}
@test "--no-workflow-files reaches run alone, and an unknown argument stops the script" {
run "$SCRIPT" --no-workflow-files
[ "$status" -eq 0 ]
[ "$(calls | grep -c '^run --line release-.* --continue --no-workflow-files --push$')" -eq 2 ]
if calls | grep -v '^run ' | grep -q -- '--no-workflow-files'; then
echo "the flag reached another command:"; calls
false
fi
: > "$CALLS"
run "$SCRIPT" --dry-run --no-workflow-files
[ "$status" -eq 0 ]
[ "$(calls | grep -c '^run --line release-.* --continue --no-workflow-files$')" -eq 2 ]
: > "$CALLS"
run "$SCRIPT" --no-workflows
[ "$status" -eq 2 ]
[ ! -s "$CALLS" ]
}
@test "without BACKPORT_BATCH_TOKEN it stops before anything, even with the job's token" {
unset BACKPORT_BATCH_TOKEN
run "$SCRIPT"
[ "$status" -eq 2 ]
[[ "$output" == *"BACKPORT_BATCH_TOKEN is empty"* ]]
[ ! -s "$CALLS" ]
}
@test "outside a worktree of the repository it stops before anything" {
export GIT_CEILING_DIRECTORIES="$BATS_TEST_TMPDIR"
mkdir "$BATS_TEST_TMPDIR/elsewhere"
cd "$BATS_TEST_TMPDIR/elsewhere"
run "$SCRIPT"
[ "$status" -eq 2 ]
[[ "$output" == *"not in a worktree of the repository"* ]]
[ ! -s "$CALLS" ]
}
@test "the team is mentioned only when BACKPORT_ALERT_TEAM is set" {
export BACKPORT_ALERT_TEAM=cozystack/maintainers
run "$SCRIPT"
[ "$status" -eq 0 ]
[ "$(calls | grep -c '^notify --line release-.* --mention cozystack/maintainers --push$')" -eq 2 ]
export BACKPORT_ALERT_TEAM=
: > "$CALLS"
run "$SCRIPT"
[ "$status" -eq 0 ]
if grep -q -- '--mention' "$CALLS"; then
echo "an empty team was mentioned:"; cat "$CALLS"
false
fi
}
@test "a line that fails goes into the summary, the other line still runs, and the job fails" {
export GITHUB_STEP_SUMMARY="${BATS_TEST_TMPDIR}/summary"
export STUB_FAIL_run=release-1.7 STUB_FAIL_check=release-1.6
run "$SCRIPT"
[ "$status" -eq 1 ]
[ "$(calls | grep -c '^run --line release-1.6 ')" -eq 1 ]
grep -qx -- '- release-1.7: the run failed with exit 2' "$GITHUB_STEP_SUMMARY"
grep -qx -- '- release-1.6: the check exited 1; see its output above' "$GITHUB_STEP_SUMMARY"
grep -qx -- '- release-1.7: something waits' "$GITHUB_STEP_SUMMARY"
# Each failure fails the job on its own.
export STUB_FAIL_run=release-1.7 STUB_FAIL_check=
run "$SCRIPT"
[ "$status" -eq 1 ]
export STUB_FAIL_run= STUB_FAIL_check=release-1.6
run "$SCRIPT"
[ "$status" -eq 1 ]
export STUB_FAIL_check= STUB_FAIL_notify=release-1.6
run "$SCRIPT"
[ "$status" -eq 1 ]
export STUB_FAIL_notify=
run "$SCRIPT"
[ "$status" -eq 0 ]
}
@test "the workflow runs Monday and Thursday as the cozystack-ci App and never with the job's token" {
[ "$(yq -r '.on.schedule[0].cron' "$WF")" = "23 6 * * 1,4" ]
[ "$(yq -r '.on.schedule | length' "$WF")" -eq 1 ]
[ "$(yq -o=json -I=0 '.permissions' "$WF")" = "{}" ]
[ "$(yq -o=json -I=0 '.jobs.batch.permissions' "$WF")" = '{"contents":"read"}' ]
[ "$(yq -r '.jobs.batch.steps[] | select(.uses == "actions/checkout*") | .with."persist-credentials"' "$WF")" = "false" ]
run_step='.jobs.batch.steps[] | select(.name == "Run the batch")'
[ "$(yq -r "$run_step | .run" "$WF")" = "hack/backport-batch-scheduled.sh --no-workflow-files" ]
[ "$(yq -r "$run_step | .env.BACKPORT_BATCH_TOKEN" "$WF")" = '${{ steps.app-token.outputs.token }}' ]
[ "$(yq -r "$run_step | .env.BACKPORT_BATCH_BOT" "$WF")" = '${{ steps.app-token.outputs.app-slug }}[bot]' ]
# The App token covers this repository alone, with Contents and Pull
# requests and nothing else: no workflows, no packages.
mint='.jobs.batch.steps[] | select(.id == "app-token")'
[ "$(yq -r "$mint | .uses" "$WF")" = "actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547" ]
[ "$(yq -r "$mint | .with.owner" "$WF")" = "cozystack" ]
[ "$(yq -r "$mint | .with.repositories" "$WF")" = "cozystack" ]
[ "$(yq -r "$mint | .with | to_entries | map(select(.key | test(\"^permission-\"))) | map(.key + \"=\" + .value) | join(\",\")" "$WF")" = "permission-contents=write,permission-pull-requests=write" ]
# A token minted after the checkout lives an hour; the job ends before.
[ "$(yq -r '.jobs.batch."timeout-minutes"' "$WF")" -le 45 ]
# The commits carry the App's bot, read from the token, not a person.
identity='.jobs.batch.steps[] | select(.name == "Use the App'"'"'s bot as git identity and credential")'
[ "$(yq -r "$identity | .env.APP_SLUG" "$WF")" = '${{ steps.app-token.outputs.app-slug }}' ]
yq -r "$identity | .run" "$WF" | grep -qF 'gh api "/users/${APP_SLUG}[bot]" --jq .id'
yq -r "$identity | .run" "$WF" | grep -qF 'user.email "${id}+${APP_SLUG}[bot]@users.noreply.github.com"'
# No step reaches for the job's own token or a person's batch token.
if grep -v '^[[:space:]]*#' "$WF" | grep -Eq 'secrets\.GITHUB_TOKEN|github\.token|secrets\.BACKPORT_BATCH_TOKEN|gh api user'; then
echo "the workflow reads the job's own token or a person's"
false
fi
}
+122
View File
@@ -0,0 +1,122 @@
#!/usr/bin/env bash
# The scheduled backport batch (docs/release/backports.md): for each line the
# batch runs for, newest first, continue the open batch pull request or open
# one, check the line, and report what waits on the batch. This is what
# .github/workflows/backport-batch.yaml runs; a maintainer can run it by hand
# from a clean worktree of the repository.
#
# Usage: backport-batch-scheduled.sh [--dry-run] [--no-workflow-files]
#
# --dry-run builds and checks each batch in this worktree and prints what it
# would report, but pushes nothing, opens or edits no pull request and writes
# no comment. Its check sees the line without the batch it just built, so it
# can exit 1 where the real run's check passes. Without --dry-run the batch's
# token has to be in BACKPORT_BATCH_TOKEN, and it is the only token gh sees:
# the job's own GITHUB_TOKEN is never a fallback, because a push made with it
# starts no CI on the batch pull request. The workflow passes the cozystack-ci
# App's token; a maintainer running this by hand passes their own.
#
# --no-workflow-files goes to run: an original that changes .github/workflows/
# is left on the batch pull request for a maintainer instead of picked, for a
# token that may not push such a commit, as the workflow's is.
#
# Environment:
# BACKPORT_BATCH the backport-batch binary (default: built from this tree)
# BACKPORT_BATCH_TOKEN the token the batch pushes, opens and comments with
# BACKPORT_BATCH_BOT the bot the scheduled batch writes as, <app>[bot],
# whose batch pull request and comment are trusted;
# cozystack-ci[bot] when unset
# BACKPORT_ALERT_TEAM the team or person the batch's comment mentions,
# org/team or a login; nobody is mentioned when it is
# unset
# GITHUB_STEP_SUMMARY when set, each finding is appended to it as a line
#
# Exits 0 when every line ran and its check passed, 1 when a line's run,
# check or report failed, and 2 on a usage or setup error.
set -euo pipefail
die() {
echo "backport-batch-scheduled: $*" >&2
exit 2
}
usage="usage: backport-batch-scheduled.sh [--dry-run] [--no-workflow-files]"
dry_run=false
# An empty array expands under set -u only as ${a[@]+"${a[@]}"} on bash before
# 4.4, which is what macOS ships.
workflows=()
for arg in "$@"; do
case "$arg" in
--dry-run) dry_run=true ;;
--no-workflow-files) workflows=(--no-workflow-files) ;;
*) die "$usage" ;;
esac
done
push=()
if ! $dry_run; then
[ -n "${BACKPORT_BATCH_TOKEN:-}" ] ||
die "BACKPORT_BATCH_TOKEN is empty; the workflow sets it to the cozystack-ci App's token, a run by hand to your own (docs/release/backports.md#the-scheduled-run). The job's GITHUB_TOKEN is not used: its pushes start no CI."
export GH_TOKEN="$BACKPORT_BATCH_TOKEN"
unset GITHUB_TOKEN
push=(--push)
fi
mention=()
if [ -n "${BACKPORT_ALERT_TEAM:-}" ]; then
mention=(--mention "$BACKPORT_ALERT_TEAM")
fi
top=$(git rev-parse --show-toplevel) || die "not in a worktree of the repository"
cd "$top"
out=$(mktemp -d)
bin=${BACKPORT_BATCH:-}
if [ -z "$bin" ]; then
# Built before the first run switches this tree to a batch branch, whose
# line may not have the tool at all.
bin="$out/backport-batch"
go build -o "$bin" ./cmd/backport-batch
fi
report() {
echo "$1"
if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
printf -- '- %s\n' "$1" >> "$GITHUB_STEP_SUMMARY"
fi
}
if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
printf '## Backport batch\n\n' >> "$GITHUB_STEP_SUMMARY"
fi
lines=$("$bin" lines) || die "cannot list the release lines"
[ -n "$lines" ] || die "no release line to run the batch for"
status=0
for line in $lines; do
echo "::group::$line"
rc=0
"$bin" run --line "$line" --continue ${workflows[@]+"${workflows[@]}"} ${push[@]+"${push[@]}"} > "$out/$line.md" || rc=$?
if [ "$rc" -ne 0 ]; then
report "$line: the run failed with exit $rc"
status=1
elif $dry_run; then
report "$line: the batch is built in this tree, its body in $out/$line.md"
fi
rc=0
"$bin" check --line "$line" || rc=$?
if [ "$rc" -ne 0 ]; then
report "$line: the check exited $rc; see its output above"
status=1
fi
rc=0
notes=$("$bin" notify --line "$line" ${mention[@]+"${mention[@]}"} ${push[@]+"${push[@]}"}) || rc=$?
while IFS= read -r note; do
[ -z "$note" ] || report "$note"
done <<< "$notes"
if [ "$rc" -ne 0 ]; then
report "$line: reporting what waits on the batch failed with exit $rc"
status=1
fi
echo "::endgroup::"
done
exit "$status"
+59
View File
@@ -0,0 +1,59 @@
#!/usr/bin/env bats
# Contract for the backport-batch-verify job of .github/workflows/pr-labeler.yaml,
# which runs `backport-batch verify` on every push to a batch pull request.
#
# The job runs under pull_request_target, so it holds a token on a pull
# request's events and must never run code from the pull request. These tests
# pin what keeps it that way: the tool is built from main, the head is only
# fetched, the token reads contents and nothing more, and no expression
# reaches a shell.
#
# Run with: bats hack/backport-batch-verify-job.bats
load test_helper
WF="${BATS_TEST_DIRNAME}/../.github/workflows/pr-labeler.yaml"
JOB='.jobs.backport-batch-verify'
@test "the job runs on pull_request_target, so main's copy reaches release lines" {
[ "$(yq -r '.on | keys | .[]' "$WF")" = "pull_request_target" ]
[ "$(yq -r "$JOB.name" "$WF")" = "Backport batch shape" ]
}
@test "the job runs only for a batch branch of this repository into a release line" {
cond="$(yq -r "$JOB.if" "$WF")"
[[ "$cond" == *"startsWith(github.event.pull_request.base.ref, 'release-')"* ]]
[[ "$cond" == *"startsWith(github.event.pull_request.head.ref, 'backport-batch/')"* ]]
[[ "$cond" == *"github.event.pull_request.head.repo.full_name == github.repository"* ]]
[ "$(grep -o '&&' <<<"$cond" | wc -l)" -eq 2 ]
}
@test "the token reads contents and nothing else" {
[ "$(yq -o=json -I=0 "$JOB.permissions" "$WF")" = '{"contents":"read"}' ]
}
@test "main is checked out, without credentials, and the head only fetched" {
[ "$(yq -r "$JOB.steps[0].uses" "$WF")" = "actions/checkout@11d5960a326750d5838078e36cf38b85af677262" ]
[ "$(yq -r "$JOB.steps[0].with.ref" "$WF")" = "main" ]
[ "$(yq -r "$JOB.steps[0].with.persist-credentials" "$WF")" = "false" ]
[ "$(yq -r "[$JOB.steps[] | select(.uses == \"actions/checkout*\")] | length" "$WF")" -eq 1 ]
[ "$(yq -r "$JOB.steps[] | select(.name == \"Build backport-batch from main\") | .run" "$WF")" = 'go build -o "$RUNNER_TEMP/backport-batch" ./cmd/backport-batch' ]
[ "$(yq -r "$JOB.steps[] | select(.name == \"Fetch the batch head as data\") | .run" "$WF")" = 'git fetch --no-tags --quiet origin "$HEAD_SHA"' ]
}
@test "the tool is built before the head is fetched, and verify gets it through env" {
names="$(yq -r "$JOB.steps[].name" "$WF")"
[ "$(grep -n 'Build backport-batch from main' <<<"$names" | cut -d: -f1)" -lt "$(grep -n 'Fetch the batch head as data' <<<"$names" | cut -d: -f1)" ]
run_line="$(yq -r "$JOB.steps[] | select(.name == \"Verify every original on the batch\") | .run" "$WF")"
[ "$run_line" = '"$RUNNER_TEMP/backport-batch" verify --line "$BASE_REF" --head "$HEAD_SHA"' ]
}
@test "no expression reaches a shell in the job" {
runs="$(yq -r "$JOB.steps[] | select(has(\"run\")) | .run" "$WF")"
[ -n "$runs" ]
if grep -q '\${{' <<<"$runs"; then
echo "a run: line carries an expression:"
echo "$runs"
false
fi
}
+36
View File
@@ -133,3 +133,39 @@ $tpl: $obj does not gate on crdEstablished for $crd"
rm -rf "$tmp"
}
# The controller's DefaultObjectsGate tells an object rendered with a stale S3
# endpoint (publishing.host changed after the render) by the
# backups.cozystack.io/endpoint annotation, because the endpoint itself sits at
# a different path in every kind. An object the chart renders without it
# counts as stale on every check, so a new template that forgets it would
# force this release every MinForceInterval forever.
@test "every rendered Strategy CR and the Velero BSL carry the rendered endpoint annotation" {
tmp=$(mktemp -d)
helm template t "$CHART" -n cozy-backup-controller \
--set backupStorage.bucketNameOverride=ext-bucket \
--set backupStorage.endpoint=https://s3.example.org \
--output-dir "$tmp/open" >/dev/null
objs=$(find "$tmp/open" -name '*.yaml' | sort | while read -r f; do
yq eval-all "select((.apiVersion // \"\" | test(\"^${STRATEGY_API}\")) or .kind == \"BackupStorageLocation\") | .kind + \"/\" + .metadata.name + \" \" + (.metadata.annotations[\"backups.cozystack.io/endpoint\"] // \"<none>\")" "$f" \
| sed "s#^#$(basename "$f") #"
done)
if ! echo "$objs" | grep -q " BackupStorageLocation/"; then
echo "render produced no BSL; nothing was checked for it" >&2
false
fi
if ! echo "$objs" | grep -q " CNPG/"; then
echo "render produced no Strategy CR; nothing was checked" >&2
false
fi
bad=$(echo "$objs" | awk '$3 != "https://s3.example.org"')
if [ -n "$bad" ]; then
echo "objects without the rendered endpoint in backups.cozystack.io/endpoint:" >&2
echo "$bad" >&2
false
fi
rm -rf "$tmp"
}
+25
View File
@@ -0,0 +1,25 @@
#!/usr/bin/env bats
# The operator's resync period is what bounds the stale-Ready window after an
# operator restart (kubernetes-sigs/cluster-api-operator#1031): about two
# periods. It reaches the manager only through patches/sync-period.diff, which
# `make update` has to reapply on every re-vendor, and through the value in
# values.yaml. Losing either silently returns the window to about 20 minutes,
# so this checks the argument the rendered Deployment actually carries.
load test_helper
REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)"
OPERATOR="$REPO_ROOT/packages/system/capi-operator"
@test "the operator manager runs with a one-minute resync period" {
local rendered args
rendered="$BATS_TEST_TMPDIR/operator.yaml"
helm template capi-operator "$OPERATOR" --namespace cozy-cluster-api > "$rendered"
args="$(yq -N 'select(.kind == "Deployment") | .spec.template.spec.containers[] | select(.name == "manager") | .args[]' "$rendered")"
[ -n "$args" ] || { echo "no manager container args rendered from $OPERATOR" >&2; return 1; }
case "$(printf '%s\n' "$args" | grep -c '^--sync-period=')" in
1) ;;
*) echo "expected exactly one --sync-period in the manager args, got:" >&2; echo "$args" >&2; return 1 ;;
esac
printf '%s\n' "$args" | grep -qx -- '--sync-period=1m' || { echo "manager args: $args" >&2; return 1; }
}
+41
View File
@@ -434,6 +434,47 @@ baseline_of_platform() {
rm -rf "$FIX"
}
@test "an allowlist entry is accepted for every suite layout the selectors see" {
# hack/select-e2e.sh enumerates a suite as a directory in or below which
# Chainsaw finds a Test: chainsaw-test.yaml, chainsaw-test.yml, numbered step
# files, or a Test nested in a subdirectory. The allowlist has to accept the
# same set, or a suite that runs cannot be exempted and the run stops before
# the first suite. That includes a chainsaw-test.y*ml that is a directory:
# the selector keeps it so Chainsaw fails on it loudly, and refusing it here
# would stop the run before Chainsaw could report it.
new_fixture
mkdir -p "$FIX/root/plain" "$FIX/root/yml" "$FIX/root/steps" "$FIX/root/nested/sub" "$FIX/root/dirtest/chainsaw-test.yaml" "$FIX/root/dirtestyml/chainsaw-test.yml"
: >"$FIX/root/plain/chainsaw-test.yaml"
: >"$FIX/root/yml/chainsaw-test.yml"
: >"$FIX/root/steps/01-install.yaml"
: >"$FIX/root/nested/sub/chainsaw-test.yaml"
printf 'plain cozystack/cozystack#1\nyml cozystack/cozystack#1\nsteps cozystack/cozystack#1\nnested cozystack/cozystack#1\ndirtest cozystack/cozystack#1\ndirtestyml cozystack/cozystack#1\n' >"$FIX/allow"
lg lg_allowlist_validate "$FIX/allow" "$FIX/root"
rm -rf "$FIX"
}
@test "a parked, empty or step-file-less suite directory cannot be allowlisted" {
# A chainsaw-test.yaml.disabled is registered nowhere and runs nowhere, so an
# exemption for it would exempt nothing today and a real suite the day the
# suffix goes. A file that only looks like a step file is not one to Chainsaw
# either: install-01.yaml has no digits before the dash, and 1a-install.yaml
# passes the find glob but not the step-file regex behind it. A directory
# with a step-file name is not a step file.
new_fixture
mkdir -p "$FIX/root/parked" "$FIX/root/empty" "$FIX/root/notsteps" "$FIX/root/globonly" "$FIX/root/dirsteps/01-install.yaml"
: >"$FIX/root/parked/chainsaw-test.yaml.disabled"
: >"$FIX/root/notsteps/install-01.yaml"
: >"$FIX/root/globonly/1a-install.yaml"
for suite in parked empty notsteps globonly dirsteps; do
printf '%s cozystack/cozystack#1\n' "$suite" >"$FIX/allow"
if lg lg_allowlist_validate "$FIX/allow" "$FIX/root" 2>/dev/null; then
echo "an allowlist entry for '$suite' was accepted" >&2
return 1
fi
done
rm -rf "$FIX"
}
@test "an allowlist whose last line has no newline still has that line validated" {
new_fixture
mkdir -p "$FIX/root/kafka"
+14
View File
@@ -520,3 +520,17 @@ EOF
fi
cleanup
}
# The workflow skips this check on backport branches, whose commits are picked
# verbatim. A fork can name a branch backport-batch/anything, so the batch's
# exemption holds only for a branch of this repository.
@test "the workflow exempts a batch branch only from this repository" {
cond="$(yq -r '.jobs.commit-trailers.if' "$REPO_ROOT/.github/workflows/pre-commit.yml")"
case "$cond" in
*"!(startsWith(github.head_ref, 'backport-batch/') && github.event.pull_request.head.repo.full_name == github.repository)"*) ;;
*)
echo "the batch exemption is not bound to this repository: $cond" >&2
exit 1
;;
esac
}
+1 -1
View File
@@ -73,7 +73,7 @@ legacy_hits=$(grep -rEn 'defaultingSanitize.*\(list "(nano|micro|small|medium|la
if [ -n "$legacy_hits" ]; then
echo "FAIL: hardcoded legacy preset literals in chart templates:" >&2
echo "$legacy_hits" >&2
echo "Replace with the instance-type form from docs/operations/resource-presets.md." >&2
echo "Replace with the instance-type form from https://cozystack.io/docs/v1/guides/resource-management/." >&2
exit 1
fi
echo "No hardcoded legacy preset literals in chart templates."
+3 -2
View File
@@ -111,8 +111,9 @@ one_value_of() {
@test "every use of the cosign installer is SHA-pinned at one cosign release" {
# Without cosign-release the action installs its own default, which moves
# with a digest bump of the action. The release jobs verify their signature
# with the cosign docs/release.md asks users for, so the version is named in
# each step, and all steps agree on it and on the action's commit.
# with the cosign docs/release/signatures-and-sbom.md asks users for, so the
# version is named in each step, and all steps agree on it and on the
# action's commit.
uses="$(workflow_code 'uses: sigstore/cosign-installer@')"
[ -n "$uses" ]
bad="$(printf '%s\n' "$uses" | grep --invert-match --extended-regexp 'installer@[0-9a-f]{40} # v[0-9.]+$' || true)"
+27
View File
@@ -488,6 +488,33 @@ pod_cidrs_of() {
rm -rf "$STATE"
}
@test "a CiliumNode list larger than one argument may hold still moves every node and names the pod on an old Ingress IP" {
init_state
add_ip web-0.tenant-c ovn-default 10.244.1.135
add_ip web-1.tenant-c ovn-default 10.244.4.135
add_ip db-0.tenant-c ovn-default 10.244.9.9
# Cilium copies every Node label onto the CiliumNode, which takes six real
# ones past the 128 KiB Linux allows a single argument. One annotation stands
# in for the labels and keeps the list over that limit even printed compact.
for i in 1 2 3 4 5 6; do
add_node "n$i" "10.244.$i.0/24" "10.244.$i.223" "10.244.$i.135"
jq '.metadata.annotations.padding = ("x" * 24576)' "$STATE/ciliumnodes/n$i.json" > "$STATE/node.tmp"
mv "$STATE/node.tmp" "$STATE/ciliumnodes/n$i.json"
done
[ "$(kubectl get ciliumnodes -o json | jq -c . | wc -c)" -gt 131072 ]
"$SCRIPT" > "$STATE/out" 2>&1
# Each pod is named with the node whose Ingress IP it holds, and a pod on an
# address that is no node's Ingress IP is not named at all.
grep -q "web-0.tenant-c holds 10.244.1.135, the Ingress IP of n1 before the move" "$STATE/out"
grep -q "web-1.tenant-c holds 10.244.4.135, the Ingress IP of n4 before the move" "$STATE/out"
if grep -q "db-0.tenant-c" "$STATE/out"; then
cat "$STATE/out"
false
fi
[ "$(pod_cidrs_of n6)" = '["10.244.248.40/29"]' ]
rm -rf "$STATE"
}
@test "a cluster without kube-ovn skips the exclusion and still moves its nodes" {
init_state
rm "$STATE/subnet.json"
@@ -0,0 +1,263 @@
#!/usr/bin/env bats
# The clickhouse-3-system-bucket-roundtrip Test edits the host CoreDNS Corefile
# and must put it back. Its restore runs from an EXIT trap under `set +e`, so a
# restore that fails without saying so leaves the rewrite in a cluster-wide
# object and still reports a green run; the next run then refuses to start.
# These run the restore against a stubbed kubectl.
load test_helper
LIB=hack/e2e-chainsaw/clickhouse/system-bucket-shared-state.sh
HELPERS=examples/backups/clickhouse/00-helpers.sh
setup() {
strict_setup
STUB=$(mktemp -d)
# A Corefile carrying the Test's marked block, as the run step leaves it.
jq -n '{apiVersion: "v1", kind: "ConfigMap",
metadata: {name: "coredns", namespace: "kube-system", resourceVersion: "7"},
data: {Corefile: ".:53 {\n # === BEGIN e2e-clickhouse-system-bucket ===\n rewrite name exact s3.example.org root-ingress-controller.tenant-root.svc.cozy.local\n # === END e2e-clickhouse-system-bucket ===\n errors\n}\n"}}' \
>"$STUB/coredns.json"
# The secret lookup follows kubectl's contract: a name together with a
# selector is refused, and a selector-only lookup prints the names in
# OWNED_SECRETS, the Secrets carrying the owner label.
cat >"$STUB/kubectl" <<'EOF'
#!/bin/bash
echo "$*" >>"$STUB/calls"
case "$*" in
*"get configmap coredns -o json"*) cat "$STUB/coredns.json" ;;
replace*)
cp "${@: -1}" "$STUB/replaced.json"
if [ -n "${REPLACE_FAILS:-}" ]; then
echo 'Error from server (Conflict): Operation cannot be fulfilled on configmaps "coredns": the object has been modified' >&2
exit 1
fi
;;
*"get secret"*)
# Positionals are "get", "secret" and then any names.
words=0 selector=
while [ $# -gt 0 ]; do
case "$1" in
-l|--selector) selector=$2; shift 2 ;;
-n|--namespace|-o|--output) shift 2 ;;
-*) shift ;;
*) words=$((words + 1)); shift ;;
esac
done
if [ -n "${LOOKUP_FAILS:-}" ]; then
echo 'Unable to connect to the server: dial tcp: i/o timeout' >&2
exit 1
fi
if [ -n "$selector" ] && [ "$words" -gt 2 ]; then
echo 'error: name cannot be provided when a selector is specified' >&2
exit 1
fi
[ "$selector" = cozystack.io/e2e-owner=clickhouse-system-bucket ] || exit 0
for s in ${OWNED_SECRETS:-}; do echo "secret/$s"; done
;;
*"delete secret "*)
if [ -n "${DELETE_FAILS:-}" ]; then
echo 'Error from server (Forbidden): secrets "seaweedfs-s3-ingress-tls" is forbidden' >&2
exit 1
fi
;;
*) echo "unexpected kubectl $*" >&2; exit 2 ;;
esac
EOF
chmod +x "$STUB/kubectl"
export STUB
}
restore() {
PATH="$STUB:$PATH" NAMESPACE=tenant-root bash -c '
set -euo pipefail
source "$1"
source "$2"
set +e
restore_shared_state
' _ "$HELPERS" "$LIB"
}
@test "a CoreDNS restore the API server refuses fails the restore" {
export REPLACE_FAILS=1
err=$(mktemp)
if restore 2>"$err"; then
echo "restore_shared_state returned 0 although kubectl replace was refused" >&2
cat "$err" >&2
false
fi
if grep -q 'Removed the s3.example.org rewrite' "$err"; then
echo "restore_shared_state reported the rewrite removed although kubectl replace was refused:" >&2
cat "$err" >&2
false
fi
}
@test "a CoreDNS restore sends the Corefile back without the marked block" {
restore
jq -e '.data.Corefile == ".:53 {\n errors\n}\n"' "$STUB/replaced.json" >/dev/null || {
echo "restored Corefile:" >&2
jq -r '.data.Corefile' "$STUB/replaced.json" >&2
false
}
# The resourceVersion read travels with the replace, so a concurrent
# writer makes it fail rather than be overwritten.
[ "$(jq -r '.metadata.resourceVersion' "$STUB/replaced.json")" = 7 ]
}
@test "the restore deletes the Ingress TLS Secret the Test created" {
OWNED_SECRETS=seaweedfs-s3-ingress-tls restore
grep -qE '^-n tenant-root delete secret seaweedfs-s3-ingress-tls( |$)' "$STUB/calls" || {
echo "restore_shared_state did not delete the Secret carrying the owner label; kubectl calls:" >&2
cat "$STUB/calls" >&2
false
}
}
@test "the restore leaves an Ingress TLS Secret without the owner label alone" {
for owned in '' e2e-other seaweedfs-s3-ingress-tls-old; do
: >"$STUB/calls"
OWNED_SECRETS=$owned restore
if grep -q 'delete secret' "$STUB/calls"; then
echo "restore_shared_state deleted a Secret although the labelled ones are '$owned'; kubectl calls:" >&2
cat "$STUB/calls" >&2
false
fi
done
}
@test "a Secret delete the API server refuses fails the restore" {
err=$(mktemp)
if OWNED_SECRETS=seaweedfs-s3-ingress-tls DELETE_FAILS=1 restore 2>"$err"; then
echo "restore_shared_state returned 0 although kubectl delete secret was refused" >&2
cat "$err" >&2
false
fi
grep -qE '^-n tenant-root delete secret seaweedfs-s3-ingress-tls( |$)' "$STUB/calls" || {
echo "restore_shared_state failed before it reached the Secret delete; kubectl calls:" >&2
cat "$STUB/calls" "$err" >&2
false
}
}
@test "a Secret lookup that fails fails the restore" {
err=$(mktemp)
if LOOKUP_FAILS=1 restore 2>"$err"; then
echo "restore_shared_state returned 0 although it could not tell whether the Secret is left over" >&2
cat "$err" >&2
false
fi
grep -q 'could not list the Secrets labelled' "$err" || {
echo "restore_shared_state failed without naming the Secret lookup:" >&2
cat "$err" >&2
false
}
}
@test "the run step's finally restores the shared objects" {
# When the step ceiling fires, Chainsaw kills the shell and the EXIT trap
# of system-bucket-roundtrip.sh never runs; the step's finally still does.
# Chainsaw runs a script's content with sh from the suite's directory.
suite=hack/e2e-chainsaw/clickhouse
finally=$(yq 'select(.metadata.name == "clickhouse-3-system-bucket-roundtrip")
| .spec.steps[] | select(.name == "run") | .finally[]
| select(.description == "restore the host CoreDNS and the S3 Ingress TLS Secret") | .script.content' \
"$suite/chainsaw-test.yaml")
[ -n "$finally" ] || { echo "the run step has no finally restoring the shared objects" >&2; false; }
(cd "$suite" && PATH="$STUB:$PATH" NAMESPACE=tenant-root sh -c "$finally")
[ -f "$STUB/replaced.json" ] || {
echo "the run step's finally did not put the Corefile back" >&2
false
}
jq -e '.data.Corefile == ".:53 {\n errors\n}\n"' "$STUB/replaced.json" >/dev/null
}
# The Altinity driver names the Backup after the BackupJob, and a Backup left
# behind under that name records an earlier run's archive, which fails the next
# run's BackupJob. It is reclaimed by name, whether or not the BackupJob
# recorded it in status.backupRef.
@test "the round-trip reclaims its Backup by name before and after the run" {
suite=hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml
test='select(.metadata.name == "clickhouse-3-system-bucket-roundtrip") | .spec.steps[]'
calls="$STUB/calls"
# Nothing is left over from an earlier run, and no BackupJob recorded a
# backupRef.
cat >"$STUB/kubectl" <<'STUBEOF'
#!/bin/bash
echo "$*" >>"$STUB/calls"
case "$*" in
*" get secret "*) exit 1 ;;
esac
STUBEOF
for selector in \
'select(.name == "preconditions") | .try[].script.content' \
'select(.name == "run") | .finally[] | select(.description == "system-bucket round-trip cleanup") | .script.content'; do
: >"$calls"
script=$(yq "$test | $selector" "$suite")
[ -n "$script" ] || { echo "no script at: $selector" >&2; false; }
PATH="$STUB:$PATH" NAMESPACE=tenant-root sh -c "$script"
grep -qE '^-n tenant-root delete backup sysbucket-backup( |$)' "$calls" || {
echo "the script at '$selector' does not delete Backup sysbucket-backup by name; kubectl calls:" >&2
cat "$calls" >&2
false
}
done
}
# cert-manager issues the Secret of a Certificate that still exists again as
# soon as it is deleted. Chainsaw deletes the Certificate it applied, and waits
# for it, before it runs the cleanup ops of the same step, so the Secret is
# deleted there and nowhere earlier.
@test "the Certificate's Secret is deleted after Chainsaw deletes the Certificate" {
test='select(.metadata.name == "clickhouse-3-system-bucket-roundtrip") | .spec.steps[]'
suite=hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml
applies=$(yq "$test | select(.name == \"create-certificate-and-probe\") | .try[] | select(has(\"apply\")) | .apply.file" "$suite")
[ "$applies" = system-bucket-tls.yaml ] || {
echo "step create-certificate-and-probe no longer applies system-bucket-tls.yaml alone: $applies" >&2
false
}
deletes=$(yq -o=json -I=0 "$test | select(.name == \"create-certificate-and-probe\") | .cleanup" "$suite")
[ "$deletes" = '[{"delete":{"ref":{"apiVersion":"v1","kind":"Secret","name":"e2e-system-bucket-s3-tls"}}}]' ] || {
echo "the cleanup of step create-certificate-and-probe is not the one delete of Secret e2e-system-bucket-s3-tls: $deletes" >&2
false
}
# The preconditions run before the apply, when Chainsaw holds no Certificate.
if yq "$test | select(.name != \"preconditions\") | (.try[], .finally[]?) | .script.content // \"\"" "$suite" | grep -qE 'delete +(secret +e2e-system-bucket-s3-tls|certificate +e2e-system-bucket-s3)( |$)'; then
echo "a step script deletes the Certificate or its Secret while Chainsaw still holds the Certificate" >&2
false
fi
}
# Chainsaw registers an applied object for deletion only when its apply created
# it; an object left by a run killed past its cleanup is patched and kept. The
# Certificate would then outlive every later run, re-issuing its Secret after
# each step cleanup, and a probe Pod whose `sleep` had ended never turns Ready
# again, so the preconditions remove all three before the apply.
@test "the preconditions remove the TLS objects and probe Pod a killed run left" {
suite=hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml
test='select(.metadata.name == "clickhouse-3-system-bucket-roundtrip") | .spec.steps[]'
calls="$STUB/calls"
cat >"$STUB/kubectl" <<'STUBEOF'
#!/bin/bash
echo "$*" >>"$STUB/calls"
case "$*" in
*" get secret "*) exit 1 ;;
esac
STUBEOF
: >"$calls"
script=$(yq "$test | select(.name == \"preconditions\") | .try[].script.content" "$suite")
PATH="$STUB:$PATH" NAMESPACE=tenant-root sh -c "$script"
line_of() { grep -nE "^-n tenant-root delete $1 --ignore-not-found --wait=true( |$)" "$calls" | head -n1 | cut -d: -f1; }
cert=$(line_of 'certificate e2e-system-bucket-s3')
secret=$(line_of 'secret e2e-system-bucket-s3-tls')
pod=$(line_of 'pod e2e-system-bucket-probe')
[ -n "$cert" ] && [ -n "$secret" ] && [ -n "$pod" ] || {
echo "the preconditions do not delete, and wait for, the Certificate, its Secret and the probe Pod; kubectl calls:" >&2
cat "$calls" >&2
false
}
[ "$cert" -lt "$secret" ] || {
echo "the preconditions delete Secret e2e-system-bucket-s3-tls before its Certificate, which cert-manager then issues again" >&2
false
}
}
+367
View File
@@ -0,0 +1,367 @@
#!/usr/bin/env bats
# Tests for hack/close-manual-backport.sh, which closes a pull request opened by
# hand against a release branch by someone outside the team.
#
# gh is stubbed: every call is recorded, the permission endpoint answers with
# $PERM_JSON (or fails under PERM_FAIL), and any other `gh api` with $PR_JSON.
# A case that expects the pull request to stay open asserts exactly which reads
# gh recorded, so a close that slips past a guard cannot go unseen.
#
# Run with: bats hack/close-manual-backport.bats
load test_helper
SCRIPT="${BATS_TEST_DIRNAME}/close-manual-backport.sh"
WF="${BATS_TEST_DIRNAME}/../.github/workflows/pr-labeler.yaml"
POLICY_URL="https://github.com/cozystack/cozystack/blob/main/docs/release/backports.md"
PERM_CALL="api repos/cozystack/cozystack/collaborators/someone/permission"
# The close-manual-backport job of pr-labeler.yaml, comment lines dropped so a
# setting demoted to a comment cannot satisfy an assertion.
guard_job() {
awk '/^ close-manual-backport:$/ { f = 1; next } f && /^ [^ ]/ { exit } f' "$WF" \
| grep -v '^[[:space:]]*#'
}
_stub_gh() {
mkdir -p "$1"
cat > "$1/gh" <<'EOF'
#!/bin/sh
printf '%s\n' "$*" >> "$GH_CALLS"
case "$1 $2" in
"api repos/"*/collaborators/*/permission)
if [ -n "${PERM_FAIL:-}" ]; then
echo 'gh: Not Found (HTTP 404)' >&2
exit 1
fi
cat "$PERM_JSON"
;;
"api "*) cat "$PR_JSON" ;;
esac
EOF
chmod +x "$1/gh"
}
setup() {
strict_setup
_stub_gh "${BATS_TEST_TMPDIR}/bin"
export PATH="${BATS_TEST_TMPDIR}/bin:$PATH"
export GH_CALLS="${BATS_TEST_TMPDIR}/calls"
export PR_JSON="${BATS_TEST_TMPDIR}/pr.json"
export PERM_JSON="${BATS_TEST_TMPDIR}/perm.json"
unset PERM_FAIL
set_permission read read
EVENT="${BATS_TEST_TMPDIR}/event.json"
# A run inside GitHub Actions has its own event here; never read it.
export GITHUB_EVENT_PATH="$EVENT"
: > "$GH_CALLS"
}
# The permission endpoint's answer: legacy permission, then role_name.
set_permission() {
printf '{"permission":"%s","role_name":"%s","user":{"login":"someone"}}\n' "$1" "$2" > "$PERM_JSON"
}
# Writes a pull_request_target event to $EVENT and its pull request object to
# $PR_JSON. Arguments: base ref, author association, author type, labels as a
# JSON array of names, and the author's login, someone by default. "-" leaves
# the association or the type out.
make_event() {
jq -n --arg base "$1" --arg assoc "$2" --arg type "$3" --argjson labels "$4" --arg login "${5:-someone}" '
({number: 42, state: "open",
base: {ref: $base, repo: {full_name: "cozystack/cozystack"}},
user: {login: $login, type: $type},
author_association: $assoc,
labels: ($labels | map({name: .}))}
| if $assoc == "-" then del(.author_association) else . end
| if $type == "-" then del(.user.type) else . end) as $pr
| {action: "opened", number: 42, pull_request: $pr,
repository: {full_name: "cozystack/cozystack"}}' > "$EVENT"
jq '.pull_request' "$EVENT" > "$PR_JSON"
}
assert_left_open() {
[ "$status" -eq 0 ] || { echo "FAIL: exit $status: $output"; false; }
if [ -s "$GH_CALLS" ]; then
echo "FAIL: gh was called:"
cat "$GH_CALLS"
false
fi
case "$output" in
*"leaving it open"*) ;;
*) echo "FAIL: no reason given: $output"; false ;;
esac
}
# Failed after the permission read, which is then the only gh call, with
# nothing closed and why said.
assert_failed_after_permission() {
[ "$status" -eq 1 ] || { echo "FAIL: exit $status: $output"; false; }
[ "$(cat "$GH_CALLS")" = "$PERM_CALL" ] || { echo "FAIL: gh calls:"; cat "$GH_CALLS"; false; }
case "$output" in
*"closing nothing"*) ;;
*) echo "FAIL: no reason given: $output"; false ;;
esac
}
# Left open after the permission read, which is then the only gh call.
assert_left_open_after_permission() {
[ "$status" -eq 0 ] || { echo "FAIL: exit $status: $output"; false; }
[ "$(cat "$GH_CALLS")" = "$PERM_CALL" ] || { echo "FAIL: gh calls:"; cat "$GH_CALLS"; false; }
case "$output" in
*"leaving it open"*) ;;
*) echo "FAIL: no reason given: $output"; false ;;
esac
}
@test "closes a pull request to release-1.6 from outside the team, linking the policy" {
for assoc in CONTRIBUTOR FIRST_TIME_CONTRIBUTOR FIRST_TIMER NONE; do
: > "$GH_CALLS"
make_event release-1.6 "$assoc" User '["kind/bug"]'
run "$SCRIPT" "$EVENT"
[ "$status" -eq 0 ] || { echo "FAIL: $assoc: exit $status: $output"; false; }
[ "$(wc -l < "$GH_CALLS" | tr -d ' ')" -eq 2 ] || { echo "FAIL: $assoc:"; cat "$GH_CALLS"; false; }
[ "$(sed -n 1p "$GH_CALLS")" = "$PERM_CALL" ]
sed -n 2p "$GH_CALLS" | grep -q '^pr close 42 --repo cozystack/cozystack --comment Thank you'
grep -qF "$POLICY_URL" "$GH_CALLS"
grep -qF 'a maintainer can reopen it.' "$GH_CALLS"
if grep -qF '—' "$GH_CALLS"; then echo "FAIL: the comment carries an em dash"; false; fi
done
}
@test "leaves OWNER, MEMBER and COLLABORATOR pull requests open" {
for assoc in OWNER MEMBER COLLABORATOR; do
: > "$GH_CALLS"
make_event release-1.6 "$assoc" User '[]'
run "$SCRIPT" "$EVENT"
assert_left_open
done
}
@test "leaves a bot's pull request open" {
make_event release-1.6 NONE Bot '[]'
run "$SCRIPT" "$EVENT"
assert_left_open
}
# The scheduled batch opens its pull requests as the cozystack-ci App, which
# GitHub can report as a contributor once it has commits here.
@test "leaves the batch's own pull request open" {
for assoc in NONE CONTRIBUTOR; do
: > "$GH_CALLS"
make_event release-1.7 "$assoc" Bot '[]' 'cozystack-ci[bot]'
run "$SCRIPT" "$EVENT"
assert_left_open
done
}
# No label keeps a pull request open: what keeps one is a maintainer
# reopening it, which the guard does not act on.
@test "closes a pull request whatever labels it carries" {
make_event release-1.6 CONTRIBUTOR User '["kind/bug","release-only","backport/include"]'
run "$SCRIPT" "$EVENT"
[ "$status" -eq 0 ] || { echo "FAIL: exit $status: $output"; false; }
sed -n 2p "$GH_CALLS" | grep -q '^pr close 42 --repo cozystack/cozystack --comment ' \
|| { echo "FAIL: not closed:"; cat "$GH_CALLS"; false; }
}
@test "acts only on an opened event, so a maintainer keeps a fix by reopening it" {
for action in reopened synchronize edited labeled ''; do
: > "$GH_CALLS"
make_event release-1.6 CONTRIBUTOR User '[]'
jq --arg a "$action" 'if $a == "" then del(.action) else .action = $a end' "$EVENT" > "$EVENT.tmp"
mv "$EVENT.tmp" "$EVENT"
run "$SCRIPT" "$EVENT"
assert_left_open
case "$output" in
*"not opened"*) ;;
*) echo "FAIL: reason does not name the event ($action): $output"; false ;;
esac
done
}
@test "leaves pull requests to main and to other branches open" {
for base in main feature-release-1.6 releases; do
: > "$GH_CALLS"
make_event "$base" CONTRIBUTOR User '[]'
run "$SCRIPT" "$EVENT"
assert_left_open
done
}
@test "does nothing when the author association is missing" {
make_event release-1.6 - User '[]'
run "$SCRIPT" "$EVENT"
assert_left_open
case "$output" in
*"no author association"*) ;;
*) echo "FAIL: reason does not name the association: $output"; false ;;
esac
}
@test "does nothing on an unknown association, a missing author type or an odd login" {
make_event release-1.6 SOMETHING_NEW User '[]'
run "$SCRIPT" "$EVENT"
assert_left_open
: > "$GH_CALLS"
make_event release-1.6 CONTRIBUTOR - '[]'
run "$SCRIPT" "$EVENT"
assert_left_open
# The login goes into the API path, so only a GitHub login shape is asked about.
: > "$GH_CALLS"
make_event release-1.6 CONTRIBUTOR User '[]'
jq '.pull_request.user.login = "someone/../../orgs/x"' "$EVENT" > "$EVENT.tmp"
mv "$EVENT.tmp" "$EVENT"
run "$SCRIPT" "$EVENT"
assert_left_open
}
# author_association shows an organization member whose membership is private
# as a CONTRIBUTOR. `permission` alone would also pass a triage member, which
# GitHub folds into read.
@test "a private member reported as CONTRIBUTOR stays open when their access is above read" {
for answer in "write write" "admin admin" "write maintain" "read triage" "read some-custom-role"; do
: > "$GH_CALLS"
set -- $answer
set_permission "$1" "$2"
make_event release-1.6 CONTRIBUTOR User '[]'
run "$SCRIPT" "$EVENT"
assert_left_open_after_permission
case "$output" in
*"permission on cozystack/cozystack is $1 (role $2)"*) ;;
*) echo "FAIL: reason does not name the access ($answer): $output"; false ;;
esac
done
}
@test "a failed or unreadable permission answer is an error and closes nothing" {
make_event release-1.6 CONTRIBUTOR User '[]'
export PERM_FAIL=1
run "$SCRIPT" "$EVENT"
assert_failed_after_permission
case "$output" in
*"cannot read someone's permission"*) ;;
*) echo "FAIL: reason does not name the failed read: $output"; false ;;
esac
unset PERM_FAIL
for answer in 'not json' '{}' '{"permission":"read"}' '{"role_name":"read"}' '{"permission":"read","role_name":null}' '[]'; do
: > "$GH_CALLS"
printf '%s\n' "$answer" > "$PERM_JSON"
run "$SCRIPT" "$EVENT"
assert_failed_after_permission
case "$output" in
*"does not give both permission and role_name"*) ;;
*) echo "FAIL: reason does not name the unreadable answer ($answer): $output"; false ;;
esac
done
}
@test "an author with read or no access is closed" {
for answer in "read read" "none none" "read none" "none read"; do
: > "$GH_CALLS"
set -- $answer
set_permission "$1" "$2"
make_event release-1.6 CONTRIBUTOR User '[]'
run "$SCRIPT" "$EVENT"
[ "$status" -eq 0 ] || { echo "FAIL: $answer: exit $status: $output"; false; }
[ "$(sed -n 1p "$GH_CALLS")" = "$PERM_CALL" ] || { echo "FAIL: $answer:"; cat "$GH_CALLS"; false; }
sed -n 2p "$GH_CALLS" | grep -q '^pr close 42 --repo cozystack/cozystack --comment ' \
|| { echo "FAIL: $answer: not closed:"; cat "$GH_CALLS"; false; }
done
}
@test "--dry-run never calls a write verb" {
make_event release-1.6 CONTRIBUTOR User '[]'
# From the event the one gh call allowed is the permission read.
run "$SCRIPT" --dry-run "$EVENT"
[ "$status" -eq 0 ]
[ "$(cat "$GH_CALLS")" = "$PERM_CALL" ] || { echo "FAIL: gh calls:"; cat "$GH_CALLS"; false; }
case "$output" in
*"would close PR #42 in cozystack/cozystack"*"$POLICY_URL"*) ;;
*) echo "FAIL: dry run did not report the close: $output"; false ;;
esac
# By number, the pull request read and the permission read.
: > "$GH_CALLS"
run "$SCRIPT" --dry-run --pr 42
[ "$status" -eq 0 ]
[ "$(cat "$GH_CALLS")" = "$(printf 'api repos/{owner}/{repo}/pulls/42\n%s' "$PERM_CALL")" ] \
|| { echo "FAIL: gh calls:"; cat "$GH_CALLS"; false; }
case "$output" in
*"would close PR #42"*) ;;
*) echo "FAIL: dry run did not report the close: $output"; false ;;
esac
}
@test "--pr reads the pull request with gh and closes it" {
make_event release-1.6 CONTRIBUTOR User '[]'
run "$SCRIPT" --pr 42
[ "$status" -eq 0 ]
[ "$(wc -l < "$GH_CALLS" | tr -d ' ')" -eq 3 ]
[ "$(sed -n 1p "$GH_CALLS")" = 'api repos/{owner}/{repo}/pulls/42' ]
[ "$(sed -n 2p "$GH_CALLS")" = "$PERM_CALL" ]
sed -n 3p "$GH_CALLS" | grep -q '^pr close 42 --repo cozystack/cozystack --comment '
}
@test "leaves a pull request that is already closed alone" {
make_event release-1.6 CONTRIBUTOR User '[]'
jq '.state = "closed"' "$PR_JSON" > "$PR_JSON.tmp"
mv "$PR_JSON.tmp" "$PR_JSON"
run "$SCRIPT" --pr 42
[ "$status" -eq 0 ]
[ "$(cat "$GH_CALLS")" = 'api repos/{owner}/{repo}/pulls/42' ]
case "$output" in
*"is not open"*) ;;
*) echo "FAIL: $output"; false ;;
esac
}
@test "an unreadable event is an error and closes nothing" {
printf 'not json' > "$EVENT"
run "$SCRIPT" "$EVENT"
[ "$status" -eq 1 ]
if [ -s "$GH_CALLS" ]; then echo "FAIL: gh was called:"; cat "$GH_CALLS"; false; fi
}
@test "a non-numeric --pr is a usage error" {
run "$SCRIPT" --pr '42; gh pr close 1'
[ "$status" -eq 2 ]
if [ -s "$GH_CALLS" ]; then echo "FAIL: gh was called:"; cat "$GH_CALLS"; false; fi
}
@test "pr-labeler runs the guard only on opened pull requests to release-*" {
job="$(guard_job)"
[ -n "$job" ]
printf '%s\n' "$job" | grep -qxF " github.event.action == 'opened' &&"
if printf '%s\n' "$job" | grep -qF 'reopened'; then echo "FAIL: the job runs on reopened"; false; fi
printf '%s\n' "$job" | grep -qF "startsWith(github.event.pull_request.base.ref, 'release-')"
printf '%s\n' "$job" | grep -qxF ' run: hack/close-manual-backport.sh "$GITHUB_EVENT_PATH"'
}
# pull_request_target hands a write token to a fork's pull request. The job is
# safe only while it holds no more than it needs and never reaches the head.
@test "the guard job holds the least token and never checks out the pull request" {
job="$(guard_job)"
[ -n "$job" ]
perms="$(printf '%s\n' "$job" | awk '/^ permissions:$/ { f = 1; next } f && /^ [^ ]/ { exit } f' \
| sed 's/[[:space:]]*#.*$//' | sort)"
[ "$perms" = "$(printf ' contents: read\n pull-requests: write')" ] || {
echo "FAIL: permissions are:"; printf '%s\n' "$perms"; false
}
# actions/checkout without a ref takes the event's own, which on
# pull_request_target is the base repository; a ref is the way to the head.
if printf '%s\n' "$job" | grep -qE '^[[:space:]]+ref:'; then echo "FAIL: the checkout names a ref"; false; fi
printf '%s\n' "$job" | grep -qxF ' persist-credentials: false'
# The token is the only expression: event fields reach the script through
# the JSON file, never through the shell.
[ "$(printf '%s\n' "$job" | grep -oF '${{' | wc -l | tr -d ' ')" -eq 1 ]
printf '%s\n' "$job" | grep -qxF ' GH_TOKEN: ${{ github.token }}'
}
+156
View File
@@ -0,0 +1,156 @@
#!/bin/sh
# Close a pull request opened by hand against a release branch by someone
# outside the team. Fixes reach the release lines through the backport batch
# (docs/release/backports.md), so a hand-made backport is work nobody needs. It
# acts only on an `opened` event: a maintainer keeps a fix that exists only on
# a release line by reopening it.
#
# Usage: close-manual-backport.sh [--dry-run] [EVENT_JSON]
# close-manual-backport.sh [--dry-run] --pr NUMBER
#
# EVENT_JSON defaults to $GITHUB_EVENT_PATH and is a pull_request or
# pull_request_target payload. --pr reads the same REST pull request object
# with `gh api`, because `gh pr view --json` has no author association. Only
# JSON is read, that object and the author's repository permission: nothing
# from the pull request is checked out or run.
#
# The pull request is closed only when the event, if there is one, is
# `opened`, the pull request is open and targets release-*, its author's
# association is not OWNER, MEMBER or COLLABORATOR, its author is not a bot,
# and GitHub answers that the author's permission and role on the repository
# are both read or none. A missing field or a value this script does not know
# leaves it open and exits 0. Exits 1, closing nothing, when the input or
# GitHub's answer on the author cannot be read, 2 on a usage error, and with
# gh's status when the close fails.
set -eu
usage() {
echo "usage: close-manual-backport.sh [--dry-run] [EVENT_JSON | --pr NUMBER]" >&2
exit 2
}
leave() {
echo "close-manual-backport: $*; leaving it open."
exit 0
}
dry_run=false
pr_number=
event=
while [ "$#" -gt 0 ]; do
case "$1" in
--dry-run) dry_run=true ;;
--pr)
[ "$#" -ge 2 ] || usage
pr_number=$2
shift
;;
-*) usage ;;
*)
[ -z "$event" ] || usage
event=$1
;;
esac
shift
done
command -v jq >/dev/null 2>&1 || { echo "close-manual-backport: jq is required" >&2; exit 1; }
if [ -n "$pr_number" ]; then
[ -z "$event" ] || usage
case "$pr_number" in '' | *[!0-9]*) usage ;; esac
if ! pr=$(gh api "repos/{owner}/{repo}/pulls/$pr_number"); then
echo "close-manual-backport: cannot read PR #$pr_number with gh" >&2
exit 1
fi
else
event=${event:-${GITHUB_EVENT_PATH:-}}
[ -n "$event" ] || usage
if ! pr=$(jq -c '.pull_request' "$event" 2>/dev/null); then
echo "close-manual-backport: cannot read the event JSON at $event" >&2
exit 1
fi
action=$(jq -r '.action | strings' "$event" 2>/dev/null || true)
[ "$action" = opened ] || leave "the event is ${action:-missing}, not opened"
fi
# A filter that fails on an unexpected shape prints nothing, which every check
# below reads as a missing field.
json_field() {
printf '%s' "$1" | jq -r "$2" 2>/dev/null || true
}
field() {
json_field "$pr" "$1"
}
[ "$(field 'type')" = object ] || leave "the input carries no pull request"
number=$(field '.number | select(type == "number" and . > 0 and . == floor) | tostring')
[ -n "$number" ] || leave "the pull request carries no number"
base=$(field '.base.ref | strings')
case "$base" in
release-*) ;;
'') leave "PR #$number carries no base ref" ;;
*) leave "PR #$number targets $base, not a release branch" ;;
esac
state=$(field '.state | strings')
[ "$state" = open ] || leave "PR #$number is not open (state: ${state:-missing})"
assoc=$(field '.author_association | strings')
case "$assoc" in
OWNER | MEMBER | COLLABORATOR) leave "PR #$number is from a repository $assoc" ;;
CONTRIBUTOR | FIRST_TIME_CONTRIBUTOR | FIRST_TIMER | NONE) ;;
'') leave "PR #$number carries no author association" ;;
*) leave "PR #$number has an author association this script does not know ($assoc)" ;;
esac
user_type=$(field '.user.type | strings')
case "$user_type" in
User) ;;
Bot) leave "PR #$number is from a bot" ;;
'') leave "PR #$number carries no author type" ;;
*) leave "PR #$number has an author type this script does not know ($user_type)" ;;
esac
repo=$(field '.base.repo.full_name | strings')
case "$repo" in
*/*/* | *[!A-Za-z0-9._/-]* | /* | */) leave "PR #$number names no usable base repository" ;;
*/*) ;;
*) leave "PR #$number names no usable base repository" ;;
esac
login=$(field '.user.login | strings')
case "$login" in
'' | *[!A-Za-z0-9-]*) leave "PR #$number carries no usable author login" ;;
esac
# author_association reports an organization member whose membership is
# private as a CONTRIBUTOR, so the author's own access to the repository is the
# check that decides. `permission` folds maintain into write and triage into
# read, which is why `role_name` has to agree. An answer that cannot be read
# fails the job: left open in silence, the pull request would look decided.
if ! access=$(gh api "repos/$repo/collaborators/$login/permission"); then
echo "close-manual-backport: PR #$number: cannot read $login's permission on $repo; closing nothing" >&2
exit 1
fi
permission=$(json_field "$access" '.permission | strings')
role=$(json_field "$access" '.role_name | strings')
if [ -z "$permission" ] || [ -z "$role" ]; then
echo "close-manual-backport: PR #$number: GitHub's answer on $login's permission on $repo does not give both permission and role_name; closing nothing" >&2
exit 1
fi
case "$permission:$role" in
read:read | read:none | none:read | none:none) ;;
*) leave "PR #$number is from $login, whose permission on $repo is $permission (role $role)" ;;
esac
policy="https://github.com/$repo/blob/main/docs/release/backports.md"
comment="Thank you for the contribution! Backports are automatic: fixes merged to \`main\` reach the release lines the batch runs for on their own, as the [backport policy]($policy) describes, so this pull request is closed. If the fix only makes sense on a release line and has no counterpart on \`main\`, a maintainer can reopen it."
if [ "$dry_run" = true ]; then
printf 'close-manual-backport: would close PR #%s in %s with this comment:\n%s\n' "$number" "$repo" "$comment"
exit 0
fi
gh pr close "$number" --repo "$repo" --comment "$comment"
echo "close-manual-backport: closed PR #$number in $repo."
+17 -15
View File
@@ -22,8 +22,11 @@ extract_has() {
}
DOCKERFILE="$PWD/packages/system/multus/images/multus-cni/Dockerfile"
KUBEOVN_VALUES="$PWD/packages/system/kubeovn/values.yaml"
DOC="$PWD/docs/vm-external-vlan.md"
# The chart's appVersion, not the values.yaml image tag: a release build
# restamps that tag to the Cozystack version (v1.7.0), the chart keeps the
# kube-ovn release it vendors.
KUBEOVN_CHART="$PWD/packages/system/kubeovn/charts/kube-ovn/Chart.yaml"
DOC="$PWD/packages/system/multus/README.md"
PLATFORM_VALUES="$PWD/packages/core/platform/values.yaml"
TEMPLATE="$PWD/packages/system/multus/templates/multus-daemonset-thick.yml"
PATCHFILE="$PWD/packages/system/multus/patches/customize-deployment.patch"
@@ -156,7 +159,7 @@ PATCHFILE="$PWD/packages/system/multus/patches/customize-deployment.patch"
if extract_has "$extract" "$plugin"; then
echo "packages/system/multus/images/multus-cni/Dockerfile now extracts" >&2
echo "./$plugin, which this package deliberately does not stage, and" >&2
echo "which docs/vm-external-vlan.md names as left alone." >&2
echo "which packages/system/multus/README.md names as left alone." >&2
if [ "$plugin" = loopback ]; then
echo "The init container would then replace the node's loopback plugin" >&2
echo "on every restart. The runtime calls loopback for every pod" >&2
@@ -168,7 +171,7 @@ PATCHFILE="$PWD/packages/system/multus/patches/customize-deployment.patch"
echo "long-running daemon and a socket on the host, which this" >&2
echo "package does not run." >&2
fi
echo "Update the guide in the same commit if this is deliberate." >&2
echo "Update the multus README in the same commit if this is deliberate." >&2
return 1
fi
done
@@ -196,7 +199,7 @@ PATCHFILE="$PWD/packages/system/multus/patches/customize-deployment.patch"
printf '%s\n' "$staged" >&2
echo "expected these 14:" >&2
printf '%s\n' "$expected" >&2
echo "docs/vm-external-vlan.md tells operators the package installs 14 of" >&2
echo "packages/system/multus/README.md tells operators the package installs 14 of" >&2
echo "the upstream reference plugins and names the four it leaves alone," >&2
echo "so that sentence is now wrong. Update it in the same commit." >&2
return 1
@@ -205,7 +208,7 @@ PATCHFILE="$PWD/packages/system/multus/patches/customize-deployment.patch"
for plugin in bridge macvlan ipvlan; do
if ! extract_has "$extract" "$plugin"; then
echo "packages/system/multus/images/multus-cni/Dockerfile no longer extracts" >&2
echo "./$plugin into /cni-plugins, but docs/vm-external-vlan.md tells operators" >&2
echo "./$plugin into /cni-plugins, but packages/system/multus/README.md tells operators" >&2
echo "the package stages it. The init container installs whatever is in that" >&2
echo "directory and skips silently when it is absent, so a NAD naming" >&2
echo "$plugin would go back to failing with" >&2
@@ -347,13 +350,12 @@ PATCHFILE="$PWD/packages/system/multus/patches/customize-deployment.patch"
checked=$(sed -n 's|^# kube-ovn-cni-plugins-checked-against: *\([^ ]*\) *$|\1|p' "$DOCKERFILE")
[ -n "$checked" ]
vendored=$(yq -r '.global.images.kubeovn.tag' "$KUBEOVN_VALUES" | sed 's|@.*||')
[ -n "$vendored" ]
[ "$vendored" != "null" ]
vendored=$(awk '$1 == "appVersion:" {gsub(/"/, "", $2); print "v" $2}' "$KUBEOVN_CHART")
[ "$vendored" != "v" ]
if [ "$checked" != "$vendored" ]; then
echo "the multus CNI plugin pin was checked against kube-ovn $checked," >&2
echo "but packages/system/kubeovn/values.yaml now vendors $vendored." >&2
echo "but packages/system/kubeovn/charts/kube-ovn/Chart.yaml now vendors $vendored." >&2
echo >&2
echo "Read CNI_PLUGINS_VERSION in kube-ovn $vendored's base image" >&2
echo "(dist/images/Dockerfile.base). If it still matches the" >&2
@@ -415,8 +417,8 @@ PATCHFILE="$PWD/packages/system/multus/patches/customize-deployment.patch"
fi
}
@test "the VLAN guide's remediation quotes what the daemonset actually emits" {
# The guide names a log line and a container; both spellings live in the
@test "the multus README's remediation quotes what the daemonset actually emits" {
# The README names a log line and a container; both spellings live in the
# manifest.
grep -qF -- 'no /cni-plugins in this image; skipping staging' "$TEMPLATE"
grep -qF -- 'no /cni-plugins in this image; skipping staging' "$DOC"
@@ -435,13 +437,13 @@ PATCHFILE="$PWD/packages/system/multus/patches/customize-deployment.patch"
# only loopback arrives from anywhere else (cilium and kube-ovn install it).
grep -qF -- 'and leaves `loopback`, `dhcp`, `dummy` and `tap` alone' "$DOC"
if grep -qF -- '`tap` to the node image' "$DOC"; then
echo "the guide again attributes the unstaged plugins to the node image;" >&2
echo "the README again attributes the unstaged plugins to the node image;" >&2
echo "staging exists because that image may not carry them" >&2
return 1
fi
if grep -qF -- 'immediately above' "$DOC"; then
echo "the guide again promises the cause sits immediately above the" >&2
echo "the README again promises the cause sits immediately above the" >&2
echo "'failed to install:' summary. It does not when more than one plugin" >&2
echo "fails: the summary is printed once, after the loop." >&2
return 1
@@ -456,7 +458,7 @@ PATCHFILE="$PWD/packages/system/multus/patches/customize-deployment.patch"
# the general case -- that one stays open deliberately.
grep -qE -- '(^|[^[:alnum:]])set the opt-out before upgrading' "$DOC"
if grep -qiE -- '(do not|never|no need to|dont|don.t) set the opt-out' "$DOC"; then
echo "the guide now tells operators NOT to set the opt-out before" >&2
echo "the README now tells operators NOT to set the opt-out before" >&2
echo "upgrading. Declining afterwards does not restore a replaced plugin," >&2
echo "so the instruction has to stay as it is." >&2
return 1
+11 -5
View File
@@ -66,15 +66,21 @@ PLATFORM ?= $(if $(filter 1,$(LOAD)),,$(DEFAULT_PLATFORMS))
BUILDX_EXTRA_ARGS ?=
COZYSTACK_VERSION = $(patsubst v%,%,$(shell git describe --tags --match 'v*'))
# SBOM=1 attaches a CycloneDX SBOM as a buildx build attestation (--sbom=true).
# Requires the docker-container buildx driver and registry referrers/attestation
# support. Kept OFF by default until an OCIR referrers/attestation probe confirms
# support — otherwise the attestation push fails the build. See #2937.
# SBOM=1 attaches an SPDX SBOM to every image as a BuildKit attestation. The
# pushed ref then names an index of the image and its attestation manifest, not
# the image itself, which hack/stitch-multiarch.sh and hack/verify-multiarch.sh
# both handle. Attestations need the docker-container buildx driver. The scanner
# is pinned by digest like every other build input, where --sbom=true would pull
# a floating tag. Renovate bumps the version and the digest together, through
# the manager in .github/renovate.json that reads the SBOM_SCANNER line below.
SBOM ?= 0
SBOM_SCANNER ?= docker.io/docker/buildkit-syft-scanner:1.12.0@sha256:ae4f3b554449e7e25548e7d8ccc029d17357348e30c6e3df01b92bc93654d6a9
# A variable, because the commas would split the arguments of $(if) below.
SBOM_ATTEST := --attest type=sbom,generator=$(SBOM_SCANNER)
BUILDX_ARGS := --provenance=false --push=$(PUSH) --load=$(LOAD) \
--label org.opencontainers.image.source=https://github.com/cozystack/cozystack \
$(if $(filter 1,$(SBOM)),--sbom=true) \
$(if $(filter 1,$(SBOM)),$(SBOM_ATTEST)) \
$(if $(strip $(BUILDER)),--builder=$(BUILDER)) \
$(if $(strip $(PLATFORM)),--platform=$(PLATFORM)) \
$(BUILDX_EXTRA_ARGS)
+28
View File
@@ -144,6 +144,34 @@ load test_helper
echo "$out" | grep -q -- '--push=0 --load=1'
}
@test "SBOM=1 attaches an SPDX attestation from a scanner pinned by digest, and the default builds none" {
# An image built with an attestation is an index, not a plain manifest, which
# the stitch and the gate have to read, so it stays off unless a build asks.
# --sbom=true would pull the scanner by a floating tag.
out=$(make -n -C packages/system/cozystack-controller image IMAGE_TAG=pr-1-abc COZYSTACK_VERSION=0 BUILDER=b)
if echo "$out" | grep -q -- '--attest\|--sbom'; then echo "FAIL: an attestation without SBOM=1"; false; fi
out=$(make -n -C packages/system/cozystack-controller image IMAGE_TAG=pr-1-abc COZYSTACK_VERSION=0 BUILDER=b SBOM=1)
echo "$out" | grep -qE -- '--attest type=sbom,generator=[^ ]+@sha256:[0-9a-f]{64} ' || { echo "FAIL: no digest-pinned scanner"; false; }
if echo "$out" | grep -q -- '--sbom'; then echo "FAIL: --sbom=true pulls a floating scanner tag"; false; fi
# The environment reaches buildx as well, which is how the release workflow sets it.
out=$(SBOM=1 make -n -C packages/system/cozystack-controller image IMAGE_TAG=pr-1-abc COZYSTACK_VERSION=0 BUILDER=b)
echo "$out" | grep -q -- '--attest type=sbom,'
}
@test "Renovate moves the SBOM scanner pin, tag and digest together" {
# Renovate reaches hack/common-envs.mk only through a custom manager, and a
# reformatted SBOM_SCANNER line drops out of its regexp without a word, so the
# pin would stay on its syft. The named groups are Renovate's: depName,
# currentValue and currentDigest, in that order.
re=$(jq -r '.customManagers[] | select(any(.managerFilePatterns[]; test("common-envs"))) | .matchStrings[0]' .github/renovate.json)
[ -n "$re" ] || { echo "FAIL: no Renovate custom manager covers hack/common-envs.mk"; false; }
ere=$(printf '%s' "$re" | sed -E 's/\(\?<[A-Za-z]+>/(/g')
pin=$(sed -nE "s#^.*${ere}.*\$#\\1:\\2@\\3#p" hack/common-envs.mk)
[ -n "$pin" ] || { echo "FAIL: the manager matches no SBOM_SCANNER line in hack/common-envs.mk"; false; }
out=$(make -n -C packages/system/cozystack-controller image IMAGE_TAG=pr-1-abc COZYSTACK_VERSION=0 BUILDER=b SBOM=1)
echo "$out" | grep -qF -- "generator=${pin} " || { echo "FAIL: the manager reads ${pin}, not the scanner the build passes"; false; }
}
@test "talos publishes nothing to the registry with PUSH=0" {
out=$(make -n -B -C packages/core/talos image-talos PUSH=0 PUBLISH_VERSIONED=1 PUBLISH_FLOATING=1 IMAGE_TAG=pr-1-abc COZYSTACK_VERSION=0)
echo "$out" | grep -q 'docker://'
+2 -2
View File
@@ -37,11 +37,11 @@ nginx_size_kib() {
cd "$REPO_ROOT"
conf=$(awk '
/^COPY <<.EOF. \/etc\/nginx\/conf\.d\/default\.conf$/ { inblock = 1; next }
/^COPY <<.EOF. \/etc\/nginx\/templates\/default\.conf\.template$/ { inblock = 1; next }
inblock && /^EOF$/ { inblock = 0; next }
inblock { print }
' "$CONSOLE_CONTAINERFILE")
[ -n "$conf" ] || { echo "no /etc/nginx/conf.d/default.conf heredoc found in $CONSOLE_CONTAINERFILE" >&2; exit 1; }
[ -n "$conf" ] || { echo "no /etc/nginx/templates/default.conf.template heredoc found in $CONSOLE_CONTAINERFILE" >&2; exit 1; }
# An extraction that silently grabbed the wrong slice would let every
# assertion below pass on an empty-ish config, so anchor on a line the
+243
View File
@@ -0,0 +1,243 @@
#!/usr/bin/env bats
# The console nginx sends the logs and metrics roots of the observability
# proxy to that proxy instead of the API server. The routing lives in the
# nginx config baked into the console image, which no chart template and no
# helm-unittest case can see, so it is pinned here.
#
# Four properties are checked. The location regex must take the four path
# shapes the proxy serves and leave every ordinary Kubernetes API path to the
# /apis and /api prefix locations; nginx runs the regex locations whenever the
# longest prefix match is a plain prefix, so the prefix locations must not
# carry ^~, which would stop that. The location must stream: tail and watch
# style responses stall behind proxy buffering and die at the default 60s
# read timeout. And the upstream must be a variable resolved per request,
# because nginx resolves a literal upstream host once at startup and refuses
# to start when it does not resolve, which is the case on every cluster that
# does not run the proxy. Finally, when nginx cannot reach the proxy it
# answers with the Kubernetes Status the console reads as not installed,
# while the proxy's own 502 passes through; that check runs the extracted
# locations in a real nginx and is skipped where none is installed.
#
# The regex is evaluated with perl, whose syntax nginx's PCRE accepts for the
# constructs used here.
# Run with: bats hack/console-nginx-observability-proxy.bats
load test_helper
REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)"
CONSOLE_CONTAINERFILE="packages/system/dashboard/images/console/Containerfile"
console_nginx_template() {
awk '
/^COPY <<.EOF. \/etc\/nginx\/templates\/default\.conf\.template$/ { inblock = 1; next }
inblock && /^EOF$/ { inblock = 0; next }
inblock { print }
' "$REPO_ROOT/$CONSOLE_CONTAINERFILE"
}
# Prints the regex of every `location ~ <regex> {` line, one per line.
regex_locations() {
printf '%s\n' "$1" | awk '$1 == "location" && $2 == "~" && $4 == "{" { print $3 }'
}
# Prints the body of the location block whose regex is $2.
location_body() {
# Through ENVIRON rather than -v, which would expand the backslashes.
printf '%s\n' "$1" | LOCATION_RE="$2" awk '
$1 == "location" && $2 == "~" && $3 == ENVIRON["LOCATION_RE"] { inblock = 1; next }
inblock && $1 == "}" { exit }
inblock { print }
'
}
matches() {
perl -e 'exit(($ARGV[1] =~ /$ARGV[0]/) ? 0 : 1)' "$1" "$2"
}
@test "console nginx routes exactly the observability proxy roots to the proxy" {
conf=$(console_nginx_template)
[ -n "$conf" ] || { echo "no /etc/nginx/templates/default.conf.template heredoc in $CONSOLE_CONTAINERFILE" >&2; exit 1; }
regexes=$(regex_locations "$conf")
[ "$(printf '%s\n' "$regexes" | grep -c .)" -eq 1 ] \
|| { echo "expected exactly one regex location, got: $regexes" >&2; exit 1; }
re=$regexes
for path in \
/apis/cozystack.io/v1alpha1/namespaces/tenant-foo/logs/select/logsql/query \
/apis/cozystack.io/v1alpha1/namespaces/tenant-foo/metrics/api/v1/query_range \
/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-foo/postgreses/db/log/select/logsql/tail \
/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-foo/vminstances/vm1/metrics/api/v1/query
do
matches "$re" "$path" || { echo "$path does not reach the proxy" >&2; exit 1; }
done
for path in \
/apis/cozystack.io/v1alpha1/namespaces/tenant-foo/logs \
/apis/cozystack.io/v1alpha1/namespaces/tenant-foo/workloadmonitors/db \
/apis/cozystack.io/v1alpha1/applicationdefinitions/postgres \
/apis/cozystack.io/v1alpha1/watch/namespaces/tenant-foo/logs/ \
/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-foo/postgreses \
/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-foo/postgreses/log \
/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-foo/postgreses/log/ \
/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-foo/postgreses/db/status \
/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-foo/postgreses/db/log \
/apis/apps.cozystack.io/v1alpha1/watch/namespaces/tenant-foo/postgreses/db/log/ \
/apis/metrics.k8s.io/v1beta1/namespaces/tenant-foo/pods/ \
/api/v1/namespaces/tenant-foo/pods/db-0/log \
/k8s/apis/cozystack.io/v1alpha1/namespaces/tenant-foo/logs/select/logsql/query
do
if matches "$re" "$path"; then
echo "FAIL: $path would reach the proxy instead of the API server" >&2
false
fi
done
# A ^~ on the /apis or /api prefix would stop nginx from trying the regex.
if printf '%s\n' "$conf" | grep -Eq '^[[:space:]]*location[[:space:]]+\^~'; then
echo "FAIL: a ^~ prefix location disables the regex location" >&2
false
fi
}
@test "the observability proxy location streams, forwards the token and resolves per request" {
conf=$(console_nginx_template)
re=$(regex_locations "$conf")
body=$(location_body "$conf" "$re")
[ -n "$body" ] || { echo "no body found for location ~ $re" >&2; exit 1; }
for directive in \
'proxy_http_version 1.1;' \
'proxy_buffering off;' \
'proxy_read_timeout 86400s;' \
'proxy_send_timeout 86400s;'
do
printf '%s\n' "$body" | grep -Fqx " $directive" \
|| { echo "the proxy location lacks '$directive'" >&2; exit 1; }
done
# nginx forwards client headers unless told otherwise; the proxy needs the
# Authorization header the gatekeeper set.
if printf '%s\n' "$body" | grep -Eq 'proxy_set_header[[:space:]]+Authorization|proxy_pass_request_headers[[:space:]]+off'; then
echo "FAIL: the proxy location drops or rewrites the Authorization header" >&2
false
fi
printf '%s\n' "$body" | grep -Eq '^[[:space:]]*resolver[[:space:]]+\$\{NGINX_LOCAL_RESOLVERS\}' \
|| { echo "the proxy location sets no resolver from the pod's resolv.conf" >&2; exit 1; }
printf '%s\n' "$body" | grep -Eq '^[[:space:]]*set[[:space:]]+\$observability_proxy[[:space:]]+observability-proxy\.cozy-observability-proxy\.svc\.\$\{CLUSTER_DOMAIN\}:80;' \
|| { echo "the proxy location does not name the Service by its fully qualified name" >&2; exit 1; }
printf '%s\n' "$body" | grep -Eq '^[[:space:]]*proxy_pass[[:space:]]+http://\$observability_proxy;' \
|| { echo "the proxy location does not proxy_pass through the variable" >&2; exit 1; }
}
@test "the console image defines every variable its nginx template substitutes" {
cd "$REPO_ROOT"
conf=$(console_nginx_template)
# The image's entrypoint substitutes only variables present in the
# environment; one that is not would reach nginx as ${NAME} and fail it.
grep -Eq '^ENV NGINX_ENTRYPOINT_LOCAL_RESOLVERS=1$' "$CONSOLE_CONTAINERFILE" \
|| { echo "the image does not ask its entrypoint for NGINX_LOCAL_RESOLVERS" >&2; exit 1; }
grep -Eq '^ENV CLUSTER_DOMAIN=cozy\.local$' "$CONSOLE_CONTAINERFILE" \
|| { echo "the image has no CLUSTER_DOMAIN default" >&2; exit 1; }
used=$(printf '%s\n' "$conf" | grep -oE '\$\{[A-Za-z_][A-Za-z0-9_]*\}' | sort -u)
expected=$(printf '%s\n' '${CLUSTER_DOMAIN}' '${NGINX_LOCAL_RESOLVERS}')
[ "$used" = "$expected" ] \
|| { echo "template substitutes: $used; the image defines: $expected" >&2; exit 1; }
}
# Prints the named location $2 of the template $1, braces included.
named_location() {
printf '%s\n' "$1" | LOCATION_NAME="$2" awk '
$1 == "location" && $2 == ENVIRON["LOCATION_NAME"] { inblock = 1 }
inblock { print }
inblock && $1 == "}" { exit }
'
}
free_port() {
python3 -c 'import socket; s = socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1])'
}
# Starts nginx with the observability locations of the console template, the
# upstream pointed at 127.0.0.1:$1, and a stand-in proxy on 127.0.0.1:$2
# that answers 502 itself. Prints the console port.
start_console_nginx() {
upstream_port=$1 standin_port=$2
conf=$(console_nginx_template)
re=$(regex_locations "$conf")
console_port=$(free_port)
{
echo "pid $BATS_TEST_TMPDIR/nginx.pid;"
echo "error_log $BATS_TEST_TMPDIR/error.log;"
echo "events {}"
echo "http {"
echo " access_log off;"
echo " client_body_temp_path $BATS_TEST_TMPDIR; proxy_temp_path $BATS_TEST_TMPDIR;"
echo " fastcgi_temp_path $BATS_TEST_TMPDIR; uwsgi_temp_path $BATS_TEST_TMPDIR; scgi_temp_path $BATS_TEST_TMPDIR;"
echo " server {"
echo " listen 127.0.0.1:$console_port;"
echo " location ~ $re {"
# The test upstream is an address, so the resolver is never asked.
location_body "$conf" "$re" \
| sed -e 's/\${NGINX_LOCAL_RESOLVERS}/127.0.0.1/' \
-e "s/^\([[:space:]]*set[[:space:]]*\$observability_proxy[[:space:]]\).*/\1127.0.0.1:$upstream_port;/"
echo " }"
named_location "$conf" "@observability_proxy_unavailable"
echo " }"
echo " server {"
echo " listen 127.0.0.1:$standin_port;"
echo " location / { return 502 'log store unreachable'; }"
echo " }"
echo "}"
} > "$BATS_TEST_TMPDIR/nginx.conf"
nginx -p "$BATS_TEST_TMPDIR" -c "$BATS_TEST_TMPDIR/nginx.conf" \
|| { cat "$BATS_TEST_TMPDIR/error.log" >&2; return 1; }
echo "$console_port"
}
teardown() {
if [ -f "${BATS_TEST_TMPDIR:-}/nginx.pid" ]; then
kill "$(cat "$BATS_TEST_TMPDIR/nginx.pid")" 2>/dev/null || true
fi
}
@test "a cluster without the proxy answers the console's not-installed Status, the proxy's own 502 passes through" {
command -v nginx >/dev/null 2>&1 || skip "nginx is not installed"
command -v curl >/dev/null 2>&1 || skip "curl is not installed"
path=/apis/cozystack.io/v1alpha1/namespaces/tenant-foo/logs/select/logsql/query
# Nothing listens on the upstream port: nginx itself fails the request.
standin=$(free_port)
port=$(start_console_nginx "$(free_port)" "$standin")
out=$(curl --silent --show-error --include --max-time 10 "http://127.0.0.1:$port$path")
printf '%s\n' "$out" | head -n 1 | grep -q ' 404' \
|| { echo "FAIL: an unreachable proxy did not answer 404: $out" >&2; false; }
printf '%s\n' "$out" | grep -iq '^content-type: application/json' \
|| { echo "FAIL: the not-installed answer is not JSON: $out" >&2; false; }
body=$(printf '%s\n' "$out" | tail -n 1)
printf '%s' "$body" | python3 -c '
import json, sys
s = json.load(sys.stdin)
assert s["kind"] == "Status" and s["apiVersion"] == "v1" and s["status"] == "Failure", s
assert s["reason"] == "NotFound" and s["code"] == 404, s
assert s["message"] == "the observability proxy is not available on this cluster", s
' || { echo "FAIL: the not-installed body is not the expected Status: $body" >&2; false; }
# The next nginx gets fresh ports, so the old master need not have
# released its sockets; it is still stopped first to keep one running.
pid=$(cat "$BATS_TEST_TMPDIR/nginx.pid")
kill "$pid"
for _ in $(seq 1 50); do kill -0 "$pid" 2>/dev/null || break; sleep 0.1; done
rm -f "$BATS_TEST_TMPDIR/nginx.pid"
# The proxy answers 502 itself: the console must see that answer.
standin=$(free_port)
port=$(start_console_nginx "$standin" "$standin")
out=$(curl --silent --show-error --include --max-time 10 "http://127.0.0.1:$port$path")
printf '%s\n' "$out" | head -n 1 | grep -q ' 502' \
|| { echo "FAIL: the proxy's own 502 did not pass through: $out" >&2; false; }
printf '%s\n' "$out" | tail -n 1 | grep -qx 'log store unreachable' \
|| { echo "FAIL: the proxy's own 502 body was replaced: $out" >&2; false; }
}
+31
View File
@@ -966,6 +966,37 @@ STUB
}
}
@test "PR E2E jobs clean up between preparation attempts but not after testing" {
tmp="$(mktemp -d)"
mkdir -p "$tmp/bin"
cat >"$tmp/bin/make" <<'STUB'
#!/bin/sh
printf '%s\n' "$*" >>"$CLEANUP_CALLS"
case "$*" in
*prepare-env-container*)
if [ ! -f "$CLEANUP_ATTEMPT" ]; then
touch "$CLEANUP_ATTEMPT"
exit 1
fi
;;
esac
STUB
chmod +x "$tmp/bin/make"
for wf in pull-requests e2e-fork; do
workflow="$HACK_DIR/../.github/workflows/$wf.yaml"
yq --unwrapScalar '.jobs.e2e.steps[] | select(.name == "Prepare environment") | .run' "$workflow" >"$tmp/prepare.sh"
CLEANUP_CALLS="$tmp/$wf.calls" CLEANUP_ATTEMPT="$tmp/$wf.attempt" SANDBOX_NAME=test PATH="$tmp/bin:$PATH" bash -eu "$tmp/prepare.sh"
printf '%s\n' 'SANDBOX_NAME=test prepare-env-container' '-C packages/core/testing SANDBOX_NAME=test delete-cluster-container' 'SANDBOX_NAME=test prepare-env-container' >"$tmp/expected"
diff "$tmp/expected" "$tmp/$wf.calls"
scripts="$(yq --unwrapScalar '.jobs.e2e.steps[] | select(.name != "Prepare environment") | .run // ""' "$workflow")"
if printf '%s\n' "$scripts" | grep -Eq 'delete-cluster-container|docker[[:space:]]+(compose.*down|rm)|zpool[[:space:]]+destroy|make.*[[:space:]]delete([[:space:]]|$)'; then
echo "FAIL: $wf destroys runner-local state outside preparation"
false
fi
done
rm -rf "$tmp"
}
@test "the step that runs this collector has a ceiling of its own everywhere" {
# Every bound inside the script protects one read. None of them protects the
# STEP: the tarball is written on the collector's last line, the upload is the
+1 -1
View File
@@ -121,7 +121,7 @@
# the number; do not expect to find it written here.
#
# TWO ceilings sit outside it, and they fail differently. The job is capped at
# `timeout-minutes: 215`, and reaching that cancels every step still to come --
# `timeout-minutes: 265`, and reaching that cancels every step still to come --
# the artifact upload among them. The step running this script is capped at 30 of
# its own, and reaching THAT ends only this step: the uploads after it still run,
# and `continue-on-error` on that step keeps the overrun out of the job's result,
+1 -1
View File
@@ -208,7 +208,7 @@ _cozy_on_exit() {
# capture would read as a complete one listing fewer crash-loops than
# actually occurred. This handler's whole worst case is ~1350s (330 + 390
# + 630, each leg including its 30s kill grace) against a job capped at
# 215 minutes with no per-step timeout —
# 265 minutes with no per-step timeout —
# deliberate, for the same reason recorded in hack/e2e-chainsaw/.chainsaw.yaml:
# a bounded, honest wait beats a kill that collects nothing.
if [ "$_prev_rc" -ne 0 ]; then
+2 -2
View File
@@ -66,14 +66,14 @@ spec:
# script: it SIGKILLs the shell alone, orphaned children hold the pipe open,
# and the op overruns (#4268 measured 606s under a 300s ceiling), so the
# nominal ~5m was a figure a busy catch could exceed, not a cap. Tests run
# sequentially, and the E2E job is capped at 215 timeout-minutes. A cascade
# sequentially, and the E2E job is capped at 265 timeout-minutes. A cascade
# reddening many suites can therefore spend a large share of the job in
# diagnostics. That is deliberate — the old ceiling
# bought its speed by collecting nothing at all — and it is bounded: both legs
# only approach their ceilings on a genuinely degraded cluster, a systemic
# failure usually reds the install gate and skips the E2E step entirely, and
# the report-upload steps are `if: always()`. That last point does NOT extend
# to a job timeout: a 215-minute cap ends the job `cancelled`, and the ~4m48s
# to a job timeout: a 265-minute cap ends the job `cancelled`, and the ~4m48s
# grace window is consumed by the earlier `Collect report` step, so both
# cozyreport.tgz and the job log are lost (measured on runs 29557003429 and
# 29568562337). If this ever does start truncating runs, bound the number of
+11 -3
View File
@@ -18,7 +18,15 @@ hack/e2e-chainsaw/
└── ...
```
Suites: `postgres`, `bucket`, `mariadb`, `mongodb`, `redis`, `valkey`, `qdrant`, `clickhouse`, `kafka`, `kafka-metadata`, `rabbitmq`, `etcd`, `openbao`, `harbor`, `foundationdb`, `external-dns`, `kuberture`, `vminstance`, `gateway`, `opensearch`, `monitoring`, `kubernetes-latest`, `kubernetes-previous`, `securitygroup`, `seaweedfs`, `computeplane`.
A suite is a directory directly under `hack/e2e-chainsaw/` in or below which Chainsaw finds a Test. Three layouts do that, and the selector (`hack/select-e2e.sh`) and the leak-guard allowlist both recognise the same three:
- `chainsaw-test.yaml`, the usual one, or `chainsaw-test.yml` when the `.yaml` is absent;
- numbered step files such as `01-install.yaml`, which Chainsaw turns into a Test on its own when neither of the above exists;
- either of those in a subdirectory of the suite, since Chainsaw walks every subdirectory of the directory it is handed.
A parked `chainsaw-test.yaml.disabled` is none of these: the directory is then not a suite, is selected by nothing and cannot be allowlisted.
Suites: `postgres`, `bucket`, `mariadb`, `mongodb`, `redis`, `valkey`, `qdrant`, `clickhouse`, `kafka`, `kafka-metadata`, `rabbitmq`, `etcd`, `openbao`, `harbor`, `foundationdb`, `external-dns`, `kuberture`, `vminstance`, `gateway`, `opensearch`, `monitoring`, `kubernetes-latest`, `kubernetes-previous`, `securitygroup`, `seaweedfs`, `computeplane`, `observability-proxy`. `kubernetes-proxmox` is parked as `chainsaw-test.yaml.disabled` and runs only in `.github/workflows/e2e-proxmox.yaml`, which needs a Proxmox hypervisor.
## What Chainsaw buys over the BATS suite
@@ -30,7 +38,7 @@ Suites: `postgres`, `bucket`, `mariadb`, `mongodb`, `redis`, `valkey`, `qdrant`,
## Test anatomy
Each suite is a directory with a `chainsaw-test.yaml` (one or more `Test` docs) plus the manifests it applies. A step has `try` (operations: `apply`, `assert`, `error`, `script`, `delete`, …), optional `catch` (diagnostics on failure), and optional `finally` (teardown regardless of outcome).
Each suite is a directory with one or more `Test` docs (in one of the layouts above, usually a `chainsaw-test.yaml`) plus the manifests it applies. A step has `try` (operations: `apply`, `assert`, `error`, `script`, `delete`, …), optional `catch` (diagnostics on failure), and optional `finally` (teardown regardless of outcome).
Declarative suites assert on `status.conditions` and concrete fields. Inherently imperative suites keep their logic in `script` steps:
@@ -55,7 +63,7 @@ Declarative suites assert on `status.conditions` and concrete fields. Inherently
## Running locally
Requires a cluster with Cozystack installed and a `tenant-test` namespace (the environment `hack/e2e-install-cozystack.bats` produces). `bucket` and `seaweedfs` additionally need `mc`, `nc`, and `jq` on the host; `kuberture`/`openbao`/`foundationdb`/`securitygroup`/`computeplane`/`kubernetes-latest`/`kubernetes-previous`/`vminstance` need `jq`.
Requires a cluster with Cozystack installed and a `tenant-test` namespace (the environment `hack/e2e-install-cozystack.bats` produces). `bucket` and `seaweedfs` additionally need `mc`, `nc`, and `jq` on the host; `kuberture`/`openbao`/`foundationdb`/`securitygroup`/`computeplane`/`kubernetes-latest`/`kubernetes-previous`/`vminstance` need `jq`; `observability-proxy` needs `curl` and `nc`.
```bash
# install chainsaw
+27 -3
View File
@@ -175,9 +175,33 @@ lg_allowlisted() {
awk -v s="$1" '$1 !~ /^#/ && $1 == s { print $2; exit }' "$2"
}
# lg_suite_has_test ROOT SUITE — true when ROOT/SUITE is a directory in or
# below which Chainsaw would find a Test: a chainsaw-test.yaml, a
# chainsaw-test.yml, or a numbered step file such as 01-install.yaml. This is
# the rule hack/select-e2e.sh enumerates the suites with (the find and sed that
# build all_apps there), spelled the same way on purpose: a layout one side
# accepts and the other refuses would run a suite the selector never selects,
# or make a suite that runs impossible to allowlist. Change both together. A
# parked chainsaw-test.yaml.disabled matches none of the names. The type test
# is the selector's too: a step-file name counts unless it is a directory, and
# a chainsaw-test.y*ml counts whatever its type, so Chainsaw reports a
# directory by that name instead of this validation stopping the run first.
lg_suite_has_test() {
local root=$1 suite=$2 path
[ -d "$root/$suite" ] || return 1
while IFS= read -r path; do
case ${path##*/} in
chainsaw-test.yaml|chainsaw-test.yml) return 0 ;;
esac
[[ ${path##*/} =~ ^[0-9]+-[^/]*\.ya?ml$ ]] && return 0
done < <(find "$root/$suite" -mindepth 1 \( ! -type d -o -name chainsaw-test.yaml -o -name chainsaw-test.yml \) \( -name chainsaw-test.yaml -o -name chainsaw-test.yml -o -name '[0-9]*-*.yaml' -o -name '[0-9]*-*.yml' \) 2>/dev/null)
return 1
}
# lg_allowlist_validate ALLOWLIST ROOT — every entry must name an existing suite
# directory under ROOT and carry an issue ref, so a renamed or removed suite
# cannot keep an exemption nobody reads.
# cannot keep an exemption nobody reads. A suite directory is one Chainsaw
# would find a Test in, whatever its layout (lg_suite_has_test).
lg_allowlist_validate() {
local list=$1 root=$2 bad=0 suite ref rest
# `|| [ -n "$suite" ]` keeps a last line with no newline, which `read`
@@ -189,8 +213,8 @@ lg_allowlist_validate() {
echo "leak-guard: allowlist line for '$suite' must be '<suite-dir> <issue-ref>'" >&2
bad=1
fi
if [ ! -f "$root/$suite/chainsaw-test.yaml" ] && [ ! -f "$root/$suite/chainsaw-test.yml" ]; then
echo "leak-guard: allowlist names '$suite', which is not a suite directory under $root" >&2
if ! lg_suite_has_test "$root" "$suite"; then
echo "leak-guard: allowlist names '$suite', which is not a suite directory under $root (no chainsaw-test.yaml, chainsaw-test.yml or numbered step file in or below it)" >&2
bad=1
fi
done <"$list"
+72 -25
View File
@@ -5230,9 +5230,10 @@ _cozy_guard_kubectl() {
cozy_guard_helmrelease() {
local _ns="$1"
local _hr="$2"
# Non-empty on a release whose install strategy the cluster is known to set,
# which is the parent Application HelmRelease and nothing else here. See the
# branch that reads it for why the check cannot be applied to the addons.
# Non-empty on a release whose install strategy the cluster is known to set:
# the parent Application HelmRelease and the worker pool release, which is an
# Application HelmRelease too. See the branch that reads it for why the check
# cannot be applied to the addons.
local _strategy_required="${3:-}"
local _probe _strategy _history _ready _read_rc
_cozy_guard_read_bounds
@@ -5334,7 +5335,7 @@ cozy_guard_helmrelease() {
# are load-bearing for this guard and not only for the chart. The csi they
# pin is helmreleases/csi.yaml, the KubeVirt driver this suite runs; on the
# Proxmox substrate the same release comes from csi-proxmox.yaml, which sets
# the strategy too and has no such pin.
# the strategy too and is pinned by csi_proxmox_install_retry_test.yaml.
if [ "${_strategy}" = "RetryOnFailure" ]; then
echo "HelmRelease ${_hr} carries a failed or uninstalled Snapshot, and its install strategy is RetryOnFailure, which retries a failed action in place instead of remediating it. An action on this release failed during the run; the release may have recovered on a retry since, and the failure is still the finding. On the tenant cilium the cause can sit outside the chart: its operator Deployment cannot schedule before a worker registers, so workers arriving more than two 10m install timeouts after the install started leave this Snapshot while still inside the node-join deadline. Compare the Snapshot times below with when the first node turned Ready before looking at the release." >&2
_cozy_guard_kubectl -n "$_ns" describe hr "${_hr}" >&2
@@ -5437,16 +5438,32 @@ cozy_guard_addon_helmreleases() {
# keeps the parent's own verdict, which an early return past the addons would
# drop.
#
# The worker pool release of parent kubernetes-<test>. The pool is a
# KubernetesNodes named <test>-md0 (see the apply in the run), and the
# aggregated apiserver names its HelmRelease kubernetes-nodes-<test>-md0. That
# name does not start with the addon prefix kubernetes-<test>-, so the addon
# listing never selects it, and it is read on its own. Derived from the parent
# rather than passed in, for the same reason as the addon prefix below.
cozy_pool_helmrelease_name() {
local _parent="$1"
printf 'kubernetes-nodes-%s-md0\n' "${_parent#kubernetes-}"
}
# The addon prefix is derived here rather than passed in, so the parent name and
# the prefix that selects its children cannot drift apart.
cozy_guard_all_helmreleases() {
local _ns="$1"
local _parent="$2"
local _failed=0
# The parent is asked to prove its install strategy read back non-empty; the
# addons are not, because most of them legitimately set none. See the branch
# in cozy_guard_helmrelease that reads this argument.
# The parent and the pool release are asked to prove their install strategy
# read back non-empty: both are Application HelmReleases, which the apiserver
# stamps. The addons are not, because most of them legitimately set none. See
# the branch in cozy_guard_helmrelease that reads this argument.
cozy_guard_helmrelease "$_ns" "${_parent}" required || _failed=1
# The pool release sits on the node-join path, through the talos-reconcile
# Job it renders, so a failed action on it is as much a finding as one on the
# parent. By the time this runs the pool has scaled, so the release exists.
cozy_guard_helmrelease "$_ns" "$(cozy_pool_helmrelease_name "${_parent}")" required || _failed=1
cozy_guard_addon_helmreleases "$_ns" "${_parent}-" || _failed=1
# These releases are now classified, and the same flag the failure-path report
# uses records it. The two are not alternatives: this call runs before the
@@ -5495,7 +5512,8 @@ _COZY_GUARD_REPORT_DONE=0
# failing path would replace the caller's own exit status, and the tenant
# crust-gather snapshot hangs off that exit.
#
# Bounded on its own clock, checked before the listing and before each release.
# Bounded on its own clock, checked before the pool release, before the listing
# and before each addon release.
# What comes after this on the exit path is the tenant snapshot, which needs the
# rest of the op, so a namespace whose apiserver answers slowly must cost a
# bounded number of seconds here rather than however many releases happen to be
@@ -5515,7 +5533,7 @@ cozy_report_helmrelease_remediation() {
local _ns="$1"
local _parent="$2"
local _label="${3:-}"
local _addon_hrs _hr _deadline _budget
local _addon_hrs _hr _pool _pool_found _read_rc _deadline _budget
[ "${_COZY_GUARD_REPORT_DONE}" -eq 0 ] || return 0
# An empty parent name means the run failed before the release was created, so
# there is no history anywhere to read yet.
@@ -5541,6 +5559,29 @@ cozy_report_helmrelease_remediation() {
cozy_diag_phase_has_time "HelmRelease remediation footprint of ${_parent}" || return 0
echo "=== ${_label}tenant HelmRelease remediation footprint (management cluster, ns ${_ns}) — classification only, this does not decide the run ==="
cozy_guard_helmrelease "$_ns" "${_parent}" required || true
# The pool release is created by a later apply than the parent, and
# run_kubernetes_test arms the exit handler between the two, so a run whose
# pool apply failed reaches this with none to read. Its absence is then a fact
# about where the run stopped, not a failed read, and is reported as such.
# Only an exit 0 with empty output means absent: --ignore-not-found makes
# NotFound that, so any non-zero exit is a read that failed and is named as
# one.
_pool=$(cozy_pool_helmrelease_name "${_parent}")
cozy_diag_phase_has_time "HelmRelease remediation footprint of ${_pool}" || return 0
if [ "$(date +%s)" -ge "${_deadline}" ]; then
echo "=== HelmRelease remediation footprint: stopped after ${_budget}s; ${_pool} and the addon releases of ${_parent} were not read, so nothing was observed about any of them ===" >&2
return 0
fi
_cozy_guard_read_bounds
_read_rc=0
_pool_found=$(_cozy_guard_kubectl get hr -n "$_ns" "${_pool}" --ignore-not-found -o name) || _read_rc=$?
if [ "${_read_rc}" -ne 0 ]; then
echo "Checking whether ${_pool} exists failed - $(_cozy_guard_read_failure "${_read_rc}"); nothing was observed about it either way." >&2
elif [ -z "${_pool_found}" ]; then
echo "» ${_pool} does not exist - the run ended before the worker pool was applied, or its HelmRelease was never created; nothing to read"
else
cozy_guard_helmrelease "$_ns" "${_pool}" required || true
fi
cozy_diag_phase_has_time "the addon HelmRelease listing of ${_parent}" || return 0
if [ "$(date +%s)" -ge "${_deadline}" ]; then
echo "=== HelmRelease remediation footprint: stopped after ${_budget}s; the addon releases of ${_parent} were not listed, so nothing was observed about any of them ===" >&2
@@ -5723,6 +5764,28 @@ ${ouroboros_addon}
version: "${k8s_version}"
EOF
# Armed here, at the parent apply, rather than once the tenant answers. The
# parent HelmRelease exists from this point on -- the aggregated apiserver
# creates it before it answers the apply above -- and everything between here
# and the tenant LB is a management-cluster wait on that release or on what it
# installs: the KamajiControlPlane, the control-plane Deployments, the
# MachineDeployment, the admin-kubeconfig Secret. That window is the one the
# guard was written for. An install action that times out while
# admin-kubeconfig is still being provisioned leaves the failed Snapshot this
# reads, and every one of those waits exits under errexit with no reading at
# all if the handler is armed after them.
#
# Ahead of the pool apply below as well, not after it: a pool apply that fails
# exits under errexit too, and armed after it that run would leave the parent
# unclassified. Armed here, it is the run the report's "pool release does not
# exist" line describes.
#
# Arming this early costs the snapshot nothing: _tenant_snapshot_on_fail
# returns on an unset CURRENT_TENANT_KC, which is assigned where the
# kubeconfig becomes usable, so it no-ops until then.
CURRENT_TENANT_PARENT_HR="kubernetes-${test_name}"
trap '_tenant_failure_on_exit' EXIT
# Worker node pool is a separate KubernetesNodes resource since the Phase 2
# split. Named "<cluster>-md0" so it produces MachineDeployment
# kubernetes-${test_name}-md0 — the same object the pre-split
@@ -5781,22 +5844,6 @@ ${talos_block}
storageClass: "${storage_class}"
version: "${k8s_version}"
EOF
# Armed here, at the apply, rather than once the tenant answers. The parent
# HelmRelease exists from this point on -- the aggregated apiserver renders it
# from the CR above -- and everything between here and the tenant LB is a
# management-cluster wait on that release or on what it installs: the
# KamajiControlPlane, the control-plane Deployments, the MachineDeployment,
# the admin-kubeconfig Secret. That window is the one the guard was written
# for. An install action that times out while admin-kubeconfig is still being
# provisioned leaves the failed Snapshot this reads, and every one of those
# waits exits under errexit with no reading at all if the handler is armed
# after them.
#
# Arming this early costs the snapshot nothing: _tenant_snapshot_on_fail
# returns on an unset CURRENT_TENANT_KC, which is assigned where the
# kubeconfig becomes usable, so it no-ops until then.
CURRENT_TENANT_PARENT_HR="kubernetes-${test_name}"
trap '_tenant_failure_on_exit' EXIT
# Wait for the tenant-test namespace to be active
kubectl wait namespace tenant-test --timeout=20s --for=jsonpath='{.status.phase}'=Active
+24 -3
View File
@@ -22,7 +22,16 @@ CLUSTER="${COZY_PROXMOX_CLUSTER:-k8s-e2e}"
PVE_SSH="${COZY_PVE_SSH:?COZY_PVE_SSH is required: the ssh destination of the Proxmox host, user@host}"
PVE_SSH_KEY="${COZY_PVE_SSH_KEY:-$HOME/.ssh/id_ed25519}"
STORAGE="${COZY_PROXMOX_STORAGE:?COZY_PROXMOX_STORAGE is required, the PVE storage id for the PVC probe}"
TENANT_KUBECONFIG="${TMPDIR:-/tmp}/proxmox-e2e-tenant.kubeconfig"
# Where this run keeps the tenant's super-admin kubeconfig. Per run rather than
# a fixed path under /tmp: the file is a credential for the whole tenant, so it
# must not outlive the run on a shared runner, and dump_diagnostics reads
# whatever is at this path, so a file left by a previous run would be reported
# as this run's tenant. On the workflow's runner this is RUNNER_TEMP, which the
# runner exports into every step and empties between jobs; apply-tenant removes
# a leftover before it does anything else, and the suite's finally removes the
# file when the run ends.
WORKDIR="${COZY_PROXMOX_WORKDIR:-${RUNNER_TEMP:-${TMPDIR:-/tmp}}}"
TENANT_KUBECONFIG="$WORKDIR/proxmox-e2e-tenant.kubeconfig"
FIXTURE="$(cd "$(dirname "$0")/.." && pwd)/kubernetes-proxmox/tenant.yaml"
log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; }
@@ -77,6 +86,10 @@ render_tenant() {
}
apply_tenant() {
# First step of the suite, so this is where a kubeconfig left by an earlier
# run on this runner is removed: from here on the path holds this run's file
# or nothing, and dump_diagnostics cannot pick up another tenant.
rm -f "$TENANT_KUBECONFIG"
render_tenant | kubectl apply -f -
}
@@ -85,8 +98,16 @@ apply_tenant() {
# carries the address a runner outside the management cluster can reach; .svc
# is the in-cluster Service the management-side controllers use.
fetch_tenant_kubeconfig() {
kubectl -n "$NS" get secret "kubernetes-${CLUSTER}-admin-kubeconfig" \
-o jsonpath='{.data.super-admin\.conf}' | base64 -d > "$TENANT_KUBECONFIG"
# Written under umask 077 in a subshell of its own: the workflow sets that
# umask only in its staging step, and a chainsaw script step runs with the
# runner's default, under which the file would be readable by every local
# user. No pipefail in POSIX sh, so a failed read leaves an empty file, which
# the size check below turns into a failure rather than an empty kubeconfig.
(
umask 077
kubectl -n "$NS" get secret "kubernetes-${CLUSTER}-admin-kubeconfig" \
-o jsonpath='{.data.super-admin\.conf}' | base64 -d > "$TENANT_KUBECONFIG"
)
[ -s "$TENANT_KUBECONFIG" ] || die "tenant kubeconfig is empty — did the control plane come up?"
}
+223 -2
View File
@@ -91,7 +91,7 @@ spec:
status:
replicas: 1
---
# ClickHouse Altinity backup/restore E2E — the two backup Tests below sit in
# ClickHouse Altinity backup/restore E2E — the three backup Tests below sit in
# this suite alongside the plain `clickhouse` Test above. Chainsaw port of the
# intended hack/e2e-apps/backup-clickhouse.bats (cozystack#2600), shaped like the
# etcd/mariadb/postgres suites:
@@ -107,8 +107,15 @@ spec:
# driving the validated example scripts
# (examples/backups/clickhouse). Also runs in
# CI, in tenant-root.
# clickhouse-3-system-bucket-roundtrip
# — backup and in-place restore through the
# shipped cozy-default BackupClass and the
# platform system bucket, for a release
# whose Pod started before its credentials
# were projected. Runs in CI, in tenant-root.
#
# Why the split: the two Tests answer different questions at different cost. The
# Why the split: the first two Tests answer different questions at different
# cost, and the third covers the system-bucket path neither of them takes. The
# contracts Test needs no object store, so it stays in the default tenant-test
# namespace next to the other app suites and is the seconds-long guard on the
# chart wiring the Altinity strategy depends on — the sidecar container/port and
@@ -377,3 +384,217 @@ spec:
set -eu
cd ../../..
NAMESPACE="$NAMESPACE" examples/backups/clickhouse/cleanup.sh
---
# Backup and in-place restore through the shipped cozy-default BackupClass and
# the platform system bucket, for a ClickHouse whose Pod started before the
# platform projected cozy-backups-creds into its namespace. The sidecar has to
# pick those credentials up while running; until it could, the first BackupJob
# of such a release failed with "no EC2 IMDS role found" and kept failing until
# the Pod was restarted.
#
# Unlike clickhouse-2-backup-roundtrip this targets the system bucket as
# advertised, https://s3.example.org, so system-bucket-roundtrip.sh makes that
# name reachable from tenant-root for the duration of its own step and undoes
# it on exit: see its header for the two shared objects it touches and the
# cert-manager behaviour that relies on. Runs in tenant-root, where the system
# bucket's SeaweedFS and its Ingress controller sit.
#
# The precondition step deletes tenant-root's projected cozy-backups-creds,
# which any BackupJob that ran in tenant-root before this Test leaves there:
# the projector runs for each BackupJob whatever its strategy, and Chainsaw
# does not fix the order of the Tests in a suite, so the earlier one may come
# from this suite or another. It writes the Secret again on the next
# BackupJob, which is the moment under test.
apiVersion: chainsaw.kyverno.io/v1alpha1
kind: Test
metadata:
name: clickhouse-3-system-bucket-roundtrip
spec:
namespace: tenant-root
catch:
- events: {}
- describe:
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
name: clickhouse-sysbucket
- describe:
apiVersion: cert-manager.io/v1
kind: Certificate
name: e2e-system-bucket-s3
- describe:
apiVersion: cert-manager.io/v1
kind: Certificate
name: seaweedfs-s3-ingress-tls
- describe:
apiVersion: networking.k8s.io/v1
kind: Ingress
name: ingress-seaweedfs-s3
- describe:
apiVersion: backups.cozystack.io/v1alpha1
kind: BackupJob
name: sysbucket-backup
- describe:
apiVersion: backups.cozystack.io/v1alpha1
kind: RestoreJob
name: sysbucket-restore
- podLogs:
selector: app.kubernetes.io/instance=clickhouse-sysbucket
container: clickhouse-backup
tail: 200
- podLogs:
selector: app=clickhouse-sysbucket-keeper
tail: 100
steps:
- name: preconditions
try:
# The deletes below wait up to 270s in all (30+30+60+30+30+30+60), so a
# stuck one is named by its kubectl --timeout before the 5m ceiling kills
# the shell with no object named. A new delete needs its own --timeout.
- script:
timeout: 5m
content: |
set -eu
if kubectl -n "$NAMESPACE" get secret seaweedfs-s3-ingress-tls -o name >/dev/null 2>&1; then
echo "Secret $NAMESPACE/seaweedfs-s3-ingress-tls already exists; this Test would have to replace it" >&2
exit 1
fi
if kubectl -n kube-system get configmap coredns -o jsonpath='{.data.Corefile}' | grep -qF 'BEGIN e2e-clickhouse-system-bucket'; then
echo "kube-system/coredns still carries the e2e-clickhouse-system-bucket block from an earlier run" >&2
exit 1
fi
# Fixed names in a persistent namespace: a leftover Succeeded job
# would satisfy the waits in the run step without a new attempt, and
# a leftover Backup of the BackupJob's name, recording an earlier
# run's archive, fails the new BackupJob.
kubectl -n "$NAMESPACE" delete restorejob sysbucket-restore --ignore-not-found --wait=true --timeout=30s
kubectl -n "$NAMESPACE" delete backupjob sysbucket-backup --ignore-not-found --wait=true --timeout=30s
kubectl -n "$NAMESPACE" delete backup sysbucket-backup --ignore-not-found --wait=true --timeout=60s
kubectl -n "$NAMESPACE" delete secret cozy-backups-creds --ignore-not-found --wait=true --timeout=30s
# Chainsaw reclaims only what its apply created, so objects a killed
# run left would be patched and kept. The Certificate goes before its
# Secret, which cert-manager issues again while the Certificate exists.
kubectl -n "$NAMESPACE" delete certificate e2e-system-bucket-s3 --ignore-not-found --wait=true --timeout=30s
kubectl -n "$NAMESPACE" delete secret e2e-system-bucket-s3-tls --ignore-not-found --wait=true --timeout=30s
kubectl -n "$NAMESPACE" delete pod e2e-system-bucket-probe --ignore-not-found --wait=true --timeout=60s
check:
($error == null): true
- name: create-certificate-and-probe
try:
- apply:
file: system-bucket-tls.yaml
- assert:
timeout: 2m
resource:
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: e2e-system-bucket-s3
status:
(conditions[?type == 'Ready']):
- status: "True"
- assert:
timeout: 3m
resource:
apiVersion: v1
kind: Pod
metadata:
name: e2e-system-bucket-probe
status:
(conditions[?type == 'Ready']):
- status: "True"
# The Certificate's Secret carries no owner reference, so it outlives the
# Certificate, and cert-manager issues it again while the Certificate
# exists. Chainsaw runs a step's cleanup ops after this step's cleaner has
# deleted the Certificate and waited for it.
cleanup:
- delete:
ref:
apiVersion: v1
kind: Secret
name: e2e-system-bucket-s3-tls
- name: create-clickhouse
try:
- apply:
file: clickhouse-system-bucket.yaml
# 11m: above the 600s install timeout cozystack-api gives this release;
# see "Sizing an HR-Ready budget" in docs/agents/e2e-testing.md.
- assert:
timeout: 11m
resource:
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: clickhouse-sysbucket
status:
(conditions[?type == 'Ready']):
- status: "True"
- assert:
timeout: 5m
resource:
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: chi-clickhouse-sysbucket-clickhouse-0-0
status:
readyReplicas: 1
# The Pod is running with no projected credentials: the state the first
# BackupJob of a fresh release meets.
- error:
timeout: 30s
resource:
apiVersion: v1
kind: Secret
metadata:
name: cozy-backups-creds
- name: run
try:
# Inner budgets of system-bucket-roundtrip.sh:
# 280s probe: a 240s deadline for CoreDNS picking up the rewrite and
# the Ingress serving the Test certificate, plus one last 30s
# attempt and its 10s pause
# 480s BackupJob to Succeeded, including the driver's 2m hold
# 480s RestoreJob to Succeeded; no hold by then, the stamp is
# already older than the backup's own 2m wait
# 90s the EXIT trap: Corefile replace plus a 60s Secret delete
# 1330s (22m10s)
# 25m leaves 2m50s for the SQL and kubectl calls between those waits, so
# an inner wait is what fires and names what stalled.
- script:
timeout: 25m
content: |
set -eu
cd ../../..
NAMESPACE="$NAMESPACE" hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh
check:
($error == null): true
finally:
# The script's EXIT trap puts the Corefile and the Ingress TLS Secret
# back, but a step ceiling kills the shell before the trap runs. A no-op
# when the trap already did it.
- description: restore the host CoreDNS and the S3 Ingress TLS Secret
script:
timeout: 2m
content: |
set -eu
cd ../../..
# Chainsaw runs this under sh; the helpers are bash.
NAMESPACE="$NAMESPACE" bash -euo pipefail -c '
source examples/backups/clickhouse/00-helpers.sh
source hack/e2e-chainsaw/clickhouse/system-bucket-shared-state.sh
restore_shared_state
'
# Chainsaw did not apply these: the jobs come from the script and the
# Backup from the controller, and deleting a BackupJob reclaims no Backup.
- description: system-bucket round-trip cleanup
script:
timeout: 3m
content: |
set -eu
kubectl -n "$NAMESPACE" delete restorejob sysbucket-restore --ignore-not-found --wait=true --timeout=30s
kubectl -n "$NAMESPACE" delete backupjob sysbucket-backup --ignore-not-found --wait=true --timeout=30s
# The driver names the Backup after the BackupJob; by name it is
# found even when status.backupRef was never written.
kubectl -n "$NAMESPACE" delete backup sysbucket-backup --ignore-not-found --wait=true --timeout=60s
@@ -8,8 +8,8 @@
# backup.enabled=true with placeholder S3 coordinates is deliberate: the
# sidecar runs `clickhouse-backup server`, which starts its HTTP API without
# validating S3 (ALLOW_EMPTY_BACKUPS=true, lazy connect — the same reason the
# platform useSystemBucket flow marks the S3 secret keys optional so the Pod
# starts before any bucket is projected). So the Pod becomes Ready and the
# platform useSystemBucket flow can start the Pod before any bucket is
# projected). So the Pod becomes Ready and the
# chart<->strategy contract is assertable with no reachable object store. The
# real S3 backup/restore round-trip lives in clickhouse-2-backup-roundtrip,
# which runs in tenant-root against the in-cluster SeaweedFS.
@@ -0,0 +1,30 @@
# ClickHouse on the platform system bucket for the
# clickhouse-3-system-bucket-roundtrip Test. No S3 coordinates are given: the
# sidecar reads them from the clickhouse-backup.yml key the platform projects
# into cozy-backups-creds at the first BackupJob, after this Pod has started.
# backup.endpointCA trusts the SeaweedFS CA that signs the certificate the Test
# serves for s3.example.org.
apiVersion: apps.cozystack.io/v1alpha1
kind: ClickHouse
metadata:
name: sysbucket
spec:
size: 5Gi
logStorageSize: 1Gi
shards: 1
replicas: 1
# The round-trip runs SQL against this server; at nano's 128Mi the first
# query OOM-kills it, so it takes the preset clickhouse-2 runs SQL on.
resourcesPreset: small
resources: {}
backup:
enabled: true
useSystemBucket: true
endpointCA:
name: e2e-system-bucket-s3-tls
key: ca.crt
clickhouseKeeper:
enabled: true
replicas: 1
size: 1Gi
resourcesPreset: micro
+174
View File
@@ -0,0 +1,174 @@
#!/bin/bash
# Backup and in-place restore of the `sysbucket` ClickHouse through the
# cozy-default BackupClass, against the platform system bucket. Run from the
# repository root by the clickhouse-3-system-bucket-roundtrip Test, which has
# already started the release with no cozy-backups-creds in its namespace.
#
# The system bucket is addressed as https://s3.example.org, the S3 Ingress host
# the COSI driver advertises. In CI that name does not resolve and its ACME
# certificate is never issued, so for the duration of this script, and only of
# this script, two shared objects are changed and the EXIT trap below puts both
# back:
#
# - the S3 Ingress TLS Secret seaweedfs-s3-ingress-tls is created from the
# Test's own certificate (SeaweedFS CA, s3.example.org). It relies on
# cert-manager leaving a Secret's data alone while the Certificate that
# names it has no completed issuance, verified with cert-manager v1.20.2:
# a fresh ACME order would replace this certificate, and ingress-nginx
# would then serve one the sidecar does not trust, which fails the probe
# below rather than the backup.
# - the host CoreDNS Corefile gets one marked `rewrite name exact` line for
# s3.example.org. The host CoreDNS is Talos-managed and imports no snippet
# files, so the block goes into the live Corefile the way ouroboros writes
# its own; ouroboros is off in CI and no other suite uses this name.
set -euo pipefail
source examples/backups/clickhouse/00-helpers.sh
source hack/e2e-chainsaw/clickhouse/system-bucket-shared-state.sh
: "${NAMESPACE:?}"
CLICKHOUSE_NAME=sysbucket
BACKUPJOB_NAME=sysbucket-backup
RESTOREJOB_NAME=sysbucket-restore
CERT_SECRET=e2e-system-bucket-s3-tls
PROBE_POD=e2e-system-bucket-probe
DNS_TARGET="root-ingress-controller.${NAMESPACE}.svc.cozy.local"
# Kubelet refreshes a ConfigMap volume within about a minute and CoreDNS's
# reload plugin polls every 30s; the rest is the 30s answer cache.
PROBE_TIMEOUT=240
# Each covers the driver's 2m hold after the credentials change plus the
# strategy Job itself.
JOB_TIMEOUT=480
workdir=$(mktemp -d)
on_exit() {
local rc=$?
set +e
restore_shared_state || rc=1
rm -rf "$workdir"
exit "$rc"
}
if kubectl -n "$NAMESPACE" get secret "$INGRESS_SECRET" -o name >/dev/null 2>&1; then
log_error "Secret ${NAMESPACE}/${INGRESS_SECRET} already exists; this Test serves its own certificate under that name and will not overwrite one it did not create"
exit 1
fi
if corefile_has_block; then
log_error "kube-system/coredns already carries the ${S3_HOST} rewrite block, left by an earlier run; remove it by hand"
exit 1
fi
trap on_exit EXIT
log_step "Serving the Test certificate for ${S3_HOST} through the platform S3 Ingress"
kubectl -n "$NAMESPACE" get secret "$CERT_SECRET" -o json \
| jq --arg name "$INGRESS_SECRET" '{apiVersion, kind, type,
metadata: {name: $name, labels: {"cozystack.io/e2e-owner": "clickhouse-system-bucket"}},
data: {"tls.crt": .data["tls.crt"], "tls.key": .data["tls.key"], "ca.crt": .data["ca.crt"]}}' \
| kubectl -n "$NAMESPACE" create -f -
log_step "Rewriting ${S3_HOST} to ${DNS_TARGET} in the host CoreDNS"
kubectl -n kube-system get configmap coredns -o json >"$workdir/coredns.json"
jq -e '.data.Corefile | startswith(".:53 {\n")' "$workdir/coredns.json" >/dev/null \
|| { log_error "kube-system/coredns Corefile does not open with '.:53 {'; refusing to edit it"; exit 1; }
# The resourceVersion read above travels with the replace, so a concurrent
# writer makes it fail instead of being overwritten.
jq --arg block "${DNS_BEGIN}"$'\n'" rewrite name exact ${S3_HOST} ${DNS_TARGET}"$'\n'"${DNS_END}" \
'.data.Corefile |= sub("^\\.:53 \\{\n"; ".:53 {\n" + $block + "\n")' \
"$workdir/coredns.json" >"$workdir/coredns-new.json"
kubectl replace -f "$workdir/coredns-new.json" >/dev/null
log_step "Checking that ${S3_HOST} serves the Test certificate from inside ${NAMESPACE}"
kubectl -n "$NAMESPACE" get secret "$CERT_SECRET" -o jsonpath='{.data.ca\.crt}' | base64 -d >"$workdir/ca.crt"
want=$(kubectl -n "$NAMESPACE" get secret "$CERT_SECRET" -o jsonpath='{.data.tls\.crt}' | base64 -d \
| openssl x509 -noout -fingerprint -sha256)
deadline=$((SECONDS + PROBE_TIMEOUT))
while true; do
# --cacert makes curl verify both the chain and the host name; %{certs}
# prints the chain it was served, leaf first.
if served=$(timeout 30 kubectl -n "$NAMESPACE" exec -i "$PROBE_POD" -- sh -c \
"cat >/tmp/ca.crt && curl -sS -o /dev/null --max-time 10 --cacert /tmp/ca.crt -w '%{certs}' https://${S3_HOST}/" \
<"$workdir/ca.crt" 2>"$workdir/probe.err"); then
got=$(printf '%s\n' "$served" | sed -n '/BEGIN CERTIFICATE/,/END CERTIFICATE/p;/END CERTIFICATE/q' \
| openssl x509 -noout -fingerprint -sha256)
if [ "$got" = "$want" ]; then
log_success "${S3_HOST} resolves in ${NAMESPACE} and serves the Test certificate"
break
fi
echo "served ${got}, want ${want}" >"$workdir/probe.err"
fi
if [ "$SECONDS" -ge "$deadline" ]; then
log_error "${S3_HOST} did not serve the Test certificate within ${PROBE_TIMEOUT}s: $(cat "$workdir/probe.err")"
kubectl -n kube-system get configmap coredns -o jsonpath='{.data.Corefile}' >&2 || true
kubectl -n "$NAMESPACE" describe ingress ingress-seaweedfs-s3 >&2 || true
exit 1
fi
# TODO(e2e-replace-fixed-timeouts): no condition reports that the host
# CoreDNS answers with the rewrite and the Ingress serves the Test
# certificate, as seen from this namespace; poll until one does.
sleep 10
done
# A restarted Pod reads its credentials afresh at start, which is what the
# old environment-based sidecar needed; the round-trip proves the fix only if
# the sidecar that started without them is the one that serves both jobs.
sidecar_identity() {
kubectl -n "$NAMESPACE" get pod "chi-clickhouse-${CLICKHOUSE_NAME}-clickhouse-0-0-0" \
-o jsonpath='{.metadata.uid} {.status.containerStatuses[?(@.name=="clickhouse-backup")].restartCount}'
}
sidecar_before=$(sidecar_identity)
[[ "$sidecar_before" =~ ^[0-9a-f-]+\ [0-9]+$ ]] \
|| { log_error "could not read the ClickHouse Pod uid and clickhouse-backup restart count (got '${sidecar_before}')"; exit 1; }
# Restore takes the archive this run's Backup recorded; a row no earlier run
# wrote is what shows the data restored is that archive's.
sentinel="before-backup-$(date -u +%Y%m%dT%H%M%SZ)-${RANDOM}"
log_step "Writing the sentinel row ${sentinel}"
clickhouse_query "$CLICKHOUSE_NAME" "CREATE TABLE IF NOT EXISTS default.sentinel (id UInt32, name String) ENGINE = MergeTree ORDER BY id"
clickhouse_query "$CLICKHOUSE_NAME" "INSERT INTO default.sentinel VALUES (1, '${sentinel}')"
log_step "Backing up through cozy-default"
kubectl apply -f - <<EOF
apiVersion: backups.cozystack.io/v1alpha1
kind: BackupJob
metadata:
name: ${BACKUPJOB_NAME}
namespace: ${NAMESPACE}
spec:
applicationRef:
apiGroup: apps.cozystack.io
kind: ClickHouse
name: ${CLICKHOUSE_NAME}
backupClassName: cozy-default
EOF
wait_for_field backupjob "$BACKUPJOB_NAME" '{.status.phase}' Succeeded "$NAMESPACE" "$JOB_TIMEOUT" Failed
backup_ref=$(kubectl -n "$NAMESPACE" get backupjob "$BACKUPJOB_NAME" -o jsonpath='{.status.backupRef.name}')
[ -n "$backup_ref" ] || { log_error "BackupJob ${BACKUPJOB_NAME} succeeded with an empty backupRef"; exit 1; }
# The sidecar started before this Secret existed; a backup that succeeded
# against it is the behaviour under test, and the key it read is this one.
endpoint=$(kubectl -n "$NAMESPACE" get secret cozy-backups-creds -o jsonpath='{.data.clickhouse-backup\.yml}' | base64 -d \
| sed -n 's/^ *endpoint: *//p')
[ "$endpoint" = "https://${S3_HOST}" ] \
|| { log_error "cozy-backups-creds clickhouse-backup.yml endpoint is '${endpoint}', expected https://${S3_HOST}"; exit 1; }
log_step "Restoring in place through cozy-default"
clickhouse_query "$CLICKHOUSE_NAME" "DROP TABLE IF EXISTS default.sentinel SYNC"
kubectl apply -f - <<EOF
apiVersion: backups.cozystack.io/v1alpha1
kind: RestoreJob
metadata:
name: ${RESTOREJOB_NAME}
namespace: ${NAMESPACE}
spec:
backupRef:
name: ${backup_ref}
EOF
wait_for_field restorejob "$RESTOREJOB_NAME" '{.status.phase}' Succeeded "$NAMESPACE" "$JOB_TIMEOUT" Failed
count=$(clickhouse_query "$CLICKHOUSE_NAME" "SELECT count() FROM default.sentinel WHERE name = '${sentinel}'" | tr -d '[:space:]')
[ "$count" = 1 ] || { log_error "after restore default.sentinel holds ${count} row(s) named ${sentinel}; expected 1"; exit 1; }
sidecar_after=$(sidecar_identity)
[ "$sidecar_after" = "$sidecar_before" ] \
|| { log_error "the ClickHouse Pod or its clickhouse-backup container restarted during the round-trip (uid restarts: '${sidecar_before}' then '${sidecar_after}')"; exit 1; }
log_success "System-bucket round-trip verified: sentinel row ${sentinel} restored"
@@ -0,0 +1,60 @@
#!/bin/bash
# The two shared objects system-bucket-roundtrip.sh changes for the duration
# of its run, and how to put them back. Sourced by that script, whose EXIT trap
# restores them, and by the finally of its Chainsaw step, which restores them
# when the step ceiling killed the script before the trap could run. Needs
# NAMESPACE and log_success/log_error from examples/backups/clickhouse/00-helpers.sh.
S3_HOST=s3.example.org
INGRESS_SECRET=seaweedfs-s3-ingress-tls
OWNER_LABEL=cozystack.io/e2e-owner=clickhouse-system-bucket
DNS_BEGIN=' # === BEGIN e2e-clickhouse-system-bucket ==='
DNS_END=' # === END e2e-clickhouse-system-bucket ==='
corefile_has_block() {
kubectl -n kube-system get configmap coredns -o jsonpath='{.data.Corefile}' | grep -qF -- "$DNS_BEGIN"
}
remove_dns_block() {
local dir
dir=$(mktemp -d)
kubectl -n kube-system get configmap coredns -o json >"$dir/coredns.json" || { rm -rf "$dir"; return 1; }
local begins
begins=$(jq -r '.data.Corefile' "$dir/coredns.json" | grep -cF -- "$DNS_BEGIN" || true)
if [ "$begins" = 0 ]; then
rm -rf "$dir"
return 0
fi
if [ "$begins" != 1 ] || ! jq -r '.data.Corefile' "$dir/coredns.json" | grep -qF -- "$DNS_END"; then
log_error "kube-system/coredns holds ${begins} '${DNS_BEGIN# *}' markers or no END marker; reconcile the Corefile by hand"
rm -rf "$dir"
return 1
fi
jq --arg b "$DNS_BEGIN" --arg e "$DNS_END" \
'.data.Corefile |= (split("\n") | (index($b)) as $i | (index($e)) as $j | .[:$i] + .[$j+1:] | join("\n"))' \
"$dir/coredns.json" >"$dir/coredns-clean.json"
# Callers run this with errexit off, so a refused replace has to return.
if ! kubectl replace -f "$dir/coredns-clean.json" >/dev/null; then
log_error "could not remove the ${S3_HOST} rewrite from kube-system/coredns; remove the marked block by hand"
rm -rf "$dir"
return 1
fi
log_success "Removed the ${S3_HOST} rewrite from kube-system/coredns"
rm -rf "$dir"
}
# Returns non-zero when either object could not be put back.
restore_shared_state() {
local rc=0
remove_dns_block || rc=1
# kubectl refuses a name together with a selector, so the label is
# looked up on its own and the name matched in what comes back.
local owned
if ! owned=$(kubectl -n "$NAMESPACE" get secret -l "$OWNER_LABEL" -o name); then
log_error "could not list the Secrets labelled ${OWNER_LABEL} in ${NAMESPACE}; delete ${INGRESS_SECRET} by hand if it carries that label"
rc=1
elif printf '%s\n' "$owned" | grep -qxF "secret/$INGRESS_SECRET"; then
kubectl -n "$NAMESPACE" delete secret "$INGRESS_SECRET" --wait=true --timeout=60s || rc=1
fi
return "$rc"
}
@@ -0,0 +1,42 @@
# Certificate and probe Pod for the clickhouse-3-system-bucket-roundtrip Test.
#
# The certificate is signed by the SeaweedFS CA already present in tenant-root,
# so its Secret carries that CA as ca.crt: the ClickHouse release trusts it via
# backup.endpointCA, and the round-trip script copies the same key pair under
# the name the platform's S3 Ingress serves for s3.example.org.
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: e2e-system-bucket-s3
spec:
secretName: e2e-system-bucket-s3-tls
dnsNames:
- s3.example.org
issuerRef:
kind: Issuer
name: seaweedfs-ca-issuer
---
# Resolves and fetches s3.example.org from inside tenant-root, where the
# ClickHouse sidecar runs, so the script can prove the DNS rewrite and the
# served certificate before it spends minutes on a BackupJob. The image is the
# one hack/e2e-install-cozystack.bats pre-pulls; it carries curl but no openssl.
apiVersion: v1
kind: Pod
metadata:
name: e2e-system-bucket-probe
spec:
restartPolicy: Never
terminationGracePeriodSeconds: 0
securityContext:
runAsNonRoot: true
runAsUser: 65534
seccompProfile:
type: RuntimeDefault
containers:
- name: probe
image: docker.io/alpine/k8s:1.36.2@sha256:44ef4942e171939b9c665a4a84beb80e2dcdb9a24330d4651cfdfd2e9deecc47
command: [sleep, "3600"]
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: [ALL]
@@ -35,7 +35,7 @@
# and the teardown. This is the third suite in a full run that brings up a
# tenant cluster, next to kubernetes-latest and kubernetes-previous. A failure
# that stalls worker join everywhere reddens all three, and their red paths
# together can exceed the E2E job's 215-minute cap, which cancels the job and
# together can exceed the E2E job's 265-minute cap, which cancels the job and
# loses its log and report. The cap is sized for the red path of the
# kubernetes pair only; this suite is not counted in it, and that is an
# accepted risk.
@@ -3,8 +3,8 @@ kind: FoundationDB
metadata:
name: test
spec:
version: v7.3
cluster:
version: "7.3.63"
processCounts:
storage: 3
stateless: -1
@@ -80,7 +80,8 @@ spec:
# wait, all on the passing path. Raising the ceiling to cover them is not
# available -- the guard in hack/run-kubernetes-serial-console_test.bats
# holds COZY_OP_CEILING equal to BOTH suites' `timeout:`, and two suites
# at 79m plus their 22m teardowns exceed the job's own 215-minute cap.
# at 79m plus their 22m teardowns take 202m, which with the install and
# the rest of a full run does not fit the job's own 265-minute cap.
# What is available is a measurement: on the green run of this branch
# (33724179053) the whole Test took 569.72s against this 4020s op, with
# kubernetes-previous, which carries no OIDC fold, at 470.19s. The
@@ -124,6 +124,12 @@ spec:
- script:
content: |
set -eu
# The tenant's super-admin kubeconfig that run-proxmox.sh fetched for
# the checks above; the same path the script derives. Removed before
# the keep_tenant exit below, because the file is a credential and
# keeping the tenant is not a reason to keep that on the runner. The
# workflow's cleanup step removes it again, for a run killed here.
rm -f "${COZY_PROXMOX_WORKDIR:-${RUNNER_TEMP:-${TMPDIR:-/tmp}}}/proxmox-e2e-tenant.kubeconfig"
# keep_tenant is honoured here as well as in the workflow's own
# teardown, or the suite would delete what the operator asked to keep.
if [ "${COZY_KEEP_TENANT:-false}" = "true" ]; then
@@ -65,7 +65,10 @@ spec:
apiVersion: apps.cozystack.io/v1alpha1
kind: KubernetesNodes
metadata:
name: md0
# <cluster>-<pool>: kubernetes-nodes refuses to render a release whose name
# does not start with kubernetes-nodes-<spec.cluster>-, so a bare "md0" never
# creates a Machine and the suite only times out waiting for one.
name: k8s-e2e-md0
namespace: tenant-e2eproxmox
spec:
cluster: k8s-e2e
@@ -0,0 +1,257 @@
# Namespace logs through the observability proxy, as the console reaches
# them: every request goes through the console nginx (a port-forward to its
# Service), which routes the logs root to the proxy, which authenticates the
# bearer token, asks a SubjectAccessReview and forwards a rebuilt query to
# the tenant's log store.
#
# Two sibling tenants, A (logsa) and B (logsb), both under tenant-test, so
# both resolve to the root log store. A viewer of A reads A's logs and is
# refused B's; parameters and headers a client adds cannot widen the scope
# (the proxy forces the namespace filter and tenant 0:0); a live tail stays
# open and keeps delivering lines past 60s.
#
# Not covered here: the OIDC gatekeeper (oauth2-proxy) in front of nginx,
# because driving a Keycloak login needs a browser; and Fluent Bit writing
# real pod logs, because the root log store is off in e2e and this suite
# seeds its own (log-store.yaml).
apiVersion: chainsaw.kyverno.io/v1alpha1
kind: Test
metadata:
name: observability-proxy-logs
spec:
catch:
- events: {}
- podLogs:
namespace: cozy-observability-proxy
selector: app.kubernetes.io/name=observability-proxy
tail: 100
- podLogs:
namespace: tenant-root
selector: app.kubernetes.io/name=e2e-observability-log-store
tail: 100
- podLogs:
namespace: cozy-dashboard
selector: app.kubernetes.io/name=console
tail: 100
steps:
- name: proxy-available
try:
- assert:
resource:
apiVersion: apps/v1
kind: Deployment
metadata:
name: observability-proxy
namespace: cozy-observability-proxy
status:
(conditions[?type == 'Available']):
- status: "True"
- name: tenants
try:
- apply:
resource:
apiVersion: apps.cozystack.io/v1alpha1
kind: Tenant
metadata:
name: logsa
spec: {}
- apply:
resource:
apiVersion: apps.cozystack.io/v1alpha1
kind: Tenant
metadata:
name: logsb
spec: {}
# 16m: tenant-rd's 15m install timeout plus the 60s start margin; see
# "Sizing an HR-Ready budget" in docs/agents/e2e-testing.md. Waited so
# that cleanup does not uninstall a release whose install still runs.
- assert:
timeout: 16m
resource:
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: tenant-logsa
status:
(conditions[?type == 'Ready']):
- status: "True"
- assert:
timeout: 16m
resource:
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: tenant-logsb
status:
(conditions[?type == 'Ready']):
- status: "True"
# The label names the store the proxy reads; the seeded store below
# stands in for the root one, so both namespaces must resolve to it.
- assert:
resource:
apiVersion: v1
kind: Namespace
metadata:
name: tenant-test-logsa
labels:
namespace.cozystack.io/monitoring: tenant-root
- assert:
resource:
apiVersion: v1
kind: Namespace
metadata:
name: tenant-test-logsb
labels:
namespace.cozystack.io/monitoring: tenant-root
- name: viewer-and-store
try:
# Applying over a real root store would rewrite its Service; refuse to.
- error:
resource:
apiVersion: v1
kind: Service
metadata:
name: vlselect-generic
namespace: tenant-root
- apply:
file: viewer.yaml
- apply:
file: log-store.yaml
- assert:
timeout: 3m
resource:
apiVersion: apps/v1
kind: Deployment
metadata:
name: e2e-observability-log-store
namespace: tenant-root
status:
(conditions[?type == 'Available']):
- status: "True"
- name: seed-logs
try:
- script:
# 30s forward + 3 x 15s ingests + 3 x 30s visibility polls, each
# capped on its own, under a 4m op.
timeout: 4m
content: |
set -eu
. ./lib.sh
trap 'for p in ${pf_pids:-}; do kill "$p" 2>/dev/null || true; done; rm -f "$body"' EXIT
forward tenant-root vlselect-generic 19471:9471
ingest tenant-test-logsa e2e-marker-a
ingest tenant-test-logsb e2e-marker-b
# Same namespace, another VictoriaLogs tenant: reachable only if a
# client AccountID header got through.
ingest tenant-test-logsa e2e-marker-account7 7
# All three must be searchable, or an absence checked later
# proves nothing.
for m in e2e-marker-a:0 e2e-marker-b:0 e2e-marker-account7:7; do
timeout 30 sh -ec "until curl --silent --fail --header 'AccountID: ${m#*:}' '$STORE/select/logsql/query?query=${m%:*}' | grep -q ${m%:*}; do sleep 1; done"
done
- name: namespace-scope
try:
- script:
# 30s forward + one 30s token request + 10 calls capped at 30s.
timeout: 8m
content: |
set -eu
. ./lib.sh
trap 'for p in ${pf_pids:-}; do kill "$p" 2>/dev/null || true; done; rm -f "$body"' EXIT
forward cozy-dashboard cozy-dashboard-console 18080:8080
token=$(k -n tenant-test-logsa create token logs-viewer --duration=10m)
auth="Authorization: Bearer $token"
code=$(call "$LOGS_A/query?query=*")
expect 401 "$code" "no token"
code=$(call "$LOGS_A/query?query=*" --header "$auth")
expect 200 "$code" "viewer reads A"
grep -q e2e-marker-a "$body" || { echo "FAIL: A's own line is missing" >&2; cat "$body" >&2; exit 1; }
absent e2e-marker-b "viewer reads A"
absent e2e-marker-account7 "viewer reads A"
code=$(call "$LOGS_B/query?query=*" --header "$auth")
expect 403 "$code" "viewer reads B"
code=$(call "$LOGS_A/query" --header "$auth" --get \
--data-urlencode 'query=*' \
--data-urlencode 'extra_stream_filters={kubernetes_namespace_name="tenant-test-logsb"}' \
--data-urlencode 'extra_filters=kubernetes_namespace_name:tenant-test-logsb')
expect 200 "$code" "client extra filters in the query string"
absent e2e-marker-b "client extra filters in the query string"
code=$(call "$LOGS_A/query" --header "$auth" \
--data-urlencode 'query=*' \
--data-urlencode 'extra_stream_filters={kubernetes_namespace_name="tenant-test-logsb"}')
expect 200 "$code" "client extra filters in a POST form"
absent e2e-marker-b "client extra filters in a POST form"
code=$(call "$LOGS_A/query?query=*" --header "$auth" --header "AccountID: 7" --header "ProjectID: 0")
expect 200 "$code" "client AccountID header"
absent e2e-marker-account7 "client AccountID header"
code=$(call "$LOGS_A/query" --header "$auth" --get \
--data-urlencode 'query={kubernetes_namespace_name="tenant-test-logsb"}')
expect 200 "$code" "B's stream named in the query"
absent e2e-marker-b "B's stream named in the query"
code=$(call "$LOGS_A/query" --header "$auth" --get \
--data-urlencode 'query=options(ignore_global_time_filter=true) *')
expect 400 "$code" "query options"
# nginx routes on the normalised path, which is B's logs root, and
# forwards the raw one; the proxy must refuse it, not read B.
code=$(call "$LOGS_A/../../../../tenant-test-logsb/logs/select/logsql/query?query=*" --path-as-is --header "$auth")
expect 400 "$code" "dot segments toward B"
code=$(call "$LOGS_A/delete?query=*" --header "$auth")
expect 404 "$code" "an endpoint outside the allowlist"
- name: tail-outlives-a-minute
try:
- script:
# 2 x 30s forwards + 30s token + 3 x 15s ingests + the 70s hold +
# the 30s delivery poll; the tail itself is cut at 150s.
timeout: 6m
content: |
set -eu
. ./lib.sh
tail_out=$(mktemp)
trap 'kill "${tail_pid:-}" 2>/dev/null || true; for p in ${pf_pids:-}; do kill "$p" 2>/dev/null || true; done; rm -f "$body" "$tail_out"' EXIT
forward tenant-root vlselect-generic 19471:9471
forward cozy-dashboard cozy-dashboard-console 18080:8080
token=$(k -n tenant-test-logsa create token logs-viewer --duration=10m)
# The tail is not subscribed yet when the next ingest runs, so it
# replays the last minute to include the early line.
timeout 150 curl --silent --no-buffer --header "Authorization: Bearer $token" \
"$CONSOLE$LOGS_A/tail?query=*&start_offset=1m" > "$tail_out" &
tail_pid=$!
ingest tenant-test-logsa e2e-tail-early
ingest tenant-test-logsb e2e-tail-b
# TODO(e2e-replace-fixed-timeouts): this hold stands in for a
# duration assertion the harness lacks; the property under test is
# that a tail survives a minute with the client sending nothing.
sleep 70
if ! kill -0 "$tail_pid" 2>/dev/null; then
echo "FAIL: the tail closed before 70s" >&2
cat "$tail_out" >&2
exit 1
fi
ingest tenant-test-logsa e2e-tail-late
# vlselect re-reads every second, five seconds behind now.
timeout 30 sh -ec "until grep -q e2e-tail-late '$tail_out'; do sleep 1; done" || {
echo "FAIL: a line written at 70s did not reach the open tail" >&2
cat "$tail_out" >&2
exit 1
}
grep -q e2e-tail-early "$tail_out" || { echo "FAIL: the early line is missing from the tail" >&2; exit 1; }
if grep -q e2e-tail-b "$tail_out"; then
echo "FAIL: the tail of A delivered B's line" >&2
exit 1
fi
@@ -0,0 +1,58 @@
# shellcheck shell=sh
# Shared by the script steps of this suite, whose cwd is this directory.
# Every call carries its own cap so a hang fails at a known budget rather
# than at the op ceiling (docs/agents/e2e-testing.md, section 8).
k() { timeout 30 kubectl "$@"; }
STORE=http://localhost:19471
CONSOLE=http://localhost:18080
# shellcheck disable=SC2034 # read by the steps that source this file
LOGS_A=/apis/cozystack.io/v1alpha1/namespaces/tenant-test-logsa/logs/select/logsql
# shellcheck disable=SC2034
LOGS_B=/apis/cozystack.io/v1alpha1/namespaces/tenant-test-logsb/logs/select/logsql
# forward <namespace> <service> <local>:<remote>. The caller's EXIT trap
# kills every pid in pf_pids; the forward gets 30s to start listening.
forward() {
kubectl port-forward -n "$1" "service/$2" "$3" >/dev/null 2>&1 &
pf_pids="${pf_pids:-} $!"
timeout 30 sh -ec "until nc -z localhost ${3%%:*}; do sleep 1; done"
}
# ingest <namespace> <message> [account]: one entry shaped as Fluent Bit
# writes it, with kubernetes_namespace_name as a stream field.
ingest() {
printf '{"kubernetes_namespace_name":"%s","log":"%s"}\n' "$1" "$2" |
timeout 15 curl --silent --show-error --fail --data-binary @- \
--header 'Content-Type: application/json' --header "AccountID: ${3:-0}" --header "ProjectID: 0" \
"$STORE/insert/jsonline?_stream_fields=kubernetes_namespace_name&_msg_field=log"
}
# call <path> [curl args...]: a request through the console nginx, the way
# the browser sends it. Prints the status; the body lands in $body.
body=$(mktemp)
call() {
path=$1
shift
timeout 30 curl --silent --show-error --output "$body" --write-out '%{http_code}' "$@" "$CONSOLE$path"
}
# expect <want-status> <got-status> <case>
expect() {
if [ "$1" != "$2" ]; then
echo "FAIL: $3: HTTP $2, want $1" >&2
cat "$body" >&2
exit 1
fi
echo "ok: $3: HTTP $2"
}
# absent <marker> <case>: the last body must not carry the marker.
absent() {
if grep -q "$1" "$body"; then
echo "FAIL: $2: the response carries $1, which is outside the scope" >&2
cat "$body" >&2
exit 1
fi
}
@@ -0,0 +1,71 @@
# A single-node VictoriaLogs under the name the proxy dials for a namespace
# whose logs belong to the root tenant: vlselect-generic.tenant-root, port
# 9471. The e2e install leaves root monitoring off, so the real one does not
# exist; the suite seeds this one itself instead of waiting for Fluent Bit,
# which keeps the test about the proxy's scope rather than log collection.
# The version is the one victoria-metrics-operator v0.68.4 deploys.
apiVersion: apps/v1
kind: Deployment
metadata:
name: e2e-observability-log-store
namespace: tenant-root
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: e2e-observability-log-store
template:
metadata:
labels:
app.kubernetes.io/name: e2e-observability-log-store
spec:
securityContext:
runAsNonRoot: true
runAsUser: 65534
seccompProfile:
type: RuntimeDefault
containers:
- name: victoria-logs
image: docker.io/victoriametrics/victoria-logs:v1.47.0
args:
- -httpListenAddr=:9471
- -storageDataPath=/storage
- -retentionPeriod=1d
ports:
- name: http
containerPort: 9471
readinessProbe:
httpGet:
path: /health
port: http
resources:
requests:
cpu: 50m
memory: 128Mi
limits:
cpu: 500m
memory: 256Mi
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop: ["ALL"]
volumeMounts:
- name: storage
mountPath: /storage
volumes:
- name: storage
emptyDir: {}
---
apiVersion: v1
kind: Service
metadata:
name: vlselect-generic
namespace: tenant-root
spec:
selector:
app.kubernetes.io/name: e2e-observability-log-store
ports:
- name: http
port: 9471
targetPort: http
@@ -0,0 +1,22 @@
# The view level of tenant A and nothing else: a ServiceAccount of tenant A
# bound to cozy:tenant:view in A's namespace only. Its token is a real bearer
# token, so the proxy's TokenReview and SubjectAccessReview run unmocked.
apiVersion: v1
kind: ServiceAccount
metadata:
name: logs-viewer
namespace: tenant-test-logsa
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: logs-viewer
namespace: tenant-test-logsa
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cozy:tenant:view
subjects:
- kind: ServiceAccount
name: logs-viewer
namespace: tenant-test-logsa
@@ -347,3 +347,148 @@ spec:
set -eu
cd ../../..
NAMESPACE="$NAMESPACE" examples/backups/postgres/cleanup.sh
---
# A system database that a version before the refusal took over is handed back
# by the next init-job run, and only what this chart marked changes hands.
#
# The helm-unittest and bats suites check the rendered SQL against a fake psql;
# only a real PostgreSQL can say that the predicates select what they claim. The
# state an older version left is rebuilt by hand as the superuser, since the
# chart now refuses to produce it:
# * postgres is commented as managed and owned by a postgres_admin role that
# carries the cozystack:managed mark, as the chart left it: it must come back
# to the superuser, and the role must be dropped;
# * template1 is commented as managed, but its template1_admin carries only the
# role comment, which a CREATEROLE holder can forge, and not the mark: the
# comment must be cleared and everything that role owns or was granted must
# stay as it is, in template1 and in a database cloned from it.
apiVersion: chainsaw.kyverno.io/v1alpha1
kind: Test
metadata:
name: postgres-3-system-databases
spec:
catch:
- events: {}
- describe:
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
name: postgres-sysdb
- podLogs:
selector: job-name=postgres-sysdb-init-job
tail: 200
steps:
- name: create-postgres
try:
- apply:
file: postgres-sysdb.yaml
# 11m: above the 600s install timeout cozystack-api gives this release;
# see "Sizing an HR-Ready budget" in docs/agents/e2e-testing.md.
- assert:
timeout: 11m
resource:
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: postgres-sysdb
status:
(conditions[?type == 'Ready']):
- status: "True"
- assert:
timeout: 2m
resource:
apiVersion: batch/v1
kind: Job
metadata:
name: postgres-sysdb-init-job
status:
(conditions[?type == 'Complete']):
- status: "True"
- name: rebuild-taken-over-system-databases
try:
- script:
timeout: 3m
content: |
set -eu
pod=$(kubectl -n "$NAMESPACE" get pod \
-l cnpg.io/cluster=postgres-sysdb,cnpg.io/instanceRole=primary -o name | head -n1)
[ -n "$pod" ] || { echo "no primary pod for postgres-sysdb" >&2; exit 1; }
sql() { kubectl -n "$NAMESPACE" exec -i "$pod" -c postgres -- psql -v ON_ERROR_STOP=1 -U postgres -d "$1"; }
sql postgres <<'SQL'
CREATE ROLE postgres_admin NOLOGIN;
COMMENT ON ROLE postgres_admin IS 'role managed by helm';
GRANT "cozystack:managed" TO postgres_admin WITH INHERIT FALSE, SET FALSE;
COMMENT ON DATABASE postgres IS 'database managed by helm';
ALTER DATABASE postgres OWNER TO postgres_admin;
CREATE ROLE template1_admin NOLOGIN;
COMMENT ON ROLE template1_admin IS 'role managed by helm';
CREATE ROLE template1_readonly NOLOGIN;
COMMENT ON DATABASE template1 IS 'database managed by helm';
ALTER DATABASE template1 OWNER TO template1_admin;
SQL
sql template1 <<'SQL'
ALTER SCHEMA public OWNER TO template1_admin;
CREATE TABLE public.kept (id int);
ALTER TABLE public.kept OWNER TO template1_admin;
GRANT SELECT ON public.kept TO template1_readonly;
SQL
sql postgres <<'SQL'
CREATE DATABASE sysdb_clone TEMPLATE template1;
SQL
# The next run must be a new Job: remove the completed one so the
# assert below cannot match it.
kubectl -n "$NAMESPACE" delete job postgres-sysdb-init-job --wait=true
# Declaring a user is enough for an upgrade, and runs the init-job.
kubectl -n "$NAMESPACE" patch postgres.apps.cozystack.io sysdb --type merge \
-p '{"spec":{"users":{"sysdbuser":{},"sysdbprobe":{}}}}'
check:
($error == null): true
- assert:
timeout: 11m
resource:
apiVersion: batch/v1
kind: Job
metadata:
name: postgres-sysdb-init-job
status:
(conditions[?type == 'Complete']):
- status: "True"
- name: verify-release
try:
- script:
timeout: 2m
content: |
set -eu
pod=$(kubectl -n "$NAMESPACE" get pod \
-l cnpg.io/cluster=postgres-sysdb,cnpg.io/instanceRole=primary -o name | head -n1)
q() { kubectl -n "$NAMESPACE" exec "$pod" -c postgres -- psql -v ON_ERROR_STOP=1 -U postgres -tA -d "$1" -c "$2"; }
fail=0
expect() {
got=$(q "$1" "$3")
if [ "$got" != "$4" ]; then
echo "FAIL: $2: got '$got', want '$4'" >&2
fail=1
fi
}
comment="SELECT coalesce(shobj_description(oid, 'pg_database'), '<none>') FROM pg_database WHERE datname = "
owner="SELECT pg_get_userbyid(datdba) FROM pg_database WHERE datname = "
expect postgres "postgres comment cleared" "$comment'postgres'" '<none>'
expect postgres "postgres back to the superuser" "$owner'postgres'" postgres
expect postgres "marked postgres_admin dropped" "SELECT count(*) FROM pg_roles WHERE rolname = 'postgres_admin'" 0
expect postgres "template1 comment cleared" "$comment'template1'" '<none>'
expect postgres "unmarked template1_admin still exists" "SELECT count(*) FROM pg_roles WHERE rolname = 'template1_admin'" 1
expect postgres "template1 still owned by the unmarked role" "$owner'template1'" template1_admin
for db in template1 sysdb_clone; do
expect "$db" "$db public still owned by the unmarked role" \
"SELECT pg_get_userbyid(nspowner) FROM pg_namespace WHERE nspname = 'public'" template1_admin
expect "$db" "$db kept table still owned by the unmarked role" \
"SELECT tableowner FROM pg_tables WHERE schemaname = 'public' AND tablename = 'kept'" template1_admin
expect "$db" "$db grant to the unmarked readonly role kept" \
"SELECT has_table_privilege('template1_readonly', 'public.kept', 'SELECT')" t
done
exit "$fail"
check:
($error == null): true
# The Postgres CR was applied by chainsaw; deleting it removes the whole
# cluster, the databases and roles built above included.
@@ -0,0 +1,18 @@
apiVersion: apps.cozystack.io/v1alpha1
kind: Postgres
metadata:
name: sysdb
spec:
external: false
size: 1Gi
replicas: 1
storageClass: ""
quorum:
minSyncReplicas: 0
maxSyncReplicas: 0
users:
sysdbuser: {}
databases:
appdb: {}
resources: {}
resourcesPreset: "nano"
+27
View File
@@ -0,0 +1,27 @@
#!/usr/bin/env bats
load test_helper
REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)"
VMALERT_TOOL_IMAGE="victoriametrics/vmalert-tool:v1.136.0@sha256:417346d7b6fdb66d8e3984887c2ad114535ec7d983eeef53381b56c776f333f1"
@test "tenant etcd failed gRPC request alerts pass their vmalert-tool cases" {
docker info >/dev/null 2>&1 || {
echo "docker is required to run the cases in $VMALERT_TOOL_IMAGE" >&2
return 1
}
work=$(mktemp -d)
helm template etcd "$REPO_ROOT/packages/extra/etcd" --namespace tenant-test \
--set '_cluster.cluster-domain=cozy.local' \
--show-only templates/prometheus-rules.yaml > "$work/rendered.yaml"
yq '.spec' "$work/rendered.yaml" > "$work/rules.yaml"
cp "$REPO_ROOT/hack/testdata/etcd-grpc-alerts/unittest.yaml" "$work/unittest.yaml"
rc=0
docker run --rm --network none -v "$work:/w:ro" "$VMALERT_TOOL_IMAGE" \
unittest --files /w/unittest.yaml > "$work/out" 2>&1 || rc=$?
cat "$work/out"
passed=$(grep -cx 'SUCCESS' "$work/out" || true)
rm -rf "$work"
[ "$rc" -eq 0 ]
[ "$passed" -eq 1 ]
}
+155
View File
@@ -0,0 +1,155 @@
#!/usr/bin/env bats
# Helm keeps every release revision in one Secret, and the apiserver refuses a
# Secret over 1 MiB. A chart past that budget installs and then fails the one
# upgrade whose manifest is large enough ("Secret sh.helm.release.v1.<name>.vN
# is invalid: data: Too long"), and that release stays failed. The stored
# release carries the whole chart directory as it reaches helm-controller:
# the packages artifact applies no .helmignore, so tests/, images/ and the
# Makefile count too. hack/helm-release-size.py reproduces the encoding.
#
# The budget is 90 percent of the limit: the estimate runs about 1 percent
# above real Secrets, and the rest is room for a chart to grow by more than
# one change before anyone has to act.
#
# Harness compatibility note: CI runs this file under Bats through
# `make bats-unit-tests`. It also remains compatible with the narrower legacy
# `hack/cozytest.sh` translator, so it avoids `run`, `$status`, `$output`,
# `skip`, and setup()/teardown(). Paths stay repo-root-relative for both
# runners.
#
# Run with: bats hack/helm-release-size.bats
load test_helper
LIMIT=1048576
BUDGET=943718
# Charts that do not render offline with default values (required values,
# cluster lookups). They are measured on their stored files and templates
# alone, a lower bound. Any other chart must render: a render failure there
# would quietly shrink the measurement to the same lower bound. A chart that
# stops rendering offline on purpose goes here; one that starts rendering
# must come off, so the list never hides a renderable chart.
NO_OFFLINE_RENDER="
packages/apps/bucket
packages/apps/clickhouse
packages/apps/foundationdb
packages/apps/harbor
packages/apps/kafka
packages/apps/kubernetes
packages/apps/kubernetes-nodes
packages/apps/mariadb
packages/apps/mongodb
packages/apps/nats
packages/apps/openbao
packages/apps/opensearch
packages/apps/postgres
packages/apps/qdrant
packages/apps/rabbitmq
packages/apps/redis
packages/apps/tenant
packages/apps/valkey
packages/apps/vm-disk
packages/apps/vm-instance
packages/apps/vpc
packages/apps/vpn
packages/core/platform
packages/extra/bootbox
packages/extra/computeplane
packages/extra/etcd
packages/extra/external-dns
packages/extra/gateway
packages/extra/info
packages/extra/ingress
packages/extra/monitoring
packages/extra/seaweedfs
packages/library/cozy-lib
packages/system/kuberture
packages/system/telepresence
packages/tests/cozy-lib-tests
"
@test "the estimator counts chart files the way Helm stores them" {
tmp=$(mktemp -d)
mkdir -p "$tmp/chart/templates" "$tmp/chart/tests"
printf 'apiVersion: v2\nname: probe\nversion: 0.0.0\n' > "$tmp/chart/Chart.yaml"
# Incompressible, so the stored size cannot be smaller than its base64.
head -c 300000 /dev/urandom > "$tmp/chart/tests/blob"
size=$(printf '' | python3 -I hack/helm-release-size.py "$tmp/chart")
if [ "$size" -lt 400000 ]; then
echo "estimate $size for a chart carrying 300000 random bytes; files are not counted" >&2
false
fi
rm -rf "$tmp"
}
@test "every chart's stored release fits the budget with its default render" {
tmp=$(mktemp -d)
: > "$tmp/over"
: > "$tmp/render"
n=0
for c in packages/*/*/; do
c=${c%/}
[ -f "$c/Chart.yaml" ] || continue
n=$((n + 1))
listed=no
case "$NO_OFFLINE_RENDER" in *"
$c
"*) listed=yes ;; esac
if helm template release "$c" -n default > "$tmp/manifest" 2> "$tmp/err"; then
if [ "$listed" = yes ]; then
echo "$c renders offline now; remove it from NO_OFFLINE_RENDER in hack/helm-release-size.bats" >> "$tmp/render"
fi
elif [ "$listed" = yes ]; then
: > "$tmp/manifest"
else
echo "$c failed to render offline; fix the render, or add it to NO_OFFLINE_RENDER in hack/helm-release-size.bats if it needs cluster state: $(grep -m 1 '^Error' "$tmp/err" | cut -c1-300)" >> "$tmp/render"
continue
fi
size=$(python3 -I hack/helm-release-size.py "$c" < "$tmp/manifest")
if [ "$size" -gt "$BUDGET" ]; then
echo "$c: $size bytes (budget $BUDGET, limit $LIMIT)" >> "$tmp/over"
fi
done
if [ "$n" -lt 50 ]; then
echo "only $n charts found under packages/; the sweep checked almost nothing" >&2
false
fi
if [ -s "$tmp/render" ]; then
cat "$tmp/render" >&2
false
fi
if [ -s "$tmp/over" ]; then
echo "charts whose stored Helm release is over budget:" >&2
cat "$tmp/over" >&2
false
fi
rm -rf "$tmp"
}
@test "backupstrategy-controller fits the budget on its largest revision" {
# The default render gates every Strategy CR off. The revision that renders
# all of them, with the Velero location on as in production, is the one
# that grew past the limit once.
c=packages/system/backupstrategy-controller
tmp=$(mktemp -d)
# Rendered to a file rather than piped: in a pipe the estimator's status
# would stand for the render's, and a failed render would be measured as an
# empty manifest and pass.
if ! helm template backupstrategy-controller "$c" -n cozy-backup-controller \
--set backupStorage.bucketNameOverride=cozy-backups-0123 \
--set velero.bslEnabled=true > "$tmp/manifest"; then
echo "$c failed to render its largest revision; nothing was measured" >&2
false
fi
if ! grep -q '^kind: Kafka$' "$tmp/manifest"; then
echo "$c rendered without the default Strategy CRs; this is not its largest revision" >&2
false
fi
size=$(python3 -I hack/helm-release-size.py "$c" < "$tmp/manifest")
rm -rf "$tmp"
if [ "$size" -gt "$BUDGET" ]; then
echo "$c: $size bytes with every default Strategy rendered (budget $BUDGET, limit $LIMIT)" >&2
false
fi
}
+106
View File
@@ -0,0 +1,106 @@
#!/usr/bin/env python3
"""Estimate the size of the Secret in which Helm stores a release of a chart.
Helm's Secret storage driver keeps each revision as one Secret whose `release`
value is base64(gzip(json(release))), and the apiserver refuses a Secret whose
data exceeds 1 MiB. A chart that crosses it installs fine at first and then
fails the one upgrade whose rendered manifest is large enough, with "Secret
sh.helm.release.v1.<name>.vN is invalid: data: Too long". The stored release
carries the whole chart, not just what renders: every file under the chart dir
that is not Chart.yaml, values.yaml, values.schema.json, templates/ or charts/
ends up in chart.files, and the production path (the packages OCI artifact
loaded by helm-controller) applies no .helmignore, so tests/, images/ and the
Makefile are stored too.
This reproduces the encoding: the release JSON with the chart's templates and
files as base64 []byte, the rendered manifest read from stdin (`helm template`
output), gzip at Helm's BestCompression, then base64. It does not reproduce
Helm's gzip byte for byte or the runtime fields (info, config); measured
against real release Secrets it lands within about 0.5 percent, which is why
callers compare it to a threshold with headroom rather than to the limit.
Usage: helm template ... | helm-release-size.py <chart-dir> [--exclude GLOB ...]
Prints the estimated stored size in bytes.
"""
import base64
import fnmatch
import gzip
import json
import os
import sys
# Paths Helm keeps out of chart.files: they are stored elsewhere in the chart
# object (metadata, values, schema, templates) or are subcharts.
NOT_FILES = {"Chart.yaml", "Chart.lock", "values.yaml", "values.schema.json"}
NOT_FILE_DIRS = ("templates/", "charts/")
def go_json(obj):
# encoding/json escapes <, > and & inside strings; match it so the size
# does not drift with how much HTML-ish text a chart carries.
s = json.dumps(obj, separators=(",", ":"), ensure_ascii=False)
return s.replace("<", "\\u003c").replace(">", "\\u003e").replace("&", "\\u0026")
def main(argv):
if len(argv) < 2:
sys.stderr.write(__doc__)
return 2
chart = argv[1].rstrip("/")
excludes = []
rest = argv[2:]
while rest:
if rest[0] != "--exclude" or len(rest) < 2:
sys.stderr.write("unexpected argument: %s\n" % rest[0])
return 2
excludes.append(rest[1])
rest = rest[2:]
templates, files = [], []
for root, dirs, names in os.walk(chart):
dirs.sort()
for n in sorted(names):
path = os.path.join(root, n)
rel = os.path.relpath(path, chart)
if any(fnmatch.fnmatch(rel, g) for g in excludes):
continue
with open(path, "rb") as fh:
data = base64.b64encode(fh.read()).decode()
entry = {"name": rel, "data": data}
if rel.startswith("templates/"):
templates.append(entry)
elif rel in NOT_FILES or rel.startswith(NOT_FILE_DIRS):
continue
else:
files.append(entry)
def read(rel):
p = os.path.join(chart, rel)
return open(p).read() if os.path.exists(p) else ""
release = {
"name": "release",
"info": {"status": "deployed", "description": "Upgrade complete"},
"chart": {
"metadata": {"name": os.path.basename(chart), "version": "0.0.0", "apiVersion": "v2"},
"lock": None,
"templates": templates,
# Helm stores the parsed values and the user config; raw text with
# comments and no config errs high (about 6 KB on
# backupstrategy-controller), the safe side for a budget.
"values": read("values.yaml"),
"schema": base64.b64encode(read("values.schema.json").encode()).decode() or None,
"files": files,
},
"manifest": sys.stdin.read(),
"version": 1,
"namespace": "default",
}
raw = go_json(release).encode()
stored = base64.b64encode(gzip.compress(raw, compresslevel=9))
print(len(stored))
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv))
+187
View File
@@ -0,0 +1,187 @@
#!/usr/bin/env bats
# -----------------------------------------------------------------------------
# packages/apps/kubernetes: the OIDC bootstrap Job must not carry the users[]
# list inside its container command.
#
# The Job's script reaches the container as one `sh -c` argument, and Linux caps
# one exec argument at 128 KiB (MAX_ARG_STRLEN). With a ClusterRoleBinding per
# user written into that script, about 240 users take it past the cap: the
# container cannot start, the post-install/post-upgrade hook fails, and the
# tenant cluster's HelmRelease stays unready. The bindings reach the Job through
# a mounted ConfigMap instead, so the script is the same size for any list.
#
# The last three tests execute the rendered script against a fake kubectl, since
# no assertion on the script's text says what the prune step does with the list:
# a condition forced either way passes every helm-unittest case.
#
# Requires: helm, yq (mikefarah v4+), jq, like hack/admin-kubeconfig-invariant.bats.
# -----------------------------------------------------------------------------
load test_helper
USERS=300
# render_users <dir> <count> renders the chart in System mode with <count>
# generated users and leaves one JSON document per object in <dir>/rendered.json.
render_users() {
jq -n --argjson n "$2" '
{oidc: {users: [range($n) | {email: "user\(.)@example.org", role: (if . % 7 == 0 then "admin" else "view" end)}]}}' \
> "$1/users.yaml"
helm template bindings packages/apps/kubernetes \
--namespace tenant-test \
--values packages/apps/kubernetes/tests/values/oidc-system.yaml \
--values "$1/users.yaml" \
> "$1/rendered.yaml"
yq --output-format=json eval-all '.' "$1/rendered.yaml" > "$1/rendered.json"
}
# render_job <dir> <users JSON> renders the chart in CustomConfig mode and unpacks
# what the bootstrap Job runs and mounts into <dir>: script.sh is its `sh -c`
# script, bindings/ holds the two files of the mounted ConfigMap. -j keeps an
# empty value an empty file, as the kubelet writes it.
render_job() {
jq -n --argjson users "$2" '
{_cluster: {"oidc-enabled": "false"},
oidc: {mode: "CustomConfig",
customConfig: {config: "apiVersion: apiserver.config.k8s.io/v1beta1\nkind: AuthenticationConfiguration\njwt: []\n"},
users: $users}}' > "$1/values.yaml"
helm template bindings packages/apps/kubernetes \
--namespace tenant-test \
--values packages/apps/kubernetes/tests/values/oidc-system.yaml \
--values "$1/values.yaml" \
> "$1/rendered.yaml"
yq --output-format=json eval-all '.' "$1/rendered.yaml" > "$1/rendered.json"
jq -s -j '.[] | select(.kind == "Job" and .metadata.name == "bindings-oidc-bootstrap")
| .spec.template.spec.containers[0].command[2]' "$1/rendered.json" > "$1/script.sh"
mkdir -p "$1/bindings"
jq -s -j '.[] | select(.kind == "ConfigMap" and .metadata.name == "bindings-oidc-bindings")
| .data["crbs.yaml"]' "$1/rendered.json" > "$1/bindings/crbs.yaml"
jq -s -j '.[] | select(.kind == "ConfigMap" and .metadata.name == "bindings-oidc-bindings")
| .data["desired.txt"]' "$1/rendered.json" > "$1/bindings/desired.txt"
}
# fake_kubectl <dir> <names> puts a kubectl first on PATH that logs every call and
# answers the script's list of existing bindings with <names>.
fake_kubectl() {
mkdir -p "$1/bin"
printf '%s' "$2" > "$1/existing"
cat > "$1/bin/kubectl" <<'KEOF'
#!/bin/sh
echo "$*" >> "$KLOG"
case "$*" in
*"get clusterrolebindings"*) cat "$EXISTING" ;;
esac
KEOF
chmod +x "$1/bin/kubectl"
}
# run_job <dir> runs the unpacked script as the container does and leaves its
# status in $rc and its output in <dir>/out. Run by bash, which has the pipefail
# the script sets and which is what the unit lane runs under; the container's own
# shell is not exercised.
run_job() {
: > "$1/kubectl.log"
echo kubeconfig > "$1/kubeconfig"
rc=0
env PATH="$1/bin:$PATH" KLOG="$1/kubectl.log" EXISTING="$1/existing" HOME="$1" \
KUBECONFIG_TENANT="$1/kubeconfig" RELEASE=bindings RELEASE_NAMESPACE=tenant-test \
OIDC_MODE=CustomConfig OIDC_BINDINGS_DIR="$1/bindings" \
bash "$1/script.sh" > "$1/out" 2>&1 || rc=$?
}
@test "the OIDC bootstrap Job script stays under the exec argument cap and carries no user" {
dir=$(mktemp -d)
render_users "$dir" "$USERS"
jq -s -r '.[] | select(.kind == "Job" and .metadata.name == "bindings-oidc-bootstrap")
| .spec.template.spec.containers[0].command[2]' "$dir/rendered.json" > "$dir/script"
size=$(wc -c < "$dir/script")
[ "$size" -gt 0 ]
[ "$size" -lt 131072 ]
if grep -q -E 'user[0-9]+@example\.org' "$dir/script"; then
echo "the Job script still carries users[] entries" >&2
false
fi
rm -rf "$dir"
}
@test "every user keeps one ClusterRoleBinding with its old name and role in the bindings ConfigMap" {
dir=$(mktemp -d)
render_users "$dir" "$USERS"
jq -s -r '.[] | select(.kind == "ConfigMap" and .metadata.name == "bindings-oidc-bindings")
| .data["crbs.yaml"]' "$dir/rendered.json" > "$dir/crbs.yaml"
jq -s -r '.[] | select(.kind == "ConfigMap" and .metadata.name == "bindings-oidc-bindings")
| .data["desired.txt"]' "$dir/rendered.json" > "$dir/desired.txt"
[ -s "$dir/crbs.yaml" ]
yq --output-format=json eval-all '.' "$dir/crbs.yaml" | jq -s '.' > "$dir/crbs.json"
[ "$(jq 'length' "$dir/crbs.json")" -eq "$USERS" ]
[ "$(jq -r '[.[].subjects[0].name] | unique | length' "$dir/crbs.json")" -eq "$USERS" ]
[ "$(jq -r '[.[] | select(.roleRef.name == "cluster-admin")] | length' "$dir/crbs.json")" -eq $(( (USERS + 6) / 7 )) ]
# The name is what the prune step and every binding already in a tenant cluster
# go by, so it must not move: cozystack-oidc- plus the first 16 hex digits of
# sha256("<release>-<email>").
for i in 0 1 6 7; do
email="user$i@example.org"
name="cozystack-oidc-$(printf '%s' "bindings-$email" | sha256sum | cut -c1-16)"
[ "$(jq -r --arg e "$email" '.[] | select(.subjects[0].name == $e) | .metadata.name' "$dir/crbs.json")" = "$name" ]
grep -q -x -F "$name" "$dir/desired.txt"
done
# The list the prune step compares against is exactly the applied names.
[ "$(jq -r '.[].metadata.name' "$dir/crbs.json" | sort)" = "$(grep . "$dir/desired.txt" | sort)" ]
rm -rf "$dir"
}
@test "the Job applies the mounted bindings and deletes only the bindings no user has" {
dir=$(mktemp -d)
render_job "$dir" '[{"email":"alice@example.org","role":"admin"},{"email":"bob@example.org","role":"view"}]'
alice="cozystack-oidc-$(printf '%s' 'bindings-alice@example.org' | sha256sum | cut -c1-16)"
bob="cozystack-oidc-$(printf '%s' 'bindings-bob@example.org' | sha256sum | cut -c1-16)"
fake_kubectl "$dir" "$alice cozystack-oidc-000000000000dead $bob cozystack-oidc-000000000000beef"
run_job "$dir"
if [ "$rc" -ne 0 ]; then
cat "$dir/out"
false
fi
grep -q -F -- "apply --filename $dir/bindings/crbs.yaml" "$dir/kubectl.log"
grep -q -F -- "delete clusterrolebinding cozystack-oidc-000000000000dead --ignore-not-found" "$dir/kubectl.log"
grep -q -F -- "delete clusterrolebinding cozystack-oidc-000000000000beef --ignore-not-found" "$dir/kubectl.log"
if grep -F -e "delete clusterrolebinding $alice" -e "delete clusterrolebinding $bob" "$dir/kubectl.log"; then
echo "the Job deleted a binding a user still has" >&2
false
fi
[ "$(grep -c 'delete clusterrolebinding' "$dir/kubectl.log")" -eq 2 ]
rm -rf "$dir"
}
@test "a release with no users applies nothing and deletes every binding it left behind" {
dir=$(mktemp -d)
render_job "$dir" '[]'
fake_kubectl "$dir" "cozystack-oidc-000000000000dead cozystack-oidc-000000000000beef"
run_job "$dir"
if [ "$rc" -ne 0 ]; then
cat "$dir/out"
false
fi
if grep -q -F -- ' apply ' "$dir/kubectl.log"; then
echo "the Job ran kubectl apply with no bindings to apply" >&2
false
fi
[ "$(grep -c 'delete clusterrolebinding' "$dir/kubectl.log")" -eq 2 ]
rm -rf "$dir"
}
@test "the Job stops before deleting anything when the desired list is not mounted" {
dir=$(mktemp -d)
render_job "$dir" '[{"email":"alice@example.org","role":"admin"}]'
fake_kubectl "$dir" "cozystack-oidc-000000000000dead cozystack-oidc-000000000000beef"
rm "$dir/bindings/desired.txt"
run_job "$dir"
[ "$rc" -ne 0 ]
if grep -q -E 'delete clusterrolebinding| apply ' "$dir/kubectl.log"; then
cat "$dir/kubectl.log"
false
fi
rm -rf "$dir"
}
+1 -1
View File
@@ -214,7 +214,7 @@ assert_labels_defined() {
count="$(stale_exempt_list exempt-issue-labels | wc -l | tr -d ' ')"
[ "$count" -eq 12 ]
count="$(stale_exempt_list exempt-pr-labels | wc -l | tr -d ' ')"
[ "$count" -eq 4 ]
[ "$count" -eq 2 ]
# The renewal path: frozen exempts both; an accepted issue or a held PR is
# deliberate, not stale.
+2 -2
View File
@@ -264,7 +264,7 @@ lineno() {
@test "external S3 (no COSI at all) keeps the Strategy CR endpoint scheme verbatim" {
prep
# provisionBucket: false on a cluster with no SeaweedFS/COSI at all
# (docs/operations/backup-classes.md describes exactly this flip), so the
# (https://cozystack.io/docs/v1/operations/services/backup-classes/ describes exactly this flip), so the
# BucketClaim CRD itself is absent and every claim query fails the way real
# kubectl fails on an unknown resource type. That must classify as external
# and must NOT abort the script under `set -euo pipefail`.
@@ -382,7 +382,7 @@ lineno() {
@test "COSI with overridden bucket coordinates is still detected (no hardcoded names)" {
prep
# backupStorage.namespace / .bucketName are supported Package-CR overrides
# (docs/operations/backup-classes.md), and this hook cannot see chart values —
# (https://cozystack.io/docs/v1/operations/services/backup-classes/), and this hook cannot see chart values —
# nothing plumbs them into the migration Job's env. So the claim may sit in any
# namespace under any name. Keying on the COSI-assigned .status.bucketName,
# which is what the projector republishes, makes the probe independent of both.
+8 -10
View File
@@ -8,7 +8,7 @@
# spec.values from the parent kubernetes HR values with a jq helper:
#
# def pick($o; ks): reduce ks[] as $k ({}; if ($o | has($k)) ...);
# ... + pick(.; ["version","talos"]) # images is narrowed to {kubectl} separately
# ... + pick(.; ["version","talos"])
#
# A subtle regression is to write `def pick(o; ks)` with a filter-parameter:
# inside `reduce ks[] as $k ({}; ...)` the `.` context is the accumulator, so
@@ -21,7 +21,7 @@
#
# The @test below drives the REAL migration script against a fake kubectl and
# asserts the captured child HelmRelease carries the tenant's non-default
# talos/version/images (plus the group fields and storageClass fallback). It
# talos/version (plus the group fields and storageClass fallback). It
# fails against the buggy `pick(o; ...)` and passes against `pick($o; ...)`.
#
# CI runs this file under Bats through `make bats-unit-tests`. It also remains
@@ -38,8 +38,9 @@ FAKEBIN="$PWD/hack/testdata/migration-56"
MIG="$PWD/packages/core/platform/images/migrations/migrations/56"
# prep resets PATH/env to a clean scenario: one tenant Kubernetes HR (test3)
# with a single pool md0 and NON-default talos/version/images so the assertions
# distinguish "copied the tenant value" from "fell back to the chart default".
# with a single pool md0 and NON-default talos/version so the assertions
# distinguish "copied the tenant value" from "fell back to the chart default",
# plus a leftover images map that must not reach the pool values.
prep() {
chmod +x "$FAKEBIN/kubectl"
WORK=$(mktemp -d)
@@ -62,7 +63,7 @@ prep() {
JSON
}
@test "child HR carries the tenant's talos/version/images (not chart defaults) so the KMT hash is preserved" {
@test "child HR carries the tenant's talos/version (not chart defaults) so the KMT hash is preserved" {
prep
rc=0
bash "$MIG" >"$WORK/out" 2>&1 || rc=$?
@@ -83,11 +84,8 @@ JSON
[ "$(jq -r '.spec.values.talos.version' "$FAKE_CHILD_HR")" = "v1.13.0" ]
[ "$(jq -r '.spec.values.talos.schematicID' "$FAKE_CHILD_HR")" = "deadbeef" ]
[ "$(jq -r '.spec.values.version' "$FAKE_CHILD_HR")" = "v1.32" ]
[ "$(jq -r '.spec.values.images.kubectl' "$FAKE_CHILD_HR")" = "example.io/kubectl:v1.32" ]
# images is narrowed to the declared kubectl key only — the parent's
# waitForKubeconfig/talosCsrSigner (undeclared in the KubernetesNodes schema)
# must NOT leak into the adopted pool values.
[ "$(jq -r '.spec.values.images | keys | join(",")' "$FAKE_CHILD_HR")" = "kubectl" ]
# Neither chart reads an images map any more, so none is carried over.
[ "$(jq -r '.spec.values | has("images")' "$FAKE_CHILD_HR")" = "false" ]
# Group fields and the storageClass cluster-level fallback carry through.
[ "$(jq -r '.spec.values.minReplicas' "$FAKE_CHILD_HR")" = "1" ]
+214
View File
@@ -0,0 +1,214 @@
#!/usr/bin/env bats
# -----------------------------------------------------------------------------
# Unit tests for platform migration 60 --> 61 (carry a FoundationDB
# application's cluster.version over to the version field that replaces it, and
# record what cannot be carried over).
#
# The chart no longer reads cluster.version, so a cluster that set it to a line
# other than 7.3 would move to the v7.3 default on its first reconcile after the
# upgrade. These drive the real migration script against a fake kubectl
# (hack/testdata/migration-60-foundationdb/) and pin that:
# - each release that set cluster.version on 7.3 or 7.4 gets version for
# that line, through a JSON patch that tests the values it read and leaves
# cluster.version in place for the old chart;
# - releases that never set cluster.version, already have version, or are not
# FoundationDB are not touched;
# - an unlabelled release is still found by its chart;
# - a 7.1 cluster, whose client the operator no longer carries, is recorded
# rather than carried, and so is a version set to a line no longer offered;
# - a value outside those lines, a value the apiserver stored as a number, a
# spec.values that is not an object at all, a carried release whose patch
# moves, and an OpenSearch, Kubernetes or KubernetesNodes release still
# setting an image override are recorded and do not block the stamp;
# - a failed scan or patch aborts before the stamp;
# - a run retried after an aborted one, which finds version already written,
# still records the patch the release moves to;
# - an OpenSearch release storing version: null is recorded;
# - the tag map the migration carries matches the chart's.
#
# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its
# own line; there is no bats `run`/`$status`/`setup`/`teardown`. Assertions are
# direct shell tests that exit non-zero on failure.
#
# Run with: hack/cozytest.sh hack/migration-60-foundationdb-version.bats
# -----------------------------------------------------------------------------
load test_helper
FAKEBIN="$PWD/hack/testdata/migration-60-foundationdb"
MIG="$PWD/packages/core/platform/images/migrations/migrations/60"
prep() {
WORK=$(mktemp -d)
export FAKE_CMDLOG="$WORK/cmdlog"
export FAKE_HR_LIST="$WORK/hrlist.json"
: > "$FAKE_CMDLOG"
export PATH="$FAKEBIN:$PATH"
export NAMESPACE=cozy-system
unset FAKE_LIST_FAIL FAKE_PATCH_FAIL MIGRATION_DRY_RUN
cat > "$FAKE_HR_LIST" <<'JSON'
{"items":[
{"metadata":{"namespace":"tenant-a","name":"foundationdb-old71","labels":{"apps.cozystack.io/application.kind":"FoundationDB"}},
"spec":{"chartRef":{"kind":"ExternalArtifact","name":"cozystack-foundationdb-application-default-foundationdb"},
"values":{"cluster":{"version":"7.1.67","redundancyMode":"double"}}}},
{"metadata":{"namespace":"tenant-a","name":"foundationdb-new74","labels":{"apps.cozystack.io/application.kind":"FoundationDB"}},
"spec":{"values":{"cluster":{"version":"7.4.3"}}}},
{"metadata":{"namespace":"tenant-b","name":"foundationdb-pinned73","labels":{"apps.cozystack.io/application.kind":"FoundationDB"}},
"spec":{"values":{"cluster":{"version":"7.3.63"}}}},
{"metadata":{"namespace":"tenant-b","name":"foundationdb-defaults","labels":{"apps.cozystack.io/application.kind":"FoundationDB"}},
"spec":{"values":{"cluster":{"redundancyMode":"double"}}}},
{"metadata":{"namespace":"tenant-b","name":"foundationdb-migrated","labels":{"apps.cozystack.io/application.kind":"FoundationDB"}},
"spec":{"values":{"version":"v7.3","cluster":{"version":"7.3.63"}}}},
{"metadata":{"namespace":"tenant-b","name":"foundationdb-stale-version","labels":{"apps.cozystack.io/application.kind":"FoundationDB"}},
"spec":{"values":{"version":"v7.1","cluster":{"version":"7.1.67"}}}},
{"metadata":{"namespace":"tenant-c","name":"foundationdb-unlabelled"},
"spec":{"chartRef":{"kind":"ExternalArtifact","name":"cozystack-foundationdb-application-default-foundationdb"},
"values":{"cluster":{"version":"7.3.5"}}}},
{"metadata":{"namespace":"tenant-c","name":"postgres-db","labels":{"apps.cozystack.io/application.kind":"Postgres"}},
"spec":{"values":{"cluster":{"version":"7.1.0"}}}},
{"metadata":{"namespace":"tenant-d","name":"foundationdb-ancient","labels":{"apps.cozystack.io/application.kind":"FoundationDB"}},
"spec":{"values":{"cluster":{"version":"6.3.25"}}}},
{"metadata":{"namespace":"tenant-d","name":"foundationdb-numeric","labels":{"apps.cozystack.io/application.kind":"FoundationDB"}},
"spec":{"values":{"cluster":{"version":7.3}}}},
{"metadata":{"namespace":"tenant-d","name":"foundationdb-scalar-values","labels":{"apps.cozystack.io/application.kind":"FoundationDB"}},
"spec":{"values":"broken"}},
{"metadata":{"namespace":"tenant-e","name":"opensearch-logs","labels":{"apps.cozystack.io/application.kind":"OpenSearch"}},
"spec":{"values":{"images":{"opensearch":"registry.example.test/opensearch:2.19.6"}}}},
{"metadata":{"namespace":"tenant-e","name":"opensearch-plain","labels":{"apps.cozystack.io/application.kind":"OpenSearch"}},
"spec":{"values":{"images":{"opensearch":""},"replicas":3}}},
{"metadata":{"namespace":"tenant-f","name":"kubernetes-mirrored","labels":{"apps.cozystack.io/application.kind":"Kubernetes"}},
"spec":{"values":{"images":{"kubectl":"","talosCsrSigner":"mirror.example.test/talos-csr-signer:1","waitForKubeconfig":"mirror.example.test/busybox:1"}}}},
{"metadata":{"namespace":"tenant-f","name":"kubernetes-plain","labels":{"apps.cozystack.io/application.kind":"Kubernetes"}},
"spec":{"values":{"images":{"kubectl":"","talosCsrSigner":"","waitForKubeconfig":""}}}},
{"metadata":{"namespace":"tenant-e","name":"opensearch-null","labels":{"apps.cozystack.io/application.kind":"OpenSearch"}},
"spec":{"values":{"version":null}}},
{"metadata":{"namespace":"tenant-f","name":"kubernetes-nodes-mirrored-md0"},
"spec":{"chartRef":{"kind":"ExternalArtifact","name":"cozystack-kubernetes-nodes-application-kubevirt-kubernetes-nodes"},
"values":{"images":{"kubectl":"mirror.example.test/kubectl:1"}}}}
]}
JSON
}
@test "carries each supported line over and leaves everything else alone" {
prep
rc=0
bash "$MIG" >"$WORK/out" 2>&1 || rc=$?
cat "$WORK/out"
cat "$FAKE_CMDLOG"
[ "$rc" -eq 0 ]
grep -qxF 'PATCH tenant-a foundationdb-new74 json [{"op":"test","path":"/spec/values","value":{"cluster":{"version":"7.4.3"}}},{"op":"add","path":"/spec/values/version","value":"v7.4"}]' "$FAKE_CMDLOG"
grep -qxF 'PATCH tenant-b foundationdb-pinned73 json [{"op":"test","path":"/spec/values","value":{"cluster":{"version":"7.3.63"}}},{"op":"add","path":"/spec/values/version","value":"v7.3"}]' "$FAKE_CMDLOG"
grep -qxF 'PATCH tenant-c foundationdb-unlabelled json [{"op":"test","path":"/spec/values","value":{"cluster":{"version":"7.3.5"}}},{"op":"add","path":"/spec/values/version","value":"v7.3"}]' "$FAKE_CMDLOG"
[ "$(grep -c '^PATCH ' "$FAKE_CMDLOG")" -eq 3 ]
if grep -q 'PATCH [^ ]* foundationdb-old71 ' "$FAKE_CMDLOG"; then echo "carried a 7.1 cluster"; exit 1; fi
if grep -q 'PATCH [^ ]* foundationdb-stale-version ' "$FAKE_CMDLOG"; then echo "patched a release that already has version"; exit 1; fi
if grep -q 'PATCH [^ ]* foundationdb-defaults ' "$FAKE_CMDLOG"; then echo "unexpected line matching 'PATCH [^ ]* foundationdb-defaults '"; exit 1; fi
if grep -q 'PATCH [^ ]* foundationdb-migrated ' "$FAKE_CMDLOG"; then echo "unexpected line matching 'PATCH [^ ]* foundationdb-migrated '"; exit 1; fi
if grep -q 'PATCH [^ ]* postgres-db ' "$FAKE_CMDLOG"; then echo "unexpected line matching 'PATCH [^ ]* postgres-db '"; exit 1; fi
if grep -q 'PATCH [^ ]* foundationdb-ancient ' "$FAKE_CMDLOG"; then echo "unexpected line matching 'PATCH [^ ]* foundationdb-ancient '"; exit 1; fi
if grep -q 'PATCH [^ ]* opensearch-plain ' "$FAKE_CMDLOG"; then echo "unexpected line matching 'PATCH [^ ]* opensearch-plain '"; exit 1; fi
if grep -q 'PATCH [^ ]* opensearch-logs ' "$FAKE_CMDLOG"; then echo "unexpected line matching 'PATCH [^ ]* opensearch-logs '"; exit 1; fi
if grep -q 'PATCH [^ ]* foundationdb-scalar-values ' "$FAKE_CMDLOG"; then echo "unexpected line matching 'PATCH [^ ]* foundationdb-scalar-values '"; exit 1; fi
if grep -q 'PATCH [^ ]* kubernetes' "$FAKE_CMDLOG"; then echo "patched a Kubernetes release"; exit 1; fi
if grep -q 'PATCH [^ ]* foundationdb-numeric ' "$FAKE_CMDLOG"; then echo "unexpected line matching 'PATCH [^ ]* foundationdb-numeric '"; exit 1; fi
annotation=$(grep '^ANNOTATE ' "$FAKE_CMDLOG")
echo "$annotation" | grep -q 'cozystack.io/migration-60-needs-attention='
echo "$annotation" | grep -q 'tenant-d/foundationdb-ancient=cluster.version:6.3.25'
echo "$annotation" | grep -q 'tenant-d/foundationdb-numeric=cluster.version:7.3'
echo "$annotation" | grep -q 'tenant-d/foundationdb-scalar-values=values-not-an-object'
echo "$annotation" | grep -q 'tenant-e/opensearch-logs=images.opensearch:registry.example.test/opensearch:2.19.6'
echo "$annotation" | grep -q 'tenant-a/foundationdb-new74=cluster.version:7.4.3->7.4.1'
echo "$annotation" | grep -q 'tenant-c/foundationdb-unlabelled=cluster.version:7.3.5->7.3.63'
echo "$annotation" | grep -q 'tenant-a/foundationdb-old71=cluster.version:7.1.67'
echo "$annotation" | grep -q 'tenant-b/foundationdb-stale-version=version:v7.1'
echo "$annotation" | grep -q 'tenant-f/kubernetes-mirrored=images.talosCsrSigner:mirror.example.test/talos-csr-signer:1;images.waitForKubeconfig:mirror.example.test/busybox:1'
echo "$annotation" | grep -q 'tenant-f/kubernetes-nodes-mirrored-md0=images.kubectl:mirror.example.test/kubectl:1'
echo "$annotation" | grep -q 'tenant-e/opensearch-null=version:null'
for quiet in opensearch-plain kubernetes-plain foundationdb-pinned73 foundationdb-migrated; do
if echo "$annotation" | grep -q "/$quiet="; then echo "recorded $quiet, which needs no attention"; exit 1; fi
done
[ "$(tail -n1 "$FAKE_CMDLOG")" = "STAMP 61" ]
if grep -q '^UNHANDLED' "$FAKE_CMDLOG"; then echo "unexpected line matching '^UNHANDLED'"; exit 1; fi
rm -rf "$WORK"
}
@test "a failed patch aborts before the stamp" {
prep
export FAKE_PATCH_FAIL=1
rc=0
bash "$MIG" >"$WORK/out" 2>&1 || rc=$?
cat "$WORK/out"
[ "$rc" -ne 0 ]
if grep -q '^STAMP' "$FAKE_CMDLOG"; then echo "unexpected line matching '^STAMP'"; exit 1; fi
rm -rf "$WORK"
}
@test "a failed fleet scan aborts before the stamp" {
prep
export FAKE_LIST_FAIL=1
rc=0
bash "$MIG" >"$WORK/out" 2>&1 || rc=$?
cat "$WORK/out"
[ "$rc" -ne 0 ]
if grep -q '^PATCH' "$FAKE_CMDLOG"; then echo "unexpected line matching '^PATCH'"; exit 1; fi
if grep -q '^STAMP' "$FAKE_CMDLOG"; then echo "unexpected line matching '^STAMP'"; exit 1; fi
rm -rf "$WORK"
}
@test "dry run reports patch changes without writing anything" {
prep
export MIGRATION_DRY_RUN=1
rc=0
bash "$MIG" >"$WORK/out" 2>&1 || rc=$?
cat "$WORK/out"
[ "$rc" -eq 0 ]
grep -q 'would set version=v7.4 on FoundationDB tenant-a/foundationdb-new74' "$WORK/out"
grep -qxF 'WARN: FoundationDB tenant-a/foundationdb-new74 runs 7.4.3, and v7.4 renders 7.4.1 after the upgrade' "$WORK/out"
grep -qxF 'WARN: FoundationDB tenant-c/foundationdb-unlabelled runs 7.3.5, and v7.3 renders 7.3.63 after the upgrade' "$WORK/out"
summary=$(grep '^==> Summary:' "$WORK/out")
printf '%s\n' "$summary" | grep -qF 'carried=3 '
printf '%s\n' "$summary" | grep -qF 'tenant-a/foundationdb-new74=cluster.version:7.4.3->7.4.1'
printf '%s\n' "$summary" | grep -qF 'tenant-c/foundationdb-unlabelled=cluster.version:7.3.5->7.3.63'
if grep -q 'WARN: FoundationDB tenant-b/foundationdb-pinned73 ' "$WORK/out"; then echo "warned on an unchanged patch"; exit 1; fi
if grep -q '^PATCH' "$FAKE_CMDLOG"; then echo "unexpected line matching '^PATCH'"; exit 1; fi
if grep -q '^ANNOTATE' "$FAKE_CMDLOG"; then echo "unexpected line matching '^ANNOTATE'"; exit 1; fi
if grep -q '^STAMP' "$FAKE_CMDLOG"; then echo "unexpected line matching '^STAMP'"; exit 1; fi
rm -rf "$WORK"
}
@test "the tag map matches the chart's versions.yaml" {
migration=$(sed -n "s/^FOUNDATIONDB_TAGS='\\(.*\\)'$/\\1/p" "$MIG" | jq -S .)
chart=$(yq -o=json '.' packages/apps/foundationdb/files/versions.yaml | jq -S .)
echo "migration: $migration"
echo "chart: $chart"
[ -n "$migration" ]
[ "$migration" = "$chart" ]
}
@test "a retry after an aborted pass still records the patch a carried release moves to" {
prep
# The state an aborted pass leaves behind: version is already written, so the
# retry has nothing to carry, but the release still moves from 7.4.3 to 7.4.1.
cat > "$FAKE_HR_LIST" <<'JSON'
{"items":[
{"metadata":{"namespace":"tenant-a","name":"foundationdb-new74","labels":{"apps.cozystack.io/application.kind":"FoundationDB"}},
"spec":{"values":{"version":"v7.4","cluster":{"version":"7.4.3"}}}},
{"metadata":{"namespace":"tenant-a","name":"foundationdb-steady73","labels":{"apps.cozystack.io/application.kind":"FoundationDB"}},
"spec":{"values":{"version":"v7.3","cluster":{"version":"7.3.63"}}}}
]}
JSON
rc=0
bash "$MIG" >"$WORK/out" 2>&1 || rc=$?
cat "$WORK/out"
cat "$FAKE_CMDLOG"
[ "$rc" -eq 0 ]
if grep -q '^PATCH' "$FAKE_CMDLOG"; then echo "patched a release that already has version"; exit 1; fi
annotation=$(grep '^ANNOTATE ' "$FAKE_CMDLOG")
echo "$annotation" | grep -q 'tenant-a/foundationdb-new74=cluster.version:7.4.3->7.4.1'
if echo "$annotation" | grep -q 'foundationdb-steady73'; then echo "recorded a release that stays on its tag"; exit 1; fi
[ "$(tail -n1 "$FAKE_CMDLOG")" = "STAMP 61" ]
rm -rf "$WORK"
}
+312
View File
@@ -0,0 +1,312 @@
#!/usr/bin/env bats
# Behavioural tests for the monitoring-agents pre-delete hook:
#
# packages/system/monitoring-agents/templates/vm-cleanup-hook.yaml
#
# The script is extracted from the rendered chart and RUN against a fake
# kubectl that keeps the VictoriaMetrics objects in a state file, with fake
# `date` and `sleep` so the 60s grace period passes in an instant. The
# helm-unittest suite in packages/system/monitoring-agents/tests/
# uninstall_test.yaml matches the script text instead, and stayed green when the
# ownership condition was removed, when the finalizer patch was made a no-op and
# when the polling condition was inverted. Each of those turns this file red.
#
# What is pinned here:
# * only objects whose Helm annotations name this release, by name and
# namespace both, are deleted or patched: an object of another release, of
# a same-named release in another namespace, or one Helm does not own, is
# left alone;
# * with the operator gone, the hook waits before touching finalizers, then
# clears them, and nothing of this release is left behind;
# * with the operator alive, it finalizes the objects itself and the hook
# never patches a finalizer;
# * a kind whose CRD is gone, or a release with nothing left, exits 0, since a
# failing pre-delete hook would itself block the uninstall.
#
# Run from the repo root with make bats-unit-tests. Direct status checks
# retain compatibility with cozytest, which has no Bats run/setup helpers.
# Inline cleanup preserves Bats' TAP handler on failed assertions.
load test_helper
MA_CHART=packages/system/monitoring-agents
MA_RELEASE=monitoring-agents
MA_NAMESPACE=cozy-monitoring
# render_hook <job-name> <out>
# Extract the named Job's shell script from the rendered chart into <out>. The
# non-empty check must be an early `return 1`: hack/cozytest.sh rewrites every
# line matching ^}$ into `return 0` plus `}`, helpers included, so a check left
# in last-command position has its status discarded.
render_hook() {
helm template "$MA_RELEASE" "$MA_CHART" --namespace "$MA_NAMESPACE" \
| yq eval "select(.kind == \"Job\" and .metadata.name == \"$1\") | .spec.template.spec.containers[0].command[2]" - > "$2"
[ -s "$2" ] || return 1
}
# write_fakes <dir>
# Emit fake `kubectl`, `date` and `sleep` into <dir>.
#
# kubectl reads and writes $STATE, one object per line as
# kind|release-name|release-namespace|namespace|name, where the two release
# fields are the Helm annotations (empty when Helm does not own the object).
# `get` prints this kind's objects in the hook's jsonpath format, or fails for a
# kind listed in $MISSING_KINDS as kubectl does when the CRD is gone. `delete`
# removes the object only when $OPERATOR_LIVE is 1, standing for an operator
# that finalizes it; otherwise the object stays, Terminating. `patch` removes it
# only when the payload actually clears the finalizers. Every call goes to $KLOG.
#
# date advances 1s per call from $CLOCK, so the 60s wait ends after sixty
# of polls, and records each time it returns in $KLOG; sleep records itself,
# and with $SLEEP_ADVANCES=1 also moves $CLOCK by its argument. The log thus
# shows how much fake time passed before each patch.
# No line below is a bare column-0 `}`, so cozytest.sh's parser leaves it alone.
write_fakes() {
cat > "$1/kubectl" <<'KEOF'
#!/bin/sh
echo "kubectl $*" >> "$KLOG"
drop() {
awk -F'|' -v k="$1" -v ns="$2" -v n="$3" '!($1 == k && $4 == ns && $5 == n)' "$STATE" > "$STATE.new"
mv "$STATE.new" "$STATE"; }
case "$1" in
get)
kind=${2%%.*}
case " ${MISSING_KINDS:-} " in
*" $kind "*)
echo "error: the server doesn't have a resource type \"$kind\"" >&2
exit 1 ;;
esac
awk -F'|' -v k="$kind" '$1 == k { print $2 "|" $3 "|" $4 "|" $5 }' "$STATE"
exit 0 ;;
-n)
ns=$2 verb=$3 kind=${4%%.*} name=$5
case "$verb" in
delete)
[ "${OPERATOR_LIVE:-0}" = 1 ] && drop "$kind" "$ns" "$name"
echo "$kind \"$name\" deleted"
exit 0 ;;
patch)
case "$*" in
*'"finalizers":null'*) drop "$kind" "$ns" "$name" ;;
esac
echo "$kind \"$name\" patched"
exit 0 ;;
esac ;;
esac
echo "fake kubectl: unexpected call: $*" >&2
exit 64
KEOF
cat > "$1/date" <<'KEOF'
#!/bin/sh
c=$(cat "$CLOCK" 2>/dev/null || echo 0)
echo $((c + 1)) > "$CLOCK"
echo "date $((1000000 + c))" >> "$KLOG"
echo $((1000000 + c))
KEOF
cat > "$1/sleep" <<'KEOF'
#!/bin/sh
echo "sleep $*" >> "$KLOG"
if [ "${SLEEP_ADVANCES:-0}" = 1 ]; then
c=$(cat "$CLOCK" 2>/dev/null || echo 0)
echo $((c + $1)) > "$CLOCK"
fi
KEOF
chmod +x "$1/kubectl" "$1/date" "$1/sleep"
}
# write_state <file>
# Three objects of this release in two namespaces; a VMRule of another release;
# a VMRule whose release name matches but whose release namespace does not, so
# only the pair of annotations tells it apart; and a VMServiceScrape that
# carries no Helm annotations.
write_state() {
cat > "$1" <<EOF
vmagents|$MA_RELEASE|$MA_NAMESPACE|$MA_NAMESPACE|vmagent
vmrules|$MA_RELEASE|$MA_NAMESPACE|$MA_NAMESPACE|alerts-node
vmservicescrapes|$MA_RELEASE|$MA_NAMESPACE|cozy-victoria-metrics-operator|operator
vmrules|other|$MA_NAMESPACE|$MA_NAMESPACE|foreign-rule
vmrules|$MA_RELEASE|other-namespace|$MA_NAMESPACE|foreign-ns-rule
vmservicescrapes|||$MA_NAMESPACE|unmanaged-scrape
EOF
}
# run_hook <dir>
# Run the rendered hook with the fakes first on PATH; its status lands in $rc.
run_hook() {
export KLOG="$1/calls" STATE="$1/state" CLOCK="$1/clock"
: > "$KLOG"
rc=0
PATH="$1:$PATH" sh "$1/hook.sh" > "$1/out" 2> "$1/err" || rc=$?
}
@test "a render that produces no script fails instead of passing as a no-op" {
# Guards the guard: every test below runs whatever render_hook produced, so an
# empty render would satisfy the exit-0 assertions without executing a line of
# the hook.
tmp=$(mktemp -d)
if render_hook no-such-job "$tmp/hook.sh"; then
echo "FAIL: render_hook reported success for a Job name the chart never emits"
false
fi
rm -rf "$tmp"
}
@test "vm cleanup hook clears the finalizers of this release's objects only, after the grace period" {
tmp=$(mktemp -d)
write_fakes "$tmp"
write_state "$tmp/state"
render_hook vm-objects-cleanup "$tmp/hook.sh"
export OPERATOR_LIVE=0 MISSING_KINDS=""
run_hook "$tmp"
if [ "$rc" -ne 0 ]; then
echo "FAIL: hook exited $rc"
cat "$tmp/out" "$tmp/err"
false
fi
for obj in "vmagents.operator.victoriametrics.com vmagent" \
"vmrules.operator.victoriametrics.com alerts-node" \
"vmservicescrapes.operator.victoriametrics.com operator"; do
if ! grep -q -- " delete $obj --wait=false" "$tmp/calls"; then
echo "FAIL: $obj was not deleted"
cat "$tmp/calls"
false
fi
done
# The objects of this release must be gone, which the fake only does for a
# patch that really clears the finalizers.
if grep -q "^[a-z]*|$MA_RELEASE|$MA_NAMESPACE|" "$tmp/state"; then
echo "FAIL: objects of this release are still there, Terminating"
cat "$tmp/state" "$tmp/calls"
false
fi
# Another release's object, one of a same-named release in another
# namespace, and an object Helm does not own are never touched.
if grep -E -q -- ' (delete|patch) [a-z.]+ (foreign-rule|foreign-ns-rule|unmanaged-scrape)( |$)' "$tmp/calls"; then
echo "FAIL: the hook touched an object this release does not own"
cat "$tmp/calls"
false
fi
if [ "$(grep -c '|' "$tmp/state")" -ne 3 ]; then
echo "FAIL: the three objects this release does not own should remain"
cat "$tmp/state"
false
fi
# The operator gets its chance first: the hook polls and sleeps before the
# first patch, and does not stop waiting before the 60s are up.
first_patch=$(grep -n ' patch ' "$tmp/calls" | head -1 | cut -d: -f1)
if [ -z "$first_patch" ]; then
echo "FAIL: no finalizer was cleared"
cat "$tmp/calls"
false
fi
# The first date call sets the deadline; the last one before the first patch
# is the time the hook gave up waiting.
start=$(grep '^date ' "$tmp/calls" | head -1 | cut -d' ' -f2)
waited_until=$(head -n "$first_patch" "$tmp/calls" | grep '^date ' | tail -1 | cut -d' ' -f2)
if [ -z "$start" ] || [ $((waited_until - start)) -lt 60 ]; then
echo "FAIL: finalizers cleared after $((waited_until - start))s; the 60s grace period was cut short"
cat "$tmp/calls"
false
fi
rm -rf "$tmp"
}
@test "vm cleanup hook clears the finalizers before its Job's deadline" {
# The grace-period test above keeps sleep free so a shortened wait shows; here
# sleep costs its argument, so a poll interval long enough to overshoot the
# Job's activeDeadlineSeconds fails instead of passing.
tmp=$(mktemp -d)
write_fakes "$tmp"
write_state "$tmp/state"
render_hook vm-objects-cleanup "$tmp/hook.sh"
job_deadline=$(helm template "$MA_RELEASE" "$MA_CHART" --namespace "$MA_NAMESPACE" \
| yq eval 'select(.kind == "Job" and .metadata.name == "vm-objects-cleanup") | .spec.activeDeadlineSeconds' -)
export OPERATOR_LIVE=0 MISSING_KINDS="" SLEEP_ADVANCES=1
run_hook "$tmp"
unset SLEEP_ADVANCES
if [ "$rc" -ne 0 ]; then
echo "FAIL: hook exited $rc"
cat "$tmp/out" "$tmp/err"
false
fi
first_patch=$(grep -n ' patch ' "$tmp/calls" | head -1 | cut -d: -f1)
start=$(grep '^date ' "$tmp/calls" | head -1 | cut -d' ' -f2)
waited_until=$(head -n "${first_patch:-0}" "$tmp/calls" | grep '^date ' | tail -1 | cut -d' ' -f2)
if [ -z "$first_patch" ] || [ -z "$job_deadline" ] || [ $((waited_until - start)) -ge "$job_deadline" ]; then
echo "FAIL: first finalizer patch at $((waited_until - start))s, Job deadline ${job_deadline}s"
cat "$tmp/calls"
false
fi
rm -rf "$tmp"
}
@test "vm cleanup hook leaves the finalizers to a live operator" {
tmp=$(mktemp -d)
write_fakes "$tmp"
write_state "$tmp/state"
render_hook vm-objects-cleanup "$tmp/hook.sh"
export OPERATOR_LIVE=1 MISSING_KINDS=""
run_hook "$tmp"
if [ "$rc" -ne 0 ]; then
echo "FAIL: hook exited $rc"
cat "$tmp/out" "$tmp/err"
false
fi
if ! grep -q ' delete vmagents.operator.victoriametrics.com vmagent ' "$tmp/calls"; then
echo "FAIL: the VMAgent was not deleted"
cat "$tmp/calls"
false
fi
if grep -q ' patch ' "$tmp/calls"; then
echo "FAIL: the hook cleared finalizers the live operator was handling"
cat "$tmp/calls"
false
fi
rm -rf "$tmp"
}
@test "vm cleanup hook exits 0 and cleans the other kinds when a CRD is gone" {
tmp=$(mktemp -d)
write_fakes "$tmp"
write_state "$tmp/state"
render_hook vm-objects-cleanup "$tmp/hook.sh"
export OPERATOR_LIVE=0 MISSING_KINDS="vmrules vmpodscrapes vmnodescrapes"
run_hook "$tmp"
if [ "$rc" -ne 0 ]; then
echo "FAIL: hook exited $rc with a CRD gone, which would block the uninstall"
cat "$tmp/out" "$tmp/err"
false
fi
if grep -q "^vmagents|$MA_RELEASE|" "$tmp/state"; then
echo "FAIL: the VMAgent was left Terminating because another kind's CRD was gone"
cat "$tmp/state" "$tmp/calls"
false
fi
rm -rf "$tmp"
}
@test "vm cleanup hook exits 0 without deleting anything when the release owns nothing" {
tmp=$(mktemp -d)
write_fakes "$tmp"
printf 'vmrules|other|%s|%s|foreign-rule\nvmrules|%s|other-namespace|%s|foreign-ns-rule\n' \
"$MA_NAMESPACE" "$MA_NAMESPACE" "$MA_RELEASE" "$MA_NAMESPACE" > "$tmp/state"
render_hook vm-objects-cleanup "$tmp/hook.sh"
export OPERATOR_LIVE=0 MISSING_KINDS=""
run_hook "$tmp"
if [ "$rc" -ne 0 ]; then
echo "FAIL: hook exited $rc with nothing to clean"
cat "$tmp/out" "$tmp/err"
false
fi
if grep -E -q ' (delete|patch) ' "$tmp/calls"; then
echo "FAIL: the hook deleted or patched with nothing of this release left"
cat "$tmp/calls"
false
fi
rm -rf "$tmp"
}
+55
View File
@@ -0,0 +1,55 @@
#!/usr/bin/env bats
load test_helper
REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)"
prepare_seed_capture() {
strict_setup
seed_tmp=$(mktemp -d)
seed_old_path=$PATH
mkdir -p "$seed_tmp/bin"
export SEED_CAPTURE="$seed_tmp/seed"
export PATH="$seed_tmp/bin:$PATH"
cat > "$seed_tmp/bin/curl" <<'EOF'
#!/usr/bin/env bash
set -eu
printf '%s\n' "$@" > "$SEED_CAPTURE.args"
cat > "$SEED_CAPTURE.body"
if ! grep --fixed-strings --line-regexp --quiet 'Content-Type: application/json' "$SEED_CAPTURE.args"; then
echo 'seed request is missing its JSON content type' >&2
exit 22
fi
EOF
chmod +x "$seed_tmp/bin/curl"
}
assert_seed_request() {
local account=$1
prepare_seed_capture
# shellcheck source=e2e-chainsaw/observability-proxy/lib.sh
. "$REPO_ROOT/hack/e2e-chainsaw/observability-proxy/lib.sh"
if [ "$account" = 0 ]; then
ingest tenant-test-logsa e2e-marker-a
else
ingest tenant-test-logsa e2e-marker-a "$account"
fi
grep --fixed-strings --line-regexp --quiet "AccountID: $account" "$SEED_CAPTURE.args"
grep --fixed-strings --line-regexp --quiet 'ProjectID: 0' "$SEED_CAPTURE.args"
grep --fixed-strings --line-regexp --quiet 'http://localhost:19471/insert/jsonline?_stream_fields=kubernetes_namespace_name&_msg_field=log' "$SEED_CAPTURE.args"
grep --fixed-strings --line-regexp --quiet -- '--data-binary' "$SEED_CAPTURE.args"
grep --fixed-strings --line-regexp --quiet '@-' "$SEED_CAPTURE.args"
printf '%s\n' '{"kubernetes_namespace_name":"tenant-test-logsa","log":"e2e-marker-a"}' > "$SEED_CAPTURE.expected"
cmp "$SEED_CAPTURE.expected" "$SEED_CAPTURE.body"
rm "$body" "$SEED_CAPTURE.args" "$SEED_CAPTURE.body" "$SEED_CAPTURE.expected" "$seed_tmp/bin/curl"
rmdir "$seed_tmp/bin" "$seed_tmp"
export PATH="$seed_old_path"
}
@test "observability seeds send JSON without consuming the body as form data" {
assert_seed_request 0
}
@test "observability seeds preserve an explicit VictoriaLogs account" {
assert_seed_request 7
}
+1 -1
View File
@@ -42,4 +42,4 @@ Cosign verification is **mandatory** on the gate for both lanes: the trust ancho
## Trust model
Signature verification happens here, at publication time, not at tap time. `cozypkg tap` and the dashboard connect flow validate an artifact's structure but do not verify its cosign signature: a cluster that taps a listed repository trusts that this index already verified it. The verification points are this CI gate (which pins each release to the entry's recorded identity) and, optionally, Flux `OCIRepository` verification at pull time. Tapping a third-party repository runs its charts in the management cluster, so operators should tap only sources they trust or that appear in a curated index like this one.
Signature verification happens here, at publication time, not at tap time. `cozypkg tap` and the dashboard connect flow validate an artifact's structure; they do not verify a cosign signature unless asked (with `--certificate-identity` / `--certificate-oidc-issuer`, which set a Flux pull-time verify policy). A cluster that taps a listed repository otherwise trusts that this index already verified it. The verification point is this CI gate, which pins each release to the entry's recorded identity. Independently of signing, a tapped repository is pinned to a digest and does not follow a moved tag: a tag-only tap is pinned to the digest it first resolves to. Installing an app from a tapped repository runs its charts in the management cluster, so operators should tap only sources they trust or that appear in a curated index like this one.
+261
View File
@@ -0,0 +1,261 @@
#!/usr/bin/env bats
# Behavioural tests for the "release system databases" pass of the Postgres
# init script:
#
# packages/apps/postgres/templates/init-script.yaml
#
# The pass is cut out of the rendered script and RUN against a fake psql that
# records which database each session connects to. The helm-unittest suite in
# packages/apps/postgres/tests/system_databases_test.yaml matches the SQL text,
# and stayed green when the template0 guard was inverted: the rendered pass then
# connected to template0, which accepts no connection, and skipped postgres and
# template1, which are the two it exists to hand back.
#
# What is pinned here:
# * no session ever connects to template0;
# * postgres and template1 are each connected to and released;
# * every taken-over system database, template0 included, has its comment
# cleared;
# * releasing template1 visits every other connectable database, for the
# trigger it passed on to them, and a failed lookup of those databases
# stops the pass before the comment that selects template1 is cleared;
# * the session that releases a system database hands its public schema
# and the database itself back only from a marked <db>_admin or
# <db>_readonly owner (or cozystack:orphaned), before the reassignment,
# an inherited public only from a marked template1_admin, and touches
# only the roles that carry the
# "cozystack:managed" mark, not a role that merely has one of the names.
#
# Run from the repo root with make bats-unit-tests. Direct status checks
# retain compatibility with cozytest, which has no Bats run/setup helpers.
# Inline cleanup preserves Bats' TAP handler on failed assertions.
load test_helper
PG_CHART=packages/apps/postgres
# render_pass <out>
# Extract the release pass, from its banner to the next one, out of the rendered
# init script. The non-empty check must be an early `return 1`: hack/cozytest.sh
# rewrites every line matching ^}$ into `return 0` plus `}`, helpers included,
# so a check left in last-command position has its status discarded.
render_pass() {
helm template pg-test "$PG_CHART" --namespace tenant-test --set-json '_cluster={"scheduling":{}}' 2>/dev/null \
| yq eval 'select(.metadata.name == "pg-test-init-script") | .stringData["init.sh"]' - \
| awk '/^echo "== release system databases"/ { f = 1 } /^echo "== delete databases"/ { f = 0 } f' > "$1"
grep -q 'SYSTEM_DBS=' "$1" || return 1
}
# write_fake_psql <dir>
# Emit a fake psql. A `-c` query answers from $SYSTEM_DBS for the system
# database lookup and from $OTHER_DBS for the connectable-database lookup,
# which exits with $CLONE_LOOKUP_FAILS instead when that is set. A
# session fed on stdin logs "connect <dbname>" ("connect -" without --dbname)
# followed by its first line and every -v db= value, to $PLOG, and its whole
# input under a "-- session <dbname> db=<value>" header to $PLOG.sql.
# No line below is a bare column-0 `}`, so cozytest.sh's parser leaves it alone.
write_fake_psql() {
cat > "$1/psql" <<'PEOF'
#!/bin/sh
dbname=- query= vdb=
while [ $# -gt 0 ]; do
case "$1" in
--dbname) dbname=$2; shift ;;
-c) query=$2; shift ;;
-v) case "$2" in db=*) vdb=${2#db=} ;; esac; shift ;;
esac
shift
done
if [ -n "$query" ]; then
case "$query" in
*"d.datname IN ('postgres', 'template0', 'template1')"*) printf '%s\n' $SYSTEM_DBS ;;
*datallowconn*)
if [ -n "${CLONE_LOOKUP_FAILS:-}" ]; then
echo "fake psql: connectable-database lookup failed" >&2
exit "$CLONE_LOOKUP_FAILS"
fi
printf '%s\n' $OTHER_DBS ;;
*) echo "fake psql: unexpected query: $query" >&2; exit 64 ;;
esac
exit 0
fi
sql=$(cat)
echo "connect $dbname db=$vdb $(printf '%s\n' "$sql" | head -n 1)" >> "$PLOG"
printf -- '-- session %s db=%s\n%s\n' "$dbname" "$vdb" "$sql" >> "$PLOG.sql"
PEOF
chmod +x "$1/psql"
}
# run_pass <dir>
# Run the extracted pass with the fake psql first on PATH; status in $rc.
run_pass() {
export PLOG="$1/calls"
: > "$PLOG"
rc=0
PATH="$1:$PATH" sh -e "$1/pass.sh" > "$1/out" 2> "$1/err" || rc=$?
}
@test "a render that produces no release pass fails instead of passing as a no-op" {
tmp=$(mktemp -d)
if PG_CHART=packages/apps/no-such-chart render_pass "$tmp/pass.sh"; then
echo "FAIL: render_pass reported success for a chart that does not exist"
false
fi
rm -rf "$tmp"
}
@test "release pass connects to postgres and template1, never to template0" {
tmp=$(mktemp -d)
write_fake_psql "$tmp"
render_pass "$tmp/pass.sh"
export SYSTEM_DBS="postgres template0 template1" OTHER_DBS="postgres app"
run_pass "$tmp"
if [ "$rc" -ne 0 ]; then
echo "FAIL: the pass exited $rc"
cat "$tmp/out" "$tmp/err"
false
fi
if grep -q '^connect template0 ' "$tmp/calls"; then
echo "FAIL: the pass connected to template0, which accepts no connection"
cat "$tmp/calls"
false
fi
for db in postgres template1; do
if ! grep -q "^connect $db db=$db DROP EVENT TRIGGER IF EXISTS trigger_auto_grant;" "$tmp/calls"; then
echo "FAIL: $db was not connected to and released"
cat "$tmp/calls"
false
fi
done
for db in postgres template0 template1; do
if ! grep -q "^connect - db=$db COMMENT ON DATABASE :\"db\" IS NULL;" "$tmp/calls"; then
echo "FAIL: the comment on $db was not cleared"
cat "$tmp/calls"
false
fi
done
rm -rf "$tmp"
}
@test "release session hands public back and touches only marked roles" {
tmp=$(mktemp -d)
write_fake_psql "$tmp"
render_pass "$tmp/pass.sh"
export SYSTEM_DBS="postgres" OTHER_DBS="postgres"
run_pass "$tmp"
if [ "$rc" -ne 0 ]; then
echo "FAIL: the pass exited $rc"
cat "$tmp/out" "$tmp/err"
false
fi
awk '/^-- session / { f = ($0 == "-- session postgres db=postgres") } f' "$tmp/calls.sql" > "$tmp/release.sql"
# Each statement must end on its whole predicate: a clause appended after the
# mark (an OR, say) would select an unmarked role again.
reassign_where="WHERE EXISTS (SELECT FROM pg_auth_members m JOIN pg_roles g ON g.oid = m.roleid JOIN pg_roles u ON u.oid = m.member WHERE g.rolname = 'cozystack:managed' AND u.rolname = r)\\gexec"
owner_marked="(o.rolname = 'cozystack:orphaned' OR o.rolname IN (:'db' || '_admin', :'db' || '_readonly') AND EXISTS (SELECT FROM pg_auth_members m JOIN pg_roles g ON g.oid = m.roleid WHERE g.rolname = 'cozystack:managed' AND m.member = o.oid))\\gexec"
public_where="WHERE n.nspname = 'public' AND $owner_marked"
database_where="WHERE d.datname = :'db' AND $owner_marked"
public_line=$(grep -n 'ALTER SCHEMA public OWNER TO' "$tmp/release.sql" | head -1)
database_line=$(grep -n 'ALTER DATABASE' "$tmp/release.sql" | head -1)
reassign_line=$(grep -n 'REASSIGN OWNED BY' "$tmp/release.sql" | head -1)
if [ "${public_line##*"$public_where"}" != "" ] || [ -z "$public_line" ]; then
echo "FAIL: the release session does not hand public back only from a marked <db>_admin or <db>_readonly owner"
cat "$tmp/release.sql"
false
fi
if [ "${database_line##*"$database_where"}" != "" ] || [ -z "$database_line" ]; then
echo "FAIL: the release session does not take the database back only from a marked owner"
cat "$tmp/release.sql"
false
fi
if [ "${reassign_line##*"$reassign_where"}" != "" ] || [ -z "$reassign_line" ]; then
echo "FAIL: the release session does not reassign only roles carrying the cozystack:managed mark"
cat "$tmp/release.sql"
false
fi
# public and the database must be handed back before REASSIGN OWNED would
# move them to cozystack:orphaned.
if [ "${public_line%%:*}" -ge "${reassign_line%%:*}" ] || [ "${database_line%%:*}" -ge "${reassign_line%%:*}" ]; then
echo "FAIL: public or the database is handed back after the reassignment has already moved it"
cat "$tmp/release.sql"
false
fi
rm -rf "$tmp"
}
@test "release pass of template1 visits every other connectable database" {
tmp=$(mktemp -d)
write_fake_psql "$tmp"
render_pass "$tmp/pass.sh"
export SYSTEM_DBS="template1" OTHER_DBS="postgres app"
run_pass "$tmp"
if [ "$rc" -ne 0 ]; then
echo "FAIL: the pass exited $rc"
cat "$tmp/out" "$tmp/err"
false
fi
for db in postgres app; do
if ! grep -q "^connect $db db= SELECT 'DROP EVENT TRIGGER IF EXISTS trigger_auto_grant'" "$tmp/calls"; then
echo "FAIL: $db was not checked for the trigger inherited from template1"
cat "$tmp/calls"
false
fi
done
# An inherited public changes hands only from a template1_admin the chart
# marked; an owner that merely has the name keeps it.
inherited_where="WHERE n.nspname = 'public' AND o.rolname = 'template1_admin' AND EXISTS (SELECT FROM pg_auth_members m JOIN pg_roles g ON g.oid = m.roleid WHERE g.rolname = 'cozystack:managed' AND m.member = o.oid)\\gexec"
inherited_line=$(awk '/^-- session / { f = ($0 == "-- session app db=") } f' "$tmp/calls.sql" | grep 'ALTER SCHEMA public OWNER TO' | head -1)
if [ -z "$inherited_line" ] || [ "${inherited_line##*"$inherited_where"}" != "" ]; then
echo "FAIL: an inherited public is handed back without checking that a marked template1_admin owns it"
cat "$tmp/calls.sql"
false
fi
rm -rf "$tmp"
}
@test "release pass of template1 stops before clearing its comment when the clone lookup fails" {
# Clearing the comment is what stops later runs from selecting template1, so
# a lookup failure that let the pass go on would skip the clone repair for
# good.
tmp=$(mktemp -d)
write_fake_psql "$tmp"
render_pass "$tmp/pass.sh"
export SYSTEM_DBS="template1" OTHER_DBS="postgres app" CLONE_LOOKUP_FAILS=73
run_pass "$tmp"
unset CLONE_LOOKUP_FAILS
if [ "$rc" -ne 73 ]; then
echo "FAIL: the pass exited $rc instead of stopping on the failed lookup (73)"
cat "$tmp/out" "$tmp/err" "$tmp/calls"
false
fi
if grep -q 'COMMENT ON DATABASE' "$tmp/calls"; then
echo "FAIL: the comment on template1 was cleared although its clones were never visited"
cat "$tmp/calls"
false
fi
rm -rf "$tmp"
}
@test "release pass of postgres alone leaves the other databases alone" {
tmp=$(mktemp -d)
write_fake_psql "$tmp"
render_pass "$tmp/pass.sh"
export SYSTEM_DBS="postgres" OTHER_DBS="postgres app"
run_pass "$tmp"
if [ "$rc" -ne 0 ]; then
echo "FAIL: the pass exited $rc"
cat "$tmp/out" "$tmp/err"
false
fi
if grep -q '^connect app ' "$tmp/calls"; then
echo "FAIL: the pass visited app although template1 was not taken over"
cat "$tmp/calls"
false
fi
rm -rf "$tmp"
}
+5 -2
View File
@@ -23,9 +23,12 @@ need_node() {
return 1
}
# The `script: |` block, de-indented to plain JavaScript.
# The `script: |` block, de-indented to plain JavaScript. The block ends at the
# first non-blank line indented less than its body, where the next job starts.
extract_script() {
awk 'f { print substr($0, 13) } /^ +script: \|$/ { f = 1 }' "$WF" > "$1"
awk 'f && length($0) && substr($0, 1, 12) != " " { exit }
f { print substr($0, 13) }
/^ +script: \|$/ { f = 1 }' "$WF" > "$1"
[ -s "$1" ]
}
+44
View File
@@ -0,0 +1,44 @@
#!/usr/bin/env bats
# Upstream releases the Proxmox CSI controller and node plugin together, and
# its chart renders both from one appVersion. Here they run in two places and
# are pinned in two packages: the controller in the management cluster
# (packages/apps/kubernetes, an images/.tag file) and the node plugin in the
# tenant (packages/system/proxmox-csi, a values override on the vendored
# chart). Both are pinned by digest because tags in that registry have been
# re-pushed, and the pin is what takes the version out of the chart's hands:
# `make update` with a new CHART_VERSION brings new templates and a new
# appVersion while both pins keep naming the old build, and nothing else
# notices.
#
# This asserts the chart's appVersion and both pins name the same release.
load test_helper
REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)"
@test "the Proxmox CSI controller and node plugin pin the vendored chart's release" {
chart="$REPO_ROOT/packages/system/proxmox-csi/charts/proxmox-csi-plugin/Chart.yaml"
values="$REPO_ROOT/packages/system/proxmox-csi/values.yaml"
tagfile="$REPO_ROOT/packages/apps/kubernetes/images/proxmox-csi-controller.tag"
app_version=$(yq -r '.appVersion' "$chart")
node_tag=$(yq -r '."proxmox-csi-plugin".node.plugin.image.tag' "$values")
controller_ref=$(tr -d '[:space:]' < "$tagfile")
[ -n "$app_version" ] && [ "$app_version" != null ] || {
echo "no appVersion in $chart" >&2
return 1
}
[ "${node_tag%%@*}" = "$app_version" ] || {
echo "node plugin tag '$node_tag' is not the chart appVersion $app_version" >&2
echo "set proxmox-csi-plugin.node.plugin.image.tag in packages/system/proxmox-csi/values.yaml" >&2
return 1
}
[ "${controller_ref%%@*}" = "ghcr.io/sergelogvinov/proxmox-csi-controller:$app_version" ] || {
echo "controller ref '$controller_ref' is not proxmox-csi-controller:$app_version" >&2
echo "update packages/apps/kubernetes/images/proxmox-csi-controller.tag" >&2
return 1
}
}
+75
View File
@@ -16,6 +16,21 @@ RUNNER="$REPO_ROOT/hack/e2e-chainsaw/_lib/run-proxmox.sh"
if grep -qE '\$\{[A-Za-z_]+:[-=?+]' "$FIXTURE"; then echo "FAIL: the fixture must not carry a shell-style default envsubst leaves in place"; false; fi
}
# kubernetes-nodes names its release kubernetes-nodes-<CR name> and refuses one
# that does not start with kubernetes-nodes-<spec.cluster>- (_helpers.tpl), so a
# pool named anything but <cluster>-<pool> fails at helm install and the suite
# only learns it from a Machine that never appears.
@test "the fixture's KubernetesNodes is named <cluster>-<pool>, as kubernetes-nodes requires" {
doc="$(sed -n '/^kind: KubernetesNodes$/,/^---$/p' "$FIXTURE")"
name="$(printf '%s\n' "$doc" | sed -n 's/^ name: *//p' | head -n 1 | tr -d '"')"
cluster="$(printf '%s\n' "$doc" | sed -n 's/^ cluster: *//p' | head -n 1 | tr -d '"')"
if [ -z "$name" ] || [ -z "$cluster" ]; then echo "FAIL: no KubernetesNodes name or spec.cluster in the fixture"; false; fi
case "$name" in
"$cluster"-?*) ;;
*) echo "FAIL: KubernetesNodes $name must be named $cluster-<pool>"; false ;;
esac
}
@test "render-tenant expands every variable with the runner's defaults" {
out="$(COZY_PVE_SSH=x COZY_PROXMOX_STORAGE=teststore "$RUNNER" render-tenant)"
if printf '%s' "$out" | grep -q '\${'; then echo "FAIL: the rendered tenant must not keep an unexpanded variable"; false; fi
@@ -61,3 +76,63 @@ RUNNER="$REPO_ROOT/hack/e2e-chainsaw/_lib/run-proxmox.sh"
rm -rf "$stub"
case "$out" in *"is gone from main-pool"*) ;; *) echo "$out" >&2; exit 1 ;; esac
}
# The tenant's super-admin kubeconfig is a credential for the whole tenant, kept
# under RUNNER_TEMP for one run. The workflow sets no path of its own, so these
# cases go through the same RUNNER_TEMP fallback the runner provides, with the
# override cleared.
KUBECONFIG_NAME=proxmox-e2e-tenant.kubeconfig
SUITE="$REPO_ROOT/hack/e2e-chainsaw/kubernetes-proxmox/chainsaw-test.yaml.disabled"
@test "tenant-node writes the tenant kubeconfig without group or other bits under umask 022" {
work="$(mktemp -d)"
stub="$(mktemp -d)"
# The secret read answers a kubeconfig and the first tenant read fails, which
# ends the check right after the fetch with the file in place.
cat > "$stub/kubectl" <<'STUB'
#!/bin/sh
case "$*" in *"get secret"*) printf '%s' YXBpVmVyc2lvbjogdjEK; exit 0 ;; esac
exit 1
STUB
chmod +x "$stub/kubectl"
rc=0
(umask 022; PATH="$stub:$PATH" RUNNER_TEMP="$work" COZY_PROXMOX_WORKDIR= COZY_PVE_SSH=x COZY_PROXMOX_STORAGE=s "$RUNNER" tenant-node) >/dev/null 2>&1 || rc=$?
mode="$(ls -l "$work/$KUBECONFIG_NAME" | cut -c1-10)"
rm -rf "$work" "$stub"
[ "$rc" -ne 0 ]
[ "$mode" = "-rw-------" ] || { echo "FAIL: the tenant kubeconfig was written $mode" >&2; exit 1; }
}
@test "apply-tenant removes a tenant kubeconfig an earlier run left behind" {
work="$(mktemp -d)"
stub="$(mktemp -d)"
printf 'an earlier run\n' > "$work/$KUBECONFIG_NAME"
printf '#!/bin/sh\ncat >/dev/null\n' > "$stub/kubectl"
chmod +x "$stub/kubectl"
PATH="$stub:$PATH" RUNNER_TEMP="$work" COZY_PROXMOX_WORKDIR= COZY_PVE_SSH=x COZY_PROXMOX_STORAGE=s "$RUNNER" apply-tenant
left=no
[ ! -e "$work/$KUBECONFIG_NAME" ] || left=yes
rm -rf "$work" "$stub"
[ "$left" = no ] || { echo "FAIL: apply-tenant left the earlier run's kubeconfig in place" >&2; exit 1; }
}
@test "the suite's finally removes the tenant kubeconfig, so diagnostics after it prints no tenant section" {
# keep_tenant exits the finally before its deletes, and the kubeconfig has to
# be gone on that path as well. Diagnostics reads whatever is at the path, so
# a file left there would be reported as the next run's tenant.
work="$(mktemp -d)"
stub="$(mktemp -d)"
printf 'this run\n' > "$work/$KUBECONFIG_NAME"
printf '#!/bin/sh\necho "stub $*"\n' > "$stub/kubectl"
printf '#!/bin/sh\nexit 0\n' > "$stub/ssh"
chmod +x "$stub/kubectl" "$stub/ssh"
finally="$(yq -r '.spec.steps[] | select(.name == "replacement-converges-unattended") | .finally[0].script.content' "$SUITE")"
[ -n "$finally" ]
PATH="$stub:$PATH" RUNNER_TEMP="$work" COZY_PROXMOX_WORKDIR= COZY_KEEP_TENANT=true sh -c "$finally" >/dev/null
out="$(PATH="$stub:$PATH" RUNNER_TEMP="$work" COZY_PROXMOX_WORKDIR= COZY_PVE_SSH=x COZY_PROXMOX_STORAGE=s "$RUNNER" diagnostics 2>&1)"
left=no
[ ! -e "$work/$KUBECONFIG_NAME" ] || left=yes
rm -rf "$work" "$stub"
[ "$left" = no ] || { echo "FAIL: the suite's finally left the tenant kubeconfig in place" >&2; exit 1; }
case "$out" in *"tenant nodes and pods"*) echo "$out" >&2; exit 1 ;; esac
}
+50
View File
@@ -39,6 +39,12 @@
# template has, so this is every worker's system disk.
# COZY_TEMPLATE_CPU default x86-64-v2-AES. Talos needs x86-64-v2, which
# the Proxmox default kvm64 does not provide.
# COZY_TALOS_FACTORY_CA path to a PEM file with the CA certificate(s) of a
# self-hosted factory or mirror served by an internal
# CA, the blocks a kubevirt pool puts in
# talos.imageFactoryCA. curl on the hypervisor trusts
# them on top of the system store, as CDI does, and
# the hypervisor's own trust store is left as it is.
# COZY_PVE_SSH_KEY default ~/.ssh/id_ed25519
set -eu
@@ -60,6 +66,21 @@ chart_talos() {
' "$VALUES"
}
# Prints the certificate blocks of a PEM file and fails unless the file holds
# complete certificate blocks and nothing else: they are pasted into the root
# program, and a private key kept in the same file must not travel with them.
pem_certificates() {
tr -d '\r' < "$1" | awk '
{ sub(/[ \t]+$/, "") }
/^-----BEGIN CERTIFICATE-----$/ && !inside { inside = 1; body = 0; print; next }
/^-----END CERTIFICATE-----$/ && inside && body { inside = 0; blocks++; print; next }
inside && /^[A-Za-z0-9+\/=]+$/ { body = 1; print; next }
!inside && /^$/ { next }
{ bad = 1; exit }
END { exit (bad || inside || !blocks) }
'
}
# grep -x matches line by line, so a value with a newline in it would pass on
# any one of its lines and carry the rest into the program as commands of their
# own. Such a value is refused before grep sees it.
@@ -81,6 +102,7 @@ FACTORY=${COZY_TALOS_FACTORY_URL:-$(chart_talos imageFactoryURL)}
FACTORY=${FACTORY%/}
DISK_SIZE=${COZY_TEMPLATE_DISK_SIZE:-20G}
CPU=${COZY_TEMPLATE_CPU:-x86-64-v2-AES}
FACTORY_CA=${COZY_TALOS_FACTORY_CA:-}
TAGS=${COZY_TEMPLATE_TAGS:-cozystack,talos,talos-$VERSION,schematic-$(printf '%s' "$SCHEMATIC" | cut -c1-8)}
# Every value below is pasted into a program that runs as root on the
@@ -93,6 +115,14 @@ matches "$SCHEMATIC" '[0-9a-f]{64}' || die "schematic '$SCHEMATIC' is not an Ima
matches "$FACTORY" 'https?://[A-Za-z0-9.-]+(:[0-9]+)?(/[A-Za-z0-9._~/-]*)?' || die "factory URL '$FACTORY' is not a plain http(s) URL"
matches "$DISK_SIZE" '[1-9][0-9]*G' || die "disk size $DISK_SIZE is not a whole number of gigabytes such as 20G"
matches "$CPU" '[A-Za-z0-9_.+-]+' || die "cpu type $CPU is not a Proxmox CPU model name"
CA_PEM=
if [ -n "$FACTORY_CA" ]; then
case $FACTORY in https://*) ;; *) die "COZY_TALOS_FACTORY_CA is set, but the factory URL '$FACTORY' is not https" ;; esac
[ -f "$FACTORY_CA" ] || die "COZY_TALOS_FACTORY_CA: $FACTORY_CA is not a file"
[ -r "$FACTORY_CA" ] || die "COZY_TALOS_FACTORY_CA: cannot read $FACTORY_CA"
CA_PEM=$(pem_certificates "$FACTORY_CA") ||
die "COZY_TALOS_FACTORY_CA: $FACTORY_CA is not a file of complete PEM certificate blocks and nothing else"
fi
# capmox lowercases the selector and compares it with the template's tags as a
# set, so the tags are normalised the same way before they are checked and
@@ -147,7 +177,27 @@ fi
work=$(mktemp -d "${TMPDIR:-/var/tmp}/talos-template.XXXXXX")
trap 'rm -rf "$work"' EXIT
EOF
if [ -z "$CA_PEM" ]; then
cat <<'EOF'
curl -fL --retry 3 -o "$work/disk.raw.xz" "$URL"
EOF
else
# The system bundle goes first so that a redirect to a publicly trusted
# host still verifies: CDI appends talos.imageFactoryCA to its system pool
# the same way. The blank line keeps a bundle without a final newline from
# running into the first block. The terminator holds an underscore, which
# no line of a checked certificate block can. A CA is accepted only for an
# https factory, and redirects are held to https as well, so the image
# cannot leave TLS verification halfway.
cat <<EOF
{ cat /etc/ssl/certs/ca-certificates.crt; echo; cat; } > "\$work/factory-ca.pem" <<'FACTORY_CA_PEM'
$CA_PEM
FACTORY_CA_PEM
curl -fL --retry 3 --proto-redir =https --cacert "\$work/factory-ca.pem" -o "\$work/disk.raw.xz" "\$URL"
EOF
fi
cat <<'EOF'
xz -d "$work/disk.raw.xz"
# The layout of the templates the Proxmox substrate was verified on. No NIC:
+63
View File
@@ -93,6 +93,69 @@ EOF
printf '%s' "$out" | grep -qxF "NAME='talos-v1.13.6-beta.1-$(chart schematicID | cut -c1-8)'"
}
# Two certificate blocks in the shape the CA checks accept. Only the shape is
# checked here, so the bodies are base64 text and not real certificates.
write_ca() {
printf '%s\n' '-----BEGIN CERTIFICATE-----' 'MIIBszCCAVmgAwIBAgIUZmFjdG9yeS1jYS1vbmU=' '-----END CERTIFICATE-----' \
'' '-----BEGIN CERTIFICATE-----' 'MIIBszCCAVmgAwIBAgIUZmFjdG9yeS1jYS10d28=' 'c2Vjb25kIGxpbmU=' '-----END CERTIFICATE-----' > "$1"
}
@test "without a factory CA the download trusts the system store alone" {
out="$("$SCRIPT" print 9002 main-pool)"
printf '%s' "$out" | grep -qx 'curl -fL --retry 3 -o "$work/disk.raw.xz" "$URL"'
if printf '%s' "$out" | grep -q -- '--cacert'; then echo "FAIL: no CA was given, so curl must not be handed one"; false; fi
}
@test "a factory CA is trusted on top of the system store for the download" {
stub="$(mktemp -d)"
stub_pve "$stub"
cat > "$stub/curl" <<EOF
#!/bin/sh
echo "curl \$*" >> "$stub/calls"
while [ \$# -gt 0 ]; do
case "\$1" in
--cacert) cp "\$2" "$stub/cacert" ;;
-o) : > "\$2" ;;
esac
shift
done
EOF
echo '[]' > "$stub/resources.json"
printf 'system bundle\n' > "$stub/system.pem"
write_ca "$stub/ca.pem"
sed 's/$/\r/' "$stub/ca.pem" > "$stub/ca-crlf.pem"
COZY_TALOS_FACTORY_URL=https://factory.internal COZY_TALOS_FACTORY_CA="$stub/ca-crlf.pem" \
"$SCRIPT" print 9002 main-pool |
sed "s|/etc/ssl/certs/ca-certificates.crt|$stub/system.pem|" |
PATH="$stub:$PATH" TMPDIR="$stub" sh >/dev/null
grep -qE '^curl -fL --retry 3 --proto-redir =https --cacert [^ ]*/factory-ca.pem -o [^ ]*/disk.raw.xz https://factory.internal/' "$stub/calls"
got="$(cat "$stub/cacert")"
want="$(printf 'system bundle\n\n'; grep -v '^$' "$stub/ca.pem")"
[ "$got" = "$want" ] || { printf 'curl was handed:\n%s\nexpected:\n%s\n' "$got" "$want" >&2; false; }
rm -rf "$stub"
}
@test "a factory CA file that is anything more than certificate blocks is refused" {
stub="$(mktemp -d)"
write_ca "$stub/ca.pem"
{ cat "$stub/ca.pem"; printf '%s\n' '-----BEGIN PRIVATE KEY-----' 'MC4CAQAwBQYDK2VwBCIEIA==' '-----END PRIVATE KEY-----'; } > "$stub/with-key.pem"
{ cat "$stub/ca.pem"; echo 'echo INJECTED'; } > "$stub/trailing.pem"
printf '%s\n' '-----BEGIN CERTIFICATE-----' 'MIIBszCCAVmgAwIBAgIU' > "$stub/open.pem"
printf '%s\n' '-----BEGIN CERTIFICATE-----' '-----END CERTIFICATE-----' > "$stub/no-body.pem"
: > "$stub/empty.pem"
for ca in with-key.pem trailing.pem open.pem no-body.pem empty.pem missing.pem .; do
rc=0
out="$(COZY_TALOS_FACTORY_URL=https://factory.internal COZY_TALOS_FACTORY_CA="$stub/$ca" "$SCRIPT" print 9002 main-pool 2>&1)" || rc=$?
[ "$rc" -ne 0 ] || { echo "accepted: $ca" >&2; false; }
case "$out" in *"set -eu"*) echo "printed a program for: $ca" >&2; false ;; esac
done
rc=0
out="$(COZY_TALOS_FACTORY_URL=http://factory.internal COZY_TALOS_FACTORY_CA="$stub/ca.pem" "$SCRIPT" print 9002 main-pool 2>&1)" || rc=$?
[ "$rc" -ne 0 ]
case "$out" in *"is not https"*) ;; *) echo "$out" >&2; false ;; esac
rm -rf "$stub"
}
@test "a template already carrying the same tag set stops the program before the download" {
stub="$(mktemp -d)"
stub_pve "$stub"
+14
View File
@@ -100,6 +100,20 @@ run_step() {
if grep -qv 'config view' "$KUBECTL_LOG"; then echo "FAIL: Stage went past the guard"; false; fi
}
@test "Stage writes an SSH key OpenSSH loads, whether or not the secret ends in a newline" {
ssh-keygen -q -t ed25519 -N '' -C e2e -f "$BATS_TEST_TMPDIR/src"
# $(...) drops the trailing newline, as `gh secret set` does with stdin.
key="$(cat "$BATS_TEST_TMPDIR/src")"
export KUBECONFIG_B64="" PVE_TOKEN_ID="" PVE_TOKEN_SECRET="" PVE_URL="" PVE_REGION=""
for PVE_SSH_KEY in "$key" "$key"$'\n'; do
export PVE_SSH_KEY
STUB_SERVER=https://192.0.2.10:6443 run_step "Stage credentials"
[ "$status" -eq 0 ]
run ssh-keygen -y -f "$COZY_PVE_SSH_KEY"
[ "$status" -eq 0 ]
done
}
@test "Stage writes the stand's Secret through the staged kubeconfig when it resolves" {
export KUBECONFIG_B64="" PVE_SSH_KEY="" PVE_TOKEN_ID="" PVE_TOKEN_SECRET="" PVE_URL="" PVE_REGION=""
STUB_SERVER=https://192.0.2.10:6443 run_step "Stage credentials"
+188 -65
View File
@@ -14,8 +14,9 @@
# full force-push cycle promote actually performs.
# * hack/validate-changelog.sh against every current-era changelog in the tree
# plus hand-built truncated / wrong-version / whitespace inputs.
# * finalize's release-body selection, executed under node with the same inputs
# the workflow gives it.
# * The github-script steps that set a release body, finalize's and
# update-releasenotes', taken verbatim from the workflows and run under node
# against fixture trees.
# * The GitHub Actions `needs` + `if:` semantics the whole non-blocking design
# rests on, run under `act` when Docker is available (see
# hack/testdata/needs-semantics.yaml).
@@ -40,6 +41,7 @@ PARSE_RC="$REPO_ROOT/hack/parse-rc-tag.sh"
SELECT="$REPO_ROOT/hack/select-changelog-source.sh"
PROMOTE="$REPO_ROOT/.github/workflows/promote-rc.yaml"
FINALIZE="$REPO_ROOT/.github/workflows/pull-requests-release.yaml"
SYNC="$REPO_ROOT/.github/workflows/update-releasenotes.yaml"
# Body of a top-level workflow job, so an assertion about one job cannot be
# satisfied — or broken — by an unrelated one. Same shape as the contract suite's.
@@ -128,6 +130,68 @@ Truncated fixture — no sections, no compare link.
EOF
}
# Exactly <count> copies of <string> and no newline; awk passes the bytes of a
# multi-byte <string> through untouched.
repeat() {
awk -v s="$1" -v n="$2" 'BEGIN { for (i = 0; i < n; i++) printf "%s", s }'
}
# node runs the workflows' github-script, so it is required, not optional: a
# test that passes without it has run nothing. The unit lane's runner has it,
# which hack/pr-labeler-contract.bats already depends on.
need_node() {
command -v node >/dev/null 2>&1 && return 0
echo "node is required to run the workflows' github-script steps" >&2
return 1
}
# Runs one github-script step under node with its script exactly as the
# workflow carries it, from <tree> as the working directory; github and core
# are stand-ins. Leaves the step's log in <out>/log, one line per call to GitHub
# or core in <out>/calls (ending in `ok`, or `threw` if the step threw), and
# each body sent with updateRelease in <out>/body.<release_id>.
#
# run_step <workflow> <job> <step> <tree> <releases.json> <out> [<tag>]
run_step() {
mkdir -p "$6"
JOB="$2" STEP="$3" yq '.jobs[strenv(JOB)].steps[] | select(.name == strenv(STEP)) | .with.script' "$1" > "$6/script.js"
[ -s "$6/script.js" ] || { echo "no script for step '$3' in $1" >&2; return 1; }
cat > "$6/harness.js" <<'JS'
const fs = require('fs');
const path = require('path');
const [scriptFile, releasesFile, outDir] = process.argv.slice(2);
let script = fs.readFileSync(scriptFile, 'utf8');
if (process.env.TAG) script = script.split('${{ steps.get_tag.outputs.tag }}').join(process.env.TAG);
if (script.includes('${{')) throw new Error('an expression in the script was left unsubstituted');
const releases = JSON.parse(fs.readFileSync(releasesFile, 'utf8'));
const calls = [];
const github = { rest: { repos: {
listReleases: async () => ({ data: releases }),
updateRelease: async (p) => {
const keys = Object.keys(p).filter((k) => !['owner', 'repo', 'release_id', 'body'].includes(k)).sort();
calls.push(['updateRelease', p.release_id, ...keys.map((k) => `${k}=${p[k]}`), ...('body' in p ? ['body'] : [])].join(' '));
if ('body' in p) fs.writeFileSync(path.join(outDir, `body.${p.release_id}`), p.body);
return { data: { ...releases.find((r) => r.id === p.release_id), ...p } };
},
} } };
const context = { repo: { owner: 'cozystack', repo: 'cozystack' } };
const core = {
setOutput: (k, v) => calls.push(`setOutput ${k}=${v}`),
setFailed: (m) => calls.push(`setFailed ${m}`),
};
const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor;
new AsyncFunction('require', 'github', 'context', 'core', script)(require, github, context, core)
.then(() => 'ok', (e) => `threw ${e.message}`)
.then((result) => fs.writeFileSync(path.join(outDir, 'calls'), calls.concat(result).join('\n') + '\n'));
JS
( cd "$4" && TAG="${7:-}" node "$6/harness.js" "$6/script.js" "$5" "$6" > "$6/log" 2>&1 ) || {
echo "node failed for step '$3':" >&2; sed 's/^/ /' "$6/log" >&2; return 1; }
[ -s "$6/calls" ] || { echo "step '$3' recorded nothing" >&2; return 1; }
}
# The body finalize publishes for v1.7.0 in place of one over 100000 bytes.
LINK_V170='The release notes for v1.7.0 are too long for a GitHub release description; the full changelog is [docs/changelogs/v1.7.0.md](https://github.com/cozystack/cozystack/blob/v1.7.0/docs/changelogs/v1.7.0.md).'
# The rc-tag parser is shared by changelog-rc.yaml and mirrored inline by the
# reusable changelog-generate.yaml. It must REJECT whitespace outright, not strip
# it: a stripped 'v1. 6.0-rc.2' silently becomes a different tag, and a caller
@@ -466,75 +530,134 @@ v1.6.0=reject:format"
return 0
}
@test "finalize: the release-body selection runs and picks the changelog" {
command -v node >/dev/null || { echo "node unavailable; skipping"; return 0; }
# finalize's Publish draft release step, run as the workflow carries it, one
# merge-commit tree per case. Whatever the tree holds, the release is
# published: with the release notes, else the changelog, else the draft's own
# body, and with a link in place of anything over 100000 bytes, since the tags
# already exist and a body GitHub refuses would strand the release.
@test "finalize: publishes the release notes, else the changelog, and a link over 100000 bytes" {
need_node
tmp="$(mktemp -d)"
cat > "$tmp/releases.json" <<'EOF'
[
{"id": 1, "draft": true, "prerelease": false, "tag_name": "v1.7.0", "name": "v1.7.0", "body": "Promoted from v1.7.0-rc.3."},
{"id": 2, "draft": false, "prerelease": false, "tag_name": "v1.6.5", "name": "v1.6.5", "body": "v1.6.5"}
]
EOF
notes=docs/releasenotes/v1.7.0.md
cl=docs/changelogs/v1.7.0.md
for c in notes blanknotes changelog neither blank at over multibyte overnotes; do
mkdir -p "$tmp/$c/docs/releasenotes" "$tmp/$c/docs/changelogs"
done
printf '# Cozystack v1.7.0\n\nThe highlights.\n' > "$tmp/notes/$notes"
printf ' \n\t\n' > "$tmp/blanknotes/$notes"
printf ' \n\n \n' > "$tmp/blank/$cl"
for c in notes blanknotes changelog; do write_changelog "$tmp/$c/$cl" 1.7.0; done
repeat a 100000 > "$tmp/at/$cl"
repeat a 100001 > "$tmp/over/$cl"
# 40000 characters, 120000 bytes: the limit is in bytes.
repeat '€' 40000 > "$tmp/multibyte/$cl"
repeat a 100001 > "$tmp/overnotes/$notes"
write_changelog "$tmp/overnotes/$cl" 1.7.0
# The same decision finalize makes, extracted verbatim in shape: read the file,
# refuse whitespace-only, otherwise use it as the body.
#
# All three cases run in ONE node invocation. Driving them from a shell loop
# produced an empty capture for the missing-file case under this repo's
# cozytest runner — reproducibly, with node exiting 0 and writing nothing, and
# not reproducible outside it. Rather than assert around a runner quirk (which
# would leave the assertion quietly meaningless), the decision table is
# computed in one process and compared as a whole.
cat > "$tmp/body.js" <<'JS'
const fs = require('fs');
// Verbatim in shape from pull-requests-release.yaml's Publish draft release step.
function selectBody(changelogPath) {
let body;
let decision;
if (fs.existsSync(changelogPath)) {
const raw = fs.readFileSync(changelogPath, 'utf8');
if (raw.trim().length === 0) {
decision = 'refuse-whitespace';
} else {
body = raw;
decision = 'use-file';
}
} else {
decision = 'absent';
}
const payload = { draft: false, ...(body ? { body } : {}) };
return decision + ':' + Object.prototype.hasOwnProperty.call(payload, 'body');
for spec in notes:$notes blanknotes:$cl changelog:$cl at:$cl over:link multibyte:link overnotes:link neither:none blank:none; do
c="${spec%%:*}"; want="${spec#*:}"
run_step "$FINALIZE" finalize 'Publish draft release' "$tmp/$c" "$tmp/releases.json" "$tmp/out.$c" v1.7.0
with_body=' body'
[ "$want" != none ] || with_body=''
expected="$(printf 'ok\nsetOutput make_latest=true\nupdateRelease 1 draft=false make_latest=true prerelease=false%s\n' "$with_body")"
[ "$(sort "$tmp/out.$c/calls")" = "$expected" ] || {
echo "$c: finalize did something other than publishing v1.7.0:" >&2
sed 's/^/ /' "$tmp/out.$c/calls" "$tmp/out.$c/log" >&2; exit 1; }
case "$want" in
none)
grep -q '^::warning::' "$tmp/out.$c/log" || {
echo "$c: the draft's body was kept without a ::warning::" >&2; exit 1; } ;;
link)
[ "$(cat "$tmp/out.$c/body.1")" = "$LINK_V170" ] || {
echo "$c: the body over 100000 bytes was not replaced by the link:" >&2
head -c 300 "$tmp/out.$c/body.1" >&2; exit 1; }
grep -q '^::warning::.* bytes, over MAX_BODY_BYTES' "$tmp/out.$c/log" || {
echo "$c: the link was published without a ::warning::" >&2; exit 1; } ;;
*)
cmp -s "$tmp/out.$c/body.1" "$tmp/$c/$want" || {
echo "$c: the body is not $want byte for byte" >&2; exit 1; }
grep -qF "Release body from $want ($(wc -c < "$tmp/$c/$want") bytes)" "$tmp/out.$c/log" || {
echo "$c: the log does not give the body's source and byte length:" >&2
sed 's/^/ /' "$tmp/out.$c/log" >&2; exit 1; } ;;
esac
done
rm -rf "$tmp"
}
const lines = process.argv.slice(2).map((p) => {
const label = p.replace(/^.*\//, '');
return label + '=' + selectBody(p);
});
fs.writeFileSync(1, lines.join('\n') + '\n');
JS
# A release with no release notes publishes exactly what finalize published
# before they existed. Every shipped changelog is a fixture; one over 100000
# bytes, which GitHub may refuse, publishes the link instead.
@test "finalize: without release notes a shipped changelog is published byte for byte, or as the link over 100000 bytes" {
need_node
tmp="$(mktemp -d)"
checked=0
verbatim=0
for f in "$REPO_ROOT"/docs/changelogs/v*.md; do
[ -f "$f" ] || continue
tag="$(basename "$f" .md)"
mkdir -p "$tmp/$tag/docs/changelogs"
cp "$f" "$tmp/$tag/docs/changelogs/"
printf '[{"id": 1, "draft": true, "prerelease": false, "tag_name": "%s", "name": "%s", "body": "draft"}]\n' "$tag" "$tag" > "$tmp/$tag.json"
run_step "$FINALIZE" finalize 'Publish draft release' "$tmp/$tag" "$tmp/$tag.json" "$tmp/out.$tag" "$tag"
if [ "$(wc -c < "$f")" -le 100000 ]; then
cmp -s "$tmp/out.$tag/body.1" "$f" || { echo "$tag: not published byte for byte" >&2; exit 1; }
verbatim=$((verbatim + 1))
else
grep -qF "/blob/$tag/docs/changelogs/$tag.md)." "$tmp/out.$tag/body.1" || {
echo "$tag: over 100000 bytes but not published as the link" >&2; exit 1; }
fi
checked=$((checked + 1))
done
[ "$checked" -ge 10 ] && [ "$verbatim" -ge 10 ] || {
echo "$checked changelog(s) checked, $verbatim verbatim; the glob broke and this verifies nothing." >&2; exit 1; }
rm -rf "$tmp"
}
write_changelog "$tmp/good.md" 1.6.0
printf ' \n\n \n' > "$tmp/ws.md"
# update-releasenotes' sync step, run as the workflow carries it over one main
# checkout. It makes finalize's choice of file, but a body over 100000 bytes is
# never synced, not even as the link: the tags.yaml backstop ports a changelog
# to main without its release notes, and it would replace the body finalize
# published.
@test "update-releasenotes: syncs the release notes, else the changelog, and never a body over 100000 bytes" {
need_node
tmp="$(mktemp -d)"
m="$tmp/main"
mkdir -p "$m/docs/releasenotes" "$m/docs/changelogs"
printf '# v1.0.1\n\nThe highlights.\n' > "$m/docs/releasenotes/v1.0.1.md"
printf ' \n' > "$m/docs/releasenotes/v1.0.2.md"
for v in 1 2 3 8 9; do write_changelog "$m/docs/changelogs/v1.0.$v.md" "1.0.$v"; done
repeat a 100001 > "$m/docs/changelogs/v1.0.5.md"
repeat '€' 40000 > "$m/docs/changelogs/v1.0.6.md"
repeat a 100000 > "$m/docs/changelogs/v1.0.7.md"
repeat a 100001 > "$m/docs/releasenotes/v1.0.9.md"
UP_TO_DATE="$(cat "$m/docs/changelogs/v1.0.8.md")" node -e '
const r = (v, body) => ({ id: v, draft: false, prerelease: false, tag_name: `v1.0.${v}`, name: `v1.0.${v}`, body });
process.stdout.write(JSON.stringify([1, 2, 3, 4, 5, 6, 7, 9].map((v) => r(v, "old body")).concat(r(8, process.env.UP_TO_DATE))));' > "$tmp/releases.json"
node "$tmp/body.js" "$tmp/good.md" "$tmp/ws.md" "$tmp/nope.md" > "$tmp/out.txt" 2>"$tmp/err.txt" || {
echo "node failed:" >&2; cat "$tmp/err.txt" >&2; exit 1; }
want="good.md=use-file:true
ws.md=refuse-whitespace:false
nope.md=absent:false"
got="$(cat "$tmp/out.txt")"
[ "$got" = "$want" ] || {
echo "finalize's release-body decision table is wrong." >&2
echo "expected:" >&2; printf '%s\n' "$want" | sed 's/^/ /' >&2
echo "got:" >&2; printf '%s\n' "$got" | sed 's/^/ /' >&2
echo "A whitespace-only or absent changelog reaching the body would publish" >&2
echo "blank release notes instead of leaving the draft's text in place." >&2
exit 1
}
# And the shape asserted here must be the shape finalize actually uses.
grep -q 'raw.trim().length === 0' "$FINALIZE" || {
echo "finalize's whitespace guard changed shape; this test now proves nothing" >&2
echo "about the real workflow." >&2
exit 1
}
run_step "$SYNC" update-releasenotes 'Update release notes from changelogs' "$m" "$tmp/releases.json" "$tmp/out"
want="updateRelease 1 body
updateRelease 2 body
updateRelease 3 body
updateRelease 7 body
ok"
[ "$(cat "$tmp/out/calls")" = "$want" ] || {
echo "The sync updated other releases than v1.0.1, v1.0.2, v1.0.3 and v1.0.7:" >&2
sed 's/^/ /' "$tmp/out/calls" "$tmp/out/log" >&2; exit 1; }
for pair in 1:releasenotes 2:changelogs 3:changelogs 7:changelogs; do
v="${pair%%:*}"
cmp -s "$tmp/out/body.$v" "$m/docs/${pair#*:}/v1.0.$v.md" || {
echo "v1.0.$v was not synced with docs/${pair#*:}/v1.0.$v.md" >&2; exit 1; }
done
for v in 5 6 9; do
grep -q "^::warning::.*over MAX_BODY_BYTES.*leaving the body of release v1.0.$v as it is" "$tmp/out/log" || {
echo "v1.0.$v, over 100000 bytes, was left alone without a ::warning:: naming it" >&2; exit 1; }
done
rm -rf "$tmp"
}
+124 -2
View File
@@ -26,6 +26,8 @@
# 1. open-pr survives a failed changelog job (the non-blocking guarantee).
# 2. The AI step is the ONLY thing allowed to fail silently.
# 3. finalize reads the changelog file as the release body.
# 3b. finalize and update-releasenotes share the 100000-byte body limit, and
# the sync fires on the release notes too.
# 4. finalize refuses whitespace-only content rather than blanking the notes.
# 5. The generator is pointed at a checkout state docs/agents/changelog.md
# actually permits.
@@ -35,9 +37,12 @@
# trip over.
# 9-11. The validator accepts every changelog of the current era (both header
# conventions, including a 19-line patch release) and rejects unusable output.
# 11b. It also rejects an entry cut off with an ellipsis and a link to
# cozystack.io/docs/next/, and nothing that merely resembles either.
# 12. The workflow calls the validator script rather than reimplementing it.
# 13. The agent instructions and the validator are overlaid from the dispatch
# ref, because the rc tag's copies predate this workflow.
# 13. The agent instructions, the release checks they send the agent to and the
# validator are overlaid from the dispatch ref, because the rc tag's copies
# predate this workflow.
# 14. open-pr survives an artifact transfer failure and keys off the file on disk.
# 15. A pre-existing changelog is validated, not merely non-empty.
# 16. promote-rc parses the rc tag ONCE in a cheap `parse` job that rejects
@@ -82,6 +87,7 @@ GENERATE="$REPO_ROOT/.github/workflows/changelog-generate.yaml"
RC_WRAPPER="$REPO_ROOT/.github/workflows/changelog-rc.yaml"
FINALIZE="$REPO_ROOT/.github/workflows/pull-requests-release.yaml"
TAGS="$REPO_ROOT/.github/workflows/tags.yaml"
SYNC="$REPO_ROOT/.github/workflows/update-releasenotes.yaml"
CHANGELOG_DOC="$REPO_ROOT/docs/agents/changelog.md"
# Body of a single top-level job block in a workflow file, from ` <name>:` up to
@@ -204,6 +210,31 @@ code_lines() { grep -vE '^[[:space:]]*#'; }
}
}
# 3b. finalize publishes a link in place of a body over the limit, because the
# tags exist by then; the sync leaves such a body alone. Both read the same
# constant, and what each does with it is executed in
# hack/release-changelog-behaviour.bats. The sync must also fire when only
# the release notes change, or editing them on main reaches no release.
@test "finalize and update-releasenotes share the 100000-byte body limit, and the sync fires on the release notes" {
for f in "$FINALIZE" "$SYNC"; do
grep -qE '^[[:space:]]*const MAX_BODY_BYTES = 100000;' "$f" || {
echo "$(basename "$f") no longer declares const MAX_BODY_BYTES = 100000 on a code line." >&2
exit 1
}
if grep -q 'release-body\.sh' "$f"; then
echo "$(basename "$f") mentions hack/release-body.sh, which no longer exists." >&2
false
fi
done
paths="$(yq '.on.push.paths[]' "$SYNC")"
for p in 'docs/changelogs/v*.md' 'docs/releasenotes/v*.md'; do
printf '%s\n' "$paths" | grep -qxF "$p" || {
echo "update-releasenotes no longer fires on pushes touching $p." >&2
exit 1
}
done
}
# 4. A whitespace-only file is worse than no file: it would blank the release
# notes more thoroughly than publishing the draft body would. The generation
# core validates structurally, but a changelog can also arrive by hand-commit,
@@ -432,6 +463,64 @@ code_lines() { grep -vE '^[[:space:]]*#'; }
rm -rf "$tmp"
}
# 11b. Two content defects that reached a generated release body: an entry cut
# off with an ellipsis before its attribution, which loses the operator
# action PR authors put at the end of a release note, and a link to
# cozystack.io/docs/next/, which the live site answers with 404. Each is
# planted in a shipped changelog, so a rejection cannot come from broken
# structure instead, and the reason must name the line and fit on one
# line, since the workflows quote it into a ::warning:: or ::error::
# annotation, which ends at the first newline. The near misses at the end
# are shapes shipped changelogs carry.
@test "the changelog validator rejects truncated entries and docs/next links, and only those" {
base="$REPO_ROOT/docs/changelogs/v1.5.1.md"
[ -f "$base" ] || { echo "fixture $base is gone; pick another shipped changelog" >&2; exit 1; }
tmp="$(mktemp -d)"
line=$(( $(wc -l < "$base") + 1 ))
for bad in \
'ellipsis|* **fix(app): restore the bucket**: The restore now carries the bucket, and the… ([**@someone**](https://github.com/someone) in #1)' \
'ellipsis|* **fix(app): restore the bucket**: The restore now carries the bucket, and the... ([**@someone**](https://github.com/someone) in #1)' \
'docs/next|Learn more in the [documentation](https://cozystack.io/docs/next/).' \
'docs/next|Learn more in the [documentation](https://cozystack.io/docs/next).'
do
want="${bad%%|*}"; text="${bad#*|}"
{ cat "$base"; printf '%s\n' "$text"; } > "$tmp/bad.md"
if "$REPO_ROOT/hack/validate-changelog.sh" "$tmp/bad.md" 1.5.1 >/dev/null 2>&1; then
echo "Validator accepted a changelog carrying: $text" >&2
exit 1
fi
reason="$("$REPO_ROOT/hack/validate-changelog.sh" "$tmp/bad.md" 1.5.1 2>&1 || true)"
case "$reason" in
*"$want"*"(line(s) $line)"*) ;;
*) echo "Rejected '$text' without naming '$want' and line $line: $reason" >&2; exit 1 ;;
esac
[ "$(printf '%s\n' "$reason" | wc -l)" -eq 1 ] || {
echo "The reason spans several lines, so a workflow annotation would carry only the first:" >&2
printf '%s\n' "$reason" | sed 's/^/ /' >&2
exit 1
}
done
# The website source tree named in prose (v1.3.0), an ellipsis inside a
# sentence (v1.4.0), a list closed with "..." right before the attribution,
# and a versioned docs link.
{
cat "$base"
printf '%s\n' \
'* **[website] Trunk docs**: keeps a permanent `content/en/docs/next/` trunk ([**@someone**](https://github.com/someone) in cozystack/website#1)' \
'* **feat(app): sizes**: the legacy names (`nano` … `2xlarge`) stay as aliases ([**@someone**](https://github.com/someone) in #2)' \
'* **feat(app): plugins**: stages the reference plugins (bridge, macvlan, ...) ([**@someone**](https://github.com/someone) in #3)' \
'Learn more in the [documentation](https://cozystack.io/docs/v1.5/operations/).'
} > "$tmp/near.md"
"$REPO_ROOT/hack/validate-changelog.sh" "$tmp/near.md" 1.5.1 >/dev/null 2>&1 || {
echo "Validator rejected text that only resembles a truncated entry or a docs/next link:" >&2
"$REPO_ROOT/hack/validate-changelog.sh" "$tmp/near.md" 1.5.1 2>&1 >/dev/null | sed 's/^/ /' >&2
exit 1
}
rm -rf "$tmp"
}
# 12. The workflow must actually call the script. Keeping the predicates in a
# tested script helps nothing if the step quietly reverts to an inline check.
# Now in the reusable generation core.
@@ -481,6 +570,39 @@ code_lines() { grep -vE '^[[:space:]]*#'; }
exit 1
}
# changelog.md sends the agent to run the docs/release/changelogs.md checks
# and the validator on its own output, by their working-tree paths. Without
# these two copies it runs the rc tag's versions, which predate every check
# added since.
for f in docs/release/changelogs.md hack/validate-changelog.sh; do
printf '%s\n' "$overlay" | code_lines | grep -qF "cp .release-tooling/$f $f" || {
echo "The refresh step does not overlay $f from the tooling checkout, so the" >&2
echo "agent following docs/agents/changelog.md section 9 runs the rc tag's copy." >&2
exit 1
}
done
# An rc tagged before the release docs were split has no docs/release/
# directory, and cp does not create one.
printf '%s\n' "$overlay" | code_lines | grep -qE '^[[:space:]]+mkdir -p docs/release$' || {
echo "The refresh step does not create docs/release before copying into it, so" >&2
echo "the copy fails on every rc tagged before the release docs were split." >&2
exit 1
}
# The tooling checkout has to list both directories the copies read from.
tooling="$(printf '%s\n' "$block" | awk '
/^ - name: Checkout release tooling from the dispatch ref$/ { inside = 1; next }
/^ - name: / { inside = 0 }
inside' | code_lines)"
for d in docs/agents docs/release; do
printf '%s\n' "$tooling" | grep -qE "^[[:space:]]+$d\$" || {
echo "The tooling checkout no longer lists $d, so .release-tooling lacks the" >&2
echo "file the refresh step copies from it." >&2
exit 1
}
done
# Every job that runs release tooling against an older checkout must source it
# from the dispatch ref. A script fetched by a script that is itself missing
# solves nothing, so this is done by a second actions/checkout, not by shell.
+44 -423
View File
@@ -1,28 +1,22 @@
#!/usr/bin/env bats
# Contract for the rc freeze, the backport target resolution that depends on it,
# the triggering and concurrency rules that decide whether a backport runs at
# all, and the agreement between those rules and cmd/backport-audit, which
# gates a cut by reproducing them. Like promote-gate-contract.bats, these tests pin executable/structural
# workflow lines rather than prose: a commented-out gate or a guard demoted to a
# comment must never satisfy the contract.
# Contract for the rc freeze and the release-line build it dispatches. Like
# promote-gate-contract.bats, these tests pin executable/structural workflow
# lines rather than prose: a commented-out gate or a guard demoted to a comment
# must never satisfy the contract.
#
# Neither workflow can be exercised by a PR lane. cut-prerelease.yaml runs only
# on workflow_dispatch from main or a release line, and its whole purpose is a
# one-way, write-once side effect — the first vX.Y.0-rc.N freezes the line, and
# there is no dry-run mode and no undo short of deleting the branch by hand.
# backport.yaml runs only on a merged main-targeted PR. So every invariant below
# is one that first reports a slip at a real release, on the one commit nobody
# wants to be debugging: getting the freeze condition wrong re-opens a frozen
# line silently, and getting the comparator wrong aims every backport at the
# wrong branch. Cheap to pin here, expensive to discover there.
# No PR lane exercises them. cut-prerelease.yaml runs only on workflow_dispatch
# from main or a release line, and its whole purpose is a one-way, write-once
# side effect — the first vX.Y.0-rc.N freezes the line, and there is no dry-run
# mode and no undo short of deleting the branch by hand. So every invariant
# below is one that first reports a slip at a real release, on the one commit
# nobody wants to be debugging: getting the freeze condition wrong re-opens a
# frozen line silently. Cheap to pin here, expensive to discover there.
load test_helper
REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)"
CUT="$REPO_ROOT/.github/workflows/cut-prerelease.yaml"
BACKPORT="$REPO_ROOT/.github/workflows/backport.yaml"
AUDIT="$REPO_ROOT/cmd/backport-audit/main.go"
# Strip YAML comments. POSIX `grep` only: the unit-test runner has no ripgrep.
# grep exits 1 when nothing is selected (legitimate for an all-comment block)
@@ -44,24 +38,6 @@ code_lines() {
[ "$rc" -le 1 ]
}
# Strip JavaScript comments. backport.yaml's logic lives in a github-script
# `script: |` block, where comments start with `//` and survive code_lines
# untouched — and that block's comments name the very APIs these tests assert
# are gone, so filtering on `#` alone would let a comment satisfy a pin.
script_lines() {
local rc=0
grep -v '^[[:space:]]*//' || rc=$?
[ "$rc" -le 1 ]
}
# Body of one job, up to the next top-level job key.
job_block() {
awk -v job=" $1:" '
$0 == job { inside = 1; next }
/^ [a-z0-9_-]+:$/ { inside = 0 }
inside' "$2"
}
# Body of one `- name:` step, up to the next step at the same indent.
step_block() {
awk -v want=" - name: $1" '
@@ -78,7 +54,6 @@ step_line() {
@test "workflows under contract exist" {
[ -f "$CUT" ]
[ -f "$BACKPORT" ]
}
# ── the freeze condition ─────────────────────────────────────────────────────
@@ -204,6 +179,39 @@ step_line() {
[ "${count:-0}" -eq 1 ]
}
@test "initial line build dispatch has Actions access and preserves failure recovery" {
step="$(STEP_NAME="Build the frozen line's packages artifact" yq -o=json '.jobs.cut.steps[] | select(.name == strenv(STEP_NAME))' "$CUT")"
script="$(printf '%s\n' "$step" | yq -r '.run')"
token="$(printf '%s\n' "$step" | yq -r '.env.GH_TOKEN')"
[ -n "$script" ] && [ "$script" != null ]
case "$token" in
'${{ github.token }}'|'${{ secrets.GITHUB_TOKEN }}') token=workflow-token ;;
'${{ steps.app-token.outputs.token }}') token=app-token ;;
*) echo "Unknown dispatch credential: $token"; return 1 ;;
esac
actions="$(yq -r '(.jobs.cut.permissions // .permissions).actions // "none"' "$CUT")"
# The installed CI app has no Actions permission; the workflow token only
# gets it when the effective job permissions request it.
gh() {
[ "$*" = 'workflow run build-release.yaml --ref release-1.7' ] || return 2
[ "$GH_TOKEN" = workflow-token ] && [ "$ACTIONS_PERMISSION" = write ] || return 1
return "$DISPATCH_STATUS"
}
export -f gh
run env GH_TOKEN="$token" ACTIONS_PERMISSION="$actions" BRANCH=release-1.7 \
DISPATCH_STATUS=0 bash -e -c "$script"
[ "$status" -eq 0 ]
[[ "$output" == *'Dispatched build-release.yaml for release-1.7;'* ]]
run env GH_TOKEN="$token" ACTIONS_PERMISSION="$actions" BRANCH=release-1.7 \
DISPATCH_STATUS=1 bash -e -c "$script"
[ "$status" -eq 0 ]
[[ "$output" == *'::warning::Could not dispatch build-release.yaml for release-1.7'* ]]
[[ "$output" == *'Start it by hand: gh workflow run build-release.yaml --ref release-1.7'* ]]
}
@test "build-release refuses a dispatch from anything but a line branch" {
BUILD_RELEASE="$REPO_ROOT/.github/workflows/build-release.yaml"
[ -f "$BUILD_RELEASE" ]
@@ -241,390 +249,3 @@ step_line() {
[ -n "$guard" ] && [ -n "$checkout" ]
[ "$guard" -lt "$checkout" ]
}
# ── backport target resolution ───────────────────────────────────────────────
# Both assertions below run over script_lines: this logic sits in a
# github-script block whose comments discuss getLatestRelease and min-1 by name.
@test "backport targets come from enumerated branches, not the latest release" {
block="$(job_block prepare "$BACKPORT")"
[ -n "$block" ]
# Enumerate real branches. getLatestRelease returns the newest published
# STABLE, which throughout a freeze window still names the PREVIOUS line — so
# a `backport` on a fix for the release being stabilised would land on the
# wrong branch and miss the release it was written for.
printf '%s\n' "$block" | script_lines | grep -qF 'github.paginate(github.rest.repos.listBranches'
count="$(printf '%s\n' "$block" | script_lines | grep -cF 'getLatestRelease' || true)"
[ "${count:-0}" -eq 0 ]
# Only branches matching release-<major>.<minor> qualify, so per-release
# staging branches (release-1.6.1) and rc branches never become targets.
printf '%s\n' "$block" | script_lines | grep -qF 'release-(\d+)\.(\d+)$'
# previous is the second-newest EXISTING line, never an arithmetic guess. The
# old min-1 derivation named a non-existent branch whenever a minor was
# skipped.
printf '%s\n' "$block" | script_lines | grep -qF 'lines.length > 1 ? lines[1].name'
count="$(printf '%s\n' "$block" | script_lines | grep -cE 'parseInt\(min\)[[:space:]]*-[[:space:]]*1|min[[:space:]]*-[[:space:]]*1' || true)"
[ "${count:-0}" -eq 0 ]
# Asking for a previous line when only one exists must fail, not silently
# backport nowhere.
printf '%s\n' "$block" | script_lines | grep -qF 'core.setFailed'
}
@test "release lines sort numerically descending so 1.10 outranks 1.9" {
block="$(job_block prepare "$BACKPORT")"
[ -n "$block" ]
# Numeric descending on major then minor is the ONLY reason release-1.10
# ranks above release-1.9. A slip to the default lexicographic sort compares
# the names as strings, puts "release-1.9" first, and aims every backport at
# a stale line — with nothing going red.
printf '%s\n' "$block" | script_lines | grep -qF '.sort((a, b) => (b.maj - a.maj) || (b.min - a.min))'
# The comparands must be numbers, not the captured strings.
printf '%s\n' "$block" | script_lines | grep -qF 'maj: Number(m[1]), min: Number(m[2])'
# And the head of that sort is what `backport` uses.
printf '%s\n' "$block" | script_lines | grep -qF 'lines[0].name'
}
# ── skip a target whose backport already merged ─────────────────────────────
# A second label event on a PR whose first backport already merged (e.g.
# backport-previous added after backport merged) re-enters this job for the
# already-delivered target. The cherry-pick is then empty, which the action's
# draft_commit_conflicts handling misreads as a real conflict and reports as a
# failed backport on the original PR — this guard skips the re-run instead.
@test "the guard checks merged, not just closed, PRs on the deterministic branch" {
block="$(job_block backport "$BACKPORT")"
[ -n "$block" ]
# The branch name the action itself would have used. Checking anything else
# answers a different question than "did THIS backport already land".
printf '%s\n' "$block" | script_lines | grep -qF 'backport-${context.payload.pull_request.number}-to-${targetBranch}'
# Closed alone is not enough: a closed-without-merge PR is a stale or
# abandoned attempt, not proof the change reached the target. Loosening this
# to any closed PR would skip a real retry after a conflict was closed
# unmerged.
printf '%s\n' "$block" | script_lines | grep -qF "state: 'closed'"
printf '%s\n' "$block" | script_lines | grep -qF 'p.merged_at !== null'
}
@test "both the checkout and the backport-action step are gated on the guard" {
block="$(job_block backport "$BACKPORT")"
[ -n "$block" ]
# Losing the gate on either step still runs the action against a target that
# already has the change, reproducing the false comment the guard exists to
# prevent.
count="$(printf '%s\n' "$block" | code_lines | grep -cF "steps.guard.outputs.already_merged != 'true'" || true)"
[ "${count:-0}" -eq 2 ]
}
@test "the guard runs before the checkout and the backport-action step" {
guard="$(step_line 'Check for existing merged backport' "$BACKPORT")"
checkout="$(step_line 'Checkout repository' "$BACKPORT")"
create="$(step_line 'Create back‑port PR' "$BACKPORT")"
[ -n "$guard" ] && [ -n "$checkout" ] && [ -n "$create" ]
[ "$guard" -lt "$checkout" ]
[ "$guard" -lt "$create" ]
}
# Mirrors release-changelog-behaviour.bats's convention for a github-script
# snippet: extract the decision expression verbatim in shape and execute it
# under node, because a grep for "merged_at" cannot tell an `.some()` from an
# `.every()` or a flipped comparison. Two cases are the ones that matter and
# are NOT interchangeable: the first-ever run (branch never existed, so the
# API returns an empty list) and a real retry (someone closed a conflicting
# backport PR without merging it). Both must NOT skip, or either the very
# first backport attempt or a legitimate re-creation after a manual close
# never runs.
@test "the guard's merged decision is correct on empty, closed-unmerged and closed-merged lists" {
command -v node >/dev/null || { echo "node unavailable; skipping"; return 0; }
tmp="$(mktemp -d)"
cat > "$tmp/guard.js" <<'JS'
function alreadyMerged(prs) {
return prs.some(p => p.merged_at !== null);
}
const cases = {
'first-run-empty': [],
'closed-unmerged': [{ merged_at: null }],
'closed-merged': [{ merged_at: '2026-01-01T00:00:00Z' }],
};
const lines = Object.entries(cases).map(([name, prs]) => name + '=' + alreadyMerged(prs));
require('fs').writeFileSync(1, lines.join('\n') + '\n');
JS
node "$tmp/guard.js" > "$tmp/out.txt" 2>"$tmp/err.txt" || {
echo "node failed:" >&2; cat "$tmp/err.txt" >&2; exit 1; }
want="first-run-empty=false
closed-unmerged=false
closed-merged=true"
got="$(cat "$tmp/out.txt")"
[ "$got" = "$want" ] || {
echo "guard decision table is wrong." >&2
echo "want:" >&2; printf '%s\n' "$want" | sed 's/^/ /' >&2
echo "got:" >&2; printf '%s\n' "$got" | sed 's/^/ /' >&2
exit 1
}
# The mirror must match the real guard's exact decision expression, not just
# the presence of "merged_at" somewhere in the step. Wrapped like the other
# tests in this file's final assertion: a bare failing command here, under a
# trap-installed EXIT handler, has been observed to abort the whole bats run
# instead of just this test on macOS.
block="$(job_block backport "$BACKPORT")"
[ -n "$block" ]
printf '%s\n' "$block" | script_lines | grep -qF 'prs.some(p => p.merged_at !== null)' || {
echo "backport.yaml's guard no longer uses prs.some(p => p.merged_at !== null)." >&2; exit 1; }
rm -rf "$tmp"
}
# ── backport concurrency ─────────────────────────────────────────────────────
# A run joins its concurrency group before `prepare`'s guard is evaluated, so
# nothing in that guard can keep a label event from disturbing the run already
# backporting the PR. The group key and `cancel-in-progress` close different
# halves of that and are pinned separately: either one alone still leaves a
# live failure, and neither reads load-bearing on its own.
@test "the trigger surface the concurrency key is built for does not widen" {
# Both halves of the key are written against one trigger delivering `closed` or
# `labeled`, and neither degrades safely if anything else arrives. The group key
# routes only `labeled` aside, so a new event lands in the main group;
# `cancel-in-progress` is positive, so it does not cancel there; and `prepare`
# does not qualify it. The result occupies the single pending slot, evicts a
# genuine backport request, and delivers nothing.
#
# Pinned as the whole surface rather than just the types list, because a SIBLING
# trigger reaches the same harm without touching `types:` at all: anything
# carrying a `pull_request` payload resolves the PR number and so lands in the
# same per-PR group. `pull_request_review` is the realistic one.
#
# Deliberately strict: this also fails on a widening that would be harmless,
# `workflow_dispatch` for instance, whose payload carries no `pull_request` so
# the key degenerates to a shared group holding no genuine request. Separating
# the harmless case needs a list of which triggers carry that payload, and that
# list drifts. A red here means read the block above, not that the pin is wrong.
triggers="$(awk '/^on:/{inside=1;next} /^[a-z]/{inside=0} inside && /^ [a-z_]+:/{print $1}' "$BACKPORT")"
[ -n "$triggers" ]
[ "$(printf '%s\n' "$triggers" | wc -l | tr -d ' ')" -eq 1 ]
printf '%s\n' "$triggers" | grep -qF 'pull_request_target:'
printf '%s\n' "$(code_lines < "$BACKPORT")" | grep -qF 'types: [closed, labeled]'
}
@test "a label event that requests no backport gets its own concurrency group" {
line="$(code_lines < "$BACKPORT" | grep '^ group: backport-')"
[ -n "$line" ]
# The per-PR discriminator, first: without it every PR's backport shares one
# group, and since a merge still cancels in that group, merging one PR kills
# another's in-flight backport. The assertions below all describe how a label
# event is separated from a merge WITHIN one PR, and every one of them stays
# green with the PR number deleted, so the widest failure of the three is the
# one nothing else here would notice.
printf '%s\n' "$line" | grep -qF 'backport-${{ github.workflow }}-${{ github.event.pull_request.number }}${{'
count="$(printf '%s\n' "$line" | grep -o "github\.event\.action == 'labeled'" | wc -l | tr -d ' ')"
[ "${count:-0}" -eq 1 ]
# The split is on the label NAMES, not on the action alone. Routing EVERY label
# event aside would move `backport` and `backport-previous` out of the group
# holding the run they have to queue behind, and one request could then evict
# the other.
#
# Pinned as the literal names, not as a count of `label.name != '`. That count
# measures the operator and says nothing about the operands, so renaming one
# name here and not in `prepare` — the half-finished rename — left it at two
# and stayed green, while a real `backport-previous` event took the `-label`
# branch and became free to cherry-pick alongside the merge run.
# Including the conjunction that joins the action check to the name checks.
# `&&` binds tighter than `||` in GitHub expressions, so flipping this one
# operator reads as `action == 'labeled' || (name != … && name != …)`, and on a
# `closed` event `github.event.label` is absent, both name checks are true, and
# EVERY event takes the suffix. A constant suffix is exactly as vacuous as no
# split at all. Every operand here was pinned before this line existed; the
# operator joining them was not.
#
# TRANSITIONAL: four names, not the original pair, because backport.yaml
# accepts both spellings while the org-level dosubot still applies the legacy
# ones. The namespaced names are genuine backport requests, so they belong on
# this side of the split exactly as the legacy ones do — routing them aside
# would hand a `kind/backport` event its own group and let it cherry-pick
# alongside the merge run, which is the failure the pin above describes. When
# dosubot's PR labelling is switched off and backport.yaml drops its legacy
# arms, this literal and the count below go back to the pair.
printf '%s\n' "$line" | grep -qF "github.event.action == 'labeled' && github.event.label.name != 'backport' && github.event.label.name != 'backport-previous' && github.event.label.name != 'kind/backport' && github.event.label.name != 'kind/backport-previous'"
# And exactly four of them. The literal above is a substring check, so appending
# a fifth exclusion satisfies it unchanged — and a fifth exclusion is not
# cosmetic: the named label stops being routed aside, lands in the main group,
# takes its single pending slot, skips in `prepare`, and evicts a genuine
# backport request while delivering nothing. Same harm as widening `types:`.
# Presence and count answer different questions; this test needs both.
count="$(printf '%s\n' "$line" | grep -o "github\.event\.label\.name != '" | wc -l | tr -d ' ')"
[ "${count:-0}" -eq 4 ]
# The suffix the condition selects, not only the condition. Asserting the
# operands alone leaves the whole split deletable — collapsing the tail to
# `&& '' || ''` puts every label back in the main group with the condition
# still reading correctly above it. Inverting it to `&& '' || '-label'` is
# worse than deleting it, because that moves the two backport requests aside
# and leaves the irrelevant labels sharing the group with the run in flight,
# which is the arrangement this file exists to prevent. Both survive every
# other assertion here.
printf '%s\n' "$line" | grep -qF "&& '-label' || ''"
}
@test "a labeled event queues behind the backport in flight instead of cancelling it" {
# Labels set with the default GITHUB_TOKEN start no run, but labels set by
# third-party GitHub Apps do, and they arrive in bursts. Under a plain `true`
# each one killed the backport the merge had just started, leaving a cancelled
# run next to a green one and the work repeated by whichever label came last.
# With `prepare` now narrowed, no burst label requalifies to repeat it,
# so dropping this arm does not restore the old noisy delivery — it loses the
# backport. `cancel-in-progress: true` is the obvious thing for the next reader
# to normalise this back to, so pin its absence too.
count="$(code_lines < "$BACKPORT" | grep -cF " cancel-in-progress: \${{ github.event.action == 'closed' }}" || true)"
[ "${count:-0}" -eq 1 ]
# Positively, not as `!= 'labeled'`. The two are equivalent only while `types:`
# holds exactly `closed` and `labeled`; the negative form hands cancellation to
# any third type added later, in the main group, which is the behaviour this
# arm removes.
count="$(code_lines < "$BACKPORT" | grep -c "cancel-in-progress:.*!= 'labeled'" || true)"
[ "${count:-0}" -eq 0 ]
count="$(code_lines < "$BACKPORT" | grep -c '^ cancel-in-progress: true$' || true)"
[ "${count:-0}" -eq 0 ]
}
@test "both backport jobs bound how long a queued request can wait" {
# These ceilings exist because of the queuing above, not for their own sake. A
# cancelling key disposed of a stuck run by killing it; queuing makes the next
# genuine request wait behind it instead, and an unbounded job on the 6-hour
# default turns one wedged job into a six-hour hole in the release line.
# Nothing else in this file makes them look load-bearing, so a tidy-up that
# drops them reads as harmless.
#
# Scope, since the test name says "how long a queued request can wait": what is
# bounded here is a stuck JOB. A run can wedge at the RUN level with every job
# already finished, and `timeout-minutes` is job-level with no run-level
# equivalent, so that state holds the group until the run is cancelled. These
# ceilings do not reach it and are not meant to.
# Asserted as a bound, not as presence. The hazard is the SIZE of the wait, so
# a pin that only checks the key exists is satisfied by `timeout-minutes: 355`,
# which restores the hole to within five minutes of the default it was written
# against.
#
# Bounded on BOTH sides, because both directions break it. Too high restores
# the hole; too low cancels a real backport that was only waiting for a runner,
# and an upper-bound-only assertion stays green at `timeout-minutes: 1`.
#
# The floor is judgement, not measurement, and says so. `timeout-minutes`
# bounds execution, and every execution measured on this workflow has been
# seconds, so no observation argues for any particular floor. No maximum is
# quoted deliberately: a superlative over a growing set goes stale on the next
# slower run. What the floor guards is a ceiling set low enough to cancel a
# cherry-pick much larger than anything seen yet; ten minutes is far above
# every leg measured. Per JOB, not per run, and not over the runner queue,
# which no `timeout-minutes` reaches.
for job in prepare backport; do
block="$(job_block "$job" "$BACKPORT" | code_lines)"
[ -n "$block" ]
minutes="$(printf '%s\n' "$block" | grep -o '^ timeout-minutes: [0-9][0-9]*' | grep -o '[0-9][0-9]*$')"
[ -n "$minutes" ]
[ "$minutes" -ge 10 ]
[ "$minutes" -le 30 ]
done
}
@test "each backport trigger reads only the labels it is entitled to" {
block="$(job_block prepare "$BACKPORT" | code_lines)"
[ -n "$block" ]
# The cumulative label set answers for the merge and for nothing else. Left
# ungated it re-enters this job on every later unrelated label — an automated
# size/* or kind/* — for a merged PR still carrying `backport`, redoing a
# backport that has already been delivered.
#
# TRANSITIONAL: four reads, not the original pair. backport.yaml accepts the
# namespaced and the legacy spelling of each label while the org-level dosubot
# still applies the legacy ones; when its PR labelling is switched off and the
# legacy arms are dropped, this literal and the count below go back to two.
printf '%s\n' "$block" | grep -qF "(github.event.action == 'closed' && (contains(github.event.pull_request.labels.*.name, 'backport') || contains(github.event.pull_request.labels.*.name, 'backport-previous') || contains(github.event.pull_request.labels.*.name, 'kind/backport') || contains(github.event.pull_request.labels.*.name, 'kind/backport-previous')))"
# And the guard reads that set exactly four times, all of them inside the gated
# disjunct above, so it cannot be joined by an ungated one that quietly
# restores the old behaviour while the assertion above stays green.
#
# Counted as OCCURRENCES, not lines. `grep -c` counts matching lines, and all
# four reads live on one line here, so it answers 1 and keeps answering 1 when a
# fifth read is appended to that same line — which is exactly the restoration
# this is meant to catch. Only an append on a NEW line would have moved it.
count="$(printf '%s\n' "$block" | grep -o 'contains(github\.event\.pull_request\.labels' | wc -l | tr -d ' ')"
[ "${count:-0}" -eq 4 ]
# A label event answers for the label it carries. Both spellings of each name,
# for the transitional reason given above.
printf '%s\n' "$block" | grep -qF "(github.event.action == 'labeled' && (github.event.label.name == 'backport' || github.event.label.name == 'backport-previous' || github.event.label.name == 'kind/backport' || github.event.label.name == 'kind/backport-previous'))"
# The two conjuncts the label logic hangs off, which the label assertions above
# would not miss. `base.ref == 'main'` is what stops the bot backporting its own
# output: its PRs target release-X.Y, so they cannot satisfy it however often a
# labeler re-applies `backport` to a `[Backport release-X.Y]` title. docs/release.md
# calls that architectural protection, which is only true while the line is here.
# With their trailing conjunctions, for the reason the group-key test spells
# out: flipping either `&&` to `||` turns the guard into a disjunction that
# fires on every merge to main and on labels applied to open PRs. That one does
# not produce a wrong backport, because the `backport` job re-checks both
# independently, but it runs the job when nothing asked for it.
# In BOTH jobs, because docs/release.md rests the architectural protection on
# both `if:` blocks. Losing either copy alone is survivable (the other still
# skips, and `backport` needs `prepare`), so this pins a duplicate rather than
# a hole — but an untested duplicate is how a duplicate stops being one.
for job in prepare backport; do
b="$(job_block "$job" "$BACKPORT" | code_lines)"
[ -n "$b" ]
printf '%s\n' "$b" | grep -qF "github.event.pull_request.base.ref == 'main' &&"
printf '%s\n' "$b" | grep -qF 'github.event.pull_request.merged == true &&'
done
}
@test "the audit accepts exactly the backport labels the bot acts on" {
# cmd/backport-audit answers "has everything labelled for this line landed on
# it" and exits non-zero while anything has not, so docs/release.md gates a
# patch cut on it. It reaches its candidates with `gh pr list --label`, which
# matches a label by exact name.
#
# Pin the two sets equal rather than pinning either one's contents: the
# audit must not miss a spelling the bot honours, and must not keep one the
# bot has retired. Both directions matter, because the legacy names are
# transitional and the two files have to drop them together.
[ -f "$AUDIT" ]
# Every name the bot tests for, from the label-detection script.
bot="$(grep -o "labels\.includes('[^']*')" "$BACKPORT" | sed "s/.*('//;s/')//" | sort -u)"
[ -n "$bot" ]
# Every backport label spelled as a Go string literal in the audit. Its own
# prose quotes these names with backticks, not double quotes, so a comment
# cannot satisfy this.
audit="$(grep -o '"\(kind/\)\{0,1\}backport\(-previous\)\{0,1\}"' "$AUDIT" | tr -d '"' | sort -u)"
[ -n "$audit" ]
[ "$bot" = "$audit" ]
# Both requests are represented, so the sets cannot agree by both having lost
# the same half.
printf '%s\n' "$audit" | grep -qx 'kind/backport'
printf '%s\n' "$audit" | grep -qx 'kind/backport-previous'
}
+6 -5
View File
@@ -1464,9 +1464,10 @@ _kind_creators() {
# What a creator can be written in, decided here rather than left to whatever
# the root happens to contain. The rule is pre-emptive: a manifest quoted in
# prose creates nothing, and a grep over every file reds on an edit to it.
# The shape exists in this repository -- docs/oidc-tenant.md carries the kind
# in a fenced example -- outside this root rather than in it, which is why
# the root is a choice about place and this is a choice about form. A parked
# The shape exists in this repository -- hack/testdata/install-wait/
# creators-filetypes/example.md carries the kind in a fenced example --
# outside this root rather than in it, which is why the root is a choice
# about place and this is a choice about form. A parked
# `.disabled` suite creates nothing either; re-enabling one restores the
# `.yaml` name the sweep reads, so that exclusion cannot outlive the parking.
#
@@ -1561,8 +1562,8 @@ EOF
#
# The sweep is hack/e2e-chainsaw only, not the repository. Widening it would
# need an exclusion for hack/testdata/, whose fixtures carry the kind on
# purpose, and for docs/, where docs/oidc-tenant.md carries it inside a
# fenced example; neither creates a cluster. What the narrow root leaves
# purpose, and for any markdown that quotes the manifest in a fenced
# example; neither creates a cluster. What the narrow root leaves
# outside is not only those: `hack/e2e-apps/` is a live lane of its own, and a
# creator appearing there would pass this case in silence -- nothing there
# creates this kind today, so it is a risk the root carries rather than a
+283 -12
View File
@@ -1,6 +1,6 @@
#!/usr/bin/env bats
# -----------------------------------------------------------------------------
# Unit tests for cozy_guard_addon_helmreleases in
# Unit tests for cozy_guard_all_helmreleases and cozy_guard_addon_helmreleases in
# hack/e2e-chainsaw/_lib/run-kubernetes.sh
#
# cozy_guard_helmrelease judges one release; cozy_guard_addon_helmreleases runs
@@ -32,9 +32,11 @@ load test_helper
# created. The stub answers the shapes the guard issues: the namespace listing
# (from ALL_HR), one release's install strategy and Ready condition together
# (<name>.strategy and <name>.ready, joined the way the guard's jsonpath joins
# them), and its history statuses (<name>.history). A missing fixture file
# stands for a field the object does not carry, which is what kubectl prints for
# an absent jsonpath - nothing.
# them), its history statuses (<name>.history), and whether it exists at all
# (`--ignore-not-found -o name`, which a <name>.ABSENT marker turns into empty
# output, and whose read the markers below take as <name>.exists). A missing
# fixture file stands for a field the object does not carry, which is what
# kubectl prints for an absent jsonpath - nothing.
#
# A FAILS marker file makes the matching read exit non-zero with a message on
# stderr, which is how the API answers a timeout or a denied request. That
@@ -96,6 +98,7 @@ for a in "$@"; do
describe) mode=describe ;;
*'status.history'*) mode=history ;;
*'install.strategy.name'*) mode=probe ;;
name) mode=exists ;;
kubernetes-*) name=$a ;;
esac
done
@@ -103,6 +106,7 @@ case "$mode" in
list) markers=LIST ;;
history) markers="$name.history" ;;
probe) markers="$name.strategy $name.ready" ;;
exists) markers="$name.exists" ;;
*) markers= ;;
esac
for m in $markers; do
@@ -145,6 +149,20 @@ case "$mode" in
"$(cat "$FIXTURES/$name.strategy" 2>/dev/null || true)" \
"$(cat "$FIXTURES/$name.ready" 2>/dev/null || true)"
;;
exists)
# `get hr <name> --ignore-not-found -o name`: an absent release prints
# nothing and exits 0, which an ABSENT marker reproduces; anything else
# exists. Without the flag kubectl answers NotFound with a non-zero exit,
# so a read that drops it is told apart here the way the cluster would.
if [ -f "$FIXTURES/$name.ABSENT" ]; then
case " $* " in
*' --ignore-not-found '*) exit 0 ;;
esac
echo "Error from server (NotFound): helmreleases.helm.toolkit.fluxcd.io \"$name\" not found" >&2
exit 1
fi
echo "helmrelease.helm.toolkit.fluxcd.io/$name"
;;
describe) echo "stub describe of $name" ;;
esac
exit 0
@@ -252,6 +270,245 @@ STUB
rm -rf "$tmp"
}
@test "the pool release is read by the full guard, and its footprint fails the run" {
# The worker pool is kubernetes-nodes-<test>-md0, an Application HelmRelease
# the apiserver stamps like the parent, on the node-join path through the
# talos-reconcile Job it renders. Its name does not start with the addon
# prefix kubernetes-<test>-, so the addon listing never selects it, and
# before this read a failed action on it was invisible to the guard on every
# path. Parent and addon are clean here; the pool alone carries the finding.
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
cozy_make_kubectl_stub "$tmp"
export FIXTURES="$tmp/fixtures"
printf 'kubernetes-test-latest-version-coredns\n' > "$FIXTURES/ALL_HR"
printf 'RetryOnFailure' > "$FIXTURES/kubernetes-test-latest-version.strategy"
printf 'deployed\n' > "$FIXTURES/kubernetes-test-latest-version.history"
printf 'deployed\n' > "$FIXTURES/kubernetes-test-latest-version-coredns.history"
printf 'RetryOnFailure' > "$FIXTURES/kubernetes-nodes-test-latest-version-md0.strategy"
printf 'failed\ndeployed\n' > "$FIXTURES/kubernetes-nodes-test-latest-version-md0.history"
rc=0
out=$(PATH="$tmp/bin:$PATH" cozy_guard_all_helmreleases tenant-test kubernetes-test-latest-version 2>&1) || rc=$?
if [ "$rc" -eq 0 ]; then
echo "expected the pool release footprint to fail the run, got: $out" >&2
exit 1
fi
if ! printf '%s\n' "$out" | grep -q -F 'HelmRelease kubernetes-nodes-test-latest-version-md0 carries a failed or uninstalled Snapshot, and its install strategy is RetryOnFailure'; then
echo "expected the pool release to be named in the verdict, got: $out" >&2
exit 1
fi
if ! grep -q -F 'kubernetes-nodes-test-latest-version-md0' "$FIXTURES/CALLS"; then
echo "expected a read of the pool release, calls were: $(cat "$FIXTURES/CALLS")" >&2
exit 1
fi
rm -rf "$tmp"
}
@test "a clean pool release leaves the full guard green" {
# The pin for the passing path: with every release clean the composite
# returns zero, so the pool read above is a read and not an unconditional
# verdict. The pool name is derived from the parent, the way the addon
# prefix is, which this fixture spells out.
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
cozy_make_kubectl_stub "$tmp"
export FIXTURES="$tmp/fixtures"
printf 'kubernetes-test-latest-version-coredns\n' > "$FIXTURES/ALL_HR"
printf 'RetryOnFailure' > "$FIXTURES/kubernetes-test-latest-version.strategy"
printf 'deployed\n' > "$FIXTURES/kubernetes-test-latest-version.history"
printf 'deployed\n' > "$FIXTURES/kubernetes-test-latest-version-coredns.history"
printf 'RetryOnFailure' > "$FIXTURES/kubernetes-nodes-test-latest-version-md0.strategy"
printf 'deployed\n' > "$FIXTURES/kubernetes-nodes-test-latest-version-md0.history"
PATH="$tmp/bin:$PATH" cozy_guard_all_helmreleases tenant-test kubernetes-test-latest-version
[ "$(cozy_pool_helmrelease_name kubernetes-test-latest-version)" = kubernetes-nodes-test-latest-version-md0 ]
rm -rf "$tmp"
}
@test "the pool release is held to a stamped install strategy like the parent" {
# The pool is an Application HelmRelease, so an empty strategy there is the
# read and never the object, the same as on the parent. Without this the
# jsonpath rename the parent check exists for would leave the pool's
# footprint on the lenient branch while the parent's check still passed.
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
cozy_make_kubectl_stub "$tmp"
export FIXTURES="$tmp/fixtures"
printf 'kubernetes-test-latest-version-coredns\n' > "$FIXTURES/ALL_HR"
printf 'RetryOnFailure' > "$FIXTURES/kubernetes-test-latest-version.strategy"
printf 'deployed\n' > "$FIXTURES/kubernetes-test-latest-version.history"
printf 'deployed\n' > "$FIXTURES/kubernetes-test-latest-version-coredns.history"
printf 'True' > "$FIXTURES/kubernetes-nodes-test-latest-version-md0.ready"
printf 'deployed\n' > "$FIXTURES/kubernetes-nodes-test-latest-version-md0.history"
rc=0
out=$(PATH="$tmp/bin:$PATH" cozy_guard_all_helmreleases tenant-test kubernetes-test-latest-version 2>&1) || rc=$?
if [ "$rc" -eq 0 ]; then
echo "expected an empty install strategy on the pool release to fail the run, got: $out" >&2
exit 1
fi
if ! printf '%s\n' "$out" | grep -q -F 'spec.install.strategy.name of kubernetes-nodes-test-latest-version-md0 came back empty'; then
echo "expected the failure to name the pool release, got: $out" >&2
exit 1
fi
rm -rf "$tmp"
}
@test "the failure-path report classifies the pool release too" {
# Same reading as the passing path, from the exit handler: a pool release
# that failed an action is named, and the report still returns zero.
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
cozy_make_kubectl_stub "$tmp"
export FIXTURES="$tmp/fixtures"
printf 'kubernetes-test-latest-version-coredns\n' > "$FIXTURES/ALL_HR"
printf 'RetryOnFailure' > "$FIXTURES/kubernetes-test-latest-version.strategy"
printf 'deployed\n' > "$FIXTURES/kubernetes-test-latest-version.history"
printf 'deployed\n' > "$FIXTURES/kubernetes-test-latest-version-coredns.history"
printf 'RetryOnFailure' > "$FIXTURES/kubernetes-nodes-test-latest-version-md0.strategy"
printf 'failed\ndeployed\n' > "$FIXTURES/kubernetes-nodes-test-latest-version-md0.history"
rc=0
out=$(PATH="$tmp/bin:$PATH" cozy_report_helmrelease_remediation tenant-test kubernetes-test-latest-version 2>&1) || rc=$?
if [ "$rc" -ne 0 ]; then
echo "expected the report to return zero, got rc=$rc: $out" >&2
exit 1
fi
if ! printf '%s\n' "$out" | grep -q -F 'HelmRelease kubernetes-nodes-test-latest-version-md0 carries a failed or uninstalled Snapshot'; then
echo "expected the pool release footprint to be classified, got: $out" >&2
exit 1
fi
rm -rf "$tmp"
}
@test "the failure-path report says when the pool release was never created" {
# The pool is applied after the parent, and the exit handler is armed between
# the two, so a run whose pool apply failed reaches the report with no pool
# release. That is a fact about where the run stopped, and the report says so
# instead of reporting a failed read of a missing object.
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
cozy_make_kubectl_stub "$tmp"
export FIXTURES="$tmp/fixtures"
printf 'kubernetes-test-latest-version-coredns\n' > "$FIXTURES/ALL_HR"
printf 'RetryOnFailure' > "$FIXTURES/kubernetes-test-latest-version.strategy"
printf 'deployed\n' > "$FIXTURES/kubernetes-test-latest-version.history"
printf 'deployed\n' > "$FIXTURES/kubernetes-test-latest-version-coredns.history"
: > "$FIXTURES/kubernetes-nodes-test-latest-version-md0.ABSENT"
rc=0
out=$(PATH="$tmp/bin:$PATH" cozy_report_helmrelease_remediation tenant-test kubernetes-test-latest-version 2>&1) || rc=$?
if [ "$rc" -ne 0 ]; then
echo "expected the report to return zero, got rc=$rc: $out" >&2
exit 1
fi
if ! printf '%s\n' "$out" | grep -q -F 'kubernetes-nodes-test-latest-version-md0 does not exist'; then
echo "expected the report to say the pool release does not exist, got: $out" >&2
exit 1
fi
if printf '%s\n' "$out" | grep -q -F 'Reading .spec.install.strategy.name and the Ready condition of kubernetes-nodes-test-latest-version-md0 failed'; then
echo "expected no failed read of the absent pool release, got: $out" >&2
exit 1
fi
rm -rf "$tmp"
}
@test "a failed existence read of the pool release is not reported as absence" {
# The report runs on the failure path, where a slow apiserver is most
# likely. A cut-off existence read said "does not exist" there, which sends
# the reader to the wrong stage of the run; it has to be named as the failed
# read it is, the way the parent's and the listing's are, and the walk goes
# on to the addons.
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
cozy_make_kubectl_stub "$tmp"
export FIXTURES="$tmp/fixtures"
printf 'kubernetes-test-latest-version-coredns\n' > "$FIXTURES/ALL_HR"
printf 'RetryOnFailure' > "$FIXTURES/kubernetes-test-latest-version.strategy"
printf 'deployed\n' > "$FIXTURES/kubernetes-test-latest-version.history"
printf 'deployed\n' > "$FIXTURES/kubernetes-test-latest-version-coredns.history"
: > "$FIXTURES/kubernetes-nodes-test-latest-version-md0.exists.CUTOFF"
rc=0
out=$(PATH="$tmp/bin:$PATH" cozy_report_helmrelease_remediation tenant-test kubernetes-test-latest-version 2>&1) || rc=$?
if [ "$rc" -ne 0 ]; then
echo "expected the report to return zero, got rc=$rc: $out" >&2
exit 1
fi
if ! printf '%s\n' "$out" | grep -q -F 'Checking whether kubernetes-nodes-test-latest-version-md0 exists failed - the read did not finish'; then
echo "expected the cut-off existence read to be named as a failed read, got: $out" >&2
exit 1
fi
if printf '%s\n' "$out" | grep -q -F 'kubernetes-nodes-test-latest-version-md0 does not exist'; then
echo "expected a failed read not to be reported as absence, got: $out" >&2
exit 1
fi
if grep -F 'kubernetes-nodes-test-latest-version-md0' "$FIXTURES/CALLS" | grep -q -v -F -e '-o name'; then
echo "expected no classification read of a release whose existence is unknown, calls were: $(cat "$FIXTURES/CALLS")" >&2
exit 1
fi
if ! printf '%s\n' "$out" | grep -q 'HelmRelease kubernetes-test-latest-version-coredns '; then
echo "expected the addons to be read after the failed pool read, got: $out" >&2
exit 1
fi
rm -rf "$tmp"
}
@test "the failure-path report does not read the pool release past its budget" {
# The parent's reads are issued before the first clock check, so on a path
# that opens no phase they can spend the whole budget on their own. The pool
# release is three more bounded reads, and starting them past the deadline
# takes them out of the tenant snapshot's time.
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
cozy_make_kubectl_stub "$tmp"
export FIXTURES="$tmp/fixtures"
printf 'kubernetes-test-latest-version-coredns\n' > "$FIXTURES/ALL_HR"
printf 'RetryOnFailure' > "$FIXTURES/kubernetes-test-latest-version.strategy"
printf 'deployed\n' > "$FIXTURES/kubernetes-test-latest-version.history"
printf 'RetryOnFailure' > "$FIXTURES/kubernetes-nodes-test-latest-version-md0.strategy"
printf 'failed\ndeployed\n' > "$FIXTURES/kubernetes-nodes-test-latest-version-md0.history"
# The deadline is stamped at 1000+90 and the pool release is checked one
# reading later, at 1100.
date_ticks="$tmp/ticks"
printf '0' > "$date_ticks"
date_now=1000
date_step=100
COZY_GUARD_REPORT_BUDGET=90
rc=0
out=$(PATH="$tmp/bin:$PATH" cozy_report_helmrelease_remediation tenant-test kubernetes-test-latest-version 2>&1) || rc=$?
date_now=
date_step=0
date_ticks=
unset COZY_GUARD_REPORT_BUDGET
if [ "$rc" -ne 0 ]; then
echo "expected the report to return zero, got rc=$rc: $out" >&2
exit 1
fi
if ! printf '%s\n' "$out" | grep -q 'HelmRelease kubernetes-test-latest-version (install strategy'; then
echo "expected the parent to have been classified before the budget ran out, got: $out" >&2
exit 1
fi
if grep -q -F 'kubernetes-nodes-test-latest-version-md0' "$FIXTURES/CALLS"; then
echo "expected no read of the pool release once the budget was spent, calls were: $(cat "$FIXTURES/CALLS")" >&2
exit 1
fi
if ! printf '%s\n' "$out" | grep -q -F 'stopped after 90s; kubernetes-nodes-test-latest-version-md0 and the addon releases of kubernetes-test-latest-version were not read'; then
echo "expected the skipped pool release to be named, got: $out" >&2
exit 1
fi
rm -rf "$tmp"
}
@test "single-release guard fails a RetryOnFailure release that failed and recovered" {
# The entry point the parent kubernetes-<test> release goes through, and the
# shape the rule exists for. The apiserver stamps RetryOnFailure on every
@@ -1110,14 +1367,15 @@ STUB
printf 'deployed\n' > "$FIXTURES/kubernetes-test-latest-version.history"
printf 'deployed\n' > "$FIXTURES/kubernetes-test-latest-version-cilium.history"
printf 'deployed\n' > "$FIXTURES/kubernetes-test-latest-version-coredns.history"
# 30s per reading against a 90s budget: the report stamps its deadline at
# 1000+90, the listing is checked at 1030, the first addon at 1060 and the
# second at 1090, which is the first reading the deadline refuses.
# 30s per reading against a 120s budget: the report stamps its deadline at
# 1000+120, the pool release is checked at 1030, the listing at 1060, the
# first addon at 1090 and the second at 1120, which is the first reading
# the deadline refuses.
date_ticks="$tmp/ticks"
printf '0' > "$date_ticks"
date_now=1000
date_step=30
COZY_GUARD_REPORT_BUDGET=90
COZY_GUARD_REPORT_BUDGET=120
rc=0
out=$(PATH="$tmp/bin:$PATH" cozy_report_helmrelease_remediation tenant-test kubernetes-test-latest-version 2>&1) || rc=$?
@@ -1137,7 +1395,7 @@ STUB
echo "expected the second addon to fall outside the budget, got: $out" >&2
exit 1
fi
if ! printf '%s\n' "$out" | grep -q -F 'stopped after 90s; kubernetes-test-latest-version-coredns and every release after it were not read'; then
if ! printf '%s\n' "$out" | grep -q -F 'stopped after 120s; kubernetes-test-latest-version-coredns and every release after it were not read'; then
echo "expected the cut-off release to be named, got: $out" >&2
exit 1
fi
@@ -1227,6 +1485,19 @@ $out" >&2
echo "expected the handler armed (line $arm) before the control-plane wait (line $cp_wait)" >&2
exit 1
fi
# And before the pool apply, which follows the parent's. A failing pool
# apply exits under errexit, so armed after it that run leaves the parent
# unclassified, and the report's absent-pool line describes a run that never
# reaches it.
pool_apply=$(grep -n '^kind: KubernetesNodes$' "$lib" | head -n 1 | cut -d: -f1)
if [ -z "$pool_apply" ]; then
echo "expected to find the KubernetesNodes apply in $lib" >&2
exit 1
fi
if [ "$arm" -ge "$pool_apply" ]; then
echo "expected the handler armed (line $arm) before the pool apply (line $pool_apply)" >&2
exit 1
fi
# The parent name is part of this one, because it is what tells the
# node-join call apart from the handler's. What follows it is not: a
# section label is presentation and may change without the wiring changing.
@@ -1320,13 +1591,13 @@ $out" >&2
printf 'RetryOnFailure' > "$FIXTURES/kubernetes-test-latest-version.strategy"
printf 'deployed\n' > "$FIXTURES/kubernetes-test-latest-version.history"
printf 'uninstalled\ndeployed\n' > "$FIXTURES/kubernetes-test-latest-version-cilium.history"
# The deadline is stamped at 1000+90 and the listing is checked one reading
# later, at 1100.
# The deadline is stamped at 1000+150, the pool release is admitted at 1100
# and the listing is checked one reading later, at 1200.
date_ticks="$tmp/ticks"
printf '0' > "$date_ticks"
date_now=1000
date_step=100
COZY_GUARD_REPORT_BUDGET=90
COZY_GUARD_REPORT_BUDGET=150
rc=0
out=$(PATH="$tmp/bin:$PATH" cozy_report_helmrelease_remediation tenant-test kubernetes-test-latest-version 2>&1) || rc=$?
+2
View File
@@ -178,6 +178,8 @@ inert_config_pattern='^(examples/|\.github/|\.claude/|\.gemini/|img/|hack/testda
# and `-type f` would be too narrow. A chainsaw-test.y*ml counts whatever its
# type: Chainsaw stats it and fails loudly on a directory, and leaving that
# suite out would hide the failure instead.
# The leak guard's allowlist accepts a suite by the same rule, in
# lg_suite_has_test (hack/e2e-chainsaw/_lib/leak-guard.sh). Change both together.
#
# Captured before the sed/sort rather than piped straight into them: a pipeline
# carries its LAST command's status, so `$(find ... | sed | sort)` would report
+1 -1
View File
@@ -141,7 +141,7 @@ suite_to_source() {
# would also change reverse test selection.
suite_runtime_sources() {
case "$1" in
clickhouse|etcd|kafka|kafka-metadata|mariadb|mongodb|postgres|rabbitmq|redis|valkey)
clickhouse|etcd|kafka|kafka-metadata|mariadb|mongodb|postgres|rabbitmq|redis|valkey|vminstance)
echo cozystack.backupstrategy-controller ;;
gateway) echo cozystack.kubeovn-webhook ;;
esac
+1 -1
View File
@@ -177,7 +177,7 @@ assert_output_graph_error() {
}
@test "backup suites keep the APIs used by their roundtrip examples" {
for suite in clickhouse etcd kafka kafka-metadata mariadb mongodb postgres rabbitmq redis valkey; do
for suite in clickhouse etcd kafka kafka-metadata mariadb mongodb postgres rabbitmq redis valkey vminstance; do
keep=$(hack/select-install.sh "$suite")
drop=$(hack/select-install.sh --disabled "$suite")
for required in cozystack.backupstrategy-controller cozystack.backup-controller \
+2 -2
View File
@@ -161,13 +161,13 @@ cleanup() {
cleanup
}
@test "docs/release.md tells users to verify the identity the script verifies" {
@test "docs/release/signatures-and-sbom.md tells users to verify the identity the script verifies" {
regexp="$(sed -n "s/^IDENTITY_REGEXP='\(.*\)'\$/\1/p" "$SCRIPT")"
if [ -z "$regexp" ]; then
echo "IDENTITY_REGEXP not found in $SCRIPT" >&2
exit 1
fi
grep -qF -- "--certificate-identity-regexp '$regexp'" "$REPO_ROOT/docs/release.md"
grep -qF -- "--certificate-identity-regexp '$regexp'" "$REPO_ROOT/docs/release/signatures-and-sbom.md"
}
@test "fails when the signature does not verify" {
+4 -3
View File
@@ -26,8 +26,9 @@
# copies along with its other assets, and a stable release gets its own.
#
# The bundle is verified before the script returns, with the same certificate
# identity users are told to check in docs/release.md, so a release whose
# signature that check would reject fails here instead of being uploaded.
# identity users are told to check in docs/release/signatures-and-sbom.md, so a
# release whose signature that check would reject fails here instead of being
# uploaded.
#
# Needs syft and cosign on PATH, and `id-token: write` in the job.
set -euo pipefail
@@ -39,7 +40,7 @@ BUNDLE=cozystack-checksums.txt.sigstore.json
# The workflows that publish releases, at the refs they run from: tags.yaml for a
# pre-release, on a v* tag push, and promote-rc.yaml for a stable one, dispatched
# from main only. A run from any other ref signs, but fails the check below.
# docs/release.md shows users the same pattern.
# docs/release/signatures-and-sbom.md shows users the same pattern.
IDENTITY_REGEXP='^https://github\.com/cozystack/cozystack/\.github/workflows/(tags\.yaml@refs/tags/v|promote-rc\.yaml@refs/heads/main$)'
OIDC_ISSUER=https://token.actions.githubusercontent.com
+20 -5
View File
@@ -10,7 +10,9 @@
# image as <image-tag>-arm64
#
# The amd64 build and the arm64 leg run on separate native runners, so each
# pushes a single-platform image. For every ref <registry>/<repo>[:<t>]@A in
# pushes a single-platform image. Built with SBOM=1 that image is an index of
# itself and its attestation manifest, and the index made below carries both
# attestations. For every ref <registry>/<repo>[:<t>]@A in
# the tree, whatever tag <t> it carries, with <tag> the <image-tag>:
#
# 1. B is the digest of <repo>:<tag>-arm64. No such image is not an error:
@@ -109,10 +111,18 @@ manifest_digest() {
}
# image_arch <repo@digest> prints the architecture of a single-platform image,
# or nothing for an index or a non-image artifact. A registry error returns
# non-zero, like manifest_digest.
# or nothing for an index of several platforms or a non-image artifact. An
# image built with SBOM=1 is an index of one linux image and its attestation
# manifest, and counts as that image. A registry error returns non-zero, like
# manifest_digest.
image_arch() {
_ia_raw="$(sk inspect --raw "docker://$1")" || return 1
_ia_plat="$(printf '%s' "$_ia_raw" | jq -r '
if .manifests then
[.manifests[] | select(.platform.os == "linux") | .platform.architecture]
| if length == 1 then .[0] else empty end
else empty end')"
if [ -n "$_ia_plat" ]; then printf '%s\n' "$_ia_plat"; return 0; fi
printf '%s' "$_ia_raw" | jq -r '.config.mediaType // empty' \
| grep -qE 'image\.config|container\.image' || return 0
_ia_cfg="$(sk inspect --config "docker://$1")" || return 1
@@ -232,10 +242,15 @@ while read -r repo a reftags <&3; do
else
[ -n "$cur" ] || { echo "::error::${repo}:${tag} does not exist, though the tree pins it" >&2; exit 1; }
raw="$(sk inspect --raw "docker://${repo}@${cur}")"
# An A built with SBOM=1 is itself an index, and the stitched one lists its
# linux image rather than A.
a_raw="$(sk inspect --raw "docker://${repo}@${a}")"
a_members="$(printf '%s' "$a_raw" | jq -c --arg a "$a" \
'if .manifests then [.manifests[] | select(.platform.os == "linux") | .digest] else [$a] end')"
# Its arm64 half need not be this run's B: a rerun rebuilds the arm64
# image, which need not reproduce it byte for byte.
if ! printf '%s' "$raw" | jq -e --arg a "$a" \
'(.manifests // []) as $m | any($m[]; .digest == $a) and any($m[]; .platform.architecture == "arm64")' >/dev/null; then
if ! printf '%s' "$raw" | jq -e --argjson ms "$a_members" \
'(.manifests // []) as $m | ($ms | length) > 0 and all($ms[]; . as $d | any($m[]; .digest == $d)) and any($m[]; .platform.architecture == "arm64")' >/dev/null; then
echo "::error::${repo}:${tag} points at ${cur}, which is neither the digest the tree pins (${a}) nor an index of it and an arm64 image" >&2
exit 1
fi
+130 -11
View File
@@ -76,8 +76,15 @@ while [ "$#" -gt 0 ]; do
--tag) tags="$tags $2"; shift 2 ;;
*)
d="${1##*@}"
arch="$(jq -r .arch "$MOCK_REG/blobs/${d#sha256:}")"
entries="$entries${entries:+,}{\"digest\":\"$d\",\"platform\":{\"os\":\"linux\",\"architecture\":\"$arch\"}}"
f="$MOCK_REG/blobs/${d#sha256:}"
# A source index contributes its entries, attestation manifests included,
# as buildx does; a plain image contributes itself.
if jq -e '.manifests' "$f" >/dev/null 2>&1; then
entries="$entries${entries:+,}$(jq -r '[.manifests[] | tojson] | join(",")' "$f")"
else
arch="$(jq -r .arch "$f")"
entries="$entries${entries:+,}{\"digest\":\"$d\",\"platform\":{\"os\":\"linux\",\"architecture\":\"$arch\"}}"
fi
shift ;;
esac
done
@@ -105,6 +112,44 @@ _tag() {
awk -v r="$1" -v t="$2" '$1 == r && $2 == t { d = $3 } END { printf "%s", d }' "$MOCK_REG/tags"
}
# _push_sbom <repo> <tag> <arch>... stores the index an image built with SBOM=1
# pushes, one linux image per arch beside an attestation manifest, tags it, and
# prints its digest.
_push_sbom() {
_ps_repo="$1"
_ps_tag="$2"
shift 2
_ps_entries=""
for _ps_arch in "$@"; do
_ps_img="{\"mediaType\":\"application/vnd.oci.image.manifest.v1+json\",\"config\":{\"mediaType\":\"application/vnd.oci.image.config.v1+json\"},\"arch\":\"$_ps_arch\",\"name\":\"$_ps_repo:$_ps_tag\"}"
_ps_hex="$(printf '%s' "$_ps_img" | sha256sum | cut -d' ' -f1)"
printf '%s' "$_ps_img" >"$MOCK_REG/blobs/$_ps_hex"
_ps_entries="$_ps_entries${_ps_entries:+,}{\"digest\":\"sha256:$_ps_hex\",\"platform\":{\"os\":\"linux\",\"architecture\":\"$_ps_arch\"}}"
done
_ps_att="{\"mediaType\":\"application/vnd.oci.image.manifest.v1+json\",\"attestation\":true,\"name\":\"$_ps_repo:$_ps_tag\"}"
_ps_hex="$(printf '%s' "$_ps_att" | sha256sum | cut -d' ' -f1)"
printf '%s' "$_ps_att" >"$MOCK_REG/blobs/$_ps_hex"
_ps_entries="$_ps_entries,{\"digest\":\"sha256:$_ps_hex\",\"platform\":{\"os\":\"unknown\",\"architecture\":\"unknown\"},\"annotations\":{\"vnd.docker.reference.type\":\"attestation-manifest\"}}"
_ps_body="{\"mediaType\":\"application/vnd.oci.image.index.v1+json\",\"manifests\":[$_ps_entries]}"
_ps_hex="$(printf '%s' "$_ps_body" | sha256sum | cut -d' ' -f1)"
printf '%s' "$_ps_body" >"$MOCK_REG/blobs/$_ps_hex"
printf '%s %s sha256:%s\n' "$_ps_repo" "$_ps_tag" "$_ps_hex" >>"$MOCK_REG/tags"
printf 'sha256:%s' "$_ps_hex"
}
# A tree stamped by an amd64 build with SBOM=1: api pinned on the index of its
# amd64 image and attestation manifest, with the arm64 leg's twin. Sets A_api
# and B_api.
_make_sbom_world() {
mkdir -p "$1/tree/system/api"
A_api=$(_push_sbom "$REG/api" main amd64)
B_api=$(_push_sbom "$REG/api" main-arm64 arm64)
PUSHED_TAGS_LOG="$1/pushed-tags"
export PUSHED_TAGS_LOG
printf '%s/api:main\n' "$REG" >"$PUSHED_TAGS_LOG"
printf 'image: %s/api:main@%s\n' "$REG" "$A_api" >"$1/tree/system/api/values.yaml"
}
# A tree stamped by an amd64 build with IMAGE_TAG=main, one image per storage
# shape and values.yaml sub-shape, each with an arm64 twin, plus the refs that
# must be left alone. Sets A_<name> to each amd64 digest.
@@ -383,24 +428,98 @@ _make_world() {
[ "$(yq -r '.jobs.prepare-release.steps[] | select(.name == "Set up flux") | .with.version' "$wf")" = "$want" ]
}
@test "the amd64 release job rebuilds matchbox as a two-arch index before it captures the tree" {
# matchbox has no arm64 twin to stitch, so this rebuild is the only thing
# that makes the ref the gate checks an index. It needs a builder that can
# push an index while make build stays on the default docker driver.
@test "the amd64 release job builds every image on a docker-container builder, with an SBOM, before it captures the tree" {
# An SBOM attestation needs the docker-container driver, and so does the
# matchbox rebuild as a two-arch index: matchbox has no arm64 twin to stitch,
# so that rebuild is the only thing that makes the ref the gate checks an
# index.
wf=.github/workflows/tags.yaml
steps=$(yq -r '.jobs.build-amd64.steps[].name' "$wf")
pos() { echo "$steps" | grep -nxF "$1" | cut -d: -f1; }
bx=$(pos 'Set up Buildx for matchbox (docker-container driver)')
bx=$(pos 'Set up Buildx (docker-container driver, docker.io mirror)')
build=$(pos 'Build')
capture=$(pos 'Capture the stamped tree')
[ -n "$bx" ] && [ -n "$build" ] && [ -n "$capture" ]
[ "$bx" -lt "$build" ] && [ "$build" -lt "$capture" ]
sel='.jobs.build-amd64.steps[] | select(.id == "matchbox_buildx")'
sel='.jobs.build-amd64.steps[] | select(.id == "buildx")'
[ "$(yq -r "$sel | .with.driver" "$wf")" = docker-container ]
[ "$(yq -r "$sel | .with.\"buildkitd-config\"" "$wf")" = hack/buildkitd.toml ]
[ "$(yq -r "$sel | .with.use" "$wf")" = false ]
run_=$(yq -r '.jobs.build-amd64.steps[] | select(.name == "Build") | .run' "$wf")
echo "$run_" | grep -qxE '[[:space:]]*make -C packages/core/talos image-matchbox PLATFORM=linux/amd64,linux/arm64 BUILDER="\$MATCHBOX_BUILDER"'
[ "$(yq -r '.jobs.build-amd64.steps[] | select(.name == "Build") | .env.MATCHBOX_BUILDER' "$wf")" = '${{ steps.matchbox_buildx.outputs.name }}' ]
build_sel='.jobs.build-amd64.steps[] | select(.name == "Build")'
[ "$(yq -r "$build_sel | .env.BUILDER" "$wf")" = '${{ steps.buildx.outputs.name }}' ]
[ "$(yq -r "$build_sel | .env.SBOM" "$wf")" = 1 ]
run_=$(yq -r "$build_sel | .run" "$wf")
echo "$run_" | grep -qxE '[[:space:]]*make -C packages/core/talos image-matchbox PLATFORM=linux/amd64,linux/arm64'
}
@test "the arm64 release leg attaches an SBOM on its docker-container builder" {
wf=.github/workflows/tags.yaml
sel='.jobs.build-arm64.steps[] | select(.name == "Build arm64 images")'
[ "$(yq -r "$sel | .env.SBOM" "$wf")" = 1 ]
[ "$(yq -r "$sel | .env.BUILDER" "$wf")" = '${{ steps.buildx.outputs.name }}' ]
[ "$(yq -r '.jobs.build-arm64.steps[] | select(.id == "buildx") | .with.driver' "$wf")" = docker-container ]
}
@test "images built with an SBOM are stitched into one index that keeps both attestations" {
tmp=$(mktemp -d)
_stub_registry "$tmp/reg"
_make_sbom_world "$tmp"
t="$tmp/tree"
PATH="$MOCK_REG/bin:$PATH" hack/stitch-multiarch.sh "$t" "$REG" main >"$tmp/out" 2>"$tmp/err"
i="$(_tag "$REG/api" main)"
[ "$i" != "$A_api" ]
# Two linux images, the ones the two builds pushed, and both attestations.
a_child=$(jq -r '.manifests[] | select(.platform.os == "linux") | .digest' "$MOCK_REG/blobs/${A_api#sha256:}")
b_child=$(jq -r '.manifests[] | select(.platform.os == "linux") | .digest' "$MOCK_REG/blobs/${B_api#sha256:}")
jq -e --arg a "$a_child" --arg b "$b_child" '
([.manifests[] | select(.platform.os == "linux") | .platform.architecture] | sort) == ["amd64", "arm64"]
and ([.manifests[] | select(.platform.os == "unknown")] | length) == 2
and any(.manifests[]; .digest == $a) and any(.manifests[]; .digest == $b)' \
"$MOCK_REG/blobs/${i#sha256:}" >/dev/null
grep -q "api:main@$i\$" "$t/system/api/values.yaml"
if grep -rq "$A_api" "$t"; then echo "FAIL: api still pinned on its amd64 index"; false; fi
grep -q '^stitch: 1 stitched (api); 0 skipped ()$' "$tmp/out"
rm -rf "$tmp"
}
@test "a rerun after a stitch of SBOM images repins on the existing index" {
# The stitched index lists the linux images of the two builds, not the amd64
# index the tree pins, so the rerun has to look through the pinned one.
tmp=$(mktemp -d)
_stub_registry "$tmp/reg"
_make_sbom_world "$tmp"
cp -R "$tmp/tree" "$tmp/stamped"
PATH="$MOCK_REG/bin:$PATH" hack/stitch-multiarch.sh "$tmp/tree" "$REG" main >/dev/null 2>&1
i="$(_tag "$REG/api" main)"
rm -rf "$tmp/tree"
cp -R "$tmp/stamped" "$tmp/tree"
PATH="$MOCK_REG/bin:$PATH" hack/stitch-multiarch.sh "$tmp/tree" "$REG" main >"$tmp/out" 2>"$tmp/err"
[ "$(_tag "$REG/api" main)" = "$i" ]
grep -q "api:main@$i\$" "$tmp/tree/system/api/values.yaml"
[ "$(grep -c 'imagetools create' "$MOCK_REG/log")" -eq 1 ]
grep -q '^stitch: 1 stitched (api); 0 skipped ()$' "$tmp/out"
rm -rf "$tmp"
}
@test "an index of two linux images is skipped, not stitched" {
# A multi-arch build in one go has no twin to join.
tmp=$(mktemp -d)
_stub_registry "$tmp/reg"
_make_sbom_world "$tmp"
both=$(_push_sbom "$REG/api" main amd64 arm64)
printf 'image: %s/api:main@%s\n' "$REG" "$both" >"$tmp/tree/system/api/values.yaml"
PATH="$MOCK_REG/bin:$PATH" hack/stitch-multiarch.sh "$tmp/tree" "$REG" main >"$tmp/out" 2>"$tmp/err"
[ "$(_tag "$REG/api" main)" = "$both" ]
grep -q "$both" "$tmp/tree/system/api/values.yaml"
grep -q 'skipping api: main and main-arm64 are not one amd64 and one arm64 image' "$tmp/err"
if grep -q 'imagetools create' "$MOCK_REG/log"; then echo "FAIL: a skipped ref was pushed"; false; fi
rm -rf "$tmp"
}
@test "an arm64 tag that holds no arm64 image is skipped, not stitched" {
+42
View File
@@ -0,0 +1,42 @@
#!/usr/bin/env bats
# A strategy CRD that inlines the upstream pod schema adds about 600 KB, and
# backupstrategy-controller stores each CRD twice in every Helm release
# revision, inside one Secret capped at 1 MiB. Three of them did once, and the
# upgrade that renders the default strategies failed with "Too long". Pod
# templates in api/backups/strategy are therefore Schemaless with
# PreserveUnknownFields (see the package doc there). hack/helm-release-size.bats
# catches the total; this names the field that caused it.
#
# Harness compatibility note: CI runs this file under Bats through
# `make bats-unit-tests`. It also remains compatible with the narrower legacy
# `hack/cozytest.sh` translator, so it avoids `run`, `$status`, `$output`,
# `skip`, and setup()/teardown(). Paths stay repo-root-relative for both
# runners.
#
# Run with: bats hack/strategy-crd-schemaless.bats
load test_helper
DEFS=packages/system/backupstrategy-controller/definitions
@test "no strategy CRD inlines a pod schema" {
tmp=$(mktemp -d)
n=0
for f in "$DEFS"/strategy.backups.cozystack.io_*.yaml; do
n=$((n + 1))
# A PodSpec is the object whose containers[] items carry an image field;
# report the property path that holds it, so the message names the field.
yq -N '[.. | select(tag == "!!map" and has("properties") and (.properties.containers.items.properties.image != null)) | path | join(".")] | .[]' "$f" \
| sed "s#^#$(basename "$f"): #" >> "$tmp/inlined"
done
if [ "$n" -lt 10 ]; then
echo "only $n strategy CRDs under $DEFS; nothing meaningful was checked" >&2
false
fi
if [ -s "$tmp/inlined" ]; then
echo "strategy CRDs inlining a pod schema; mark the Go field +kubebuilder:validation:Schemaless and +kubebuilder:pruning:PreserveUnknownFields:" >&2
cat "$tmp/inlined" >&2
false
fi
rm -rf "$tmp"
}
+100
View File
@@ -0,0 +1,100 @@
rule_files:
- rules.yaml
evaluation_interval: 15s
tests:
- name: watch streams of a stopped API server end with Canceled
interval: 30s
input_series:
- series: 'grpc_server_handled_total{job="tenant-test/etcd-pod-scrape", instance="10.0.0.1:2381", grpc_service="etcdserverpb.Watch", grpc_method="Watch", grpc_type="bidi_stream", grpc_code="Canceled"}'
values: '0+19x80'
- series: 'grpc_server_handled_total{job="tenant-test/etcd-pod-scrape", instance="10.0.0.1:2381", grpc_service="etcdserverpb.Watch", grpc_method="Watch", grpc_type="bidi_stream", grpc_code="Unavailable"}'
values: '2x80'
- series: 'grpc_server_handled_total{job="tenant-test/etcd-pod-scrape", instance="10.0.0.1:2381", grpc_service="etcdserverpb.KV", grpc_method="Range", grpc_type="unary", grpc_code="OK"}'
values: '0+30x80'
alert_rule_test:
- eval_time: 6m
groupname: etcd
alertname: etcdHighNumberOfFailedGRPCRequests
- eval_time: 12m
groupname: etcd
alertname: etcdHighNumberOfFailedGRPCRequests
- eval_time: 30m
groupname: etcd
alertname: etcdHighNumberOfFailedGRPCRequests
- name: range calls fail with Unavailable
interval: 30s
input_series:
- series: 'grpc_server_handled_total{job="tenant-test/etcd-pod-scrape", instance="10.0.0.1:2381", grpc_service="etcdserverpb.KV", grpc_method="Range", grpc_type="unary", grpc_code="OK"}'
values: '0+3x80'
- series: 'grpc_server_handled_total{job="tenant-test/etcd-pod-scrape", instance="10.0.0.1:2381", grpc_service="etcdserverpb.KV", grpc_method="Range", grpc_type="unary", grpc_code="Unavailable"}'
values: '0+1x80'
alert_rule_test:
- eval_time: 4m
groupname: etcd
alertname: etcdHighNumberOfFailedGRPCRequests
- eval_time: 7m
groupname: etcd
alertname: etcdHighNumberOfFailedGRPCRequests
exp_alerts:
- exp_labels:
job: tenant-test/etcd-pod-scrape
instance: 10.0.0.1:2381
grpc_service: etcdserverpb.KV
grpc_method: Range
severity: critical
exp_annotations:
summary: "etcd cluster 'tenant-test/etcd-pod-scrape': '25' of requests for 'Range' failed on etcd instance '10.0.0.1:2381'."
- eval_time: 12m
groupname: etcd
alertname: etcdHighNumberOfFailedGRPCRequests
exp_alerts:
- exp_labels:
job: tenant-test/etcd-pod-scrape
instance: 10.0.0.1:2381
grpc_service: etcdserverpb.KV
grpc_method: Range
severity: warning
exp_annotations:
summary: "etcd cluster 'tenant-test/etcd-pod-scrape': '25' of requests for 'Range' failed on etcd instance '10.0.0.1:2381'."
- exp_labels:
job: tenant-test/etcd-pod-scrape
instance: 10.0.0.1:2381
grpc_service: etcdserverpb.KV
grpc_method: Range
severity: critical
exp_annotations:
summary: "etcd cluster 'tenant-test/etcd-pod-scrape': '25' of requests for 'Range' failed on etcd instance '10.0.0.1:2381'."
- name: canceled watch streams next to failing range calls
interval: 30s
input_series:
- series: 'grpc_server_handled_total{job="tenant-test/etcd-pod-scrape", instance="10.0.0.1:2381", grpc_service="etcdserverpb.Watch", grpc_method="Watch", grpc_type="bidi_stream", grpc_code="Canceled"}'
values: '0+19x80'
- series: 'grpc_server_handled_total{job="tenant-test/etcd-pod-scrape", instance="10.0.0.1:2381", grpc_service="etcdserverpb.KV", grpc_method="Range", grpc_type="unary", grpc_code="OK"}'
values: '0+3x80'
- series: 'grpc_server_handled_total{job="tenant-test/etcd-pod-scrape", instance="10.0.0.1:2381", grpc_service="etcdserverpb.KV", grpc_method="Range", grpc_type="unary", grpc_code="Unavailable"}'
values: '0+1x80'
alert_rule_test:
- eval_time: 12m
groupname: etcd
alertname: etcdHighNumberOfFailedGRPCRequests
exp_alerts:
- exp_labels:
job: tenant-test/etcd-pod-scrape
instance: 10.0.0.1:2381
grpc_service: etcdserverpb.KV
grpc_method: Range
severity: warning
exp_annotations:
summary: "etcd cluster 'tenant-test/etcd-pod-scrape': '25' of requests for 'Range' failed on etcd instance '10.0.0.1:2381'."
- exp_labels:
job: tenant-test/etcd-pod-scrape
instance: 10.0.0.1:2381
grpc_service: etcdserverpb.KV
grpc_method: Range
severity: critical
exp_annotations:
summary: "etcd cluster 'tenant-test/etcd-pod-scrape': '25' of requests for 'Range' failed on etcd instance '10.0.0.1:2381'."
+52
View File
@@ -0,0 +1,52 @@
#!/bin/sh
# Fake kubectl for hack/migration-60-foundationdb-version.bats. It serves only
# the calls migration 60 (and lib/cozystack-version.sh) make and appends one
# line per call to $FAKE_CMDLOG.
#
# FAKE_CMDLOG file to append a record of each call to
# FAKE_HR_LIST file holding the HelmRelease list JSON the fleet scan returns
# FAKE_LIST_FAIL non-empty => the fleet scan fails
# FAKE_PATCH_FAIL non-empty => every patch fails
#
# Any other call fails loudly, so drift between the script and this fake does
# not pass unnoticed.
set -u
log() { printf '%s\n' "$*" >> "$FAKE_CMDLOG"; }
arg_after() { k="$1"; shift; p=""; for a in "$@"; do [ "$p" = "$k" ] && { printf '%s' "$a"; return; }; p="$a"; done; }
args="$*"
case "$args" in
"get helmreleases.helm.toolkit.fluxcd.io --all-namespaces --output json")
if [ -n "${FAKE_LIST_FAIL:-}" ]; then
echo 'error: the server was unable to return a response in the time allotted' >&2
exit 1
fi
cat "$FAKE_HR_LIST"
exit 0
;;
"patch helmreleases.helm.toolkit.fluxcd.io "*)
if [ -n "${FAKE_PATCH_FAIL:-}" ]; then
echo 'The request is invalid: the server rejected our request due to an error in our request' >&2
exit 1
fi
ns=$(arg_after --namespace "$@")
log "PATCH $ns $(arg_after "$ns" "$@") $(arg_after --type "$@") $(arg_after --patch "$@")"
exit 0
;;
"annotate configmap "*)
log "ANNOTATE $*"
exit 0
;;
"apply --filename -")
v=$(sed -n 's/^ version: "\{0,1\}\([0-9]\{1,\}\)"\{0,1\}$/\1/p' | head -1)
log "STAMP ${v:-unknown}"
exit 0
;;
esac
log "UNHANDLED $args"
echo "fake kubectl: unmodelled call: $args" >&2
exit 1
+18
View File
@@ -59,4 +59,22 @@ grep -qE "compare/.*\.\.\.v${VERSION}([^0-9.]|\$)" "$CL" \
grep -qE '^## ' "$CL" \
|| fail "no '## ' sections — not a complete changelog"
# The two checks below read content rather than shape, and each reports its
# line numbers inside the one-line reason: the workflows quote the reason into
# a ::warning:: or ::error:: annotation, which ends at the first newline.
# An ellipsis right before the attribution is a PR text pasted up to a length
# limit. The tail it lost is where PR authors put what the operator has to do,
# so the entry is not merely short but wrong.
truncated="$(grep -nE '(…|\.\.\.)[[:space:]]*\(\[\*\*@' "$CL" | cut -d: -f1 | tr '\n' ' ')"
[ -z "$truncated" ] \
|| fail "entries end in an ellipsis before their attribution (line(s) ${truncated% }): a PR text was cut off; rewrite each from the whole release note"
# cozystack.io serves only versioned docs paths and answers /docs/next/ with
# 404, so a release body must not link it. Anchored on the host, because
# content/en/docs/next/ in prose names the website's source tree and is fine.
nextdocs="$(grep -nE 'cozystack\.io/docs/next([^[:alnum:]_.-]|$)' "$CL" | cut -d: -f1 | tr '\n' ' ')"
[ -z "$nextdocs" ] \
|| fail "links point at cozystack.io/docs/next/, which the live site does not serve (line(s) ${nextdocs% }): link https://cozystack.io/docs/vX.Y/<path>/ instead"
echo "OK: $CL is publishable as the v${VERSION} release body ($(wc -l < "$CL" | tr -d ' ') lines)"
+2
View File
@@ -117,6 +117,8 @@ while IFS= read -r ref; do
try=1
while :; do
if raw="$(skopeo inspect --raw "docker://${target}" 2>"$err")"; then
# An attestation manifest (SBOM=1) is an index entry with platform
# unknown/unknown, which selecting os == linux leaves out.
missing="$(printf '%s' "$raw" | jq -r '
if .manifests then
[.manifests[].platform | select(.os == "linux") | .architecture] as $a