#!/usr/bin/env bash
#
# Unit tests for scripts/update.sh — runs in seconds, no server, no deploy.
#
# update.sh is source-able: when sourced it defines every function and then
# returns *before* the run block (the BASH_SOURCE guard). That lets us exercise
# the config-refresh + deployment-detection logic against throwaway fixtures
# instead of a real /etc and a real cloud box — which is what made this script
# so painful to get right.
#
# Each unit-under-test runs in a subshell that re-enables `set -Eeuo pipefail`,
# so a regression of the kind that bit 1.7.0 (an unguarded command silently
# aborting under set -e) is caught here as a failed assertion.
#
# Run:  bash scripts/test/test_update.sh
set -uo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
UPDATE_SH="$SCRIPT_DIR/../update.sh"

# Shared stub factories + the fresh-box fixture builder.
# shellcheck source=stubs.sh
source "$SCRIPT_DIR/stubs.sh"

PASS=0
FAIL=0
SKIP=0
ok()   { PASS=$((PASS + 1)); printf '  \033[32m✔\033[0m %s\n' "$1"; }
bad()  { FAIL=$((FAIL + 1)); printf '  \033[31m✘\033[0m %s\n' "$1"; }
skip() { SKIP=$((SKIP + 1)); printf '  \033[33m∼\033[0m %s (skipped)\n' "$1"; }

WORK="$(mktemp -d)"
trap 'rm -rf "$WORK"' EXIT

# --------------------------------------------------------------------------
# Stub the external commands the functions may shell out to, so the tests
# never touch the host's nginx/systemd/docker.
# --------------------------------------------------------------------------
STUB_BIN="$WORK/bin"
mkdir -p "$STUB_BIN"
make_stub_ok "$STUB_BIN" systemctl nginx npm curl
# docker stub: `docker ps ...` lists the fixture container names; anything else
# (image inspect/tag/compose/...) is a harmless no-op.
cat > "$STUB_BIN/docker" <<'EOF'
#!/usr/bin/env bash
case "${1:-}" in
  ps) for n in ${SERVERKIT_TEST_CONTAINERS:-}; do printf '%s\n' "$n"; done ;;
  *)  exit 0 ;;
esac
EOF
chmod +x "$STUB_BIN"/*
export PATH="$STUB_BIN:$PATH"

# --------------------------------------------------------------------------
# Source update.sh (functions only). Keep logging off and point the install
# dir at the sandbox so the derived DIR_A/DIR_B land under $WORK.
# --------------------------------------------------------------------------
export SERVERKIT_NO_LOG=1
export SERVERKIT_DIR="$WORK/opt/serverkit"
# shellcheck disable=SC1090
source "$UPDATE_SH"
set +e +u   # hand control back to the harness; tests re-arm set -e per subshell

printf '\nupdate.sh unit tests\n\n'

# --------------------------------------------------------------------------
# T1 — the headline regression: refresh_config must NOT die when the live
# nginx has no serverkit.conf (HTTP-only boxes). This is the exact 1.7.0
# silent-death that left the updater stuck reporting the old version.
# --------------------------------------------------------------------------
t="$WORK/t1"
mkdir -p "$t/nginx/sites-available" "$t/nginx/sites-enabled" "$t/target/nginx/sites-available"
printf 'http {\n}\n' > "$t/nginx/nginx.conf"
printf 'server { listen 80; }\n' > "$t/target/nginx/sites-available/serverkit-insecure.conf"
if (
    set -Eeuo pipefail
    NGINX_DIR="$t/nginx"; LETSENCRYPT_DIR="$t/le"; SYSTEMD_DIR="$t/sysd"; CONFIG_DIR="$t/cfg"; DRY_RUN=0
    refresh_config "$t/target"
) >/dev/null 2>&1; then
    ok "refresh_config survives a missing serverkit.conf (the 1.7.0 silent-death bug)"
else
    bad "refresh_config DIED on a missing serverkit.conf — the set -e/pipefail regression is back"
fi

# --------------------------------------------------------------------------
# T2 — refresh_config still works when a serverkit.conf with a real cert path
# is present (the grep finds a match).
# --------------------------------------------------------------------------
t="$WORK/t2"
mkdir -p "$t/nginx/sites-available" "$t/nginx/sites-enabled" "$t/target/nginx/sites-available" "$t/le/live/example.com"
printf 'http {\n}\n' > "$t/nginx/nginx.conf"
printf 'ssl_certificate %s/live/example.com/fullchain.pem;\n' "$t/le" > "$t/nginx/sites-available/serverkit.conf"
printf 'server { listen 80; }\n' > "$t/target/nginx/sites-available/serverkit-insecure.conf"
if (
    set -Eeuo pipefail
    NGINX_DIR="$t/nginx"; LETSENCRYPT_DIR="$t/le"; SYSTEMD_DIR="$t/sysd"; CONFIG_DIR="$t/cfg"; DRY_RUN=0
    refresh_config "$t/target"
) >/dev/null 2>&1; then
    ok "refresh_config handles a present serverkit.conf with a cert path"
else
    bad "refresh_config failed with a present serverkit.conf"
fi

# --------------------------------------------------------------------------
# T3 — deployment-shape detection (the bug that made 1.7.0 take the wrong path
# on an all-Docker box).
# --------------------------------------------------------------------------
t="$WORK/t3"; mkdir -p "$t/install"
touch "$t/install/docker-compose.yml"
if (
    set -Eeuo pipefail
    INSTALL_DIR="$t/install"
    export SERVERKIT_TEST_CONTAINERS="serverkit-backend serverkit-frontend"
    is_docker_deployment
); then
    ok "is_docker_deployment → docker when compose + container and no host venv"
else
    bad "is_docker_deployment should pick the docker path for an all-Docker box"
fi

mkdir -p "$t/install/venv/bin"
printf '#!/bin/sh\n' > "$t/install/venv/bin/python"; chmod +x "$t/install/venv/bin/python"
if (
    set -Eeuo pipefail
    INSTALL_DIR="$t/install"
    export SERVERKIT_TEST_CONTAINERS="serverkit-backend"
    is_docker_deployment
); then
    bad "is_docker_deployment should fall back to hybrid when a host venv exists"
else
    ok "is_docker_deployment → hybrid when a host venv exists (precedence)"
fi

# --------------------------------------------------------------------------
# T4 — blue/green slot resolution.
# --------------------------------------------------------------------------
t="$WORK/t4"; mkdir -p "$t/serverkit-a" "$t/serverkit-b"
ln -sfn "$t/serverkit-a" "$t/serverkit" 2>/dev/null || true
if [ ! -L "$t/serverkit" ]; then
    skip "active/next slot flip — symlinks unsupported here (works on Linux CI)"
else
    exp="$(readlink -f "$t/serverkit-a")|$t/serverkit-b"
    if ! res="$(
        set -Eeuo pipefail
        INSTALL_DIR="$t/serverkit"; DIR_A="$t/serverkit-a"; DIR_B="$t/serverkit-b"
        printf '%s|%s' "$(active_real_dir)" "$(next_real_dir)"
    )"; then
        bad "active/next slot resolution aborted under set -Eeuo pipefail"
    elif [ "$res" = "$exp" ]; then
        ok "active/next slot flip (A active → B is next)"
    else
        bad "active/next slot wrong: got [$res] expected [$exp]"
    fi
fi

# --------------------------------------------------------------------------
# T5 — the loud-failure reporter actually emits a labelled diagnostic.
# --------------------------------------------------------------------------
# UPDATE_LOG pinned non-empty so the "full log:" line is part of what T5
# asserts; the empty-UPDATE_LOG return-0 contract is covered by the fresh-box
# loop below (report_failure is in UPDATE_OBSERVERS).
if out="$(LAST_PHASE='Refreshing Configuration' UPDATE_LOG='/dev/null' \
          report_failure 2 42 'grep ... serverkit.conf' 2>&1)" \
   && printf '%s' "$out" | grep -q 'Update aborted'; then
    ok "report_failure emits a labelled 'Update aborted' diagnostic"
else
    bad "report_failure produced no diagnostic (or returned non-zero)"
fi

# --------------------------------------------------------------------------
# T6 — self-update bootstrap skips cleanly under each opt-out, and never
# re-execs (would replace this test process) when there is nothing to do.
# --------------------------------------------------------------------------
self_update_skips() {
    # Each guard runs in a subshell with set -e; a clean return keeps the test
    # process alive, and any stray `exec` would visibly break the harness.
    ( set -Eeuo pipefail; SERVERKIT_UPDATER_REEXECED=1; DRY_RUN=0; maybe_reexec_latest_updater ) &&
    ( set -Eeuo pipefail; SERVERKIT_NO_SELF_UPDATE=1;  DRY_RUN=0; maybe_reexec_latest_updater ) &&
    ( set -Eeuo pipefail; DRY_RUN=1;                              maybe_reexec_latest_updater ) &&
    ( set -Eeuo pipefail; DRY_RUN=0; SERVERKIT_OFFLINE_TARBALL=/x; maybe_reexec_latest_updater )
}
if self_update_skips >/dev/null 2>&1; then
    ok "self-update no-ops under re-exec/opt-out/dry-run/offline guards"
else
    bad "self-update guard returned non-zero (would block or loop the updater)"
fi

# --------------------------------------------------------------------------
# T7 — the run lock refuses a second concurrent update.
# --------------------------------------------------------------------------
if command -v flock >/dev/null 2>&1; then
    lock="$WORK/update.lock"
    ( flock -n 9 || exit 1; sleep 3 ) 9>"$lock" &   # hold the lock
    held=$!
    sleep 0.3
    if ( set -Eeuo pipefail; LOCK_FILE="$lock"; DRY_RUN=0; acquire_update_lock ) >/dev/null 2>&1; then
        bad "acquire_update_lock should refuse while the lock is held"
    else
        ok "acquire_update_lock refuses a concurrent run while locked"
    fi
    kill "$held" 2>/dev/null || true; wait "$held" 2>/dev/null || true
    if ( set -Eeuo pipefail; LOCK_FILE="$WORK/free.lock"; DRY_RUN=0; acquire_update_lock ) >/dev/null 2>&1; then
        ok "acquire_update_lock succeeds when the lock is free"
    else
        bad "acquire_update_lock failed on a free lock"
    fi
else
    skip "run-lock test — flock unavailable here (runs on Linux CI)"
fi

# --------------------------------------------------------------------------
# T8 — version comparison: versions_equal ignores a leading "v".
# --------------------------------------------------------------------------
if ( set -Eeuo pipefail; versions_equal v1.7.1 1.7.1 ) && \
   ( set -Eeuo pipefail; versions_equal 1.7.1 1.7.1 ) && \
   ! ( set -Eeuo pipefail; versions_equal 1.7.0 1.7.1 ); then
    ok "versions_equal matches across a leading 'v' and rejects mismatches"
else
    bad "versions_equal comparison is wrong"
fi

# --------------------------------------------------------------------------
# T9 — is_already_current short-circuits to "proceed" (non-zero) under --force
# and offline, without any network/git access.
# --------------------------------------------------------------------------
if ( set -Eeuo pipefail; FORCE_UPDATE=1; is_already_current ); then
    bad "is_already_current must proceed (non-zero) under --force"
else
    ok "is_already_current proceeds under --force (skips the version check)"
fi
if ( set -Eeuo pipefail; FORCE_UPDATE=0; SERVERKIT_OFFLINE_TARBALL=/x; is_already_current ); then
    bad "is_already_current must proceed (non-zero) when offline"
else
    ok "is_already_current proceeds when offline (can't compare)"
fi

# --------------------------------------------------------------------------
# T10 — the rollback-safety fix: migrate_database must run the migration
# against the NEW slot's database copy (slot-absolute path), never the
# /opt/serverkit symlink that still resolves to the live old slot. A flask
# stub captures the DATABASE_URL the migration actually used.
# --------------------------------------------------------------------------
t="$WORK/t10/serverkit-b"
mkdir -p "$t/venv/bin" "$t/backend/instance"
: > "$t/venv/bin/activate"                              # sourceable no-op
: > "$t/backend/instance/serverkit.db"                  # the slot's DB copy
printf '#!/usr/bin/env bash\necho ok\n' > "$t/venv/bin/python"  # integrity probe: healthy DB
chmod +x "$t/venv/bin/python"
printf 'DATABASE_URL=sqlite:///opt/serverkit/backend/instance/serverkit.db\n' > "$t/.env"
FLASK_CAP="$WORK/t10/flask-saw-dburl"
cat > "$STUB_BIN/flask" <<EOF
#!/usr/bin/env bash
printf '%s' "\${DATABASE_URL:-NONE}" > "$FLASK_CAP"
exit 0
EOF
chmod +x "$STUB_BIN/flask"
mig_rc=0
(
    set -Eeuo pipefail
    DRY_RUN=0
    migrate_database "$t"
) >/dev/null 2>&1 || mig_rc=$?
saw="$(tr -d '\r' < "$FLASK_CAP" 2>/dev/null || true)"
if [ "$mig_rc" -eq 0 ] && [ "$saw" = "sqlite:///$t/backend/instance/serverkit.db" ]; then
    ok "migrate_database targets the new slot's DB, leaving the old slot untouched"
else
    bad "migrate_database rc=$mig_rc, used [$saw], expected rc 0 + the slot-absolute new-slot DB path"
fi
rm -f "$STUB_BIN/flask"

# --------------------------------------------------------------------------
# T10b — the corrupt-DB fail-fast: when the integrity probe reports real
# corruption, migrate_database must halt BEFORE flask runs, with the update
# still on the old slot. A python stub simulates PRAGMA integrity_check
# returning 'database disk image is malformed'.
# --------------------------------------------------------------------------
t="$WORK/t10b/serverkit-b"
mkdir -p "$t/venv/bin" "$t/backend/instance"
: > "$t/venv/bin/activate"
: > "$t/backend/instance/serverkit.db"
printf 'DATABASE_URL=sqlite:///opt/serverkit/backend/instance/serverkit.db\n' > "$t/.env"
printf '#!/usr/bin/env bash\necho "database disk image is malformed"\n' > "$t/venv/bin/python"
chmod +x "$t/venv/bin/python"
FLASK_CAP="$WORK/t10b/flask-ran"
cat > "$STUB_BIN/flask" <<EOF
#!/usr/bin/env bash
touch "$FLASK_CAP"
exit 0
EOF
chmod +x "$STUB_BIN/flask"
mig_rc=0
(
    set -Eeuo pipefail
    DRY_RUN=0
    migrate_database "$t"
) >/dev/null 2>&1 || mig_rc=$?
if [ "$mig_rc" -ne 0 ] && [ ! -e "$FLASK_CAP" ]; then
    ok "migrate_database halts on a corrupt SQLite DB before flask runs (fail-fast)"
else
    bad "migrate_database corrupt-DB: rc=$mig_rc (want non-zero), flask-ran=$([ -e "$FLASK_CAP" ] && echo yes || echo no) (want no)"
fi
rm -f "$STUB_BIN/flask"

# --------------------------------------------------------------------------
# T10c — a probe that can't run at all (no usable venv python) is NOT proof
# of corruption: warn and proceed so flask can surface any real problem.
# --------------------------------------------------------------------------
t="$WORK/t10c/serverkit-b"
mkdir -p "$t/venv/bin" "$t/backend/instance"
: > "$t/venv/bin/activate"
: > "$t/backend/instance/serverkit.db"
printf 'DATABASE_URL=sqlite:///opt/serverkit/backend/instance/serverkit.db\n' > "$t/.env"
printf '#!/usr/bin/env bash\nexit 0\n' > "$STUB_BIN/flask"
chmod +x "$STUB_BIN/flask"
mig_rc=0
(
    set -Eeuo pipefail
    DRY_RUN=0
    migrate_database "$t"
) >/dev/null 2>&1 || mig_rc=$?
if [ "$mig_rc" -eq 0 ]; then
    ok "migrate_database proceeds (with a warning) when the integrity probe can't run"
else
    bad "migrate_database probe-unavailable: rc=$mig_rc, expected rc 0 (warn + proceed)"
fi
rm -f "$STUB_BIN/flask"

# --------------------------------------------------------------------------
# T10d — copy_sqlite_db snapshots a LIVE database consistently via the SQLite
# online backup API. A plain cp of a hot DB can capture half a transaction
# and come out "database disk image is malformed" — the exact failure that
# sank the migration on a real 1.7.35 -> 1.7.65 update.
# --------------------------------------------------------------------------
t="$WORK/t10d"; mkdir -p "$t"
PY=""
for c in python3.14 python3.13 python3.12 python3.11 python3; do
    if command -v "$c" >/dev/null 2>&1 && "$c" -c 'import sqlite3' 2>/dev/null; then
        PY="$c"; break
    fi
done
if [ -z "$PY" ]; then
    skip "copy_sqlite_db snapshots a live DB consistently (no working python3 on this host)"
else
    "$PY" - "$t/live.db" <<'PYEOF'
import sqlite3, sys
c = sqlite3.connect(sys.argv[1])
c.execute('CREATE TABLE t(id INTEGER PRIMARY KEY, v TEXT)')
c.executemany('INSERT INTO t(v) VALUES (?)', [('row%d' % i,) for i in range(1000)])
c.commit(); c.close()
PYEOF
    snap_rc=0
    (
        set -Eeuo pipefail
        DRY_RUN=0
        copy_sqlite_db "$t/live.db" "$t/snap.db"
    ) >/dev/null 2>&1 || snap_rc=$?
    snap_check="$("$PY" - "$t/snap.db" <<'PYEOF'
import sqlite3, sys
c = sqlite3.connect(sys.argv[1])
print(c.execute('PRAGMA integrity_check').fetchone()[0],
      c.execute('SELECT COUNT(*) FROM t').fetchone()[0])
PYEOF
    )"
    if [ "$snap_rc" -eq 0 ] && [ "$snap_check" = "ok 1000" ]; then
        ok "copy_sqlite_db snapshots a live DB consistently (integrity ok, all rows)"
    else
        bad "copy_sqlite_db: rc=$snap_rc, snapshot check=[$snap_check], expected 'ok 1000'"
    fi
fi

# Contract: no racy plain-cp of the LIVE database anywhere in update.sh —
# the pre-upgrade backup and both deploy paths must go through copy_sqlite_db.
if ! grep -qF 'cp "$db_file" "$backup_file"' "$UPDATE_SH" \
   && ! grep -qF 'cp "$INSTALL_DIR/backend/instance/serverkit.db"' "$UPDATE_SH"; then
    ok "live SQLite DB is never plain-cp'd (consistent snapshot only)"
else
    bad "a racy plain-cp of the live serverkit.db is still in update.sh"
fi

# --------------------------------------------------------------------------
# T10e — PostgreSQL installs get a real pre-upgrade safety net: backup_current
# must pg_dump the external DB (sanitizing SQLAlchemy driver suffixes libpq
# can't parse) and must HALT when the dump fails — updating a shared,
# migrate-in-place database with no dump is flying without a parachute.
# --------------------------------------------------------------------------
t="$WORK/t10e"; mkdir -p "$t/serverkit" "$t/backups"
printf 'DATABASE_URL=postgresql+psycopg2://user:secret@db.host:5432/serverkit\n' > "$t/serverkit/.env"
cat > "$STUB_BIN/pg_dump" <<EOF
#!/usr/bin/env bash
printf '%s\n' "\$1" > "$WORK/t10e/url.txt"
out=""; prev=""
for a in "\$@"; do [ "\$prev" = "-f" ] && out="\$a"; prev="\$a"; done
printf 'PGDMP fake contents\n' > "\$out"
exit \${PG_DUMP_RC:-0}
EOF
chmod +x "$STUB_BIN/pg_dump"
make_stub_pg_restore "$STUB_BIN"
bk_rc=0
(
    set -Eeuo pipefail
    DRY_RUN=0 INSTALL_DIR="$t/serverkit" BACKUP_DIR="$t/backups"
    backup_current
) >/dev/null 2>&1 || bk_rc=$?
dump_file="$(ls "$t/backups"/serverkit-pre-upgrade-*.dump 2>/dev/null | head -1)"
if [ "$bk_rc" -eq 0 ] && [ -n "$dump_file" ] \
   && [ "$(cat "$WORK/t10e/url.txt" 2>/dev/null)" = "postgresql://user:secret@db.host:5432/serverkit" ]; then
    ok "backup_current pg_dumps PostgreSQL installs (driver suffix sanitized)"
else
    bad "backup_current postgres: rc=$bk_rc, dump=[$dump_file], url=[$(cat "$WORK/t10e/url.txt" 2>/dev/null)]"
fi
rm -rf "$t/backups"; mkdir -p "$t/backups"
bk_rc=0
(
    set -Eeuo pipefail
    DRY_RUN=0 INSTALL_DIR="$t/serverkit" BACKUP_DIR="$t/backups"
    export PG_DUMP_RC=1
    backup_current
) >/dev/null 2>&1 || bk_rc=$?
if [ "$bk_rc" -ne 0 ]; then
    ok "backup_current halts when the PostgreSQL dump fails (no safety net, no update)"
else
    bad "backup_current proceeded despite a failed pg_dump (rc=0)"
fi

# The engine is decided by DATABASE_URL, never by which files happen to exist.
# An install migrated SQLite -> PostgreSQL keeps its old serverkit.db (every
# deploy copies it forward on file-existence alone), and backing THAT up
# instead of dumping left PRE_UPGRADE_DUMP unset — quietly disabling
# migrate_database's auto-restore, which reads it as "${PRE_UPGRADE_DUMP:-}",
# on the one engine that migrates in place and cannot roll back by switching
# slots. Needs no python3: the PostgreSQL path is pg_dump + pg_restore only.
t="$WORK/t10e2"; mkdir -p "$t/serverkit/backend/instance" "$t/backups"
printf 'DATABASE_URL=postgresql://user:secret@db.host:5432/serverkit\n' > "$t/serverkit/.env"
printf 'stale sqlite file left over from before the postgres migration\n' \
    > "$t/serverkit/backend/instance/serverkit.db"
lo_rc=0
(
    set -Eeuo pipefail
    DRY_RUN=0 INSTALL_DIR="$t/serverkit" BACKUP_DIR="$t/backups"
    backup_current
    printf '%s\n' "${PRE_UPGRADE_DUMP:-}" > "$t/dump-var.txt"
) >/dev/null 2>&1 || lo_rc=$?
lo_dump="$(ls "$t/backups"/serverkit-pre-upgrade-*.dump 2>/dev/null | head -1)"
lo_db="$(ls "$t/backups"/serverkit-pre-upgrade-*.db 2>/dev/null | head -1)"
if [ "$lo_rc" -eq 0 ] && [ -n "$lo_dump" ] && [ -z "$lo_db" ] \
   && [ "$(cat "$t/dump-var.txt" 2>/dev/null)" = "$lo_dump" ]; then
    ok "a leftover serverkit.db never diverts a PostgreSQL install away from pg_dump"
else
    bad "pg install w/ stale sqlite file: rc=$lo_rc, dump=[$lo_dump], db=[$lo_db], PRE_UPGRADE_DUMP=[$(cat "$t/dump-var.txt" 2>/dev/null)]"
fi

# An engine this updater cannot dump (MySQL) must say so. "No SQLite database"
# reads like a benign nothing-to-do while the update proceeds with no safety
# net at all. The warning names the SCHEME only — a DATABASE_URL carries a
# password, and this text lands in the update log.
t="$WORK/t10e3"; mkdir -p "$t/serverkit" "$t/backups"
printf 'DATABASE_URL=mysql://user:hunter2@db.host:3306/serverkit\n' > "$t/serverkit/.env"
my_rc=0
my_out="$(
    set -Eeuo pipefail
    DRY_RUN=0 INSTALL_DIR="$t/serverkit" BACKUP_DIR="$t/backups"
    backup_current 2>&1
)" || my_rc=$?
if [ "$my_rc" -eq 0 ] \
   && printf '%s' "$my_out" | grep -q "mysql" \
   && printf '%s' "$my_out" | grep -q "NO database backup" \
   && ! printf '%s' "$my_out" | grep -q "hunter2"; then
    ok "an undumpable engine is named in the warning, without leaking the password"
else
    bad "mysql warning: rc=$my_rc, out=[$my_out]"
fi
rm -f "$STUB_BIN/pg_dump" "$STUB_BIN/pg_restore"

# --------------------------------------------------------------------------
# T10f — backup verification helpers: a backup is only a safety net when it
# is PROVEN valid. verify_sqlite_db accepts a real DB and rejects garbage;
# verify_pg_dump rejects empty/truncated dumps.
# --------------------------------------------------------------------------
t="$WORK/t10f"; mkdir -p "$t"
PY=""
for c in python3.14 python3.13 python3.12 python3.11 python3; do
    if command -v "$c" >/dev/null 2>&1 && "$c" -c 'import sqlite3' 2>/dev/null; then
        PY="$c"; break
    fi
done
if [ -z "$PY" ]; then
    skip "verify_sqlite_db accepts valid DBs and rejects garbage (no working python3)"
else
    "$PY" - "$t/good.db" <<'PYEOF'
import sqlite3, sys
c = sqlite3.connect(sys.argv[1])
c.execute('CREATE TABLE t(a)')
c.commit(); c.close()
PYEOF
    printf 'garbage not a database' > "$t/bad.db"
    v_rc=0
    (
        set -Eeuo pipefail
        verify_sqlite_db "$t/good.db" && ! verify_sqlite_db "$t/bad.db"
    ) >/dev/null 2>&1 || v_rc=$?
    if [ "$v_rc" -eq 0 ]; then
        ok "verify_sqlite_db accepts a valid DB and rejects a malformed one"
    else
        bad "verify_sqlite_db misclassified a fixture (rc=$v_rc)"
    fi
fi
: > "$t/empty.dump"; printf 'PGDMP\nstuff\n' > "$t/full.dump"
# A non-archive that is merely NON-EMPTY is the fixture that separates the two
# verify_pg_dump branches: the size check passes it, only the TOC listing
# catches it. Requires the stub — see make_stub_pg_restore.
printf 'ERROR: connection refused\n' > "$t/garbage.dump"
make_stub_pg_restore "$STUB_BIN"
v_rc=0
(
    set -Eeuo pipefail
    verify_pg_dump "$t/full.dump" && ! verify_pg_dump "$t/empty.dump" \
        && ! verify_pg_dump "$t/missing.dump" && ! verify_pg_dump "$t/garbage.dump"
) >/dev/null 2>&1 || v_rc=$?
if [ "$v_rc" -eq 0 ]; then
    ok "verify_pg_dump rejects empty/missing/non-archive dumps (TOC check via pg_restore)"
else
    bad "verify_pg_dump misclassified a fixture (rc=$v_rc)"
fi
rm -f "$STUB_BIN/pg_restore"

# --------------------------------------------------------------------------
# T10i — the SQLite helpers must NOT inherit the venv-builder's 3.11/3.12
# constraint. Snapshotting and integrity-checking a DB needs only an
# interpreter carrying the stdlib sqlite3 module; demanding 3.11/3.12 blocked
# updates on stock EL9 (python3.9) and Ubuntu 22.04 (python3.10) boxes for a
# reason unrelated to the data — including all-Docker installs, which return
# before preflight_check and do every real step inside the container, yet
# verified the extracted DB with the HOST's interpreter.
# --------------------------------------------------------------------------
t="$WORK/t10i"; mkdir -p "$t/bin" "$t/slot/backend/instance" "$t/slot/venv/bin"
if [ -z "$PY" ]; then
    skip "SQLite helpers work without a 3.11/3.12 interpreter (no working python3)"
else
    REAL_PY="$(command -v "$PY")"
    # An interpreter that answers a version probe with 3.9 but is otherwise a
    # real Python: makes locate_python fail exactly as it does on EL9, while
    # `import sqlite3` keeps working. Shadowing all four candidate names keeps
    # the precondition true regardless of what the host actually has.
    for n in python3.14 python3.13 python3.12 python3.11 python3 python; do
        cat > "$t/bin/$n" <<EOF
#!/usr/bin/env bash
case "\$*" in
    *version_info*) echo "3.9"; exit 0 ;;
esac
exec "$REAL_PY" "\$@"
EOF
        chmod +x "$t/bin/$n"
    done
    cp "$t/bin/python3" "$t/slot/venv/bin/python"
    "$PY" - "$t/slot/backend/instance/serverkit.db" <<'PYEOF'
import sqlite3, sys
c = sqlite3.connect(sys.argv[1])
c.execute('CREATE TABLE t(id INTEGER PRIMARY KEY, v TEXT)')
c.executemany('INSERT INTO t(v) VALUES (?)', [('row%d' % i,) for i in range(100)])
c.commit(); c.close()
PYEOF
    el9_rc=0
    (
        set -Eeuo pipefail
        PATH="$t/bin:$PATH"
        DRY_RUN=0
        # Precondition: to the venv builder this box now looks like EL9.
        ! locate_python
        # The sqlite helpers still resolve an interpreter — and prefer the
        # install's own over anything on PATH, so they stay correct on a box
        # whose system python is unusable.
        [ "$(locate_sqlite_python "$t/slot/venv/bin/python")" = "$t/slot/venv/bin/python" ]
        copy_sqlite_db "$t/slot/backend/instance/serverkit.db" "$t/snap.db"
        verify_sqlite_db "$t/snap.db"
    ) >/dev/null 2>&1 || el9_rc=$?
    rows="$("$PY" -c "import sqlite3,sys;print(sqlite3.connect(sys.argv[1]).execute('SELECT COUNT(*) FROM t').fetchone()[0])" "$t/snap.db" 2>/dev/null || true)"
    if [ "$el9_rc" -eq 0 ] && [ "$rows" = "100" ]; then
        ok "SQLite snapshot/verify need only sqlite3, not the gated Python 3.11-3.14 (EL9/22.04 boxes)"
    else
        bad "sqlite helpers still gated on locate_python: rc=$el9_rc, rows=[$rows], expected 100"
    fi
fi

# --------------------------------------------------------------------------
# T10g — a failed IN-PLACE PostgreSQL migration must trigger an automatic
# restore from the verified pre-upgrade dump: the DB is shared, so without a
# restore the half-migrated schema stays live against the old code.
# --------------------------------------------------------------------------
t="$WORK/t10g/serverkit-b"
mkdir -p "$t/venv/bin" "$t/backend/instance"
: > "$t/venv/bin/activate"
printf 'DATABASE_URL=postgresql://user:secret@db.host:5432/serverkit\n' > "$t/.env"
printf 'PGDMP fake contents\n' > "$WORK/t10g.dump"
cat > "$STUB_BIN/flask" <<'EOF'
#!/usr/bin/env bash
exit 1   # migration blows up mid-flight
EOF
cat > "$STUB_BIN/pg_restore" <<EOF
#!/usr/bin/env bash
echo "\$@" > "$WORK/t10g/restore-args.txt"
exit 0
EOF
chmod +x "$STUB_BIN/flask" "$STUB_BIN/pg_restore"
mig_rc=0
(
    set -Eeuo pipefail
    DRY_RUN=0
    PRE_UPGRADE_DUMP="$WORK/t10g.dump"
    migrate_database "$t"
) >/dev/null 2>&1 || mig_rc=$?
if [ "$mig_rc" -ne 0 ] && grep -q -- "--clean --if-exists -d postgresql://user:secret@db.host:5432/serverkit $WORK/t10g.dump" "$WORK/t10g/restore-args.txt" 2>/dev/null; then
    ok "failed in-place PostgreSQL migration auto-restores the pre-upgrade dump"
else
    bad "pg auto-restore: rc=$mig_rc, args=[$(cat "$WORK/t10g/restore-args.txt" 2>/dev/null)]"
fi
rm -f "$STUB_BIN/flask" "$STUB_BIN/pg_restore"

# --------------------------------------------------------------------------
# T10h — backup retention is capped (default 5, SERVERKIT_BACKUP_RETENTION
# overrides) across ALL backup kinds: full-size DB copies pile up fast on
# small VPSes (a real box carried 6.3G of stale backups on a 25G disk).
# --------------------------------------------------------------------------
t="$WORK/t10h"; mkdir -p "$t/backups"
for i in 1 2 3 4 5 6; do
    mkdir -p "$t/backups/serverkit-tree-2026010$i"
    : > "$t/backups/serverkit-pre-upgrade-2026010$i-00000$i.db"
    : > "$t/backups/serverkit-pre-upgrade-2026010$i-00000$i.dump"
    touch -d "2026-01-0$i 00:00" "$t/backups/serverkit-tree-2026010$i" \
        "$t/backups/serverkit-pre-upgrade-2026010$i-00000$i.db" \
        "$t/backups/serverkit-pre-upgrade-2026010$i-00000$i.dump" 2>/dev/null || true
done
ret_rc=0
(
    set -Eeuo pipefail
    DRY_RUN=0 BACKUP_DIR="$t/backups" SERVERKIT_BACKUP_RETENTION=3
    cleanup
) >/dev/null 2>&1 || ret_rc=$?
left_trees=$(find "$t/backups" -mindepth 1 -maxdepth 1 -name 'serverkit-tree-*' | wc -l)
left_dbs=$(find "$t/backups" -mindepth 1 -maxdepth 1 -name '*.db' | wc -l)
left_dumps=$(find "$t/backups" -mindepth 1 -maxdepth 1 -name '*.dump' | wc -l)
if [ "$ret_rc" -eq 0 ] && [ "$left_trees" -eq 3 ] && [ "$left_dbs" -eq 3 ] && [ "$left_dumps" -eq 3 ]; then
    ok "cleanup prunes all backup kinds to SERVERKIT_BACKUP_RETENTION (3 kept)"
else
    bad "retention: rc=$ret_rc, left trees=$left_trees db=$left_dbs dump=$left_dumps (want 3/3/3)"
fi

# --------------------------------------------------------------------------
# T10i — plan 85 B3: with SERVERKIT_BACKUP_RETENTION unset, a filesystem under
# 50 GiB keeps ONE upgrade snapshot (each is a full DB copy), a larger one
# keeps 3, and an unreadable size keeps the old default of 3.
# --------------------------------------------------------------------------
t="$WORK/t10i"; mkdir -p "$t/bin" "$t/backups"
_keep_with_df() {  # $1 = total KiB df reports ('' = df fails)
    if [ -n "$1" ]; then
        printf '#!/usr/bin/env bash\necho "Filesystem 1024-blocks Used Available Capacity Mounted"\necho "/dev/vda1 %s 1 1 1%% /"\n' "$1" > "$t/bin/df"
    else
        printf '#!/usr/bin/env bash\nexit 1\n' > "$t/bin/df"
    fi
    chmod +x "$t/bin/df"
    ( set -Eeuo pipefail; export PATH="$t/bin:$PATH"; BACKUP_DIR="$t/backups"; default_backup_keep )
}
k_small="$(_keep_with_df 26214400)"     # 25 GiB
k_large="$(_keep_with_df 524288000)"    # 500 GiB
k_unknown="$(_keep_with_df '')"
if [ "$k_small" = 1 ] && [ "$k_large" = 3 ] && [ "$k_unknown" = 3 ]; then
    ok "default_backup_keep: 1 snapshot under 50 GiB, 3 above or when unknown"
else
    bad "default_backup_keep: small=$k_small large=$k_large unknown=$k_unknown (want 1/3/3)"
fi

# --------------------------------------------------------------------------
# T10j — plan 85 B2: on a blue/green install the active slot stays on disk as
# the rollback, so backup_current must not write a serverkit-tree-* copy of
# it (a second full copy of the same code per update). A plain-directory
# install still gets its tree backup.
# --------------------------------------------------------------------------
t="$WORK/t10j"
_tree_backups_after_backup_current() {  # $1 = bluegreen|plain
    rm -rf "$t"; mkdir -p "$t/opt/serverkit-a/backend" "$t/backups"
    if [ "$1" = bluegreen ]; then
        ln -s "$t/opt/serverkit-a" "$t/opt/serverkit"
    else
        mv "$t/opt/serverkit-a" "$t/opt/serverkit"
    fi
    (
        set -Eeuo pipefail
        DRY_RUN=0; INSTALL_DIR="$t/opt/serverkit"; BACKUP_DIR="$t/backups"
        SERVERKIT_BACKUP_RETENTION=3
        backup_current
    ) >/dev/null 2>&1 || true
    find "$t/backups" -mindepth 1 -maxdepth 1 -name 'serverkit-tree-*' | wc -l
}
trees_bg="$(_tree_backups_after_backup_current bluegreen)"
trees_plain="$(_tree_backups_after_backup_current plain)"
if [ "$trees_bg" -eq 0 ] && [ "$trees_plain" -eq 1 ]; then
    ok "backup_current skips the tree copy on blue/green, keeps it for a plain install"
else
    bad "tree backups: bluegreen=$trees_bg plain=$trees_plain (want 0/1)"
fi

# --------------------------------------------------------------------------
# T11 — zero-downtime regression: reload_nginx_graceful must RELOAD a running
# nginx and must NEVER stop it. Host nginx fronts every managed app, so a stop
# during a panel update used to black out unrelated sites. A recording systemctl
# stub (PATH-prepended ahead of the global stub) captures every invocation.
# --------------------------------------------------------------------------
t="$WORK/t11"; mkdir -p "$t/bin"
CALL_LOG="$t/calls.log"; : > "$CALL_LOG"
cat > "$t/bin/systemctl" <<EOF
#!/usr/bin/env bash
printf '%s\n' "\$*" >> "$CALL_LOG"
exit 0                       # is-active --quiet nginx → running
EOF
cat > "$t/bin/nginx" <<EOF
#!/usr/bin/env bash
printf 'nginx %s\n' "\$*" >> "$CALL_LOG"
exit 0                       # nginx -t passes
EOF
chmod +x "$t/bin"/*
if (
    set -Eeuo pipefail
    export PATH="$t/bin:$PATH"
    DRY_RUN=0
    reload_nginx_graceful
) >/dev/null 2>&1; then
    if grep -q 'reload nginx' "$CALL_LOG" && ! grep -q 'stop nginx' "$CALL_LOG"; then
        ok "reload_nginx_graceful reloads a running nginx and never stops it (zero-downtime)"
    else
        bad "reload_nginx_graceful must reload (not stop) nginx; saw: $(tr '\n' ';' < "$CALL_LOG")"
    fi
else
    bad "reload_nginx_graceful returned non-zero against a healthy running nginx"
fi

# --------------------------------------------------------------------------
# T12 — when nginx is NOT running, reload_nginx_graceful starts it (instead of
# reloading a dead service) and still never issues a stop. The is-active gate
# reports inactive on its first probe, then active so wait_for_service returns
# immediately (keeps the test sub-second).
# --------------------------------------------------------------------------
t="$WORK/t12"; mkdir -p "$t/bin"
CALL_LOG="$t/calls.log"; : > "$CALL_LOG"; : > "$t/probe"
cat > "$t/bin/systemctl" <<EOF
#!/usr/bin/env bash
printf '%s\n' "\$*" >> "$CALL_LOG"
if [ "\$*" = "is-active --quiet nginx" ]; then
    n=\$(cat "$t/probe" 2>/dev/null || echo 0); echo \$((n + 1)) > "$t/probe"
    [ "\$n" -ge 1 ] && exit 0 || exit 1      # 1st probe: down → start branch; then up
fi
exit 0
EOF
chmod +x "$t/bin"/*
if (
    set -Eeuo pipefail
    export PATH="$t/bin:$PATH"
    DRY_RUN=0
    reload_nginx_graceful
) >/dev/null 2>&1; then
    if grep -q 'start nginx' "$CALL_LOG" && ! grep -q 'stop nginx' "$CALL_LOG" \
       && ! grep -q 'reload nginx' "$CALL_LOG"; then
        ok "reload_nginx_graceful starts a stopped nginx (never reloads a dead unit, never stops)"
    else
        bad "reload_nginx_graceful should start (not reload/stop) a dead nginx; saw: $(tr '\n' ';' < "$CALL_LOG")"
    fi
else
    bad "reload_nginx_graceful returned non-zero while starting a stopped nginx"
fi

# --------------------------------------------------------------------------
# T13 — guard against the old behaviour creeping back: the update.sh source must
# not contain a literal `systemctl stop nginx`. The forward and rollback paths
# both route nginx through reload_nginx_graceful now.
# --------------------------------------------------------------------------
if grep -nq 'systemctl stop nginx' "$UPDATE_SH"; then
    bad "update.sh still contains 'systemctl stop nginx' — apps would black out on update"
else
    ok "update.sh never stops nginx (no 'systemctl stop nginx' anywhere)"
fi

# --------------------------------------------------------------------------
# T14 — the panel frontend is served statically from $INSTALL_DIR/frontend/dist.
# refresh_config must repoint the shipped `root` (default /opt/serverkit) at a
# customised SERVERKIT_DIR, or a custom install dir would 404 the whole panel
# after an upgrade.
# --------------------------------------------------------------------------
t="$WORK/t14"
mkdir -p "$t/nginx/sites-available" "$t/nginx/sites-enabled" "$t/target/nginx/sites-available"
printf 'http {\n}\n' > "$t/nginx/nginx.conf"
printf 'server {\n  root /opt/serverkit/frontend/dist;\n  location / { try_files $uri /index.html; }\n}\n' \
    > "$t/target/nginx/sites-available/serverkit-insecure.conf"
t14_rc=0
(
    set -Eeuo pipefail
    NGINX_DIR="$t/nginx"; LETSENCRYPT_DIR="$t/le"; SYSTEMD_DIR="$t/sysd"; CONFIG_DIR="$t/cfg"; DRY_RUN=0
    INSTALL_DIR="$WORK/opt/serverkit"     # non-default → substitution must fire
    refresh_config "$t/target"
) >/dev/null 2>&1 || t14_rc=$?
installed="$t/nginx/sites-available/serverkit-insecure.conf"
if [ "$t14_rc" -eq 0 ] \
   && grep -q "root $WORK/opt/serverkit/frontend/dist;" "$installed" \
   && ! grep -q "root /opt/serverkit/frontend/dist;" "$installed"; then
    ok "refresh_config repoints the static-frontend root at a custom SERVERKIT_DIR"
else
    bad "refresh_config rc=$t14_rc or did not rewrite the frontend dist root: $(grep -n root "$installed" | tr '\n' ';')"
fi

# --------------------------------------------------------------------------
# T15 — the shipped nginx sites serve the SPA statically (host nginx, no
# container): each must carry a frontend/dist root + a try_files SPA fallback
# and must NOT proxy the retired frontend container on :3847.
# --------------------------------------------------------------------------
SK_ROOT="$SCRIPT_DIR/../.."
for f in serverkit.conf serverkit-insecure.conf; do
    cfg="$SK_ROOT/nginx/sites-available/$f"
    if grep -q 'frontend/dist' "$cfg" && grep -q 'try_files' "$cfg" \
       && ! grep -q '127.0.0.1:3847' "$cfg"; then
        ok "$f serves the SPA statically (dist root + try_files, no :3847 proxy)"
    else
        bad "$f is not a clean static-serve config (still proxying :3847?)"
    fi
    # The HTML entrypoint must revalidate every load (expires -1 → no-cache):
    # a cached index.html references hashed assets that vanish on the next
    # blue/green switch, bricking the panel behind any caching layer.
    if grep -q 'expires -1;' "$cfg"; then
        ok "$f marks the SPA entrypoint no-cache (expires -1)"
    else
        bad "$f lets the HTML entrypoint be cached — stale index.html breaks after a slot switch"
    fi
done

# --------------------------------------------------------------------------
# T15b — report_stale_panel_vhosts: a leftover custom vhost proxying the
# retired :3847 frontend container serves a permanently stale panel bundle
# (2026-07-03 incident: looked exactly like a browser/CDN cache problem).
# The check must WARN and name the file — and never abort, even with no
# sites-enabled dir at all (fresh box; also covered by the loop below).
# --------------------------------------------------------------------------
t="$WORK/t15b"; mkdir -p "$t/nginx/sites-enabled"
printf 'server {\n  location / {\n    proxy_pass http://127.0.0.1:3847;\n  }\n}\n' \
    > "$t/nginx/sites-enabled/legacy-panel.conf"
printf 'server {\n  root /opt/serverkit/frontend/dist;\n}\n' \
    > "$t/nginx/sites-enabled/clean.conf"
if out="$( set -Eeuo pipefail; NGINX_DIR="$t/nginx"; INSTALL_DIR="/opt/serverkit"
           report_stale_panel_vhosts 2>&1 )" \
   && printf '%s' "$out" | grep -q 'legacy-panel.conf' \
   && ! printf '%s' "$out" | grep -q 'clean.conf'; then
    ok "report_stale_panel_vhosts warns about (only) vhosts proxying :3847"
else
    bad "report_stale_panel_vhosts missed the stale vhost or flagged a clean one: [$out]"
fi
if out="$( set -Eeuo pipefail; NGINX_DIR="$t/nowhere"; INSTALL_DIR="/opt/serverkit"
           report_stale_panel_vhosts 2>&1 )" && [ -z "$out" ]; then
    ok "report_stale_panel_vhosts is silent (rc 0) with no sites-enabled dir"
else
    bad "report_stale_panel_vhosts failed or spoke on a missing dir: [$out]"
fi

# --------------------------------------------------------------------------
# T16 — app-uptime verification: discover_app_upstreams must extract the unique
# set of app container upstreams from the per-app nginx location snippets (this
# is the list the updater probes to prove apps stayed up).
# --------------------------------------------------------------------------
t="$WORK/t16"; mkdir -p "$t/loc"
printf 'location /app1 { proxy_pass http://127.0.0.1:8001; }\n' > "$t/loc/app1.conf"
printf 'location /app2  { proxy_pass http://127.0.0.1:8002/; }\nlocation /app2b { proxy_pass http://127.0.0.1:8001; }\n' > "$t/loc/app2.conf"
if ! res="$( set -Eeuo pipefail; APP_LOCATIONS_DIR="$t/loc"; discover_app_upstreams | tr '\n' ',' )"; then
    bad "discover_app_upstreams aborted on a populated app-locations directory"
elif [ "$res" = "127.0.0.1:8001,127.0.0.1:8002," ]; then
    ok "discover_app_upstreams extracts the unique app upstreams from location snippets"
else
    bad "discover_app_upstreams returned [$res], expected the two unique upstreams"
fi

# T16b — empty app-locations directory must not abort under set -euo pipefail.
mkdir -p "$t/empty"
if res="$( set -Eeuo pipefail; APP_LOCATIONS_DIR="$t/empty"; discover_app_upstreams )"; then
    if [ -z "$res" ]; then
        ok "discover_app_upstreams tolerates an empty app-locations directory"
    else
        bad "discover_app_upstreams returned [$res] for an empty directory, expected empty"
    fi
else
    bad "discover_app_upstreams aborted on an empty app-locations directory"
fi

# --------------------------------------------------------------------------
# T17 — report_app_uptime_regressions flags an app that was reachable before the
# update and is not after (and ignores one that was already down), returning
# non-zero so the operator is warned; the clean case returns success.
# --------------------------------------------------------------------------
before=$'127.0.0.1:8001 up\n127.0.0.1:8002 up\n127.0.0.1:8003 down'
after=$'127.0.0.1:8001 up\n127.0.0.1:8002 down\n127.0.0.1:8003 down'
if ( set -Eeuo pipefail; report_app_uptime_regressions "$before" "$after" ) >/dev/null 2>&1; then
    bad "report_app_uptime_regressions should flag the app that went up->down"
else
    ok "report_app_uptime_regressions flags an app that went down across the update"
fi
if ( set -Eeuo pipefail; report_app_uptime_regressions "$before" "$before" ) >/dev/null 2>&1; then
    ok "report_app_uptime_regressions passes when every app that was up is still up"
else
    bad "report_app_uptime_regressions should pass when nothing regressed"
fi

# --------------------------------------------------------------------------
# T18 — preserve_installed_plugins (#48): user-installed plugin dirs are
# carried into the new tree; dirs the new tree already ships are NOT
# overwritten; __pycache__ is skipped; and the function never dies under
# set -e (it runs inside deploy_source/deploy_release).
# --------------------------------------------------------------------------
t="$WORK/t18"
mkdir -p "$t/old/backend/app/plugins/third-party" \
         "$t/old/frontend/src/plugins/third-party" \
         "$t/old/backend/app/plugins/shipped-plugin" \
         "$t/old/backend/app/plugins/__pycache__" \
         "$t/new/backend/app/plugins/shipped-plugin"
printf 'user-code\n' > "$t/old/backend/app/plugins/third-party/blueprint.py"
printf 'user-ui\n'   > "$t/old/frontend/src/plugins/third-party/index.jsx"
printf 'old-copy\n'  > "$t/old/backend/app/plugins/shipped-plugin/__init__.py"
printf 'stale\n'     > "$t/old/backend/app/plugins/__pycache__/x.pyc"
printf 'repo-copy\n' > "$t/new/backend/app/plugins/shipped-plugin/__init__.py"
if (
    # One && chain, not sequential statements: bash suppresses set -e inside
    # an if-condition subshell, so only an explicit chain makes every
    # assertion gate the result (same fix as test_install.sh T16).
    set -Eeuo pipefail
    preserve_installed_plugins "$t/old" "$t/new" \
        && [ -f "$t/new/backend/app/plugins/third-party/blueprint.py" ] \
        && [ -f "$t/new/frontend/src/plugins/third-party/index.jsx" ] \
        && [ ! -d "$t/new/backend/app/plugins/__pycache__" ] \
        && grep -q repo-copy "$t/new/backend/app/plugins/shipped-plugin/__init__.py"
) >/dev/null 2>&1; then
    ok "preserve_installed_plugins carries user plugins forward without clobbering repo-shipped ones"
else
    bad "preserve_installed_plugins lost a user plugin, clobbered a shipped one, or died under set -e"
fi

# --------------------------------------------------------------------------
# T19 — U1: backup pruning must be a no-op on an empty/missing BACKUP_DIR.
# The old `ls -t <glob> | tail | xargs` prune exited 2 on an unmatched glob,
# aborting the updater AFTER a successful update on any box with no prior
# backups. Also prove the retention itself: newest 10 kept, oldest removed.
# --------------------------------------------------------------------------
t="$WORK/t19"; mkdir -p "$t/empty" "$t/ret"
prune_survives() {
    ( set -Eeuo pipefail; BACKUP_DIR="$t/empty"
      prune_old_backups 'serverkit-tree-*' 10 \
          && prune_old_backups 'serverkit-pre-upgrade-*.db' 10 ) &&
    ( set -Eeuo pipefail; BACKUP_DIR="$t/gone"; prune_old_backups 'serverkit-tree-*' 10 ) &&
    ( set -Eeuo pipefail; BACKUP_DIR="$t/empty"; INSTALL_DIR="$t/no-install"; DRY_RUN=0; cleanup )
}
if prune_survives >/dev/null 2>&1; then
    ok "cleanup/prune are a clean no-op on an empty or missing BACKUP_DIR (post-update abort bug)"
else
    bad "backup pruning DIED on an empty/missing BACKUP_DIR — the ls-glob abort is back"
fi

for i in 01 02 03 04 05 06 07 08 09 10 11 12; do
    mkdir -p "$t/ret/serverkit-tree-2026$i"
    touch -d "2026-01-$i 12:00:00" "$t/ret/serverkit-tree-2026$i"
done
( set -Eeuo pipefail; BACKUP_DIR="$t/ret"; prune_old_backups 'serverkit-tree-*' 10 ) >/dev/null 2>&1
remaining="$(find "$t/ret" -mindepth 1 -maxdepth 1 -name 'serverkit-tree-*' | wc -l)"
if [ "$remaining" -eq 10 ] && [ ! -d "$t/ret/serverkit-tree-202601" ] \
   && [ ! -d "$t/ret/serverkit-tree-202602" ] && [ -d "$t/ret/serverkit-tree-202612" ]; then
    ok "prune_old_backups keeps the newest 10 backups and removes the oldest"
else
    bad "prune retention wrong: $remaining left, expected the newest 10"
fi

# --------------------------------------------------------------------------
# T20 — U2: download_release's stdout is CAPTURED by its caller
# (tarball="\$(download_release ...)"), so it must print exactly one line — the
# tarball path — with every step/good/warn routed to stderr. A progress line on
# stdout used to hand tar a 4-line "filename", breaking every --release update.
# --------------------------------------------------------------------------
t="$WORK/t20"; mkdir -p "$t/bin" "$t/payload"
case "$(uname -m)" in
    x86_64|amd64)  tarch="amd64" ;;
    aarch64|arm64) tarch="arm64" ;;
    *)             tarch="" ;;
esac
if [ -z "$tarch" ] || ! command -v sha256sum >/dev/null 2>&1; then
    skip "download_release stdout purity — needs sha256sum + a known arch (runs on Linux CI)"
else
    printf 'fake-release-bytes\n' > "$t/payload/tarball"
    sha="$(sha256sum "$t/payload/tarball" | cut -d' ' -f1)"
    printf '%s  serverkit-v9.9.9-linux-%s.tar.gz\n' "$sha" "$tarch" > "$t/payload/checksums.txt"
    # curl stub that actually materialises the -o target, so the mirror path
    # runs end-to-end (download + checksum verification) without a network.
    cat > "$t/bin/curl" <<EOF
#!/usr/bin/env bash
out=""; url=""
while [ \$# -gt 0 ]; do
    case "\$1" in
        -o) out="\$2"; shift 2 ;;
        -*) shift ;;
        *)  url="\$1"; shift ;;
    esac
done
case "\$url" in
    *checksums.txt) cp "$t/payload/checksums.txt" "\$out" ;;
    *)              cp "$t/payload/tarball" "\$out" ;;
esac
EOF
    chmod +x "$t/bin/curl"
    out="$(
        set -Eeuo pipefail
        export PATH="$t/bin:$PATH"
        SERVERKIT_OFFLINE_TARBALL=""
        SERVERKIT_MIRROR_URL="http://mirror.invalid"
        download_release v9.9.9 2>"$t/stderr.txt"
    )"
    rc=$?
    if [ "$rc" -eq 0 ] && [ -n "$out" ] && [ -f "$out" ] \
       && [ "$(printf '%s' "$out" | wc -l)" -eq 0 ] \
       && grep -q 'Downloading release tarball' "$t/stderr.txt"; then
        ok "download_release captures to a single existing path (progress → stderr)"
    else
        bad "download_release polluted its captured stdout (rc=$rc): [$out]"
    fi
    off="$t/offline.tar.gz"; printf 'x' > "$off"
    if ! out2="$( set -Eeuo pipefail; SERVERKIT_OFFLINE_TARBALL="$off"; download_release v9.9.9 2>/dev/null )"; then
        bad "download_release (offline tarball) aborted under set -Eeuo pipefail"
    elif [ "$out2" = "$off" ]; then
        ok "download_release (offline tarball) returns exactly the tarball path"
    else
        bad "download_release (offline) returned [$out2], expected [$off]"
    fi
fi

# --------------------------------------------------------------------------
# T20c — a missing checksums.txt used to be "skipping verification" and the
# unverified tarball replaced the running panel. It must refuse unless the
# operator opts out explicitly (SERVERKIT_ALLOW_UNVERIFIED=1).
# --------------------------------------------------------------------------
t="$WORK/t20c"; mkdir -p "$t/bin"
printf 'fake-release-bytes\n' > "$t/tarball"
cat > "$t/bin/curl" <<EOF
#!/usr/bin/env bash
out=""; url=""
while [ \$# -gt 0 ]; do
    case "\$1" in -o) out="\$2"; shift 2 ;; -*) shift ;; *) url="\$1"; shift ;; esac
done
case "\$url" in
    *checksums.txt) exit 22 ;;
    *)              cp "$t/tarball" "\$out" ;;
esac
EOF
chmod +x "$t/bin/curl"
_t20c() {
    (
        set -Eeuo pipefail
        export PATH="$t/bin:$PATH"
        SERVERKIT_OFFLINE_TARBALL=""; SERVERKIT_MIRROR_URL="http://mirror.invalid"
        SERVERKIT_ALLOW_UNVERIFIED="${1:-0}"
        download_release v9.9.9
    ) 2>"$t/stderr.txt"
}
if out="$(_t20c 0)"; then
    bad "download_release accepted a release with no checksums.txt: [$out]"
elif grep -q 'refusing an unverified release' "$t/stderr.txt"; then
    ok "download_release refuses a release whose checksums.txt is missing"
else
    bad "download_release failed without saying why: $(cat "$t/stderr.txt")"
fi
if out="$(_t20c 1)" && [ -f "$out" ] && grep -q 'UNVERIFIED' "$t/stderr.txt"; then
    ok "SERVERKIT_ALLOW_UNVERIFIED=1 installs it, loudly"
    rm -rf "$(dirname "$out")"
else
    bad "SERVERKIT_ALLOW_UNVERIFIED=1 did not allow the unverified release"
fi

# --------------------------------------------------------------------------
# T20b — deploy_release removes its download and unpack dirs. It used to leave
# the ~70 MB tarball's mktemp dir behind on every update (a 25 GB box had 11
# of them, 644 MB), and a failed download stranded its dir as well. An
# operator's offline tarball must survive; stale leftovers are swept.
# --------------------------------------------------------------------------
t="$WORK/t20b"; mkdir -p "$t/bin" "$t/tmp" "$t/src/serverkit/scripts" "$t/live" "$t/slot"
if [ -z "$tarch" ] || ! command -v sha256sum >/dev/null 2>&1 || ! command -v tar >/dev/null 2>&1; then
    skip "deploy_release scratch cleanup — needs sha256sum, tar + a known arch (runs on Linux CI)"
else
    printf '#!/bin/sh\n' > "$t/src/serverkit/serverkit"
    tar czf "$t/release.tar.gz" -C "$t/src" serverkit
    sha="$(sha256sum "$t/release.tar.gz" | cut -d' ' -f1)"
    printf '%s  serverkit-v9.9.9-linux-%s.tar.gz\n' "$sha" "$tarch" > "$t/checksums.txt"
    cat > "$t/bin/curl" <<EOF
#!/usr/bin/env bash
out=""; url=""
while [ \$# -gt 0 ]; do
    case "\$1" in -o) out="\$2"; shift 2 ;; -*) shift ;; *) url="\$1"; shift ;; esac
done
[ -n "\${T20B_CURL_FAIL:-}" ] && exit 22
case "\$url" in
    *checksums.txt) cp "$t/checksums.txt" "\$out" ;;
    *)              cp "$t/release.tar.gz" "\$out" ;;
esac
EOF
    chmod +x "$t/bin/curl"
    # A leftover from an older updater (swept) and a fresh one (kept: it could
    # be a concurrent download).
    mkdir -p "$t/tmp/tmp.OLDLEFTOVER" "$t/tmp/tmp.FRESHDL"
    : > "$t/tmp/tmp.OLDLEFTOVER/serverkit-v1.0.0-linux-amd64.tar.gz"
    : > "$t/tmp/tmp.FRESHDL/serverkit-v1.0.1-linux-amd64.tar.gz"
    touch -d '2 hours ago' "$t/tmp/tmp.OLDLEFTOVER/serverkit-v1.0.0-linux-amd64.tar.gz"

    _t20b_deploy() {
        (
            set -Eeuo pipefail
            export PATH="$t/bin:$PATH" TMPDIR="$t/tmp"
            DRY_RUN=0; INSTALL_DIR="$t/live"; SERVERKIT_MIRROR_URL="http://mirror.invalid"
            SERVERKIT_OFFLINE_TARBALL="${1:-}"
            preserve_installed_plugins() { :; }
            trap cleanup_on_exit EXIT
            deploy_release "$t/slot/target" v9.9.9
        ) >/dev/null 2>&1
    }
    _t20b_scratch() { find "$t/tmp" -mindepth 1 -maxdepth 1 ! -name tmp.FRESHDL | wc -l; }

    if _t20b_deploy && [ -x "$t/slot/target/serverkit" ] && [ "$(_t20b_scratch)" -eq 0 ] \
       && [ -d "$t/tmp/tmp.FRESHDL" ]; then
        ok "deploy_release leaves no download/unpack dir behind and sweeps stale ones"
    else
        bad "deploy_release left scratch in TMPDIR: [$(ls "$t/tmp" | tr '\n' ' ')]"
    fi

    if _t20b_deploy "$t/release.tar.gz" && [ -f "$t/release.tar.gz" ] && [ "$(_t20b_scratch)" -eq 0 ]; then
        ok "deploy_release keeps an operator's offline tarball"
    else
        bad "deploy_release deleted the offline tarball or left scratch: [$(ls "$t/tmp" | tr '\n' ' ')]"
    fi

    if ! T20B_CURL_FAIL=1 _t20b_deploy && [ "$(_t20b_scratch)" -eq 0 ]; then
        ok "a failed download leaves no dir behind"
    else
        bad "a failed download stranded its dir: [$(ls "$t/tmp" | tr '\n' ' ')]"
    fi
fi

# --------------------------------------------------------------------------
# T21 — U5/U8: a health-check-triggered rollback halts, which fires the EXIT
# trap (cleanup_on_exit) — that trap must NOT run a second rollback on top of
# the one that just finished. And the rollback itself must never abort
# mid-flight: a failed daemon-reload still has to end in a started backend.
# --------------------------------------------------------------------------
t="$WORK/t21"; mkdir -p "$t/bin" "$t/prev"
CALL_LOG="$t/calls.log"; : > "$CALL_LOG"
cat > "$t/bin/systemctl" <<EOF
#!/usr/bin/env bash
printf '%s\n' "\$*" >> "$CALL_LOG"
[ "\$1" = "daemon-reload" ] && exit 1               # fails mid-rollback (U8)
if [ "\$1" = "is-active" ]; then
    grep -q '^start ' "$CALL_LOG" && exit 0         # active once a start was issued
    exit 1                                          # inactive before that
fi
exit 0
EOF
chmod +x "$t/bin/systemctl"
(
    set -Eeuo pipefail
    export PATH="$t/bin:$PATH"
    DRY_RUN=0
    INSTALL_DIR="$t/serverkit"
    PREVIOUS_DIR="$t/prev"
    # atomic_switch needs real symlinks (unsupported on this fs — see T4); the
    # unit under test is rollback's re-entry discipline, so stub the switch.
    atomic_switch() { printf 'switched-to %s\n' "$1" >> "$CALL_LOG"; }
    trap cleanup_on_exit EXIT
    rollback
) > "$t/out.txt" 2>&1
n="$(grep -c 'rolling back to previous slot' "$t/out.txt")"
if [ "$n" = "1" ]; then
    ok "rollback runs exactly once through the halt → EXIT-trap path (no double rollback)"
else
    bad "rollback transcript appeared $n time(s), expected exactly once"
fi
if grep -q '^daemon-reload' "$CALL_LOG" && grep -q '^start serverkit' "$CALL_LOG"; then
    ok "rollback survives a failed daemon-reload and still starts the backend (never aborts mid-flight)"
else
    bad "rollback aborted mid-flight on a failed daemon-reload; calls: $(tr '\n' ';' < "$CALL_LOG")"
fi

# --------------------------------------------------------------------------
# T22 — U4: nginx down AND refusing to start must not abort the updater
# post-switch (which would then abort AGAIN inside the rollback's EXIT trap).
# reload_nginx_graceful warns and returns 0 no matter what nginx does.
# --------------------------------------------------------------------------
t="$WORK/t22"; mkdir -p "$t/bin"
CALL_LOG="$t/calls.log"; : > "$CALL_LOG"
cat > "$t/bin/systemctl" <<EOF
#!/usr/bin/env bash
printf '%s\n' "\$*" >> "$CALL_LOG"
[ "\$1" = "start" ] && exit 1                       # nginx refuses to start
if [ "\$1" = "is-active" ]; then
    grep -q '^start ' "$CALL_LOG" && exit 0 || exit 1
fi
exit 0
EOF
chmod +x "$t/bin/systemctl"
if (
    set -Eeuo pipefail
    export PATH="$t/bin:$PATH"
    DRY_RUN=0
    reload_nginx_graceful
) >/dev/null 2>&1; then
    ok "reload_nginx_graceful never propagates a failed nginx start (post-switch abort averted)"
else
    bad "reload_nginx_graceful aborted when nginx was down and refused to start"
fi

# --------------------------------------------------------------------------
# T23 — U13: refresh_config runs `systemctl daemon-reload` pre-switch; a
# failing reload (degraded systemd, container, chroot) must warn, not abort.
# --------------------------------------------------------------------------
t="$WORK/t23"
mkdir -p "$t/nginx/sites-available" "$t/nginx/sites-enabled" "$t/target/nginx/sites-available" "$t/bin"
printf 'http {\n}\n' > "$t/nginx/nginx.conf"
printf 'server { listen 80; }\n' > "$t/target/nginx/sites-available/serverkit-insecure.conf"
cat > "$t/bin/systemctl" <<'EOF'
#!/usr/bin/env bash
[ "$1" = "daemon-reload" ] && exit 1
exit 0
EOF
chmod +x "$t/bin/systemctl"
if (
    set -Eeuo pipefail
    export PATH="$t/bin:$PATH"
    NGINX_DIR="$t/nginx"; LETSENCRYPT_DIR="$t/le"; SYSTEMD_DIR="$t/sysd"; CONFIG_DIR="$t/cfg"; DRY_RUN=0
    refresh_config "$t/target"
) >/dev/null 2>&1; then
    ok "refresh_config warns (not aborts) when systemctl daemon-reload fails"
else
    bad "refresh_config aborted on a failing daemon-reload"
fi

# --------------------------------------------------------------------------
# T24 — U9: migrate_database used to `cd` into the new slot's backend and leak
# that cwd into the main shell for the rest of the update. It now runs the
# activation + cd in a subshell, leaving the caller's cwd untouched.
# --------------------------------------------------------------------------
t="$WORK/t24/serverkit-b"
mkdir -p "$t/venv/bin" "$t/backend/instance"
: > "$t/venv/bin/activate"
: > "$t/backend/instance/serverkit.db"
printf '#!/usr/bin/env bash\necho ok\n' > "$t/venv/bin/python"  # integrity probe: healthy DB
chmod +x "$t/venv/bin/python"
printf 'DATABASE_URL=sqlite:////opt/serverkit/backend/instance/serverkit.db\n' > "$t/.env"
printf '#!/usr/bin/env bash\nexit 0\n' > "$STUB_BIN/flask"
chmod +x "$STUB_BIN/flask"
exp="$(cd "$WORK" && pwd)"
t24_rc=0
res="$(
    # && chain: this capture sits on the left of `||`, where bash suppresses
    # set -e even inside the substitution — sequential statements would let a
    # failing cd/migrate slip through ungated.
    set -Eeuo pipefail
    DRY_RUN=0
    cd "$WORK" \
        && migrate_database "$t" >/dev/null 2>&1 \
        && pwd
)" || t24_rc=$?
rm -f "$STUB_BIN/flask"
if [ "$t24_rc" -eq 0 ] && [ "$res" = "$exp" ]; then
    ok "migrate_database leaves the caller's cwd untouched (no cd leak)"
else
    bad "migrate_database rc=$t24_rc or leaked its cd: caller cwd is now [$res]"
fi

# --------------------------------------------------------------------------
# T25 — U12: the EXIT trap (cleanup_on_exit) must preserve the script's exit
# code AND return 0 — returning \$rc re-fired the ERR trap from inside the
# trap, appending a second, misleading "Update aborted" report.
# --------------------------------------------------------------------------
t="$WORK/t25"; mkdir -p "$t"
(
    set -Eeuo pipefail
    trap 'printf "ERR-REFIRED\n"' ERR
    trap cleanup_on_exit EXIT
    DRY_RUN=1
    exit 3
) > "$t/out.txt" 2>&1
rc=$?
if [ "$rc" -eq 3 ] && ! grep -q 'ERR-REFIRED' "$t/out.txt"; then
    ok "cleanup_on_exit preserves the exit code and never re-fires the ERR trap"
else
    bad "cleanup_on_exit changed rc to $rc or re-fired the ERR trap"
fi

# --------------------------------------------------------------------------
# T26 — regression guards baked into the source (same spirit as T13): the
# audited failure shapes must never creep back in.
# --------------------------------------------------------------------------
if ! grep -qE '^[^#]*ls -t "\$BACKUP_DIR"' "$UPDATE_SH"; then    # code lines only — the fix's comment cites the old idiom
    ok "update.sh never prunes backups via an ls-glob (aborts on empty BACKUP_DIR)"
else
    bad "an 'ls -t \"\$BACKUP_DIR\"' prune is back — unmatched globs abort under pipefail"
fi
if grep -q 'required+=(npm)' "$UPDATE_SH"; then
    ok "preflight requires npm for source-mode updates (no mid-update discovery)"
else
    bad "npm is missing from the source-mode preflight required-tools list"
fi
if grep -qF '${ORIG_ARGS[@]+"${ORIG_ARGS[@]}"}' "$UPDATE_SH"; then
    ok "self-update re-exec guards the empty-ORIG_ARGS expansion (set -u on bash < 4.4)"
else
    bad "the re-exec expands ORIG_ARGS unguarded — fatal with no args under set -u on old bash"
fi
if ! grep -qF 'frontend.*Up' "$UPDATE_SH"; then
    ok "health_check no longer probes the retired frontend container"
else
    bad "the vestigial 'frontend container Up' check is still in update.sh"
fi

# --------------------------------------------------------------------------
# T27 — the fresh-minimal-box loop. On 2026-07-02 a production update aborted
# because discover_app_upstreams() grep'd an EMPTY /etc/nginx/serverkit-
# locations dir (the default state of every fresh install) and set -Eeuo
# pipefail killed the bare APP_BASELINE="$(snapshot_app_reachability)"
# assignment. The function HAD a unit test — but only with a populated
# fixture. This loop runs EVERY observation/discovery/snapshot/report/probe
# function against the emptiest valid world a fresh box presents (zero apps,
# zero containers, no optional confs, dead network, empty state), under the
# script's own set -Eeuo pipefail.
#
# POLICY: every new observation/discovery/snapshot/report function added to
# scripts/update.sh MUST be appended here (add args/disposition to the case
# table when needed). Dispositions:
#   must0 — output is captured by assignment somewhere (X="$(fn)"), so a
#           non-zero exit IS the outage: the function must exit 0.
#   pred  — used only in conditional position (if fn; ...), where a clean
#           non-zero answer is legitimate: it must merely return normally
#           (no set -u crash, no signal death).
# --------------------------------------------------------------------------
FRESH="$WORK/freshbox"
make_fresh_box_fixture "$FRESH"

UPDATE_OBSERVERS=(
    local_version versions_equal remote_release_tag is_already_current
    version_gate active_real_dir next_real_dir discover_app_upstreams
    probe_app_upstreams snapshot_app_reachability report_app_uptime_regressions
    locate_python wait_for_service quick_health is_docker_deployment
    snapshot_docker_state report_failure report_stale_panel_vhosts
)

loop_fail=0
for fn in "${UPDATE_OBSERVERS[@]}"; do
    args=(); mode=must0
    case "$fn" in
        versions_equal)                args=(1.7.0 v1.7.0) ;;
        report_failure)                args=(2 42 'grep conf') ;;
        report_app_uptime_regressions) args=("" "") ;;
        wait_for_service)              args=(serverkit inactive 1) ;;
        quick_health)                  args=(1); mode=pred ;;
        is_already_current|is_docker_deployment|locate_python) mode=pred ;;
    esac
    out="$(
        {
            set -Eeuo pipefail
            export PATH="$FRESH/bin:$PATH"
            INSTALL_DIR="$FRESH/opt/serverkit"; SERVERKIT_DIR="$INSTALL_DIR"
            DIR_A="$FRESH/opt/serverkit-a"; DIR_B="$FRESH/opt/serverkit-b"
            APP_LOCATIONS_DIR="$FRESH/etc/nginx/serverkit-locations"
            NGINX_DIR="$FRESH/etc/nginx"; CONFIG_DIR="$FRESH/etc/serverkit"
            BACKUP_DIR="$FRESH/var/backups"; LOG_DIR="$FRESH/var/log"
            DRY_RUN=0; FORCE_UPDATE=0; USE_RELEASE=0
            RELEASE_VERSION=""; TARGET_BRANCH=""; SERVERKIT_OFFLINE_TARBALL=""
            "$fn" ${args[@]+"${args[@]}"}
        } </dev/null 2>&1
    )"
    rc=$?
    if [ "$mode" = "must0" ] && [ "$rc" -ne 0 ]; then
        bad "fresh-box loop: $fn exited $rc on a zero-app box: [$(printf '%s' "$out" | tail -c 160)]"
        loop_fail=1
    elif [ "$mode" = "pred" ] && { [ "$rc" -gt 128 ] || printf '%s' "$out" | grep -q 'unbound variable'; }; then
        bad "fresh-box loop: predicate $fn crashed (rc=$rc) on a zero-app box: [$(printf '%s' "$out" | tail -c 160)]"
        loop_fail=1
    fi
done
if [ "$loop_fail" = "0" ]; then
    ok "fresh-box loop: all ${#UPDATE_OBSERVERS[@]} observation/discovery functions survive a zero-app box"
fi

# --------------------------------------------------------------------------
# T28 — the best-effort contract as an executable promise: the app-uptime
# verification composed exactly the way update.sh's run block wires it —
#     APP_BASELINE="$(snapshot_app_reachability)"
#     ... stop/switch/start ...
#     phase "Verifying App Uptime"
#     APP_AFTER="$(snapshot_app_reachability)"
#     report_app_uptime_regressions "$APP_BASELINE" "$APP_AFTER" || true
# — must exit 0 under set -Eeuo pipefail on (a) a fresh zero-app box (the
# exact 2026-07-02 world) and (b) a box where an app genuinely regressed:
# the report warns loudly, but the `|| true` keeps verification from failing
# an already-healthy update. The outage commit made that promise only in
# prose; this test makes it executable.
# --------------------------------------------------------------------------
t="$WORK/t28"; mkdir -p "$t/binup" "$t/bindown" "$t/loc"
make_stub_ok          "$t/binup" curl      # every upstream answers → "up"
make_stub_curl_fail22 "$t/bindown"         # every upstream refuses → "down"

t28_rc=0
out="$(
    {
        # && chain: this capture sits on the left of `||`, where bash
        # suppresses set -e even inside the substitution. The chain gates on
        # every statement, which is exactly what set -e does to the real
        # (sequential) run block.
        set -Eeuo pipefail
        export PATH="$FRESH/bin:$PATH"
        APP_LOCATIONS_DIR="$FRESH/etc/nginx/serverkit-locations"; DRY_RUN=0
        APP_BASELINE="$(snapshot_app_reachability)" \
            && phase "Verifying App Uptime" \
            && APP_AFTER="$(snapshot_app_reachability)" \
            && { report_app_uptime_regressions "$APP_BASELINE" "$APP_AFTER" || true; }
    } 2>&1
)" || t28_rc=$?
if [ "$t28_rc" -eq 0 ] && printf '%s' "$out" | grep -q 'No managed apps'; then
    ok "contract: the run block's uptime-verification chain exits 0 on a fresh zero-app box"
else
    bad "contract: uptime verification FAILED a fresh-box update (rc=$t28_rc): [$out]"
fi

printf 'location /app { proxy_pass http://127.0.0.1:8009; }\n' > "$t/loc/app.conf"
t28_rc=0
out="$(
    {
        set -Eeuo pipefail
        APP_LOCATIONS_DIR="$t/loc"; DRY_RUN=0
        export PATH="$t/binup:$PATH"       # app reachable pre-switch
        APP_BASELINE="$(snapshot_app_reachability)" \
            && export PATH="$t/bindown:$PATH" `# app dead post-switch → regression` \
            && phase "Verifying App Uptime" \
            && APP_AFTER="$(snapshot_app_reachability)" \
            && { report_app_uptime_regressions "$APP_BASELINE" "$APP_AFTER" || true; }
    } 2>&1
)" || t28_rc=$?
if [ "$t28_rc" -eq 0 ] && printf '%s' "$out" | grep -q 'DOWN now'; then
    ok "contract: a real app regression warns loudly but never fails the healthy update"
else
    bad "contract: regression path rc=$t28_rc (must be 0, with the loud warning): [$out]"
fi

# --------------------------------------------------------------------------
# T29 — refresh_bash_completion: regenerates the completion file from the
# switched-in CLI, honours DRY_RUN, and no-ops on boxes without a
# bash-completion directory. (SERVERKIT_COMPLETION_DIR is the test seam.)
# --------------------------------------------------------------------------
t="$WORK/t29"; mkdir -p "$t/comp" "$t/comp-dry"
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
t29_rc=0
out="$(
    {
        set -Eeuo pipefail
        SERVERKIT_COMPLETION_DIR="$t/comp"; INSTALL_DIR="$REPO_ROOT"; DRY_RUN=0
        refresh_bash_completion
    } 2>&1
)" || t29_rc=$?
t29b_rc=0
out_dry="$(
    {
        set -Eeuo pipefail
        SERVERKIT_COMPLETION_DIR="$t/comp-dry"; INSTALL_DIR="$REPO_ROOT"; DRY_RUN=1
        refresh_bash_completion
    } 2>&1
)" || t29b_rc=$?
t29c_rc=0
(
    set -Eeuo pipefail
    SERVERKIT_COMPLETION_DIR="$t/absent"; INSTALL_DIR="$REPO_ROOT"; DRY_RUN=0
    refresh_bash_completion
) >/dev/null 2>&1 || t29c_rc=$?
if [ "$t29_rc" -eq 0 ] && grep -q 'complete -F _serverkit serverkit' "$t/comp/serverkit" \
   && [ "$t29b_rc" -eq 0 ] && printf '%s' "$out_dry" | grep -q 'dry-run' \
   && [ ! -f "$t/comp-dry/serverkit" ] \
   && [ "$t29c_rc" -eq 0 ]; then
    ok "refresh_bash_completion writes the file, honours dry-run, skips absent dirs"
else
    bad "refresh_bash_completion broken: rc=$t29_rc/$t29b_rc/$t29c_rc out=[$out] dry=[$out_dry]"
fi

# --------------------------------------------------------------------------
# T30 — retired plugins (the serverkit-workflows update-breaker of 1.7.25→):
# the carry-forward must NOT preserve a slug in RETIRED_PLUGIN_SLUGS (its dead
# imports would sink the bundle build in the new slot) while still carrying
# real user plugins; and remove_retired_plugins strips leftovers from an
# in-place tree (docker mode).
# --------------------------------------------------------------------------
t="$WORK/t30"
mkdir -p "$t/old/frontend/src/plugins/serverkit-workflows" \
         "$t/old/frontend/src/plugins/user-plugin" \
         "$t/new" \
         "$t/tree/frontend/src/plugins/serverkit-workflows" \
         "$t/tree/backend/app/plugins/serverkit-workflows"
printf 'import "reactflow";\n' > "$t/old/frontend/src/plugins/serverkit-workflows/index.jsx"
printf 'export default 1;\n'   > "$t/old/frontend/src/plugins/user-plugin/index.jsx"
if (
    set -Eeuo pipefail
    preserve_installed_plugins "$t/old" "$t/new" \
        && [ ! -e "$t/new/frontend/src/plugins/serverkit-workflows" ] \
        && [ -f "$t/new/frontend/src/plugins/user-plugin/index.jsx" ] \
        && remove_retired_plugins "$t/tree" \
        && [ ! -e "$t/tree/frontend/src/plugins/serverkit-workflows" ] \
        && [ ! -e "$t/tree/backend/app/plugins/serverkit-workflows" ]
) >/dev/null 2>&1; then
    ok "retired plugins are neither carried forward nor left in an in-place tree"
else
    bad "a retired plugin leaked into the new tree (the workflows update-breaker)"
fi

# --------------------------------------------------------------------------
# T31 — quarantine_carried_plugins: after a failed bundle build the carried
# frontends move aside (into BACKUP_DIR) so the build can be retried; with
# nothing carried it returns 1 so the caller halts with the original error.
# --------------------------------------------------------------------------
t="$WORK/t31"
mkdir -p "$t/new/frontend/src/plugins/broken-plugin"
printf 'import "gone";\n' > "$t/new/frontend/src/plugins/broken-plugin/index.jsx"
if (
    set -Eeuo pipefail
    BACKUP_DIR="$t/backups"
    CARRIED_FRONTEND_PLUGINS=" broken-plugin"
    quarantine_carried_plugins "$t/new" \
        && [ ! -e "$t/new/frontend/src/plugins/broken-plugin" ] \
        && ls "$t/backups"/quarantined-plugins-*/broken-plugin/index.jsx >/dev/null \
        && ! quarantine_carried_plugins "$t/new"
) >/dev/null 2>&1; then
    ok "quarantine_carried_plugins sets carried plugins aside; no-op when nothing carried"
else
    bad "quarantine_carried_plugins failed to move a carried plugin aside (or false-positived)"
fi

# --------------------------------------------------------------------------
# T32 — _node_build_ok gates on vite 8's Node floor (^20.19.0 || >=22.12.0) so a
# source-mode update on an old-Node box fails fast with an actionable message
# instead of a cryptic rolldown MODULE_NOT_FOUND mid-build (plan 47 / vite 8).
# --------------------------------------------------------------------------
t="$WORK/t32"; mkdir -p "$t/bin"
_mk_node_stub() { printf '#!/usr/bin/env bash\n[ "$1" = "--version" ] && echo "v%s"\nexit 0\n' "$1" > "$t/bin/node"; chmod +x "$t/bin/node"; }
node_gate_ok=1
for v in 20.19.0 22.12.0 24.1.0; do
    _mk_node_stub "$v"
    ( set -Eeuo pipefail; PATH="$t/bin:$PATH"; _node_build_ok ) || node_gate_ok=0
done
for v in 18.20.0 20.17.0 20.18.9 21.7.0 22.11.0; do
    _mk_node_stub "$v"
    ( set -Eeuo pipefail; PATH="$t/bin:$PATH"; _node_build_ok ) && node_gate_ok=0
done
if [ "$node_gate_ok" = 1 ]; then
    ok "_node_build_ok accepts 20.19+/22.12+ and rejects older (incl. 21.x, 22.11)"
else
    bad "_node_build_ok mis-gated a Node version against the vite 8 floor"
fi

# --------------------------------------------------------------------------
# T33 — build_frontend_bundle refuses (returns 1, never touches npm) on a
# too-old Node, so the caller rolls back instead of a half-built bundle.
# --------------------------------------------------------------------------
t="$WORK/t33"; mkdir -p "$t/bin" "$t/tree/frontend"
printf '#!/usr/bin/env bash\n[ "$1" = "--version" ] && echo "v20.17.0"\nexit 0\n' > "$t/bin/node"; chmod +x "$t/bin/node"
printf '#!/usr/bin/env bash\necho ran >> "%s/npmlog"\nexit 0\n' "$t" > "$t/bin/npm"; chmod +x "$t/bin/npm"
if (
    set -Eeuo pipefail
    PATH="$t/bin:$PATH"
    ! build_frontend_bundle "$t/tree"
) >/dev/null 2>&1 && [ ! -e "$t/npmlog" ]; then
    ok "build_frontend_bundle refuses on too-old Node without touching npm"
else
    bad "build_frontend_bundle ran npm (or returned 0) on a too-old Node"
fi

# --------------------------------------------------------------------------
# T34 — refresh_config renders the systemd unit with @BIND_HOST@ substituted
# (loopback), not left as a literal placeholder that would break gunicorn's bind
# after an update (parity with install.sh; the gap the operator caught).
# --------------------------------------------------------------------------
t="$WORK/t34"
mkdir -p "$t/nginx/sites-available" "$t/nginx/sites-enabled" \
         "$t/target/nginx/sites-available" "$t/target/templates" "$t/sysd"
printf 'http {\n}\n' > "$t/nginx/nginx.conf"
printf 'server { listen 80; }\n' > "$t/target/nginx/sites-available/serverkit-insecure.conf"
printf 'ExecStart=gunicorn -b @BIND_HOST@:@PORT@ run:app\n' \
    > "$t/target/templates/serverkit-backend.service.in"
(
    set -Eeuo pipefail
    NGINX_DIR="$t/nginx"; LETSENCRYPT_DIR="$t/le"; SYSTEMD_DIR="$t/sysd"; CONFIG_DIR="$t/cfg"
    INSTALL_DIR="$t/opt"; VENV_DIR="$t/venv"; LOG_DIR="$t/log"; DRY_RUN=0
    refresh_config "$t/target"
) >/dev/null 2>&1
if [ -f "$t/sysd/serverkit.service" ] \
   && ! grep -q '@BIND_HOST@' "$t/sysd/serverkit.service" \
   && grep -q '127.0.0.1:' "$t/sysd/serverkit.service"; then
    ok "refresh_config substitutes @BIND_HOST@ in the systemd unit (loopback bind)"
else
    bad "refresh_config left a literal @BIND_HOST@ (gunicorn would fail to bind after update)"
fi

# --------------------------------------------------------------------------
# T35 — updates default to release-mode (pre-built, Node-free) so the server
# never needs Node/npm; --source, --branch and BUILD_FROM_SOURCE=1 opt into an
# on-box source rebuild. Re-source update.sh with args and read the resolved
# (USE_RELEASE|TARGET_BRANCH) the deploy dispatch keys off (plan 47 / vite 8).
# --------------------------------------------------------------------------
_mode() { ( SERVERKIT_NO_SELF_UPDATE=1 source "$UPDATE_SH" "$@" >/dev/null 2>&1
            printf '%s|%s' "$USE_RELEASE" "$TARGET_BRANCH" ); }
d="$(_mode)"; s="$(_mode --source)"; b="$(_mode --branch dev)"; r="$(_mode --release)"
bfs="$(BUILD_FROM_SOURCE=1 _mode)"
if [ "$d" = "1|" ] && [ "$s" = "0|" ] && [ "$b" = "0|dev" ] \
   && [ "$r" = "1|" ] && [ "$bfs" = "0|" ]; then
    ok "update defaults to release (Node-free); --source/--branch/BUILD_FROM_SOURCE opt into on-box rebuild"
else
    bad "update mode mapping wrong: default=$d source=$s branch=$b release=$r build_from_source=$bfs"
fi

# --------------------------------------------------------------------------
# Install profiles: the minimal profile ships without Docker on purpose, so
# preflight must not demand it — otherwise every minimal box is installable but
# permanently un-updatable.
# --------------------------------------------------------------------------
pt="$WORK/profile"; mkdir -p "$pt/slot"
mkenv() { printf 'DATABASE_URL=sqlite:///x\n%s\n' "$1" > "$pt/slot/.env"; }

read_profile() {  # read_profile — echo install_profile() under a fixture slot
    ( set -Eeuo pipefail
      INSTALL_DIR="$pt/slot"; DIR_A="$pt/slot"; DIR_B="$pt/other"
      install_profile )
}

mkenv 'SERVERKIT_PROFILE=minimal'
if [ "$(read_profile)" = "minimal" ]; then
    ok "install_profile reads SERVERKIT_PROFILE from the active slot's .env"
else
    bad "install_profile misread a minimal .env: got [$(read_profile)]"
fi

mkenv 'SERVERKIT_PROFILE=banana'
if [ "$(read_profile)" = "standard" ]; then
    ok "install_profile falls back to standard on a bogus value"
else
    bad "install_profile accepted a bogus profile: got [$(read_profile)]"
fi

# Every install predating profiles has no such key and is effectively standard.
mkenv 'PORT=80'
if [ "$(read_profile)" = "standard" ]; then
    ok "install_profile treats a pre-profile .env as standard"
else
    bad "install_profile mishandled a pre-profile .env: got [$(read_profile)]"
fi

# No .env at all (fresh base dir) must not abort under pipefail.
if res="$( set -Eeuo pipefail; INSTALL_DIR="$pt/missing"; DIR_A="$pt/missing"
           DIR_B="$pt/other"; install_profile )"; then
    ok "install_profile survives a missing .env (got [$res])"
else
    bad "install_profile aborted with no .env under set -Eeuo pipefail"
fi

# The actual regression: minimal + no docker binary → Docker is not required.
# have_docker() is the seam; hiding docker by trimming PATH would also hide the
# coreutils install_profile parses with, and the assertion would then pass for
# the wrong reason.
mkenv 'SERVERKIT_PROFILE=minimal'
if ( set -Eeuo pipefail
     INSTALL_DIR="$pt/slot"; DIR_A="$pt/slot"; DIR_B="$pt/other"
     have_docker() { return 1; }
     update_needs_docker ); then
    bad "update_needs_docker still demands Docker on a minimal Dockerless box (update would halt)"
else
    ok "update_needs_docker: minimal + no docker → Docker not required"
fi

# ...but if the operator later installed Docker, keep validating it.
if ( set -Eeuo pipefail
     INSTALL_DIR="$pt/slot"; DIR_A="$pt/slot"; DIR_B="$pt/other"
     have_docker() { return 0; }
     update_needs_docker ); then
    ok "update_needs_docker: minimal + docker present → still validated"
else
    bad "update_needs_docker skipped a Docker that is actually installed"
fi

# A standard install without Docker is real drift and must still halt.
mkenv 'SERVERKIT_PROFILE=standard'
if ( set -Eeuo pipefail
     INSTALL_DIR="$pt/slot"; DIR_A="$pt/slot"; DIR_B="$pt/other"
     have_docker() { return 1; }
     update_needs_docker ); then
    ok "update_needs_docker: standard profile still requires Docker"
else
    bad "update_needs_docker let a standard install skip Docker"
fi

# An all-Docker deployment needs Docker no matter what the profile says.
mkenv 'SERVERKIT_PROFILE=minimal'
if ( set -Eeuo pipefail
     INSTALL_DIR="$pt/slot"; DIR_A="$pt/slot"; DIR_B="$pt/other"
     have_docker() { return 1; }
     is_docker_deployment() { return 0; }
     update_needs_docker ); then
    ok "update_needs_docker: all-Docker deployment overrides the minimal profile"
else
    bad "update_needs_docker skipped Docker for an all-Docker deployment"
fi

# --------------------------------------------------------------------------
# T10i — retention runs BEFORE the new backup is written, not only in
# cleanup(). cleanup() is the LAST phase, so it never ran on an update that
# died earlier — including one that died because the disk was full. A box was
# found carrying six snapshots under a cap of five for exactly that reason.
# --------------------------------------------------------------------------
t="$WORK/t10i"; mkdir -p "$t/backups"
for i in 1 2 3 4 5; do
    mkdir -p "$t/backups/serverkit-tree-2026010$i"
    : > "$t/backups/serverkit-pre-upgrade-2026010$i-00000$i.db"
    touch -d "2026-01-0$i 00:00" "$t/backups/serverkit-tree-2026010$i" \
        "$t/backups/serverkit-pre-upgrade-2026010$i-00000$i.db" 2>/dev/null || true
done
pre_rc=0
(
    set -Eeuo pipefail
    DRY_RUN=0 BACKUP_DIR="$t/backups" SERVERKIT_BACKUP_RETENTION=2
    trim_backups
) >/dev/null 2>&1 || pre_rc=$?
pre_trees=$(find "$t/backups" -mindepth 1 -maxdepth 1 -name 'serverkit-tree-*' | wc -l)
pre_dbs=$(find "$t/backups" -mindepth 1 -maxdepth 1 -name '*.db' | wc -l)
if [ "$pre_rc" -eq 0 ] && [ "$pre_trees" -eq 2 ] && [ "$pre_dbs" -eq 2 ]; then
    ok "trim_backups caps every kind and is callable before the backup is written"
else
    bad "trim_backups: rc=$pre_rc trees=$pre_trees dbs=$pre_dbs (want 2/2)"
fi

if grep -A16 '^backup_current()' "$UPDATE_SH" | grep -q 'trim_backups'; then
    ok "backup_current trims BEFORE writing the new snapshot (survives a failed run)"
else
    bad "backup_current does not trim before writing — a failed update leaves backups over cap"
fi

# --------------------------------------------------------------------------
# T10j — the size budget drops oldest-first until backups fit their share of
# the disk. A count cap alone is not a disk guarantee: five snapshots of an
# 800 MB database is 8 GB, a third of a 25 GB droplet.
# --------------------------------------------------------------------------
t="$WORK/t10j"; mkdir -p "$t/backups"
for i in 1 2 3; do
    dd if=/dev/zero of="$t/backups/serverkit-pre-upgrade-2026010$i-00000$i.db" \
        bs=1024 count=200 2>/dev/null
    touch -d "2026-01-0$i 00:00" "$t/backups/serverkit-pre-upgrade-2026010$i-00000$i.db" 2>/dev/null || true
done
bud_rc=0
(
    set -Eeuo pipefail
    DRY_RUN=0 BACKUP_DIR="$t/backups"
    SERVERKIT_BACKUP_MAX_PERCENT=0
    enforce_backup_budget
) >/dev/null 2>&1 || bud_rc=$?
left=$(find "$t/backups" -mindepth 1 -maxdepth 1 -name 'serverkit-pre-upgrade-*' | wc -l)
newest_left=$(find "$t/backups" -name 'serverkit-pre-upgrade-20260103-*' | wc -l)
if [ "$bud_rc" -eq 0 ] && [ "$left" -eq 1 ] && [ "$newest_left" -eq 1 ]; then
    ok "enforce_backup_budget trims oldest-first and always keeps one restore point"
else
    bad "budget: rc=$bud_rc left=$left newest_kept=$newest_left (want 1/1)"
fi

t="$WORK/t10k"; mkdir -p "$t/backups"
: > "$t/backups/serverkit-pre-upgrade-20260101-000001.db"
: > "$t/backups/serverkit-pre-upgrade-20260102-000002.db"
(
    set -Eeuo pipefail
    DRY_RUN=0 BACKUP_DIR="$t/backups" SERVERKIT_BACKUP_MAX_PERCENT=99
    enforce_backup_budget
) >/dev/null 2>&1 || true
if [ "$(find "$t/backups" -name '*.db' | wc -l)" -eq 2 ]; then
    ok "enforce_backup_budget is a no-op when backups already fit the budget"
else
    bad "enforce_backup_budget deleted backups that were within budget"
fi

# --------------------------------------------------------------------------
# T10l — the disk preflight sizes its requirement to what the update will
# actually write (2 GiB + TWO copies of the database), not a flat 2 GiB. With
# an 800 MB DB the old check passed at 2.1 GiB free and then filled the disk.
# --------------------------------------------------------------------------
if grep -q 'db_kb \* 2' "$UPDATE_SH"; then
    ok "preflight sizes the disk requirement to two database copies"
else
    bad "preflight still uses a flat disk floor — an 800 MB DB overruns it"
fi
if grep -q 'trimming old backups' "$UPDATE_SH"; then
    ok "preflight reclaims stale snapshots before halting on low disk"
else
    bad "preflight halts on low disk without first trimming reclaimable backups"
fi

# --------------------------------------------------------------------------
if (
    set -Eeuo pipefail
    source_tree="$WORK/snapshot-source"
    target_tree="$WORK/snapshot-target"
    mkdir -p "$source_tree/backend/instance" "$source_tree/backend/venv" "$source_tree/node_modules"
    printf 'code' > "$source_tree/app.py"
    printf 'settings' > "$source_tree/backend/instance/settings.json"
    touch "$source_tree/backend/instance/serverkit.db" \
        "$source_tree/backend/instance/serverkit.db-wal" \
        "$source_tree/backend/instance/serverkit.db-shm" \
        "$source_tree/backend/venv/dependency" "$source_tree/node_modules/dependency"
    rsync() { return 127; }
    snapshot_install_tree "$source_tree" "$target_tree"
    [ "$(cat "$target_tree/app.py")" = code ]
    [ "$(cat "$target_tree/backend/instance/settings.json")" = settings ]
    [ ! -e "$target_tree/backend/instance/serverkit.db" ]
    [ ! -e "$target_tree/backend/instance/serverkit.db-wal" ]
    [ ! -e "$target_tree/backend/instance/serverkit.db-shm" ]
    [ ! -e "$target_tree/backend/venv" ]
    [ ! -e "$target_tree/node_modules" ]
); then
    ok "tree snapshot fallback preserves code and excludes database copies and dependencies"
else
    bad "tree snapshot fallback loses files or duplicates excluded data"
fi

printf '\n%d passed, %d failed, %d skipped\n\n' "$PASS" "$FAIL" "$SKIP"
[ "$FAIL" -eq 0 ]