#!/bin/bash # # ServerKit updater — atomic blue/green, pre-flight checked, offline-capable. # # Updates default to a PRE-BUILT release tarball — the frontend is built once in # CI and shipped, so the server needs no Node/npm and no on-box SPA build (that's # what makes updates work uniformly on old distros, ARM/Pi, and tiny boxes). Opt # into an on-box source rebuild with --source / --branch / BUILD_FROM_SOURCE=1 # (that path needs a compatible Node — see install.sh's node floor). # # Usage: # bash /opt/serverkit/scripts/update.sh # pre-built release (default, Node-free) # bash /opt/serverkit/scripts/update.sh --dry-run # bash /opt/serverkit/scripts/update.sh --source # rebuild from main on this box (needs Node) # bash /opt/serverkit/scripts/update.sh --branch dev # rebuild from a branch on this box (needs Node) # bash /opt/serverkit/scripts/update.sh --release [v1.7.0] # SERVERKIT_OFFLINE_TARBALL=/tmp/serverkit-v1.7.0-linux-amd64.tar.gz bash /opt/serverkit/scripts/update.sh # # -E (errtrace) makes the ERR trap fire for failures *inside* functions/subshells # too — without it a silent death deep in a helper would never be reported. set -Eeuo pipefail # --------------------------------------------------------------------------- # Configuration + argument parsing # --------------------------------------------------------------------------- DRY_RUN=0 FORCE_UPDATE=0 TARGET_BRANCH="" # Default to a pre-built release update (Node-free). --source / --branch / # BUILD_FROM_SOURCE=1 opt into an on-box source rebuild instead. USE_RELEASE="${INSTALL_FROM_RELEASE:-1}" [ "${BUILD_FROM_SOURCE:-0}" = "1" ] && USE_RELEASE=0 RELEASE_VERSION="${SERVERKIT_VERSION:-}" # Captured before parsing so the self-update re-exec can forward them verbatim. ORIG_ARGS=("$@") while [[ $# -gt 0 ]]; do case "$1" in --dry-run|-n) DRY_RUN=1 shift ;; --force|-f) FORCE_UPDATE=1 shift ;; --source|--main) # Rebuild from the main branch on this box (needs a compatible Node). USE_RELEASE=0 TARGET_BRANCH="" shift ;; --branch|-b) # A branch update is an on-box source rebuild — override the release default. TARGET_BRANCH="$2" USE_RELEASE=0 shift 2 ;; --release|-r) USE_RELEASE=1 if [[ -n "${2:-}" ]] && [[ ! "$2" =~ ^- ]]; then RELEASE_VERSION="$2" shift 2 else shift fi ;; --help|-h) cat <<'EOF' Usage: update.sh [OPTIONS] Options: --dry-run, -n Show what would happen without making changes --force, -f Skip version comparison and update anyway --release [version], -r Update from a pre-built release tarball (DEFAULT; Node-free) --source, --main Rebuild from the main branch on this box (needs Node) --branch , -b Rebuild from a git branch on this box (needs Node) --help, -h Show this help message Environment: SERVERKIT_DIR Active install directory (default: /opt/serverkit) SERVERKIT_VENV_DIR Python venv path (default: $SERVERKIT_DIR/venv) SERVERKIT_OFFLINE_TARBALL Local release tarball to use instead of downloading SERVERKIT_MIRROR_URL Base URL for release tarballs/checksums GITHUB_REPO GitHub org/repo for source and releases EOF exit 0 ;; *) echo "Unknown option: $1" >&2 exit 1 ;; esac done # --------------------------------------------------------------------------- # Paths # --------------------------------------------------------------------------- SERVERKIT_DIR="${SERVERKIT_DIR:-/opt/serverkit}" INSTALL_DIR="$SERVERKIT_DIR" BASE_NAME="$(basename "$INSTALL_DIR")" BASE_DIR="$(dirname "$INSTALL_DIR")" DIR_A="$BASE_DIR/${BASE_NAME}-a" DIR_B="$BASE_DIR/${BASE_NAME}-b" VENV_DIR="${SERVERKIT_VENV_DIR:-$INSTALL_DIR/venv}" BACKUP_DIR="${SERVERKIT_BACKUP_DIR:-/var/backups/serverkit}" LOG_DIR="/var/log/serverkit" CONFIG_DIR="${SERVERKIT_CONFIG_DIR:-/etc/serverkit}" LOCK_FILE="${SERVERKIT_LOCK_FILE:-/var/lock/serverkit-update.lock}" # System integration dirs — overridable so the config-refresh logic can be # exercised against fixtures in tests instead of the host's real /etc. NGINX_DIR="${SERVERKIT_NGINX_DIR:-/etc/nginx}" LETSENCRYPT_DIR="${SERVERKIT_LETSENCRYPT_DIR:-/etc/letsencrypt}" SYSTEMD_DIR="${SERVERKIT_SYSTEMD_DIR:-/etc/systemd/system}" # Per-app nginx location snippets — one proxy_pass to each managed app's # container. The updater probes these to prove apps stayed up across the switch. APP_LOCATIONS_DIR="${SERVERKIT_APP_LOCATIONS_DIR:-/etc/nginx/serverkit-locations}" GITHUB_REPO="${GITHUB_REPO:-jhd3197/ServerKit}" SERVERKIT_OFFLINE_TARBALL="${SERVERKIT_OFFLINE_TARBALL:-}" SERVERKIT_MIRROR_URL="${SERVERKIT_MIRROR_URL:-}" BACKEND_SERVICE="serverkit" # --------------------------------------------------------------------------- # Terminal styling # --------------------------------------------------------------------------- if [ -t 1 ] && [ -z "${NO_COLOR:-}" ] && [ "${TERM:-dumb}" != "dumb" ]; then ESC=$'\033' RST="${ESC}[0m"; BLD="${ESC}[1m" paint() { printf '%s[38;2;%d;%d;%dm' "$ESC" "$1" "$2" "$3"; } else RST=''; BLD='' paint() { :; } fi V3="$(paint 139 92 246)"; V4="$(paint 124 58 237)" PAPER="$(paint 237 233 254)"; FOG="$(paint 113 108 140)" HUE_OK="$(paint 52 211 153)"; HUE_WARN="$(paint 250 204 21)" HUE_ERR="$(paint 248 113 113)"; HUE_LINK="$(paint 103 232 249)" good() { printf ' %s✔%s %s\n' "$HUE_OK" "$RST" "$1"; } warn() { printf ' %s▴%s %s\n' "$HUE_WARN" "$RST" "$1"; } # Every hard failure points at the full log so a stuck box is debuggable from a # single line (UPDATE_LOG is empty until init_logging runs, hence the guard). halt() { printf ' %s✘%s %s\n' "$HUE_ERR" "$RST" "$1" >&2 [ -n "${UPDATE_LOG:-}" ] && printf ' %sfull log: %s%s\n' "$FOG" "$UPDATE_LOG" "$RST" >&2 exit 1 } step() { printf ' %s❯%s %s\n' "$HUE_LINK" "$RST" "$1"; } info() { printf ' %s•%s %s\n' "$FOG" "$RST" "$1"; } STARTED_AT=0 PHASE_N=0 clock() { [ "$STARTED_AT" -gt 0 ] || { printf ''; return; } local secs=$(( $(date +%s) - STARTED_AT )) printf '%dm %02ds' "$((secs / 60))" "$((secs % 60))" } LAST_PHASE="startup" phase() { PHASE_N=$((PHASE_N + 1)) LAST_PHASE="$1" printf '\n %s%s%02d%s %s%s%s %s%s%s\n' \ "$BLD" "$V3" "$PHASE_N" "$RST" "$BLD" "$1" "$RST" "$FOG" "$(clock)" "$RST" printf ' %s%s%s\n\n' "$V4" "──────────────────────────────────────" "$RST" } # --------------------------------------------------------------------------- # L5 — Loud failures + always-on logging # --------------------------------------------------------------------------- # `set -euo pipefail` makes any unguarded command abort the script. Without this # layer that abort is *silent* — you just land back at the prompt with no idea # which phase, line, or command died (exactly the failure mode that made this # updater so hard to debug). init_logging mirrors everything to a timestamped # log; the ERR trap turns every abort into a labelled, actionable message. UPDATE_LOG="" init_logging() { [ -n "${SERVERKIT_NO_LOG:-}" ] && return 0 mkdir -p "$LOG_DIR" 2>/dev/null || true if [ -d "$LOG_DIR" ] && [ -w "$LOG_DIR" ]; then UPDATE_LOG="$LOG_DIR/update-$(date +%Y%m%d-%H%M%S).log" # Keep output on the terminal *and* append it to the log. Colors were # already resolved above from the real TTY, so they survive the pipe. exec > >(tee -a "$UPDATE_LOG") 2>&1 info "Logging to $UPDATE_LOG" fi } report_failure() { local rc=$1 line=$2 cmd=$3 printf '\n %s✘ Update aborted%s during %s%s%s\n' \ "$HUE_ERR" "$RST" "$BLD" "$LAST_PHASE" "$RST" >&2 printf ' %sexit %s · line %s · %s%s\n' "$FOG" "$rc" "$line" "$cmd" "$RST" >&2 # `if`, not a trailing `&&` list: with UPDATE_LOG empty (fresh box before # init_logging, SERVERKIT_NO_LOG=1) the list form returns 1 from the # reporter itself — the same species as the July 2 outage. if [ -n "$UPDATE_LOG" ]; then printf ' %sfull log: %s%s\n' "$FOG" "$UPDATE_LOG" "$RST" >&2 fi } # --------------------------------------------------------------------------- # Self-updating bootstrap + run lock # --------------------------------------------------------------------------- # The updater that runs is whatever is installed on the box — so a *stale* # update.sh (one predating a bug fix or a new deployment shape) fails in ways # the current code already handles, and it can't fix itself. This is the exact # trap that left a box stuck: an old updater died before it could install the # fixed one. Before doing any work, fetch the newest update.sh for the target # ref and re-exec into it. From this version on, "just run serverkit update" # is reliable no matter how old the box is. # # CHANNEL DECISION (2026-07, scripts-reliability round 2): stable installs keep # fetching this file from *main* — no release buffer. The flip side is that any # updater bug merged to main is live on every box instantly, so merges touching # scripts/** are gated by the real install+update e2e job in scripts-ci.yml # (update-e2e). Fetching from the latest release tag was considered and # rejected: a broken *released* updater would need a whole new release to fix, # while main + a CI gate keeps the one-push-fixes-the-fleet property that # resolved the 2026-07-02 incident. SELF_PATH="${BASH_SOURCE[0]}" maybe_reexec_latest_updater() { [ -n "${SERVERKIT_UPDATER_REEXECED:-}" ] && return 0 # already the latest [ "${SERVERKIT_NO_SELF_UPDATE:-0}" = "1" ] && return 0 # opt-out / tests [ "$DRY_RUN" = "1" ] && return 0 [ -n "$SERVERKIT_OFFLINE_TARBALL" ] && return 0 # offline: nothing to fetch command -v curl &>/dev/null || return 0 local ref="main" [ -n "$TARGET_BRANCH" ] && ref="$TARGET_BRANCH" [ "$USE_RELEASE" = "1" ] && [ -n "$RELEASE_VERSION" ] && ref="$RELEASE_VERSION" local url tmp url="https://raw.githubusercontent.com/${GITHUB_REPO}/${ref}/scripts/update.sh" tmp="$(mktemp)" if ! curl -fsSL --max-time 20 "$url" -o "$tmp" 2>/dev/null || [ ! -s "$tmp" ]; then rm -f "$tmp"; return 0 # fetch failed → use local fi # Swap only if it genuinely differs AND is valid bash (never re-exec into a # truncated/corrupt download). if ! cmp -s "$tmp" "$SELF_PATH" && bash -n "$tmp" 2>/dev/null; then info "Refreshing the updater itself from ${ref}..." chmod +x "$tmp" 2>/dev/null || true # ${arr[@]+...}: expanding an EMPTY array as "${arr[@]}" is an # unbound-variable error under set -u on bash < 4.4 (older distros). SERVERKIT_UPDATER_REEXECED=1 exec bash "$tmp" ${ORIG_ARGS[@]+"${ORIG_ARGS[@]}"} fi rm -f "$tmp" } # Stop two concurrent updates clobbering each other (e.g. both cloning into the # same blue/green slot, or racing docker compose). The lock auto-releases when # the script exits, since the fd closes. acquire_update_lock() { [ "$DRY_RUN" = "1" ] && return 0 command -v flock &>/dev/null || return 0 mkdir -p "$(dirname "$LOCK_FILE")" 2>/dev/null || true # Brace-group so a failed open (unwritable path) is caught here instead of # the redirection error aborting the whole script — then just skip locking. { exec 9>"$LOCK_FILE"; } 2>/dev/null || return 0 if ! flock -n 9; then halt "Another 'serverkit update' is already running (lock: $LOCK_FILE)." fi } # --------------------------------------------------------------------------- # Utility helpers # --------------------------------------------------------------------------- run_or_dry() { if [ "$DRY_RUN" = "1" ]; then info "[dry-run] would run: $*" else "$@" fi } # Like run_or_dry, but runs the command inside . Under --dry-run it reports # the working directory it *would* run in instead of silently dropping the cd — # a chained `run_or_dry cd X && run_or_dry docker compose ...` used to print the # compose command with no hint of where it would execute. run_in_dir() { local dir="$1"; shift if [ "$DRY_RUN" = "1" ]; then info "[dry-run] (cwd: $dir) would run: $*" else ( cd "$dir" && "$@" ) fi } # --------------------------------------------------------------------------- # Version comparison — skip the whole update when already current. # --------------------------------------------------------------------------- local_version() { tr -d '\n\r ' < "$INSTALL_DIR/VERSION" 2>/dev/null || true } # True when two version strings match ignoring a leading "v" (1.7.1 == v1.7.1). versions_equal() { [ "${1#v}" = "${2#v}" ] } # Resolve the release tag we would update to (pinned, or the latest on GitHub). remote_release_tag() { if [ -n "$RELEASE_VERSION" ]; then printf '%s' "$RELEASE_VERSION" return 0 fi curl -sf "https://api.github.com/repos/${GITHUB_REPO}/releases/latest" 2>/dev/null \ | grep '"tag_name"' | head -1 | cut -d'"' -f4 || true } # Return 0 when the install is already at the target version/ref (the caller # should then skip). Best-effort: --force always proceeds; offline always # proceeds (can't compare); an indeterminate comparison proceeds with the # update rather than wrongly blocking it. is_already_current() { [ "$FORCE_UPDATE" = "1" ] && return 1 [ -n "$SERVERKIT_OFFLINE_TARBALL" ] && return 1 if [ "$USE_RELEASE" = "1" ]; then local target local_v target="$(remote_release_tag)" [ -n "$target" ] || return 1 local_v="$(local_version)" [ -n "$local_v" ] || return 1 versions_equal "$local_v" "$target" && return 0 return 1 fi # Branch / main mode: compare the local checkout's HEAD against the remote # branch HEAD. ls-remote needs no fetch and works on shallow clones. local branch local_head remote_head branch="${TARGET_BRANCH:-main}" git -C "$INSTALL_DIR" rev-parse --is-inside-work-tree >/dev/null 2>&1 || return 1 local_head="$(git -C "$INSTALL_DIR" rev-parse HEAD 2>/dev/null || true)" remote_head="$(git -C "$INSTALL_DIR" ls-remote "https://github.com/${GITHUB_REPO}.git" \ "refs/heads/$branch" 2>/dev/null | awk '{print $1}' | head -1)" [ -n "$local_head" ] && [ -n "$remote_head" ] && [ "$local_head" = "$remote_head" ] && return 0 return 1 } # If already current, announce it and exit cleanly (unless --force). version_gate() { if is_already_current; then local v; v="$(local_version)" good "Already up to date (version ${v:-unknown}). Use --force to update anyway." exit 0 fi } wait_for_service() { local unit="$1" target_state="$2" timeout="${3:-30}" local waited=0 while [ "$waited" -lt "$timeout" ]; do if systemctl is-active --quiet "$unit" 2>/dev/null; then [ "$target_state" = "active" ] && return 0 else [ "$target_state" = "inactive" ] && return 0 fi sleep 1 waited=$((waited + 1)) done return 1 } # Apply the (possibly refreshed) nginx config WITHOUT dropping traffic. # # Host nginx is the front door for every managed app — it reverse-proxies their # containers via /etc/nginx/serverkit-locations/*.conf. Stopping it (the old # behaviour) blacked out every hosted site for the whole switch window, so a # panel update became an outage for unrelated apps. `nginx -s reload` keeps the # listening sockets and in-flight connections alive while swapping config, so # apps never blink. The updater only ever changes the *config* (refresh_config), # never anything nginx serves from the blue/green slot, so a reload is all that # is ever needed. If nginx happens to be down, start it instead. reload_nginx_graceful() { if [ "$DRY_RUN" = "1" ]; then info "[dry-run] would reload nginx (graceful, zero-downtime)" return 0 fi if systemctl is-active --quiet nginx 2>/dev/null; then # Validate before reloading: a bad config would otherwise leave nginx # running the OLD config (reload is rejected) — non-fatal, but warn loudly. if nginx -t >/dev/null 2>&1; then systemctl reload nginx 2>/dev/null \ || run_or_dry systemctl reload nginx \ || warn "nginx reload failed — apps keep serving the previous config" else warn "nginx config test failed — skipping reload (apps keep serving the previous config)" fi else warn "nginx was not running — starting it" # Never propagate a failed start: nginx being down is a pre-existing # condition, and aborting here (post-switch) would fire a rollback that # cannot fix nginx either — warn and let the operator handle it. run_or_dry systemctl start nginx \ || warn "nginx failed to start — check 'systemctl status nginx'" wait_for_service nginx active 15 || warn "nginx did not report active within 15 seconds" fi } # Label a slot's SPA bundle for SELinux-enforcing hosts so nginx can read the # static panel assets it now serves (see install.sh selinux_label_frontend_dist). # Best-effort and a no-op on permissive/disabled boxes or where the tools are # absent. Takes the slot's real path (blue/green resolves through a symlink). selinux_label_dist() { local dir="$1" [ "$DRY_RUN" = "1" ] && return 0 command -v selinuxenabled &>/dev/null && selinuxenabled 2>/dev/null || return 0 local dist="${dir}/frontend/dist" [ -d "$dist" ] || return 0 if command -v semanage &>/dev/null && command -v restorecon &>/dev/null; then semanage fcontext -a -t httpd_sys_content_t "${dist}(/.*)?" 2>/dev/null \ || semanage fcontext -m -t httpd_sys_content_t "${dist}(/.*)?" 2>/dev/null || true restorecon -R "$dist" 2>/dev/null || true elif command -v chcon &>/dev/null; then chcon -R -t httpd_sys_content_t "$dist" 2>/dev/null || true fi } # --------------------------------------------------------------------------- # Managed-app uptime verification # --------------------------------------------------------------------------- # The whole point of the reload-not-stop discipline is that hosted apps never go # down when the panel updates. These helpers turn that promise into something the # updater actually checks: snapshot which app upstreams are reachable before the # switch, re-probe after the reload, and shout if any app that WAS serving has # stopped answering. # Discover the upstreams (host:port) host nginx reverse-proxies for managed apps # by scanning the per-app location snippets. One unique "host:port" per line. discover_app_upstreams() { [ -d "$APP_LOCATIONS_DIR" ] || return 0 grep -rhoE 'proxy_pass[[:space:]]+https?://127\.0\.0\.1:[0-9]+' "$APP_LOCATIONS_DIR" 2>/dev/null \ | grep -oE '127\.0\.0\.1:[0-9]+' | sort -u || true } # Probe each upstream on stdin; emit "host:port up" / "host:port down". "up" means # the app container answered at all (ANY HTTP status — a 4xx/5xx app is still # reachable; a refused/timed-out connection is not), which is exactly what nginx # needs to keep fronting it. probe_app_upstreams() { local up while IFS= read -r up; do [ -n "$up" ] || continue if curl -s -o /dev/null --max-time 4 "http://$up/" 2>/dev/null; then printf '%s up\n' "$up" else printf '%s down\n' "$up" fi done } # A reachability snapshot for every fronted app: "host:port state" lines. snapshot_app_reachability() { [ "$DRY_RUN" = "1" ] && return 0 discover_app_upstreams | probe_app_upstreams } # Compare a pre-update snapshot with a post-update one. Warn for any app that was # reachable before and is not now (the update disrupted a workload). Non-zero if # any regression is found; an app that was already down before the update is not # counted against us. report_app_uptime_regressions() { local before="$1" after="$2" regressed=0 total=0 line up state while IFS= read -r line; do [ -n "$line" ] || continue up="${line% *}"; state="${line##* }" total=$((total + 1)) [ "$state" = "up" ] || continue if printf '%s\n' "$after" | grep -qx "$up down"; then warn "App upstream $up was reachable before the update but is DOWN now" regressed=$((regressed + 1)) fi done </dev/null \ | head -1 | cut -d= -f2- | tr -d ' \r')" fi fi case "$profile" in minimal|standard|full) printf '%s' "$profile" ;; *) printf 'standard' ;; esac } # Whether this update has to insist on Docker. The minimal profile ships # without it on purpose, so demanding it here would make every minimal box # installable but un-updatable. Two escape hatches keep that honest: an # all-Docker deployment obviously needs it, and if Docker is present at all we # still validate compose so a half-provisioned box is caught early. # Split out so the unit tests can redefine it. Restricting PATH to hide docker # would also hide the coreutils install_profile parses with, and the test would # then pass for the wrong reason. have_docker() { command -v docker &>/dev/null; } update_needs_docker() { [ "$(install_profile)" = "minimal" ] || return 0 is_docker_deployment && return 0 have_docker && return 0 return 1 } # --------------------------------------------------------------------------- # Pre-flight checks # --------------------------------------------------------------------------- preflight_check() { phase "Pre-flight Checks" # Root if [ "$EUID" -ne 0 ]; then halt "Run as root." fi good "Running as root" # Install directory exists or can be created if [ ! -e "$BASE_DIR" ]; then halt "Base directory $BASE_DIR does not exist." fi good "Install base directory exists" # Python version local py_bin py_version py_bin="$(locate_python 2>/dev/null || true)" if [ -z "$py_bin" ]; then halt "Python 3.11-3.14 is required. Install a supported python3.1x and python3-venv." fi py_version="$($py_bin -c 'import sys; print(".".join(map(str, sys.version_info[:2])))')" good "Python $py_version available ($py_bin)" # Required commands. Source-mode updates build the SPA on the host, so npm # must be checked up front — discovering it missing mid-update would abort # after the new slot is already half-built. Release tarballs ship a # prebuilt dist and stay npm-free. local cmd missing=() required=(git curl tar rsync systemctl nginx python3) [ "$USE_RELEASE" = "1" ] || required+=(npm) # Docker is conditional — see update_needs_docker(). if update_needs_docker; then required+=(docker) fi for cmd in "${required[@]}"; do command -v "$cmd" &>/dev/null || missing+=("$cmd") done if update_needs_docker; then if ! docker compose version &>/dev/null && ! docker-compose --version &>/dev/null; then missing+=("docker compose") fi fi if [ ${#missing[@]} -gt 0 ]; then halt "Missing required tools: ${missing[*]}" fi good "Required tools available" if ! update_needs_docker; then good "Docker not required (minimal profile)" fi # Disk space. # # A fixed 2 GiB floor is not enough on its own: this update is about to # write TWO full copies of the database (a pre-upgrade snapshot and a tree # backup that contains it) plus a release tree. With an 800 MB database # that is ~2 GiB of new files, so a box with 2.1 GiB free passes the old # check and then fills the disk mid-update. Size the requirement to what # will actually be written, and reclaim stale snapshots before giving up. local avail_kb need_kb db_kb active_dir # -P (POSIX format) pins each filesystem to one line — a long device name # otherwise wraps and shifts the column the awk parse reads. avail_kb="$(df -Pk "$BASE_DIR" | awk 'NR==2 {print $4}')" active_dir="$(active_real_dir 2>/dev/null || echo "$INSTALL_DIR")" db_kb="$(du -sk "$active_dir/backend/instance/serverkit.db" 2>/dev/null | awk '{print $1}')" [ -n "$db_kb" ] || db_kb=0 # 2 GiB floor + room for both database copies. need_kb=$(( 2 * 1024 * 1024 + db_kb * 2 )) if [ "$avail_kb" -lt "$need_kb" ]; then warn "Low disk: $((avail_kb / 1024)) MiB free, need $((need_kb / 1024)) MiB — trimming old backups" trim_backups enforce_backup_budget avail_kb="$(df -Pk "$BASE_DIR" | awk 'NR==2 {print $4}')" fi if [ "$avail_kb" -lt "$need_kb" ]; then halt "Insufficient disk space on $BASE_DIR: $((avail_kb / 1024)) MiB free, need $((need_kb / 1024)) MiB (2 GiB + two $((db_kb / 1024)) MiB database copies). Run 'sudo serverkit disk' to reclaim space." fi good "Disk space OK ($((avail_kb / 1024 / 1024)) GiB free, need $((need_kb / 1024 / 1024)) GiB)" # Memory (need 512 MiB free) local mem_avail_mb mem_avail_mb="$(awk '/MemAvailable:/ {print int($2/1024)}' /proc/meminfo 2>/dev/null || echo 0)" if [ "$mem_avail_mb" -lt 512 ]; then warn "Low memory: ${mem_avail_mb} MiB available (recommend >= 512 MiB)." else good "Memory OK (${mem_avail_mb} MiB available)" fi # Network reachability (skip in offline mode) if [ -n "$SERVERKIT_OFFLINE_TARBALL" ]; then info "Offline tarball set; skipping network checks" elif [ "$USE_RELEASE" = "1" ] || [ -z "$TARGET_BRANCH" ]; then if ! curl -sfI "https://github.com" >/dev/null 2>&1; then halt "Cannot reach github.com. Set SERVERKIT_OFFLINE_TARBALL or fix network." fi good "Network reachability OK" fi # Current backend health (warn only; it may already be down) if curl -sf --max-time 5 http://127.0.0.1:5000/api/v1/system/health >/dev/null 2>&1; then good "Backend currently healthy" else warn "Backend is not currently responding on :5000 — will proceed anyway" fi } # --------------------------------------------------------------------------- # Python virtual environment # --------------------------------------------------------------------------- locate_python() { local c v for c in python3.14 python3.13 python3.12 python3.11 python3; do if command -v "$c" &>/dev/null; then v="$($c -c 'import sys;print(".".join(map(str,sys.version_info[:2])))' 2>/dev/null || true)" # An EMPTY version means the binary/shim exists but can't actually # execute Python — and an empty string would pass the sort checks # below ("" sorts before everything). Reject it and move on. [ -n "$v" ] || continue if printf '%s\n%s' "3.11" "$v" | sort -C -V && \ printf '%s\n%s' "$v" "3.14" | sort -C -V; then printf '%s' "$c" return 0 fi fi done return 1 } rebuild_virtualenv() { local target_dir="$1" step "Rebuilding Python virtual environment..." local py_bin py_bin="$(locate_python)" || halt "ServerKit requires Python 3.11-3.14." if [ "$DRY_RUN" = "1" ]; then info "[dry-run] would recreate venv at $target_dir using $py_bin" return 0 fi rm -rf "$target_dir" "$py_bin" -m venv "$target_dir" # shellcheck source=/dev/null source "$target_dir/bin/activate" pip install --upgrade pip --quiet pip install -r "$target_dir/../backend/requirements.txt" --quiet pip install gunicorn --quiet good "Virtual environment rebuilt at $target_dir" } # Ensure the target directory has a usable venv. If a pre-built one exists AND # is bound to this slot's path, use it; otherwise rebuild from requirements. require_venv() { local target_dir="$1" if [ -f "$target_dir/bin/activate" ] && [ -x "$target_dir/bin/python" ]; then # A venv is path-bound: bin/activate and every entry-point shebang # embed the absolute path the venv was created at. The release # tarball's prebuilt venv is baked as $INSTALL_DIR/venv, which — # until atomic_switch flips the symlink — resolves to the OLD slot. # Running it would execute the new code with the old slot's # dependencies (and run the OLD flask binary for migrations), so a # venv bound to any other path must be rebuilt in place. local bound_path resolved_bound resolved_target # Matches both VIRTUAL_ENV="/path" (classic venv) and the unquoted # export VIRTUAL_ENV=/path form; skips cygpath-style indirection. bound_path="$(grep -m1 -oE 'VIRTUAL_ENV="?/[^")]+"?' "$target_dir/bin/activate" | cut -d= -f2- | tr -d '"')" # Compare RESOLVED paths: on a fresh install the symlink already points # at the only slot, so the baked path resolves to the target and the # prebuilt venv is genuinely usable (fast path). During an update it # resolves to the OLD slot — that mismatch triggers the rebuild. resolved_bound="$(readlink -f "$bound_path" 2>/dev/null || true)" resolved_target="$(readlink -f "$target_dir" 2>/dev/null || echo "$target_dir")" if [ -n "$resolved_bound" ] && [ "$resolved_bound" = "$resolved_target" ]; then good "Virtual environment ready at $target_dir" return 0 fi warn "Virtual environment at $target_dir is bound to '${bound_path:-unknown path}' — rebuilding for this slot" rebuild_virtualenv "$target_dir" return 0 fi warn "Virtual environment missing at $target_dir" rebuild_virtualenv "$target_dir" } # --------------------------------------------------------------------------- # Database migration # --------------------------------------------------------------------------- migrate_database() { local work_dir="$1" local venv="$work_dir/venv" step "Running database migrations..." if [ "$DRY_RUN" = "1" ]; then info "[dry-run] would run flask db upgrade in $work_dir/backend" return 0 fi # For SQLite installs, migrate the COPY we placed in the *new* slot, not the # database the .env points at via the /opt/serverkit symlink (which still # resolves to the live OLD slot until atomic_switch). Touching the old slot's # DB here would corrupt rollback: reverting the symlink would hand the old # code a database already upgraded to the new schema. Pointing the migration # at the slot-absolute path keeps the old slot's DB byte-for-byte intact. # # Subshell: the venv activation and the cd must not leak into the main # shell — the rest of the update keeps running from the caller's cwd. local slot_db="$work_dir/backend/instance/serverkit.db" local use_slot_db=0 if grep -qE '^DATABASE_URL=sqlite' "$work_dir/.env" 2>/dev/null && [ -f "$slot_db" ]; then use_slot_db=1 # Fail fast with an actionable message when the SQLite slot copy is # corrupt, instead of dying deep inside app boot with "database disk # image is malformed". The sqlite3 CLI isn't guaranteed to be # installed — probe via the venv's Python instead. An EMPTY result # means the probe itself couldn't run (no usable venv python) — that # is not proof of corruption, so warn and let `flask db upgrade` # surface any real problem rather than halting on a false positive. local integrity integrity="$("$venv/bin/python" -c \ "import sqlite3,sys;print(sqlite3.connect(sys.argv[1]).execute('PRAGMA integrity_check').fetchone()[0])" \ "$slot_db" 2>/dev/null || true)" if [ -n "$integrity" ] && [ "$integrity" != "ok" ]; then halt "SQLite database is corrupt ($integrity). Repair or restore it from $BACKUP_DIR and re-run the update. The previous installation is still active." elif [ -z "$integrity" ]; then warn "Could not pre-check SQLite integrity (venv python unavailable) — proceeding" fi fi if ! ( # shellcheck source=/dev/null source "$venv/bin/activate" cd "$work_dir/backend" # SERVERKIT_SKIP_BACKGROUND=1: create_app() must not start queue # consumers / schedulers / collectors while we only want migrations — # they query the DB before `flask db upgrade` runs, and any error # there aborts the migration (belt & braces; the app also detects the # flask CLI on its own). if [ "$use_slot_db" = "1" ]; then DATABASE_URL="sqlite:///$slot_db" FLASK_ENV=production SERVERKIT_SKIP_BACKGROUND=1 flask db upgrade else # Non-SQLite (e.g. PostgreSQL): the DB is shared/external, so there # is no per-slot copy to isolate — migrate it directly. FLASK_ENV=production SERVERKIT_SKIP_BACKGROUND=1 flask db upgrade fi ); then # PostgreSQL (and other external DBs): the migration ran IN PLACE # against the SHARED database — there is no untouched slot copy to # fall back on. Put the database back the way we found it using the # verified pre-upgrade dump before reporting the failure. if [ "$use_slot_db" != "1" ] && [ -n "${PRE_UPGRADE_DUMP:-}" ] && [ -f "${PRE_UPGRADE_DUMP:-}" ]; then warn "Migration failed — restoring the pre-upgrade dump into the shared database..." local pg_url pg_url="$(pg_url_from_env "$work_dir")" if command -v pg_restore >/dev/null 2>&1 \ && pg_restore --clean --if-exists -d "$pg_url" "$PRE_UPGRADE_DUMP"; then halt "Database migration failed; the pre-upgrade dump was restored, so the database is back to its pre-update state. The previous installation is still active." fi halt "Database migration failed AND the automatic restore failed. Restore MANUALLY with: pg_restore --clean --if-exists -d '$pg_url' '$PRE_UPGRADE_DUMP' — the previous installation is still active." fi halt "Database migration failed. The previous installation is still active." fi good "Database migrated" } # --------------------------------------------------------------------------- # Release download + checksum verification # --------------------------------------------------------------------------- # Prints exactly ONE line on stdout: the path to the verified tarball. The # caller captures that stdout (tarball="$(download_release ...)"), so every # bit of progress in here is routed to stderr — a step/good line on stdout # would be captured into the "path" and break the tar that follows. stderr # still reaches the terminal and the log (init_logging merges 2>&1 into tee). download_release() { local version="$1" local arch output tmp_dir base_url checksum_url tarball_url case "$(uname -m)" in x86_64|amd64) arch="amd64" ;; aarch64|arm64) arch="arm64" ;; *) halt "Unsupported architecture: $(uname -m)" ;; esac if [ -n "$SERVERKIT_OFFLINE_TARBALL" ]; then [ -f "$SERVERKIT_OFFLINE_TARBALL" ] || halt "Offline tarball not found: $SERVERKIT_OFFLINE_TARBALL" echo "$SERVERKIT_OFFLINE_TARBALL" return 0 fi if [ -n "$SERVERKIT_MIRROR_URL" ]; then base_url="$SERVERKIT_MIRROR_URL" else base_url="https://github.com/${GITHUB_REPO}/releases/download/${version}" fi tarball_url="${base_url}/serverkit-${version}-linux-${arch}.tar.gz" checksum_url="${base_url}/checksums.txt" tmp_dir="$(mktemp -d)" output="$tmp_dir/serverkit-${version}-linux-${arch}.tar.gz" # This runs in a command substitution, so the caller never learns tmp_dir # on failure: remove it here before halting, or every failed download # strands a directory in /tmp. step "Downloading release tarball (${arch})..." >&2 if ! curl -sfL "$tarball_url" -o "$output"; then rm -rf "$tmp_dir" halt "Failed to download $tarball_url" fi step "Verifying checksum..." >&2 if curl -sfL "$checksum_url" -o "$tmp_dir/checksums.txt"; then cd "$tmp_dir" if ! sha256sum -c <(grep "serverkit-${version}-linux-${arch}.tar.gz" checksums.txt) >/dev/null 2>&1; then cd / rm -rf "$tmp_dir" halt "Checksum verification failed for release tarball." fi good "Checksum verified" >&2 elif [ "${SERVERKIT_ALLOW_UNVERIFIED:-0}" = "1" ]; then warn "No checksums.txt — installing UNVERIFIED (SERVERKIT_ALLOW_UNVERIFIED=1)" >&2 else # Every release publishes checksums.txt; its absence means a broken # mirror or a tampered download, and this replaces the running panel. rm -rf "$tmp_dir" halt "Could not download checksums.txt from ${base_url} — refusing an unverified release. Set SERVERKIT_ALLOW_UNVERIFIED=1 to override." fi echo "$output" } # Remove release downloads that earlier updates left in the temp dir (before # the download dir was cleaned up, every update stranded ~70 MB there). Only # directories holding a release tarball and untouched for an hour are taken, # so a concurrent download is never pulled out from under itself. sweep_stale_downloads() { local tarball while IFS= read -r tarball; do [ -n "$tarball" ] || continue rm -rf "$(dirname "$tarball")" 2>/dev/null || true done < <(find "${TMPDIR:-/tmp}" -mindepth 2 -maxdepth 2 -path '*/tmp.*/serverkit-*-linux-*.tar.gz' \ -mmin +60 2>/dev/null || true) return 0 } # --------------------------------------------------------------------------- # Blue/green directory management # --------------------------------------------------------------------------- ensure_bluegreen_layout() { # Convert legacy single-directory installs into the blue/green symlink layout. if [ -d "$INSTALL_DIR" ] && [ ! -L "$INSTALL_DIR" ]; then step "Migrating to blue/green layout..." if [ "$DRY_RUN" = "1" ]; then info "[dry-run] would move $INSTALL_DIR → $DIR_A and symlink $INSTALL_DIR → $DIR_A" return 0 fi mv "$INSTALL_DIR" "$DIR_A" ln -s "$DIR_A" "$INSTALL_DIR" good "Migrated to blue/green layout" fi # Ensure both slots exist. if [ "$DRY_RUN" = "0" ]; then mkdir -p "$DIR_A" "$DIR_B" fi } atomic_switch() { local target="$1" if [ "$DRY_RUN" = "1" ]; then info "[dry-run] would switch symlink $INSTALL_DIR → $target" return 0 fi ln -sfn "$target" "${INSTALL_DIR}.tmp" mv -Tf "${INSTALL_DIR}.tmp" "$INSTALL_DIR" good "Switched active install to $target" } # --------------------------------------------------------------------------- # Consistent SQLite copy. A plain `cp` of the LIVE panel database races the # backend's own writes: the copy can capture half a transaction and come out # "database disk image is malformed" — which then sinks the migration in the # new slot, and silently poisons the pre-upgrade backup that was supposed to # be the safety net. The SQLite online backup API snapshots a consistent DB # even while the panel is writing, using any interpreter that can import # sqlite3; with none available we fail rather than fall back to a racy cp. # --------------------------------------------------------------------------- # Snapshotting or integrity-checking a SQLite file needs nothing more than an # interpreter carrying the stdlib `sqlite3` module — NOT the gated 3.11-3.14 # that rebuild_virtualenv and preflight_check legitimately demand. Reusing # locate_python here silently coupled the DATA-safety paths to the venv-builder # constraint, and blocked updates for a reason unrelated to the data: # * an all-Docker install returns before preflight_check ever runs and does # all its work in the container, yet backup_docker_db verified the extracted # DB on the HOST — so a stock EL9 (python3.9) or Ubuntu 22.04 (python3.10) # host halted with "Could not obtain a VALID database backup", blaming the # backup for a missing interpreter. # * backup_current halts when it cannot snapshot, so the same boxes could # never update at all. # Prefer the install's own venv interpreter: on a real box it is guaranteed # present, guaranteed to have the module, and independent of what is on PATH. locate_sqlite_python() { local c for c in "$@" python3.14 python3.13 python3.12 python3.11 python3 python; do [ -n "$c" ] || continue command -v "$c" >/dev/null 2>&1 || continue "$c" -c 'import sqlite3' >/dev/null 2>&1 || continue printf '%s' "$c" return 0 done return 1 } copy_sqlite_db() { local src="$1" dest="$2" py slot # The DB lives at /backend/instance/serverkit.db — derive that slot's # interpreter as the preferred candidate. A src outside a slot leaves the # path unchanged, yielding a non-executable candidate that is simply skipped. slot="${src%/backend/instance/*}" py="$(locate_sqlite_python "$slot/venv/bin/python")" \ || { warn "No Python with the sqlite3 module available for a consistent DB snapshot"; return 1; } "$py" - "$src" "$dest" <<'PYEOF' import sqlite3, sys src, dest = sys.argv[1], sys.argv[2] s = sqlite3.connect(src) d = sqlite3.connect(dest) s.backup(d) d.close() s.close() PYEOF } # A backup that was never verified is a hope, not a safety net. Verify every # artifact BEFORE it is allowed to justify proceeding with the update. verify_sqlite_db() { local db="$1" py active hint="" [ -s "$db" ] || return 1 # Backups live in BACKUP_DIR, outside any slot, so hint at the active # install's interpreter instead of deriving one from the file's path. active="$(active_real_dir 2>/dev/null || true)" if [ -n "$active" ]; then hint="$active/venv/bin/python" fi py="$(locate_sqlite_python "$hint")" || return 1 [ "$("$py" -c "import sqlite3,sys;print(sqlite3.connect(sys.argv[1]).execute('PRAGMA integrity_check').fetchone()[0])" "$db" 2>/dev/null || true)" = "ok" ] } verify_pg_dump() { local dump="$1" [ -s "$dump" ] || return 1 # Structural validation when pg_restore is available: list the dump's TOC. if command -v pg_restore >/dev/null 2>&1; then pg_restore -l "$dump" >/dev/null 2>&1 || return 1 fi return 0 } # Extract the DATABASE_URL from a slot's .env, normalized for libpq: strip # SQLAlchemy driver suffixes (postgresql+psycopg2://) and wrapping quotes. pg_url_from_env() { grep -E '^DATABASE_URL=' "$1/.env" 2>/dev/null | head -1 | cut -d= -f2- \ | tr -d "\"'" | sed -E 's|^postgres(ql)?\+[a-z0-9]+://|postgresql://|' } # --------------------------------------------------------------------------- # Backup # --------------------------------------------------------------------------- snapshot_install_tree() { local source_dir="$1" target_dir="$2" rsync -a --exclude=venv --exclude=backups --exclude=node_modules \ --exclude='backend/instance/*.db' \ --exclude='backend/instance/*.db-wal' \ --exclude='backend/instance/*.db-shm' \ "$source_dir/" "$target_dir/" 2>/dev/null && return 0 # Preserve the exclusions even on hosts without rsync; never stage a # second database copy just to delete it after copying the tree. mkdir -p "$target_dir" || return 1 (set -o pipefail tar -C "$source_dir" --exclude=venv --exclude=backups --exclude=node_modules \ --exclude='./backend/instance/*.db' \ --exclude='./backend/instance/*.db-wal' \ --exclude='./backend/instance/*.db-shm' -cf - . \ | tar -C "$target_dir" -xpf - ) } backup_current() { phase "Database Backup" mkdir -p "$BACKUP_DIR" # Trim BEFORE writing, not only in cleanup(). # # cleanup() is the last phase of the update, so it never ran on a run that # died earlier — including the run that died BECAUSE the disk was full. # Retention that only executes on success cannot bound a failure loop: a # box was found carrying six snapshots under a cap of five for exactly # this reason. Trimming first also means peak usage is `keep` snapshots # rather than `keep + 1`. if [ "$DRY_RUN" != "1" ]; then trim_backups enforce_backup_budget fi local active db_file backup_file active="$(active_real_dir)" db_file="$active/backend/instance/serverkit.db" # Branch on the DECLARED engine, not on which files happen to exist. An # install migrated SQLite -> PostgreSQL keeps its old serverkit.db on disk # (both deploy paths copy it forward on file-existence alone, so it is # carried into every future slot), and a file-first test then backed up # that abandoned file, skipped the dump, and left PRE_UPGRADE_DUMP unset — # silently disabling the auto-restore in migrate_database, which reads it # as "${PRE_UPGRADE_DUMP:-}" and just does nothing. PostgreSQL is the ONE # install type that migrates in place and cannot roll back by switching # slots, so that dump is its only way home. migrate_database has always # keyed off DATABASE_URL; this keeps the two functions in agreement. if grep -qE '^DATABASE_URL=postgres' "$active/.env" 2>/dev/null; then # PostgreSQL: the DB is external and SHARED between slots — there is # no per-slot file to snapshot, and the migration runs in place. A # pre-upgrade dump is the only safety net AND the only way back: # without it, a rollback hands the old code a newer schema. backup_file="$BACKUP_DIR/serverkit-pre-upgrade-$(date +%Y%m%d-%H%M%S).dump" command -v pg_dump >/dev/null 2>&1 \ || halt "PostgreSQL install but pg_dump is unavailable — install postgresql-client so the update has a safety net" local pg_url pg_url="$(pg_url_from_env "$active")" if ! run_or_dry pg_dump "$pg_url" -Fc -f "$backup_file"; then rm -f "$backup_file" halt "PostgreSQL backup failed — refusing to update without a safety net" fi if [ "$DRY_RUN" != "1" ] && ! verify_pg_dump "$backup_file"; then rm -f "$backup_file" halt "PostgreSQL backup verification failed — refusing to update without a VALID safety net" fi # Remember the dump path so migrate_database can offer/perform an # automatic restore if the in-place migration fails. PRE_UPGRADE_DUMP="$backup_file" good "Database backed up to $backup_file" elif [ -f "$db_file" ]; then backup_file="$BACKUP_DIR/serverkit-pre-upgrade-$(date +%Y%m%d-%H%M%S).db" # A torn backup is worse than none: it looks like a safety net but # restores a malformed DB. Snapshot consistently, VERIFY the result, # and refuse to proceed without a proven-valid net. Failed or invalid # artifacts are removed so they are never mistaken for a good backup. if ! run_or_dry copy_sqlite_db "$db_file" "$backup_file"; then rm -f "$backup_file" halt "Database backup failed — refusing to update without a safety net" fi if [ "$DRY_RUN" != "1" ] && ! verify_sqlite_db "$backup_file"; then rm -f "$backup_file" halt "Database backup verification failed — refusing to update without a VALID safety net" fi good "Database backed up to $backup_file" else # Neither engine matched. Say which one, and say plainly that the # update is proceeding unprotected — "No SQLite database" reads like a # benign nothing-to-do on, say, a MySQL install that in fact has no # safety net at all. local db_scheme db_scheme="$(grep -E '^DATABASE_URL=' "$active/.env" 2>/dev/null | head -1 \ | cut -d= -f2- | tr -d "\"'" | cut -d: -f1)" if [ -n "$db_scheme" ] && [ "$db_scheme" != "sqlite" ]; then warn "DATABASE_URL uses '$db_scheme', which this updater cannot dump — proceeding with NO database backup" else warn "No SQLite database at $db_file — skipping DB backup" fi fi local tree_backup tree_backup="$BACKUP_DIR/serverkit-tree-$(date +%Y%m%d-%H%M%S)" if [ -L "$INSTALL_DIR" ] && [ -d "$active" ]; then # Blue/green: the update installs into the OTHER slot, so this one # stays on disk as the rollback. A tree copy of it was a second full # copy of the same code (~1 GB per update on a small VPS). info "Install tree kept in place as the rollback slot: $active" elif [ -d "$active" ]; then # The SQLite database is NOT part of the tree snapshot: it was just # written as the pre-upgrade copy above, and rollback restores from # that copy, never from the tree. Copying it here too doubled every # update's backup footprint (a 450 MB database became 900 MB per run), # which is how small VPSes kept filling their disk with backups. if run_or_dry snapshot_install_tree "$active" "$tree_backup"; then good "Install tree backed up to $tree_backup" else warn "Install tree backup failed: $tree_backup may be incomplete" fi fi } # --------------------------------------------------------------------------- # Carry user-installed plugins across a redeploy (#48). # # URL/registry/upload-installed extensions live as extracted dirs under # backend/app/plugins/ and frontend/src/plugins/. A fresh # clone/tarball only contains the repo-tracked ones, so anything the user # installed would silently vanish — copy forward every plugin dir the new # tree doesn't already ship. Runs BEFORE the frontend build so carried # frontends compile into the new bundle. The backend also has a boot-time # repair pass as a backstop (plugin_service.repair_missing_plugins). # --------------------------------------------------------------------------- # Plugins deliberately deleted from the product (replaced or retired) — e.g. # serverkit-workflows, superseded by serverkit-tramo (plan 45). Never carry # these forward: their frontends import dependencies the new tree no longer # ships (workflows → reactflow), which sinks the whole bundle build. # Space-separated slugs; append here whenever an extension is retired. # Mirror of extension_migration.RETIRED_EXTENSION_SLUGS in the backend. RETIRED_PLUGIN_SLUGS="serverkit-workflows" plugin_is_retired() { local slug for slug in $RETIRED_PLUGIN_SLUGS; do [ "$1" = "$slug" ] && return 0 done return 1 } # Strip retired plugin leftovers from a tree (docker in-place mode, where # untracked plugin dirs survive the hard reset and would otherwise sink the # bundle rebuild). remove_retired_plugins() { local tree="$1" slug sub for slug in $RETIRED_PLUGIN_SLUGS; do for sub in backend/app/plugins frontend/src/plugins; do if [ -d "$tree/$sub/$slug" ]; then rm -rf "$tree/$sub/$slug" 2>/dev/null || true info "Removed retired plugin: $sub/$slug" fi done done return 0 } # Frontend plugin dirs copied forward by preserve_installed_plugins this run. # If the bundle build then fails these are the prime suspects — see # quarantine_carried_plugins. CARRIED_FRONTEND_PLUGINS="" preserve_installed_plugins() { local src="$1" target="$2" local sub dir name for sub in backend/app/plugins frontend/src/plugins; do [ -d "$src/$sub" ] || continue for dir in "$src/$sub"/*/; do [ -d "$dir" ] || continue name="$(basename "$dir")" [ "$name" = "__pycache__" ] && continue if plugin_is_retired "$name"; then info "Not carrying forward retired plugin: $sub/$name" continue fi if [ ! -e "$target/$sub/$name" ]; then mkdir -p "$target/$sub" 2>/dev/null || true if cp -a "$dir" "$target/$sub/$name" 2>/dev/null; then info "Preserved installed plugin: $sub/$name" if [ "$sub" = "frontend/src/plugins" ]; then CARRIED_FRONTEND_PLUGINS="$CARRIED_FRONTEND_PLUGINS $name" fi else warn "Could not preserve plugin $sub/$name" fi fi done done return 0 } # Vite 8 / rolldown (the frontend bundler) require Node ^20.19.0 || >=22.12.0. # Returns 1 (no output) on an absent/too-old Node so callers can warn+rollback or # halt with an actionable message, instead of npm silently skipping rolldown's # native binary and `npm run build` dying with a cryptic MODULE_NOT_FOUND. _node_build_ok() { command -v node >/dev/null 2>&1 || return 1 local v M m v="$(node --version 2>/dev/null | sed -E 's/^v//')" M="${v%%.*}"; m="${v#*.}"; m="${m%%.*}" case "$M" in ''|*[!0-9]*) return 1;; esac case "$m" in ''|*[!0-9]*) return 1;; esac if [ "$M" -eq 20 ] && [ "$m" -ge 19 ]; then return 0; fi if [ "$M" -eq 22 ] && [ "$m" -ge 12 ]; then return 0; fi if [ "$M" -ge 23 ]; then return 0; fi return 1 } # Build the SPA bundle in a target tree. Pass "reuse" as $2 to skip npm ci # (node_modules already intact from a just-failed build). Subshell so the cd # cannot leak into the main shell. build_frontend_bundle() { if ! _node_build_ok; then warn "Node.js $(node --version 2>/dev/null || echo 'not found') is too old to build the frontend — vite 8 needs 20.19+ or 22.12+. Upgrade Node (e.g. NodeSource 22 LTS) and re-run the update." return 1 fi ( cd "$1/frontend" && \ { [ "${2:-}" = "reuse" ] || npm ci --prefer-offline 2>&1 | tail -3; } && \ NODE_OPTIONS="--max-old-space-size=1024" npm run build 2>&1 | tail -5 ) } # Re-apply this install's PERSISTENT favicon tint to the staged dist. The color # is assigned once (on first install/update) and stored in /etc/serverkit so the # mark keeps the same per-install color across updates instead of repainting — # and an install that predates the feature picks one up on its first update. # Mirror of install.sh randomize_favicon (kept standalone; the two installers # already keep separate build fns). Also serves it at /favicon.ico so a blind # favicon-hash fetch varies too. apply_favicon_tint() { local root="$1" local fav="$root/frontend/dist/favicon.svg" [ -f "$fav" ] || return 0 local color_file="/etc/serverkit/favicon-color" local hsl="" [ -s "$color_file" ] && hsl="$(cat "$color_file" 2>/dev/null)" if [ -z "$hsl" ]; then local hues=(210 225 235 250 265 285 320 345 12 25 160 175 190) local base="${hues[$RANDOM % ${#hues[@]}]}" local jitter=$(( RANDOM % 21 - 10 )) local h=$(( (base + jitter + 360) % 360 )) local s=$(( 48 + RANDOM % 28 )) local l=$(( 32 + RANDOM % 15 )) hsl="hsl(${h}, ${s}%, ${l}%)" mkdir -p /etc/serverkit 2>/dev/null || true printf '%s\n' "$hsl" > "$color_file" 2>/dev/null || true fi sed -i -E "s|(/dev/null || true cp "$fav" "$root/frontend/dist/favicon.ico" 2>/dev/null || true } # A plugin carried forward from the old install can be incompatible with the # new tree (imports that no longer resolve) and sink the whole bundle build. # Rather than blocking the update, move the carried frontends aside so the # caller can retry — fail the plugin, never the update. Returns 1 when there # was nothing to quarantine (the failure is not plugin-borne). QUARANTINED_PLUGINS="" quarantine_carried_plugins() { local target="$1" qdir name moved="" [ -n "$CARRIED_FRONTEND_PLUGINS" ] || return 1 qdir="$BACKUP_DIR/quarantined-plugins-$(date +%Y%m%d-%H%M%S)" mkdir -p "$qdir" 2>/dev/null || return 1 for name in $CARRIED_FRONTEND_PLUGINS; do [ -d "$target/frontend/src/plugins/$name" ] || continue if mv "$target/frontend/src/plugins/$name" "$qdir/" 2>/dev/null; then moved="$moved $name" fi done CARRIED_FRONTEND_PLUGINS="" [ -n "$moved" ] || { rmdir "$qdir" 2>/dev/null || true; return 1; } QUARANTINED_PLUGINS="${moved# }" warn "Carried-forward plugin(s) quarantined after build failure: $QUARANTINED_PLUGINS" warn "Copies kept in $qdir — reinstall compatible versions from the Marketplace" return 0 } # --------------------------------------------------------------------------- # Deploy source into the next directory # --------------------------------------------------------------------------- deploy_source() { local target="$1" local branch="${2:-main}" phase "Updating Source" if [ "$DRY_RUN" = "1" ]; then info "[dry-run] would clone/pull $GITHUB_REPO:$branch into $target" return 0 fi # Preserve .env and database across rewrites. The DB snapshot must be # consistent (online backup API) — a racy cp of the live DB can come out # malformed and sink the migration in the new slot. local tmp_env tmp_db live_db tmp_env="$(mktemp)" tmp_db="$(mktemp)" cp "$INSTALL_DIR/.env" "$tmp_env" 2>/dev/null || true live_db="$INSTALL_DIR/backend/instance/serverkit.db" if [ -f "$live_db" ]; then copy_sqlite_db "$live_db" "$tmp_db" \ || halt "Could not snapshot the live database consistently — aborting before any switch" fi rm -rf "$target" git clone --depth 1 --branch "$branch" "https://github.com/${GITHUB_REPO}.git" "$target" \ || halt "Failed to clone ${GITHUB_REPO}:$branch" cp "$tmp_env" "$target/.env" 2>/dev/null || true if [ -s "$tmp_db" ]; then mkdir -p "$target/backend/instance" cp "$tmp_db" "$target/backend/instance/serverkit.db" 2>/dev/null || true fi rm -f "$tmp_env" "$tmp_db" preserve_installed_plugins "$INSTALL_DIR" "$target" chmod +x "$target/serverkit" chmod +x "$target/scripts/"*.sh 2>/dev/null || true good "Source updated to $branch in $target" } # --------------------------------------------------------------------------- # Deploy release tarball into the next directory # --------------------------------------------------------------------------- deploy_release() { local target="$1" local version="$2" phase "Downloading Release" local tarball stage unpacked sweep_stale_downloads tarball="$(download_release "$version")" # The download dir is ours to remove; an operator's offline tarball is not. # Both scratch dirs are recorded globally so cleanup_on_exit removes them # when a halt below aborts the update. if [ "$tarball" != "${SERVERKIT_OFFLINE_TARBALL:-}" ]; then RELEASE_DOWNLOAD_DIR="$(dirname "$tarball")" fi if [ "$DRY_RUN" = "1" ]; then info "[dry-run] would unpack $tarball into $target" remove_release_scratch return 0 fi stage="$(mktemp -d)" RELEASE_STAGE_DIR="$stage" tar xzf "$tarball" -C "$stage" unpacked="$stage/serverkit" [ ! -d "$unpacked" ] && unpacked="$stage/opt/serverkit" if [ ! -d "$unpacked" ]; then # `|| true`: head exiting first can SIGPIPE find (rc 141 under # pipefail); an empty result is handled by the halt just below. unpacked="$(find "$stage" -maxdepth 2 -type d -name serverkit | head -n1 || true)" fi [ -d "$unpacked" ] || halt "Release tarball layout is unrecognized" # Preserve live state. Same consistent-snapshot rule as deploy_source: # never plain-cp the live DB into the new slot. cp "$INSTALL_DIR/.env" "$unpacked/.env" 2>/dev/null || true local live_db="$INSTALL_DIR/backend/instance/serverkit.db" if [ -f "$live_db" ]; then mkdir -p "$unpacked/backend/instance" copy_sqlite_db "$live_db" "$unpacked/backend/instance/serverkit.db" \ || halt "Could not snapshot the live database consistently — aborting before any switch" fi preserve_installed_plugins "$INSTALL_DIR" "$unpacked" rm -rf "$target" cp -a "$unpacked" "$target" remove_release_scratch chmod +x "$target/serverkit" chmod +x "$target/scripts/"*.sh 2>/dev/null || true good "Release $version deployed to $target" } # --------------------------------------------------------------------------- # Configuration refresh # --------------------------------------------------------------------------- refresh_config() { local target="$1" phase "Refreshing Configuration" if [ "$DRY_RUN" = "1" ]; then info "[dry-run] would refresh nginx + systemd configs from $target" return 0 fi mkdir -p "$NGINX_DIR/sites-available" "$NGINX_DIR/sites-enabled" # Recover panel domain from live nginx config. The trailing `|| true` keeps a # no-match/missing-file grep (exit 1/2) from tripping `set -euo pipefail` and # silently killing the updater before the atomic switch — HTTP-only boxes have # no serverkit.conf, so this grep finds nothing. local prior_panel_domain="" prior_panel_domain=$(grep -oE "$LETSENCRYPT_DIR/live/[^/]+/" \ "$NGINX_DIR/sites-available/serverkit.conf" 2>/dev/null | head -n1 | \ sed -E 's|.*/live/([^/]+)/|\1|' || true) [ "$prior_panel_domain" = "YOUR_DOMAIN" ] && prior_panel_domain="" if [ -f "$target/nginx/sites-available/serverkit.conf" ]; then cp "$target/nginx/sites-available/serverkit.conf" "$NGINX_DIR/sites-available/" fi if [ -f "$target/nginx/sites-available/serverkit-insecure.conf" ]; then cp "$target/nginx/sites-available/serverkit-insecure.conf" "$NGINX_DIR/sites-available/" fi # The panel frontend is served statically by host nginx from # $INSTALL_DIR/frontend/dist (the /opt/serverkit symlink). The shipped config # roots at the default /opt/serverkit; re-point it when SERVERKIT_DIR differs, # exactly as install.sh does, so a custom install dir survives upgrades. if [ "$INSTALL_DIR" != "/opt/serverkit" ]; then local conf for conf in serverkit.conf serverkit-insecure.conf; do [ -f "$NGINX_DIR/sites-available/$conf" ] && \ sed -i "s|/opt/serverkit/frontend/dist|$INSTALL_DIR/frontend/dist|g" \ "$NGINX_DIR/sites-available/$conf" done fi # TLS floor — prefer a reversible conf.d snippet when nginx.conf doesn't # already declare these (a second declaration in the same http{} context is a # "duplicate ssl_protocols" error); otherwise rewrite in place. Mirrors # install.sh's harden_global_tls. if [ -f "$NGINX_DIR/nginx.conf" ]; then local ciphers='ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384' local has_proto=0 has_ciphers=0 grep -qE '^[[:space:]]*ssl_protocols[[:space:]]' "$NGINX_DIR/nginx.conf" && has_proto=1 grep -qE '^[[:space:]]*ssl_ciphers[[:space:]]' "$NGINX_DIR/nginx.conf" && has_ciphers=1 if [ "$has_proto" = "0" ] && [ "$has_ciphers" = "0" ] && \ grep -qE 'include[[:space:]]+/etc/nginx/conf\.d/\*\.conf' "$NGINX_DIR/nginx.conf"; then mkdir -p "$NGINX_DIR/conf.d" cat > "$NGINX_DIR/conf.d/serverkit-tls.conf" </dev/null || true if [ "$has_proto" = "1" ]; then sed -i -E 's|^([[:space:]]*)ssl_protocols[[:space:]].*|\1ssl_protocols TLSv1.2 TLSv1.3;|' "$NGINX_DIR/nginx.conf" else sed -i '/http {/a \ ssl_protocols TLSv1.2 TLSv1.3;' "$NGINX_DIR/nginx.conf" fi if [ "$has_ciphers" = "1" ]; then sed -i -E "s|^([[:space:]]*)ssl_ciphers[[:space:]].*|\1ssl_ciphers ${ciphers};|" "$NGINX_DIR/nginx.conf" else sed -i "/http {/a \\ ssl_ciphers ${ciphers};" "$NGINX_DIR/nginx.conf" fi fi fi # SSL mode local ssl_mode="insecure" if [ -f "$CONFIG_DIR/ssl-mode" ]; then ssl_mode="$(cat "$CONFIG_DIR/ssl-mode")" fi if [ "$ssl_mode" = "secure" ] && [ -f "$NGINX_DIR/sites-available/serverkit.conf" ]; then local panel_domain="" if [ -f "$CONFIG_DIR/panel-domain" ]; then panel_domain="$(cat "$CONFIG_DIR/panel-domain" 2>/dev/null || true)" fi [ -z "$panel_domain" ] && panel_domain="$prior_panel_domain" if [ -n "$panel_domain" ] && [ -d "$LETSENCRYPT_DIR/live/$panel_domain" ]; then sed -i "s|/etc/letsencrypt/live/YOUR_DOMAIN/|$LETSENCRYPT_DIR/live/$panel_domain/|g" \ "$NGINX_DIR/sites-available/serverkit.conf" ln -sf "$NGINX_DIR/sites-available/serverkit.conf" "$NGINX_DIR/sites-enabled/serverkit.conf" else warn "SSL mode is 'secure' but no certificate found for '${panel_domain:-unknown}'" ln -sf "$NGINX_DIR/sites-available/serverkit-insecure.conf" "$NGINX_DIR/sites-enabled/serverkit.conf" fi else ln -sf "$NGINX_DIR/sites-available/serverkit-insecure.conf" "$NGINX_DIR/sites-enabled/serverkit.conf" fi # Service unit — render from the template so a custom SERVERKIT_DIR / venv / # log path survives upgrades (the unit references the /opt/serverkit symlink, # so blue/green switches need no re-render). Fall back to a plain unit if an # older tree still ships one. local svc_template="$target/templates/serverkit-backend.service.in" # Bind the API to loopback by default — host nginx fronts it on :80/:443, so # the raw gunicorn port must not be world-reachable. Mirror install.sh so an # update re-renders @BIND_HOST@ instead of leaving the literal placeholder # (which would make gunicorn fail to bind). Override with SERVERKIT_BIND_HOST. local bind_host="${SERVERKIT_BIND_HOST:-127.0.0.1}" if [ -f "$svc_template" ]; then sed -e "s|@SERVERKIT_DIR@|$INSTALL_DIR|g" \ -e "s|@SERVERKIT_VENV_DIR@|$VENV_DIR|g" \ -e "s|@PORT@|5000|g" \ -e "s|@BIND_HOST@|$bind_host|g" \ -e "s|@USER@|root|g" \ -e "s|@LOG_DIR@|$LOG_DIR|g" \ "$svc_template" > "$SYSTEMD_DIR/serverkit.service" elif [ -f "$target/serverkit-backend.service" ]; then cp "$target/serverkit-backend.service" "$SYSTEMD_DIR/serverkit.service" fi systemctl daemon-reload 2>/dev/null \ || warn "systemd daemon-reload failed — the backend may restart with a stale unit" report_stale_panel_vhosts good "Configuration refreshed" } # The panel used to be served by a frontend container on :3847; it is now # served statically by host nginx. A leftover custom vhost proxying to :3847 # keeps serving whatever bundle the (never-updated) container holds — the # panel looks "stuck on an old version" for that hostname only, which reads # like a browser/CDN cache problem and wastes hours (2026-07-03 incident). # Observation only (R1): warn and continue, tolerate a missing/empty dir. report_stale_panel_vhosts() { local hits="" hits="$(grep -rls 'proxy_pass http://127\.0\.0\.1:3847' \ "$NGINX_DIR/sites-enabled/" 2>/dev/null | sort -u | tr '\n' ' ' || true)" if [ -n "$hits" ]; then warn "Vhost(s) still proxy the panel to the retired :3847 frontend container: ${hits}" warn " These serve a STALE panel bundle. Repoint them at the static SPA" warn " (root $INSTALL_DIR/frontend/dist + /api and /socket.io → :5000)," warn " mirroring sites-available/serverkit.conf." fi } # --------------------------------------------------------------------------- # Rollback # --------------------------------------------------------------------------- # Probe the backend health endpoint; returns 0 if healthy within s. # Side-effect free (unlike health_check, which triggers a rollback) so it is # safe to call *after* a rollback to confirm the restored version came back. quick_health() { local timeout="${1:-30}" waited=0 while [ "$waited" -lt "$timeout" ]; do curl -sf --max-time 5 http://127.0.0.1:5000/api/v1/system/health >/dev/null 2>&1 && return 0 sleep 2 waited=$((waited + 2)) done return 1 } rollback() { # Flag FIRST: halt() below exits the script, which fires the EXIT trap # (cleanup_on_exit) — without this flag already set, that trap would run a # second, redundant rollback on top of the one that just finished. ROLLING_BACK=1 warn "Update failed — rolling back to previous slot..." if [ -z "${PREVIOUS_DIR:-}" ] || [ ! -d "$PREVIOUS_DIR" ]; then halt "Cannot roll back: previous installation directory not available" fi # Same zero-downtime discipline as the forward path: cycle only the backend, # never stop nginx (it fronts every hosted app), and reload its config after # the slot is switched back. systemctl stop "$BACKEND_SERVICE" 2>/dev/null || true wait_for_service "$BACKEND_SERVICE" inactive 30 || true atomic_switch "$PREVIOUS_DIR" # A rollback must never abort mid-flight over one failed step — everything # from here on is best-effort so the backend still gets started. systemctl daemon-reload 2>/dev/null \ || warn "systemd daemon-reload failed during rollback — continuing" systemctl start "$BACKEND_SERVICE" 2>/dev/null || true wait_for_service "$BACKEND_SERVICE" active 30 || true # Frontend is static (served from the restored slot); just re-label + reload. selinux_label_dist "$PREVIOUS_DIR" reload_nginx_graceful # Confirm the restored version actually answers — a rollback that itself # comes up unhealthy is a far worse state to leave the operator guessing in. if quick_health 30; then halt "Rolled back to $(active_real_dir) and it is healthy. Inspect logs: journalctl -u serverkit -n 50" else halt "Rolled back to $(active_real_dir) but it is STILL UNHEALTHY — manual intervention needed. Logs: journalctl -u serverkit -n 50" fi } # Remove deploy_release's download and unpack dirs. Warn-and-continue: a # scratch dir that will not delete must never fail an update or its rollback. remove_release_scratch() { if [ -n "${RELEASE_DOWNLOAD_DIR:-}" ]; then rm -rf "$RELEASE_DOWNLOAD_DIR" 2>/dev/null || true RELEASE_DOWNLOAD_DIR="" fi if [ -n "${RELEASE_STAGE_DIR:-}" ]; then rm -rf "$RELEASE_STAGE_DIR" 2>/dev/null || true RELEASE_STAGE_DIR="" fi return 0 } # If the update fails after we have switched the symlink, roll back to the # previous slot automatically. Registered as the EXIT trap by the run block # below; defined here (above the source guard) so it stays unit-testable. cleanup_on_exit() { local rc=$? remove_release_scratch [ "$rc" -eq 0 ] && return 0 if [ "$DRY_RUN" = "0" ] && [ -n "${PREVIOUS_DIR:-}" ] && \ [ "${ROLLING_BACK:-0}" != "1" ] && [ "${HEALTH_PASSED:-0}" != "1" ]; then ROLLING_BACK=1 rollback fi # Always return 0: the script's exit code is already $rc (an EXIT trap does # not change it) — returning $rc here only re-fires the ERR trap, which # appends a second, misleading "Update aborted" report. return 0 } # --------------------------------------------------------------------------- # Health check # --------------------------------------------------------------------------- health_check() { phase "Health Check" if [ "$DRY_RUN" = "1" ]; then info "[dry-run] would probe http://127.0.0.1:5000/api/v1/system/health" return 0 fi step "Waiting for backend..." local waited=0 while [ "$waited" -lt 60 ]; do if curl -sf --max-time 5 http://127.0.0.1:5000/api/v1/system/health >/dev/null 2>&1; then good "Backend healthy" break fi sleep 2 waited=$((waited + 2)) done if [ "$waited" -ge 60 ]; then rollback fi if ! curl -sf --max-time 5 http://127.0.0.1:5000/api/v1/system/health >/dev/null 2>&1; then rollback fi # No frontend probe: the panel frontend is static files served by host # nginx from the active slot — there is no frontend container anymore. HEALTH_PASSED=1 } # --------------------------------------------------------------------------- # Cleanup # --------------------------------------------------------------------------- # Trim a family of backups down to its newest entries. Deliberately # find-based: the old `ls -t "$BACKUP_DIR"/ | tail | xargs` aborted the # whole updater under `set -Eeuo pipefail` whenever the glob matched nothing # (ls exits 2) — i.e. on every box with no prior backups of that family, AFTER # an otherwise successful update. find treats "nothing matched" as a clean # empty result, and an absent BACKUP_DIR is an explicit no-op. prune_old_backups() { local pattern="$1" keep="$2" [ -d "$BACKUP_DIR" ] || return 0 find "$BACKUP_DIR" -mindepth 1 -maxdepth 1 -name "$pattern" -printf '%T@\t%p\n' 2>/dev/null \ | sort -rn | tail -n +"$((keep + 1))" | cut -f2- \ | xargs -r -d '\n' rm -rf || true } # Bytes currently held by all backup artifacts. backups_size_kb() { [ -d "$BACKUP_DIR" ] || { echo 0; return 0; } du -sk "$BACKUP_DIR" 2>/dev/null | awk '{print $1}' || echo 0 } # How many upgrade snapshots to keep when SERVERKIT_BACKUP_RETENTION is unset: # 1 on a filesystem under 50 GiB, else 3. Each snapshot is a full database # copy, and on a 25 GB VPS three of them compete with the apps for the disk. # An unreadable size keeps the old default. default_backup_keep() { local total_kb total_kb="$(df -Pk "$BACKUP_DIR" 2>/dev/null | awk 'NR==2 {print $2}' || true)" if [ -z "$total_kb" ]; then total_kb="$(df -Pk / 2>/dev/null | awk 'NR==2 {print $2}' || true)" fi if [ -n "$total_kb" ] && [ "$total_kb" -lt $((50 * 1024 * 1024)) ] 2>/dev/null; then echo 1 else echo 3 fi } # Trim every backup kind to the retention cap. Safe to call repeatedly. trim_backups() { local keep="${1:-${SERVERKIT_BACKUP_RETENTION:-$(default_backup_keep)}}" prune_old_backups 'serverkit-tree-*' "$keep" prune_old_backups 'serverkit-pre-upgrade-*.db' "$keep" prune_old_backups 'serverkit-pre-upgrade-*.dump' "$keep" } # Drop the OLDEST snapshots until the backup directory fits its size budget. # # A count cap alone is not a disk guarantee: five snapshots of an 800 MB # database is 8 GB, which is a third of a 25 GB VPS. The budget is a share of # the filesystem (default 15%), and at least one snapshot is always kept — # an update with no way back is worse than a full disk. enforce_backup_budget() { [ -d "$BACKUP_DIR" ] || return 0 local pct="${SERVERKIT_BACKUP_MAX_PERCENT:-15}" local total_kb budget_kb used_kb total_kb="$(df -Pk "$BACKUP_DIR" | awk 'NR==2 {print $2}')" [ -n "$total_kb" ] && [ "$total_kb" -gt 0 ] || return 0 budget_kb=$(( total_kb * pct / 100 )) local guard=0 while [ "$guard" -lt 50 ]; do used_kb="$(backups_size_kb)" [ "$used_kb" -le "$budget_kb" ] && return 0 # Oldest stamp still on disk, across every artifact kind. local oldest oldest="$(find "$BACKUP_DIR" -mindepth 1 -maxdepth 1 \ \( -name 'serverkit-tree-*' -o -name 'serverkit-pre-upgrade-*' \) \ -printf '%T@\t%p\n' 2>/dev/null | sort -n | head -1 | cut -f2-)" [ -n "$oldest" ] || return 0 # Never delete the last remaining snapshot set. local remaining remaining="$(find "$BACKUP_DIR" -mindepth 1 -maxdepth 1 \ -name 'serverkit-pre-upgrade-*' 2>/dev/null | wc -l)" if [ "$remaining" -le 1 ] && [[ "$(basename "$oldest")" == serverkit-pre-upgrade-* ]]; then warn "Backups exceed ${pct}% of the disk but only one restore point remains — keeping it" return 0 fi info "Backup budget exceeded — removing oldest backup $(basename "$oldest")" rm -rf "$oldest" || return 0 guard=$((guard + 1)) done } cleanup() { phase "Cleanup" if [ "$DRY_RUN" = "1" ]; then info "[dry-run] would trim old backups" return 0 fi # Retention is capped (default 3, SERVERKIT_BACKUP_RETENTION overrides) # AND budgeted by size: pre-upgrade DB backups are full-size copies, so # five of an 800 MB database is 8 GB — a third of a 25 GB droplet. Seen in # the wild twice: 6.3G of stale backups, then a disk filled to 0 bytes. # # backup_current already trimmed before writing; this is the second pass # that also reclaims the snapshot this run just superseded. trim_backups enforce_backup_budget local new_version new_version="$(cat "$INSTALL_DIR/VERSION" 2>/dev/null | tr -d '\n\r ' || echo "unknown")" curl -s "https://serverkit.ai/track/update?v=${new_version}" >/dev/null 2>&1 || true good "Cleanup complete" } # --------------------------------------------------------------------------- # Firewall — keep 80/443 open across updates. # --------------------------------------------------------------------------- # Idempotent and best-effort: a box installed before firewall automation existed # gets its ports opened on the next update; an already-configured box is a no-op. # Records what we opened in install-state.json so uninstall can undo it. Sources # the helpers from the (now-current) install tree, which carries scripts/lib. ensure_firewall() { local lib="$INSTALL_DIR/scripts/lib" [ -f "$lib/firewall.sh" ] || return 0 # shellcheck source=/dev/null source "$lib/firewall.sh" if [ -f "$lib/state.sh" ]; then # shellcheck source=/dev/null source "$lib/state.sh" fi local backend backend="$(firewall_detect)" if [ "$backend" = "none" ]; then return 0 fi if [ "$DRY_RUN" = "1" ]; then info "[dry-run] would ensure firewall ($backend) allows 80/tcp and 443/tcp" return 0 fi firewall_open "$backend" 80/tcp 443/tcp || true if command -v state_set >/dev/null 2>&1; then state_set firewall_backend "$backend" || true state_append firewall_ports 80/tcp || true state_append firewall_ports 443/tcp || true fi good "Firewall ensured ($backend): 80/tcp and 443/tcp open" } refresh_bash_completion() { # Regenerate bash tab-completion from the switched-in CLI so new commands # complete immediately after an update. Best-effort: boxes without the # bash-completion package simply skip it. local dir="${SERVERKIT_COMPLETION_DIR:-/etc/bash_completion.d}" [ -d "$dir" ] || return 0 if [ "$DRY_RUN" = "1" ]; then info "[dry-run] would refresh $dir/serverkit" return 0 fi bash "$INSTALL_DIR/serverkit" completion > "$dir/serverkit" 2>/dev/null \ || { rm -f "$dir/serverkit"; warn "Could not refresh bash completion (non-fatal)"; } } # --------------------------------------------------------------------------- # All-Docker deployment (compose) detection + update path # --------------------------------------------------------------------------- # A ServerKit box runs in one of two shapes: # * Hybrid (canonical): backend in a host venv under systemd `serverkit`, # only the frontend in Docker. The blue/green + venv + systemd flow below # targets this shape. # * All-Docker: both backend AND frontend run as docker-compose services # (container_name serverkit-backend / serverkit-frontend), no host venv. # Running the hybrid flow on an all-Docker box builds a useless host venv, # migrates the wrong (non-container) database, and stops a systemd unit that # never serves traffic — which is exactly the failure that ends in # "/opt/serverkit/venv/bin/activate: No such file or directory". Detect the # all-Docker shape and route it to a dedicated compose update instead. is_docker_deployment() { [ -f "$INSTALL_DIR/docker-compose.yml" ] || return 1 # A usable host venv means this is the hybrid shape — take precedence. [ -x "$INSTALL_DIR/venv/bin/python" ] && return 1 # The backend container is defined by the all-Docker compose project. docker ps -a --format '{{.Names}}' 2>/dev/null | grep -qx 'serverkit-backend' } dc() { if docker compose version &>/dev/null; then ( cd "$INSTALL_DIR" && docker compose "$@" ) else ( cd "$INSTALL_DIR" && docker-compose "$@" ) fi } backup_docker_db() { mkdir -p "$BACKUP_DIR" if [ "$DRY_RUN" = "1" ]; then info "[dry-run] would copy the SQLite DB out of the backend container" return 0 fi local backup_file backup_file="$BACKUP_DIR/serverkit-pre-upgrade-$(date +%Y%m%d-%H%M%S).db" # The live DB lives in the serverkit-data volume at /app/instance inside # the container, not in the source tree. Prefer a consistent snapshot via # the container's Python (SQLite online backup API) — a plain docker cp # of the live file races the backend's writes and can come out malformed. local py snap_ok=0 for py in python python3; do if docker exec -i serverkit-backend "$py" - /app/instance/serverkit.db /tmp/.sk-snap.db <<'PYEOF' 2>/dev/null; then import sqlite3, sys s = sqlite3.connect(sys.argv[1]) d = sqlite3.connect(sys.argv[2]) s.backup(d) d.close() s.close() PYEOF snap_ok=1 break fi done if [ "$snap_ok" = "1" ] && docker cp serverkit-backend:/tmp/.sk-snap.db "$backup_file" 2>/dev/null; then docker exec serverkit-backend rm -f /tmp/.sk-snap.db 2>/dev/null || true else docker cp serverkit-backend:/app/instance/serverkit.db "$backup_file" 2>/dev/null || true fi # Same rule as the host path: an unverified backup is not a safety net. if [ -f "$backup_file" ] && verify_sqlite_db "$backup_file"; then good "Database backed up to $backup_file" else rm -f "$backup_file" halt "Could not obtain a VALID database backup from the container — refusing to update without a safety net" fi } # L6 — Heal layout left by an interrupted or legacy (hybrid) run before the # docker update touches anything. heal_layout_for_docker() { # Pin the compose project name. A blue/green symlink otherwise makes compose # derive the project name from the link target and warn — worse, a changed # name would orphan the running containers. export COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-$BASE_NAME}" [ "$DRY_RUN" = "1" ] && return 0 # On an all-Docker box the blue/green slots are vestigial. If the install is # a symlink (a prior hybrid run migrated it), reclaim the *inactive* slot it # left behind — the live tree + DB volume are elsewhere, so this is safe. if [ -L "$INSTALL_DIR" ]; then local active inactive active="$(active_real_dir)" inactive="$DIR_A"; [ "$active" = "$DIR_A" ] && inactive="$DIR_B" if [ -d "$inactive" ]; then warn "Reclaiming stale slot from a prior run: $inactive" rm -rf "$inactive" fi fi } # L4 — Snapshot the current docker deployment so a bad upgrade can be reverted. DOCKER_PREV_SHA="" DOCKER_ROLLBACK_READY=0 snapshot_docker_state() { [ "$DRY_RUN" = "1" ] && return 0 DOCKER_PREV_SHA="$(git -C "$INSTALL_DIR" rev-parse HEAD 2>/dev/null || true)" # Tag the currently-running images so we can re-point :latest back to them, # and stash the host-built bundle that the new build is about to overwrite. local img for img in serverkit-backend serverkit-frontend; do docker image inspect "$img:latest" >/dev/null 2>&1 && \ docker tag "$img:latest" "$img:rollback" 2>/dev/null || true done if [ -d "$INSTALL_DIR/frontend/dist" ]; then rm -rf "$INSTALL_DIR/frontend/dist.rollback" cp -a "$INSTALL_DIR/frontend/dist" "$INSTALL_DIR/frontend/dist.rollback" 2>/dev/null || true fi DOCKER_ROLLBACK_READY=1 } rollback_docker() { [ "$DOCKER_ROLLBACK_READY" = "1" ] || halt "New version is unhealthy and no rollback snapshot exists — inspect: cd $INSTALL_DIR && docker compose logs backend" warn "New version unhealthy — rolling back to the previous deployment..." # Guard the reset on a non-empty SHA: an empty SHA would make `git reset # --hard ''` reset to HEAD (a no-op at best, surprising at worst). The image # re-tag + bundle restore below still recover the previous deployment even # when no commit was recorded. if [ -n "$DOCKER_PREV_SHA" ]; then git -C "$INSTALL_DIR" reset --hard "$DOCKER_PREV_SHA" 2>&1 | tail -n2 || \ warn "git reset to $DOCKER_PREV_SHA failed during rollback" else warn "No previous commit recorded — skipping git reset (images/bundle still restored)" fi if [ -d "$INSTALL_DIR/frontend/dist.rollback" ]; then rm -rf "$INSTALL_DIR/frontend/dist" mv "$INSTALL_DIR/frontend/dist.rollback" "$INSTALL_DIR/frontend/dist" fi local img for img in serverkit-backend serverkit-frontend; do docker image inspect "$img:rollback" >/dev/null 2>&1 && \ docker tag "$img:rollback" "$img:latest" 2>/dev/null || true done dc up -d 2>&1 | tail -n10 || true # Confirm the restored deployment came back healthy. local waited=0 status="" while [ "$waited" -lt 60 ]; do status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' serverkit-backend 2>/dev/null || echo missing)" { [ "$status" = "healthy" ] || [ "$status" = "none" ]; } && break sleep 3 waited=$((waited + 3)) done if [ "$status" = "healthy" ] || [ "$status" = "none" ]; then halt "Rolled back to previous version (${DOCKER_PREV_SHA:-unknown}) and it is healthy. Logs: cd $INSTALL_DIR && docker compose logs backend" else halt "Rolled back to previous version (${DOCKER_PREV_SHA:-unknown}) but it is STILL UNHEALTHY (status: $status) — manual intervention needed. Logs: cd $INSTALL_DIR && docker compose logs backend" fi } # Drop rollback artifacts once an upgrade is confirmed healthy. clear_docker_rollback() { [ "$DRY_RUN" = "1" ] && return 0 rm -rf "$INSTALL_DIR/frontend/dist.rollback" 2>/dev/null || true local img for img in serverkit-backend serverkit-frontend; do docker image inspect "$img:rollback" >/dev/null 2>&1 && \ docker rmi "$img:rollback" >/dev/null 2>&1 || true done } update_docker_compose() { printf '\n %s%sServerKit Updater — Docker deployment%s\n' "$BLD" "$PAPER" "$RST" STARTED_AT=$(date +%s) [ "$DRY_RUN" = "1" ] && warn "DRY RUN — no changes will be made" command -v docker &>/dev/null || halt "docker is required but not installed" command -v git &>/dev/null || halt "git is required but not installed" heal_layout_for_docker # Resolve the git ref to update to (branch / release tag / main). local ref="origin/main" local mode="main" if [ -n "$TARGET_BRANCH" ]; then ref="origin/$TARGET_BRANCH" mode="branch:$TARGET_BRANCH" elif [ "$USE_RELEASE" = "1" ]; then # `|| true`: head exiting first can SIGPIPE git (rc 141 under # pipefail); an empty ref is caught by the halt just below. ref="${RELEASE_VERSION:-$(git -C "$INSTALL_DIR" tag -l 'v*' --sort=-v:refname | head -n1 || true)}" [ -n "$ref" ] || halt "Could not determine a release tag to update to" mode="release" fi # Skip when already current (unless --force). Capture the starting version # for the summary before we touch the tree. local old_version old_version="$(local_version)" version_gate phase "Database Backup" backup_docker_db # Snapshot the current (still-running, last-known-good) deployment before we # mutate the tree, rebuild the bundle, or rebuild images — so an unhealthy # upgrade can be reverted. snapshot_docker_state phase "Syncing Source" if [ "$DRY_RUN" = "1" ]; then info "[dry-run] would git fetch and reset $INSTALL_DIR to $ref" else # The compose build context IS the live tree, so update it in place # (no blue/green). Untracked .env, instance DB and frontend/dist are # preserved across a hard reset. git -C "$INSTALL_DIR" fetch --all --tags --prune 2>&1 | tail -n2 || halt "git fetch failed" git -C "$INSTALL_DIR" reset --hard "$ref" 2>&1 | tail -n2 || halt "git reset to $ref failed" chmod +x "$INSTALL_DIR/serverkit" "$INSTALL_DIR/scripts/"*.sh 2>/dev/null || true # Untracked plugin dirs survive the hard reset — drop retired ones so # they can't sink the bundle rebuild below. remove_retired_plugins "$INSTALL_DIR" good "Source synced to $ref" fi # When the compose serves a host-built bundle (bind-mounted ./frontend/dist) # the image's own assets are shadowed, so the bundle must be rebuilt on the # host. Otherwise the frontend image carries its assets and needs no host build. if grep -qE '\./frontend/dist[ :]' "$INSTALL_DIR/docker-compose.yml" 2>/dev/null; then phase "Building Frontend" if [ "$DRY_RUN" = "1" ]; then info "[dry-run] would npm ci + npm run build in frontend/" elif command -v npm &>/dev/null; then _node_build_ok || halt "Node.js $(node --version 2>/dev/null || echo 'not found') is too old to build the frontend — vite 8 needs 20.19+ or 22.12+. Upgrade Node (e.g. NodeSource 22 LTS) and re-run." ( cd "$INSTALL_DIR/frontend" && npm ci 2>&1 | tail -n3 && \ NODE_OPTIONS="--max-old-space-size=1024" npm run build 2>&1 | tail -n5 ) \ || halt "Frontend build failed" good "Frontend assets rebuilt" else warn "npm not on host; relying on the frontend image build for assets" fi fi phase "Building Images" if [ "$DRY_RUN" = "1" ]; then info "[dry-run] would docker compose build" else # Build before recreating: running containers are untouched until # 'up -d', so a failed build leaves the current version serving. dc build 2>&1 | tail -n15 || halt "docker compose build failed — current version still running" good "Images built" fi phase "Recreating Containers" if [ "$DRY_RUN" = "1" ]; then info "[dry-run] would docker compose up -d (backend auto-migrates on boot)" else dc up -d 2>&1 | tail -n15 || halt "docker compose up -d failed" # If the frontend serves a host-built bundle, restart it so nginx picks # up freshly built assets even when its own image/config did not change. if grep -qE '\./frontend/dist[ :]' "$INSTALL_DIR/docker-compose.yml" 2>/dev/null; then dc restart frontend 2>&1 | tail -n3 || true fi good "Containers recreated" fi phase "Health Check" if [ "$DRY_RUN" = "1" ]; then info "[dry-run] would wait for serverkit-backend to report healthy" else local waited=0 status="" while [ "$waited" -lt 90 ]; do status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' serverkit-backend 2>/dev/null || echo missing)" [ "$status" = "healthy" ] && { good "Backend healthy"; break; } [ "$status" = "none" ] && { good "Backend container running (no healthcheck defined)"; break; } sleep 3 waited=$((waited + 3)) done if [ "$status" != "healthy" ] && [ "$status" != "none" ]; then warn "Backend did not report healthy within 90s (status: $status)" rollback_docker # L4 — revert to the snapshot; this halts the script fi fi # Healthy (or no healthcheck defined): the upgrade stuck, drop the snapshot. clear_docker_rollback # Keep the firewall in sync (idempotent, best-effort). ensure_firewall || true local new_version new_version="$(cat "$INSTALL_DIR/VERSION" 2>/dev/null | tr -d '\n\r ' || echo unknown)" curl -s "https://serverkit.ai/track/update?v=${new_version}" >/dev/null 2>&1 || true printf '\n %s%s✔ Update complete (Docker)%s %s%s%s\n\n' \ "$BLD" "$HUE_OK" "$RST" "$FOG" "$(clock)" "$RST" printf ' Updated %s → %s\n' "${old_version:-unknown}" "$new_version" printf ' Mode docker/%s\n' "$mode" printf ' Duration %s\n' "$(clock)" printf ' Backend %s\n' "$(docker inspect -f '{{.State.Status}}' serverkit-backend 2>/dev/null || echo unknown)" printf ' Frontend %s\n' "$(docker inspect -f '{{.State.Status}}' serverkit-frontend 2>/dev/null || echo unknown)" [ -n "${UPDATE_LOG:-}" ] && printf ' Log %s\n' "$UPDATE_LOG" printf '\n %sCLI%s serverkit status\n\n' "$BLD" "$RST" } # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- # Sourcing this file (e.g. from scripts/test/test_update.sh) exposes every # function above for unit testing without running an update. Only a direct # execution falls through to the run below. [ "${BASH_SOURCE[0]}" = "${0}" ] || return 0 maybe_reexec_latest_updater # may exec into the newest updater and not return acquire_update_lock init_logging # L5 — turn any unguarded failure (in the main flow OR a helper, thanks to -E) # into a labelled report instead of a silent drop back to the prompt. trap 'report_failure "$?" "$LINENO" "$BASH_COMMAND"' ERR # All-Docker deployments don't use the host venv / systemd / blue-green flow — # route them to the compose path. This is what prevents the missing-venv crash. if is_docker_deployment; then update_docker_compose exit 0 fi printf '\n %s%sServerKit Updater%s\n' "$BLD" "$PAPER" "$RST" STARTED_AT=$(date +%s) [ "$DRY_RUN" = "1" ] && warn "DRY RUN — no changes will be made" # Roll back automatically if the update fails after the symlink switch (the # handler, cleanup_on_exit, is defined next to rollback() above). trap cleanup_on_exit EXIT preflight_check # Capture where we are coming from and what mode this is, then skip the whole # update when the box is already current (unless --force). Doing this before any # backup/deploy work means an up-to-date box does zero churn. OLD_VERSION="$(local_version)" if [ "$USE_RELEASE" = "1" ]; then UPDATE_MODE="release" elif [ -n "$TARGET_BRANCH" ]; then UPDATE_MODE="branch:$TARGET_BRANCH" else UPDATE_MODE="main" fi version_gate ensure_bluegreen_layout backup_current # Determine update mode. NEXT_DIR="$(next_real_dir)" if [ "$USE_RELEASE" = "1" ]; then if [ -z "$RELEASE_VERSION" ]; then RELEASE_VERSION="$(curl -sf "https://api.github.com/repos/${GITHUB_REPO}/releases/latest" \ | grep '"tag_name"' | head -1 | cut -d'"' -f4 || true)" [ -n "$RELEASE_VERSION" ] || halt "Could not determine the latest release" fi step "Updating to release $RELEASE_VERSION" deploy_release "$NEXT_DIR" "$RELEASE_VERSION" elif [ -n "$TARGET_BRANCH" ]; then step "Updating to branch $TARGET_BRANCH" deploy_source "$NEXT_DIR" "$TARGET_BRANCH" else deploy_source "$NEXT_DIR" "main" fi # Ensure venv in the new tree. require_venv "$NEXT_DIR/venv" # Run database migrations before switching. migrate_database "$NEXT_DIR" # Sync templates. if [ "$DRY_RUN" = "0" ]; then mkdir -p /etc/serverkit/templates cp -r "$NEXT_DIR/backend/templates/"*.yaml /etc/serverkit/templates/ 2>/dev/null || true cp -r "$NEXT_DIR/backend/templates/"*.yml /etc/serverkit/templates/ 2>/dev/null || true fi # Build frontend if dist is missing (source mode or older release). if [ ! -d "$NEXT_DIR/frontend/dist" ]; then step "Building frontend..." if [ "$DRY_RUN" = "0" ]; then # The build is guarded so a failure halts loudly while the old slot # still serves. When plugins were carried forward from the old install # they are the usual culprit (imports the new tree no longer satisfies) # — quarantine them and give the build one retry before giving up. if ! build_frontend_bundle "$NEXT_DIR"; then quarantine_carried_plugins "$NEXT_DIR" \ || halt "Frontend build failed — previous installation still active" step "Retrying frontend build without carried-forward plugins..." build_frontend_bundle "$NEXT_DIR" reuse \ || halt "Frontend build failed — previous installation still active" fi else info "[dry-run] would npm ci + npm run build in $NEXT_DIR/frontend" fi fi # Re-apply the persistent per-install favicon tint to the staged dist (whether # just rebuilt above or shipped prebuilt), so the color carries across updates. [ "$DRY_RUN" = "0" ] && apply_favicon_tint "$NEXT_DIR" # Refresh nginx/systemd configs in the active tree before switch. refresh_config "$NEXT_DIR" # Stop services. # # nginx is deliberately NOT stopped: it fronts every managed app, so taking it # down would black out unrelated sites for the whole switch. Only the panel # backend is cycled (a brief panel-API gap that never touches hosted apps), and # nginx picks up any refreshed config via a graceful reload after the switch. # # First snapshot which hosted apps are reachable through nginx right now, so we # can prove afterwards that the update did not knock any of them offline. APP_BASELINE="$(snapshot_app_reachability)" phase "Stopping Services" # Guarded like the rollback path: `systemctl stop` exits non-zero when the unit # is not loaded (exit 5) — a reason to warn, never to abort a healthy update. run_or_dry systemctl stop "$BACKEND_SERVICE" \ || warn "Backend service did not stop cleanly (unit may not be loaded) — continuing" wait_for_service "$BACKEND_SERVICE" inactive 30 || warn "Backend did not stop within 30 seconds" # Record the currently active directory before switching. PREVIOUS_DIR="$(active_real_dir)" # Atomic switch. atomic_switch "$NEXT_DIR" # Start services. # # The frontend is now static files under the switched-in slot's frontend/dist, # served directly by host nginx (no container to recreate). The atomic switch # already swapped the served assets; nginx only needs a graceful config reload. phase "Starting Services" run_or_dry systemctl start "$BACKEND_SERVICE" wait_for_service "$BACKEND_SERVICE" active 30 || warn "Backend did not report active within 30 seconds" # Re-label the now-active slot's bundle for SELinux hosts before nginx serves it. selinux_label_dist "$NEXT_DIR" # Graceful, zero-downtime config swap — never a stop/start (see reload_nginx_graceful). reload_nginx_graceful good "Services started" # Health check. health_check # Verify the panel update did not take any hosted app down. Compare the # pre-switch reachability snapshot with a fresh probe now that nginx has # reloaded. Best-effort: a regression is reported loudly but does not fail the # (already-healthy) update — the operator decides what to do about an app that # was likely already unhealthy. if [ "$DRY_RUN" = "0" ]; then phase "Verifying App Uptime" APP_AFTER="$(snapshot_app_reachability)" report_app_uptime_regressions "$APP_BASELINE" "$APP_AFTER" || true fi # Keep the firewall in sync (idempotent, best-effort). ensure_firewall || true # Keep bash tab-completion in sync with the updated CLI (best-effort). refresh_bash_completion || true # Cleanup. cleanup # Summary. NEW_VERSION="$(local_version)" printf '\n %s%s✔ Update complete%s %s%s%s\n\n' \ "$BLD" "$HUE_OK" "$RST" "$FOG" "$(clock)" "$RST" printf ' Updated %s → %s\n' "${OLD_VERSION:-unknown}" "${NEW_VERSION:-unknown}" printf ' Mode %s\n' "${UPDATE_MODE:-main}" printf ' Duration %s\n' "$(clock)" printf ' Active %s\n' "$(active_real_dir)" printf ' Backend %s\n' "$(systemctl is-active serverkit 2>/dev/null || echo unknown)" printf ' Nginx %s\n\n' "$(systemctl is-active nginx 2>/dev/null || echo unknown)" [ -n "${UPDATE_LOG:-}" ] && printf ' Log %s\n\n' "$UPDATE_LOG" printf ' %sCLI%s serverkit status\n\n' "$BLD" "$RST"