admin/ServerKit
1
0 Fork 0
mirror of https://github.com/jhd3197/ServerKit.git synced 2026-10-23 03:27:43 +09:00
Code Issues Packages Projects Releases Wiki Activity
Files
main
ServerKit/scripts/update.sh
T
Juan Denis e6e94dd23a Refuse release tarballs without a checksum 
Both the installer and the updater treated a missing checksums.txt as
"skip verification" and installed the tarball anyway. Every release
publishes one, so its absence means a broken mirror or a tampered
download. The installer now falls back to a source build and the
updater refuses; SERVERKIT_ALLOW_UNVERIFIED=1 overrides for mirrors
that do not host checksums.
2026-10-10 18:20:27 -04:00

2315 lines
100 KiB
Bash
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/bin/bash
#
# ServerKit updater — atomic blue/green, pre-flight checked, offline-capable.
#
# Updates default to a PRE-BUILT release tarball — the frontend is built once in
# CI and shipped, so the server needs no Node/npm and no on-box SPA build (that's
# what makes updates work uniformly on old distros, ARM/Pi, and tiny boxes). Opt
# into an on-box source rebuild with --source / --branch / BUILD_FROM_SOURCE=1
# (that path needs a compatible Node — see install.sh's node floor).
#
# Usage:
# bash /opt/serverkit/scripts/update.sh # pre-built release (default, Node-free)
# bash /opt/serverkit/scripts/update.sh --dry-run
# bash /opt/serverkit/scripts/update.sh --source # rebuild from main on this box (needs Node)
# bash /opt/serverkit/scripts/update.sh --branch dev # rebuild from a branch on this box (needs Node)
# bash /opt/serverkit/scripts/update.sh --release [v1.7.0]
# SERVERKIT_OFFLINE_TARBALL=/tmp/serverkit-v1.7.0-linux-amd64.tar.gz bash /opt/serverkit/scripts/update.sh
#
# -E (errtrace) makes the ERR trap fire for failures *inside* functions/subshells
# too — without it a silent death deep in a helper would never be reported.
set -Eeuo pipefail
# ---------------------------------------------------------------------------
# Configuration + argument parsing
# ---------------------------------------------------------------------------
DRY_RUN=0
FORCE_UPDATE=0
TARGET_BRANCH=""
# Default to a pre-built release update (Node-free). --source / --branch /
# BUILD_FROM_SOURCE=1 opt into an on-box source rebuild instead.
USE_RELEASE="${INSTALL_FROM_RELEASE:-1}"
[ "${BUILD_FROM_SOURCE:-0}" = "1" ] && USE_RELEASE=0
RELEASE_VERSION="${SERVERKIT_VERSION:-}"
# Captured before parsing so the self-update re-exec can forward them verbatim.
ORIG_ARGS=("$@")
while [[ $# -gt 0 ]]; do
case "$1" in
--dry-run|-n)
DRY_RUN=1
shift
;;
--force|-f)
FORCE_UPDATE=1
shift
;;
--source|--main)
# Rebuild from the main branch on this box (needs a compatible Node).
USE_RELEASE=0
TARGET_BRANCH=""
shift
;;
--branch|-b)
# A branch update is an on-box source rebuild — override the release default.
TARGET_BRANCH="$2"
USE_RELEASE=0
shift 2
;;
--release|-r)
USE_RELEASE=1
if [[ -n "${2:-}" ]] && [[ ! "$2" =~ ^- ]]; then
RELEASE_VERSION="$2"
shift 2
else
shift
fi
;;
--help|-h)
cat <<'EOF'
Usage: update.sh [OPTIONS]
Options:
--dry-run, -n Show what would happen without making changes
--force, -f Skip version comparison and update anyway
--release [version], -r Update from a pre-built release tarball (DEFAULT; Node-free)
--source, --main Rebuild from the main branch on this box (needs Node)
--branch <name>, -b Rebuild from a git branch on this box (needs Node)
--help, -h Show this help message
Environment:
SERVERKIT_DIR Active install directory (default: /opt/serverkit)
SERVERKIT_VENV_DIR Python venv path (default: $SERVERKIT_DIR/venv)
SERVERKIT_OFFLINE_TARBALL Local release tarball to use instead of downloading
SERVERKIT_MIRROR_URL Base URL for release tarballs/checksums
GITHUB_REPO GitHub org/repo for source and releases
EOF
exit 0
;;
*)
echo "Unknown option: $1" >&2
exit 1
;;
esac
done
# ---------------------------------------------------------------------------
# Paths
# ---------------------------------------------------------------------------
SERVERKIT_DIR="${SERVERKIT_DIR:-/opt/serverkit}"
INSTALL_DIR="$SERVERKIT_DIR"
BASE_NAME="$(basename "$INSTALL_DIR")"
BASE_DIR="$(dirname "$INSTALL_DIR")"
DIR_A="$BASE_DIR/${BASE_NAME}-a"
DIR_B="$BASE_DIR/${BASE_NAME}-b"
VENV_DIR="${SERVERKIT_VENV_DIR:-$INSTALL_DIR/venv}"
BACKUP_DIR="${SERVERKIT_BACKUP_DIR:-/var/backups/serverkit}"
LOG_DIR="/var/log/serverkit"
CONFIG_DIR="${SERVERKIT_CONFIG_DIR:-/etc/serverkit}"
LOCK_FILE="${SERVERKIT_LOCK_FILE:-/var/lock/serverkit-update.lock}"
# System integration dirs — overridable so the config-refresh logic can be
# exercised against fixtures in tests instead of the host's real /etc.
NGINX_DIR="${SERVERKIT_NGINX_DIR:-/etc/nginx}"
LETSENCRYPT_DIR="${SERVERKIT_LETSENCRYPT_DIR:-/etc/letsencrypt}"
SYSTEMD_DIR="${SERVERKIT_SYSTEMD_DIR:-/etc/systemd/system}"
# Per-app nginx location snippets — one proxy_pass to each managed app's
# container. The updater probes these to prove apps stayed up across the switch.
APP_LOCATIONS_DIR="${SERVERKIT_APP_LOCATIONS_DIR:-/etc/nginx/serverkit-locations}"
GITHUB_REPO="${GITHUB_REPO:-jhd3197/ServerKit}"
SERVERKIT_OFFLINE_TARBALL="${SERVERKIT_OFFLINE_TARBALL:-}"
SERVERKIT_MIRROR_URL="${SERVERKIT_MIRROR_URL:-}"
BACKEND_SERVICE="serverkit"
# ---------------------------------------------------------------------------
# Terminal styling
# ---------------------------------------------------------------------------
if [ -t 1 ] && [ -z "${NO_COLOR:-}" ] && [ "${TERM:-dumb}" != "dumb" ]; then
ESC=$'\033'
RST="${ESC}[0m"; BLD="${ESC}[1m"
paint() { printf '%s[38;2;%d;%d;%dm' "$ESC" "$1" "$2" "$3"; }
else
RST=''; BLD=''
paint() { :; }
fi
V3="$(paint 139 92 246)"; V4="$(paint 124 58 237)"
PAPER="$(paint 237 233 254)"; FOG="$(paint 113 108 140)"
HUE_OK="$(paint 52 211 153)"; HUE_WARN="$(paint 250 204 21)"
HUE_ERR="$(paint 248 113 113)"; HUE_LINK="$(paint 103 232 249)"
good() { printf ' %s✔%s %s\n' "$HUE_OK" "$RST" "$1"; }
warn() { printf ' %s▴%s %s\n' "$HUE_WARN" "$RST" "$1"; }
# Every hard failure points at the full log so a stuck box is debuggable from a
# single line (UPDATE_LOG is empty until init_logging runs, hence the guard).
halt() {
printf ' %s✘%s %s\n' "$HUE_ERR" "$RST" "$1" >&2
[ -n "${UPDATE_LOG:-}" ] && printf ' %sfull log: %s%s\n' "$FOG" "$UPDATE_LOG" "$RST" >&2
exit 1
}
step() { printf ' %s❯%s %s\n' "$HUE_LINK" "$RST" "$1"; }
info() { printf ' %s•%s %s\n' "$FOG" "$RST" "$1"; }
STARTED_AT=0
PHASE_N=0
clock() {
[ "$STARTED_AT" -gt 0 ] || { printf ''; return; }
local secs=$(( $(date +%s) - STARTED_AT ))
printf '%dm %02ds' "$((secs / 60))" "$((secs % 60))"
}
LAST_PHASE="startup"
phase() {
PHASE_N=$((PHASE_N + 1))
LAST_PHASE="$1"
printf '\n %s%s%02d%s %s%s%s %s%s%s\n' \
"$BLD" "$V3" "$PHASE_N" "$RST" "$BLD" "$1" "$RST" "$FOG" "$(clock)" "$RST"
printf ' %s%s%s\n\n' "$V4" "──────────────────────────────────────" "$RST"
}
# ---------------------------------------------------------------------------
# L5 — Loud failures + always-on logging
# ---------------------------------------------------------------------------
# `set -euo pipefail` makes any unguarded command abort the script. Without this
# layer that abort is *silent* — you just land back at the prompt with no idea
# which phase, line, or command died (exactly the failure mode that made this
# updater so hard to debug). init_logging mirrors everything to a timestamped
# log; the ERR trap turns every abort into a labelled, actionable message.
UPDATE_LOG=""
init_logging() {
[ -n "${SERVERKIT_NO_LOG:-}" ] && return 0
mkdir -p "$LOG_DIR" 2>/dev/null || true
if [ -d "$LOG_DIR" ] && [ -w "$LOG_DIR" ]; then
UPDATE_LOG="$LOG_DIR/update-$(date +%Y%m%d-%H%M%S).log"
# Keep output on the terminal *and* append it to the log. Colors were
# already resolved above from the real TTY, so they survive the pipe.
exec > >(tee -a "$UPDATE_LOG") 2>&1
info "Logging to $UPDATE_LOG"
fi
}
report_failure() {
local rc=$1 line=$2 cmd=$3
printf '\n %s✘ Update aborted%s during %s%s%s\n' \
"$HUE_ERR" "$RST" "$BLD" "$LAST_PHASE" "$RST" >&2
printf ' %sexit %s · line %s · %s%s\n' "$FOG" "$rc" "$line" "$cmd" "$RST" >&2
# `if`, not a trailing `&&` list: with UPDATE_LOG empty (fresh box before
# init_logging, SERVERKIT_NO_LOG=1) the list form returns 1 from the
# reporter itself — the same species as the July 2 outage.
if [ -n "$UPDATE_LOG" ]; then
printf ' %sfull log: %s%s\n' "$FOG" "$UPDATE_LOG" "$RST" >&2
fi
}
# ---------------------------------------------------------------------------
# Self-updating bootstrap + run lock
# ---------------------------------------------------------------------------
# The updater that runs is whatever is installed on the box — so a *stale*
# update.sh (one predating a bug fix or a new deployment shape) fails in ways
# the current code already handles, and it can't fix itself. This is the exact
# trap that left a box stuck: an old updater died before it could install the
# fixed one. Before doing any work, fetch the newest update.sh for the target
# ref and re-exec into it. From this version on, "just run serverkit update"
# is reliable no matter how old the box is.
#
# CHANNEL DECISION (2026-07, scripts-reliability round 2): stable installs keep
# fetching this file from *main* — no release buffer. The flip side is that any
# updater bug merged to main is live on every box instantly, so merges touching
# scripts/** are gated by the real install+update e2e job in scripts-ci.yml
# (update-e2e). Fetching from the latest release tag was considered and
# rejected: a broken *released* updater would need a whole new release to fix,
# while main + a CI gate keeps the one-push-fixes-the-fleet property that
# resolved the 2026-07-02 incident.
SELF_PATH="${BASH_SOURCE[0]}"
maybe_reexec_latest_updater() {
[ -n "${SERVERKIT_UPDATER_REEXECED:-}" ] && return 0 # already the latest
[ "${SERVERKIT_NO_SELF_UPDATE:-0}" = "1" ] && return 0 # opt-out / tests
[ "$DRY_RUN" = "1" ] && return 0
[ -n "$SERVERKIT_OFFLINE_TARBALL" ] && return 0 # offline: nothing to fetch
command -v curl &>/dev/null || return 0
local ref="main"
[ -n "$TARGET_BRANCH" ] && ref="$TARGET_BRANCH"
[ "$USE_RELEASE" = "1" ] && [ -n "$RELEASE_VERSION" ] && ref="$RELEASE_VERSION"
local url tmp
url="https://raw.githubusercontent.com/${GITHUB_REPO}/${ref}/scripts/update.sh"
tmp="$(mktemp)"
if ! curl -fsSL --max-time 20 "$url" -o "$tmp" 2>/dev/null || [ ! -s "$tmp" ]; then
rm -f "$tmp"; return 0 # fetch failed → use local
fi
# Swap only if it genuinely differs AND is valid bash (never re-exec into a
# truncated/corrupt download).
if ! cmp -s "$tmp" "$SELF_PATH" && bash -n "$tmp" 2>/dev/null; then
info "Refreshing the updater itself from ${ref}..."
chmod +x "$tmp" 2>/dev/null || true
# ${arr[@]+...}: expanding an EMPTY array as "${arr[@]}" is an
# unbound-variable error under set -u on bash < 4.4 (older distros).
SERVERKIT_UPDATER_REEXECED=1 exec bash "$tmp" ${ORIG_ARGS[@]+"${ORIG_ARGS[@]}"}
fi
rm -f "$tmp"
}
# Stop two concurrent updates clobbering each other (e.g. both cloning into the
# same blue/green slot, or racing docker compose). The lock auto-releases when
# the script exits, since the fd closes.
acquire_update_lock() {
[ "$DRY_RUN" = "1" ] && return 0
command -v flock &>/dev/null || return 0
mkdir -p "$(dirname "$LOCK_FILE")" 2>/dev/null || true
# Brace-group so a failed open (unwritable path) is caught here instead of
# the redirection error aborting the whole script — then just skip locking.
{ exec 9>"$LOCK_FILE"; } 2>/dev/null || return 0
if ! flock -n 9; then
halt "Another 'serverkit update' is already running (lock: $LOCK_FILE)."
fi
}
# ---------------------------------------------------------------------------
# Utility helpers
# ---------------------------------------------------------------------------
run_or_dry() {
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would run: $*"
else
"$@"
fi
}
# Like run_or_dry, but runs the command inside <dir>. Under --dry-run it reports
# the working directory it *would* run in instead of silently dropping the cd —
# a chained `run_or_dry cd X && run_or_dry docker compose ...` used to print the
# compose command with no hint of where it would execute.
run_in_dir() {
local dir="$1"; shift
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] (cwd: $dir) would run: $*"
else
( cd "$dir" && "$@" )
fi
}
# ---------------------------------------------------------------------------
# Version comparison — skip the whole update when already current.
# ---------------------------------------------------------------------------
local_version() {
tr -d '\n\r ' < "$INSTALL_DIR/VERSION" 2>/dev/null || true
}
# True when two version strings match ignoring a leading "v" (1.7.1 == v1.7.1).
versions_equal() {
[ "${1#v}" = "${2#v}" ]
}
# Resolve the release tag we would update to (pinned, or the latest on GitHub).
remote_release_tag() {
if [ -n "$RELEASE_VERSION" ]; then
printf '%s' "$RELEASE_VERSION"
return 0
fi
curl -sf "https://api.github.com/repos/${GITHUB_REPO}/releases/latest" 2>/dev/null \
| grep '"tag_name"' | head -1 | cut -d'"' -f4 || true
}
# Return 0 when the install is already at the target version/ref (the caller
# should then skip). Best-effort: --force always proceeds; offline always
# proceeds (can't compare); an indeterminate comparison proceeds with the
# update rather than wrongly blocking it.
is_already_current() {
[ "$FORCE_UPDATE" = "1" ] && return 1
[ -n "$SERVERKIT_OFFLINE_TARBALL" ] && return 1
if [ "$USE_RELEASE" = "1" ]; then
local target local_v
target="$(remote_release_tag)"
[ -n "$target" ] || return 1
local_v="$(local_version)"
[ -n "$local_v" ] || return 1
versions_equal "$local_v" "$target" && return 0
return 1
fi
# Branch / main mode: compare the local checkout's HEAD against the remote
# branch HEAD. ls-remote needs no fetch and works on shallow clones.
local branch local_head remote_head
branch="${TARGET_BRANCH:-main}"
git -C "$INSTALL_DIR" rev-parse --is-inside-work-tree >/dev/null 2>&1 || return 1
local_head="$(git -C "$INSTALL_DIR" rev-parse HEAD 2>/dev/null || true)"
remote_head="$(git -C "$INSTALL_DIR" ls-remote "https://github.com/${GITHUB_REPO}.git" \
"refs/heads/$branch" 2>/dev/null | awk '{print $1}' | head -1)"
[ -n "$local_head" ] && [ -n "$remote_head" ] && [ "$local_head" = "$remote_head" ] && return 0
return 1
}
# If already current, announce it and exit cleanly (unless --force).
version_gate() {
if is_already_current; then
local v; v="$(local_version)"
good "Already up to date (version ${v:-unknown}). Use --force to update anyway."
exit 0
fi
}
wait_for_service() {
local unit="$1" target_state="$2" timeout="${3:-30}"
local waited=0
while [ "$waited" -lt "$timeout" ]; do
if systemctl is-active --quiet "$unit" 2>/dev/null; then
[ "$target_state" = "active" ] && return 0
else
[ "$target_state" = "inactive" ] && return 0
fi
sleep 1
waited=$((waited + 1))
done
return 1
}
# Apply the (possibly refreshed) nginx config WITHOUT dropping traffic.
#
# Host nginx is the front door for every managed app — it reverse-proxies their
# containers via /etc/nginx/serverkit-locations/*.conf. Stopping it (the old
# behaviour) blacked out every hosted site for the whole switch window, so a
# panel update became an outage for unrelated apps. `nginx -s reload` keeps the
# listening sockets and in-flight connections alive while swapping config, so
# apps never blink. The updater only ever changes the *config* (refresh_config),
# never anything nginx serves from the blue/green slot, so a reload is all that
# is ever needed. If nginx happens to be down, start it instead.
reload_nginx_graceful() {
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would reload nginx (graceful, zero-downtime)"
return 0
fi
if systemctl is-active --quiet nginx 2>/dev/null; then
# Validate before reloading: a bad config would otherwise leave nginx
# running the OLD config (reload is rejected) — non-fatal, but warn loudly.
if nginx -t >/dev/null 2>&1; then
systemctl reload nginx 2>/dev/null \
|| run_or_dry systemctl reload nginx \
|| warn "nginx reload failed — apps keep serving the previous config"
else
warn "nginx config test failed — skipping reload (apps keep serving the previous config)"
fi
else
warn "nginx was not running — starting it"
# Never propagate a failed start: nginx being down is a pre-existing
# condition, and aborting here (post-switch) would fire a rollback that
# cannot fix nginx either — warn and let the operator handle it.
run_or_dry systemctl start nginx \
|| warn "nginx failed to start — check 'systemctl status nginx'"
wait_for_service nginx active 15 || warn "nginx did not report active within 15 seconds"
fi
}
# Label a slot's SPA bundle for SELinux-enforcing hosts so nginx can read the
# static panel assets it now serves (see install.sh selinux_label_frontend_dist).
# Best-effort and a no-op on permissive/disabled boxes or where the tools are
# absent. Takes the slot's real path (blue/green resolves through a symlink).
selinux_label_dist() {
local dir="$1"
[ "$DRY_RUN" = "1" ] && return 0
command -v selinuxenabled &>/dev/null && selinuxenabled 2>/dev/null || return 0
local dist="${dir}/frontend/dist"
[ -d "$dist" ] || return 0
if command -v semanage &>/dev/null && command -v restorecon &>/dev/null; then
semanage fcontext -a -t httpd_sys_content_t "${dist}(/.*)?" 2>/dev/null \
|| semanage fcontext -m -t httpd_sys_content_t "${dist}(/.*)?" 2>/dev/null || true
restorecon -R "$dist" 2>/dev/null || true
elif command -v chcon &>/dev/null; then
chcon -R -t httpd_sys_content_t "$dist" 2>/dev/null || true
fi
}
# ---------------------------------------------------------------------------
# Managed-app uptime verification
# ---------------------------------------------------------------------------
# The whole point of the reload-not-stop discipline is that hosted apps never go
# down when the panel updates. These helpers turn that promise into something the
# updater actually checks: snapshot which app upstreams are reachable before the
# switch, re-probe after the reload, and shout if any app that WAS serving has
# stopped answering.
# Discover the upstreams (host:port) host nginx reverse-proxies for managed apps
# by scanning the per-app location snippets. One unique "host:port" per line.
discover_app_upstreams() {
[ -d "$APP_LOCATIONS_DIR" ] || return 0
grep -rhoE 'proxy_pass[[:space:]]+https?://127\.0\.0\.1:[0-9]+' "$APP_LOCATIONS_DIR" 2>/dev/null \
| grep -oE '127\.0\.0\.1:[0-9]+' | sort -u || true
}
# Probe each upstream on stdin; emit "host:port up" / "host:port down". "up" means
# the app container answered at all (ANY HTTP status — a 4xx/5xx app is still
# reachable; a refused/timed-out connection is not), which is exactly what nginx
# needs to keep fronting it.
probe_app_upstreams() {
local up
while IFS= read -r up; do
[ -n "$up" ] || continue
if curl -s -o /dev/null --max-time 4 "http://$up/" 2>/dev/null; then
printf '%s up\n' "$up"
else
printf '%s down\n' "$up"
fi
done
}
# A reachability snapshot for every fronted app: "host:port state" lines.
snapshot_app_reachability() {
[ "$DRY_RUN" = "1" ] && return 0
discover_app_upstreams | probe_app_upstreams
}
# Compare a pre-update snapshot with a post-update one. Warn for any app that was
# reachable before and is not now (the update disrupted a workload). Non-zero if
# any regression is found; an app that was already down before the update is not
# counted against us.
report_app_uptime_regressions() {
local before="$1" after="$2" regressed=0 total=0 line up state
while IFS= read -r line; do
[ -n "$line" ] || continue
up="${line% *}"; state="${line##* }"
total=$((total + 1))
[ "$state" = "up" ] || continue
if printf '%s\n' "$after" | grep -qx "$up down"; then
warn "App upstream $up was reachable before the update but is DOWN now"
regressed=$((regressed + 1))
fi
done <<EOF
$before
EOF
if [ "$total" -eq 0 ]; then
info "No managed apps fronted by nginx — nothing to verify"
elif [ "$regressed" -eq 0 ]; then
good "All $total managed app(s) stayed reachable across the update"
else
warn "$regressed of $total managed app(s) went down across the update"
fi
[ "$regressed" -eq 0 ]
}
# Resolve the currently active real directory behind the symlink.
active_real_dir() {
if [ -L "$INSTALL_DIR" ]; then
readlink -f "$INSTALL_DIR"
elif [ -d "$INSTALL_DIR" ]; then
echo "$INSTALL_DIR"
else
echo ""
fi
}
# Return the inactive blue/green directory.
next_real_dir() {
local active
active="$(active_real_dir)"
if [ "$active" = "$DIR_A" ]; then
echo "$DIR_B"
else
echo "$DIR_A"
fi
}
# Read the install profile recorded by install.sh in the active slot's .env.
# Anything unrecognised — including every install predating profiles — reports
# "standard", which is exactly what those installs are.
install_profile() {
local active env_file profile=""
active="$(active_real_dir)"
if [ -n "$active" ]; then
env_file="$active/.env"
if [ -f "$env_file" ]; then
profile="$(grep -E '^SERVERKIT_PROFILE=' "$env_file" 2>/dev/null \
| head -1 | cut -d= -f2- | tr -d ' \r')"
fi
fi
case "$profile" in
minimal|standard|full) printf '%s' "$profile" ;;
*) printf 'standard' ;;
esac
}
# Whether this update has to insist on Docker. The minimal profile ships
# without it on purpose, so demanding it here would make every minimal box
# installable but un-updatable. Two escape hatches keep that honest: an
# all-Docker deployment obviously needs it, and if Docker is present at all we
# still validate compose so a half-provisioned box is caught early.
# Split out so the unit tests can redefine it. Restricting PATH to hide docker
# would also hide the coreutils install_profile parses with, and the test would
# then pass for the wrong reason.
have_docker() { command -v docker &>/dev/null; }
update_needs_docker() {
[ "$(install_profile)" = "minimal" ] || return 0
is_docker_deployment && return 0
have_docker && return 0
return 1
}
# ---------------------------------------------------------------------------
# Pre-flight checks
# ---------------------------------------------------------------------------
preflight_check() {
phase "Pre-flight Checks"
# Root
if [ "$EUID" -ne 0 ]; then
halt "Run as root."
fi
good "Running as root"
# Install directory exists or can be created
if [ ! -e "$BASE_DIR" ]; then
halt "Base directory $BASE_DIR does not exist."
fi
good "Install base directory exists"
# Python version
local py_bin py_version
py_bin="$(locate_python 2>/dev/null || true)"
if [ -z "$py_bin" ]; then
halt "Python 3.11-3.14 is required. Install a supported python3.1x and python3-venv."
fi
py_version="$($py_bin -c 'import sys; print(".".join(map(str, sys.version_info[:2])))')"
good "Python $py_version available ($py_bin)"
# Required commands. Source-mode updates build the SPA on the host, so npm
# must be checked up front — discovering it missing mid-update would abort
# after the new slot is already half-built. Release tarballs ship a
# prebuilt dist and stay npm-free.
local cmd missing=() required=(git curl tar rsync systemctl nginx python3)
[ "$USE_RELEASE" = "1" ] || required+=(npm)
# Docker is conditional — see update_needs_docker().
if update_needs_docker; then
required+=(docker)
fi
for cmd in "${required[@]}"; do
command -v "$cmd" &>/dev/null || missing+=("$cmd")
done
if update_needs_docker; then
if ! docker compose version &>/dev/null && ! docker-compose --version &>/dev/null; then
missing+=("docker compose")
fi
fi
if [ ${#missing[@]} -gt 0 ]; then
halt "Missing required tools: ${missing[*]}"
fi
good "Required tools available"
if ! update_needs_docker; then
good "Docker not required (minimal profile)"
fi
# Disk space.
#
# A fixed 2 GiB floor is not enough on its own: this update is about to
# write TWO full copies of the database (a pre-upgrade snapshot and a tree
# backup that contains it) plus a release tree. With an 800 MB database
# that is ~2 GiB of new files, so a box with 2.1 GiB free passes the old
# check and then fills the disk mid-update. Size the requirement to what
# will actually be written, and reclaim stale snapshots before giving up.
local avail_kb need_kb db_kb active_dir
# -P (POSIX format) pins each filesystem to one line — a long device name
# otherwise wraps and shifts the column the awk parse reads.
avail_kb="$(df -Pk "$BASE_DIR" | awk 'NR==2 {print $4}')"
active_dir="$(active_real_dir 2>/dev/null || echo "$INSTALL_DIR")"
db_kb="$(du -sk "$active_dir/backend/instance/serverkit.db" 2>/dev/null | awk '{print $1}')"
[ -n "$db_kb" ] || db_kb=0
# 2 GiB floor + room for both database copies.
need_kb=$(( 2 * 1024 * 1024 + db_kb * 2 ))
if [ "$avail_kb" -lt "$need_kb" ]; then
warn "Low disk: $((avail_kb / 1024)) MiB free, need $((need_kb / 1024)) MiB — trimming old backups"
trim_backups
enforce_backup_budget
avail_kb="$(df -Pk "$BASE_DIR" | awk 'NR==2 {print $4}')"
fi
if [ "$avail_kb" -lt "$need_kb" ]; then
halt "Insufficient disk space on $BASE_DIR: $((avail_kb / 1024)) MiB free, need $((need_kb / 1024)) MiB (2 GiB + two $((db_kb / 1024)) MiB database copies). Run 'sudo serverkit disk' to reclaim space."
fi
good "Disk space OK ($((avail_kb / 1024 / 1024)) GiB free, need $((need_kb / 1024 / 1024)) GiB)"
# Memory (need 512 MiB free)
local mem_avail_mb
mem_avail_mb="$(awk '/MemAvailable:/ {print int($2/1024)}' /proc/meminfo 2>/dev/null || echo 0)"
if [ "$mem_avail_mb" -lt 512 ]; then
warn "Low memory: ${mem_avail_mb} MiB available (recommend >= 512 MiB)."
else
good "Memory OK (${mem_avail_mb} MiB available)"
fi
# Network reachability (skip in offline mode)
if [ -n "$SERVERKIT_OFFLINE_TARBALL" ]; then
info "Offline tarball set; skipping network checks"
elif [ "$USE_RELEASE" = "1" ] || [ -z "$TARGET_BRANCH" ]; then
if ! curl -sfI "https://github.com" >/dev/null 2>&1; then
halt "Cannot reach github.com. Set SERVERKIT_OFFLINE_TARBALL or fix network."
fi
good "Network reachability OK"
fi
# Current backend health (warn only; it may already be down)
if curl -sf --max-time 5 http://127.0.0.1:5000/api/v1/system/health >/dev/null 2>&1; then
good "Backend currently healthy"
else
warn "Backend is not currently responding on :5000 — will proceed anyway"
fi
}
# ---------------------------------------------------------------------------
# Python virtual environment
# ---------------------------------------------------------------------------
locate_python() {
local c v
for c in python3.14 python3.13 python3.12 python3.11 python3; do
if command -v "$c" &>/dev/null; then
v="$($c -c 'import sys;print(".".join(map(str,sys.version_info[:2])))' 2>/dev/null || true)"
# An EMPTY version means the binary/shim exists but can't actually
# execute Python — and an empty string would pass the sort checks
# below ("" sorts before everything). Reject it and move on.
[ -n "$v" ] || continue
if printf '%s\n%s' "3.11" "$v" | sort -C -V && \
printf '%s\n%s' "$v" "3.14" | sort -C -V; then
printf '%s' "$c"
return 0
fi
fi
done
return 1
}
rebuild_virtualenv() {
local target_dir="$1"
step "Rebuilding Python virtual environment..."
local py_bin
py_bin="$(locate_python)" || halt "ServerKit requires Python 3.11-3.14."
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would recreate venv at $target_dir using $py_bin"
return 0
fi
rm -rf "$target_dir"
"$py_bin" -m venv "$target_dir"
# shellcheck source=/dev/null
source "$target_dir/bin/activate"
pip install --upgrade pip --quiet
pip install -r "$target_dir/../backend/requirements.txt" --quiet
pip install gunicorn --quiet
good "Virtual environment rebuilt at $target_dir"
}
# Ensure the target directory has a usable venv. If a pre-built one exists AND
# is bound to this slot's path, use it; otherwise rebuild from requirements.
require_venv() {
local target_dir="$1"
if [ -f "$target_dir/bin/activate" ] && [ -x "$target_dir/bin/python" ]; then
# A venv is path-bound: bin/activate and every entry-point shebang
# embed the absolute path the venv was created at. The release
# tarball's prebuilt venv is baked as $INSTALL_DIR/venv, which —
# until atomic_switch flips the symlink — resolves to the OLD slot.
# Running it would execute the new code with the old slot's
# dependencies (and run the OLD flask binary for migrations), so a
# venv bound to any other path must be rebuilt in place.
local bound_path resolved_bound resolved_target
# Matches both VIRTUAL_ENV="/path" (classic venv) and the unquoted
# export VIRTUAL_ENV=/path form; skips cygpath-style indirection.
bound_path="$(grep -m1 -oE 'VIRTUAL_ENV="?/[^")]+"?' "$target_dir/bin/activate" | cut -d= -f2- | tr -d '"')"
# Compare RESOLVED paths: on a fresh install the symlink already points
# at the only slot, so the baked path resolves to the target and the
# prebuilt venv is genuinely usable (fast path). During an update it
# resolves to the OLD slot — that mismatch triggers the rebuild.
resolved_bound="$(readlink -f "$bound_path" 2>/dev/null || true)"
resolved_target="$(readlink -f "$target_dir" 2>/dev/null || echo "$target_dir")"
if [ -n "$resolved_bound" ] && [ "$resolved_bound" = "$resolved_target" ]; then
good "Virtual environment ready at $target_dir"
return 0
fi
warn "Virtual environment at $target_dir is bound to '${bound_path:-unknown path}' — rebuilding for this slot"
rebuild_virtualenv "$target_dir"
return 0
fi
warn "Virtual environment missing at $target_dir"
rebuild_virtualenv "$target_dir"
}
# ---------------------------------------------------------------------------
# Database migration
# ---------------------------------------------------------------------------
migrate_database() {
local work_dir="$1"
local venv="$work_dir/venv"
step "Running database migrations..."
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would run flask db upgrade in $work_dir/backend"
return 0
fi
# For SQLite installs, migrate the COPY we placed in the *new* slot, not the
# database the .env points at via the /opt/serverkit symlink (which still
# resolves to the live OLD slot until atomic_switch). Touching the old slot's
# DB here would corrupt rollback: reverting the symlink would hand the old
# code a database already upgraded to the new schema. Pointing the migration
# at the slot-absolute path keeps the old slot's DB byte-for-byte intact.
#
# Subshell: the venv activation and the cd must not leak into the main
# shell — the rest of the update keeps running from the caller's cwd.
local slot_db="$work_dir/backend/instance/serverkit.db"
local use_slot_db=0
if grep -qE '^DATABASE_URL=sqlite' "$work_dir/.env" 2>/dev/null && [ -f "$slot_db" ]; then
use_slot_db=1
# Fail fast with an actionable message when the SQLite slot copy is
# corrupt, instead of dying deep inside app boot with "database disk
# image is malformed". The sqlite3 CLI isn't guaranteed to be
# installed — probe via the venv's Python instead. An EMPTY result
# means the probe itself couldn't run (no usable venv python) — that
# is not proof of corruption, so warn and let `flask db upgrade`
# surface any real problem rather than halting on a false positive.
local integrity
integrity="$("$venv/bin/python" -c \
"import sqlite3,sys;print(sqlite3.connect(sys.argv[1]).execute('PRAGMA integrity_check').fetchone()[0])" \
"$slot_db" 2>/dev/null || true)"
if [ -n "$integrity" ] && [ "$integrity" != "ok" ]; then
halt "SQLite database is corrupt ($integrity). Repair or restore it from $BACKUP_DIR and re-run the update. The previous installation is still active."
elif [ -z "$integrity" ]; then
warn "Could not pre-check SQLite integrity (venv python unavailable) — proceeding"
fi
fi
if ! (
# shellcheck source=/dev/null
source "$venv/bin/activate"
cd "$work_dir/backend"
# SERVERKIT_SKIP_BACKGROUND=1: create_app() must not start queue
# consumers / schedulers / collectors while we only want migrations —
# they query the DB before `flask db upgrade` runs, and any error
# there aborts the migration (belt & braces; the app also detects the
# flask CLI on its own).
if [ "$use_slot_db" = "1" ]; then
DATABASE_URL="sqlite:///$slot_db" FLASK_ENV=production SERVERKIT_SKIP_BACKGROUND=1 flask db upgrade
else
# Non-SQLite (e.g. PostgreSQL): the DB is shared/external, so there
# is no per-slot copy to isolate — migrate it directly.
FLASK_ENV=production SERVERKIT_SKIP_BACKGROUND=1 flask db upgrade
fi
); then
# PostgreSQL (and other external DBs): the migration ran IN PLACE
# against the SHARED database — there is no untouched slot copy to
# fall back on. Put the database back the way we found it using the
# verified pre-upgrade dump before reporting the failure.
if [ "$use_slot_db" != "1" ] && [ -n "${PRE_UPGRADE_DUMP:-}" ] && [ -f "${PRE_UPGRADE_DUMP:-}" ]; then
warn "Migration failed — restoring the pre-upgrade dump into the shared database..."
local pg_url
pg_url="$(pg_url_from_env "$work_dir")"
if command -v pg_restore >/dev/null 2>&1 \
&& pg_restore --clean --if-exists -d "$pg_url" "$PRE_UPGRADE_DUMP"; then
halt "Database migration failed; the pre-upgrade dump was restored, so the database is back to its pre-update state. The previous installation is still active."
fi
halt "Database migration failed AND the automatic restore failed. Restore MANUALLY with: pg_restore --clean --if-exists -d '$pg_url' '$PRE_UPGRADE_DUMP' — the previous installation is still active."
fi
halt "Database migration failed. The previous installation is still active."
fi
good "Database migrated"
}
# ---------------------------------------------------------------------------
# Release download + checksum verification
# ---------------------------------------------------------------------------
# Prints exactly ONE line on stdout: the path to the verified tarball. The
# caller captures that stdout (tarball="$(download_release ...)"), so every
# bit of progress in here is routed to stderr — a step/good line on stdout
# would be captured into the "path" and break the tar that follows. stderr
# still reaches the terminal and the log (init_logging merges 2>&1 into tee).
download_release() {
local version="$1"
local arch output tmp_dir base_url checksum_url tarball_url
case "$(uname -m)" in
x86_64|amd64) arch="amd64" ;;
aarch64|arm64) arch="arm64" ;;
*) halt "Unsupported architecture: $(uname -m)" ;;
esac
if [ -n "$SERVERKIT_OFFLINE_TARBALL" ]; then
[ -f "$SERVERKIT_OFFLINE_TARBALL" ] || halt "Offline tarball not found: $SERVERKIT_OFFLINE_TARBALL"
echo "$SERVERKIT_OFFLINE_TARBALL"
return 0
fi
if [ -n "$SERVERKIT_MIRROR_URL" ]; then
base_url="$SERVERKIT_MIRROR_URL"
else
base_url="https://github.com/${GITHUB_REPO}/releases/download/${version}"
fi
tarball_url="${base_url}/serverkit-${version}-linux-${arch}.tar.gz"
checksum_url="${base_url}/checksums.txt"
tmp_dir="$(mktemp -d)"
output="$tmp_dir/serverkit-${version}-linux-${arch}.tar.gz"
# This runs in a command substitution, so the caller never learns tmp_dir
# on failure: remove it here before halting, or every failed download
# strands a directory in /tmp.
step "Downloading release tarball (${arch})..." >&2
if ! curl -sfL "$tarball_url" -o "$output"; then
rm -rf "$tmp_dir"
halt "Failed to download $tarball_url"
fi
step "Verifying checksum..." >&2
if curl -sfL "$checksum_url" -o "$tmp_dir/checksums.txt"; then
cd "$tmp_dir"
if ! sha256sum -c <(grep "serverkit-${version}-linux-${arch}.tar.gz" checksums.txt) >/dev/null 2>&1; then
cd /
rm -rf "$tmp_dir"
halt "Checksum verification failed for release tarball."
fi
good "Checksum verified" >&2
elif [ "${SERVERKIT_ALLOW_UNVERIFIED:-0}" = "1" ]; then
warn "No checksums.txt — installing UNVERIFIED (SERVERKIT_ALLOW_UNVERIFIED=1)" >&2
else
# Every release publishes checksums.txt; its absence means a broken
# mirror or a tampered download, and this replaces the running panel.
rm -rf "$tmp_dir"
halt "Could not download checksums.txt from ${base_url} — refusing an unverified release. Set SERVERKIT_ALLOW_UNVERIFIED=1 to override."
fi
echo "$output"
}
# Remove release downloads that earlier updates left in the temp dir (before
# the download dir was cleaned up, every update stranded ~70 MB there). Only
# directories holding a release tarball and untouched for an hour are taken,
# so a concurrent download is never pulled out from under itself.
sweep_stale_downloads() {
local tarball
while IFS= read -r tarball; do
[ -n "$tarball" ] || continue
rm -rf "$(dirname "$tarball")" 2>/dev/null || true
done < <(find "${TMPDIR:-/tmp}" -mindepth 2 -maxdepth 2 -path '*/tmp.*/serverkit-*-linux-*.tar.gz' \
-mmin +60 2>/dev/null || true)
return 0
}
# ---------------------------------------------------------------------------
# Blue/green directory management
# ---------------------------------------------------------------------------
ensure_bluegreen_layout() {
# Convert legacy single-directory installs into the blue/green symlink layout.
if [ -d "$INSTALL_DIR" ] && [ ! -L "$INSTALL_DIR" ]; then
step "Migrating to blue/green layout..."
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would move $INSTALL_DIR → $DIR_A and symlink $INSTALL_DIR → $DIR_A"
return 0
fi
mv "$INSTALL_DIR" "$DIR_A"
ln -s "$DIR_A" "$INSTALL_DIR"
good "Migrated to blue/green layout"
fi
# Ensure both slots exist.
if [ "$DRY_RUN" = "0" ]; then
mkdir -p "$DIR_A" "$DIR_B"
fi
}
atomic_switch() {
local target="$1"
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would switch symlink $INSTALL_DIR → $target"
return 0
fi
ln -sfn "$target" "${INSTALL_DIR}.tmp"
mv -Tf "${INSTALL_DIR}.tmp" "$INSTALL_DIR"
good "Switched active install to $target"
}
# ---------------------------------------------------------------------------
# Consistent SQLite copy. A plain `cp` of the LIVE panel database races the
# backend's own writes: the copy can capture half a transaction and come out
# "database disk image is malformed" — which then sinks the migration in the
# new slot, and silently poisons the pre-upgrade backup that was supposed to
# be the safety net. The SQLite online backup API snapshots a consistent DB
# even while the panel is writing, using any interpreter that can import
# sqlite3; with none available we fail rather than fall back to a racy cp.
# ---------------------------------------------------------------------------
# Snapshotting or integrity-checking a SQLite file needs nothing more than an
# interpreter carrying the stdlib `sqlite3` module — NOT the gated 3.11-3.14
# that rebuild_virtualenv and preflight_check legitimately demand. Reusing
# locate_python here silently coupled the DATA-safety paths to the venv-builder
# constraint, and blocked updates for a reason unrelated to the data:
# * an all-Docker install returns before preflight_check ever runs and does
# all its work in the container, yet backup_docker_db verified the extracted
# DB on the HOST — so a stock EL9 (python3.9) or Ubuntu 22.04 (python3.10)
# host halted with "Could not obtain a VALID database backup", blaming the
# backup for a missing interpreter.
# * backup_current halts when it cannot snapshot, so the same boxes could
# never update at all.
# Prefer the install's own venv interpreter: on a real box it is guaranteed
# present, guaranteed to have the module, and independent of what is on PATH.
locate_sqlite_python() {
local c
for c in "$@" python3.14 python3.13 python3.12 python3.11 python3 python; do
[ -n "$c" ] || continue
command -v "$c" >/dev/null 2>&1 || continue
"$c" -c 'import sqlite3' >/dev/null 2>&1 || continue
printf '%s' "$c"
return 0
done
return 1
}
copy_sqlite_db() {
local src="$1" dest="$2" py slot
# The DB lives at <slot>/backend/instance/serverkit.db — derive that slot's
# interpreter as the preferred candidate. A src outside a slot leaves the
# path unchanged, yielding a non-executable candidate that is simply skipped.
slot="${src%/backend/instance/*}"
py="$(locate_sqlite_python "$slot/venv/bin/python")" \
|| { warn "No Python with the sqlite3 module available for a consistent DB snapshot"; return 1; }
"$py" - "$src" "$dest" <<'PYEOF'
import sqlite3, sys
src, dest = sys.argv[1], sys.argv[2]
s = sqlite3.connect(src)
d = sqlite3.connect(dest)
s.backup(d)
d.close()
s.close()
PYEOF
}
# A backup that was never verified is a hope, not a safety net. Verify every
# artifact BEFORE it is allowed to justify proceeding with the update.
verify_sqlite_db() {
local db="$1" py active hint=""
[ -s "$db" ] || return 1
# Backups live in BACKUP_DIR, outside any slot, so hint at the active
# install's interpreter instead of deriving one from the file's path.
active="$(active_real_dir 2>/dev/null || true)"
if [ -n "$active" ]; then
hint="$active/venv/bin/python"
fi
py="$(locate_sqlite_python "$hint")" || return 1
[ "$("$py" -c "import sqlite3,sys;print(sqlite3.connect(sys.argv[1]).execute('PRAGMA integrity_check').fetchone()[0])" "$db" 2>/dev/null || true)" = "ok" ]
}
verify_pg_dump() {
local dump="$1"
[ -s "$dump" ] || return 1
# Structural validation when pg_restore is available: list the dump's TOC.
if command -v pg_restore >/dev/null 2>&1; then
pg_restore -l "$dump" >/dev/null 2>&1 || return 1
fi
return 0
}
# Extract the DATABASE_URL from a slot's .env, normalized for libpq: strip
# SQLAlchemy driver suffixes (postgresql+psycopg2://) and wrapping quotes.
pg_url_from_env() {
grep -E '^DATABASE_URL=' "$1/.env" 2>/dev/null | head -1 | cut -d= -f2- \
| tr -d "\"'" | sed -E 's|^postgres(ql)?\+[a-z0-9]+://|postgresql://|'
}
# ---------------------------------------------------------------------------
# Backup
# ---------------------------------------------------------------------------
snapshot_install_tree() {
local source_dir="$1" target_dir="$2"
rsync -a --exclude=venv --exclude=backups --exclude=node_modules \
--exclude='backend/instance/*.db' \
--exclude='backend/instance/*.db-wal' \
--exclude='backend/instance/*.db-shm' \
"$source_dir/" "$target_dir/" 2>/dev/null && return 0
# Preserve the exclusions even on hosts without rsync; never stage a
# second database copy just to delete it after copying the tree.
mkdir -p "$target_dir" || return 1
(set -o pipefail
tar -C "$source_dir" --exclude=venv --exclude=backups --exclude=node_modules \
--exclude='./backend/instance/*.db' \
--exclude='./backend/instance/*.db-wal' \
--exclude='./backend/instance/*.db-shm' -cf - . \
| tar -C "$target_dir" -xpf -
)
}
backup_current() {
phase "Database Backup"
mkdir -p "$BACKUP_DIR"
# Trim BEFORE writing, not only in cleanup().
#
# cleanup() is the last phase of the update, so it never ran on a run that
# died earlier — including the run that died BECAUSE the disk was full.
# Retention that only executes on success cannot bound a failure loop: a
# box was found carrying six snapshots under a cap of five for exactly
# this reason. Trimming first also means peak usage is `keep` snapshots
# rather than `keep + 1`.
if [ "$DRY_RUN" != "1" ]; then
trim_backups
enforce_backup_budget
fi
local active db_file backup_file
active="$(active_real_dir)"
db_file="$active/backend/instance/serverkit.db"
# Branch on the DECLARED engine, not on which files happen to exist. An
# install migrated SQLite -> PostgreSQL keeps its old serverkit.db on disk
# (both deploy paths copy it forward on file-existence alone, so it is
# carried into every future slot), and a file-first test then backed up
# that abandoned file, skipped the dump, and left PRE_UPGRADE_DUMP unset —
# silently disabling the auto-restore in migrate_database, which reads it
# as "${PRE_UPGRADE_DUMP:-}" and just does nothing. PostgreSQL is the ONE
# install type that migrates in place and cannot roll back by switching
# slots, so that dump is its only way home. migrate_database has always
# keyed off DATABASE_URL; this keeps the two functions in agreement.
if grep -qE '^DATABASE_URL=postgres' "$active/.env" 2>/dev/null; then
# PostgreSQL: the DB is external and SHARED between slots — there is
# no per-slot file to snapshot, and the migration runs in place. A
# pre-upgrade dump is the only safety net AND the only way back:
# without it, a rollback hands the old code a newer schema.
backup_file="$BACKUP_DIR/serverkit-pre-upgrade-$(date +%Y%m%d-%H%M%S).dump"
command -v pg_dump >/dev/null 2>&1 \
|| halt "PostgreSQL install but pg_dump is unavailable — install postgresql-client so the update has a safety net"
local pg_url
pg_url="$(pg_url_from_env "$active")"
if ! run_or_dry pg_dump "$pg_url" -Fc -f "$backup_file"; then
rm -f "$backup_file"
halt "PostgreSQL backup failed — refusing to update without a safety net"
fi
if [ "$DRY_RUN" != "1" ] && ! verify_pg_dump "$backup_file"; then
rm -f "$backup_file"
halt "PostgreSQL backup verification failed — refusing to update without a VALID safety net"
fi
# Remember the dump path so migrate_database can offer/perform an
# automatic restore if the in-place migration fails.
PRE_UPGRADE_DUMP="$backup_file"
good "Database backed up to $backup_file"
elif [ -f "$db_file" ]; then
backup_file="$BACKUP_DIR/serverkit-pre-upgrade-$(date +%Y%m%d-%H%M%S).db"
# A torn backup is worse than none: it looks like a safety net but
# restores a malformed DB. Snapshot consistently, VERIFY the result,
# and refuse to proceed without a proven-valid net. Failed or invalid
# artifacts are removed so they are never mistaken for a good backup.
if ! run_or_dry copy_sqlite_db "$db_file" "$backup_file"; then
rm -f "$backup_file"
halt "Database backup failed — refusing to update without a safety net"
fi
if [ "$DRY_RUN" != "1" ] && ! verify_sqlite_db "$backup_file"; then
rm -f "$backup_file"
halt "Database backup verification failed — refusing to update without a VALID safety net"
fi
good "Database backed up to $backup_file"
else
# Neither engine matched. Say which one, and say plainly that the
# update is proceeding unprotected — "No SQLite database" reads like a
# benign nothing-to-do on, say, a MySQL install that in fact has no
# safety net at all.
local db_scheme
db_scheme="$(grep -E '^DATABASE_URL=' "$active/.env" 2>/dev/null | head -1 \
| cut -d= -f2- | tr -d "\"'" | cut -d: -f1)"
if [ -n "$db_scheme" ] && [ "$db_scheme" != "sqlite" ]; then
warn "DATABASE_URL uses '$db_scheme', which this updater cannot dump — proceeding with NO database backup"
else
warn "No SQLite database at $db_file — skipping DB backup"
fi
fi
local tree_backup
tree_backup="$BACKUP_DIR/serverkit-tree-$(date +%Y%m%d-%H%M%S)"
if [ -L "$INSTALL_DIR" ] && [ -d "$active" ]; then
# Blue/green: the update installs into the OTHER slot, so this one
# stays on disk as the rollback. A tree copy of it was a second full
# copy of the same code (~1 GB per update on a small VPS).
info "Install tree kept in place as the rollback slot: $active"
elif [ -d "$active" ]; then
# The SQLite database is NOT part of the tree snapshot: it was just
# written as the pre-upgrade copy above, and rollback restores from
# that copy, never from the tree. Copying it here too doubled every
# update's backup footprint (a 450 MB database became 900 MB per run),
# which is how small VPSes kept filling their disk with backups.
if run_or_dry snapshot_install_tree "$active" "$tree_backup"; then
good "Install tree backed up to $tree_backup"
else
warn "Install tree backup failed: $tree_backup may be incomplete"
fi
fi
}
# ---------------------------------------------------------------------------
# Carry user-installed plugins across a redeploy (#48).
#
# URL/registry/upload-installed extensions live as extracted dirs under
# backend/app/plugins/<slug> and frontend/src/plugins/<slug>. A fresh
# clone/tarball only contains the repo-tracked ones, so anything the user
# installed would silently vanish — copy forward every plugin dir the new
# tree doesn't already ship. Runs BEFORE the frontend build so carried
# frontends compile into the new bundle. The backend also has a boot-time
# repair pass as a backstop (plugin_service.repair_missing_plugins).
# ---------------------------------------------------------------------------
# Plugins deliberately deleted from the product (replaced or retired) — e.g.
# serverkit-workflows, superseded by serverkit-tramo (plan 45). Never carry
# these forward: their frontends import dependencies the new tree no longer
# ships (workflows → reactflow), which sinks the whole bundle build.
# Space-separated slugs; append here whenever an extension is retired.
# Mirror of extension_migration.RETIRED_EXTENSION_SLUGS in the backend.
RETIRED_PLUGIN_SLUGS="serverkit-workflows"
plugin_is_retired() {
local slug
for slug in $RETIRED_PLUGIN_SLUGS; do
[ "$1" = "$slug" ] && return 0
done
return 1
}
# Strip retired plugin leftovers from a tree (docker in-place mode, where
# untracked plugin dirs survive the hard reset and would otherwise sink the
# bundle rebuild).
remove_retired_plugins() {
local tree="$1" slug sub
for slug in $RETIRED_PLUGIN_SLUGS; do
for sub in backend/app/plugins frontend/src/plugins; do
if [ -d "$tree/$sub/$slug" ]; then
rm -rf "$tree/$sub/$slug" 2>/dev/null || true
info "Removed retired plugin: $sub/$slug"
fi
done
done
return 0
}
# Frontend plugin dirs copied forward by preserve_installed_plugins this run.
# If the bundle build then fails these are the prime suspects — see
# quarantine_carried_plugins.
CARRIED_FRONTEND_PLUGINS=""
preserve_installed_plugins() {
local src="$1" target="$2"
local sub dir name
for sub in backend/app/plugins frontend/src/plugins; do
[ -d "$src/$sub" ] || continue
for dir in "$src/$sub"/*/; do
[ -d "$dir" ] || continue
name="$(basename "$dir")"
[ "$name" = "__pycache__" ] && continue
if plugin_is_retired "$name"; then
info "Not carrying forward retired plugin: $sub/$name"
continue
fi
if [ ! -e "$target/$sub/$name" ]; then
mkdir -p "$target/$sub" 2>/dev/null || true
if cp -a "$dir" "$target/$sub/$name" 2>/dev/null; then
info "Preserved installed plugin: $sub/$name"
if [ "$sub" = "frontend/src/plugins" ]; then
CARRIED_FRONTEND_PLUGINS="$CARRIED_FRONTEND_PLUGINS $name"
fi
else
warn "Could not preserve plugin $sub/$name"
fi
fi
done
done
return 0
}
# Vite 8 / rolldown (the frontend bundler) require Node ^20.19.0 || >=22.12.0.
# Returns 1 (no output) on an absent/too-old Node so callers can warn+rollback or
# halt with an actionable message, instead of npm silently skipping rolldown's
# native binary and `npm run build` dying with a cryptic MODULE_NOT_FOUND.
_node_build_ok() {
command -v node >/dev/null 2>&1 || return 1
local v M m
v="$(node --version 2>/dev/null | sed -E 's/^v//')"
M="${v%%.*}"; m="${v#*.}"; m="${m%%.*}"
case "$M" in ''|*[!0-9]*) return 1;; esac
case "$m" in ''|*[!0-9]*) return 1;; esac
if [ "$M" -eq 20 ] && [ "$m" -ge 19 ]; then return 0; fi
if [ "$M" -eq 22 ] && [ "$m" -ge 12 ]; then return 0; fi
if [ "$M" -ge 23 ]; then return 0; fi
return 1
}
# Build the SPA bundle in a target tree. Pass "reuse" as $2 to skip npm ci
# (node_modules already intact from a just-failed build). Subshell so the cd
# cannot leak into the main shell.
build_frontend_bundle() {
if ! _node_build_ok; then
warn "Node.js $(node --version 2>/dev/null || echo 'not found') is too old to build the frontend — vite 8 needs 20.19+ or 22.12+. Upgrade Node (e.g. NodeSource 22 LTS) and re-run the update."
return 1
fi
( cd "$1/frontend" && \
{ [ "${2:-}" = "reuse" ] || npm ci --prefer-offline 2>&1 | tail -3; } && \
NODE_OPTIONS="--max-old-space-size=1024" npm run build 2>&1 | tail -5 )
}
# Re-apply this install's PERSISTENT favicon tint to the staged dist. The color
# is assigned once (on first install/update) and stored in /etc/serverkit so the
# mark keeps the same per-install color across updates instead of repainting —
# and an install that predates the feature picks one up on its first update.
# Mirror of install.sh randomize_favicon (kept standalone; the two installers
# already keep separate build fns). Also serves it at /favicon.ico so a blind
# favicon-hash fetch varies too.
apply_favicon_tint() {
local root="$1"
local fav="$root/frontend/dist/favicon.svg"
[ -f "$fav" ] || return 0
local color_file="/etc/serverkit/favicon-color"
local hsl=""
[ -s "$color_file" ] && hsl="$(cat "$color_file" 2>/dev/null)"
if [ -z "$hsl" ]; then
local hues=(210 225 235 250 265 285 320 345 12 25 160 175 190)
local base="${hues[$RANDOM % ${#hues[@]}]}"
local jitter=$(( RANDOM % 21 - 10 ))
local h=$(( (base + jitter + 360) % 360 ))
local s=$(( 48 + RANDOM % 28 ))
local l=$(( 32 + RANDOM % 15 ))
hsl="hsl(${h}, ${s}%, ${l}%)"
mkdir -p /etc/serverkit 2>/dev/null || true
printf '%s\n' "$hsl" > "$color_file" 2>/dev/null || true
fi
sed -i -E "s|(<rect width=\"32\" height=\"32\" rx=\"7\" fill=\")[^\"]+(\")|\1${hsl}\2|" "$fav" 2>/dev/null || true
cp "$fav" "$root/frontend/dist/favicon.ico" 2>/dev/null || true
}
# A plugin carried forward from the old install can be incompatible with the
# new tree (imports that no longer resolve) and sink the whole bundle build.
# Rather than blocking the update, move the carried frontends aside so the
# caller can retry — fail the plugin, never the update. Returns 1 when there
# was nothing to quarantine (the failure is not plugin-borne).
QUARANTINED_PLUGINS=""
quarantine_carried_plugins() {
local target="$1" qdir name moved=""
[ -n "$CARRIED_FRONTEND_PLUGINS" ] || return 1
qdir="$BACKUP_DIR/quarantined-plugins-$(date +%Y%m%d-%H%M%S)"
mkdir -p "$qdir" 2>/dev/null || return 1
for name in $CARRIED_FRONTEND_PLUGINS; do
[ -d "$target/frontend/src/plugins/$name" ] || continue
if mv "$target/frontend/src/plugins/$name" "$qdir/" 2>/dev/null; then
moved="$moved $name"
fi
done
CARRIED_FRONTEND_PLUGINS=""
[ -n "$moved" ] || { rmdir "$qdir" 2>/dev/null || true; return 1; }
QUARANTINED_PLUGINS="${moved# }"
warn "Carried-forward plugin(s) quarantined after build failure: $QUARANTINED_PLUGINS"
warn "Copies kept in $qdir — reinstall compatible versions from the Marketplace"
return 0
}
# ---------------------------------------------------------------------------
# Deploy source into the next directory
# ---------------------------------------------------------------------------
deploy_source() {
local target="$1"
local branch="${2:-main}"
phase "Updating Source"
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would clone/pull $GITHUB_REPO:$branch into $target"
return 0
fi
# Preserve .env and database across rewrites. The DB snapshot must be
# consistent (online backup API) — a racy cp of the live DB can come out
# malformed and sink the migration in the new slot.
local tmp_env tmp_db live_db
tmp_env="$(mktemp)"
tmp_db="$(mktemp)"
cp "$INSTALL_DIR/.env" "$tmp_env" 2>/dev/null || true
live_db="$INSTALL_DIR/backend/instance/serverkit.db"
if [ -f "$live_db" ]; then
copy_sqlite_db "$live_db" "$tmp_db" \
|| halt "Could not snapshot the live database consistently — aborting before any switch"
fi
rm -rf "$target"
git clone --depth 1 --branch "$branch" "https://github.com/${GITHUB_REPO}.git" "$target" \
|| halt "Failed to clone ${GITHUB_REPO}:$branch"
cp "$tmp_env" "$target/.env" 2>/dev/null || true
if [ -s "$tmp_db" ]; then
mkdir -p "$target/backend/instance"
cp "$tmp_db" "$target/backend/instance/serverkit.db" 2>/dev/null || true
fi
rm -f "$tmp_env" "$tmp_db"
preserve_installed_plugins "$INSTALL_DIR" "$target"
chmod +x "$target/serverkit"
chmod +x "$target/scripts/"*.sh 2>/dev/null || true
good "Source updated to $branch in $target"
}
# ---------------------------------------------------------------------------
# Deploy release tarball into the next directory
# ---------------------------------------------------------------------------
deploy_release() {
local target="$1"
local version="$2"
phase "Downloading Release"
local tarball stage unpacked
sweep_stale_downloads
tarball="$(download_release "$version")"
# The download dir is ours to remove; an operator's offline tarball is not.
# Both scratch dirs are recorded globally so cleanup_on_exit removes them
# when a halt below aborts the update.
if [ "$tarball" != "${SERVERKIT_OFFLINE_TARBALL:-}" ]; then
RELEASE_DOWNLOAD_DIR="$(dirname "$tarball")"
fi
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would unpack $tarball into $target"
remove_release_scratch
return 0
fi
stage="$(mktemp -d)"
RELEASE_STAGE_DIR="$stage"
tar xzf "$tarball" -C "$stage"
unpacked="$stage/serverkit"
[ ! -d "$unpacked" ] && unpacked="$stage/opt/serverkit"
if [ ! -d "$unpacked" ]; then
# `|| true`: head exiting first can SIGPIPE find (rc 141 under
# pipefail); an empty result is handled by the halt just below.
unpacked="$(find "$stage" -maxdepth 2 -type d -name serverkit | head -n1 || true)"
fi
[ -d "$unpacked" ] || halt "Release tarball layout is unrecognized"
# Preserve live state. Same consistent-snapshot rule as deploy_source:
# never plain-cp the live DB into the new slot.
cp "$INSTALL_DIR/.env" "$unpacked/.env" 2>/dev/null || true
local live_db="$INSTALL_DIR/backend/instance/serverkit.db"
if [ -f "$live_db" ]; then
mkdir -p "$unpacked/backend/instance"
copy_sqlite_db "$live_db" "$unpacked/backend/instance/serverkit.db" \
|| halt "Could not snapshot the live database consistently — aborting before any switch"
fi
preserve_installed_plugins "$INSTALL_DIR" "$unpacked"
rm -rf "$target"
cp -a "$unpacked" "$target"
remove_release_scratch
chmod +x "$target/serverkit"
chmod +x "$target/scripts/"*.sh 2>/dev/null || true
good "Release $version deployed to $target"
}
# ---------------------------------------------------------------------------
# Configuration refresh
# ---------------------------------------------------------------------------
refresh_config() {
local target="$1"
phase "Refreshing Configuration"
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would refresh nginx + systemd configs from $target"
return 0
fi
mkdir -p "$NGINX_DIR/sites-available" "$NGINX_DIR/sites-enabled"
# Recover panel domain from live nginx config. The trailing `|| true` keeps a
# no-match/missing-file grep (exit 1/2) from tripping `set -euo pipefail` and
# silently killing the updater before the atomic switch — HTTP-only boxes have
# no serverkit.conf, so this grep finds nothing.
local prior_panel_domain=""
prior_panel_domain=$(grep -oE "$LETSENCRYPT_DIR/live/[^/]+/" \
"$NGINX_DIR/sites-available/serverkit.conf" 2>/dev/null | head -n1 | \
sed -E 's|.*/live/([^/]+)/|\1|' || true)
[ "$prior_panel_domain" = "YOUR_DOMAIN" ] && prior_panel_domain=""
if [ -f "$target/nginx/sites-available/serverkit.conf" ]; then
cp "$target/nginx/sites-available/serverkit.conf" "$NGINX_DIR/sites-available/"
fi
if [ -f "$target/nginx/sites-available/serverkit-insecure.conf" ]; then
cp "$target/nginx/sites-available/serverkit-insecure.conf" "$NGINX_DIR/sites-available/"
fi
# The panel frontend is served statically by host nginx from
# $INSTALL_DIR/frontend/dist (the /opt/serverkit symlink). The shipped config
# roots at the default /opt/serverkit; re-point it when SERVERKIT_DIR differs,
# exactly as install.sh does, so a custom install dir survives upgrades.
if [ "$INSTALL_DIR" != "/opt/serverkit" ]; then
local conf
for conf in serverkit.conf serverkit-insecure.conf; do
[ -f "$NGINX_DIR/sites-available/$conf" ] && \
sed -i "s|/opt/serverkit/frontend/dist|$INSTALL_DIR/frontend/dist|g" \
"$NGINX_DIR/sites-available/$conf"
done
fi
# TLS floor — prefer a reversible conf.d snippet when nginx.conf doesn't
# already declare these (a second declaration in the same http{} context is a
# "duplicate ssl_protocols" error); otherwise rewrite in place. Mirrors
# install.sh's harden_global_tls.
if [ -f "$NGINX_DIR/nginx.conf" ]; then
local ciphers='ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384'
local has_proto=0 has_ciphers=0
grep -qE '^[[:space:]]*ssl_protocols[[:space:]]' "$NGINX_DIR/nginx.conf" && has_proto=1
grep -qE '^[[:space:]]*ssl_ciphers[[:space:]]' "$NGINX_DIR/nginx.conf" && has_ciphers=1
if [ "$has_proto" = "0" ] && [ "$has_ciphers" = "0" ] && \
grep -qE 'include[[:space:]]+/etc/nginx/conf\.d/\*\.conf' "$NGINX_DIR/nginx.conf"; then
mkdir -p "$NGINX_DIR/conf.d"
cat > "$NGINX_DIR/conf.d/serverkit-tls.conf" <<EOF
# Auto-generated by ServerKit — server-wide TLS floor. Safe to remove.
ssl_protocols TLSv1.2 TLSv1.3;
ssl_ciphers ${ciphers};
EOF
else
rm -f "$NGINX_DIR/conf.d/serverkit-tls.conf" 2>/dev/null || true
if [ "$has_proto" = "1" ]; then
sed -i -E 's|^([[:space:]]*)ssl_protocols[[:space:]].*|\1ssl_protocols TLSv1.2 TLSv1.3;|' "$NGINX_DIR/nginx.conf"
else
sed -i '/http {/a \ ssl_protocols TLSv1.2 TLSv1.3;' "$NGINX_DIR/nginx.conf"
fi
if [ "$has_ciphers" = "1" ]; then
sed -i -E "s|^([[:space:]]*)ssl_ciphers[[:space:]].*|\1ssl_ciphers ${ciphers};|" "$NGINX_DIR/nginx.conf"
else
sed -i "/http {/a \\ ssl_ciphers ${ciphers};" "$NGINX_DIR/nginx.conf"
fi
fi
fi
# SSL mode
local ssl_mode="insecure"
if [ -f "$CONFIG_DIR/ssl-mode" ]; then
ssl_mode="$(cat "$CONFIG_DIR/ssl-mode")"
fi
if [ "$ssl_mode" = "secure" ] && [ -f "$NGINX_DIR/sites-available/serverkit.conf" ]; then
local panel_domain=""
if [ -f "$CONFIG_DIR/panel-domain" ]; then
panel_domain="$(cat "$CONFIG_DIR/panel-domain" 2>/dev/null || true)"
fi
[ -z "$panel_domain" ] && panel_domain="$prior_panel_domain"
if [ -n "$panel_domain" ] && [ -d "$LETSENCRYPT_DIR/live/$panel_domain" ]; then
sed -i "s|/etc/letsencrypt/live/YOUR_DOMAIN/|$LETSENCRYPT_DIR/live/$panel_domain/|g" \
"$NGINX_DIR/sites-available/serverkit.conf"
ln -sf "$NGINX_DIR/sites-available/serverkit.conf" "$NGINX_DIR/sites-enabled/serverkit.conf"
else
warn "SSL mode is 'secure' but no certificate found for '${panel_domain:-unknown}'"
ln -sf "$NGINX_DIR/sites-available/serverkit-insecure.conf" "$NGINX_DIR/sites-enabled/serverkit.conf"
fi
else
ln -sf "$NGINX_DIR/sites-available/serverkit-insecure.conf" "$NGINX_DIR/sites-enabled/serverkit.conf"
fi
# Service unit — render from the template so a custom SERVERKIT_DIR / venv /
# log path survives upgrades (the unit references the /opt/serverkit symlink,
# so blue/green switches need no re-render). Fall back to a plain unit if an
# older tree still ships one.
local svc_template="$target/templates/serverkit-backend.service.in"
# Bind the API to loopback by default — host nginx fronts it on :80/:443, so
# the raw gunicorn port must not be world-reachable. Mirror install.sh so an
# update re-renders @BIND_HOST@ instead of leaving the literal placeholder
# (which would make gunicorn fail to bind). Override with SERVERKIT_BIND_HOST.
local bind_host="${SERVERKIT_BIND_HOST:-127.0.0.1}"
if [ -f "$svc_template" ]; then
sed -e "s|@SERVERKIT_DIR@|$INSTALL_DIR|g" \
-e "s|@SERVERKIT_VENV_DIR@|$VENV_DIR|g" \
-e "s|@PORT@|5000|g" \
-e "s|@BIND_HOST@|$bind_host|g" \
-e "s|@USER@|root|g" \
-e "s|@LOG_DIR@|$LOG_DIR|g" \
"$svc_template" > "$SYSTEMD_DIR/serverkit.service"
elif [ -f "$target/serverkit-backend.service" ]; then
cp "$target/serverkit-backend.service" "$SYSTEMD_DIR/serverkit.service"
fi
systemctl daemon-reload 2>/dev/null \
|| warn "systemd daemon-reload failed — the backend may restart with a stale unit"
report_stale_panel_vhosts
good "Configuration refreshed"
}
# The panel used to be served by a frontend container on :3847; it is now
# served statically by host nginx. A leftover custom vhost proxying to :3847
# keeps serving whatever bundle the (never-updated) container holds — the
# panel looks "stuck on an old version" for that hostname only, which reads
# like a browser/CDN cache problem and wastes hours (2026-07-03 incident).
# Observation only (R1): warn and continue, tolerate a missing/empty dir.
report_stale_panel_vhosts() {
local hits=""
hits="$(grep -rls 'proxy_pass http://127\.0\.0\.1:3847' \
"$NGINX_DIR/sites-enabled/" 2>/dev/null | sort -u | tr '\n' ' ' || true)"
if [ -n "$hits" ]; then
warn "Vhost(s) still proxy the panel to the retired :3847 frontend container: ${hits}"
warn " These serve a STALE panel bundle. Repoint them at the static SPA"
warn " (root $INSTALL_DIR/frontend/dist + /api and /socket.io → :5000),"
warn " mirroring sites-available/serverkit.conf."
fi
}
# ---------------------------------------------------------------------------
# Rollback
# ---------------------------------------------------------------------------
# Probe the backend health endpoint; returns 0 if healthy within <timeout>s.
# Side-effect free (unlike health_check, which triggers a rollback) so it is
# safe to call *after* a rollback to confirm the restored version came back.
quick_health() {
local timeout="${1:-30}" waited=0
while [ "$waited" -lt "$timeout" ]; do
curl -sf --max-time 5 http://127.0.0.1:5000/api/v1/system/health >/dev/null 2>&1 && return 0
sleep 2
waited=$((waited + 2))
done
return 1
}
rollback() {
# Flag FIRST: halt() below exits the script, which fires the EXIT trap
# (cleanup_on_exit) — without this flag already set, that trap would run a
# second, redundant rollback on top of the one that just finished.
ROLLING_BACK=1
warn "Update failed — rolling back to previous slot..."
if [ -z "${PREVIOUS_DIR:-}" ] || [ ! -d "$PREVIOUS_DIR" ]; then
halt "Cannot roll back: previous installation directory not available"
fi
# Same zero-downtime discipline as the forward path: cycle only the backend,
# never stop nginx (it fronts every hosted app), and reload its config after
# the slot is switched back.
systemctl stop "$BACKEND_SERVICE" 2>/dev/null || true
wait_for_service "$BACKEND_SERVICE" inactive 30 || true
atomic_switch "$PREVIOUS_DIR"
# A rollback must never abort mid-flight over one failed step — everything
# from here on is best-effort so the backend still gets started.
systemctl daemon-reload 2>/dev/null \
|| warn "systemd daemon-reload failed during rollback — continuing"
systemctl start "$BACKEND_SERVICE" 2>/dev/null || true
wait_for_service "$BACKEND_SERVICE" active 30 || true
# Frontend is static (served from the restored slot); just re-label + reload.
selinux_label_dist "$PREVIOUS_DIR"
reload_nginx_graceful
# Confirm the restored version actually answers — a rollback that itself
# comes up unhealthy is a far worse state to leave the operator guessing in.
if quick_health 30; then
halt "Rolled back to $(active_real_dir) and it is healthy. Inspect logs: journalctl -u serverkit -n 50"
else
halt "Rolled back to $(active_real_dir) but it is STILL UNHEALTHY — manual intervention needed. Logs: journalctl -u serverkit -n 50"
fi
}
# Remove deploy_release's download and unpack dirs. Warn-and-continue: a
# scratch dir that will not delete must never fail an update or its rollback.
remove_release_scratch() {
if [ -n "${RELEASE_DOWNLOAD_DIR:-}" ]; then
rm -rf "$RELEASE_DOWNLOAD_DIR" 2>/dev/null || true
RELEASE_DOWNLOAD_DIR=""
fi
if [ -n "${RELEASE_STAGE_DIR:-}" ]; then
rm -rf "$RELEASE_STAGE_DIR" 2>/dev/null || true
RELEASE_STAGE_DIR=""
fi
return 0
}
# If the update fails after we have switched the symlink, roll back to the
# previous slot automatically. Registered as the EXIT trap by the run block
# below; defined here (above the source guard) so it stays unit-testable.
cleanup_on_exit() {
local rc=$?
remove_release_scratch
[ "$rc" -eq 0 ] && return 0
if [ "$DRY_RUN" = "0" ] && [ -n "${PREVIOUS_DIR:-}" ] && \
[ "${ROLLING_BACK:-0}" != "1" ] && [ "${HEALTH_PASSED:-0}" != "1" ]; then
ROLLING_BACK=1
rollback
fi
# Always return 0: the script's exit code is already $rc (an EXIT trap does
# not change it) — returning $rc here only re-fires the ERR trap, which
# appends a second, misleading "Update aborted" report.
return 0
}
# ---------------------------------------------------------------------------
# Health check
# ---------------------------------------------------------------------------
health_check() {
phase "Health Check"
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would probe http://127.0.0.1:5000/api/v1/system/health"
return 0
fi
step "Waiting for backend..."
local waited=0
while [ "$waited" -lt 60 ]; do
if curl -sf --max-time 5 http://127.0.0.1:5000/api/v1/system/health >/dev/null 2>&1; then
good "Backend healthy"
break
fi
sleep 2
waited=$((waited + 2))
done
if [ "$waited" -ge 60 ]; then
rollback
fi
if ! curl -sf --max-time 5 http://127.0.0.1:5000/api/v1/system/health >/dev/null 2>&1; then
rollback
fi
# No frontend probe: the panel frontend is static files served by host
# nginx from the active slot — there is no frontend container anymore.
HEALTH_PASSED=1
}
# ---------------------------------------------------------------------------
# Cleanup
# ---------------------------------------------------------------------------
# Trim a family of backups down to its newest <keep> entries. Deliberately
# find-based: the old `ls -t "$BACKUP_DIR"/<glob> | tail | xargs` aborted the
# whole updater under `set -Eeuo pipefail` whenever the glob matched nothing
# (ls exits 2) — i.e. on every box with no prior backups of that family, AFTER
# an otherwise successful update. find treats "nothing matched" as a clean
# empty result, and an absent BACKUP_DIR is an explicit no-op.
prune_old_backups() {
local pattern="$1" keep="$2"
[ -d "$BACKUP_DIR" ] || return 0
find "$BACKUP_DIR" -mindepth 1 -maxdepth 1 -name "$pattern" -printf '%T@\t%p\n' 2>/dev/null \
| sort -rn | tail -n +"$((keep + 1))" | cut -f2- \
| xargs -r -d '\n' rm -rf || true
}
# Bytes currently held by all backup artifacts.
backups_size_kb() {
[ -d "$BACKUP_DIR" ] || { echo 0; return 0; }
du -sk "$BACKUP_DIR" 2>/dev/null | awk '{print $1}' || echo 0
}
# How many upgrade snapshots to keep when SERVERKIT_BACKUP_RETENTION is unset:
# 1 on a filesystem under 50 GiB, else 3. Each snapshot is a full database
# copy, and on a 25 GB VPS three of them compete with the apps for the disk.
# An unreadable size keeps the old default.
default_backup_keep() {
local total_kb
total_kb="$(df -Pk "$BACKUP_DIR" 2>/dev/null | awk 'NR==2 {print $2}' || true)"
if [ -z "$total_kb" ]; then
total_kb="$(df -Pk / 2>/dev/null | awk 'NR==2 {print $2}' || true)"
fi
if [ -n "$total_kb" ] && [ "$total_kb" -lt $((50 * 1024 * 1024)) ] 2>/dev/null; then
echo 1
else
echo 3
fi
}
# Trim every backup kind to the retention cap. Safe to call repeatedly.
trim_backups() {
local keep="${1:-${SERVERKIT_BACKUP_RETENTION:-$(default_backup_keep)}}"
prune_old_backups 'serverkit-tree-*' "$keep"
prune_old_backups 'serverkit-pre-upgrade-*.db' "$keep"
prune_old_backups 'serverkit-pre-upgrade-*.dump' "$keep"
}
# Drop the OLDEST snapshots until the backup directory fits its size budget.
#
# A count cap alone is not a disk guarantee: five snapshots of an 800 MB
# database is 8 GB, which is a third of a 25 GB VPS. The budget is a share of
# the filesystem (default 15%), and at least one snapshot is always kept —
# an update with no way back is worse than a full disk.
enforce_backup_budget() {
[ -d "$BACKUP_DIR" ] || return 0
local pct="${SERVERKIT_BACKUP_MAX_PERCENT:-15}"
local total_kb budget_kb used_kb
total_kb="$(df -Pk "$BACKUP_DIR" | awk 'NR==2 {print $2}')"
[ -n "$total_kb" ] && [ "$total_kb" -gt 0 ] || return 0
budget_kb=$(( total_kb * pct / 100 ))
local guard=0
while [ "$guard" -lt 50 ]; do
used_kb="$(backups_size_kb)"
[ "$used_kb" -le "$budget_kb" ] && return 0
# Oldest stamp still on disk, across every artifact kind.
local oldest
oldest="$(find "$BACKUP_DIR" -mindepth 1 -maxdepth 1 \
\( -name 'serverkit-tree-*' -o -name 'serverkit-pre-upgrade-*' \) \
-printf '%T@\t%p\n' 2>/dev/null | sort -n | head -1 | cut -f2-)"
[ -n "$oldest" ] || return 0
# Never delete the last remaining snapshot set.
local remaining
remaining="$(find "$BACKUP_DIR" -mindepth 1 -maxdepth 1 \
-name 'serverkit-pre-upgrade-*' 2>/dev/null | wc -l)"
if [ "$remaining" -le 1 ] && [[ "$(basename "$oldest")" == serverkit-pre-upgrade-* ]]; then
warn "Backups exceed ${pct}% of the disk but only one restore point remains — keeping it"
return 0
fi
info "Backup budget exceeded — removing oldest backup $(basename "$oldest")"
rm -rf "$oldest" || return 0
guard=$((guard + 1))
done
}
cleanup() {
phase "Cleanup"
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would trim old backups"
return 0
fi
# Retention is capped (default 3, SERVERKIT_BACKUP_RETENTION overrides)
# AND budgeted by size: pre-upgrade DB backups are full-size copies, so
# five of an 800 MB database is 8 GB — a third of a 25 GB droplet. Seen in
# the wild twice: 6.3G of stale backups, then a disk filled to 0 bytes.
#
# backup_current already trimmed before writing; this is the second pass
# that also reclaims the snapshot this run just superseded.
trim_backups
enforce_backup_budget
local new_version
new_version="$(cat "$INSTALL_DIR/VERSION" 2>/dev/null | tr -d '\n\r ' || echo "unknown")"
curl -s "https://serverkit.ai/track/update?v=${new_version}" >/dev/null 2>&1 || true
good "Cleanup complete"
}
# ---------------------------------------------------------------------------
# Firewall — keep 80/443 open across updates.
# ---------------------------------------------------------------------------
# Idempotent and best-effort: a box installed before firewall automation existed
# gets its ports opened on the next update; an already-configured box is a no-op.
# Records what we opened in install-state.json so uninstall can undo it. Sources
# the helpers from the (now-current) install tree, which carries scripts/lib.
ensure_firewall() {
local lib="$INSTALL_DIR/scripts/lib"
[ -f "$lib/firewall.sh" ] || return 0
# shellcheck source=/dev/null
source "$lib/firewall.sh"
if [ -f "$lib/state.sh" ]; then
# shellcheck source=/dev/null
source "$lib/state.sh"
fi
local backend
backend="$(firewall_detect)"
if [ "$backend" = "none" ]; then
return 0
fi
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would ensure firewall ($backend) allows 80/tcp and 443/tcp"
return 0
fi
firewall_open "$backend" 80/tcp 443/tcp || true
if command -v state_set >/dev/null 2>&1; then
state_set firewall_backend "$backend" || true
state_append firewall_ports 80/tcp || true
state_append firewall_ports 443/tcp || true
fi
good "Firewall ensured ($backend): 80/tcp and 443/tcp open"
}
refresh_bash_completion() {
# Regenerate bash tab-completion from the switched-in CLI so new commands
# complete immediately after an update. Best-effort: boxes without the
# bash-completion package simply skip it.
local dir="${SERVERKIT_COMPLETION_DIR:-/etc/bash_completion.d}"
[ -d "$dir" ] || return 0
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would refresh $dir/serverkit"
return 0
fi
bash "$INSTALL_DIR/serverkit" completion > "$dir/serverkit" 2>/dev/null \
|| { rm -f "$dir/serverkit"; warn "Could not refresh bash completion (non-fatal)"; }
}
# ---------------------------------------------------------------------------
# All-Docker deployment (compose) detection + update path
# ---------------------------------------------------------------------------
# A ServerKit box runs in one of two shapes:
# * Hybrid (canonical): backend in a host venv under systemd `serverkit`,
# only the frontend in Docker. The blue/green + venv + systemd flow below
# targets this shape.
# * All-Docker: both backend AND frontend run as docker-compose services
# (container_name serverkit-backend / serverkit-frontend), no host venv.
# Running the hybrid flow on an all-Docker box builds a useless host venv,
# migrates the wrong (non-container) database, and stops a systemd unit that
# never serves traffic — which is exactly the failure that ends in
# "/opt/serverkit/venv/bin/activate: No such file or directory". Detect the
# all-Docker shape and route it to a dedicated compose update instead.
is_docker_deployment() {
[ -f "$INSTALL_DIR/docker-compose.yml" ] || return 1
# A usable host venv means this is the hybrid shape — take precedence.
[ -x "$INSTALL_DIR/venv/bin/python" ] && return 1
# The backend container is defined by the all-Docker compose project.
docker ps -a --format '{{.Names}}' 2>/dev/null | grep -qx 'serverkit-backend'
}
dc() {
if docker compose version &>/dev/null; then
( cd "$INSTALL_DIR" && docker compose "$@" )
else
( cd "$INSTALL_DIR" && docker-compose "$@" )
fi
}
backup_docker_db() {
mkdir -p "$BACKUP_DIR"
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would copy the SQLite DB out of the backend container"
return 0
fi
local backup_file
backup_file="$BACKUP_DIR/serverkit-pre-upgrade-$(date +%Y%m%d-%H%M%S).db"
# The live DB lives in the serverkit-data volume at /app/instance inside
# the container, not in the source tree. Prefer a consistent snapshot via
# the container's Python (SQLite online backup API) — a plain docker cp
# of the live file races the backend's writes and can come out malformed.
local py snap_ok=0
for py in python python3; do
if docker exec -i serverkit-backend "$py" - /app/instance/serverkit.db /tmp/.sk-snap.db <<'PYEOF' 2>/dev/null; then
import sqlite3, sys
s = sqlite3.connect(sys.argv[1])
d = sqlite3.connect(sys.argv[2])
s.backup(d)
d.close()
s.close()
PYEOF
snap_ok=1
break
fi
done
if [ "$snap_ok" = "1" ] && docker cp serverkit-backend:/tmp/.sk-snap.db "$backup_file" 2>/dev/null; then
docker exec serverkit-backend rm -f /tmp/.sk-snap.db 2>/dev/null || true
else
docker cp serverkit-backend:/app/instance/serverkit.db "$backup_file" 2>/dev/null || true
fi
# Same rule as the host path: an unverified backup is not a safety net.
if [ -f "$backup_file" ] && verify_sqlite_db "$backup_file"; then
good "Database backed up to $backup_file"
else
rm -f "$backup_file"
halt "Could not obtain a VALID database backup from the container — refusing to update without a safety net"
fi
}
# L6 — Heal layout left by an interrupted or legacy (hybrid) run before the
# docker update touches anything.
heal_layout_for_docker() {
# Pin the compose project name. A blue/green symlink otherwise makes compose
# derive the project name from the link target and warn — worse, a changed
# name would orphan the running containers.
export COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-$BASE_NAME}"
[ "$DRY_RUN" = "1" ] && return 0
# On an all-Docker box the blue/green slots are vestigial. If the install is
# a symlink (a prior hybrid run migrated it), reclaim the *inactive* slot it
# left behind — the live tree + DB volume are elsewhere, so this is safe.
if [ -L "$INSTALL_DIR" ]; then
local active inactive
active="$(active_real_dir)"
inactive="$DIR_A"; [ "$active" = "$DIR_A" ] && inactive="$DIR_B"
if [ -d "$inactive" ]; then
warn "Reclaiming stale slot from a prior run: $inactive"
rm -rf "$inactive"
fi
fi
}
# L4 — Snapshot the current docker deployment so a bad upgrade can be reverted.
DOCKER_PREV_SHA=""
DOCKER_ROLLBACK_READY=0
snapshot_docker_state() {
[ "$DRY_RUN" = "1" ] && return 0
DOCKER_PREV_SHA="$(git -C "$INSTALL_DIR" rev-parse HEAD 2>/dev/null || true)"
# Tag the currently-running images so we can re-point :latest back to them,
# and stash the host-built bundle that the new build is about to overwrite.
local img
for img in serverkit-backend serverkit-frontend; do
docker image inspect "$img:latest" >/dev/null 2>&1 && \
docker tag "$img:latest" "$img:rollback" 2>/dev/null || true
done
if [ -d "$INSTALL_DIR/frontend/dist" ]; then
rm -rf "$INSTALL_DIR/frontend/dist.rollback"
cp -a "$INSTALL_DIR/frontend/dist" "$INSTALL_DIR/frontend/dist.rollback" 2>/dev/null || true
fi
DOCKER_ROLLBACK_READY=1
}
rollback_docker() {
[ "$DOCKER_ROLLBACK_READY" = "1" ] || halt "New version is unhealthy and no rollback snapshot exists — inspect: cd $INSTALL_DIR && docker compose logs backend"
warn "New version unhealthy — rolling back to the previous deployment..."
# Guard the reset on a non-empty SHA: an empty SHA would make `git reset
# --hard ''` reset to HEAD (a no-op at best, surprising at worst). The image
# re-tag + bundle restore below still recover the previous deployment even
# when no commit was recorded.
if [ -n "$DOCKER_PREV_SHA" ]; then
git -C "$INSTALL_DIR" reset --hard "$DOCKER_PREV_SHA" 2>&1 | tail -n2 || \
warn "git reset to $DOCKER_PREV_SHA failed during rollback"
else
warn "No previous commit recorded — skipping git reset (images/bundle still restored)"
fi
if [ -d "$INSTALL_DIR/frontend/dist.rollback" ]; then
rm -rf "$INSTALL_DIR/frontend/dist"
mv "$INSTALL_DIR/frontend/dist.rollback" "$INSTALL_DIR/frontend/dist"
fi
local img
for img in serverkit-backend serverkit-frontend; do
docker image inspect "$img:rollback" >/dev/null 2>&1 && \
docker tag "$img:rollback" "$img:latest" 2>/dev/null || true
done
dc up -d 2>&1 | tail -n10 || true
# Confirm the restored deployment came back healthy.
local waited=0 status=""
while [ "$waited" -lt 60 ]; do
status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' serverkit-backend 2>/dev/null || echo missing)"
{ [ "$status" = "healthy" ] || [ "$status" = "none" ]; } && break
sleep 3
waited=$((waited + 3))
done
if [ "$status" = "healthy" ] || [ "$status" = "none" ]; then
halt "Rolled back to previous version (${DOCKER_PREV_SHA:-unknown}) and it is healthy. Logs: cd $INSTALL_DIR && docker compose logs backend"
else
halt "Rolled back to previous version (${DOCKER_PREV_SHA:-unknown}) but it is STILL UNHEALTHY (status: $status) — manual intervention needed. Logs: cd $INSTALL_DIR && docker compose logs backend"
fi
}
# Drop rollback artifacts once an upgrade is confirmed healthy.
clear_docker_rollback() {
[ "$DRY_RUN" = "1" ] && return 0
rm -rf "$INSTALL_DIR/frontend/dist.rollback" 2>/dev/null || true
local img
for img in serverkit-backend serverkit-frontend; do
docker image inspect "$img:rollback" >/dev/null 2>&1 && \
docker rmi "$img:rollback" >/dev/null 2>&1 || true
done
}
update_docker_compose() {
printf '\n %s%sServerKit Updater — Docker deployment%s\n' "$BLD" "$PAPER" "$RST"
STARTED_AT=$(date +%s)
[ "$DRY_RUN" = "1" ] && warn "DRY RUN — no changes will be made"
command -v docker &>/dev/null || halt "docker is required but not installed"
command -v git &>/dev/null || halt "git is required but not installed"
heal_layout_for_docker
# Resolve the git ref to update to (branch / release tag / main).
local ref="origin/main"
local mode="main"
if [ -n "$TARGET_BRANCH" ]; then
ref="origin/$TARGET_BRANCH"
mode="branch:$TARGET_BRANCH"
elif [ "$USE_RELEASE" = "1" ]; then
# `|| true`: head exiting first can SIGPIPE git (rc 141 under
# pipefail); an empty ref is caught by the halt just below.
ref="${RELEASE_VERSION:-$(git -C "$INSTALL_DIR" tag -l 'v*' --sort=-v:refname | head -n1 || true)}"
[ -n "$ref" ] || halt "Could not determine a release tag to update to"
mode="release"
fi
# Skip when already current (unless --force). Capture the starting version
# for the summary before we touch the tree.
local old_version
old_version="$(local_version)"
version_gate
phase "Database Backup"
backup_docker_db
# Snapshot the current (still-running, last-known-good) deployment before we
# mutate the tree, rebuild the bundle, or rebuild images — so an unhealthy
# upgrade can be reverted.
snapshot_docker_state
phase "Syncing Source"
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would git fetch and reset $INSTALL_DIR to $ref"
else
# The compose build context IS the live tree, so update it in place
# (no blue/green). Untracked .env, instance DB and frontend/dist are
# preserved across a hard reset.
git -C "$INSTALL_DIR" fetch --all --tags --prune 2>&1 | tail -n2 || halt "git fetch failed"
git -C "$INSTALL_DIR" reset --hard "$ref" 2>&1 | tail -n2 || halt "git reset to $ref failed"
chmod +x "$INSTALL_DIR/serverkit" "$INSTALL_DIR/scripts/"*.sh 2>/dev/null || true
# Untracked plugin dirs survive the hard reset — drop retired ones so
# they can't sink the bundle rebuild below.
remove_retired_plugins "$INSTALL_DIR"
good "Source synced to $ref"
fi
# When the compose serves a host-built bundle (bind-mounted ./frontend/dist)
# the image's own assets are shadowed, so the bundle must be rebuilt on the
# host. Otherwise the frontend image carries its assets and needs no host build.
if grep -qE '\./frontend/dist[ :]' "$INSTALL_DIR/docker-compose.yml" 2>/dev/null; then
phase "Building Frontend"
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would npm ci + npm run build in frontend/"
elif command -v npm &>/dev/null; then
_node_build_ok || halt "Node.js $(node --version 2>/dev/null || echo 'not found') is too old to build the frontend — vite 8 needs 20.19+ or 22.12+. Upgrade Node (e.g. NodeSource 22 LTS) and re-run."
( cd "$INSTALL_DIR/frontend" && npm ci 2>&1 | tail -n3 && \
NODE_OPTIONS="--max-old-space-size=1024" npm run build 2>&1 | tail -n5 ) \
|| halt "Frontend build failed"
good "Frontend assets rebuilt"
else
warn "npm not on host; relying on the frontend image build for assets"
fi
fi
phase "Building Images"
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would docker compose build"
else
# Build before recreating: running containers are untouched until
# 'up -d', so a failed build leaves the current version serving.
dc build 2>&1 | tail -n15 || halt "docker compose build failed — current version still running"
good "Images built"
fi
phase "Recreating Containers"
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would docker compose up -d (backend auto-migrates on boot)"
else
dc up -d 2>&1 | tail -n15 || halt "docker compose up -d failed"
# If the frontend serves a host-built bundle, restart it so nginx picks
# up freshly built assets even when its own image/config did not change.
if grep -qE '\./frontend/dist[ :]' "$INSTALL_DIR/docker-compose.yml" 2>/dev/null; then
dc restart frontend 2>&1 | tail -n3 || true
fi
good "Containers recreated"
fi
phase "Health Check"
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would wait for serverkit-backend to report healthy"
else
local waited=0 status=""
while [ "$waited" -lt 90 ]; do
status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' serverkit-backend 2>/dev/null || echo missing)"
[ "$status" = "healthy" ] && { good "Backend healthy"; break; }
[ "$status" = "none" ] && { good "Backend container running (no healthcheck defined)"; break; }
sleep 3
waited=$((waited + 3))
done
if [ "$status" != "healthy" ] && [ "$status" != "none" ]; then
warn "Backend did not report healthy within 90s (status: $status)"
rollback_docker # L4 — revert to the snapshot; this halts the script
fi
fi
# Healthy (or no healthcheck defined): the upgrade stuck, drop the snapshot.
clear_docker_rollback
# Keep the firewall in sync (idempotent, best-effort).
ensure_firewall || true
local new_version
new_version="$(cat "$INSTALL_DIR/VERSION" 2>/dev/null | tr -d '\n\r ' || echo unknown)"
curl -s "https://serverkit.ai/track/update?v=${new_version}" >/dev/null 2>&1 || true
printf '\n %s%s✔ Update complete (Docker)%s %s%s%s\n\n' \
"$BLD" "$HUE_OK" "$RST" "$FOG" "$(clock)" "$RST"
printf ' Updated %s → %s\n' "${old_version:-unknown}" "$new_version"
printf ' Mode docker/%s\n' "$mode"
printf ' Duration %s\n' "$(clock)"
printf ' Backend %s\n' "$(docker inspect -f '{{.State.Status}}' serverkit-backend 2>/dev/null || echo unknown)"
printf ' Frontend %s\n' "$(docker inspect -f '{{.State.Status}}' serverkit-frontend 2>/dev/null || echo unknown)"
[ -n "${UPDATE_LOG:-}" ] && printf ' Log %s\n' "$UPDATE_LOG"
printf '\n %sCLI%s serverkit status\n\n' "$BLD" "$RST"
}
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
# Sourcing this file (e.g. from scripts/test/test_update.sh) exposes every
# function above for unit testing without running an update. Only a direct
# execution falls through to the run below.
[ "${BASH_SOURCE[0]}" = "${0}" ] || return 0
maybe_reexec_latest_updater # may exec into the newest updater and not return
acquire_update_lock
init_logging
# L5 — turn any unguarded failure (in the main flow OR a helper, thanks to -E)
# into a labelled report instead of a silent drop back to the prompt.
trap 'report_failure "$?" "$LINENO" "$BASH_COMMAND"' ERR
# All-Docker deployments don't use the host venv / systemd / blue-green flow —
# route them to the compose path. This is what prevents the missing-venv crash.
if is_docker_deployment; then
update_docker_compose
exit 0
fi
printf '\n %s%sServerKit Updater%s\n' "$BLD" "$PAPER" "$RST"
STARTED_AT=$(date +%s)
[ "$DRY_RUN" = "1" ] && warn "DRY RUN — no changes will be made"
# Roll back automatically if the update fails after the symlink switch (the
# handler, cleanup_on_exit, is defined next to rollback() above).
trap cleanup_on_exit EXIT
preflight_check
# Capture where we are coming from and what mode this is, then skip the whole
# update when the box is already current (unless --force). Doing this before any
# backup/deploy work means an up-to-date box does zero churn.
OLD_VERSION="$(local_version)"
if [ "$USE_RELEASE" = "1" ]; then
UPDATE_MODE="release"
elif [ -n "$TARGET_BRANCH" ]; then
UPDATE_MODE="branch:$TARGET_BRANCH"
else
UPDATE_MODE="main"
fi
version_gate
ensure_bluegreen_layout
backup_current
# Determine update mode.
NEXT_DIR="$(next_real_dir)"
if [ "$USE_RELEASE" = "1" ]; then
if [ -z "$RELEASE_VERSION" ]; then
RELEASE_VERSION="$(curl -sf "https://api.github.com/repos/${GITHUB_REPO}/releases/latest" \
| grep '"tag_name"' | head -1 | cut -d'"' -f4 || true)"
[ -n "$RELEASE_VERSION" ] || halt "Could not determine the latest release"
fi
step "Updating to release $RELEASE_VERSION"
deploy_release "$NEXT_DIR" "$RELEASE_VERSION"
elif [ -n "$TARGET_BRANCH" ]; then
step "Updating to branch $TARGET_BRANCH"
deploy_source "$NEXT_DIR" "$TARGET_BRANCH"
else
deploy_source "$NEXT_DIR" "main"
fi
# Ensure venv in the new tree.
require_venv "$NEXT_DIR/venv"
# Run database migrations before switching.
migrate_database "$NEXT_DIR"
# Sync templates.
if [ "$DRY_RUN" = "0" ]; then
mkdir -p /etc/serverkit/templates
cp -r "$NEXT_DIR/backend/templates/"*.yaml /etc/serverkit/templates/ 2>/dev/null || true
cp -r "$NEXT_DIR/backend/templates/"*.yml /etc/serverkit/templates/ 2>/dev/null || true
fi
# Build frontend if dist is missing (source mode or older release).
if [ ! -d "$NEXT_DIR/frontend/dist" ]; then
step "Building frontend..."
if [ "$DRY_RUN" = "0" ]; then
# The build is guarded so a failure halts loudly while the old slot
# still serves. When plugins were carried forward from the old install
# they are the usual culprit (imports the new tree no longer satisfies)
# — quarantine them and give the build one retry before giving up.
if ! build_frontend_bundle "$NEXT_DIR"; then
quarantine_carried_plugins "$NEXT_DIR" \
|| halt "Frontend build failed — previous installation still active"
step "Retrying frontend build without carried-forward plugins..."
build_frontend_bundle "$NEXT_DIR" reuse \
|| halt "Frontend build failed — previous installation still active"
fi
else
info "[dry-run] would npm ci + npm run build in $NEXT_DIR/frontend"
fi
fi
# Re-apply the persistent per-install favicon tint to the staged dist (whether
# just rebuilt above or shipped prebuilt), so the color carries across updates.
[ "$DRY_RUN" = "0" ] && apply_favicon_tint "$NEXT_DIR"
# Refresh nginx/systemd configs in the active tree before switch.
refresh_config "$NEXT_DIR"
# Stop services.
#
# nginx is deliberately NOT stopped: it fronts every managed app, so taking it
# down would black out unrelated sites for the whole switch. Only the panel
# backend is cycled (a brief panel-API gap that never touches hosted apps), and
# nginx picks up any refreshed config via a graceful reload after the switch.
#
# First snapshot which hosted apps are reachable through nginx right now, so we
# can prove afterwards that the update did not knock any of them offline.
APP_BASELINE="$(snapshot_app_reachability)"
phase "Stopping Services"
# Guarded like the rollback path: `systemctl stop` exits non-zero when the unit
# is not loaded (exit 5) — a reason to warn, never to abort a healthy update.
run_or_dry systemctl stop "$BACKEND_SERVICE" \
|| warn "Backend service did not stop cleanly (unit may not be loaded) — continuing"
wait_for_service "$BACKEND_SERVICE" inactive 30 || warn "Backend did not stop within 30 seconds"
# Record the currently active directory before switching.
PREVIOUS_DIR="$(active_real_dir)"
# Atomic switch.
atomic_switch "$NEXT_DIR"
# Start services.
#
# The frontend is now static files under the switched-in slot's frontend/dist,
# served directly by host nginx (no container to recreate). The atomic switch
# already swapped the served assets; nginx only needs a graceful config reload.
phase "Starting Services"
run_or_dry systemctl start "$BACKEND_SERVICE"
wait_for_service "$BACKEND_SERVICE" active 30 || warn "Backend did not report active within 30 seconds"
# Re-label the now-active slot's bundle for SELinux hosts before nginx serves it.
selinux_label_dist "$NEXT_DIR"
# Graceful, zero-downtime config swap — never a stop/start (see reload_nginx_graceful).
reload_nginx_graceful
good "Services started"
# Health check.
health_check
# Verify the panel update did not take any hosted app down. Compare the
# pre-switch reachability snapshot with a fresh probe now that nginx has
# reloaded. Best-effort: a regression is reported loudly but does not fail the
# (already-healthy) update — the operator decides what to do about an app that
# was likely already unhealthy.
if [ "$DRY_RUN" = "0" ]; then
phase "Verifying App Uptime"
APP_AFTER="$(snapshot_app_reachability)"
report_app_uptime_regressions "$APP_BASELINE" "$APP_AFTER" || true
fi
# Keep the firewall in sync (idempotent, best-effort).
ensure_firewall || true
# Keep bash tab-completion in sync with the updated CLI (best-effort).
refresh_bash_completion || true
# Cleanup.
cleanup
# Summary.
NEW_VERSION="$(local_version)"
printf '\n %s%s✔ Update complete%s %s%s%s\n\n' \
"$BLD" "$HUE_OK" "$RST" "$FOG" "$(clock)" "$RST"
printf ' Updated %s → %s\n' "${OLD_VERSION:-unknown}" "${NEW_VERSION:-unknown}"
printf ' Mode %s\n' "${UPDATE_MODE:-main}"
printf ' Duration %s\n' "$(clock)"
printf ' Active %s\n' "$(active_real_dir)"
printf ' Backend %s\n' "$(systemctl is-active serverkit 2>/dev/null || echo unknown)"
printf ' Nginx %s\n\n' "$(systemctl is-active nginx 2>/dev/null || echo unknown)"
[ -n "${UPDATE_LOG:-}" ] && printf ' Log %s\n\n' "$UPDATE_LOG"
printf ' %sCLI%s serverkit status\n\n' "$BLD" "$RST"
Reference in new issue View Git Blame Copy Permalink