ServerKit/.github/workflows/scripts-ci.yml

name: Scripts CI

# Static + unit checks for the shell scripts that install/update/uninstall
# ServerKit on a server. These ran nowhere before — so a bug like the 1.7.0
# updater silently dying on `set -euo pipefail` was only ever found on a real
# box. shellcheck catches that whole class statically; the test_*.sh suites
# exercise the install/update/uninstall + lib logic against fixtures with no
# server, on every supported distro family.
#
# The unit suites *source* the scripts, though — the top-level run block
# (exactly where the 2026-07-02 update abort lived) executes in none of them.
# The update-e2e job closes that gap with a REAL install + REAL update on the
# runner VM, so the code path a user's box actually runs is gated on every PR.

on:
  push:
    branches: [dev]
    paths:
      - 'scripts/**'
      - 'templates/**'
      - 'install.sh'
      - 'uninstall.sh'
      - 'serverkit'
      - '.github/workflows/main-promotion.yml'
      - '.github/workflows/release.yml'
      - '.github/workflows/version-bump.yml'
      - '.github/workflows/scripts-ci.yml'
  pull_request:
    branches: [main]
    paths:
      - 'scripts/**'
      - 'templates/**'
      - 'install.sh'
      - 'uninstall.sh'
      - 'serverkit'
      - '.github/workflows/main-promotion.yml'
      - '.github/workflows/release.yml'
      - '.github/workflows/version-bump.yml'
      - '.github/workflows/scripts-ci.yml'
  workflow_call:

jobs:
  shell:
    name: lint + unit
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Syntax check (bash -n)
        run: |
          status=0
          for f in serverkit install.sh uninstall.sh scripts/*.sh scripts/lib/*.sh scripts/test/*.sh; do
            [ -f "$f" ] || continue
            if bash -n "$f"; then
              echo "ok   $f"
            else
              echo "FAIL $f"; status=1
            fi
          done
          exit $status

      - name: ShellCheck (error severity — installer/updater/uninstaller surface)
        run: |
          sudo apt-get update -qq
          sudo apt-get install -y -qq shellcheck
          # Errors fail the job. The CLI + installer carry pre-existing advisory
          # warnings (unused color vars, etc.), so they are gated at error level.
          shellcheck --severity=error \
            scripts/update.sh scripts/install.sh install.sh uninstall.sh serverkit \
            scripts/lib/*.sh scripts/test/*.sh

      - name: ShellCheck (warning severity — new libs must stay clean)
        run: |
          # The lib/ abstraction was written warning-clean; keep it that way so
          # quality doesn't rot. -x follows `source`d files.
          shellcheck --severity=warning -x scripts/lib/*.sh

      - name: Unit tests (update + install + lib + cli + agent install)
        run: |
          bash scripts/test/test_update.sh
          bash scripts/test/test_install.sh
          bash scripts/test/test_lib.sh
          bash scripts/test/test_cli.sh
          bash scripts/test/test_agent_install.sh
          bash scripts/test/test_stage.sh
          bash scripts/test/test_release_workflow.sh

      - name: nginx -t smoke (shipped vhosts parse on a real nginx)
        run: |
          # The panel frontend is served straight from these vhosts (static SPA +
          # /api + /socket.io proxy). Parse them with the real nginx so a broken
          # directive fails here, not with a 502 on someone's server. The smoke
          # script stages each vhost against nginx and skips if nginx is absent —
          # so install it first to make sure the step actually runs.
          sudo apt-get install -y -qq nginx openssl
          bash scripts/test/test_nginx_conf.sh

  cross-distro:
    name: unit (${{ matrix.image }})
    runs-on: ubuntu-latest
    # ServerKit targets several Linux families; the scripts' logic (symlink
    # resolution, locking, refresh_config, deployment/OS/pkg/init detection,
    # the uninstall matrix) must behave the same on Debian-, RHEL-, and
    # SUSE-derived hosts, not just the Ubuntu runner. Run the unit suites inside
    # each distro's container so a portability regression fails here instead of
    # on someone's server.
    strategy:
      fail-fast: false
      matrix:
        image:
          - 'ubuntu:22.04'
          - 'ubuntu:24.04'
          - 'debian:12'
          # Debian 13 (trixie) ships ONLY python3.13 — no 3.11, no 3.12. Issue
          # #99: the installer's version gate could not be satisfied from its
          # repos and fell through to a source build. Every list we had stopped
          # at debian:12, which passes because bookworm still carries
          # python3.11, so nothing caught it. Newest-stable belongs here.
          - 'debian:13'
          - 'rockylinux:9'
          - 'almalinux:9'
          - 'fedora:40'
          - 'opensuse/leap:15.5'
    steps:
      - uses: actions/checkout@v4

      - name: Pull ${{ matrix.image }} (Docker Hub, then mirror)
        env:
          IMAGE: ${{ matrix.image }}
        run: |
          # `docker run` pulls implicitly, so an unreachable registry kills the
          # matrix leg with exit 125 before a single test runs. Docker Hub did
          # exactly that on two consecutive runs — a sustained egress blackout
          # from the runner ("Client.Timeout exceeded while awaiting headers"),
          # not a rate limit, so retrying Hub alone rides it out for two
          # minutes and then fails anyway. Fall back to Google's public Hub
          # mirror, a separate failure domain, and re-tag to the original name
          # so the run step below needs no changes.
          #
          # ONLY the pull is retried. Wrapping `docker run` would re-run the
          # suites on failure too, quietly turning a flaky TEST green.
          case "$IMAGE" in
            */*) MIRROR="mirror.gcr.io/$IMAGE" ;;          # namespaced: opensuse/leap
            *)   MIRROR="mirror.gcr.io/library/$IMAGE" ;;  # official image
          esac

          try_pull() {
            ref="$1"; tries="$2"
            for i in $(seq 1 "$tries"); do
              if docker pull --quiet "$ref"; then
                return 0
              fi
              echo "::warning::pull of $ref failed (attempt $i/$tries)"
              if [ "$i" -lt "$tries" ]; then
                sleep $((i * 5))
              fi
            done
            return 1
          }

          if try_pull "$IMAGE" 3; then
            exit 0
          fi
          echo "::warning::Docker Hub unreachable — falling back to $MIRROR"
          if try_pull "$MIRROR" 3; then
            # Re-tag so `docker run "$IMAGE"` resolves locally and never
            # reaches for Hub again (default pull policy is "missing").
            docker tag "$MIRROR" "$IMAGE"
            exit 0
          fi
          echo "::error::could not pull $IMAGE from Docker Hub or $MIRROR"
          exit 1

      - name: bash -n + unit suites on ${{ matrix.image }}
        run: |
          docker run --rm -v "$PWD:/src" -w /src "${{ matrix.image }}" sh -c '
            set -e
            # bash ships in every image here; the suites stub the real tools
            # (curl/docker/systemd/flask) so nothing else is required. Best-effort
            # bash install for any stripped-down variant — never fatal (EL9, for
            # one, has a curl/curl-minimal conflict we deliberately do not touch).
            command -v bash >/dev/null 2>&1 || {
              { command -v apt-get >/dev/null 2>&1 && apt-get update -qq && apt-get install -y -qq bash; } ||
              { command -v dnf     >/dev/null 2>&1 && dnf install -y -q bash; } ||
              { command -v zypper  >/dev/null 2>&1 && zypper --non-interactive install bash; } || true
            }
            echo "bash $(bash --version | head -1)"
            for f in serverkit install.sh uninstall.sh scripts/*.sh scripts/lib/*.sh scripts/test/*.sh; do
              [ -f "$f" ] || continue
              bash -n "$f" && echo "ok   $f"
            done
            bash scripts/test/test_update.sh
            bash scripts/test/test_install.sh
            bash scripts/test/test_lib.sh
            bash scripts/test/test_cli.sh
            bash scripts/test/test_agent_install.sh
            bash scripts/test/test_stage.sh
            bash scripts/test/test_release_workflow.sh
          '

  # The probe-honesty integration legs (plan 75 §D) — real ufw/nginx/firewalld
  # in 4 Docker shapes (sbinless PATH, non-root+sudo, rocky/firewalld,
  # cap-dropped LXC shape). The harness existed since plan 75 but was wired
  # into NO workflow, so the one pytest suite that runs firewall probes
  # against real binaries never ran in CI (plan 82 §E).
  probe-matrix:
    name: probe matrix (real firewall probes)
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v4
      - name: Run probe matrix
        run: bash scripts/test/probe-matrix.sh

  python-provisioning:
    name: python provisioning (${{ matrix.image }})
    runs-on: ubuntu-latest
    timeout-minutes: 25
    # The suites above stub every external tool, which is the right trade for
    # logic but leaves ONE thing untested: whether the distro's real repos can
    # actually give us a supported Python. That gap shipped issue #99 — Debian
    # 13 carries only python3.13, the gate wanted 3.11-3.12, and the installer
    # fell through to a source build that cannot succeed (nothing in install.sh
    # installs a compiler). Seven distro images were green throughout.
    #
    # provision_python needs no systemd, no nginx and no ports, so it runs
    # honestly in the same plain containers. This is the leg that turns "a new
    # distro dropped our Python" from an issue report into a red build.
    strategy:
      fail-fast: false
      matrix:
        image:
          - 'ubuntu:22.04'
          - 'ubuntu:24.04'
          - 'debian:12'
          - 'debian:13'
          - 'rockylinux:9'
          - 'almalinux:9'
          - 'fedora:40'
          - 'opensuse/leap:15.5'
    steps:
      - uses: actions/checkout@v4

      - name: provision_python on ${{ matrix.image }}
        run: |
          docker run --rm -v "$PWD:/src:ro" -w /src "${{ matrix.image }}" sh -c '
            command -v bash >/dev/null 2>&1 || {
              { command -v apt-get >/dev/null 2>&1 && apt-get update -qq && apt-get install -y -qq bash; } ||
              { command -v dnf     >/dev/null 2>&1 && dnf install -y -q bash; } ||
              { command -v zypper  >/dev/null 2>&1 && zypper --non-interactive install bash; } || true
            }
            bash scripts/test/test_provision_python.sh
          '

  cross-arch-arm64:
    name: unit (arm64 / raspberry-pi proxy)
    runs-on: ubuntu-24.04-arm
    # Raspberry Pi / ARM homelab proxy: the E2E VM harness can't virtualize
    # ARM (Hyper-V on Windows is x86_64 only), so this hosted ARM64 runner is
    # the closest CI approximation of 64-bit Raspberry Pi OS / Ubuntu ARM.
    # Runs the same unit suites natively (no QEMU) so an arch-sensitive
    # regression (symlink resolution, arch detection, asset/package naming)
    # fails here instead of on someone's Pi. Real-hardware validation (GPIO,
    # SD-card I/O, 32-bit ARM) still needs a physical device.
    steps:
      - uses: actions/checkout@v4
      - name: bash -n + unit suites on arm64
        run: |
          set -e
          for f in serverkit install.sh uninstall.sh scripts/*.sh scripts/lib/*.sh scripts/test/*.sh; do
            [ -f "$f" ] || continue
            bash -n "$f" && echo "ok   $f"
          done
          bash scripts/test/test_update.sh
          bash scripts/test/test_install.sh
          bash scripts/test/test_lib.sh
          bash scripts/test/test_cli.sh
          bash scripts/test/test_agent_install.sh
          bash scripts/test/test_stage.sh

  update-e2e:
    name: e2e real install + update
    # The unit suites source the scripts and test functions in isolation — the
    # top-level run block never executes there, and that run block is exactly
    # where the 2026-07-02 update abort lived. This job is the missing gate: a
    # REAL install (the default non-interactive, no-domain, plain-HTTP path)
    # followed by a REAL run of THIS branch's updater, on the runner VM (full
    # systemd + docker — the same shape as a user's box). --dry-run is
    # deliberately NOT used: dry-run returns early from the very functions that
    # have broken. This job mutates the runner (/opt/serverkit, systemd units,
    # nginx), so it must stay independent — never fold it into the unit job.
    runs-on: ubuntu-latest
    timeout-minutes: 45
    steps:
      - uses: actions/checkout@v4

      - name: Install ServerKit (non-interactive, no domain, plain HTTP)
        run: |
          # `sudo env`, not `... | sudo bash`: sudo resets the environment, so
          # the switches must ride on the env command. PATH is forwarded so the
          # runner's preinstalled node/npm/docker toolchain stays visible to
          # root. No PANEL_DOMAIN + non-tty stdin is the documented no-prompt
          # default; SERVERKIT_SKIP_SSL keeps the run HTTP-only (no certbot
          # attempt on a runner with no DNS). install.sh copes with whatever
          # web-server state the runner image ships — it is built for real,
          # already-populated boxes.
          sudo env PATH="$PATH" SERVERKIT_SKIP_SSL=1 bash install.sh

      - name: Panel healthy after install
        run: |
          # Same probe install.sh and the VM harness gate on.
          for i in $(seq 1 30); do
            curl -fsS http://127.0.0.1:5000/api/v1/system/health >/dev/null 2>&1 && {
              echo "backend healthy after install (probe $i)"; exit 0; }
            sleep 2
          done
          echo "backend never became healthy after install" >&2
          exit 1

      - name: Real update with THIS branch's updater (--force)
        run: |
          # SERVERKIT_NO_SELF_UPDATE=1 stops the bootstrap from re-execing into
          # the updater published on the target ref — the object under test is
          # scripts/update.sh from this checkout. --force skips the version
          # gate so the full deploy → migrate → switch → health path always
          # executes.
          sudo env PATH="$PATH" SERVERKIT_NO_SELF_UPDATE=1 bash scripts/update.sh --force

      - name: Panel healthy after update
        run: |
          for i in $(seq 1 45); do
            curl -fsS http://127.0.0.1:5000/api/v1/system/health >/dev/null 2>&1 && {
              echo "backend healthy after update (probe $i)"; exit 0; }
            sleep 2
          done
          echo "backend never became healthy after update" >&2
          exit 1

      - name: Version gate skips a redundant re-run
        run: |
          # A second run without --force should hit the version gate and skip.
          # The message check is advisory (the remote may have advanced between
          # the two runs, making a second full update legitimate) — only the
          # exit code gates.
          out="$(sudo env PATH="$PATH" SERVERKIT_NO_SELF_UPDATE=1 bash scripts/update.sh)"
          printf '%s\n' "$out"
          printf '%s' "$out" | grep -qi "already up to date" \
            || echo "::warning::second run did not report 'Already up to date' (remote may have advanced)"

      - name: Dump install/update state
        if: failure()
        run: |
          set +e
          echo '===== /var/log/serverkit (updater logs) ====='
          sudo sh -c 'for f in /var/log/serverkit/*.log; do
            [ -f "$f" ] || continue
            echo "----- $f (last 300 lines) -----"
            tail -n 300 "$f"
          done'
          echo '===== systemd ====='
          sudo systemctl status serverkit --no-pager -l
          sudo journalctl -u serverkit --no-pager -n 200
          echo '===== nginx ====='
          sudo systemctl status nginx --no-pager -l
          sudo nginx -t
          sudo tail -n 100 /var/log/nginx/error.log
          echo '===== layout ====='
          ls -la /opt/ /opt/serverkit/
          true

  shellcheck-advisory:
    name: shellcheck full ruleset (advisory)
    # Everything shellcheck can say about the two scripts users actually pipe
    # to bash, at maximum sensitivity. Deliberately non-gating: --enable=all
    # carries style opinions the codebase does not follow, so this exists to be
    # read when touching install.sh / scripts/update.sh — not chased to green.
    # The gating checks stay in the lint + unit job above.
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v4

      - name: ShellCheck --enable=all (advisory — never fails the build)
        continue-on-error: true
        run: |
          sudo apt-get update -qq
          sudo apt-get install -y -qq shellcheck
          shellcheck --enable=all install.sh scripts/update.sh