ServerKit/.github/workflows/scripts-ci.yml

name: Scripts CI

# Static + unit checks for the shell scripts that install/update/uninstall
# ServerKit on a server. These ran nowhere before — so a bug like the 1.7.0
# updater silently dying on `set -euo pipefail` was only ever found on a real
# box. shellcheck catches that whole class statically; the test_*.sh suites
# exercise the install/update/uninstall + lib logic against fixtures with no
# server, on every supported distro family.
#
# The unit suites *source* the scripts, though — the top-level run block
# (exactly where the 2026-07-02 update abort lived) executes in none of them.
# The update-e2e job closes that gap with a REAL install + REAL update on the
# runner VM, so the code path a user's box actually runs is gated on every PR.

on:
  push:
    branches: [dev, main]
    paths:
      - 'scripts/**'
      - 'templates/**'
      - 'install.sh'
      - 'uninstall.sh'
      - 'serverkit'
      - '.github/workflows/scripts-ci.yml'
  pull_request:
    branches: [dev, main]
    paths:
      - 'scripts/**'
      - 'templates/**'
      - 'install.sh'
      - 'uninstall.sh'
      - 'serverkit'
      - '.github/workflows/scripts-ci.yml'
  workflow_call:

jobs:
  shell:
    name: lint + unit
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Syntax check (bash -n)
        run: |
          status=0
          for f in serverkit install.sh uninstall.sh scripts/*.sh scripts/lib/*.sh scripts/test/*.sh; do
            [ -f "$f" ] || continue
            if bash -n "$f"; then
              echo "ok   $f"
            else
              echo "FAIL $f"; status=1
            fi
          done
          exit $status

      - name: ShellCheck (error severity — installer/updater/uninstaller surface)
        run: |
          sudo apt-get update -qq
          sudo apt-get install -y -qq shellcheck
          # Errors fail the job. The CLI + installer carry pre-existing advisory
          # warnings (unused color vars, etc.), so they are gated at error level.
          shellcheck --severity=error \
            scripts/update.sh scripts/install.sh install.sh uninstall.sh serverkit \
            scripts/lib/*.sh scripts/test/*.sh

      - name: ShellCheck (warning severity — new libs must stay clean)
        run: |
          # The lib/ abstraction was written warning-clean; keep it that way so
          # quality doesn't rot. -x follows `source`d files.
          shellcheck --severity=warning -x scripts/lib/*.sh

      - name: Unit tests (update + install + lib + cli + agent install)
        run: |
          bash scripts/test/test_update.sh
          bash scripts/test/test_install.sh
          bash scripts/test/test_lib.sh
          bash scripts/test/test_cli.sh
          bash scripts/test/test_agent_install.sh
          bash scripts/test/test_stage.sh

      - name: nginx -t smoke (shipped vhosts parse on a real nginx)
        run: |
          # The panel frontend is served straight from these vhosts (static SPA +
          # /api + /socket.io proxy). Parse them with the real nginx so a broken
          # directive fails here, not with a 502 on someone's server. The smoke
          # script stages each vhost against nginx and skips if nginx is absent —
          # so install it first to make sure the step actually runs.
          sudo apt-get install -y -qq nginx openssl
          bash scripts/test/test_nginx_conf.sh

  cross-distro:
    name: unit (${{ matrix.image }})
    runs-on: ubuntu-latest
    # ServerKit targets several Linux families; the scripts' logic (symlink
    # resolution, locking, refresh_config, deployment/OS/pkg/init detection,
    # the uninstall matrix) must behave the same on Debian-, RHEL-, and
    # SUSE-derived hosts, not just the Ubuntu runner. Run the unit suites inside
    # each distro's container so a portability regression fails here instead of
    # on someone's server.
    strategy:
      fail-fast: false
      matrix:
        image:
          - 'ubuntu:22.04'
          - 'ubuntu:24.04'
          - 'debian:12'
          - 'rockylinux:9'
          - 'almalinux:9'
          - 'fedora:40'
          - 'opensuse/leap:15.5'
    steps:
      - uses: actions/checkout@v4
      - name: bash -n + unit suites on ${{ matrix.image }}
        run: |
          docker run --rm -v "$PWD:/src" -w /src "${{ matrix.image }}" sh -c '
            set -e
            # bash ships in every image here; the suites stub the real tools
            # (curl/docker/systemd/flask) so nothing else is required. Best-effort
            # bash install for any stripped-down variant — never fatal (EL9, for
            # one, has a curl/curl-minimal conflict we deliberately do not touch).
            command -v bash >/dev/null 2>&1 || {
              { command -v apt-get >/dev/null 2>&1 && apt-get update -qq && apt-get install -y -qq bash; } ||
              { command -v dnf     >/dev/null 2>&1 && dnf install -y -q bash; } ||
              { command -v zypper  >/dev/null 2>&1 && zypper --non-interactive install bash; } || true
            }
            echo "bash $(bash --version | head -1)"
            for f in serverkit install.sh uninstall.sh scripts/*.sh scripts/lib/*.sh scripts/test/*.sh; do
              [ -f "$f" ] || continue
              bash -n "$f" && echo "ok   $f"
            done
            bash scripts/test/test_update.sh
            bash scripts/test/test_install.sh
            bash scripts/test/test_lib.sh
            bash scripts/test/test_cli.sh
            bash scripts/test/test_agent_install.sh
            bash scripts/test/test_stage.sh
          '

  update-e2e:
    name: e2e real install + update
    # The unit suites source the scripts and test functions in isolation — the
    # top-level run block never executes there, and that run block is exactly
    # where the 2026-07-02 update abort lived. This job is the missing gate: a
    # REAL install (the default non-interactive, no-domain, plain-HTTP path)
    # followed by a REAL run of THIS branch's updater, on the runner VM (full
    # systemd + docker — the same shape as a user's box). --dry-run is
    # deliberately NOT used: dry-run returns early from the very functions that
    # have broken. This job mutates the runner (/opt/serverkit, systemd units,
    # nginx), so it must stay independent — never fold it into the unit job.
    runs-on: ubuntu-latest
    timeout-minutes: 45
    steps:
      - uses: actions/checkout@v4

      - name: Install ServerKit (non-interactive, no domain, plain HTTP)
        run: |
          # `sudo env`, not `... | sudo bash`: sudo resets the environment, so
          # the switches must ride on the env command. PATH is forwarded so the
          # runner's preinstalled node/npm/docker toolchain stays visible to
          # root. No PANEL_DOMAIN + non-tty stdin is the documented no-prompt
          # default; SERVERKIT_SKIP_SSL keeps the run HTTP-only (no certbot
          # attempt on a runner with no DNS). install.sh copes with whatever
          # web-server state the runner image ships — it is built for real,
          # already-populated boxes.
          sudo env PATH="$PATH" SERVERKIT_SKIP_SSL=1 bash install.sh

      - name: Panel healthy after install
        run: |
          # Same probe install.sh and the VM harness gate on.
          for i in $(seq 1 30); do
            curl -fsS http://127.0.0.1:5000/api/v1/system/health >/dev/null 2>&1 && {
              echo "backend healthy after install (probe $i)"; exit 0; }
            sleep 2
          done
          echo "backend never became healthy after install" >&2
          exit 1

      - name: Real update with THIS branch's updater (--force)
        run: |
          # SERVERKIT_NO_SELF_UPDATE=1 stops the bootstrap from re-execing into
          # the updater published on the target ref — the object under test is
          # scripts/update.sh from this checkout. --force skips the version
          # gate so the full deploy → migrate → switch → health path always
          # executes.
          sudo env PATH="$PATH" SERVERKIT_NO_SELF_UPDATE=1 bash scripts/update.sh --force

      - name: Panel healthy after update
        run: |
          for i in $(seq 1 45); do
            curl -fsS http://127.0.0.1:5000/api/v1/system/health >/dev/null 2>&1 && {
              echo "backend healthy after update (probe $i)"; exit 0; }
            sleep 2
          done
          echo "backend never became healthy after update" >&2
          exit 1

      - name: Version gate skips a redundant re-run
        run: |
          # A second run without --force should hit the version gate and skip.
          # The message check is advisory (the remote may have advanced between
          # the two runs, making a second full update legitimate) — only the
          # exit code gates.
          out="$(sudo env PATH="$PATH" SERVERKIT_NO_SELF_UPDATE=1 bash scripts/update.sh)"
          printf '%s\n' "$out"
          printf '%s' "$out" | grep -qi "already up to date" \
            || echo "::warning::second run did not report 'Already up to date' (remote may have advanced)"

      - name: Dump install/update state
        if: failure()
        run: |
          set +e
          echo '===== /var/log/serverkit (updater logs) ====='
          sudo sh -c 'for f in /var/log/serverkit/*.log; do
            [ -f "$f" ] || continue
            echo "----- $f (last 300 lines) -----"
            tail -n 300 "$f"
          done'
          echo '===== systemd ====='
          sudo systemctl status serverkit --no-pager -l
          sudo journalctl -u serverkit --no-pager -n 200
          echo '===== nginx ====='
          sudo systemctl status nginx --no-pager -l
          sudo nginx -t
          sudo tail -n 100 /var/log/nginx/error.log
          echo '===== layout ====='
          ls -la /opt/ /opt/serverkit/
          true

  shellcheck-advisory:
    name: shellcheck full ruleset (advisory)
    # Everything shellcheck can say about the two scripts users actually pipe
    # to bash, at maximum sensitivity. Deliberately non-gating: --enable=all
    # carries style opinions the codebase does not follow, so this exists to be
    # read when touching install.sh / scripts/update.sh — not chased to green.
    # The gating checks stay in the lint + unit job above.
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v4

      - name: ShellCheck --enable=all (advisory — never fails the build)
        continue-on-error: true
        run: |
          sudo apt-get update -qq
          sudo apt-get install -y -qq shellcheck
          shellcheck --enable=all install.sh scripts/update.sh