release: archivebox 0.9.34rc66

release: archivebox 0.9.34rc65
release: archivebox 0.9.34rc64
2026-06-06 23:40:29 -07:00 · 2026-06-06 23:34:58 -07:00 · 2026-06-06 23:28:53 -07:00 · 2026-06-06 23:27:54 -07:00 · 2026-06-06 23:19:13 -07:00 · 2026-06-06 23:09:44 -07:00
657 changed files with 114407 additions and 29341 deletions
@@ -1,17 +1,31 @@
 .DS_Store
 ._*
+/.heartbeat.json
+/package.json
+/package-lock.json
+__pycache__
+**/__pycache__
 *.pyc
-__pycache__/
+*.pyo
+*.py[cod]
+*$py.class
 .mypy_cache/
 .pytest_cache/
+.ruff_cache/
+.uv-cache/
 .github/
 .pdm-build/
 .pdm-python
 .eggs/
 .git/
+!.git/
+.git/*
 .vscode/
 !.git/HEAD
-!.git/refs/heads/*
+!.git/packed-refs
+!.git/refs/
+!.git/refs/heads/
+!.git/refs/heads/**

 venv/
 .venv/
@@ -19,9 +33,24 @@ venv/
 .docker_venv/
 .docker-venv/
 node_modules/
+abx-dl/
+abxpkg/
+abx-plugins/
+abxbus/
 chrome/
 chromeprofile/
 chrome_profile/
+lib/
+out/
+users/
+archive/
+crawls/
+snapshots/
+logs/
+archivebox-docker-smoke*/
+archivebox-compose-smoke*/
+core
+*.core

 pdm.dev.lock
 pdm.lock
@@ -38,8 +67,20 @@ website/
 typings/

 tmp/
+.tmp/
 data/
 data*/
+-
+personas/
+sources/
 output/
 index.sqlite3
 index.sqlite3-wal
+queue.sqlite3
+*.sqlite*
+data.*
+.archivebox_id
+ArchiveBox.conf
+*.stdout
+*.stderr
+*.log
@@ -19,6 +19,10 @@ on:
  schedule:
    - cron: '33 17 * * 6'

+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
  analyze:
    name: Analyze (${{ matrix.language }})
@@ -1,210 +0,0 @@
-name: Build Debian package
-
-on:
-  workflow_dispatch:
-  workflow_call:
-  push:
-    branches: ['**']
-    paths:
-      - 'pkg/debian/**'
-      - 'bin/build_deb.sh'
-      - 'bin/release_deb.sh'
-      - '.github/workflows/debian.yml'
-      - 'pyproject.toml'
-  # release trigger is handled by release.yml to avoid double-runs
-
-permissions:
-  contents: write
-
-jobs:
-  build:
-    runs-on: ubuntu-24.04
-    strategy:
-      matrix:
-        arch: [amd64, arm64]
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Get version
-        id: version
-        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
-
-      - name: Install nfpm
-        env:
-          NFPM_VERSION: "2.45.1"
-        run: |
-          curl -fsSL "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${NFPM_VERSION}_Linux_x86_64.tar.gz" \
-            | sudo tar -xz -C /usr/local/bin nfpm
-          nfpm --version
-
-      - name: Build .deb package
-        run: |
-          export VERSION="${{ steps.version.outputs.version }}"
-          export ARCH="${{ matrix.arch }}"
-          ./bin/build_deb.sh
-
-      - name: Verify .deb package contents
-        run: |
-          DEB_FILE="$(ls dist/archivebox*.deb | head -1)"
-          echo "=== Package info ==="
-          dpkg-deb --info "$DEB_FILE"
-          echo ""
-          echo "=== Package contents ==="
-          dpkg-deb --contents "$DEB_FILE"
-          echo ""
-          echo "=== Control fields ==="
-          dpkg-deb --field "$DEB_FILE"
-
-      - name: Upload .deb artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: archivebox-${{ steps.version.outputs.version }}-${{ matrix.arch }}.deb
-          path: dist/*.deb
-
-  test:
-    needs: build
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - arch: amd64
-            runner: ubuntu-24.04
-          - arch: arm64
-            runner: ubuntu-24.04-arm
-    runs-on: ${{ matrix.runner }}
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v4
-        with:
-          enable-cache: true
-
-      - name: Install build dependencies
-        uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0
-        with:
-          packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1
-          version: 1.0
-
-      - name: Build local wheel
-        run: |
-          uv sync --frozen --all-extras --no-install-project --no-install-workspace
-          uv build --wheel --out-dir /tmp/wheels/
-
-      - name: Download .deb artifact
-        uses: actions/download-artifact@v4
-        with:
-          pattern: archivebox-*-${{ matrix.arch }}.deb
-          merge-multiple: true
-
-      - name: Install system dependencies
-        run: |
-          sudo apt-get install -y python3.13 python3.13-venv python3-pip git curl wget
-
-      - name: Pre-seed virtualenv with local wheel before dpkg install
-        run: |
-          # CI-only: pre-seed the venv with a local wheel so we test the
-          # unreleased code, not whatever is on PyPI. We explicitly use python3.13
-          # here (matching the system dep installed above) to ensure the venv is
-          # created with the correct Python version. On real installs, install.sh
-          # handles this by preferring python3.13 and failing if < 3.13.
-          # When postinstall.sh runs during dpkg -i, it finds the venv already
-          # populated and just upgrades deps.
-          sudo mkdir -p /opt/archivebox
-          sudo python3.13 -m venv /opt/archivebox/venv
-          sudo /opt/archivebox/venv/bin/python3 -m pip install --quiet --upgrade pip setuptools
-          sudo /opt/archivebox/venv/bin/pip install --quiet /tmp/wheels/archivebox-*.whl
-          echo "[√] Pre-seeded /opt/archivebox/venv with local wheel"
-
-      - name: Install .deb package
-        run: |
-          # dpkg install will run postinstall which creates the user,
-          # sets up systemd, and tries pip install (which finds it already installed)
-          sudo dpkg -i archivebox*.deb || sudo apt-get install -f -y
-
-      - name: Verify archivebox is installed
-        run: |
-          which archivebox
-          archivebox version
-
-      - name: Verify wrapper script works
-        run: |
-          /usr/bin/archivebox version
-          /usr/bin/archivebox --help | head -5
-
-      - name: Test archivebox init as archivebox user
-        run: |
-          # The postinstall should have created the user
-          id archivebox
-          sudo mkdir -p /tmp/archivebox-test
-          sudo chown archivebox:archivebox /tmp/archivebox-test
-          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox init'
-
-      - name: Test archivebox status
-        run: |
-          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox status'
-
-      - name: Test archivebox add
-        run: |
-          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox add "https://example.com"'
-
-      - name: Verify systemd service file exists
-        run: |
-          test -f /usr/lib/systemd/system/archivebox.service
-          cat /usr/lib/systemd/system/archivebox.service
-
-  # Upload .deb to GitHub Release only after tests pass
-  release:
-    if: github.event_name == 'release' || github.event_name == 'workflow_call'
-    needs: [build, test]
-    runs-on: ubuntu-24.04
-    permissions:
-      contents: write
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Determine release tag
-        id: tag
-        run: |
-          TAG="${{ github.event.release.tag_name }}"
-          if [ -z "$TAG" ]; then
-            TAG="v$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')"
-            echo "[i] No release tag in event context, using version from pyproject.toml: $TAG"
-          fi
-          echo "tag=$TAG" >> "$GITHUB_OUTPUT"
-
-      - name: Download all .deb artifacts
-        uses: actions/download-artifact@v4
-        with:
-          pattern: archivebox-*.deb
-          merge-multiple: true
-
-      - name: Upload .deb to GitHub Release
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          TAG="${{ steps.tag.outputs.tag }}"
-          # Verify the release exists before uploading
-          if ! gh release view "$TAG" >/dev/null 2>&1; then
-            echo "[!] No GitHub Release found for tag $TAG."
-            if [ -n "${{ github.event.release.tag_name }}" ]; then
-              echo "[X] This was triggered by a release event — the release should exist. Failing."
-              exit 1
-            fi
-            echo "[i] Skipping upload (workflow_dispatch without a release)."
-            echo "    Create a release first or trigger via the release event."
-            exit 0
-          fi
-          gh release upload "$TAG" *.deb --clobber
@@ -0,0 +1,43 @@
+name: Deploy Publicsite to GitHub Pages
+
+on:
+  push:
+    branches:
+      - dev
+    paths:
+      - publicsite/**
+      - .github/workflows/deploy-publicsite.yml
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: github-pages-publicsite
+  cancel-in-progress: true
+
+jobs:
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Setup Pages
+        uses: actions/configure-pages@v5
+
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: ./publicsite
+
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4
@@ -5,17 +5,40 @@ on:
  workflow_call:
  push:
    branches:
-      - '**'
+      - "**"
    tags:
      - 'v*'
  # pull_request:

 env:
-  DOCKER_IMAGE: archivebox-ci
+  DOCKERHUB_IMAGE: archivebox/archivebox
+  GHCR_IMAGE: ghcr.io/archivebox/archivebox
+  ABX_DL_IMAGE: archivebox/abx-dl:latest
+
+permissions:
+  contents: read
+  packages: write
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true

 jobs:
-  buildx:
-    runs-on: ubuntu-latest
+  build:
+    name: build ${{ matrix.platform }}
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: linux/amd64
+            runner: ubuntu-24.04
+            cache_scope: docker-amd64
+            artifact_name: digest-linux-amd64
+          - platform: linux/arm64
+            runner: ubuntu-24.04-arm
+            cache_scope: docker-arm64
+            artifact_name: digest-linux-arm64
    steps:
      - name: Checkout
        uses: actions/checkout@v4
@@ -23,8 +46,43 @@ jobs:
        #   submodules: true
        #   fetch-depth: 1

-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+      - name: Wait for released ArchiveBox deps on PyPI
+        run: |
+          python3 - <<'PY'
+          import json
+          import re
+          import sys
+          import time
+          import tomllib
+          import urllib.request
+
+          watched = {"abxbus", "abxpkg", "abx-plugins", "abx-dl"}
+          deps = tomllib.loads(open("pyproject.toml", "rb").read().decode())["project"]["dependencies"]
+          required = {}
+          for dep in deps:
+              for name in watched:
+                  match = re.match(rf"{re.escape(name)}\s*(==|>=)\s*([^,;\s]+)", dep)
+                  if match:
+                      required[name] = match.group(2)
+
+          deadline = time.monotonic() + 300
+          missing = required.copy()
+          while missing and time.monotonic() < deadline:
+              for name, version in list(missing.items()):
+                  with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/json", timeout=20) as resp:
+                      releases = json.load(resp)["releases"]
+                  if version in releases:
+                      print(f"{name} {version} is available on PyPI")
+                      missing.pop(name)
+                  else:
+                      print(f"{name} {version} is not available on PyPI yet")
+              if missing:
+                  time.sleep(10)
+
+          if missing:
+              print(f"Missing PyPI releases after wait: {missing}", file=sys.stderr)
+              sys.exit(1)
+          PY

      - name: Set up Docker Buildx
        id: buildx
@@ -32,7 +90,6 @@ jobs:
        with:
          version: latest
          install: true
-          platforms: linux/amd64,linux/arm64

      - name: Builder instance name
        run: echo ${{ steps.buildx.outputs.name }}
@@ -40,13 +97,23 @@ jobs:
      - name: Available platforms
        run: echo ${{ steps.buildx.outputs.platforms }}

-      - name: Cache Docker layers
-        uses: actions/cache@v4
-        with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildx-
+      - name: Wait for published abx-dl image
+        id: abx_dl_image
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          deadline=$((SECONDS + 1800))
+          until docker buildx imagetools inspect "${ABX_DL_IMAGE}" >/tmp/abx-dl-image.json; do
+            if (( SECONDS >= deadline )); then
+              echo "Timed out waiting for published ${ABX_DL_IMAGE}" >&2
+              exit 1
+            fi
+            echo "${ABX_DL_IMAGE} is not published yet; waiting..."
+            sleep 30
+          done
+
+          echo "image=${ABX_DL_IMAGE}" >> "$GITHUB_OUTPUT"
+          docker buildx imagetools inspect "${ABX_DL_IMAGE}"

      - name: Login to Docker Hub
        uses: docker/login-action@v3
@@ -62,68 +129,203 @@ jobs:
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

-      - name: Collect Full Release Docker tags
-        # https://github.com/docker/metadata-action
+      - name: Collect Docker labels
        id: docker_meta
-        uses: docker/metadata-action@v5
-        if: github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call'
-        with:
-          images: archivebox/archivebox,ghcr.io/archivebox/archivebox
-          tags: |
-              # :stable
-              type=ref,event=branch
-              # :0.7.3
-              type=semver,pattern={{version}}
-              # :0.7
-              type=semver,pattern={{major}}.{{minor}}
-              # :sha-463ea54
-              type=sha
-              # :latest
-              type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'stable') }}
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          VERSION="$(python3 - <<'PY'
+          import tomllib

-      - name: Collect Non-Release Docker tags
-        # https://github.com/docker/metadata-action
-        id: docker_meta_non_release
-        uses: docker/metadata-action@v5
-        if: github.event_name != 'workflow_dispatch' && github.event_name != 'workflow_call'
-        with:
-          images: archivebox/archivebox,ghcr.io/archivebox/archivebox
-          tags: |
-              # :stable
-              type=ref,event=branch
-              # :sha-463ea54
-              type=sha
+          with open("pyproject.toml", "rb") as f:
+              print(tomllib.load(f)["project"]["version"])
+          PY
+          )"

-      - name: Build and push
+          {
+            echo 'labels<<EOF'
+            echo "org.opencontainers.image.version=${VERSION}"
+            echo "org.opencontainers.image.revision=${GITHUB_SHA}"
+            echo 'org.opencontainers.image.source=https://github.com/ArchiveBox/ArchiveBox'
+            echo "io.archivebox.abx-dl.image=${{ steps.abx_dl_image.outputs.image }}"
+            echo 'EOF'
+            echo "version=${VERSION}"
+          } >> "$GITHUB_OUTPUT"
+
+          echo "[+] Building ${{ matrix.platform }} for ${VERSION} using ${{ steps.abx_dl_image.outputs.image }}"
+
+      - name: Build and push digest
        id: docker_build
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
        with:
          context: ./
          file: ./Dockerfile
          builder: ${{ steps.buildx.outputs.name }}
          push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') && steps.docker_meta.outputs.tags || steps.docker_meta_non_release.outputs.tags }}
-          labels: ${{ (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') && steps.docker_meta.outputs.labels || steps.docker_meta_non_release.outputs.labels }}
-          cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache-new
-          platforms: linux/amd64,linux/arm64
+          tags: |
+            ${{ env.DOCKERHUB_IMAGE }}
+            ${{ env.GHCR_IMAGE }}
+          labels: ${{ steps.docker_meta.outputs.labels }}
+          build-args: |
+            ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
+          cache-from: type=gha,scope=${{ matrix.cache_scope }}
+          cache-to: type=gha,mode=max,scope=${{ matrix.cache_scope }}
+          platforms: ${{ matrix.platform }}
+          outputs: type=image,push-by-digest=true,name-canonical=true,push=true

      - name: Image digest
        run: echo ${{ steps.docker_build.outputs.digest }}

+      - name: Export digest
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          mkdir -p /tmp/digests
+          digest="${{ steps.docker_build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+
+      - name: Upload digest
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.artifact_name }}
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  publish:
+    name: publish multiarch tags
+    runs-on: ubuntu-24.04
+    needs:
+      - build
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          version: latest
+          install: true
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        if: github.event_name != 'pull_request'
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp/digests
+          pattern: digest-*
+          merge-multiple: true
+
+      - name: Collect Docker tags
+        id: docker_meta
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          VERSION="$(python3 - <<'PY'
+          import tomllib
+
+          with open("pyproject.toml", "rb") as f:
+              print(tomllib.load(f)["project"]["version"])
+          PY
+          )"
+          BRANCH_TAG="$(printf '%s' "${GITHUB_REF_NAME}" | tr -c 'A-Za-z0-9_.-' '-' | sed -E 's/^-+//; s/-+$//; s/-+/-/g' | cut -c1-128)"
+          SHORT_SHA="${GITHUB_SHA::12}"
+          test -n "$BRANCH_TAG"
+          test -n "$SHORT_SHA"
+
+          {
+            echo 'dockerhub_tags<<EOF'
+            if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
+              echo "${DOCKERHUB_IMAGE}:latest"
+            fi
+            echo "${DOCKERHUB_IMAGE}:${BRANCH_TAG}"
+            echo "${DOCKERHUB_IMAGE}:${VERSION}"
+            echo "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
+            echo 'EOF'
+            echo 'ghcr_tags<<EOF'
+            if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
+              echo "${GHCR_IMAGE}:latest"
+            fi
+            echo "${GHCR_IMAGE}:${BRANCH_TAG}"
+            echo "${GHCR_IMAGE}:${VERSION}"
+            echo "${GHCR_IMAGE}:sha-${SHORT_SHA}"
+            echo 'EOF'
+            echo "version=${VERSION}"
+            echo "branch_tag=${BRANCH_TAG}"
+            echo "short_sha=${SHORT_SHA}"
+          } >> "$GITHUB_OUTPUT"
+
+          echo "[+] Publishing Docker Hub tags:"
+          if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
+            printf '%s\n' "${DOCKERHUB_IMAGE}:latest"
+          fi
+          printf '%s\n' "${DOCKERHUB_IMAGE}:${BRANCH_TAG}" "${DOCKERHUB_IMAGE}:${VERSION}" "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
+          echo "[+] Publishing GHCR tags:"
+          if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
+            printf '%s\n' "${GHCR_IMAGE}:latest"
+          fi
+          printf '%s\n' "${GHCR_IMAGE}:${BRANCH_TAG}" "${GHCR_IMAGE}:${VERSION}" "${GHCR_IMAGE}:sha-${SHORT_SHA}"
+
+      - name: Create Docker Hub manifest
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          mapfile -t DIGESTS < <(find /tmp/digests -maxdepth 1 -type f -printf '%f\n' | sort)
+          [[ "${#DIGESTS[@]}" -gt 0 ]]
+
+          TAG_ARGS=()
+          while IFS= read -r tag; do
+            [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag")
+          done <<< "${{ steps.docker_meta.outputs.dockerhub_tags }}"
+
+          REFS=()
+          for digest in "${DIGESTS[@]}"; do
+            REFS+=("${DOCKERHUB_IMAGE}@sha256:${digest}")
+          done
+
+          docker buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
+
+      - name: Create GHCR manifest
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          mapfile -t DIGESTS < <(find /tmp/digests -maxdepth 1 -type f -printf '%f\n' | sort)
+          [[ "${#DIGESTS[@]}" -gt 0 ]]
+
+          TAG_ARGS=()
+          while IFS= read -r tag; do
+            [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag")
+          done <<< "${{ steps.docker_meta.outputs.ghcr_tags }}"
+
+          REFS=()
+          for digest in "${DIGESTS[@]}"; do
+            REFS+=("${GHCR_IMAGE}@sha256:${digest}")
+          done
+
+          docker buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
+
+      - name: Inspect published images
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          docker buildx imagetools inspect "${DOCKERHUB_IMAGE}:sha-${{ steps.docker_meta.outputs.short_sha }}"
+          docker buildx imagetools inspect "${GHCR_IMAGE}:sha-${{ steps.docker_meta.outputs.short_sha }}"
+
      - name: Update README
        uses: peter-evans/dockerhub-description@v4
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
          repository: archivebox/archivebox
-
-      # This ugly bit is necessary if you don't want your cache to grow forever
-      # until it hits GitHub's limit of 5GB.
-      # Temp fix
-      # https://github.com/docker/build-push-action/issues/252
-      # https://github.com/moby/buildkit/issues/1896
-      - name: Move cache
-        run: |
-          rm -rf /tmp/.buildx-cache
-          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
@@ -1,64 +0,0 @@
-# Simple workflow for deploying static content to GitHub Pages
-name: Deploy static content to Pages
-
-on:
-  # Runs on pushes targeting the default branch
-  push:
-    branches: ["dev"]
-
-  # Allows you to run this workflow manually from the Actions tab
-  workflow_dispatch:
-
-# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
-permissions:
-  contents: read
-  pages: write
-  id-token: write
-
-# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
-# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
-concurrency:
-  group: "pages"
-  cancel-in-progress: false
-
-jobs:
-  # Single deploy job since we're just deploying
-  deploy:
-    environment:
-      name: github-pages
-      url: ${{ steps.deployment.outputs.page_url }}
-    runs-on: ubuntu-latest
-    steps:
-
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          submodules: true
-          fetch-depth: 1
-      - name: Copy README.md into place
-        run: |
-          rm -f ./website/README.md
-          cp ./README.md ./website/README.md
-      - name: Setup Pages
-        uses: actions/configure-pages@v5
-      - name: Build with Jekyll
-        uses: actions/jekyll-build-pages@v1
-        with:
-          source: ./website
-          destination: ./_site
-      - name: Upload artifact
-        uses: actions/upload-pages-artifact@v3
-
-      # - name: Checkout
-      #   uses: actions/checkout@v4
-      # - name: Setup Pages
-      #   uses: actions/configure-pages@v5
-      # - name: Upload artifact
-      #   uses: actions/upload-pages-artifact@v3
-      #   with:
-      #     # Upload entire repository
-      #     path: './website'
-
-      - name: Deploy to GitHub Pages
-        id: deployment
-        uses: actions/deploy-pages@v4
@@ -1,229 +0,0 @@
-name: Build Homebrew formula
-
-on:
-  workflow_dispatch:
-  workflow_call:
-  push:
-    branches: ['**']
-    paths:
-      - 'brew_dist/**'
-      - 'bin/build_brew.sh'
-      - 'bin/release_brew.sh'
-      - '.github/workflows/homebrew.yml'
-      - 'pyproject.toml'
-  # release trigger is handled by release.yml to avoid double-runs
-
-permissions:
-  contents: read
-
-jobs:
-  build-and-test:
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [macos-latest, ubuntu-24.04]
-    runs-on: ${{ matrix.os }}
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Get version
-        id: version
-        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
-
-      - name: Validate formula template syntax
-        run: ruby -c brew_dist/archivebox.rb
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v4
-        with:
-          enable-cache: true
-
-      - name: Install build dependencies (Linux)
-        if: runner.os == 'Linux'
-        uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0
-        with:
-          packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1
-          version: 1.0
-
-      - name: Build local sdist
-        run: |
-          uv sync --frozen --all-extras --no-install-project --no-install-workspace
-          uv build --sdist --out-dir /tmp/sdist/
-
-      - name: Generate formula from local sdist
-        run: |
-          VERSION="${{ steps.version.outputs.version }}"
-          SDIST_PATH="$(ls /tmp/sdist/archivebox-*.tar.gz | head -1)"
-          SDIST_SHA256="$(shasum -a 256 "$SDIST_PATH" | awk '{print $1}')"
-
-          # Install archivebox + poet into a temp venv to generate resource stanzas
-          python3 -m venv /tmp/poet-venv
-          source /tmp/poet-venv/bin/activate
-          pip install --quiet "$SDIST_PATH" homebrew-pypi-poet
-          echo "[+] Generating resource stanzas with homebrew-pypi-poet..."
-          RESOURCES="$(poet archivebox)"
-          deactivate
-
-          # For CI: use file:// URL pointing to local sdist
-          # For release: this gets overridden with the PyPI URL
-          SDIST_URL="file://${SDIST_PATH}"
-
-          cat > /tmp/archivebox.rb << RUBY
-class Archivebox < Formula
-  include Language::Python::Virtualenv
-
-  desc "Self-hosted internet archiving solution"
-  homepage "https://github.com/ArchiveBox/ArchiveBox"
-  url "${SDIST_URL}"
-  sha256 "${SDIST_SHA256}"
-  license "MIT"
-
-  depends_on "python@3.13"
-
-${RESOURCES}
-
-  def install
-    virtualenv_install_with_resources
-  end
-
-  def post_install
-    data_dir = var/"archivebox"
-    data_dir.mkpath
-    ENV["DATA_DIR"] = data_dir.to_s
-    system bin/"archivebox", "init"
-  end
-
-  def caveats
-    <<~EOS
-      ArchiveBox data is stored in:
-        #{var}/archivebox
-
-      To start archiving, run:
-        cd #{var}/archivebox && archivebox add 'https://example.com'
-
-      To start the web UI:
-        cd #{var}/archivebox && archivebox server 0.0.0.0:8000
-    EOS
-  end
-
-  test do
-    assert_match version.to_s, shell_output("#{bin}/archivebox version")
-  end
-end
-RUBY
-
-          echo "[√] Generated formula:"
-          ruby -c /tmp/archivebox.rb
-          cat /tmp/archivebox.rb
-
-      - name: Install Homebrew (Linux only)
-        if: runner.os == 'Linux'
-        run: |
-          NONINTERACTIVE=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
-          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
-          echo "HOMEBREW_PREFIX=$HOMEBREW_PREFIX" >> "$GITHUB_ENV"
-          echo "HOMEBREW_CELLAR=$HOMEBREW_CELLAR" >> "$GITHUB_ENV"
-          echo "HOMEBREW_REPOSITORY=$HOMEBREW_REPOSITORY" >> "$GITHUB_ENV"
-          echo "$HOMEBREW_PREFIX/bin" >> "$GITHUB_PATH"
-          echo "$HOMEBREW_PREFIX/sbin" >> "$GITHUB_PATH"
-
-      - name: Install brew dependencies
-        run: brew install python@3.13
-
-      - name: Install archivebox via brew from local formula
-        run: |
-          brew install --build-from-source --verbose /tmp/archivebox.rb
-
-      - name: Verify archivebox version
-        run: archivebox version
-
-      - name: Test archivebox init
-        run: |
-          mkdir -p /tmp/archivebox-test && cd /tmp/archivebox-test
-          archivebox init
-
-      - name: Test archivebox status
-        run: |
-          cd /tmp/archivebox-test
-          archivebox status
-
-  # On release only: generate the real formula with PyPI URL and push to tap
-  release:
-    if: github.event_name == 'release' || github.event_name == 'workflow_call'
-    needs: build-and-test
-    runs-on: macos-latest
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Get version
-        id: version
-        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - name: Wait for PyPI package availability
-        run: |
-          VERSION="${{ steps.version.outputs.version }}"
-          echo "[+] Waiting for archivebox==${VERSION} to be available on PyPI..."
-          for i in $(seq 1 30); do
-            if pip index versions archivebox 2>/dev/null | grep -q "$VERSION"; then
-              echo "[√] archivebox==${VERSION} is available on PyPI"
-              break
-            fi
-            if [ "$i" -eq 30 ]; then
-              echo "[!] Timed out waiting for PyPI. Trying to install anyway..."
-              break
-            fi
-            echo "    Attempt $i/30 - not yet available, waiting 30s..."
-            sleep 30
-          done
-
-      - name: Generate release formula via build_brew.sh
-        run: ./bin/build_brew.sh
-
-      - name: Test formula install
-        run: |
-          brew install --build-from-source brew_dist/archivebox.rb
-          archivebox version
-
-      - name: Upload formula artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: archivebox.rb
-          path: brew_dist/archivebox.rb
-
-      - name: Push to homebrew-archivebox tap
-        env:
-          GH_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }}
-        run: |
-          VERSION="${{ steps.version.outputs.version }}"
-          git clone "https://x-access-token:${GH_TOKEN}@github.com/ArchiveBox/homebrew-archivebox.git" /tmp/tap
-
-          cp brew_dist/archivebox.rb /tmp/tap/Formula/archivebox.rb 2>/dev/null || \
-            cp brew_dist/archivebox.rb /tmp/tap/archivebox.rb
-
-          cd /tmp/tap
-          git config user.name "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-          git add -A
-          if git diff --cached --quiet; then
-            echo "[i] No changes to formula, skipping push."
-          else
-            git commit -m "Update archivebox to v${VERSION}"
-            git push origin HEAD
-            echo "[√] Formula pushed to homebrew-archivebox tap"
-          fi
@@ -1,58 +0,0 @@
-# Sample workflow for building and deploying a Jekyll site to GitHub Pages
-name: Build GitHub Pages website
-
-on:
-  # Runs on pushes targeting the default branch
-  push:
-    branches: ["dev"]
-
-  # Allows you to run this workflow manually from the Actions tab
-  workflow_dispatch:
-
-# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
-permissions:
-  contents: read
-  pages: write
-  id-token: write
-
-# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
-# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
-concurrency:
-  group: "pages"
-  cancel-in-progress: true
-
-jobs:
-  # Build job
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          submodules: true
-          fetch-depth: 1
-      - name: Copy README.md into place
-        run: |
-          rm ./website/README.md
-          cp ./README.md ./website/README.md
-      - name: Setup Pages
-        uses: actions/configure-pages@v5
-      - name: Build with Jekyll
-        uses: actions/jekyll-build-pages@v1
-        with:
-          source: ./website
-          destination: ./_site
-      - name: Upload artifact
-        uses: actions/upload-pages-artifact@v3
-
-  # Deployment job
-  deploy:
-    environment:
-      name: github-pages
-      url: ${{ steps.deployment.outputs.page_url }}
-    runs-on: ubuntu-latest
-    needs: build
-    steps:
-      - name: Deploy to GitHub Pages
-        id: deployment
-        uses: actions/deploy-pages@v4
@@ -4,6 +4,14 @@ on:
  workflow_dispatch:
  push:

+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  UV_NO_SOURCES: "1"
+  PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
+
 jobs:
  lint:
    runs-on: ubuntu-22.04
@@ -13,6 +21,15 @@ jobs:
          submodules: true
          fetch-depth: 1

+      - name: Clone abxpkg
+        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
+
+      - name: Clone abx-plugins
+        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+
+      - name: Clone abx-dl
+        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
+
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
@@ -22,10 +39,12 @@ jobs:
      - name: Install uv
        uses: astral-sh/setup-uv@v6
        with:
-          enable-cache: true
+          enable-cache: false

      - name: Install dependencies with uv
-        run: uv sync --all-extras --all-groups --no-sources --no-cache
+        run: |
+          uv venv
+          uv pip install --all-groups -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[sonic,debug]"

      - name: Run prek
-        run: uv run prek run --all-files
+        run: uv run --no-sync --no-sources prek run --all-files
@@ -9,6 +9,10 @@ on:
    tags:
      - 'v*'

+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 env:
  PYTHON_VERSION: "3.13"

@@ -18,13 +22,14 @@ jobs:
      id-token: write

    runs-on: ubuntu-24.04
+    environment: pypi
    steps:
      - uses: actions/checkout@v4

      - name: Install uv
-        uses: astral-sh/setup-uv@v4
+        uses: astral-sh/setup-uv@v6
        with:
-          enable-cache: true
+          enable-cache: false

      - name: Set up Python
        uses: actions/setup-python@v5
@@ -38,8 +43,46 @@ jobs:
          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
          version: 1.0

+      - name: Wait for released ArchiveBox deps on PyPI
+        run: |
+          python - <<'PY'
+          import json
+          import re
+          import sys
+          import time
+          import tomllib
+          import urllib.request
+
+          watched = {"abxbus", "abxpkg", "abx-plugins", "abx-dl"}
+          deps = tomllib.loads(open("pyproject.toml", "rb").read().decode())["project"]["dependencies"]
+          required = {}
+          for dep in deps:
+              for name in watched:
+                  match = re.match(rf"{re.escape(name)}\s*(==|>=)\s*([^,;\s]+)", dep)
+                  if match:
+                      required[name] = match.group(2)
+
+          deadline = time.monotonic() + 300
+          missing = required.copy()
+          while missing and time.monotonic() < deadline:
+              for name, version in list(missing.items()):
+                  with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/json", timeout=20) as resp:
+                      releases = json.load(resp)["releases"]
+                  if version in releases:
+                      print(f"{name} {version} is available on PyPI")
+                      missing.pop(name)
+                  else:
+                      print(f"{name} {version} is not available on PyPI yet")
+              if missing:
+                  time.sleep(10)
+
+          if missing:
+              print(f"Missing PyPI releases after wait: {missing}", file=sys.stderr)
+              sys.exit(1)
+          PY
+
      - name: UV install archivebox dev + run sub-dependencies
-        run: uv sync --frozen --all-extras --no-install-project --no-install-workspace
+        run: uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache

      - name: UV build archivebox and archivebox/pkgs/* packages
        run: |
@@ -53,13 +96,13 @@ jobs:
          password: ${{ secrets.PYPI_PAT_SECRET }}

      - name: UV install archivebox and archivebox/pkgs/* locally for tests
-        run: uv sync --frozen --all-extras
+        run: uv sync --all-extras --no-sources --no-cache

      - name: UV run archivebox init + archivebox version
        run: |
          mkdir -p data && cd data
-          uv run archivebox init \
-          && uv run archivebox version
+          uv run --no-sync --no-sources archivebox init \
+          && uv run --no-sync --no-sources archivebox version
          # && uv run archivebox add 'https://example.com' \
          # && uv run archivebox status \
          # || (echo "UV Failed to run archivebox!" && exit 1)
@@ -10,6 +10,10 @@ permissions:
  contents: write
  id-token: write

+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
  release-state:
    runs-on: ubuntu-latest
@@ -26,7 +30,7 @@ jobs:

      - uses: astral-sh/setup-uv@v6
        with:
-          enable-cache: true
+          enable-cache: false

      - uses: actions/setup-node@v4
        with:
@@ -2,33 +2,32 @@ name: Release

 # Orchestrates the full release pipeline:
 #   1. Build and publish pip package to PyPI
-#   2. Build .deb packages and Homebrew formula (in parallel, after pip)
-#   3. Build Docker images (in parallel with deb/brew)
+#   2. Build Homebrew formula after pip
+#   3. Build Docker images in parallel
 #
 # Individual workflows also run on push for CI (see their own triggers).
-# This workflow ensures the correct ordering during a release.
+# Manual fallback for the hosted release pipeline. The normal prerelease loop
+# is run by bin/release.sh locally so published GitHub releases do not launch a
+# duplicate release job.

 on:
-  release:
-    types: [published]
+  workflow_dispatch:

 permissions:
  contents: write
  packages: write
  id-token: write

+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
  pip:
    name: Publish to PyPI
    uses: ./.github/workflows/pip.yml
    secrets: inherit

-  debian:
-    name: Build .deb packages
-    needs: pip
-    uses: ./.github/workflows/debian.yml
-    secrets: inherit
-
  homebrew:
    name: Update Homebrew formula
    needs: pip
@@ -1,22 +1,29 @@
 name: Parallel Tests

 on:
+  workflow_dispatch:
  pull_request:
    branches: [dev, main, master]
  push:
    branches: [dev]

+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 env:
  PYTHONIOENCODING: utf-8
  PYTHONLEGACYWINDOWSSTDIO: utf-8
  USE_COLOR: False
+  UV_NO_SOURCES: "1"

 jobs:
  discover-tests:
    name: Discover test files
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
    outputs:
      test-files: ${{ steps.set-matrix.outputs.test-files }}
+      plugin-tests: ${{ steps.set-plugin-matrix.outputs.plugin-tests }}

    steps:
      - uses: actions/checkout@v4
@@ -42,8 +49,12 @@ jobs:

            # Extract a display name for the test
            name="main/$(basename $test_file .py | sed 's/^test_//')"
+            needs_chromium=false
+            if grep -Eiq '(chrom|archivewebpage|PLUGINS=.*title|--plugins=.*title|SAVE_TITLE.*[Tt]rue)' "$test_file"; then
+              needs_chromium=true
+            fi

-            json_array+="{\"path\":\"$test_file\",\"name\":\"$name\"}"
+            json_array+="{\"path\":\"$test_file\",\"name\":\"$name\",\"needs_chromium\":$needs_chromium}"
          done
          json_array+="]"

@@ -51,10 +62,46 @@ jobs:
          echo "Found $(echo $all_tests | wc -w) test files"
          echo "$json_array" | jq '.'

+      - name: Clone abx-plugins
+        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+
+      - name: Discover plugin tests
+        id: set-plugin-matrix
+        run: |
+          plugin_tests=$(find abx-plugins/abx_plugins/plugins -maxdepth 2 -type d -name tests | sed 's#abx-plugins/abx_plugins/plugins/##; s#/tests##' | sort)
+
+          json_array="["
+          first=true
+          for plugin_name in $plugin_tests; do
+            if [ "$first" = true ]; then
+              first=false
+            else
+              json_array+=","
+            fi
+
+            needs_chromium=false
+            if grep -Riq "chrom" "abx-plugins/abx_plugins/plugins/$plugin_name"; then
+              needs_chromium=true
+            fi
+
+            json_array+="{\"plugin\":\"$plugin_name\",\"name\":\"plugin/$plugin_name\",\"needs_chromium\":$needs_chromium}"
+          done
+          json_array+="]"
+
+          echo "plugin-tests=$json_array" >> $GITHUB_OUTPUT
+          echo "Found $(echo $plugin_tests | wc -w) plugin test suites"
+          echo "$json_array" | jq '.'
+
  run-tests:
    name: ${{ matrix.test.name }}
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
    needs: discover-tests
+    env:
+      PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
+      CHROME_HEADLESS: "true"
+      CHROME_SANDBOX: "false"
+      PERSONAS_DIR: /tmp/abx-personas
+      CHROME_USER_DATA_DIR: /tmp/abx-personas/Default/chrome_profile

    strategy:
      fail-fast: false
@@ -68,14 +115,23 @@ jobs:
          submodules: true
          fetch-depth: 1

+      - name: Clone abxpkg
+        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
+
+      - name: Clone abx-plugins
+        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+
+      - name: Clone abx-dl
+        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
+
      - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
          architecture: x64

      - name: Install uv
-        uses: astral-sh/setup-uv@v4
+        uses: astral-sh/setup-uv@v6
        with:
          version: "latest"

@@ -85,10 +141,10 @@ jobs:
          node-version: 22

      - name: Cache uv
-        uses: actions/cache@v3
+        uses: actions/cache@v4
        with:
          path: ~/.cache/uv
-          key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml', 'uv.lock') }}
+          key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml') }}
          restore-keys: |
            ${{ runner.os }}-${{ matrix.python }}-uv-

@@ -99,16 +155,70 @@ jobs:

      - name: Install dependencies with uv
        run: |
-          uv sync --dev --all-extras --no-sources
+          uv venv
+          uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
+
+      - name: Install Chrome through ArchiveBox
+        if: ${{ matrix.test.needs_chromium }}
+        env:
+          MIN_CHROMIUM_MAJOR: "149"
+          LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_INSTALL_TIMEOUT: "900"
+          ABX_CI_DATA_DIR: ${{ runner.temp }}/archivebox-ci-data
+        run: |
+          set -euo pipefail
+          mkdir -p "$LIB_DIR" "$ABX_CI_DATA_DIR"
+          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
+          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
+
+          uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox install chrome
+
+          candidate="$(uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="chromium", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
+          if [ ! -x "$candidate" ]; then
+            echo "ArchiveBox did not install an executable Chromium binary: ${candidate:-not found}" >&2
+            exit 1
+          fi
+          version="$("$candidate" --version || true)"
+          major="$(printf '%s\n' "$version" | sed -E 's/.* ([0-9]+)\..*/\1/' | head -1)"
+          case "$major" in
+            ''|*[!0-9]*) major=0 ;;
+          esac
+          if [ "$major" -lt "$MIN_CHROMIUM_MAJOR" ]; then
+            echo "ArchiveBox installed Chromium is too old: ${version:-unknown}" >&2
+            exit 1
+          fi
+          echo "Using ArchiveBox-installed Chromium: $candidate ($version)"
+          echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV"

      - name: Run test - ${{ matrix.test.name }}
        run: |
          mkdir -p tests/out
-          uv run pytest -xvs "${{ matrix.test.path }}" --basetemp=tests/out --ignore=archivebox/pkgs
+          set +e
+          uv run --no-sync --no-sources pytest -xvs "${{ matrix.test.path }}" --basetemp=tests/out --ignore=archivebox/pkgs
+          status=$?
+          set -e
+          if [ "$status" -eq 5 ]; then
+            echo "No tests collected from ${{ matrix.test.path }}; treating as an empty test module."
+            exit 0
+          fi
+          exit "$status"

  plugin-tests:
-    name: Plugin tests
-    runs-on: ubuntu-22.04
+    name: ${{ matrix.plugin.name }}
+    runs-on: ubuntu-24.04
+    needs: discover-tests
+    env:
+      PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
+      CHROME_HEADLESS: "true"
+      CHROME_SANDBOX: "false"
+      PERSONAS_DIR: /tmp/abx-personas
+      CHROME_USER_DATA_DIR: /tmp/abx-personas/Default/chrome_profile
+
+    strategy:
+      fail-fast: false
+      matrix:
+        plugin: ${{ fromJson(needs.discover-tests.outputs.plugin-tests) }}

    steps:
      - uses: actions/checkout@v4
@@ -116,14 +226,23 @@ jobs:
          submodules: true
          fetch-depth: 1

+      - name: Clone abxpkg
+        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
+
+      - name: Clone abx-plugins
+        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+
+      - name: Clone abx-dl
+        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
+
      - name: Set up Python 3.13
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
        with:
          python-version: "3.13"
          architecture: x64

      - name: Install uv
-        uses: astral-sh/setup-uv@v4
+        uses: astral-sh/setup-uv@v6
        with:
          version: "latest"

@@ -133,10 +252,10 @@ jobs:
          node-version: 22

      - name: Cache uv
-        uses: actions/cache@v3
+        uses: actions/cache@v4
        with:
          path: ~/.cache/uv
-          key: ${{ runner.os }}-3.13-uv-${{ hashFiles('pyproject.toml', 'uv.lock') }}
+          key: ${{ runner.os }}-3.13-uv-${{ hashFiles('pyproject.toml') }}
          restore-keys: |
            ${{ runner.os }}-3.13-uv-

@@ -147,12 +266,46 @@ jobs:

      - name: Install dependencies with uv
        run: |
-          uv sync --dev --all-extras --no-sources
+          uv venv
+          uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"

-      - name: Run plugin tests
+      - name: Install Chrome through ArchiveBox
+        if: ${{ matrix.plugin.needs_chromium }}
+        env:
+          MIN_CHROMIUM_MAJOR: "149"
+          LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_INSTALL_TIMEOUT: "900"
+          ABX_CI_DATA_DIR: ${{ runner.temp }}/archivebox-ci-data
+        run: |
+          set -euo pipefail
+          mkdir -p "$LIB_DIR" "$ABX_CI_DATA_DIR"
+          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
+          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
+
+          uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox install chrome
+
+          candidate="$(uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="chromium", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
+          if [ ! -x "$candidate" ]; then
+            echo "ArchiveBox did not install an executable Chromium binary: ${candidate:-not found}" >&2
+            exit 1
+          fi
+          version="$("$candidate" --version || true)"
+          major="$(printf '%s\n' "$version" | sed -E 's/.* ([0-9]+)\..*/\1/' | head -1)"
+          case "$major" in
+            ''|*[!0-9]*) major=0 ;;
+          esac
+          if [ "$major" -lt "$MIN_CHROMIUM_MAJOR" ]; then
+            echo "ArchiveBox installed Chromium is too old: ${version:-unknown}" >&2
+            exit 1
+          fi
+          echo "Using ArchiveBox-installed Chromium: $candidate ($version)"
+          echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV"
+
+      - name: Run plugin tests - ${{ matrix.plugin.name }}
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
          API_KEY_2CAPTCHA: ${{ secrets.TWOCAPTCHA_API_KEY }}
        run: |
-          uv run bash ./bin/test_plugins.sh --no-coverage
+          uv run --no-sync --no-sources bash ./bin/test_plugins.sh "${{ matrix.plugin.plugin }}" --no-coverage
@@ -1,15 +1,22 @@
 name: Run tests
 on: [push]

+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 env:
  DOCKER_IMAGE: archivebox-ci
  PYTHONIOENCODING: utf-8
  PYTHONLEGACYWINDOWSSTDIO: utf-8
  USE_COLOR: False
+  UV_NO_SOURCES: "1"

 jobs:
  python_tests:
    runs-on: ${{ matrix.os }}
+    env:
+      PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl

    strategy:
      matrix:
@@ -23,6 +30,15 @@ jobs:
          submodules: true
          fetch-depth: 1

+      - name: Clone abxpkg
+        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
+
+      - name: Clone abx-plugins
+        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+
+      - name: Clone abx-dl
+        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
+
      ### Setup Python & JS Languages
      - name: Set up Python ${{ matrix.python }}
        uses: actions/setup-python@v4
@@ -45,7 +61,7 @@ jobs:
        uses: actions/cache@v3
        with:
          path: ~/.cache/uv
-          key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml', 'uv.lock') }}
+          key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml') }}
          restore-keys: |
            ${{ runner.os }}-${{ matrix.python }}-uv-

@@ -56,7 +72,8 @@ jobs:

      - name: Install dependencies with uv
        run: |
-          uv sync --dev --all-extras --no-sources
+          uv venv
+          uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"

      ### Run the tests
      - name: Directory listing for debugging
@@ -67,14 +84,14 @@ jobs:
      - name: Archivebox version
        run: |
          mkdir -p tests/out/data
-          DATA_DIR="$PWD/tests/out/data" uv run archivebox version
+          DATA_DIR="$PWD/tests/out/data" uv run --no-sync --no-sources archivebox version

      - name: Test built package with pytest
        # TODO: remove this exception for windows once we get tests passing on that platform
        if: ${{ !contains(matrix.os, 'windows') }}
        run: |
          mkdir -p tests/out
-          uv run pytest -s archivebox/tests --basetemp=tests/out --ignore=archivebox/pkgs
+          uv run --no-sync --no-sources pytest -s archivebox/tests --basetemp=tests/out --ignore=archivebox/pkgs

      - name: Run plugin tests
        if: ${{ !contains(matrix.os, 'windows') }}
@@ -83,7 +100,7 @@ jobs:
          TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
          API_KEY_2CAPTCHA: ${{ secrets.TWOCAPTCHA_API_KEY }}
        run: |
-          uv run bash ./bin/test_plugins.sh --no-coverage
+          uv run --no-sync --no-sources bash ./bin/test_plugins.sh --no-coverage

  docker_tests:
    runs-on: ubuntu-latest
@@ -0,0 +1,35 @@
+name: Update Homebrew tap
+
+on:
+  push:
+    branches: [dev]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: update-homebrew-tap-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  dispatch:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Dispatch tap update
+        env:
+          HOMEBREW_TAP_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }}
+        run: |
+          if [ -z "$HOMEBREW_TAP_TOKEN" ]; then
+            echo "HOMEBREW_TAP_TOKEN is not configured; tap hourly fallback will update the formula."
+            exit 0
+          fi
+
+          curl --fail-with-body \
+            -X POST \
+            -H "Accept: application/vnd.github+json" \
+            -H "Authorization: Bearer ${HOMEBREW_TAP_TOKEN}" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            https://api.github.com/repos/ArchiveBox/homebrew-archivebox/dispatches \
+            -d '{"event_type":"archivebox-dev-updated"}'
@@ -1,5 +1,8 @@
 .DS_Store
 ._*
+/.heartbeat.json
+/package.json
+/package-lock.json
 *.pyc
 __pycache__/
 .mypy_cache/
@@ -35,6 +38,7 @@ dist/

 # Data folders
 lib/
+out/
 tmp/
 data/
 data*/
@@ -42,6 +46,9 @@ archivebox/tests/data/
 archive/
 output/
 logs/
+/-
+/personas/
+/sources/
 index.sqlite3
 queue.sqlite3
 *.sqlite*
@@ -56,3 +63,6 @@ ArchiveBox.conf
 # vim
 *.sw?
 .vscode
+
+# Local Claude Code task locks (not project state)
+.claude/scheduled_tasks.lock
@@ -1,3 +0,0 @@
-[submodule "docs"]
-    path = docs
-    url = https://github.com/ArchiveBox/ArchiveBox.wiki.git
@@ -29,12 +29,12 @@ repos:
    hooks:
      - id: ruff-format
        name: ruff-format
-        entry: uv run --active ruff format
+        entry: uv run --active --no-sync --no-sources ruff format
        language: system
        types_or: [python, pyi]
      - id: ruff-check
        name: ruff-check
-        entry: uv run --active ruff check --fix
+        entry: uv run --active --no-sync --no-sources ruff check --fix
        language: system
        types_or: [python, pyi]

@@ -1,34 +1,152 @@
-# This is the Dockerfile for ArchiveBox, it bundles the following main dependencies:
-#     python3.13, uv, python3-ldap
-#     curl, wget, git, dig, ping, tree, nano
-#     node, npm, single-file, readability-extractor, postlight-parser
-#     ArchiveBox, yt-dlp, playwright, chromium
-# Usage:
-#     git clone https://github.com/ArchiveBox/ArchiveBox && cd ArchiveBox
-#     docker build . -t archivebox
-#     docker run -v "$PWD/data":/data archivebox init
-#     docker run -v "$PWD/data":/data archivebox add 'https://example.com'
-#     docker run -v "$PWD/data":/data -it archivebox manage createsuperuser
-#     docker run -v "$PWD/data":/data -p 8000:8000 archivebox server
-# Multi-arch build:
-#     docker buildx create --use
-#     docker buildx build . --platform=linux/amd64,linux/arm64 --push -t archivebox/archivebox:dev -t archivebox/archivebox:sha-abc123
-# Read more here: https://github.com/ArchiveBox/ArchiveBox#archivebox-development
+# syntax=docker/dockerfile:1.7

+# Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image.
+# abx-dl owns Python, Node, Chromium, and downloader plugin runtimes.
+# ArchiveBox owns ripgrep, sonic, supervisor, Django, and the app runtime.
+# Build abx-dl first, then point this file at it:
+#   docker buildx build ../abx-dl -f ../abx-dl/Dockerfile \
+#       --build-context abxbus=../abxbus \
+#       --build-context abxpkg=../abxpkg \
+#       --build-context abx-plugins=../abx-plugins \
+#       -t archivebox/abx-dl:dev
+#   docker buildx build . -f Dockerfile \
+#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:latest \
+#       -t archivebox:multistage

-#########################################################################################
+ARG ABX_DL_IMAGE=archivebox/abx-dl:latest

-### Example: Using ArchiveBox in your own project's Dockerfile ########
+FROM archivebox/sonic:1.4.9 AS sonic
+FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base

-# FROM python:3.13-slim
-# WORKDIR /data
-# RUN pip install archivebox>=0.9.0   # use latest release here
-# RUN archivebox install
-# RUN useradd -ms /bin/bash archivebox && chown -R archivebox /data
+ARG TARGETPLATFORM
+ARG TARGETOS
+ARG TARGETARCH
+ARG TARGETVARIANT

-#########################################################################################
+ENV TZ=UTC \
+    LANGUAGE=en_US:en \
+    LC_ALL=C.UTF-8 \
+    LANG=C.UTF-8 \
+    DEBIAN_FRONTEND=noninteractive \
+    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
+    PYTHONIOENCODING=UTF-8 \
+    PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    PIP_NO_COMPILE=1 \
+    PIP_ONLY_BINARY=aiohttp \
+    npm_config_loglevel=error

-FROM ubuntu:24.04
+ENV PYTHON_VERSION=3.13 \
+    NODE_VERSION=24
+
+ENV ARCHIVEBOX_USER=archivebox \
+    DEFAULT_PUID=911 \
+    DEFAULT_PGID=911 \
+    IN_DOCKER=True
+
+ENV CODE_DIR=/app \
+    DATA_DIR=/data \
+    LIB_DIR=/opt/archivebox/lib \
+    ABXPKG_LIB_DIR=/opt/archivebox/lib \
+    PLAYWRIGHT_BROWSERS_PATH=/opt/archivebox/lib/playwright/cache \
+    PERSONAS_DIR=/data/personas \
+    CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \
+    CHROME_HEADLESS=true \
+    CHROME_SANDBOX=false \
+    CHROME_ISOLATION=crawl \
+    CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
+
+ENV TMP_DIR=/tmp/archivebox \
+    PIP_VENV_PYTHON=/venv/bin/python3 \
+    GOOGLE_API_KEY=no \
+    GOOGLE_DEFAULT_CLIENT_ID=no \
+    GOOGLE_DEFAULT_CLIENT_SECRET=no
+
+ENV HOME=/home/archivebox \
+    XDG_CONFIG_HOME=/home/archivebox/.config \
+    XDG_CACHE_HOME=/home/archivebox/.cache \
+    ABXPKG_INSTALL_TIMEOUT=600 \
+    ABXPKG_POSTINSTALL_SCRIPTS=True \
+    ABXPKG_MIN_RELEASE_AGE=0 \
+    TIMEOUT=600
+
+ENV UV_COMPILE_BYTECODE=false \
+    UV_PYTHON_PREFERENCE=managed \
+    UV_PYTHON_INSTALL_DIR=/opt/uv/python \
+    ARCHIVEBOX_PYTHON_INSTALL_DIR=/opt/archivebox/python \
+    UV_LINK_MODE=copy \
+    UV_PROJECT_ENVIRONMENT=/venv \
+    VIRTUAL_ENV=/venv \
+    PATH="/venv/bin:/opt/node/bin:/opt/archivebox/lib/bin:$PATH"
+
+SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
+WORKDIR "$CODE_DIR"
+
+RUN cp /VERSION.txt /ABX-DL-VERSION.txt \
+    && (echo "[i] Docker build for ArchiveBox multistage starting..." \
+    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
+    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
+    && uname -a \
+    && sed -n '1,7p' /etc/os-release \
+    && which node && node --version \
+    && which uv && uv self version \
+    ) | tee -a /VERSION.txt
+
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    echo "[+] APT Installing ArchiveBox search dependency ripgrep for $TARGETPLATFORM..." \
+    && apt-get update -qq \
+    && apt-get install -qq -y --no-install-recommends ripgrep \
+    && rm -rf /var/lib/apt/lists/*
+
+FROM archivebox-runtime-base AS archivebox-builder
+
+WORKDIR "$CODE_DIR"
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
+    echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..." \
+    && export UV_PYTHON_INSTALL_DIR="$ARCHIVEBOX_PYTHON_INSTALL_DIR" \
+    && echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
+    && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends \
+    && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests \
+    && rm -f /etc/apt/apt.conf.d/docker-clean \
+    && apt-get update -qq \
+    && apt-get install -qq -y --no-install-recommends \
+        build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
+    && /usr/bin/uv venv --clear /venv --python "${PYTHON_VERSION}" \
+    && /usr/bin/uv pip install setuptools pip wheel \
+    && /usr/bin/uv sync \
+        --refresh \
+        --no-dev \
+        --inexact \
+        --all-extras \
+        --no-install-project \
+        --no-install-workspace \
+        --no-sources \
+    && apt-get purge -y build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
+    && apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
+RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
+    && COMMIT_HASH="$( \
+        if [[ -f "$CODE_DIR/.git/HEAD" ]]; then \
+            HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"; \
+            if [[ "$HEAD_REF" =~ ^[0-9a-fA-F]{40}$ ]]; then \
+                echo "$HEAD_REF"; \
+            elif [[ "$HEAD_REF" == ref:\ * ]]; then \
+                REF_PATH="${HEAD_REF#ref: }"; \
+                cat "$CODE_DIR/.git/$REF_PATH" 2>/dev/null || awk -v ref="$REF_PATH" '$2 == ref {print $1}' "$CODE_DIR/.git/packed-refs" 2>/dev/null || true; \
+            fi; \
+        fi)" \
+    && if [[ "$COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt; fi \
+    && /usr/bin/uv pip install --no-deps "$CODE_DIR" \
+    && (/usr/bin/uv pip show archivebox && which archivebox) | tee -a /VERSION.txt \
+    && rm -rf "$CODE_DIR/.git"
+
+FROM archivebox-runtime-base

 LABEL name="archivebox" \
    maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
@@ -39,358 +157,74 @@ LABEL name="archivebox" \
    org.opencontainers.image.vendor="ArchiveBox" \
    org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \
    org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \
-    com.docker.image.source.entrypoint="Dockerfile" \
-    # TODO: release ArchiveBox as a Docker Desktop extension (requires these labels):
-    # https://docs.docker.com/desktop/extensions-sdk/architecture/metadata/
-    com.docker.desktop.extension.api.version=">= 1.4.7" \
-    com.docker.desktop.extension.icon="https://archivebox.io/icon.png" \
-    com.docker.extension.publisher-url="https://archivebox.io" \
-    com.docker.extension.screenshots='[{"alt": "Screenshot of Admin UI", "url": "https://github.com/ArchiveBox/ArchiveBox/assets/511499/e8e0b6f8-8fdf-4b7f-8124-c10d8699bdb2"}]' \
-    com.docker.extension.detailed-description='See here for detailed documentation: https://wiki.archivebox.io' \
-    com.docker.extension.changelog='See here for release notes: https://github.com/ArchiveBox/ArchiveBox/releases' \
-    com.docker.extension.categories='database,utility-tools'
+    com.docker.image.source.entrypoint="Dockerfile"

-ARG TARGETPLATFORM
-ARG TARGETOS
-ARG TARGETARCH
-ARG TARGETVARIANT
-######### Environment Variables #################################
-
-# Global build-time and runtime environment constants + default pkg manager config
-ENV TZ=UTC \
-    LANGUAGE=en_US:en \
-    LC_ALL=C.UTF-8 \
-    LANG=C.UTF-8 \
-    DEBIAN_FRONTEND=noninteractive \
-    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
-    PYTHONIOENCODING=UTF-8 \
-    PYTHONUNBUFFERED=1 \
-    PIP_DISABLE_PIP_VERSION_CHECK=1 \
-    npm_config_loglevel=error
-
-# Language Version config
-ENV PYTHON_VERSION=3.13 \
-    NODE_VERSION=22
-
-# Non-root User config
-ENV ARCHIVEBOX_USER="archivebox" \
-    DEFAULT_PUID=911 \
-    DEFAULT_PGID=911 \
-    IN_DOCKER=True
-
-# ArchiveBox Source Code + Lib + Data paths
-ENV CODE_DIR=/app \
-    DATA_DIR=/data \
-    PLAYWRIGHT_BROWSERS_PATH=/browsers
-
-# Bash SHELL config
-# http://redsymbol.net/articles/unofficial-bash-strict-mode/
-SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] 
-
-######### System Environment ####################################
-
-# Detect ArchiveBox version number by reading pyproject.toml (also serves to invalidate the entire build cache whenever pyproject.toml changes)
-WORKDIR "$CODE_DIR"
-
-# Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up back-to-back Docker builds)
-RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
-    && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-intall-recommends \
-    && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-intall-suggests \
-    && rm -f /etc/apt/apt.conf.d/docker-clean
-
-# Print debug info about build and save it to disk, for human eyes only, not used by anything else
-RUN (echo "[i] Docker build for ArchiveBox starting..." \
-    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})" \
-    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
-    && echo \
-    && echo "PYTHON=${PYTHON_VERSION} NODE=${NODE_VERSION} PATH=${PATH}" \
-    && echo "CODE_DIR=${CODE_DIR} DATA_DIR=${DATA_DIR}" \
-    && echo \
-    && uname -a \
-    && cat /etc/os-release | head -n7 \
-    && which bash && bash --version | head -n1 \
-    && which dpkg && dpkg --version | head -n1 \
-    && echo -e '\n\n' && env && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
-# Create non-privileged user for archivebox and chrome
-RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
-    && groupadd --system $ARCHIVEBOX_USER \
-    && useradd --system --create-home --gid $ARCHIVEBOX_USER --groups audio,video $ARCHIVEBOX_USER \
-    && usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
-    && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
-    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u $ARCHIVEBOX_USER) PGID=$(id -g $ARCHIVEBOX_USER)\n\n" \
-    | tee -a /VERSION.txt
-    # DEFAULT_PUID and DEFAULT_PID are overridden by PUID and PGID in /bin/docker_entrypoint.sh at runtime
-    # https://docs.linuxserver.io/general/understanding-puid-and-pgid
-
-# Install system apt dependencies (adding backports to access more recent apt updates)
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    echo "[+] APT Installing base system dependencies for $TARGETPLATFORM..." \
-    && mkdir -p /etc/apt/keyrings \
-    && apt-get update -qq \
-    && apt-get install -qq -y \
-        # 1. packaging dependencies
-        apt-transport-https ca-certificates apt-utils gnupg2 curl wget \
-        # 2. docker and init system dependencies
-        zlib1g-dev dumb-init gosu cron unzip grep dnsutils \
-        # 3. frivolous CLI helpers to make debugging failed archiving easier
-        tree nano iputils-ping \
-        # nano iputils-ping dnsutils htop procps jq yq
-    && rm -rf /var/lib/apt/lists/*
-
-# Install apt binary dependencies for extractors
-# COPY --from=selenium/ffmpeg:latest /usr/local/bin/ffmpeg /usr/local/bin/ffmpeg
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    echo "[+] APT Installing extractor dependencies for $TARGETPLATFORM..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends \
-        git ripgrep \
-        # Packages we have also needed in the past:
-        # youtube-dl wget2 aria2 python3-pyxattr rtmpdump libfribidi-bin mpv \
-        # curl wget (already installed above)
-    && rm -rf /var/lib/apt/lists/* \
-    # Save version info
-    && ( \
-        which curl && curl --version | head -n1 \
-        && which wget && wget --version 2>&1 | head -n1 \
-        && which git && git --version 2>&1 | head -n1 \
-        # && which ffmpeg && (ffmpeg --version 2>&1 | head -n1) || true \
-        && which rg && rg --version 2>&1 | head -n1 \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
-# Install sonic search backend
-COPY --from=archivebox/sonic:1.4.9 /usr/local/bin/sonic /usr/local/bin/sonic
+COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
 COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
-RUN (which sonic && sonic --version) | tee -a /VERSION.txt

-######### Language Environments ####################################
+COPY --from=archivebox-builder /opt/archivebox/python /opt/archivebox/python
+COPY --from=archivebox-builder /venv /venv
+COPY --from=archivebox-builder /app /app
+COPY --from=archivebox-builder /VERSION.txt /VERSION.txt

-# Set up Python environment
-# NOT NEEDED because we're using a pre-built python image, keeping this here in case we switch back to custom-building our own:
-#RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-#    --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-# RUN echo "[+] APT Installing PYTHON $PYTHON_VERSION for $TARGETPLATFORM (skipped, provided by base image)..." \
-    # && apt-get update -qq \
-    # && apt-get install -qq -y --no-upgrade \
-    #     python${PYTHON_VERSION} python${PYTHON_VERSION}-minimal python3-pip python${PYTHON_VERSION}-venv pipx \
-    # && rm -rf /var/lib/apt/lists/* \
-    # tell PDM to allow using global system python site packages
-    # && rm /usr/lib/python3*/EXTERNALLY-MANAGED \
-    # && ln -s "$(which python${PYTHON_VERSION})" /usr/bin/python \
-    # create global virtual environment GLOBAL_VENV to use (better than using pip install --global)
-    # && python3 -m venv --system-site-packages --symlinks $GLOBAL_VENV \
-    # && python3 -m venv --system-site-packages $GLOBAL_VENV \
-    # && python3 -m venv $GLOBAL_VENV \
-    # install global dependencies / python build dependencies in GLOBAL_VENV
-    # && pip install --upgrade pip setuptools wheel \
-    # Save version info
-    # && ( \
-    #     which python3 && python3 --version | grep " $PYTHON_VERSION" \
-    #     && which pip && pip --version \
-    #     # && which pdm && pdm --version \
-    #     && echo -e '\n\n' \
-    # ) | tee -a /VERSION.txt
-
-
-# Set up Node environment
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
-    echo "[+] APT Installing NODE $NODE_VERSION for $TARGETPLATFORM..." \
-    && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_VERSION}.x nodistro main" >> /etc/apt/sources.list.d/nodejs.list \
-    && curl -fsSL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
-    && apt-get update -qq \
-    && apt-get install -qq -y --no-upgrade libatomic1 \
-    && apt-get install -y --no-upgrade \
-        nodejs \
-    && rm -rf /var/lib/apt/lists/* \
-    # Update NPM to latest version
-    && npm i -g npm --cache /root/.npm \
-    # Save version info
-    && ( \
-        which node && node --version \
-        && which npm && npm --version \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
-
-# Set up uv and main app /venv
-RUN curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR=/bin sh
-ENV UV_COMPILE_BYTECODE=1 \
-    UV_PYTHON_PREFERENCE=managed \
-    UV_PYTHON_INSTALL_DIR=/opt/uv/python \
-    UV_LINK_MODE=copy \
-    UV_PROJECT_ENVIRONMENT=/venv
-WORKDIR "$CODE_DIR"
-# COPY --chown=root:root --chmod=755 pyproject.toml "$CODE_DIR/"
-RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[+] UV Creating /venv using python ${PYTHON_VERSION} for ${TARGETPLATFORM}..." \
-    && uv venv /venv --python ${PYTHON_VERSION}
-ENV VIRTUAL_ENV=/venv PATH="/venv/bin:$PATH"
-RUN uv pip install setuptools pip \
-    && ( \
-        which python3 && python3 --version \
-        && which uv && uv self version \
-        && uv python find --system && uv python find \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
-
-######### ArchiveBox & Extractor Dependencies ##################################
-
-# Install ArchiveBox C-compiled/apt-installed Python dependencies in app /venv (currently only used for python-ldap)
-WORKDIR "$CODE_DIR"
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    #--mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-    echo "[+] APT Installing + Compiling python3-ldap for PIP archivebox[ldap] on ${TARGETPLATFORM}..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends \
-        build-essential gcc \
-        python3-dev libssl-dev libldap2-dev libsasl2-dev python3-ldap \
-        python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \
-    && uv pip install \
-        "python-ldap>=3.4.3" \
-    && apt-get purge -y \
-        python3-dev build-essential gcc \
-    && apt-get autoremove -y \
-    && rm -rf /var/lib/apt/lists/*
-
-
-# Install apt font & rendering dependencies for chromium browser
-# TODO: figure out how much of this overlaps with `playwright install-deps chromium`
-# RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-
-# Install chromium browser binary using playwright
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[+] APT Installing CHROMIUM dependencies, fonts, and display libraries for $TARGETPLATFORM..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y \
-        #fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-khmeros fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
-        #at-spi2-common fonts-liberation fonts-noto-color-emoji fonts-tlwg-loma-otf fonts-unifont libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libavahi-client3 \
-        #libavahi-common-data libavahi-common3 libcups2 libfontenc1 libice6 libnspr4 libnss3 libsm6 libunwind8 \
-        #libxaw7 libxcomposite1 libxdamage1 libxfont2 \
-        libxkbfile1 libxmu6 libxpm4 libxt6 x11-xkb-utils x11-utils xfonts-encodings \
-        # xfonts-scalable xfonts-utils xserver-common xvfb \
-        # chrome can run without dbus/upower technically, it complains about missing dbus but should run ok anyway
-        # libxss1 dbus dbus-x11 upower \
-    # && service dbus start \
-    && echo "[+] PIP Installing playwright into /venv and CHROMIUM binary into $PLAYWRIGHT_BROWSERS_PATH..." \
-    && uv pip install "playwright>=1.49.1" \
-    && uv run playwright install chromium --no-shell --with-deps \
-    && export CHROME_BINARY="$(uv run python -c 'from playwright.sync_api import sync_playwright; print(sync_playwright().start().chromium.executable_path)')" \
-    && ln -s "$CHROME_BINARY" /usr/bin/chromium-browser \
-    && ln -s /browsers/ffmpeg-*/ffmpeg-linux /usr/bin/ffmpeg \
-    && mkdir -p "/home/${ARCHIVEBOX_USER}/.config/chromium/Crash Reports/pending/" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "/home/${ARCHIVEBOX_USER}/.config" \
-    && mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \
-    && chown -R $ARCHIVEBOX_USER "$PLAYWRIGHT_BROWSERS_PATH" \
-    # delete extra full copy of node that playwright installs (saves >100mb)
-    && rm -f /venv/lib/python$PYTHON_VERSION/site-packages/playwright/driver/node \
-    # Save version info
-    && rm -rf /var/lib/apt/lists/* \
-    && ( \
-        uv pip show playwright \
-        && which chromium-browser && /usr/bin/chromium-browser --version || /usr/lib/chromium/chromium --version \
-        && which ffmpeg && ffmpeg -version \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
-# Install Node extractor dependencies
-ENV PATH="/home/$ARCHIVEBOX_USER/.npm/bin:$PATH"
-USER $ARCHIVEBOX_USER
-WORKDIR "/home/$ARCHIVEBOX_USER/.npm"
-RUN --mount=type=cache,target=/home/archivebox/.npm_cache,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT,uid=$DEFAULT_PUID,gid=$DEFAULT_PGID \
-    echo "[+] NPM Installing node extractor dependencies into /home/$ARCHIVEBOX_USER/.npm..." \
-    && npm config set prefix "/home/$ARCHIVEBOX_USER/.npm" \
-    && npm install --global --prefer-offline --no-fund --no-audit --cache "/home/$ARCHIVEBOX_USER/.npm_cache" \
-        "@postlight/parser@^2.2.3" \
-        "readability-extractor@github:ArchiveBox/readability-extractor" \
-        "single-file-cli@^1.1.54" \
-        "puppeteer@^23.5.0" \
-        "@puppeteer/browsers@^2.4.0" \
-    && rm -Rf "/home/$ARCHIVEBOX_USER/.cache/puppeteer"
-USER root
-WORKDIR "$CODE_DIR"
-RUN ( \
-        which node && node --version \
-        && which npm && npm version \
-        && which postlight-parser \
-        && which readability-extractor && readability-extractor --version \
-        && which single-file && single-file --version \
-        && which puppeteer && puppeteer --version \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
-######### Build Dependencies ####################################
-
-
-# Install ArchiveBox Python venv dependencies from uv.lock
-RUN --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
-    --mount=type=bind,source=uv.lock,target=/app/uv.lock \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[+] PIP Installing ArchiveBox dependencies from pyproject.toml and uv.lock..." \
-    && uv sync \
-        --frozen \
-        --inexact \
-        --all-extras \
-        --no-install-project \
-        --no-install-workspace
-    # installs the pip packages that archivebox depends on, defined in pyproject.toml and uv.lock dependencies
-
-# Install ArchiveBox Python package + workspace dependencies from source
-COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
-RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
-    && pip install \
-        --no-deps \
-        "$CODE_DIR" \
-    && ( \
-        pip show archivebox \
-        && which archivebox \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-    # installs archivebox itself, and any other vendored packages in pkgs/*, defined in pyproject.toml workspaces
-
-####################################################
-
-# Setup ArchiveBox runtime config
-ENV TMP_DIR=/tmp/archivebox \
-    LIB_DIR=/usr/share/archivebox/lib \
-    GOOGLE_API_KEY=no \
-    GOOGLE_DEFAULT_CLIENT_ID=no \
-    GOOGLE_DEFAULT_CLIENT_SECRET=no
+RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
+    && printf 'export PATH="/venv/bin:/opt/node/bin:/opt/archivebox/lib/bin:$PATH"\n' > /etc/profile.d/archivebox-path.sh \
+    && ln -sf /venv/bin/archivebox /usr/local/bin/archivebox \
+    && ln -sf /venv/bin/daphne /usr/local/bin/daphne \
+    && ln -sf /venv/bin/supervisord /usr/local/bin/supervisord \
+    && ln -sf /venv/bin/supervisorctl /usr/local/bin/supervisorctl \
+    && getent group "$ARCHIVEBOX_USER" >/dev/null || groupadd --system "$ARCHIVEBOX_USER" \
+    && id -u "$ARCHIVEBOX_USER" >/dev/null 2>&1 || useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \
+    && usermod --append --groups audio,video "$ARCHIVEBOX_USER" \
+    && [[ "$(id -u "$ARCHIVEBOX_USER")" == "$DEFAULT_PUID" ]] || usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
+    && [[ "$(id -g "$ARCHIVEBOX_USER")" == "$DEFAULT_PGID" ]] || groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
+    && (which sonic && sonic --version) | tee -a /VERSION.txt \
+    && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
+    && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER" "/home/$ARCHIVEBOX_USER/.config" "/home/$ARCHIVEBOX_USER/.cache" \
+    && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER/.config/abx" "/home/$ARCHIVEBOX_USER/.cache/abxbus" "/home/$ARCHIVEBOX_USER/.cache/pnpm" "/home/$ARCHIVEBOX_USER/.cache/uv" \
+    && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
+    && openssl rand -hex 16 > /etc/machine-id \
+    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u "$ARCHIVEBOX_USER") PGID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
+    && echo -e "TMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt

 WORKDIR "$DATA_DIR"
-RUN openssl rand -hex 16 > /etc/machine-id \
-    && mkdir -p "$TMP_DIR" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$TMP_DIR" \
-    && mkdir -p "$LIB_DIR" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
-    && echo -e "\nTMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
+RUN echo "[+] Initializing image collection..." \
+    && find "$DATA_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} + \
+    && PUID=0 PGID=0 archivebox init \
+    && (chown "$DEFAULT_PUID:$DEFAULT_PGID" \
+        "$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \
+        "$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \
+        "$DATA_DIR"/archive "$DATA_DIR"/archive/users "$DATA_DIR"/personas \
+        "$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \
+        "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
+        2>/dev/null || true) \
+    && find "$TMP_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} +

-# Print version for nice docker finish summary
-RUN (echo -e "\n\n[√] Finished Docker build successfully. Saving build summary in: /VERSION.txt" \
-    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})\n" \
+RUN chmod +x "$CODE_DIR"/bin/*.sh \
+    && chmod g+w "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"
+
+RUN "$LIB_DIR/bin/chromium" --version | tee -a /VERSION.txt \
+    && "$LIB_DIR/uv/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \
+    && /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \
+    && /usr/local/bin/sonic --version | tee -a /VERSION.txt \
+    && /venv/bin/supervisord --version | tee -a /VERSION.txt \
+    && for forbidden_bin in gcc g++ make; do ! command -v "$forbidden_bin" || (echo "Unexpected build tool in runtime: $forbidden_bin=$(command -v "$forbidden_bin")" >&2 && exit 1); done \
+    && stat -c "%U:%G %a %n" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$LIB_DIR" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \
+    && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
+
+RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully." \
+    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
    && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
    ) | tee -a /VERSION.txt

-# Verify ArchiveBox is installed and print version info
-RUN chmod +x "$CODE_DIR"/bin/*.sh \
-    && gosu "$DEFAULT_PUID" archivebox version 2>&1 | tee -a /VERSION.txt || true
-
-####################################################
-
-# Expose ArchiveBox's main interfaces to the outside world
 WORKDIR "$DATA_DIR"
 VOLUME "$DATA_DIR"
 EXPOSE 8000

 HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
-    CMD curl --silent 'http://admin.archivebox.localhost:8000/health/' | grep -q 'OK'
+    CMD curl --fail --silent --show-error --max-time 5 --connect-timeout 2 'http://admin.archivebox.localhost:8000/health/' | grep -q 'OK'

 ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh"]
 CMD ["archivebox", "server", "--init", "0.0.0.0:8000"]
@@ -0,0 +1,224 @@
+# syntax=docker/dockerfile:1.7
+
+# Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image.
+# abx-dl owns Python, Node, Chromium, and downloader plugin runtimes.
+# ArchiveBox owns ripgrep, sonic, supervisor, Django, and the app runtime.
+# Build abx-dl first, then point this file at it:
+#   docker buildx build ../abx-dl -f ../abx-dl/Dockerfile \
+#       --build-context abxbus=../abxbus \
+#       --build-context abxpkg=../abxpkg \
+#       --build-context abx-plugins=../abx-plugins \
+#       -t archivebox/abx-dl:dev
+#   docker buildx build . -f Dockerfile \
+#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:latest \
+#       -t archivebox:multistage
+
+ARG ABX_DL_IMAGE=archivebox/abx-dl:latest
+
+FROM archivebox/sonic:1.4.9 AS sonic
+FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base
+
+ARG TARGETPLATFORM
+ARG TARGETOS
+ARG TARGETARCH
+ARG TARGETVARIANT
+
+ENV TZ=UTC \
+    LANGUAGE=en_US:en \
+    LC_ALL=C.UTF-8 \
+    LANG=C.UTF-8 \
+    DEBIAN_FRONTEND=noninteractive \
+    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
+    PYTHONIOENCODING=UTF-8 \
+    PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    PIP_NO_COMPILE=1 \
+    PIP_ONLY_BINARY=aiohttp \
+    npm_config_loglevel=error
+
+ENV PYTHON_VERSION=3.13 \
+    NODE_VERSION=22.22.3
+
+ENV ARCHIVEBOX_USER=archivebox \
+    DEFAULT_PUID=911 \
+    DEFAULT_PGID=911 \
+    IN_DOCKER=True
+
+ENV CODE_DIR=/app \
+    DATA_DIR=/data \
+    LIB_DIR=/opt/archivebox/lib \
+    ABXPKG_LIB_DIR=/opt/archivebox/lib \
+    PLAYWRIGHT_BROWSERS_PATH=/opt/archivebox/lib/playwright/cache \
+    PERSONAS_DIR=/data/personas \
+    CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \
+    CHROME_HEADLESS=true \
+    CHROME_SANDBOX=false \
+    CHROME_ISOLATION=crawl \
+    CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
+
+ENV TMP_DIR=/tmp/archivebox \
+    PIP_VENV_PYTHON=/venv/bin/python3 \
+    GOOGLE_API_KEY=no \
+    GOOGLE_DEFAULT_CLIENT_ID=no \
+    GOOGLE_DEFAULT_CLIENT_SECRET=no
+
+ENV HOME=/home/archivebox \
+    XDG_CONFIG_HOME=/home/archivebox/.config \
+    XDG_CACHE_HOME=/home/archivebox/.cache \
+    ABXPKG_INSTALL_TIMEOUT=600 \
+    ABXPKG_POSTINSTALL_SCRIPTS=True \
+    ABXPKG_MIN_RELEASE_AGE=0 \
+    TIMEOUT=600
+
+ENV UV_COMPILE_BYTECODE=false \
+    UV_PYTHON_PREFERENCE=managed \
+    UV_PYTHON_INSTALL_DIR=/opt/uv/python \
+    ARCHIVEBOX_PYTHON_INSTALL_DIR=/opt/archivebox/python \
+    UV_LINK_MODE=copy \
+    UV_PROJECT_ENVIRONMENT=/venv \
+    VIRTUAL_ENV=/venv \
+    PATH="/venv/bin:/opt/node/bin:/opt/archivebox/lib/bin:$PATH"
+
+SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
+WORKDIR "$CODE_DIR"
+
+RUN cp /VERSION.txt /ABX-DL-VERSION.txt \
+    && (echo "[i] Docker build for ArchiveBox multistage starting..." \
+    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
+    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
+    && uname -a \
+    && sed -n '1,7p' /etc/os-release \
+    && which node && node --version \
+    && which uv && uv self version \
+    ) | tee -a /VERSION.txt
+
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    echo "[+] APT Installing ArchiveBox search dependency ripgrep for $TARGETPLATFORM..." \
+    && apt-get update -qq \
+    && apt-get install -qq -y --no-install-recommends ripgrep \
+    && rm -rf /var/lib/apt/lists/*
+
+FROM archivebox-runtime-base AS archivebox-builder
+
+WORKDIR "$CODE_DIR"
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
+    echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..." \
+    && export UV_PYTHON_INSTALL_DIR="$ARCHIVEBOX_PYTHON_INSTALL_DIR" \
+    && echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
+    && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends \
+    && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests \
+    && rm -f /etc/apt/apt.conf.d/docker-clean \
+    && apt-get update -qq \
+    && apt-get install -qq -y --no-install-recommends \
+        build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
+    && /usr/bin/uv venv --clear /venv --python "${PYTHON_VERSION}" \
+    && /usr/bin/uv pip install setuptools pip wheel \
+    && /usr/bin/uv sync \
+        --refresh \
+        --no-dev \
+        --inexact \
+        --all-extras \
+        --no-install-project \
+        --no-install-workspace \
+        --no-sources \
+    && apt-get purge -y build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
+    && apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
+RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
+    && COMMIT_HASH="$( \
+        if [[ -f "$CODE_DIR/.git/HEAD" ]]; then \
+            HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"; \
+            if [[ "$HEAD_REF" =~ ^[0-9a-fA-F]{40}$ ]]; then \
+                echo "$HEAD_REF"; \
+            elif [[ "$HEAD_REF" == ref:\ * ]]; then \
+                REF_PATH="${HEAD_REF#ref: }"; \
+                cat "$CODE_DIR/.git/$REF_PATH" 2>/dev/null || awk -v ref="$REF_PATH" '$2 == ref {print $1}' "$CODE_DIR/.git/packed-refs" 2>/dev/null || true; \
+            fi; \
+        fi)" \
+    && if [[ "$COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt; fi \
+    && /usr/bin/uv pip install --no-deps "$CODE_DIR" \
+    && (/usr/bin/uv pip show archivebox && which archivebox) | tee -a /VERSION.txt \
+    && rm -rf "$CODE_DIR/.git"
+
+FROM archivebox-runtime-base
+
+LABEL name="archivebox" \
+    maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
+    description="All-in-one self-hosted internet archiving solution" \
+    homepage="https://github.com/ArchiveBox/ArchiveBox" \
+    documentation="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker" \
+    org.opencontainers.image.title="ArchiveBox" \
+    org.opencontainers.image.vendor="ArchiveBox" \
+    org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \
+    org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \
+    com.docker.image.source.entrypoint="Dockerfile"
+
+COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
+COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
+
+COPY --from=archivebox-builder /opt/archivebox/python /opt/archivebox/python
+COPY --from=archivebox-builder /venv /venv
+COPY --from=archivebox-builder /app /app
+COPY --from=archivebox-builder /VERSION.txt /VERSION.txt
+
+RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
+    && getent group "$ARCHIVEBOX_USER" >/dev/null || groupadd --system "$ARCHIVEBOX_USER" \
+    && id -u "$ARCHIVEBOX_USER" >/dev/null 2>&1 || useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \
+    && usermod --append --groups audio,video "$ARCHIVEBOX_USER" \
+    && [[ "$(id -u "$ARCHIVEBOX_USER")" == "$DEFAULT_PUID" ]] || usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
+    && [[ "$(id -g "$ARCHIVEBOX_USER")" == "$DEFAULT_PGID" ]] || groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
+    && (which sonic && sonic --version) | tee -a /VERSION.txt \
+    && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
+    && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER" "/home/$ARCHIVEBOX_USER/.config" "/home/$ARCHIVEBOX_USER/.cache" \
+    && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER/.config/abx" "/home/$ARCHIVEBOX_USER/.cache/abxbus" "/home/$ARCHIVEBOX_USER/.cache/pnpm" "/home/$ARCHIVEBOX_USER/.cache/uv" \
+    && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
+    && openssl rand -hex 16 > /etc/machine-id \
+    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u "$ARCHIVEBOX_USER") PGID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
+    && echo -e "TMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
+
+WORKDIR "$DATA_DIR"
+RUN echo "[+] Initializing image collection..." \
+    && find "$DATA_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} + \
+    && PUID=0 PGID=0 archivebox init \
+    && (chown "$DEFAULT_PUID:$DEFAULT_PGID" \
+        "$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \
+        "$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \
+        "$DATA_DIR"/archive "$DATA_DIR"/archive/users "$DATA_DIR"/personas \
+        "$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \
+        "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
+        2>/dev/null || true)
+
+RUN chmod +x "$CODE_DIR"/bin/*.sh \
+    && chmod g+w "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"
+
+RUN "$LIB_DIR/bin/chromium" --version | tee -a /VERSION.txt \
+    && "$LIB_DIR/uv/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \
+    && /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \
+    && /usr/local/bin/sonic --version | tee -a /VERSION.txt \
+    && /venv/bin/supervisord --version | tee -a /VERSION.txt \
+    && for forbidden_bin in gcc g++ make; do ! command -v "$forbidden_bin" || (echo "Unexpected build tool in runtime: $forbidden_bin=$(command -v "$forbidden_bin")" >&2 && exit 1); done \
+    && stat -c "%U:%G %a %n" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$LIB_DIR" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \
+    && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
+
+RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully." \
+    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
+    && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
+    ) | tee -a /VERSION.txt
+
+WORKDIR "$DATA_DIR"
+VOLUME "$DATA_DIR"
+EXPOSE 8000
+
+HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
+    CMD curl --fail --silent --show-error --max-time 5 --connect-timeout 2 'http://admin.archivebox.localhost:8000/health/' | grep -q 'OK'
+
+ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh"]
+CMD ["archivebox", "server", "--init", "0.0.0.0:8000"]
@@ -77,7 +77,7 @@ The goal is to sleep soundly knowing the part of the internet you care about wil
 <pre lang="bash"><code style="white-space: pre-line"># Option A: Get ArchiveBox with Docker Compose (recommended):
 mkdir -p ~/archivebox/data && cd ~/archivebox
 curl -fsSL 'https://docker-compose.archivebox.io' > docker-compose.yml   # edit options in this file as-needed
-docker compose run archivebox init --install
+docker compose run archivebox init
 # docker compose run archivebox add 'https://example.com'
 # docker compose run archivebox help
 # docker compose up
@@ -85,7 +85,7 @@ docker compose run archivebox init --install
 <br/>
 # Option B: Or use it as a plain Docker container:
 mkdir -p ~/archivebox/data && cd ~/archivebox/data
-docker run -it -v $PWD:/data archivebox/archivebox init --install
+docker run -it -v $PWD:/data archivebox/archivebox init
 # docker run -it -v $PWD:/data archivebox/archivebox add 'https://example.com'
 # docker run -it -v $PWD:/data archivebox/archivebox help
 # docker run -it -v $PWD:/data -p 8000:8000 archivebox/archivebox
@@ -94,7 +94,8 @@ docker run -it -v $PWD:/data archivebox/archivebox init --install
 # Option C: Or install it with your preferred pkg manager (see Quickstart below for apt, brew, and more)
 pip install archivebox
 mkdir -p ~/archivebox/data && cd ~/archivebox/data
-archivebox init --install
+archivebox init
+archivebox install
 # archivebox add 'https://example.com'
 # archivebox help
 # archivebox server 0.0.0.0:8000
@@ -105,7 +106,7 @@ curl -fsSL 'https://get.archivebox.io' | bash
 </code></pre>
 <br/>
 <sub>Open <a href="http://web.archivebox.localhost:8000"><code>http://web.archivebox.localhost:8000</code></a> for the public UI and <a href="http://admin.archivebox.localhost:8000"><code>http://admin.archivebox.localhost:8000</code></a> for the admin UI ➡️</sub><br/>
-<sub>Set <code>LISTEN_HOST</code> to change the base domain; <code>web.</code> and <code>admin.</code> subdomains are used automatically.</sub>
+<sub>Set <code>BIND_ADDR</code> to change the base domain; <code>web.</code> and <code>admin.</code> subdomains are used automatically.</sub>
 </details>
 <br/>

@@ -189,9 +190,9 @@ ArchiveBox is free for everyone to self-host, but we also provide support, secur
 curl -fsSL 'https://docker-compose.archivebox.io' > docker-compose.yml
 </code></pre></li>
 <li>Run the initial setup to create an admin user (or set ADMIN_USER/PASS in docker-compose.yml)
-<pre lang="bash"><code style="white-space: pre-line">docker compose run archivebox init --install
+<pre lang="bash"><code style="white-space: pre-line">docker compose run archivebox init
 </code></pre></li>
-<li>Next steps: Start the server then login to the Web UI <a href="http://127.0.0.1:8000">http://127.0.0.1:8000</a> ⇢ Admin.
+<li>Next steps: Start the server then login to the Web UI <a href="http://archivebox.localhost:8000">http://archivebox.localhost:8000</a> ⇢ Admin.
 <pre lang="bash"><code style="white-space: pre-line">docker compose up
 # completely optional, CLI can always be used without running a server
 # docker compose run [-T] archivebox [subcommand] [--help]
@@ -213,10 +214,10 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <li>Install <a href="https://docs.docker.com/get-docker/">Docker</a> on your system (if not already installed).</li>
 <li>Create a new empty directory and initialize your collection (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
-docker run -v $PWD:/data -it archivebox/archivebox init --install
+docker run -v $PWD:/data -it archivebox/archivebox init
 </code></pre>
 </li>
-<li>Optional: Start the server then login to the Web UI <a href="http://127.0.0.1:8000">http://127.0.0.1:8000</a> ⇢ Admin.
+<li>Optional: Start the server then login to the Web UI <a href="http://archivebox.localhost:8000">http://archivebox.localhost:8000</a> ⇢ Admin.
 <pre lang="bash"><code style="white-space: pre-line">docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox
 # completely optional, CLI can always be used without running a server
 # docker run -v $PWD:/data -it [subcommand] [--help]
@@ -270,11 +271,11 @@ archivebox version
 </li>
 <li>Create a new empty directory and initialize your collection (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data   # for example
-archivebox init --install   # instantialize a new collection
-# (--setup auto-installs and link JS dependencies: singlefile, readability, mercury, etc.)
+archivebox init     # instantialize a new collection
+archivebox install  # install all the runtime dependencies (e.g. chrome, single-file, yt-dlp, etc.)
 </code></pre>
 </li>
-<li>Optional: Start the server then login to the Web UI <a href="http://127.0.0.1:8000">http://127.0.0.1:8000</a> ⇢ Admin.
+<li>Optional: Start the server then login to the Web UI <a href="http://archivebox.localhost:8000">http://archivebox.localhost:8000</a> ⇢ Admin.
 <pre lang="bash"><code style="white-space: pre-line">archivebox server 0.0.0.0:8000
 # completely optional, CLI can always be used without running a server
 # archivebox [subcommand] [--help]
@@ -294,12 +295,10 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <summary><b><img src="https://user-images.githubusercontent.com/511499/117448075-49597580-af0c-11eb-91ba-f34fff10096b.png" alt="aptitude" height="28px" align="top"/> <code>apt</code></b> (Ubuntu/Debian/etc.)</summary>
 <br/>
 <ol>
-<li>Download and install the <code>.deb</code> package from the <a href="https://github.com/ArchiveBox/ArchiveBox/releases">latest release</a>.
-<pre lang="bash"><code style="white-space: pre-line"># download the .deb for your architecture (amd64 or arm64)
-ARCH="$(dpkg --print-architecture)"
-VERSION="$(curl -fsSL https://api.github.com/repos/ArchiveBox/ArchiveBox/releases/latest | python3 -c "import sys,json; print(json.load(sys.stdin)['tag_name'].lstrip('v'))")"
-curl -fsSL "https://github.com/ArchiveBox/ArchiveBox/releases/latest/download/archivebox_${VERSION}_${ARCH}.deb" -o /tmp/archivebox.deb
-sudo apt install /tmp/archivebox.deb
+<li>Add the third-party ArchiveBox apt repo and install <code>archivebox</code>.
+<pre lang="bash"><code style="white-space: pre-line">echo 'deb [trusted=yes] https://archivebox.github.io/debian-archivebox dev main' | sudo tee /etc/apt/sources.list.d/archivebox.list
+sudo apt update
+sudo apt install archivebox
 archivebox version                         # make sure all dependencies are installed
 </code></pre>
 </li>
@@ -309,7 +308,7 @@ archivebox init --install
 </code></pre>
 <br/>
 </li>
-<li>Optional: Start the server then login to the Web UI <a href="http://127.0.0.1:8000">http://127.0.0.1:8000</a> ⇢ Admin.
+<li>Optional: Start the server then login to the Web UI <a href="http://archivebox.localhost:8000">http://archivebox.localhost8000</a> ⇢ Admin.
 <pre lang="bash"><code style="white-space: pre-line">archivebox server 0.0.0.0:8000
 # completely optional, CLI can always be used without running a server
 # archivebox [subcommand] [--help]
@@ -336,10 +335,11 @@ archivebox version                         # make sure all dependencies are inst
 </li>
 <li>Create a new empty directory and initialize your collection (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
-archivebox init --install
+archivebox init
+archivebox install
 </code></pre>
 </li>
-<li>Optional: Start the server then login to the Web UI <a href="http://127.0.0.1:8000">http://127.0.0.1:8000</a> ⇢ Admin.
+<li>Optional: Start the server then login to the Web UI <a href="http://archivebox.localhost:8000">http://archivebox.localhost:8000</a> ⇢ Admin.
 <pre lang="bash"><code style="white-space: pre-line">archivebox server 0.0.0.0:8000
 # completely optional, CLI can always be used without running a server
 # archivebox [subcommand] [--help]
@@ -586,7 +586,7 @@ docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox
 </code></pre>

 <sup>Open <a href="http://web.archivebox.localhost:8000"><code>http://web.archivebox.localhost:8000</code></a> for the public UI and <a href="http://admin.archivebox.localhost:8000"><code>http://admin.archivebox.localhost:8000</code></a> for the admin UI ➡️</sup><br/>
-<sup>Set <code>LISTEN_HOST</code> to change the base domain; <code>web.</code> and <code>admin.</code> subdomains are used automatically.</sup>
+<sup>Set <code>BIND_ADDR</code> to change the base domain; <code>web.</code> and <code>admin.</code> subdomains are used automatically.</sup>
 <br/><br/>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#ui-usage">Usage: Web UI</a> wiki. ➡️</i>
 <br/><br/>
@@ -796,7 +796,7 @@ ArchiveBox bundles industry-standard tools like [Google Chrome](https://github.c
 <li>Database: <a href="https://docs.djangoproject.com/en/6.0/ref/databases/#sqlite-notes">Django ORM</a> saving to <a href="https://www.sqlite.org/mostdeployed.html">SQLite3</a> <code>./data/index.sqlite3</code></li>
 <li>Job Queue: Custom orchestrator using <code>supervisord</code> for worker management</li>
 <li>Build/test/lint: <a href="https://github.com/astral-sh/uv"><code>uv</code></a> / <code>pyright</code>+<code>ty</code>+<code>pytest</code> / <code>ruff</code></li>
-<li>Subdependencies: <a href="https://github.com/ArchiveBox/abx-pkg"><code>abx-pkg</code></a> installs apt/brew/pip/npm pkgs at runtime (e.g. <code>yt-dlp</code>, <code>singlefile</code>, <code>readability</code>, <code>git</code>)</li>
+<li>Subdependencies: <a href="https://github.com/ArchiveBox/abxpkg"><code>abxpkg</code></a> installs apt/brew/pip/npm pkgs at runtime (e.g. <code>yt-dlp</code>, <code>singlefile</code>, <code>readability</code>, <code>git</code>)</li>
 </ul>


@@ -1342,26 +1342,35 @@ For low hanging fruit / easy first tickets, see: <a href="https://github.com/Arc

 <details><summary><i>Click to expand...</i></summary>

-#### 1. Clone the main code repo (making sure to pull the submodules as well)
+#### 1. Setup the monorepo
+
+First make sure you have `uv` installed: https://docs.astral.sh/uv/getting-started/installation/

 ```bash
-git clone --recurse-submodules https://github.com/ArchiveBox/ArchiveBox
-cd ArchiveBox
-git checkout dev  # or the branch you want to test
-git submodule update --init --recursive
-git pull --recurse-submodules
+git clone https://github.com/ArchiveBox/monorepo
+cd monorepo
+./bin/setup.sh
+
+# activate the monorepo venv, then cd into archivebox
+source .venv/bin/activate
+cd archivebox
 ```

+Repos included in monorepo setup:
+
+- https://github.com/ArchiveBox/abxbus
+- https://github.com/ArchiveBox/abxpkg
+- https://github.com/ArchiveBox/abx-plugins
+- https://github.com/ArchiveBox/abx-dl
+- https://github.com/ArchiveBox/ArchiveBox
+
+
 #### 2. Option A: Install the Python, JS, and system dependencies directly on your machine

 ```bash
-# Install ArchiveBox + python dependencies
-pip install uv
-./bin/lock_pkgs.sh         # (aka `uv venv; uv sync;` + generate requirements.txt)
-source .venv/bin/activate  # activate the venv
-
 # Install ArchiveBox runtime dependencies
 mkdir -p data && cd data
+archivebox init
 archivebox install         # detect and install all extractor dependencies

 # Run the development server w/ autoreloading (but no bg workers)
@@ -1508,13 +1517,9 @@ archivebox manage runscript testdata
 archivebox manage reset_db

 # use django-tui to interactively explore commands
-pip install django-tui
+uv pip install django-tui
 # ensure django-tui is in INSTALLED_APPS: core/settings.py
 archivebox manage tui
-
-# show python and JS package dependency trees
-pdm list --tree
-npm ls --all
 ```

 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/dc3e9f8c-9544-46e0-a7f0-30f571b72022" width="600px" alt="ArchiveBox ORM models relatinoship graph"/>
@@ -1530,40 +1535,11 @@ npm ls --all

 #### Contributing a new extractor

-<details><summary><i>Click to expand...</i></summary>

-<br/><br/>
+Extractors are maintained in a separate repo here: https://github.com/ArchiveBox/abx-plugins (included in monorepo setup).

-ArchiveBox [`extractors`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/extractors/media.py) are external binaries or Python/Node scripts that ArchiveBox runs to archive content on a page.
+Copy a similar plugin as a template to modify, then open a new PR to add it in that repo.

-Extractors take the URL of a page to archive, write their output to the filesystem `data/archive/TIMESTAMP/EXTRACTOR/...`, and return an [`ArchiveResult`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/core/models.py#:~:text=return%20qs-,class%20ArchiveResult,-(models.Model)%3A) entry which is saved to the database (visible on the `Log` page in the UI).
-
-*Check out how we added **[`archivebox/extractors/singlefile.py`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/extractors/singlefile.py)** as an example of the process: [Issue #399](https://github.com/ArchiveBox/ArchiveBox/issues/399) + [PR #403](https://github.com/ArchiveBox/ArchiveBox/pull/403).*
-
-<br/>
-
-
-**The process to contribute a new extractor is like this:**
-
-> [!IMPORTANT]
-> This process is getting much easier after v0.8.x, there is a new plugin system under development: https://github.com/ArchiveBox/ArchiveBox/releases/tag/v0.8.4-rc
-
-1. [Open an issue](https://github.com/ArchiveBox/ArchiveBox/issues/new?assignees=&labels=changes%3A+behavior%2Cstatus%3A+idea+phase&template=feature_request.md&title=Feature+Request%3A+...) with your propsoed implementation (please link to the pages of any new external dependencies you plan on using)
-2. Ensure any dependencies needed are easily installable via a package managers like `apt`, `brew`, `pip3`, `npm`
-   (Ideally, prefer to use external programs available via `pip3` or `npm`, however we do support using any binary installable via package manager that exposes a CLI/Python API and writes output to stdout or the filesystem.)
-3. Create a new file in [`archivebox/extractors/EXTRACTOR.py`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/extractors) (copy an existing extractor like [`singlefile.py`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/extractors/singlefile.py) as a template)
-4. Add config settings to enable/disable any new dependencies and the extractor as a whole, e.g. `USE_DEPENDENCYNAME`, `SAVE_EXTRACTORNAME`, `EXTRACTORNAME_SOMEOTHEROPTION` in [`archivebox/config.py`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/config.py)
-5. Add a preview section to [`archivebox/templates/core/snapshot.html`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/snapshot.html) to view the output, and a column to [`archivebox/templates/core/index_row.html`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/index_row.html) with an icon for your extractor
-6. Add an integration test for your extractor in [`tests/test_extractors.py`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/tests/test_extractors.py)
-7. [Submit your PR for review!](https://github.com/ArchiveBox/ArchiveBox/blob/dev/.github/CONTRIBUTING.md) 🎉
-8. Once merged, please document it in these places and anywhere else you see info about other extractors:
-  - https://github.com/ArchiveBox/ArchiveBox#output-formats
-  - https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#archive-method-toggles
-  - https://github.com/ArchiveBox/ArchiveBox/wiki/Install#dependencies
-
-<br/><br/>
-
-</details>

 #### Build the docs, pip package, and docker image

@@ -16,8 +16,6 @@ import sys
 from pathlib import Path
 from typing import Protocol, cast

-from abx_plugins import get_plugins_dir
-

 class _ReconfigurableStream(Protocol):
    def reconfigure(self, *, line_buffering: bool) -> object: ...
@@ -59,38 +57,59 @@ check_not_root()
 check_not_inside_source_dir()
 check_io_encoding()

-# Install monkey patches for third-party libraries
-from .misc.monkey_patches import *  # noqa
-
-# Plugin directories
-BUILTIN_PLUGINS_DIR = Path(get_plugins_dir()).resolve()
-USER_PLUGINS_DIR = (
-    Path(
-        os.environ.get("ARCHIVEBOX_USER_PLUGINS_DIR") or os.environ.get("USER_PLUGINS_DIR") or os.environ.get("DATA_DIR", os.getcwd()),
-    )
-    / "custom_plugins"
-)
-
-# These are kept for backwards compatibility with existing code
-# that checks for plugins. The new hook system uses discover_hooks()
-ALL_PLUGINS = {
-    "builtin": BUILTIN_PLUGINS_DIR,
-    "user": USER_PLUGINS_DIR,
-}
-LOADED_PLUGINS = ALL_PLUGINS
-
-# Setup basic config, constants, paths, and version
-from .config.constants import CONSTANTS  # noqa
-from .config.paths import PACKAGE_DIR, DATA_DIR, ARCHIVE_DIR  # noqa
 from .config.version import VERSION  # noqa

-# Set MACHINE_ID env var so hook scripts can use it
-os.environ.setdefault("MACHINE_ID", CONSTANTS.MACHINE_ID)

 __version__ = VERSION
 __author__ = "ArchiveBox"
 __license__ = "MIT"

+
+def __getattr__(name: str):
+    if name == "CONSTANTS":
+        from .config.constants import CONSTANTS
+
+        os.environ.setdefault("MACHINE_ID", CONSTANTS.MACHINE_ID)
+        return CONSTANTS
+    if name == "DATA_DIR":
+        from .config.paths import DATA_DIR
+
+        return DATA_DIR
+    if name == "VERSION":
+        return VERSION
+    if name in ("BUILTIN_PLUGINS_DIR", "USER_PLUGINS_DIR", "ALL_PLUGINS", "LOADED_PLUGINS"):
+        from abx_plugins import get_plugins_dir
+        from .config.constants import CONSTANTS
+
+        builtin_plugins_dir = Path(get_plugins_dir()).resolve()
+        user_plugins_dir = CONSTANTS.USER_PLUGINS_DIR
+        plugins = {
+            "builtin": builtin_plugins_dir,
+            "user": user_plugins_dir,
+        }
+        values = {
+            "BUILTIN_PLUGINS_DIR": builtin_plugins_dir,
+            "USER_PLUGINS_DIR": user_plugins_dir,
+            "ALL_PLUGINS": plugins,
+            "LOADED_PLUGINS": plugins,
+        }
+        return values[name]
+    raise AttributeError(name)
+
+
+__all__ = (
+    "ASCII_LOGO",
+    "ASCII_ICON",
+    "PACKAGE_DIR",
+    "DATA_DIR",
+    "CONSTANTS",
+    "VERSION",
+    "BUILTIN_PLUGINS_DIR",
+    "USER_PLUGINS_DIR",
+    "ALL_PLUGINS",
+    "LOADED_PLUGINS",
+)
+
 ASCII_ICON = """
 ██████████████████████████████████████████████████████████████████████████████████████████████████ 
 ██████████████████████████████████████████████████████████████████████████████████████████████████ 
@@ -17,4 +17,4 @@ ASCII_LOGO_MINI = r"""
 """

 if __name__ == "__main__":
-    main(args=sys.argv[1:], stdin=sys.stdin)
+    main(args=sys.argv[1:])
@@ -1,15 +1,23 @@
 __package__ = "archivebox.api"

+from django import forms
 from django.contrib import admin
 from django.http import HttpRequest
-from signal_webhooks.admin import WebhookAdmin
-from signal_webhooks.utils import get_webhook_model
+from django.utils.text import capfirst
+from signal_webhooks.admin import WebhookAdmin, WebhookModelForm
+from signal_webhooks.settings import webhook_settings
+from signal_webhooks.utils import get_webhook_model, model_from_reference

 from archivebox.base_models.admin import BaseModelAdmin

 from archivebox.api.models import APIToken


+def _webhook_fields(*names: str) -> tuple[str, ...]:
+    model_fields = {field.name for field in get_webhook_model()._meta.fields}
+    return tuple(name for name in names if name in model_fields)
+
+
 class APITokenAdmin(BaseModelAdmin):
    list_display = ("created_at", "id", "created_by", "token_redacted", "expires")
    sort_fields = ("id", "created_at", "created_by", "expires")
@@ -45,44 +53,58 @@ class APITokenAdmin(BaseModelAdmin):
    list_per_page = 100


+class OutboundWebhookAdminForm(WebhookModelForm):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.fields["ref"] = forms.ChoiceField(
+            label=self.fields["ref"].label,
+            help_text=self.fields["ref"].help_text,
+            choices=[
+                (ref, f"{capfirst(model_from_reference(ref, check_hooks=False)._meta.verbose_name_plural)} ({ref})")
+                for ref in sorted(webhook_settings.HOOKS)
+            ],
+        )
+
+
 class CustomWebhookAdmin(WebhookAdmin, BaseModelAdmin):
+    form = OutboundWebhookAdminForm
    list_display = ("created_at", "created_by", "id", *WebhookAdmin.list_display)
-    sort_fields = ("created_at", "created_by", "id", "referenced_model", "endpoint", "last_success", "last_error")
-    readonly_fields = ("created_at", "modified_at", *WebhookAdmin.readonly_fields)
+    sort_fields = _webhook_fields("created_at", "created_by", "id", "ref", "endpoint", "last_success", "last_failure")
+    readonly_fields = _webhook_fields("created_at", "modified_at", *WebhookAdmin.readonly_fields)

    fieldsets = (
        (
            "Webhook",
            {
-                "fields": ("name", "signal", "referenced_model", "endpoint"),
+                "fields": _webhook_fields("name", "signal", "ref", "endpoint", "headers", "keep_last_response"),
                "classes": ("card", "wide"),
            },
        ),
        (
            "Authentication",
            {
-                "fields": ("auth_token",),
+                "fields": _webhook_fields("auth_token"),
                "classes": ("card",),
            },
        ),
        (
            "Status",
            {
-                "fields": ("enabled", "last_success", "last_error"),
+                "fields": _webhook_fields("enabled", "last_success", "last_failure", "last_response"),
                "classes": ("card",),
            },
        ),
        (
            "Owner",
            {
-                "fields": ("created_by",),
+                "fields": _webhook_fields("created_by"),
                "classes": ("card",),
            },
        ),
        (
            "Timestamps",
            {
-                "fields": ("created_at", "modified_at"),
+                "fields": _webhook_fields("created_at", "modified_at"),
                "classes": ("card",),
            },
        ),
@@ -7,7 +7,7 @@ from django.http import HttpRequest
 from django.contrib.auth import authenticate
 from django.contrib.auth.models import User

-from ninja.security import HttpBearer, APIKeyQuery, APIKeyHeader, HttpBasicAuth
+from ninja.security import HttpBearer, APIKeyQuery, APIKeyHeader
 from ninja.errors import HttpError


@@ -51,6 +51,27 @@ def auth_using_token(token: str | None, request: HttpRequest | None = None) -> U
    return user


+def token_from_request(request: HttpRequest) -> str:
+    token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key") or ""
+    auth_header = request.headers.get("Authorization", "")
+    if not token and auth_header.lower().startswith("bearer "):
+        token = auth_header.split(None, 1)[1].strip()
+    return token
+
+
+def authenticated_user_from_request(request: HttpRequest) -> User | None:
+    user = request.user
+    if user.is_authenticated and user.is_active:
+        return user
+
+    token = token_from_request(request)
+    token_user = auth_using_token(token=token, request=request) if token else None
+    if token_user and token_user.is_active:
+        request.user = token_user
+        return token_user
+    return None
+
+
 def auth_using_password(username: str | None, password: str | None, request: HttpRequest | None = None) -> User | None:
    """Given a username and password, check if they are valid and return the corresponding user"""
    user: User | None = None
@@ -106,33 +127,6 @@ class QueryParamTokenAuth(APIKeyQuery):
        return _require_superuser(auth_using_token(token=key, request=request), request, self.__class__.__name__)


-class UsernameAndPasswordAuth(HttpBasicAuth):
-    """Allow authenticating by passing username & password via HTTP Basic Authentication (not recommended)"""
-
-    def authenticate(self, request: HttpRequest, username: str, password: str) -> User | None:
-        return _require_superuser(
-            auth_using_password(username=username, password=password, request=request),
-            request,
-            self.__class__.__name__,
-        )
-
-
-class DjangoSessionAuth:
-    """Allow authenticating with existing Django session cookies (same-origin only)."""
-
-    def __call__(self, request: HttpRequest) -> User | None:
-        return self.authenticate(request)
-
-    def authenticate(self, request: HttpRequest, **kwargs) -> User | None:
-        user = getattr(request, "user", None)
-        if isinstance(user, User) and user.is_authenticated:
-            setattr(request, "_api_auth_method", self.__class__.__name__)
-            if not user.is_superuser:
-                raise HttpError(403, "Valid session but User does not have permission (make sure user.is_superuser=True)")
-            return user
-        return None
-
-
 ### Enabled Auth Methods

 API_AUTH_METHODS = [
@@ -1,7 +1,7 @@
 __package__ = "archivebox.api"

 import secrets
-from archivebox.uuid_compat import uuid7
+from archivebox.uuid_compat import CompactUUIDField, uuid7

 from django.conf import settings
 from django.db import models
@@ -17,7 +17,7 @@ def generate_secret_token() -> str:


 class APIToken(models.Model):
-    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
    created_at = models.DateTimeField(default=timezone.now, db_index=True)
    modified_at = models.DateTimeField(auto_now=True)
@@ -41,7 +41,7 @@ class APIToken(models.Model):


 class OutboundWebhook(WebhookBase):
-    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
    created_at = models.DateTimeField(default=timezone.now, db_index=True)
    modified_at = models.DateTimeField(auto_now=True)
@@ -1,12 +1,25 @@
 __package__ = "archivebox.api"

+from urllib.parse import quote
+
+from django.http import HttpRequest, HttpResponseRedirect
+from django.shortcuts import redirect
 from django.urls import path
 from django.views.generic.base import RedirectView

+from archivebox.core.routes_util import build_web_url
 from .v1_api import urls as v1_api_urls

+
+def archive_redirect_view(request: HttpRequest, url: str) -> HttpResponseRedirect:
+    if request.META.get("QUERY_STRING"):
+        url = f"{url}?{request.META['QUERY_STRING']}"
+    return redirect(build_web_url(f"/web/{quote(url, safe=':/')}", request=request))
+
+
 urlpatterns = [
    path("", RedirectView.as_view(url="/api/v1/docs")),
+    path("archive/<path:url>", archive_redirect_view, name="api-archive-redirect"),
    path("v1/", RedirectView.as_view(url="/api/v1/docs")),
    path("v1/", v1_api_urls),
    path("v1", RedirectView.as_view(url="/api/v1/docs")),
@@ -44,6 +44,7 @@ def register_urls(api: NinjaAPI) -> NinjaAPI:
    api.add_router("/crawls/", "archivebox.api.v1_crawls.router")
    api.add_router("/cli/", "archivebox.api.v1_cli.router")
    api.add_router("/machine/", "archivebox.api.v1_machine.router")
+    api.add_router("/personas/", "archivebox.api.v1_personas.router")
    return api


@@ -67,14 +68,14 @@ class NinjaAPIWithIOCapture(NinjaAPI):
        # response['X-ArchiveBox-View'] = self.get_openapi_operation_id(request) or 'Unknown'

        # Add Auth Headers to response
-        api_token_attr = getattr(request, "_api_token", None)
+        api_token_attr = request.__dict__.get("_api_token")
        api_token = api_token_attr if isinstance(api_token_attr, APIToken) else None
        token_expiry = api_token.expires.isoformat() if api_token and api_token.expires else "Never"

-        response["X-ArchiveBox-Auth-Method"] = str(getattr(request, "_api_auth_method", "None"))
+        response["X-ArchiveBox-Auth-Method"] = str(request.__dict__.get("_api_auth_method", "None"))
        response["X-ArchiveBox-Auth-Expires"] = token_expiry
        response["X-ArchiveBox-Auth-Token-Id"] = str(api_token.id) if api_token else "None"
-        response["X-ArchiveBox-Auth-User-Id"] = str(request.user.pk) if getattr(request.user, "pk", None) else "None"
+        response["X-ArchiveBox-Auth-User-Id"] = str(request.user.pk) if request.user.pk else "None"
        response["X-ArchiveBox-Auth-User-Username"] = request.user.username if isinstance(request.user, User) else "None"

        # import ipdb; ipdb.set_trace()
@@ -8,9 +8,11 @@ from enum import Enum
 from django.http import HttpRequest

 from ninja import Router, Schema
+from ninja.errors import HttpError
+from pydantic import Field

 from archivebox.misc.util import ansi_to_html
-from archivebox.config.common import ARCHIVING_CONFIG
+from archivebox.core.models import SnapshotQuerySet


 # from .auth import API_AUTH_METHODS
@@ -22,6 +24,7 @@ router = Router(tags=["ArchiveBox CLI Sub-Commands"])
 # Schemas

 JSONType = list[Any] | dict[str, Any] | bool | int | str | None
+FILTER_PATTERNS_EXAMPLES = [["https://example.com"]]


 class CLICommandResponseSchema(Schema):
@@ -33,47 +36,51 @@ class CLICommandResponseSchema(Schema):
    stderr: str


-class FilterTypeChoices(str, Enum):
-    exact = "exact"
-    substring = "substring"
-    regex = "regex"
-    domain = "domain"
-    tag = "tag"
-    timestamp = "timestamp"
-
-
-class StatusChoices(str, Enum):
-    indexed = "indexed"
-    archived = "archived"
-    unarchived = "unarchived"
-    present = "present"
-    valid = "valid"
-    invalid = "invalid"
-    duplicate = "duplicate"
-    orphaned = "orphaned"
-    corrupted = "corrupted"
-    unrecognized = "unrecognized"
+FilterTypeChoices = Enum(
+    "FilterTypeChoices",
+    {filter_type: filter_type for filter_type in SnapshotQuerySet.FILTER_TYPE_CHOICES},
+    type=str,
+)


 class AddCommandSchema(Schema):
    urls: list[str]
+    snapshot_ids: list[str] | None = None
    tag: str = ""
    depth: int = 0
+    max_urls: int = 0
+    crawl_max_size: int = 0
+    crawl_timeout: int = 0
+    snapshot_max_size: int = 0
    parser: str = "auto"
    plugins: str = ""
-    update: bool = not ARCHIVING_CONFIG.ONLY_NEW  # Default to the opposite of ARCHIVING_CONFIG.ONLY_NEW
+    only_new: bool | None = None
+    update: bool = False
    overwrite: bool = False
    index_only: bool = False


-class UpdateCommandSchema(Schema):
-    resume: str | None = None
+class SnapshotFilterCommandSchema(Schema):
    after: float | None = 0
-    before: float | None = 999999999999999
+    before: float | None = None
    filter_type: str | None = FilterTypeChoices.substring
-    filter_patterns: list[str] | None = ["https://example.com"]
+    filter_patterns: list[str] | None = Field(default=None, examples=FILTER_PATTERNS_EXAMPLES)
+    status: str | None = None
+    url__icontains: str | None = None
+    url__istartswith: str | None = None
+    tag: str | None = None
+    crawl_id: str | None = None
+    limit: int | None = None
+    sort: str | None = None
+    search: str | None = None
+
+
+class UpdateCommandSchema(SnapshotFilterCommandSchema):
+    resume: str | None = None
    batch_size: int = 100
    continuous: bool = False
+    index_only: bool = False
+    migrate_only: bool = False


 class ScheduleCommandSchema(Schema):
@@ -86,47 +93,55 @@ class ScheduleCommandSchema(Schema):
    every: str | None = None
    tag: str = ""
    depth: int = 0
+    only_new: bool | None = None
+    update: bool = False
    overwrite: bool = False
-    update: bool = not ARCHIVING_CONFIG.ONLY_NEW
    clear: bool = False


-class ListCommandSchema(Schema):
-    filter_patterns: list[str] | None = ["https://example.com"]
-    filter_type: str = FilterTypeChoices.substring
-    status: StatusChoices = StatusChoices.indexed
-    after: float | None = 0
-    before: float | None = 999999999999999
-    sort: str = "bookmarked_at"
+class ListCommandSchema(SnapshotFilterCommandSchema):
    as_json: bool = True
    as_html: bool = False
    as_csv: str | None = "timestamp,url"
    with_headers: bool = False


-class RemoveCommandSchema(Schema):
-    delete: bool = True
-    after: float | None = 0
-    before: float | None = 999999999999999
+class RemoveCommandSchema(SnapshotFilterCommandSchema):
    filter_type: str = FilterTypeChoices.exact
-    filter_patterns: list[str] | None = ["https://example.com"]
+    timeout: float = 60.0
+
+
+def snapshot_filter_kwargs(args: SnapshotFilterCommandSchema, *, default_filter_type: str) -> dict[str, Any]:
+    kwargs = args.dict()
+    kwargs["filter_patterns"] = kwargs.get("filter_patterns") or []
+    kwargs["filter_type"] = kwargs.get("filter_type") or default_filter_type
+    return kwargs


@router.post("/add", response=CLICommandResponseSchema, summary="archivebox add [args] [urls]")
 def cli_add(request: HttpRequest, args: AddCommandSchema):
    from archivebox.cli.archivebox_add import add

+    config_overrides: dict[str, object] = {}
+    if args.only_new is not None:
+        config_overrides["ONLY_NEW"] = bool(args.only_new)
+    if args.update or args.overwrite:
+        config_overrides["ONLY_NEW"] = False
    crawl, snapshots = add(
        urls=args.urls,
+        snapshot_ids=args.snapshot_ids,
        tag=args.tag,
        depth=args.depth,
-        update=args.update,
+        max_urls=args.max_urls,
+        crawl_max_size=args.crawl_max_size,
+        crawl_timeout=args.crawl_timeout,
+        snapshot_max_size=args.snapshot_max_size,
        index_only=args.index_only,
-        overwrite=args.overwrite,
        plugins=args.plugins,
        parser=args.parser,
        bg=True,  # Always run in background for API calls
        created_by_id=request.user.pk,
+        config=config_overrides or None,
    )

    snapshot_ids = [str(snapshot_id) for snapshot_id in snapshots.values_list("id", flat=True)]
@@ -136,8 +151,8 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
        "snapshot_ids": snapshot_ids,
        "queued_urls": args.urls,
    }
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")

    return {
        "success": True,
@@ -151,23 +166,39 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):

@router.post("/update", response=CLICommandResponseSchema, summary="archivebox update [args] [filter_patterns]")
 def cli_update(request: HttpRequest, args: UpdateCommandSchema):
-    from archivebox.cli.archivebox_update import update
+    from archivebox.cli.archivebox_update import _build_filtered_snapshots_queryset, update
+    from archivebox.core.snapshot_status import normalize_snapshot_status

-    result = update(
-        filter_patterns=args.filter_patterns or [],
-        filter_type=args.filter_type or FilterTypeChoices.substring,
-        after=args.after,
-        before=args.before,
-        resume=args.resume,
-        batch_size=args.batch_size,
-        continuous=args.continuous,
+    try:
+        status = normalize_snapshot_status(args.status)
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
+
+    update_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.substring)
+    update_kwargs["status"] = status
+    update_kwargs["stop_daemon_stack"] = False
+
+    is_filtered_update = any(
+        (update_kwargs.get(key) for key in (*SnapshotQuerySet.FILTER_ARG_KEYS, "resume") if key != "filter_type"),
    )
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
+    matched_snapshot_ids = []
+    if is_filtered_update:
+        matched_snapshot_ids = [
+            str(snapshot_id) for snapshot_id in _build_filtered_snapshots_queryset(**update_kwargs).values_list("id", flat=True)
+        ]
+
+    update(**update_kwargs)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
    return {
        "success": True,
        "errors": [],
-        "result": result,
+        "result": {
+            "matched_count": len(matched_snapshot_ids),
+            "snapshot_ids": matched_snapshot_ids,
+        }
+        if is_filtered_update
+        else None,
        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
    }
@@ -177,6 +208,11 @@ def cli_update(request: HttpRequest, args: UpdateCommandSchema):
 def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
    from archivebox.cli.archivebox_schedule import schedule

+    config_overrides: dict[str, object] = {}
+    if args.only_new is not None:
+        config_overrides["ONLY_NEW"] = bool(args.only_new)
+    if args.update or args.overwrite:
+        config_overrides["ONLY_NEW"] = False
    result = schedule(
        import_path=args.import_path,
        add=args.add,
@@ -188,12 +224,11 @@ def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
        every=args.every,
        tag=args.tag,
        depth=args.depth,
-        overwrite=args.overwrite,
-        update=args.update,
+        config=config_overrides or None,
    )

-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
    return {
        "success": True,
        "errors": [],
@@ -206,32 +241,38 @@ def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):

@router.post("/search", response=CLICommandResponseSchema, summary="archivebox search [args] [filter_patterns]")
 def cli_search(request: HttpRequest, args: ListCommandSchema):
-    from archivebox.cli.archivebox_search import search
+    from archivebox.cli.archivebox_snapshot import build_snapshot_queryset

-    result = search(
-        filter_patterns=args.filter_patterns,
-        filter_type=args.filter_type,
-        status=args.status,
-        after=args.after,
-        before=args.before,
-        sort=args.sort,
-        csv=args.as_csv,
-        json=args.as_json,
-        html=args.as_html,
-        with_headers=args.with_headers,
-    )
+    search_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.substring)
+    as_json = search_kwargs.pop("as_json")
+    as_html = search_kwargs.pop("as_html")
+    as_csv = search_kwargs.pop("as_csv")
+    with_headers = search_kwargs.pop("with_headers")
+    try:
+        snapshots = build_snapshot_queryset(**search_kwargs).select_related("crawl", "crawl__created_by")
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err

    result_format = "txt"
-    if args.as_json:
+    if as_json:
        result_format = "json"
-        result = json.loads(result)
-    elif args.as_html:
+        result = [
+            json.loads(json.dumps(snapshot.to_dict(extended=True), default=str))
+            for snapshot in snapshots.prefetch_related("tags").iterator(chunk_size=500)
+        ]
+    elif as_html:
        result_format = "html"
-    elif args.as_csv:
+        result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500))
+    elif as_csv:
        result_format = "csv"
+        cols = [col.strip() for col in as_csv.split(",") if col.strip()]
+        rows = [snapshot.to_csv(cols=cols, separator=",") for snapshot in snapshots.prefetch_related("tags").iterator(chunk_size=500)]
+        result = "\n".join((",".join(cols), *rows) if with_headers else rows)
+    else:
+        result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500))

-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
    return {
        "success": True,
        "errors": [],
@@ -245,38 +286,26 @@ def cli_search(request: HttpRequest, args: ListCommandSchema):
@router.post("/remove", response=CLICommandResponseSchema, summary="archivebox remove [args] [filter_patterns]")
 def cli_remove(request: HttpRequest, args: RemoveCommandSchema):
    from archivebox.cli.archivebox_remove import remove
-    from archivebox.cli.archivebox_search import get_snapshots
    from archivebox.core.models import Snapshot

-    filter_patterns = args.filter_patterns or []
-    snapshots_to_remove = get_snapshots(
-        filter_patterns=filter_patterns,
-        filter_type=args.filter_type,
-        after=args.after,
-        before=args.before,
-    )
-    removed_snapshot_ids = [str(snapshot_id) for snapshot_id in snapshots_to_remove.values_list("id", flat=True)]
+    remove_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.exact)
+    timeout_arg = remove_kwargs.pop("timeout")
+    timeout = min(float(timeout_arg if timeout_arg is not None else 60.0), 60.0)
+    try:
+        snapshots_to_remove = Snapshot.objects.order_by("-created_at").search(**remove_kwargs)
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err

-    remove(
+    result = remove(
        yes=True,  # no way to interactively ask for confirmation via API, so we force yes
-        delete=args.delete,
        snapshots=snapshots_to_remove,
-        before=args.before,
-        after=args.after,
-        filter_type=args.filter_type,
-        filter_patterns=filter_patterns,
+        timeout=timeout,
    )
-
-    result = {
-        "removed_count": len(removed_snapshot_ids),
-        "removed_snapshot_ids": removed_snapshot_ids,
-        "remaining_snapshots": Snapshot.objects.count(),
-    }
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
    return {
-        "success": True,
-        "errors": [],
+        "success": bool(result["success"]),
+        "errors": [str(result["error"])] if result["error"] else [],
        "result": result,
        "result_format": "json",
        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
@@ -1,8 +1,10 @@
 __package__ = "archivebox.api"

+from pathlib import Path
 from uuid import UUID
 from datetime import datetime
-from django.http import HttpRequest
+from django.http import FileResponse, HttpRequest
+from django.shortcuts import redirect
 from django.utils import timezone

 from django.contrib.auth import get_user_model
@@ -12,9 +14,17 @@ from ninja import Router, Schema
 from ninja.errors import HttpError

 from archivebox.core.models import Snapshot
+from archivebox.core.permissions import (
+    PERMISSIONS_PUBLIC,
+    PERMISSIONS_UNLISTED,
+    is_admin_user,
+    normalize_permissions,
+)
+from archivebox.config.common import get_config
 from archivebox.crawls.models import Crawl
+from archivebox.misc.util import filter_queryset_by_uuid_substring

-from .auth import API_AUTH_METHODS
+from .auth import API_AUTH_METHODS, authenticated_user_from_request

 router = Router(tags=["Crawl Models"], auth=API_AUTH_METHODS)

@@ -31,11 +41,10 @@ class CrawlSchema(Schema):

    status: str
    retry_at: datetime | None
+    is_paused: bool

    urls: str
    max_depth: int
-    max_urls: int
-    max_size: int
    tags_str: str
    config: dict

@@ -49,17 +58,26 @@ class CrawlSchema(Schema):
    def resolve_created_by_username(obj):
        user_model = get_user_model()
        user = user_model.objects.get(id=obj.created_by_id)
-        username = getattr(user, "username", None)
+        username = user.username
        return username if isinstance(username, str) else str(user)

+    @staticmethod
+    def resolve_config(obj):
+        # Redact credential values so REST responses can never leak the raw
+        # token/secret/api-key that the operator stored in Crawl.config.
+        from archivebox.config.common import redact_sensitive_config
+
+        return redact_sensitive_config(obj.config)
+
    @staticmethod
    def resolve_snapshots(obj, context):
-        if bool(getattr(context["request"], "with_snapshots", False)):
+        if bool(context["request"].__dict__.get("with_snapshots", False)):
            return obj.snapshot_set.all().distinct()
        return Snapshot.objects.none()


 class CrawlUpdateSchema(Schema):
+    action: str | None = None
    status: str | None = None
    retry_at: datetime | None = None
    tags: list[str] | None = None
@@ -69,8 +87,6 @@ class CrawlUpdateSchema(Schema):
 class CrawlCreateSchema(Schema):
    urls: list[str]
    max_depth: int = 0
-    max_urls: int = 0
-    max_size: int = 0
    tags: list[str] | None = None
    tags_str: str = ""
    label: str = ""
@@ -103,58 +119,110 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
        raise HttpError(400, "At least one URL is required")
    if data.max_depth not in (0, 1, 2, 3, 4):
        raise HttpError(400, "max_depth must be between 0 and 4")
-    if data.max_urls < 0:
-        raise HttpError(400, "max_urls must be >= 0")
-    if data.max_size < 0:
-        raise HttpError(400, "max_size must be >= 0")

    tags = normalize_tag_list(data.tags, data.tags_str)
+    config = dict(data.config or {})
+    config.setdefault("PERMISSIONS", str(get_config().PERMISSIONS))
    crawl = Crawl.objects.create(
        urls="\n".join(urls),
        max_depth=data.max_depth,
-        max_urls=data.max_urls,
-        max_size=data.max_size,
        tags_str=",".join(tags),
        label=data.label,
        notes=data.notes,
-        config=data.config,
+        config=config,
        status=Crawl.StatusChoices.QUEUED,
        retry_at=timezone.now(),
        created_by=request.user if isinstance(request.user, User) else None,
    )
-    crawl.create_snapshots_from_urls()
    return crawl


-@router.get("/crawl/{crawl_id}", response=CrawlSchema | str, url_name="get_crawl")
+def get_crawl_by_ref(crawl_id: str):
+    return filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
+
+
+@router.get("/crawl/{crawl_id}", response=CrawlSchema, url_name="get_crawl")
 def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_snapshots: bool = False, with_archiveresults: bool = False):
    """Get a specific Crawl by id."""
    setattr(request, "with_snapshots", with_snapshots)
    setattr(request, "with_archiveresults", with_archiveresults)
-    crawl = Crawl.objects.get(id__icontains=crawl_id)
+    crawl = get_crawl_by_ref(crawl_id)

    if crawl and as_rss:
-        # return snapshots as XML rss feed
-        urls = [
-            {"url": snapshot.url, "title": snapshot.title, "bookmarked_at": snapshot.bookmarked_at, "tags": snapshot.tags_str}
-            for snapshot in crawl.snapshot_set.all()
-        ]
-        xml = '<rss version="2.0"><channel>'
-        for url in urls:
-            xml += f"<item><url>{url['url']}</url><title>{url['title']}</title><bookmarked_at>{url['bookmarked_at']}</bookmarked_at><tags>{url['tags']}</tags></item>"
-        xml += "</channel></rss>"
-        return xml
+        query = request.GET.copy()
+        query.pop("as_rss", None)
+        query["crawl_id"] = str(crawl.id)
+        return redirect(f"/api/v1/core/snapshots.rss?{query.urlencode()}")

    return crawl


+def crawl_file(request: HttpRequest, crawl_id: str, path: str):
+    # Try to resolve the crawl first; if it doesn't exist, return 404.
+    try:
+        crawl = get_crawl_by_ref(crawl_id)
+    except Crawl.DoesNotExist:
+        raise HttpError(404, "Crawl not found")
+
+    user = authenticated_user_from_request(request)
+
+    # Gate access using the same model as SnapshotView/can_view_snapshot:
+    # admins always pass; owners can see their own crawls; otherwise the crawl
+    # must be PUBLIC or UNLISTED. Don't disclose existence of private crawls.
+    if not is_admin_user(request):
+        permissions = normalize_permissions(crawl.permissions)
+        is_owner = bool(user and crawl.created_by_id == user.id)
+        if not is_owner and permissions not in {PERMISSIONS_PUBLIC, PERMISSIONS_UNLISTED}:
+            raise HttpError(404, "Crawl not found")
+
+    crawl_root = Path(crawl.output_dir).resolve()
+    file_path = (crawl_root / path).resolve()
+    if not file_path.is_file() or crawl_root not in file_path.parents:
+        raise HttpError(404, "Crawl file not found")
+
+    response = FileResponse(file_path.open("rb"))
+    response["Cache-Control"] = "no-store, no-cache, max-age=0, must-revalidate"
+    response["Pragma"] = "no-cache"
+    response["Expires"] = "0"
+    response["X-Content-Type-Options"] = "nosniff"
+    return response
+
+
+@router.get("/crawl/{crawl_id}/files/{filename}", auth=None, url_name="crawl_file_root")
+def crawl_file_root(request: HttpRequest, crawl_id: str, filename: str):
+    return crawl_file(request, crawl_id, filename)
+
+
+@router.get("/crawl/{crawl_id}/files/{folder}/{filename}", auth=None, url_name="crawl_file_nested_1")
+def crawl_file_nested_1(request: HttpRequest, crawl_id: str, folder: str, filename: str):
+    return crawl_file(request, crawl_id, f"{folder}/{filename}")
+
+
+@router.get("/crawl/{crawl_id}/files/{folder}/{subfolder}/{filename}", auth=None, url_name="crawl_file_nested_2")
+def crawl_file_nested_2(request: HttpRequest, crawl_id: str, folder: str, subfolder: str, filename: str):
+    return crawl_file(request, crawl_id, f"{folder}/{subfolder}/{filename}")
+
+
@router.patch("/crawl/{crawl_id}", response=CrawlSchema, url_name="patch_crawl")
 def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
    """Update a crawl (e.g., set status=sealed to cancel queued work)."""
-    crawl = Crawl.objects.get(id__icontains=crawl_id)
+    crawl = get_crawl_by_ref(crawl_id)
    payload = data.dict(exclude_unset=True)
    update_fields = ["modified_at"]

+    action = payload.pop("action", None)
+    if action:
+        if action == "pause":
+            crawl.pause()
+            return crawl
+        if action in ("resume", "unpause"):
+            crawl.resume()
+            return crawl
+        if action == "cancel":
+            crawl.cancel()
+            return crawl
+        raise HttpError(400, f"Invalid action: {action}")
+
    tags = payload.pop("tags", None)
    tags_str = payload.pop("tags_str", None)
    if tags is not None or tags_str is not None:
@@ -164,9 +232,10 @@ def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
    if "status" in payload:
        if payload["status"] not in Crawl.StatusChoices.values:
            raise HttpError(400, f"Invalid status: {payload['status']}")
+        if payload["status"] == Crawl.StatusChoices.SEALED:
+            crawl.cancel()
+            return crawl
        crawl.status = payload["status"]
-        if crawl.status == Crawl.StatusChoices.SEALED and "retry_at" not in payload:
-            crawl.retry_at = None
        update_fields.append("status")

    if "retry_at" in payload:
@@ -174,22 +243,12 @@ def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
        update_fields.append("retry_at")

    crawl.save(update_fields=update_fields)
-
-    if payload.get("status") == Crawl.StatusChoices.SEALED:
-        Snapshot.objects.filter(
-            crawl=crawl,
-            status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED],
-        ).update(
-            status=Snapshot.StatusChoices.SEALED,
-            retry_at=None,
-            modified_at=timezone.now(),
-        )
    return crawl


@router.delete("/crawl/{crawl_id}", response=CrawlDeleteResponseSchema, url_name="delete_crawl")
 def delete_crawl(request: HttpRequest, crawl_id: str):
-    crawl = Crawl.objects.get(id__icontains=crawl_id)
+    crawl = get_crawl_by_ref(crawl_id)
    crawl_id_str = str(crawl.id)
    snapshot_count = crawl.snapshot_set.count()
    deleted_count, _ = crawl.delete()
@@ -0,0 +1,168 @@
+__package__ = "archivebox.api"
+
+import json
+from datetime import datetime
+from typing import Any
+from uuid import UUID
+
+from django.db.models import Q
+from django.http import HttpRequest
+from ninja import Router, Schema
+from ninja.pagination import paginate
+from pydantic import Field
+
+from archivebox.api.v1_core import CustomPagination
+from archivebox.personas.importers import validate_persona_name
+from archivebox.personas.models import Persona
+
+
+router = Router(tags=["Personas"])
+
+
+class PersonaBrowserSettingsSchema(Schema):
+    user_agent: str = ""
+    viewport_size: str = ""
+    viewport_device_scale_factor: float | None = None
+    language: str = ""
+    timezone: str = ""
+    geolocation: dict[str, Any] | None = None
+
+
+class PersonaSyncSchema(Schema):
+    extension_persona_id: str
+    name: str
+    settings: PersonaBrowserSettingsSchema = Field(default_factory=PersonaBrowserSettingsSchema)
+    cookies_txt: str = ""
+    auth_json: dict[str, Any] = Field(default_factory=dict)
+
+
+class PersonaSchema(Schema):
+    TYPE: str = "personas.models.Persona"
+    id: UUID
+    name: str
+    created_at: datetime
+    created_by_id: str
+    created_by_username: str
+    config: dict[str, Any] | None
+
+    @staticmethod
+    def resolve_created_by_id(obj):
+        return str(obj.created_by.pk)
+
+    @staticmethod
+    def resolve_created_by_username(obj) -> str:
+        return obj.created_by.username
+
+    @staticmethod
+    def resolve_config(obj):
+        # Redact credential values so REST responses don't leak the raw
+        # token/secret/api-key the operator stored in Persona.config.
+        from archivebox.config.common import redact_sensitive_config
+
+        return redact_sensitive_config(obj.config)
+
+
+class PersonaSyncResponseSchema(Schema):
+    success: bool
+    created: bool
+    persona: PersonaSchema
+    cookies_file_written: bool
+    auth_file_written: bool
+
+
+def browser_settings_to_config(extension_persona_id: str, settings: PersonaBrowserSettingsSchema) -> dict[str, Any]:
+    config: dict[str, Any] = {
+        "BROWSER_EXTENSION_PERSONA_ID": extension_persona_id,
+        "BROWSER_EXTENSION_SYNCED_AT": datetime.utcnow().isoformat() + "Z",
+    }
+
+    if settings.user_agent:
+        config.update(
+            {
+                "USER_AGENT": settings.user_agent,
+                "CHROME_USER_AGENT": settings.user_agent,
+                "WGET_USER_AGENT": settings.user_agent,
+                "CURL_USER_AGENT": settings.user_agent,
+            },
+        )
+    if settings.viewport_size:
+        config.update(
+            {
+                "RESOLUTION": settings.viewport_size,
+                "CHROME_RESOLUTION": settings.viewport_size,
+            },
+        )
+    if settings.viewport_device_scale_factor is not None:
+        config["BROWSER_DEVICE_SCALE_FACTOR"] = settings.viewport_device_scale_factor
+    if settings.language:
+        config["BROWSER_LANGUAGE"] = settings.language
+    if settings.timezone:
+        config["BROWSER_TIMEZONE"] = settings.timezone
+    if settings.geolocation:
+        config["BROWSER_GEOLOCATION"] = settings.geolocation
+
+    return config
+
+
+def find_persona(extension_persona_id: str, name: str) -> Persona | None:
+    return (
+        Persona.objects.filter(
+            Q(config__BROWSER_EXTENSION_PERSONA_ID=extension_persona_id) | Q(name=name),
+        )
+        .order_by("created_at")
+        .first()
+    )
+
+
+@router.get("/personas", response=list[PersonaSchema], url_name="get_personas")
+@paginate(CustomPagination)
+def get_personas(request: HttpRequest):
+    """List personas available on this ArchiveBox server."""
+    return Persona.objects.all().order_by("name")
+
+
+@router.post("/sync", response=PersonaSyncResponseSchema, url_name="sync_persona")
+def sync_persona(request: HttpRequest, payload: PersonaSyncSchema):
+    """
+    Create or update a Persona from a browser extension profile export.
+
+    The extension sends browser settings plus portable auth artifacts. The server
+    keeps browser override settings in Persona.config and writes cookies.txt /
+    auth.json into the persona directory for extractors to consume.
+    """
+    name = payload.name.strip()
+    is_valid, error_message = validate_persona_name(name)
+    if not is_valid:
+        raise ValueError(error_message)
+
+    persona = find_persona(payload.extension_persona_id, name)
+    created = persona is None
+    if persona is None:
+        persona = Persona(name=name)
+        if request.user.is_authenticated:
+            persona.created_by = request.user
+
+    persona.config = {
+        **(persona.config or {}),
+        **browser_settings_to_config(payload.extension_persona_id, payload.settings),
+    }
+    persona.save()
+    persona.ensure_dirs()
+
+    cookies_written = False
+    if payload.cookies_txt.strip():
+        (persona.path / "cookies.txt").write_text(payload.cookies_txt)
+        cookies_written = True
+
+    auth_written = False
+    if payload.auth_json:
+        (persona.path / "auth.json").write_text(json.dumps(payload.auth_json, indent=2, sort_keys=True) + "\n")
+        auth_written = True
+
+    return {
+        "success": True,
+        "created": created,
+        "persona": persona,
+        "cookies_file_written": cookies_written,
+        "auth_file_written": auth_written,
+    }
@@ -0,0 +1,32 @@
+__package__ = "archivebox.api"
+
+from typing import Any
+from collections.abc import Callable
+import logging
+
+from django.db import transaction
+from signal_webhooks.handlers import sync_task_handler
+
+
+logger = logging.getLogger(__name__)
+
+
+def warning_error_handler(hook: Any, error: Exception | None) -> None:
+    if error is not None:
+        logger.warning("Outbound webhook %r failed: %s", hook.name, error)
+        return
+
+    logger.warning("Outbound webhook %r returned a non-success response.", hook.name)
+
+
+def transaction_on_commit_task_handler(hook: Callable[..., None], **kwargs: Any) -> None:
+    def run_webhook() -> None:
+        try:
+            sync_task_handler(hook, **kwargs)
+        except Exception:
+            logger.warning("Outbound webhook failed after transaction commit.", exc_info=True)
+
+    try:
+        transaction.on_commit(run_webhook)
+    except Exception:
+        logger.warning("Could not schedule outbound webhook after transaction commit.", exc_info=True)
@@ -3,18 +3,50 @@
 __package__ = "archivebox.base_models"

 import json
+import uuid
 from collections.abc import Mapping
-from typing import NotRequired, TypedDict
+from typing import NotRequired, TypedDict, cast

 from django import forms
 from django.contrib import admin
 from django.db import models
 from django.forms.renderers import BaseRenderer
 from django.http import HttpRequest, QueryDict
+from django.urls import path, register_converter
 from django.utils.safestring import SafeString, mark_safe
 from django_object_actions import DjangoObjectActions


+class HexUUIDConverter:
+    """URL path converter that canonicalizes UUIDs to their 32-char hex form.
+
+    Accepts both the hyphenated (``aaaaaaaa-bbbb-...``) and bare-hex
+    (``aaaaaaaabbbb...``) UUID strings on the way in (Django's UUIDField
+    parses either), but ``to_url`` always emits the bare-hex form. This is
+    what makes ``reverse("admin:app_model_change", args=[obj.pk])`` produce
+    ``/admin/app/model/06a1a8facb0d.../change/`` instead of the default
+    hyphenated rendering — admin links throughout the app reverse through
+    this converter once ``BaseModelAdmin.get_urls`` swaps in
+    ``<hexuuid:object_id>`` below.
+    """
+
+    regex = r"[0-9a-fA-F]{32}|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"
+
+    def to_python(self, value: str) -> str:
+        # Strip hyphens but stay as a string — Django admin treats the
+        # captured object_id as a string and calls ``model._meta.pk.to_python``
+        # itself, so we don't want to short-circuit that.
+        return value.replace("-", "")
+
+    def to_url(self, value) -> str:
+        if isinstance(value, uuid.UUID):
+            return value.hex
+        return str(value).replace("-", "")
+
+
+register_converter(HexUUIDConverter, "hexuuid")
+
+
 class ConfigOption(TypedDict):
    plugin: str
    type: str | list[str]
@@ -44,22 +76,23 @@ class KeyValueWidget(forms.Widget):
    def _get_config_options(self) -> dict[str, ConfigOption]:
        """Get available config options from plugins."""
        try:
-            from archivebox.hooks import discover_plugin_configs
+            from archivebox.config.common import config_field_metadata

-            plugin_configs = discover_plugin_configs()
            options: dict[str, ConfigOption] = {}
-            for plugin_name, schema in plugin_configs.items():
-                for key, prop in schema.get("properties", {}).items():
-                    option: ConfigOption = {
-                        "plugin": plugin_name,
-                        "type": prop.get("type", "string"),
-                        "default": prop.get("default", ""),
-                        "description": prop.get("description", ""),
-                    }
+            for key, metadata in config_field_metadata().items():
+                option_type = metadata.get("type", "string")
+                option: ConfigOption = {
+                    "plugin": str(metadata.get("plugin", "archivebox")),
+                    "type": cast(str | list[str], option_type if isinstance(option_type, (str, list)) else str(option_type)),
+                    "default": metadata.get("default", ""),
+                    "description": str(metadata.get("description", "")),
+                }
+                schema = metadata.get("schema")
+                if isinstance(schema, Mapping):
                    for schema_key in ("enum", "pattern", "minimum", "maximum"):
-                        if schema_key in prop:
-                            option[schema_key] = prop[schema_key]
-                    options[key] = option
+                        if schema_key in schema:
+                            option[schema_key] = schema[schema_key]
+                options[key] = option
            return options
        except Exception:
            return {}
@@ -100,7 +133,7 @@ class KeyValueWidget(forms.Widget):
        config_meta_json = json.dumps(config_options)

        html = f'''
-        <div id="{widget_id}_container" class="key-value-editor" style="max-width: 700px;">
+        <div id="{widget_id}_container" class="key-value-editor" style="width: 100%; max-width: none;">
            <datalist id="{widget_id}_keys">
                {datalist_options}
            </datalist>
@@ -210,9 +243,6 @@ class KeyValueWidget(forms.Widget):
                            return 'Example: ["value"]';
                        }}
                        if (types.includes('object')) {{
-                            if (key === 'SAVE_ALLOWLIST' || key === 'SAVE_DENYLIST') {{
-                                return 'Example: {{"^https://example\\\\.com": ["wget"]}}';
-                            }}
                            return 'Example: {{"key": "value"}}';
                        }}
                        return '';
@@ -221,8 +251,6 @@ class KeyValueWidget(forms.Widget):
                    function isRegexConfigKey_{widget_id}(key) {{
                        return key === 'URL_ALLOWLIST' ||
                            key === 'URL_DENYLIST' ||
-                            key === 'SAVE_ALLOWLIST' ||
-                            key === 'SAVE_DENYLIST' ||
                            key.endsWith('_PATTERN') ||
                            key.includes('REGEX');
                    }}
@@ -589,7 +617,7 @@ class KeyValueWidget(forms.Widget):
                        var newRow = document.createElement('div');
                        newRow.className = 'key-value-row';
                        newRow.style.cssText = 'margin-bottom: 6px;';
-                        newRow.innerHTML = '<div style="display: flex; gap: 8px; align-items: center;">' +
+                        newRow.innerHTML = '<div class="kv-inputs" style="display: flex; gap: 8px; align-items: center;">' +
                            '<input type="text" class="kv-key" placeholder="KEY" list="{widget_id}_keys" ' +
                            'style="flex: 1; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">' +
                            '<input type="text" class="kv-value" placeholder="value" ' +
@@ -616,17 +644,74 @@ class KeyValueWidget(forms.Widget):

                    window.updateHiddenField_{widget_id} = updateHiddenField_{widget_id};

+                    function focusConfigKeyFromHash_{widget_id}() {{
+                        // Deep-link affordance: ``…/change/#SOME_KEY`` jumps directly
+                        // to (or creates) the matching row in this editor. Used by
+                        // the in-banner "pin via admin" link and the
+                        // ``→ Edit <KEY> in Machine.config`` shortcut on the live
+                        // config detail page.
+                        var hash = (window.location.hash || '').replace(/^#/, '').trim();
+                        if (!hash || !/^[A-Z][A-Z0-9_]*$/.test(hash)) {{
+                            return;
+                        }}
+                        var container = document.getElementById('{widget_id}_rows');
+                        if (!container) {{
+                            return;
+                        }}
+                        var match = null;
+                        container.querySelectorAll('.key-value-row').forEach(function(row) {{
+                            if (match) {{ return; }}
+                            var keyInput = row.querySelector('.kv-key');
+                            if (keyInput && keyInput.value.trim() === hash) {{
+                                match = row;
+                            }}
+                        }});
+                        if (!match) {{
+                            // No existing row for this key — prepopulate one with the
+                            // key filled in but value left blank so the operator just
+                            // types/pastes the value and hits save.
+                            window.addKeyValueRow_{widget_id}();
+                            var rows = container.querySelectorAll('.key-value-row');
+                            match = rows[rows.length - 1];
+                            var keyInput = match.querySelector('.kv-key');
+                            if (keyInput) {{
+                                keyInput.value = hash;
+                                keyInput.dispatchEvent(new Event('input', {{ bubbles: true }}));
+                            }}
+                        }}
+                        if (!match) {{
+                            return;
+                        }}
+                        match.scrollIntoView({{ behavior: 'smooth', block: 'center' }});
+                        var prevOutline = match.style.outline;
+                        match.style.outline = '2px solid #f59e0b';
+                        match.style.outlineOffset = '2px';
+                        match.style.transition = 'outline 1.2s ease-out';
+                        setTimeout(function() {{
+                            match.style.outline = prevOutline || 'none';
+                        }}, 1400);
+                        var valueInput = match.querySelector('.kv-value');
+                        if (valueInput) {{
+                            valueInput.focus();
+                            try {{ valueInput.setSelectionRange(valueInput.value.length, valueInput.value.length); }} catch (e) {{}}
+                        }}
+                    }}
+
                    // Initialize on load
                    document.addEventListener('DOMContentLoaded', function() {{
                        initializeRows_{widget_id}();
                        updateHiddenField_{widget_id}();
+                        focusConfigKeyFromHash_{widget_id}();
                    }});
                    // Also run immediately in case DOM is already ready
                    if (document.readyState !== 'loading') {{
                        initializeRows_{widget_id}();
                        updateHiddenField_{widget_id}();
+                        focusConfigKeyFromHash_{widget_id}();
                    }}

+                    window.addEventListener('hashchange', focusConfigKeyFromHash_{widget_id});
+
                    // Update on any input change
                    var rowsEl_{widget_id} = document.getElementById('{widget_id}_rows');

@@ -653,12 +738,34 @@ class KeyValueWidget(forms.Widget):
        return mark_safe(html)

    def _render_row(self, widget_id: str, key: str, value: str) -> str:
+        from archivebox.config.common import is_sensitive_config_key
+
+        # Sensitive keys (``*TOKEN*``, ``*SECRET*``, ``*API_KEY*``, ``*APIKEY*``) are
+        # rendered write-only: the input is a password field with a placeholder
+        # showing the value is set, but the raw value is NEVER sent to the browser.
+        # When the user submits the form with the field left blank, the
+        # ``ConfigEditorMixin.save_model`` hook re-merges the previously-saved
+        # value so leaving it untouched is a no-op rather than a destructive clear.
+        is_sensitive = is_sensitive_config_key(key)
+        has_value = bool(value)
+        if is_sensitive:
+            input_type = "password"
+            rendered_value = ""
+            placeholder = (
+                "•••••• (saved — enter new value to replace, clear by deleting row)" if has_value else "value (will be saved write-only)"
+            )
+            extra_attrs = ' autocomplete="off" data-sensitive="1"' + (' data-had-value="1"' if has_value else "")
+        else:
+            input_type = "text"
+            rendered_value = self._escape(value)
+            placeholder = "value"
+            extra_attrs = ""
        return f'''
            <div class="key-value-row" style="margin-bottom: 6px;">
-                <div style="display: flex; gap: 8px; align-items: center;">
+                <div class="kv-inputs" style="display: flex; gap: 8px; align-items: center;">
                    <input type="text" class="kv-key" value="{self._escape(key)}" placeholder="KEY" list="{widget_id}_keys"
                           style="flex: 1; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">
-                    <input type="text" class="kv-value" value="{self._escape(value)}" placeholder="value"
+                    <input type="{input_type}" class="kv-value" value="{rendered_value}" placeholder="{self._escape(placeholder)}"{extra_attrs}
                           style="flex: 2; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">
                    <datalist class="kv-value-options"></datalist>
                    <button type="button" onclick="removeKeyValueRow_{widget_id}(this)"
@@ -693,7 +800,7 @@ class ConfigEditorMixin(admin.ModelAdmin):

    def formfield_for_dbfield(
        self,
-        db_field: models.Field[object, object],
+        db_field: models.Field,
        request: HttpRequest,
        **kwargs: object,
    ) -> forms.Field | None:
@@ -702,6 +809,37 @@ class ConfigEditorMixin(admin.ModelAdmin):
            kwargs["widget"] = KeyValueWidget()
        return super().formfield_for_dbfield(db_field, request, **kwargs)

+    def save_model(self, request: HttpRequest, obj, form, change):
+        """Preserve write-only redacted credentials on save.
+
+        The KeyValueWidget renders sensitive keys (``*TOKEN*``, ``*SECRET*``,
+        ``*API_KEY*``, ``*APIKEY*``) with an empty value + password input —
+        the real value never leaves the server. On submit, an empty value
+        for a sensitive key that was previously set means "leave untouched",
+        not "clear it." We honor that here by re-merging the stored value
+        before the row is written. Explicitly removing the row in the UI
+        still clears it (the key is gone from the submitted JSON, so there's
+        nothing to merge over).
+        """
+        from archivebox.config.common import is_sensitive_config_key
+
+        if change and obj.pk and obj.config is not None:
+            try:
+                stored = type(obj).objects.filter(pk=obj.pk).values_list("config", flat=True).first() or {}
+            except Exception:
+                stored = {}
+            if isinstance(stored, dict):
+                new_config = dict(obj.config or {})
+                for key, new_value in list(new_config.items()):
+                    if not is_sensitive_config_key(key):
+                        continue
+                    if new_value not in (None, "") and new_value != "********":
+                        continue
+                    if key in stored:
+                        new_config[key] = stored[key]
+                obj.config = new_config
+        super().save_model(request, obj, form, change)
+

 class BaseModelAdmin(DjangoObjectActions, admin.ModelAdmin):
    list_display = ("id", "created_at", "created_by")
@@ -724,3 +862,30 @@ class BaseModelAdmin(DjangoObjectActions, admin.ModelAdmin):
        if "created_by" in form.base_fields:
            form.base_fields["created_by"].initial = request.user
        return form
+
+    def get_urls(self):
+        """Swap the per-object admin URLs from ``<path:object_id>`` to
+        ``<hexuuid:object_id>`` so canonical change/delete/history URLs use the
+        32-char hex form. The hyphenated form still resolves because the
+        converter's regex accepts both — Django reverses through ``to_url``
+        which always emits hex, so links in templates / changelists / inline
+        formsets all canonicalize automatically.
+
+        Non-UUID PKs (an ``IntegerField`` PK on some legacy table, for example)
+        won't match the converter's regex and fall back to the default
+        ``<path:object_id>`` patterns we still include after our swap.
+        """
+        info = self.opts.app_label, self.opts.model_name
+        object_routes = [
+            path("<hexuuid:object_id>/history/", self.admin_site.admin_view(self.history_view), name="%s_%s_history" % info),
+            path("<hexuuid:object_id>/delete/", self.admin_site.admin_view(self.delete_view), name="%s_%s_delete" % info),
+            path("<hexuuid:object_id>/change/", self.admin_site.admin_view(self.change_view), name="%s_%s_change" % info),
+        ]
+        # Append after super().get_urls() so our patterns are the
+        # *last-registered* ones with the canonical admin URL names — Django's
+        # reverse() picks the later registration when names collide, which is
+        # how we make ``reverse("admin:app_model_change", args=[obj.pk])``
+        # emit the hex form. The original ``<path:object_id>`` routes stay in
+        # place as a fallback for non-UUID PKs and for resolving inbound
+        # hyphenated URLs (the ``hexuuid`` regex accepts both forms anyway).
+        return super().get_urls() + object_routes
@@ -2,11 +2,17 @@

 __package__ = "archivebox.base_models"

-from archivebox.uuid_compat import uuid7
+import json
+import shutil
+from typing import Any
+
+from archivebox.uuid_compat import CompactUUIDField, uuid7
 from pathlib import Path

 from django.db import models
 from django.db.models import F
+from django.db import transaction
+from django.db.models.signals import pre_delete
 from django.utils import timezone
 from django.contrib.auth import get_user_model
 from django.urls import reverse_lazy
@@ -14,6 +20,27 @@ from django.conf import settings

 from django_stubs_ext.db.models import TypedModelMeta

+from archivebox.config import CONSTANTS
+
+
+def normalize_config_json_values(config: Any) -> Any:
+    if not isinstance(config, dict):
+        return config
+
+    normalized = dict(config)
+    for key, value in list(normalized.items()):
+        if not isinstance(value, str) or len(value) < 2:
+            continue
+        if value[:1] != '"' or value[-1:] != '"':
+            continue
+        try:
+            decoded = json.loads(value)
+        except ValueError:
+            continue
+        if isinstance(decoded, str):
+            normalized[key] = decoded
+    return normalized
+

 def get_or_create_system_user_pk(username="system"):
    User = get_user_model()
@@ -32,7 +59,7 @@ class AutoDateTimeField(models.DateTimeField):
    """DateTimeField that automatically updates on save (legacy compatibility)."""

    def pre_save(self, model_instance, add):
-        if add or not getattr(model_instance, self.attname):
+        if add or self.attname not in model_instance.__dict__ or not model_instance.__dict__[self.attname]:
            value = timezone.now()
            setattr(model_instance, self.attname, value)
            return value
@@ -40,7 +67,7 @@ class AutoDateTimeField(models.DateTimeField):


 class ModelWithUUID(models.Model):
-    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
    created_at = models.DateTimeField(default=timezone.now, db_index=True)
    modified_at = models.DateTimeField(auto_now=True)
    created_by = models.ForeignKey(
@@ -88,6 +115,10 @@ class ModelWithHealthStats(models.Model):
    class Meta(TypedModelMeta):
        abstract = True

+    @property
+    def admin_change_url(self) -> str:
+        return f"/admin/{self._meta.app_label}/{self._meta.model_name}/{self.pk}/change/"
+
    @property
    def health(self) -> int:
        total = max(self.num_uses_failed + self.num_uses_succeeded, 1)
@@ -96,7 +127,12 @@ class ModelWithHealthStats(models.Model):
    def increment_health_stats(self, success: bool):
        """Atomically increment success or failure counter using F() expression."""
        field = "num_uses_succeeded" if success else "num_uses_failed"
-        type(self).objects.filter(pk=self.pk).update(**{field: F(field) + 1})
+        type(self).objects.filter(pk=self.pk).update(
+            **{
+                field: F(field) + 1,
+                "modified_at": timezone.now(),
+            },
+        )


 class ModelWithConfig(models.Model):
@@ -107,14 +143,99 @@ class ModelWithConfig(models.Model):
    class Meta(TypedModelMeta):
        abstract = True

+    def save(self, *args, **kwargs):
+        normalized_config = normalize_config_json_values(self.config)
+        if normalized_config != self.config:
+            self.config = normalized_config
+            update_fields = kwargs.get("update_fields")
+            if update_fields is not None:
+                kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "config"]))
+        super().save(*args, **kwargs)
+
+
+class ModelWithDeleteAfter(models.Model):
+    delete_after_final_statuses: tuple[str, ...] = ()
+    delete_at = models.DateTimeField(default=None, null=True, blank=True, db_index=True)
+
+    class Meta(TypedModelMeta):
+        abstract = True
+
+    def save(self, *args, **kwargs):
+        update_fields = kwargs.get("update_fields")
+        if self.delete_at is None:
+            self.set_delete_at_from_config()
+            if self.delete_at is not None and update_fields is not None:
+                kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "delete_at"]))
+        super().save(*args, **kwargs)
+
+    def get_delete_after_config_value(self):
+        from archivebox.config.common import get_config
+
+        return get_config(include_machine=False, resolve_plugins=False).DELETE_AFTER
+
+    def set_delete_at_from_config(self, config_value=None) -> bool:
+        if self.delete_at is not None:
+            return False
+
+        from archivebox.config.common import parse_delete_after
+
+        duration = parse_delete_after(self.get_delete_after_config_value() if config_value is None else config_value)
+        if duration is None:
+            return False
+
+        self.delete_at = (self.created_at or timezone.now()) + duration
+        return True
+
+    @classmethod
+    def missing_delete_at_candidates(cls):
+        return cls.objects.none()
+
+    @classmethod
+    def delete_expired(cls, *, batch_size: int = 100, backfill_missing: bool = True) -> int:
+        if backfill_missing:
+            missing_delete_at = list(cls.missing_delete_at_candidates().order_by("created_at", "pk")[:batch_size])
+            for obj in missing_delete_at:
+                if obj.set_delete_at_from_config():
+                    cls.objects.filter(pk=obj.pk, delete_at__isnull=True).update(
+                        delete_at=obj.delete_at,
+                        modified_at=timezone.now(),
+                    )
+
+        # Keep the expiration sweep anchored on delete_at. Some large tables
+        # have millions of final-status rows but almost no retained rows; adding
+        # the status filter before SQLite has narrowed by delete_at can make the
+        # runner scan the hot status index before it claims new work.
+        due_pks = list(
+            cls.objects.filter(delete_at__isnull=False, delete_at__lte=timezone.now())
+            .order_by("delete_at", "pk")
+            .values_list("pk", flat=True)[:batch_size],
+        )
+        if not due_pks:
+            return 0
+
+        queryset = cls.objects.filter(pk__in=due_pks)
+        if cls.delete_after_final_statuses:
+            queryset = queryset.filter(status__in=cls.delete_after_final_statuses)
+
+        count = 0
+        expired = list(queryset.order_by("delete_at", "pk"))
+        for obj in expired:
+            obj.delete()
+            count += 1
+        return count
+

 class ModelWithOutputDir(ModelWithUUID):
    class Meta(ModelWithUUID.Meta):
        abstract = True

+    _delete_signal_registered = False
+
    def save(self, *args, **kwargs):
        super().save(*args, **kwargs)
-        Path(self.output_dir).mkdir(parents=True, exist_ok=True)
+        output_dir = Path(self.output_dir)
+        # Avoid holding SQLite write transactions open across slow filesystem work.
+        transaction.on_commit(lambda: output_dir.mkdir(parents=True, exist_ok=True))
        # Note: index.json is deprecated, models should use write_index_jsonl() for full data

    @property
@@ -132,3 +253,51 @@ class ModelWithOutputDir(ModelWithUUID):
    @property
    def output_dir(self) -> Path:
        raise NotImplementedError(f"{self.__class__.__name__} must implement output_dir property")
+
+    def output_paths_for_delete(self) -> tuple[Path, ...]:
+        return (Path(self.output_dir),)
+
+    @classmethod
+    def validate_output_paths_for_delete(cls, paths) -> tuple[Path, ...]:
+        data_dir = CONSTANTS.DATA_DIR.resolve()
+        safe_paths = []
+        for raw_path in paths:
+            path = Path(raw_path)
+            is_safe = False
+            for candidate in (path.absolute(), path.resolve()):
+                try:
+                    candidate.relative_to(data_dir)
+                    is_safe = True
+                    break
+                except ValueError:
+                    continue
+            if not is_safe:
+                raise ValueError(f"Refusing to delete output path outside DATA_DIR: {path}")
+            safe_paths.append(path)
+        return tuple(safe_paths)
+
+    @classmethod
+    def delete_output_paths(cls, paths) -> None:
+        for path in cls.validate_output_paths_for_delete(paths):
+            if path.is_symlink() or path.is_file():
+                path.unlink(missing_ok=True)
+            elif path.is_dir():
+                shutil.rmtree(path, ignore_errors=True)
+
+    @classmethod
+    def register_delete_signal(cls) -> None:
+        if cls._delete_signal_registered:
+            return
+
+        def schedule_output_dir_cleanup(sender, instance, **kwargs):
+            if not isinstance(instance, ModelWithOutputDir):
+                return
+            paths = instance.validate_output_paths_for_delete(instance.output_paths_for_delete())
+            transaction.on_commit(lambda paths=paths: instance.delete_output_paths(paths))
+
+        pre_delete.connect(
+            schedule_output_dir_cleanup,
+            dispatch_uid="archivebox.output_dir_cleanup_on_delete",
+            weak=False,
+        )
+        cls._delete_signal_registered = True
@@ -5,15 +5,25 @@ import sys
 from importlib import import_module

 import rich_click as click
-from rich import print
+from rich.console import Console

 from archivebox.config.version import VERSION

+STDERR = Console(stderr=True)
+

 if "--debug" in sys.argv:
    os.environ["DEBUG"] = "True"
    sys.argv.remove("--debug")

+# Universal `--init` flag: when passed to ANY subcommand (e.g. `archivebox server --init`,
+# `archivebox add --init`, `archivebox shell --init`), run a `quick` archivebox init before
+# the subcommand executes. Strip it from argv here so each subcommand's own click parser
+# never sees it. Ignored for `help` and `init` themselves.
+if "--init" in sys.argv:
+    sys.argv = [arg for arg in sys.argv if arg != "--init"]
+    os.environ["ARCHIVEBOX_WANTS_INIT"] = "1"
+

 class ArchiveBoxGroup(click.Group):
    """lazy loading click group for archivebox commands"""
@@ -56,10 +66,6 @@ class ArchiveBoxGroup(click.Group):
        # Introspection commands
        "pluginmap": "archivebox.cli.archivebox_pluginmap.main",
    }
-    legacy_model_commands = {
-        "crawl": "archivebox.cli.archivebox_crawl_compat.main",
-        "snapshot": "archivebox.cli.archivebox_snapshot_compat.main",
-    }
    all_subcommands = {
        **meta_commands,
        **setup_commands,
@@ -71,51 +77,43 @@ class ArchiveBoxGroup(click.Group):
        "import": "add",
        "archive": "add",
    }
-    legacy_model_subcommands = {
-        "crawl": {"create", "list", "update", "delete"},
-        "snapshot": {"create", "list", "update", "delete"},
-    }

    @classmethod
    def get_canonical_name(cls, cmd_name):
        return cls.renamed_commands.get(cmd_name, cmd_name)

    @classmethod
-    def _should_use_legacy_model_command(cls, cmd_name: str) -> bool:
-        if cmd_name not in cls.legacy_model_commands:
-            return False
+    def _needs_django_for_lazy_import(cls, cmd_name: str) -> bool:
+        wants_help = any(arg in ("-h", "--help", "--version") for arg in sys.argv[1:])
+        return not wants_help and (cmd_name in cls.archive_commands or cmd_name in cls.model_commands)

-        try:
-            arg_idx = sys.argv.index(cmd_name)
-        except ValueError:
-            return False
+    @classmethod
+    def _setup_django_for_lazy_import(cls, cmd_name: str) -> None:
+        if not cls._needs_django_for_lazy_import(cmd_name):
+            return

-        remaining_args = sys.argv[arg_idx + 1 :]
-        if not remaining_args:
-            return False
+        from django.apps import apps

-        first_arg = remaining_args[0]
-        if first_arg in ("-h", "--help"):
-            return False
+        if apps.ready:
+            return

-        return first_arg not in cls.legacy_model_subcommands[cmd_name]
+        from archivebox.config.django import setup_django
+
+        setup_django()

    def get_command(self, ctx, cmd_name):
        # handle renamed commands
        if cmd_name in self.renamed_commands:
            new_name = self.renamed_commands[cmd_name]
-            print(
+            STDERR.print(
                f" [violet]Hint:[/violet] `archivebox {cmd_name}` has been renamed to `archivebox {new_name}`",
-                file=sys.stderr,
            )
            cmd_name = new_name
            ctx.invoked_subcommand = cmd_name

-        if self._should_use_legacy_model_command(cmd_name):
-            return self._lazy_load(self.legacy_model_commands[cmd_name])
-
        # handle lazy loading of commands
        if cmd_name in self.all_subcommands:
+            self._setup_django_for_lazy_import(cmd_name)
            return self._lazy_load(cmd_name)

        # fall-back to using click's default command lookup
@@ -130,9 +128,9 @@ class ArchiveBoxGroup(click.Group):

        # print(f'LAZY LOADING {import_path}')
        mod = import_module(modname)
-        func = getattr(mod, funcname)
+        func = vars(mod)[funcname]

-        if not hasattr(func, "__doc__"):
+        if func.__doc__ is None:
            raise ValueError(f"lazy loading of {import_path} failed - no docstring found on method")

        # if not isinstance(cmd, click.BaseCommand):
@@ -156,7 +154,8 @@ def cli(ctx, help=False):

    # if the subcommand is in archive_commands or model_commands,
    # then we need to set up the django environment and check that we're in a valid data folder
-    if subcommand in ArchiveBoxGroup.archive_commands or subcommand in ArchiveBoxGroup.model_commands:
+    wants_help = any(arg in ("-h", "--help", "--version") for arg in sys.argv[1:])
+    if not wants_help and (subcommand in ArchiveBoxGroup.archive_commands or subcommand in ArchiveBoxGroup.model_commands):
        # print('SETUP DJANGO AND CHECK DATA FOLDER')
        try:
            if subcommand == "server":
@@ -165,34 +164,59 @@ def cli(ctx, help=False):
                    os.environ["ARCHIVEBOX_RUNSERVER"] = "1"
                    if "--reload" in sys.argv:
                        os.environ["ARCHIVEBOX_AUTORELOAD"] = "1"
-                        from archivebox.config.common import STORAGE_CONFIG
-
-                        os.environ["ARCHIVEBOX_RUNSERVER_PIDFILE"] = str(STORAGE_CONFIG.TMP_DIR / "runserver.pid")

            from archivebox.config.django import setup_django
-            from archivebox.misc.checks import check_data_folder
+            from archivebox.misc.checks import check_data_folder, check_migrations

            setup_django()
+            if os.environ.get("ARCHIVEBOX_WANTS_INIT") == "1" and subcommand not in ("init", "help"):
+                # Universal `--init` was passed: build/upgrade the data folder before
+                # the regular preflight runs, so it succeeds on a fresh dir and an
+                # out-of-date schema both. Drop the env var afterwards so spawned
+                # subprocesses (supervisord workers, daphne, runner, etc.) inherit
+                # a clean env and don't re-trigger init in every child.
+                from archivebox.cli.archivebox_init import init as archivebox_init
+
+                archivebox_init(quick=True)
+                os.environ.pop("ARCHIVEBOX_WANTS_INIT", None)
            check_data_folder()
+            if subcommand != "update":
+                check_migrations(auto_apply=True)
        except Exception as e:
-            print(f"[red][X] Error setting up Django or checking data folder: {e}[/red]", file=sys.stderr)
+            STDERR.print(f"[red][X] Error setting up Django or checking data folder: {e}[/red]")
            if subcommand not in ("manage", "shell"):  # not all management commands need django to be setup beforehand
                raise


-def main(args=None, prog_name=None, stdin=None):
+def main(args=None, prog_name=None):
    # show `docker run archivebox xyz` in help messages if running in docker
    IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes")
    IS_TTY = sys.stdin.isatty()
    prog_name = prog_name or (f"docker compose run{'' if IS_TTY else ' -T'} archivebox" if IN_DOCKER else "archivebox")

-    # stdin param allows passing input data from caller (used by __main__.py)
-    # currently not used by click-based CLI, but kept for backwards compatibility
+    previous_unraisablehook = sys.unraisablehook

+    def ignore_shutdown_unraisable(unraisable):
+        if isinstance(unraisable.exc_value, (KeyboardInterrupt, SystemExit)):
+            return
+        previous_unraisablehook(unraisable)
+
+    sys.unraisablehook = ignore_shutdown_unraisable
    try:
-        cli(args=args, prog_name=prog_name)
+        cli(args=args, prog_name=prog_name, standalone_mode=False)
+    except click.Abort:
+        STDERR.print("\n[red][X] Got CTRL+C. Exiting...[/red]")
+        raise SystemExit(130) from None
+    except click.ClickException as err:
+        err.show()
+        raise SystemExit(err.exit_code) from None
+    except click.exceptions.Exit as err:
+        raise SystemExit(err.exit_code) from None
    except KeyboardInterrupt:
-        print("\n\n[red][X] Got CTRL+C. Exiting...[/red]")
+        STDERR.print("\n[red][X] Got CTRL+C. Exiting...[/red]")
+        raise SystemExit(130) from None
+    finally:
+        sys.unraisablehook = previous_unraisablehook


 if __name__ == "__main__":
@@ -1,28 +1,26 @@
 #!/usr/bin/env python3
+from __future__ import annotations

 __package__ = "archivebox.cli"
 __command__ = "archivebox add"

 import sys
+import json
+import os
 from pathlib import Path

-from typing import TYPE_CHECKING
+from typing import Any, TYPE_CHECKING

 import rich_click as click

-from django.utils import timezone
-from django.db.models import QuerySet
-
 from archivebox.misc.util import enforce_types, docstring
 from archivebox.misc.util import parse_filesize_to_bytes
-from archivebox import CONSTANTS
-from archivebox.config.common import ARCHIVING_CONFIG, SERVER_CONFIG
-from archivebox.config.permissions import USER, HOSTNAME


 if TYPE_CHECKING:
-    from archivebox.core.models import Snapshot
+    from django.db.models import QuerySet
    from archivebox.crawls.models import Crawl
+    from archivebox.core.models import Snapshot


 def _collect_input_urls(args: tuple[str, ...]) -> list[str]:
@@ -47,21 +45,24 @@ def _collect_input_urls(args: tuple[str, ...]) -> list[str]:
@enforce_types
 def add(
    urls: str | list[str],
+    snapshot_ids: list[str] | None = None,
    depth: int | str = 0,
    max_urls: int = 0,
-    max_size: int | str = 0,
+    crawl_max_size: int | str = 0,
+    crawl_timeout: int = 0,
+    snapshot_max_size: int | str = 0,
+    crawl_max_concurrent_snapshots: int | None = None,
    tag: str = "",
    url_allowlist: str = "",
    url_denylist: str = "",
    parser: str = "auto",
    plugins: str = "",
    persona: str = "Default",
-    overwrite: bool = False,
-    update: bool | None = None,
    index_only: bool = False,
    bg: bool = False,
    created_by_id: int | None = None,
-) -> tuple["Crawl", QuerySet["Snapshot"]]:
+    config: dict[str, Any] | None = None,
+) -> tuple[Crawl, QuerySet[Snapshot]]:
    """Add a new URL or list of URLs to your archive.

    The flow is:
@@ -78,34 +79,74 @@ def add(

    depth = int(depth)
    max_urls = int(max_urls or 0)
-    max_size = parse_filesize_to_bytes(max_size)
+    crawl_max_size = parse_filesize_to_bytes(crawl_max_size)
+    crawl_timeout = int(crawl_timeout or 0)
+    snapshot_max_size = parse_filesize_to_bytes(snapshot_max_size)
+    from archivebox import CONSTANTS
+    from archivebox.config.permissions import USER, HOSTNAME
+    from archivebox.config.common import get_config
+
+    config_overrides = dict(config or {})
+    runtime_config = get_config()
+    crawl_max_concurrent_snapshots_override = crawl_max_concurrent_snapshots is not None
+    if crawl_max_concurrent_snapshots is None:
+        crawl_max_concurrent_snapshots = runtime_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS
+    crawl_max_concurrent_snapshots = int(crawl_max_concurrent_snapshots)

    if depth not in (0, 1, 2, 3, 4):
        raise ValueError("Depth must be 0-4")
    if max_urls < 0:
        raise ValueError("max_urls must be >= 0")
-    if max_size < 0:
-        raise ValueError("max_size must be >= 0")
+    if crawl_max_size < 0:
+        raise ValueError("crawl_max_size must be >= 0")
+    if crawl_timeout < 0:
+        raise ValueError("crawl_timeout must be >= 0")
+    if snapshot_max_size < 0:
+        raise ValueError("snapshot_max_size must be >= 0")
+    if crawl_max_concurrent_snapshots < 1:
+        raise ValueError("crawl_max_concurrent_snapshots must be >= 1")

    # import models once django is set up
-    from archivebox.core.models import Snapshot
    from archivebox.crawls.models import Crawl
+    from archivebox.core.models import Snapshot
    from archivebox.base_models.models import get_or_create_system_user_pk
    from archivebox.personas.models import Persona
    from archivebox.misc.logging_util import printable_filesize
    from archivebox.misc.system import get_dir_size
-    from archivebox.config.configset import get_config
-    from archivebox.services.runner import run_crawl
+    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
+    from django.utils import timezone

    created_by_id = created_by_id or get_or_create_system_user_pk()
    started_at = timezone.now()
-    if update is None:
-        update = not ARCHIVING_CONFIG.ONLY_NEW
+
+    if isinstance(urls, str):
+        url_list = [line.strip() for line in urls.splitlines() if line.strip()]
+    else:
+        url_list = [str(url).strip() for url in urls if str(url).strip()]
+    if snapshot_ids and len(snapshot_ids) != len(url_list):
+        raise ValueError("snapshot_ids length must match urls length")
+    admitted_urls: list[str] = []
+    admitted_snapshot_ids: list[str] | None = [] if snapshot_ids else None
+    for index, url in enumerate(url_list):
+        if Snapshot.is_archivebox_internal_url(url):
+            print(f"[yellow][!] Skipping internal ArchiveBox URL: {url}[/yellow]")
+            continue
+        admitted_urls.append(url)
+        if admitted_snapshot_ids is not None and snapshot_ids:
+            admitted_snapshot_ids.append(snapshot_ids[index])

    # 1. Save the provided URLs to sources/2024-11-05__23-59-59__cli_add.txt
    sources_file = CONSTANTS.SOURCES_DIR / f"{timezone.now().strftime('%Y-%m-%d__%H-%M-%S')}__cli_add.txt"
    sources_file.parent.mkdir(parents=True, exist_ok=True)
-    sources_file.write_text(urls if isinstance(urls, str) else "\n".join(urls))
+    if admitted_snapshot_ids is not None:
+        sources_file.write_text(
+            "\n".join(
+                json.dumps({"url": url, "id": admitted_snapshot_ids[index], "tags": tag, "depth": 0})
+                for index, url in enumerate(admitted_urls)
+            ),
+        )
+    else:
+        sources_file.write_text("\n".join(admitted_urls))

    # 2. Create a new Crawl with inline URLs
    cli_args = [*sys.argv]
@@ -115,32 +156,46 @@ def add(

    timestamp = timezone.now().strftime("%Y-%m-%d__%H-%M-%S")

-    # Read URLs directly into crawl
    urls_content = sources_file.read_text()
    persona_name = (persona or "Default").strip() or "Default"
-    plugins = plugins or str(get_config().get("PLUGINS") or "")
+    plugins = plugins or ""
    persona_obj, _ = Persona.objects.get_or_create(name=persona_name)
    persona_obj.ensure_dirs()
+    effective_persona_config = get_config(persona=persona_obj)
+
+    crawl_config = {
+        "PERMISSIONS": str(effective_persona_config.PERMISSIONS),
+        **({"INDEX_ONLY": True} if index_only else {}),
+        **({"PLUGINS": plugins} if plugins else {}),
+        **(
+            {"CRAWL_MAX_CONCURRENT_SNAPSHOTS": crawl_max_concurrent_snapshots}
+            if crawl_max_concurrent_snapshots_override
+            and crawl_max_concurrent_snapshots != int(effective_persona_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS)
+            else {}
+        ),
+        **({"CRAWL_MAX_URLS": max_urls} if max_urls else {}),
+        **({"CRAWL_MAX_SIZE": crawl_max_size} if crawl_max_size else {}),
+        **({"CRAWL_TIMEOUT": crawl_timeout} if crawl_timeout else {}),
+        **({"SNAPSHOT_MAX_SIZE": snapshot_max_size} if snapshot_max_size else {}),
+        **({"PARSER": parser} if parser != "auto" else {}),
+        **({"URL_ALLOWLIST": url_allowlist} if url_allowlist else {}),
+        **({"URL_DENYLIST": url_denylist} if url_denylist else {}),
+    }
+    # Caller-supplied overrides (e.g. {"ONLY_NEW": False}) are the highest
+    # priority for crawl-frozen keys. Runtime-derived execution keys are
+    # stripped by Crawl.save() and rederived when hooks run.
+    crawl_config.update(config_overrides)

    crawl = Crawl.objects.create(
        urls=urls_content,
        max_depth=depth,
-        max_urls=max_urls,
-        max_size=max_size,
        tags_str=tag,
        persona_id=persona_obj.id,
        label=f"{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]",
        created_by_id=created_by_id,
-        config={
-            "ONLY_NEW": not update,
-            "INDEX_ONLY": index_only,
-            "OVERWRITE": overwrite,
-            "PLUGINS": plugins,
-            "DEFAULT_PERSONA": persona_name,
-            "PARSER": parser,
-            **({"URL_ALLOWLIST": url_allowlist} if url_allowlist else {}),
-            **({"URL_DENYLIST": url_denylist} if url_denylist else {}),
-        },
+        status=Crawl.StatusChoices.QUEUED,
+        retry_at=None if index_only else timezone.now(),
+        config=crawl_config,
    )

    print(f"[green]\\[+] Created Crawl {crawl.id} with max_depth={depth}[/green]")
@@ -152,27 +207,8 @@ def add(
    #    Discovered URLs become child Snapshots (depth+1)

    if index_only:
-        # Just create the crawl but don't start processing
-        print("[yellow]\\[*] Index-only mode - crawl created but not started[/yellow]")
-        # Create snapshots for all URLs in the crawl
-        for url in crawl.get_urls_list():
-            snapshot, _ = Snapshot.objects.update_or_create(
-                crawl=crawl,
-                url=url,
-                defaults={
-                    "status": Snapshot.INITIAL_STATE,
-                    "retry_at": timezone.now(),
-                    "timestamp": str(timezone.now().timestamp()),
-                    "depth": 0,
-                },
-            )
-            if tag:
-                snapshot.save_tags(tag.split(","))
-            snapshot.ensure_crawl_symlink()
-        return crawl, crawl.snapshot_set.all()
-
-    if bg:
-        crawl.create_snapshots_from_urls()
+        print("[yellow]\\[*] Index-only mode - URLs queued, runner not started[/yellow]")
+        return crawl, crawl.snapshot_set.none()

    # 5. Start the crawl runner to process the queue
    #    The runner will:
@@ -182,25 +218,67 @@ def add(
    #    - Repeat until max_depth reached

    if bg:
-        # Background mode: just queue work and return (background runner via server will pick it up)
+        # Background mode: just queue work and return (background runner via server will pick it up).
        print(
            "[yellow]\\[*] URLs queued. The background runner will process them (run `archivebox server` or `archivebox run --daemon` if not already running).[/yellow]",
        )
    else:
        # Foreground mode: run full crawl runner until all work is done
        print("[green]\\[*] Starting crawl runner to process crawl...[/green]")
-        run_crawl(str(crawl.id))
+        from archivebox.machine.models import Process
+        from archivebox.core.takeover_util import command_owns_foreground_runner, current_command, standby_until_foreground_runner_needed
+        from archivebox.workers.supervisord_util import get_existing_supervisord_process, run_runner_worker, stop_own_supervisord_process
+
+        command = current_command(Process.TypeChoices.ADD, data_dir=CONSTANTS.DATA_DIR, url=first_url)
+        exit_code = 0
+        try:
+            try:
+                with foreground_shutdown_signals(first_signal_message=None), foreground_parent_watchdog():
+                    while True:
+                        standby_until_foreground_runner_needed(command, data_dir=CONSTANTS.DATA_DIR)
+                        exit_code = run_runner_worker(
+                            ["--crawl-id", str(crawl.id)],
+                            name=f"worker_runner_add_{os.getpid()}",
+                            interactive_interrupts=True,
+                        )
+                        crawl.refresh_from_db(fields=["status", "retry_at"])
+                        if exit_code == 0 and crawl.status == crawl.StatusChoices.SEALED:
+                            break
+                        # A shared supervisord can be stopped by another
+                        # foreground owner (for example `archivebox server`
+                        # shutting down) and stop the one-shot add runner while
+                        # this add's crawl is still runnable. Keep the add
+                        # process alive so it can re-enter the normal
+                        # foreground-runner ownership loop and finish its crawl.
+                        supervisor_gone = exit_code == 1 and get_existing_supervisord_process(quiet=True) is None
+                        if crawl.status in crawl.RUNNABLE_STATES and (exit_code in (0, 130, 143) or supervisor_gone):
+                            continue
+                        if exit_code == 0:
+                            break
+                        if not command_owns_foreground_runner(command, data_dir=CONSTANTS.DATA_DIR):
+                            continue
+                        raise SystemExit(exit_code)
+            except KeyboardInterrupt:
+                exit_code = 130
+                print("\n[red][X] archivebox add interrupted.[/red]")
+                print("[yellow]Hint: resume this crawl with:[/yellow]")
+                print(f"    [green]archivebox run --crawl-id={crawl.id}[/green]")
+                raise SystemExit(exit_code)
+        finally:
+            command.mark_exited(exit_code=exit_code)
+            stop_own_supervisord_process()

        # Print summary for foreground runs
        try:
            crawl.refresh_from_db()
-            snapshots_count = crawl.snapshot_set.count()
            try:
                from django.db.models import Count, Sum

-                totals = crawl.snapshot_set.aggregate(snapshot_count=Count("id"), total_bytes=Sum("archiveresult__output_size"))
-                total_bytes = int(totals["total_bytes"] or 0) if totals["snapshot_count"] else 0
+                totals = crawl.snapshot_set.aggregate(snapshot_count=Count("id"), total_bytes=Sum("output_size"))
+                snapshots_count = int(totals["snapshot_count"] or 0)
+                total_bytes = int(totals["total_bytes"] or 0)
            except Exception:
+                snapshots_count = crawl.snapshot_set.count()
                total_bytes, _, _ = get_dir_size(crawl.output_dir)
            total_size = printable_filesize(total_bytes)
            total_time = timezone.now() - started_at
@@ -221,12 +299,9 @@ def add(
            except Exception:
                rel_output_str = str(crawl.output_dir)

-            bind_addr = SERVER_CONFIG.BIND_ADDR or "127.0.0.1:8000"
-            if bind_addr.startswith("http://") or bind_addr.startswith("https://"):
-                base_url = bind_addr
-            else:
-                base_url = f"http://{bind_addr}"
-            admin_url = f"{base_url}/admin/crawls/crawl/{crawl.id}/change/"
+            from archivebox.core.routes_util import build_admin_url
+
+            admin_url = build_admin_url(f"/admin/crawls/crawl/{crawl.id}/change/", config=config)

            print("\n[bold]crawl output saved to:[/bold]")
            print(f"  {rel_output_str}")
@@ -252,34 +327,65 @@ def add(
    help="Recursively archive linked pages up to N hops away",
 )
@click.option("--max-urls", type=int, default=0, help="Maximum number of URLs to snapshot for this crawl (0 = unlimited)")
-@click.option("--max-size", default="0", help="Maximum total crawl size in bytes or units like 45mb / 1gb (0 = unlimited)")
+@click.option("--crawl-max-size", default="0", help="Maximum total crawl size in bytes or units like 45mb / 1gb (0 = unlimited)")
+@click.option("--crawl-timeout", type=int, default=0, help="Maximum total crawl runtime in seconds (0 = unlimited)")
+@click.option("--snapshot-max-size", default="0", help="Maximum per-snapshot size in bytes or units like 45mb / 1gb (0 = unlimited)")
+@click.option("--crawl-max-concurrent-snapshots", type=int, default=None, help="Maximum snapshots to archive concurrently within one crawl")
@click.option("--tag", "-t", default="", help="Comma-separated list of tags to add to each snapshot e.g. tag1,tag2,tag3")
@click.option("--url-allowlist", "--domain-allowlist", default="", help="Comma-separated URL/domain allowlist for this crawl")
@click.option("--url-denylist", "--domain-denylist", default="", help="Comma-separated URL/domain denylist for this crawl")
@click.option("--parser", default="auto", help="Parser for reading input URLs (auto, txt, html, rss, json, jsonl, netscape, ...)")
@click.option("--plugins", "-p", default="", help="Comma-separated list of plugins to run e.g. title,favicon,screenshot,singlefile,...")
@click.option("--persona", default="Default", help="Authentication profile to use when archiving")
-@click.option("--overwrite", "-F", is_flag=True, help="Overwrite existing data if URLs have been archived previously")
-@click.option("--update", is_flag=True, default=None, help="Retry any previously skipped/failed URLs when re-adding them")
+@click.option(
+    "--only-new/--no-only-new",
+    "only_new",
+    default=None,
+    help="Skip URLs that already have a snapshot (default: inherit from ONLY_NEW config). "
+    "Pass --no-only-new to force re-archive of URLs that already exist.",
+)
@click.option("--index-only", is_flag=True, help="Just add the URLs to the index without archiving them now")
+@click.option("--overwrite", is_flag=True, help="Re-archive URLs even if they already exist (alias for --no-only-new)")
+@click.option("--update", is_flag=True, help="Re-archive URLs even if they already exist (alias for --no-only-new)")
@click.option("--bg", is_flag=True, help="Run archiving in background (queue work and return immediately)")
@click.argument("urls", nargs=-1, type=click.Path())
@docstring(add.__doc__)
 def main(**kwargs):
    """Add a new URL or list of URLs to your archive"""

-    raw_urls = kwargs.pop("urls")
-    urls = _collect_input_urls(raw_urls)
-    if not urls:
-        raise click.UsageError("No URLs provided. Pass URLs as arguments or via stdin.")
-    if int(kwargs.get("max_urls") or 0) < 0:
-        raise click.BadParameter("max_urls must be 0 or a positive integer.", param_hint="--max-urls")
-    try:
-        kwargs["max_size"] = parse_filesize_to_bytes(kwargs.get("max_size"))
-    except ValueError as err:
-        raise click.BadParameter(str(err), param_hint="--max-size") from err
+    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals

-    add(urls=urls, **kwargs)
+    with foreground_shutdown_signals(), foreground_parent_watchdog():
+        raw_urls = kwargs.pop("urls")
+        urls = _collect_input_urls(raw_urls)
+        if not urls:
+            raise click.UsageError("No URLs provided. Pass URLs as arguments or via stdin.")
+        if int(kwargs.get("max_urls") or 0) < 0:
+            raise click.BadParameter("max_urls must be 0 or a positive integer.", param_hint="--max-urls")
+        if int(kwargs.get("crawl_timeout") or 0) < 0:
+            raise click.BadParameter("crawl_timeout must be 0 or a positive integer.", param_hint="--crawl-timeout")
+        try:
+            kwargs["crawl_max_size"] = parse_filesize_to_bytes(kwargs.get("crawl_max_size"))
+        except ValueError as err:
+            raise click.BadParameter(str(err), param_hint="--crawl-max-size") from err
+        try:
+            kwargs["snapshot_max_size"] = parse_filesize_to_bytes(kwargs.get("snapshot_max_size"))
+        except ValueError as err:
+            raise click.BadParameter(str(err), param_hint="--snapshot-max-size") from err
+        if kwargs.get("crawl_max_concurrent_snapshots") is not None and int(kwargs["crawl_max_concurrent_snapshots"]) < 1:
+            raise click.BadParameter("crawl_max_concurrent_snapshots must be at least 1.", param_hint="--crawl-max-concurrent-snapshots")
+
+        # Translate --only-new/--no-only-new into a crawl config override.
+        # add() takes config overrides as a dict; no per-flag kwargs.
+        overwrite = kwargs.pop("overwrite", False)
+        update = kwargs.pop("update", False)
+        only_new = kwargs.pop("only_new", None)
+        if overwrite or update:
+            only_new = False
+        if only_new is not None:
+            kwargs["config"] = {"ONLY_NEW": bool(only_new)}
+
+        add(urls=urls, **kwargs)


 if __name__ == "__main__":
@@ -38,7 +38,7 @@ import sys
 import rich_click as click
 from rich import print as rprint

-from archivebox.cli.cli_utils import apply_filters
+from archivebox.cli.cli_util import apply_filters


 def build_archiveresult_request(snapshot_id: str, plugin: str, hook_name: str = "", status: str = "queued") -> dict:
@@ -73,8 +73,8 @@ def create_archiveresults(
        0: Success
        1: Failure
    """
-    from archivebox.config.configset import get_config
-    from archivebox.hooks import discover_hooks
+    from archivebox.config.common import get_config
+    from archivebox.plugins.hooks import discover_hooks
    from archivebox.misc.jsonl import read_stdin, write_record, TYPE_SNAPSHOT, TYPE_ARCHIVERESULT
    from archivebox.core.models import Snapshot

@@ -152,7 +152,7 @@ def create_archiveresults(
            config = get_config(crawl=snapshot.crawl, snapshot=snapshot)
            hooks = discover_hooks("Snapshot", config=config)
            for hook_path in hooks:
-                hook_name = hook_path.name
+                hook_name = hook_path.stem
                plugin_name = hook_path.parent.name
                if not is_tty:
                    write_record(build_archiveresult_request(snapshot.id, plugin_name, hook_name=hook_name, status=status))
@@ -33,7 +33,7 @@ import sys
 import rich_click as click
 from rich import print as rprint

-from archivebox.cli.cli_utils import apply_filters
+from archivebox.cli.cli_util import apply_filters


 # =============================================================================
@@ -3,14 +3,19 @@
 __package__ = "archivebox.cli"

 import sys
+import toml
 import rich_click as click
 from rich import print
-from benedict import benedict
+from pathlib import Path

 from archivebox.misc.util import docstring, enforce_types
 from archivebox.misc.toml_util import CustomTOMLEncoder


+def _format_toml(config: dict) -> str:
+    return toml.dumps(config, encoder=CustomTOMLEncoder()).strip().replace("\n\n", "\n")
+
+
@enforce_types
 def config(
    *keys,
@@ -24,13 +29,29 @@ def config(

    from archivebox.misc.checks import check_data_folder
    from archivebox.misc.logging_util import printable_config
-    from archivebox.config.collection import load_all_config, write_config_file, get_real_name
-    from archivebox.config.configset import get_flat_config, get_all_configs
+    from abx_plugins.plugins.base.utils import resolve_alias
+    from archivebox.config.collection import write_config_file
+    from archivebox.config import CONSTANTS_CONFIG
+    from archivebox.config.common import ArchiveBoxConfig, get_config, get_all_configs
+    from archivebox.plugins.discovery import discover_plugin_configs

    check_data_folder()

-    FLAT_CONFIG = get_flat_config()
+    FLAT_CONFIG = get_config().as_dict()
+    runtime_derived_keys = ArchiveBoxConfig.runtime_derived_config_keys()
+    readonly_config = {key: val for key, val in CONSTANTS_CONFIG.items() if key.isupper() and isinstance(val, Path)}
+    writable_config = {key: val for key, val in FLAT_CONFIG.items() if key not in runtime_derived_keys and key not in readonly_config}
+    readable_config = {**writable_config, **readonly_config}
    CONFIGS = get_all_configs()
+    plugin_schemas = {
+        plugin_name: schema.get("properties", {}) for plugin_name, schema in discover_plugin_configs().items() if isinstance(schema, dict)
+    }
+    core_config_aliases = {
+        alias.upper(): field_name
+        for field_name, field in ArchiveBoxConfig.model_fields.items()
+        for alias in (field_name, str(field.alias or ""))
+        if alias
+    }

    config_options: list[str] = list(kwargs.pop("key=value", []) or keys or [f"{key}={val}" for key, val in kwargs.items()])
    no_args = not (get or set or reset or config_options)
@@ -38,19 +59,26 @@ def config(
    matching_config = {}
    if search:
        if config_options:
-            config_options = [get_real_name(key) for key in config_options]
-            matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
+            config_options = [
+                core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options
+            ]
+            matching_config = {key: readable_config[key] for key in config_options if key in readable_config}
            for config_section in CONFIGS.values():
-                aliases = getattr(config_section, "aliases", {})
+                aliases = {str(field.alias): field_name for field_name, field in type(config_section).model_fields.items() if field.alias}

                for search_key in config_options:
                    # search all aliases in the section
                    for alias_key, key in aliases.items():
-                        if search_key.lower() in alias_key.lower():
+                        if key in readable_config and search_key.lower() in alias_key.lower():
                            matching_config[key] = dict(config_section)[key]

                    # search all keys and values in the section
                    for existing_key, value in dict(config_section).items():
+                        if existing_key in readable_config and (
+                            search_key.lower() in existing_key.lower() or search_key.lower() in str(value).lower()
+                        ):
+                            matching_config[existing_key] = value
+                    for existing_key, value in readonly_config.items():
                        if search_key.lower() in existing_key.lower() or search_key.lower() in str(value).lower():
                            matching_config[existing_key] = value

@@ -59,46 +87,48 @@ def config(

    elif get or no_args:
        if config_options:
-            config_options = [get_real_name(key) for key in config_options]
-            matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
-            failed_config = [key for key in config_options if key not in FLAT_CONFIG]
+            config_options = [
+                core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options
+            ]
+            matching_config = {key: readable_config[key] for key in config_options if key in readable_config}
+            failed_config = [key for key in config_options if key not in readable_config]
            if failed_config:
                print("\n[red][X] These options failed to get[/red]")
                print("    {}".format("\n    ".join(config_options)))
                raise SystemExit(1)
        else:
-            matching_config = FLAT_CONFIG
+            matching_config = readable_config

        # Display core config sections
        for config_section in CONFIGS.values():
-            section_header = getattr(config_section, "toml_section_header", "")
+            section_header = config_section.toml_section_header
            if isinstance(section_header, str) and section_header:
                print(f"[grey53]\\[{section_header}][/grey53]")
            else:
                print("[grey53]\\[CONSTANTS]                                        # (read-only)[/grey53]")

            kv_in_section = {key: val for key, val in dict(config_section).items() if key in matching_config}
-            print(benedict(kv_in_section).to_toml(encoder=CustomTOMLEncoder()).strip().replace("\n\n", "\n"))
+            print(_format_toml(kv_in_section))
            print("[grey53]################################################################[/grey53]")

-        # Display plugin config section
-        from archivebox.hooks import discover_plugin_configs
+        readonly_keys = {key: val for key, val in readonly_config.items() if key in matching_config}
+        if readonly_keys:
+            print("[grey53]\\[CONSTANTS]                                        # (read-only)[/grey53]")
+            print(_format_toml(readonly_keys))
+            print("[grey53]################################################################[/grey53]")

-        plugin_configs = discover_plugin_configs()
        plugin_keys = {}

        # Collect all plugin config keys
-        for plugin_name, schema in plugin_configs.items():
-            if "properties" not in schema:
-                continue
-            for key in schema["properties"].keys():
-                if key in matching_config:
+        for schema in plugin_schemas.values():
+            for key in schema.keys():
+                if key in matching_config and key in writable_config:
                    plugin_keys[key] = matching_config[key]

        # Display all plugin config in single [PLUGINS] section
        if plugin_keys:
            print("[grey53]\\[PLUGINS][/grey53]")
-            print(benedict(plugin_keys).to_toml(encoder=CustomTOMLEncoder()).strip().replace("\n\n", "\n"))
+            print(_format_toml(plugin_keys))
            print("[grey53]################################################################[/grey53]")

        raise SystemExit(not matching_config)
@@ -116,13 +146,13 @@ def config(

            raw_key, val = line.split("=", 1)
            raw_key = raw_key.upper().strip()
-            key = get_real_name(raw_key)
+            key = core_config_aliases.get(raw_key) or resolve_alias(raw_key, plugin_schemas)
            if key != raw_key:
                print(
                    f"[yellow][i] Note: The config option {raw_key} has been renamed to {key}, please use the new name going forwards.[/yellow]",
                )

-            if key in FLAT_CONFIG:
+            if key in writable_config:
                new_config[key] = val.strip()
            else:
                failed_options.append(line)
@@ -130,7 +160,7 @@ def config(
        if new_config:
            before = FLAT_CONFIG
            matching_config = write_config_file(new_config)
-            after = {**load_all_config(), **get_flat_config()}
+            after = get_config().as_dict()
            print(printable_config(matching_config))

            side_effect_changes = {}
@@ -39,7 +39,7 @@ from collections.abc import Iterable
 import rich_click as click
 from rich import print as rprint

-from archivebox.cli.cli_utils import apply_filters
+from archivebox.cli.cli_util import apply_filters


 # =============================================================================
@@ -247,17 +247,30 @@ def update_crawls(
        try:
            crawl = Crawl.objects.get(id=crawl_id)

-            # Apply updates from CLI flags
            if status:
-                crawl.status = status
-                crawl.retry_at = timezone.now()
+                if status not in Crawl.StatusChoices.values:
+                    rprint(f"[red]Invalid crawl status: {status}[/red]", file=sys.stderr)
+                    continue
+                if status == Crawl.StatusChoices.SEALED:
+                    crawl.cancel()
+                elif status == Crawl.StatusChoices.PAUSED:
+                    crawl.pause()
+                elif status == Crawl.StatusChoices.QUEUED:
+                    if crawl.status == Crawl.StatusChoices.PAUSED:
+                        crawl.resume()
+                    else:
+                        crawl.update_and_requeue(status=Crawl.StatusChoices.QUEUED, retry_at=timezone.now())
+                elif status == Crawl.StatusChoices.STARTED:
+                    crawl.update_and_requeue(status=Crawl.StatusChoices.STARTED, retry_at=timezone.now())
            if max_depth is not None:
+                crawl.safe_update({"max_depth": max_depth, "modified_at": timezone.now()}, refresh=False)
                crawl.max_depth = max_depth
-
-            crawl.save()
+            elif not status:
+                crawl.safe_update({"modified_at": timezone.now()}, refresh=False)
            updated_count += 1

            if not is_tty:
+                crawl.refresh_from_db()
                write_record(crawl.to_json())

        except Crawl.DoesNotExist:
@@ -1,27 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox crawl"
-
-import sys
-
-import rich_click as click
-
-from archivebox.cli.archivebox_add import add
-
-
-@click.command(context_settings={"ignore_unknown_options": True})
-@click.option("--depth", "-d", type=int, default=0, help="Max crawl depth (default: 0)")
-@click.option("--tag", "-t", default="", help="Comma-separated tags to add")
-@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
-@click.option("--wait/--no-wait", "wait", default=True, help="Accepted for backwards compatibility")
-@click.argument("urls", nargs=-1)
-def main(depth: int, tag: str, status: str, wait: bool, urls: tuple[str, ...]):
-    """Backwards-compatible `archivebox crawl URL...` entrypoint."""
-    del status, wait
-    add(list(urls), depth=depth, tag=tag, index_only=True, bg=True)
-    sys.exit(0)
-
-
-if __name__ == "__main__":
-    main()
@@ -32,6 +32,7 @@ __command__ = "archivebox extract"

 import sys
 from collections import defaultdict
+from itertools import product

 import rich_click as click

@@ -47,10 +48,11 @@ def process_archiveresult_by_id(archiveresult_id: str) -> int:
    from rich import print as rprint
    from django.utils import timezone
    from archivebox.core.models import ArchiveResult
+    from archivebox.api.v1_core import _uuid_ref_query
    from archivebox.services.runner import run_crawl

    try:
-        archiveresult = ArchiveResult.objects.get(id=archiveresult_id)
+        archiveresult = ArchiveResult.objects.get(_uuid_ref_query("id", archiveresult_id))
    except ArchiveResult.DoesNotExist:
        rprint(f"[red]ArchiveResult {archiveresult_id} not found[/red]", file=sys.stderr)
        return 1
@@ -58,19 +60,35 @@ def process_archiveresult_by_id(archiveresult_id: str) -> int:
    rprint(f"[blue]Extracting {archiveresult.plugin} for {archiveresult.snapshot.url}[/blue]", file=sys.stderr)

    try:
+        was_paused = archiveresult.snapshot.is_paused
        archiveresult.reset_for_retry()
        snapshot = archiveresult.snapshot
-        snapshot.status = snapshot.StatusChoices.QUEUED
-        snapshot.retry_at = timezone.now()
-        snapshot.save(update_fields=["status", "retry_at", "modified_at"])
-
+        if not was_paused:
+            snapshot.queue_for_extraction()
+        else:
+            # A paused snapshot may still accept explicit maintenance for one
+            # ArchiveResult, but this path must not transition it back to
+            # queued/startable work. Guard: only set retry_at while the row is
+            # still paused — concurrent resume would otherwise see a stale
+            # retry_at marker.
+            snapshot.safe_update(
+                {"retry_at": timezone.now()},
+                refresh=False,
+                extra_filter={"status": snapshot.StatusChoices.PAUSED},
+            )
        crawl = snapshot.crawl
-        if crawl.status != crawl.StatusChoices.STARTED:
-            crawl.status = crawl.StatusChoices.QUEUED
-        crawl.retry_at = timezone.now()
-        crawl.save(update_fields=["status", "retry_at", "modified_at"])
+        if not crawl.claim_processing_lock(lock_seconds=10):
+            rprint(
+                f"[yellow]Crawl {crawl.id} is already owned by another runner[/yellow]",
+                file=sys.stderr,
+            )
+            return 1

-        run_crawl(str(crawl.id), snapshot_ids=[str(snapshot.id)], selected_plugins=[archiveresult.plugin])
+        try:
+            run_crawl(str(snapshot.crawl_id), snapshot_ids=[str(snapshot.id)], selected_plugins=[archiveresult.plugin])
+        finally:
+            if was_paused:
+                snapshot.restore_paused_scheduler_marker()
        archiveresult.refresh_from_db()

        if archiveresult.status == ArchiveResult.StatusChoices.SUCCEEDED:
@@ -98,6 +116,8 @@ def run_plugins(
    plugins: str = "",
    wait: bool = True,
    emit_results: bool = True,
+    show_progress: bool = True,
+    preserve_queued: bool = False,
 ) -> int:
    """
    Run plugins on Snapshots from input.
@@ -118,7 +138,9 @@ def run_plugins(
        TYPE_ARCHIVERESULT,
    )
    from archivebox.core.models import Snapshot
+    from archivebox.core.models import ArchiveResult
    from archivebox.services.runner import run_crawl
+    from abx_dl.models import discover_plugins

    is_tty = sys.stdout.isatty()

@@ -145,7 +167,7 @@ def run_plugins(
        if record_type == TYPE_SNAPSHOT:
            snapshot_id = record.get("id")
            if snapshot_id:
-                snapshot_ids.add(snapshot_id)
+                snapshot_ids.add(str(snapshot_id))
            elif record.get("url"):
                # Look up by URL (get most recent if multiple exist)
                snap = Snapshot.objects.filter(url=record["url"]).order_by("-created_at").first()
@@ -157,60 +179,201 @@ def run_plugins(
        elif record_type == TYPE_ARCHIVERESULT:
            snapshot_id = record.get("snapshot_id")
            if snapshot_id:
-                snapshot_ids.add(snapshot_id)
+                snapshot_ids.add(str(snapshot_id))
                plugin_name = record.get("plugin")
                if plugin_name and not plugins_list:
                    requested_plugins_by_snapshot[str(snapshot_id)].add(str(plugin_name))

        elif "id" in record:
            # Assume it's a snapshot ID
-            snapshot_ids.add(record["id"])
+            snapshot_ids.add(str(record["id"]))

    if not snapshot_ids:
        rprint("[red]No valid snapshot IDs found in input[/red]", file=sys.stderr)
        return 1

-    # Get snapshots and ensure they have pending ArchiveResults
-    processed_count = 0
-    for snapshot_id in snapshot_ids:
-        try:
-            snapshot = Snapshot.objects.get(id=snapshot_id)
-        except Snapshot.DoesNotExist:
-            rprint(f"[yellow]Snapshot {snapshot_id} not found[/yellow]", file=sys.stderr)
-            continue
+    existing_snapshots = list(Snapshot.objects.filter(id__in=snapshot_ids).values_list("id", "crawl_id"))
+    existing_snapshot_ids = {str(snapshot_id) for snapshot_id, _crawl_id in existing_snapshots}
+    existing_crawl_ids = {str(crawl_id) for _snapshot_id, crawl_id in existing_snapshots}
+    missing_snapshot_ids = sorted(str(snapshot_id) for snapshot_id in snapshot_ids - existing_snapshot_ids)
+    for snapshot_id in missing_snapshot_ids:
+        rprint(f"[yellow]Snapshot {snapshot_id} not found[/yellow]", file=sys.stderr)

-        requested_plugin_names = set(plugins_list) | requested_plugins_by_snapshot.get(str(snapshot.id), set())
-        for plugin_name in requested_plugin_names:
-            existing_result = snapshot.archiveresult_set.filter(plugin=plugin_name).order_by("-created_at").first()
-            if existing_result:
-                existing_result.reset_for_retry()
+    # Queue only the target plugin rows. Bulk updates keep large reindex runs
+    # from doing one SELECT+UPDATE per snapshot/plugin before hooks even start.
+    requested_pairs: set[tuple[str, str]] = set()
+    if plugins_list:
+        requested_pairs.update((snapshot_id, plugin_name) for snapshot_id, plugin_name in product(existing_snapshot_ids, plugins_list))
+    else:
+        requested_pairs.update(
+            (snapshot_id, plugin_name)
+            for snapshot_id, plugin_names in requested_plugins_by_snapshot.items()
+            if snapshot_id in existing_snapshot_ids
+            for plugin_name in plugin_names
+        )
+    plugins_by_name = discover_plugins(runtime="archivebox")
+    requested_rows: set[tuple[str, str, str]] = set()
+    for snapshot_id, plugin_name in requested_pairs:
+        plugin = plugins_by_name.get(plugin_name)
+        hooks = plugin.filter_hooks("Snapshot") if plugin is not None else []
+        if hooks:
+            requested_rows.update((snapshot_id, plugin_name, hook.name) for hook in hooks)
+        else:
+            requested_rows.add((snapshot_id, plugin_name, ""))

-        # Reset snapshot status to allow processing
-        if snapshot.status == Snapshot.StatusChoices.SEALED:
-            snapshot.status = Snapshot.StatusChoices.STARTED
-            snapshot.retry_at = timezone.now()
-            snapshot.save()
+    queued_rows: set[tuple[str, str, str]] = set()
+    if preserve_queued and requested_rows:
+        queued_rows = {
+            (str(snapshot_id), plugin_name, hook_name)
+            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
+                snapshot_id__in=existing_snapshot_ids,
+                plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in requested_rows},
+                status=ArchiveResult.StatusChoices.QUEUED,
+            ).values_list("snapshot_id", "plugin", "hook_name")
+        }
+    rows_to_queue = requested_rows - queued_rows

-        processed_count += 1
+    reset_fields = {
+        "status": ArchiveResult.StatusChoices.QUEUED,
+        "output_str": "",
+        "output_json": None,
+        "output_files": {},
+        "output_size": 0,
+        "output_mimetypes": "",
+        "start_ts": None,
+        "end_ts": None,
+        "modified_at": timezone.now(),
+    }
+    if rows_to_queue and plugins_list:
+        rows_to_reset_by_hook: dict[tuple[str, str], set[str]] = defaultdict(set)
+        for snapshot_id, plugin_name, hook_name in rows_to_queue:
+            rows_to_reset_by_hook[(plugin_name, hook_name)].add(snapshot_id)
+        for (plugin_name, hook_name), plugin_snapshot_ids in rows_to_reset_by_hook.items():
+            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name, hook_name=hook_name).update(
+                **reset_fields,
+            )
+    elif rows_to_queue and requested_plugins_by_snapshot:
+        snapshot_ids_by_hook: dict[tuple[str, str], set[str]] = defaultdict(set)
+        for snapshot_id, plugin_name, hook_name in rows_to_queue:
+            snapshot_ids_by_hook[(plugin_name, hook_name)].add(snapshot_id)
+        for (plugin_name, hook_name), plugin_snapshot_ids in snapshot_ids_by_hook.items():
+            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name, hook_name=hook_name).update(
+                **reset_fields,
+            )
+    existing_rows = (
+        {
+            (str(snapshot_id), plugin_name, hook_name)
+            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
+                snapshot_id__in=existing_snapshot_ids,
+                plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in rows_to_queue},
+            ).values_list("snapshot_id", "plugin", "hook_name")
+        }
+        if rows_to_queue
+        else set()
+    )
+    missing_rows = rows_to_queue - existing_rows
+    if missing_rows:
+        ArchiveResult.objects.bulk_create(
+            [
+                ArchiveResult(
+                    snapshot_id=snapshot_id,
+                    plugin=plugin_name,
+                    hook_name=hook_name,
+                    status=ArchiveResult.StatusChoices.QUEUED,
+                )
+                for snapshot_id, plugin_name, hook_name in sorted(missing_rows)
+            ],
+            batch_size=500,
+        )
+
+    processed_count = len(existing_snapshot_ids)
+    queue_at = timezone.now()
+    if existing_snapshot_ids:
+        if requested_rows:
+            # Targeted ArchiveResult retries use retry_at as the scheduling
+            # signal and keep sealed snapshots sealed so extractors are not
+            # re-run outside the explicitly queued plugin rows. Paused snapshots
+            # also keep status=paused here: `retry_at` only asks the orchestrator
+            # to process the queued plugin rows, and run_due_snapshot restores
+            # retry_at=MAX afterward instead of resuming the snapshot lifecycle.
+            affected_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in rows_to_queue}
+            if preserve_queued and queued_rows:
+                queued_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in queued_rows}
+                affected_snapshot_ids.update(
+                    str(snapshot_id)
+                    for snapshot_id in Snapshot.objects.filter(id__in=queued_snapshot_ids)
+                    .filter(retry_at__gt=queue_at)
+                    .values_list("id", flat=True)
+                )
+                affected_snapshot_ids.update(
+                    str(snapshot_id)
+                    for snapshot_id in Snapshot.objects.filter(id__in=queued_snapshot_ids, retry_at__isnull=True).values_list(
+                        "id",
+                        flat=True,
+                    )
+                )
+            for snapshot in Snapshot.objects.filter(id__in=affected_snapshot_ids).only("id", "status", "modified_at"):
+                # Guard the read-time status so we never bump retry_at on a
+                # row that's been re-queued / started by a concurrent runner.
+                snapshot.safe_update(
+                    {"retry_at": queue_at, "modified_at": queue_at},
+                    refresh=False,
+                    extra_filter={"status": snapshot.status},
+                )
+        else:
+            # No plugin rows were requested, so this is a full snapshot retry.
+            for snapshot in Snapshot.objects.filter(id__in=existing_snapshot_ids).only("id", "status", "retry_at", "modified_at"):
+                snapshot.safe_update(
+                    {
+                        "status": Snapshot.StatusChoices.QUEUED,
+                        "retry_at": queue_at,
+                        "current_step": 0,
+                        "modified_at": queue_at,
+                    },
+                    refresh=False,
+                    extra_filter={"status": snapshot.status},
+                )
+    if existing_crawl_ids and not requested_rows:
+        from archivebox.crawls.models import Crawl
+
+        for crawl in Crawl.objects.filter(id__in=existing_crawl_ids).only("id", "status", "retry_at", "modified_at"):
+            update_fields = {
+                "retry_at": queue_at,
+                "modified_at": queue_at,
+            }
+            if crawl.status != Crawl.StatusChoices.STARTED:
+                update_fields["status"] = Crawl.StatusChoices.QUEUED
+            crawl.safe_update(
+                update_fields,
+                refresh=False,
+                extra_filter={"status": crawl.status},
+            )

    if processed_count == 0:
        rprint("[red]No snapshots to process[/red]", file=sys.stderr)
        return 1

-    rprint(f"[blue]Queued {processed_count} snapshots for extraction[/blue]", file=sys.stderr)
+    if show_progress:
+        rprint(f"[blue]Queued {processed_count} snapshots for extraction[/blue]", file=sys.stderr)

    # Run orchestrator if --wait (default)
    if wait:
-        rprint("[blue]Running plugins...[/blue]", file=sys.stderr)
+        if show_progress:
+            rprint("[blue]Running plugins...[/blue]", file=sys.stderr)
        snapshot_ids_by_crawl: dict[str, set[str]] = defaultdict(set)
-        for snapshot_id in snapshot_ids:
-            try:
-                snapshot = Snapshot.objects.only("id", "crawl_id").get(id=snapshot_id)
-            except Snapshot.DoesNotExist:
-                continue
-            snapshot_ids_by_crawl[str(snapshot.crawl_id)].add(str(snapshot.id))
+        for snapshot_id, crawl_id in existing_snapshots:
+            snapshot_ids_by_crawl[str(crawl_id)].add(str(snapshot_id))

        for crawl_id, crawl_snapshot_ids in snapshot_ids_by_crawl.items():
+            from archivebox.crawls.models import Crawl
+
+            crawl = Crawl.objects.get(id=crawl_id)
+            if not crawl.claim_processing_lock(lock_seconds=10):
+                rprint(
+                    f"[yellow]Crawl {crawl_id} is already owned by another runner[/yellow]",
+                    file=sys.stderr,
+                )
+                return 1
            selected_plugins = (
                plugins_list
                or sorted(
@@ -222,6 +385,7 @@ def run_plugins(
                crawl_id,
                snapshot_ids=sorted(crawl_snapshot_ids),
                selected_plugins=selected_plugins,
+                show_progress=show_progress,
            )

    if not emit_results:
@@ -255,16 +419,11 @@ def run_plugins(


 def is_archiveresult_id(value: str) -> bool:
-    """Check if value looks like an ArchiveResult UUID."""
-    import re
-
-    uuid_pattern = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.I)
-    if not uuid_pattern.match(value):
-        return False
-    # Verify it's actually an ArchiveResult (not a Snapshot or other object)
+    """Check if value resolves to an ArchiveResult ID."""
    from archivebox.core.models import ArchiveResult
+    from archivebox.api.v1_core import _uuid_ref_query

-    return ArchiveResult.objects.filter(id=value).exists()
+    return ArchiveResult.objects.filter(_uuid_ref_query("id", value)).exists()


@click.command()
@@ -2,6 +2,8 @@
 __package__ = "archivebox.cli"
 __command__ = "archivebox help"

+import ast
+import importlib.util
 import os
 from pathlib import Path

@@ -10,27 +12,47 @@ from rich import print
 from rich.panel import Panel


+def _command_doc(cmd: str, import_path: str) -> str:
+    def first_doc_line(docstring: str | None) -> str:
+        lines = (docstring or "").splitlines()
+        return lines[0] if lines else ""
+
+    modname, _ = import_path.rsplit(".", 1)
+    spec = importlib.util.find_spec(modname)
+    if spec is None or spec.origin is None:
+        return ""
+
+    try:
+        tree = ast.parse(Path(spec.origin).read_text())
+    except OSError:
+        return ""
+
+    for name in (cmd, "main"):
+        for node in tree.body:
+            if isinstance(node, ast.FunctionDef) and node.name == name:
+                return first_doc_line(ast.get_docstring(node))
+
+    return first_doc_line(ast.get_docstring(tree))
+
+
 def help() -> None:
    """Print the ArchiveBox help message and usage"""

    from archivebox.cli import ArchiveBoxGroup
    from archivebox.config import CONSTANTS
+    from archivebox.config.common import get_config
    from archivebox.config.permissions import IN_DOCKER
    from archivebox.misc.logging_util import log_cli_command

    log_cli_command("help", [], None, ".")

    COMMANDS_HELP_TEXT = (
-        "\n    ".join(
-            f"[green]{cmd.ljust(20)}[/green] {ArchiveBoxGroup._lazy_load(cmd).__doc__}" for cmd in ArchiveBoxGroup.meta_commands.keys()
-        )
+        "\n    ".join(f"[green]{cmd.ljust(20)}[/green] {_command_doc(cmd, path)}" for cmd, path in ArchiveBoxGroup.meta_commands.items())
+        + "\n\n    "
+        + "\n    ".join(f"[green]{cmd.ljust(20)}[/green] {_command_doc(cmd, path)}" for cmd, path in ArchiveBoxGroup.setup_commands.items())
        + "\n\n    "
        + "\n    ".join(
-            f"[green]{cmd.ljust(20)}[/green] {ArchiveBoxGroup._lazy_load(cmd).__doc__}" for cmd in ArchiveBoxGroup.setup_commands.keys()
-        )
-        + "\n\n    "
-        + "\n    ".join(
-            f"[green]{cmd.ljust(20)}[/green] {ArchiveBoxGroup._lazy_load(cmd).__doc__}" for cmd in ArchiveBoxGroup.archive_commands.keys()
+            f"[green]{cmd.ljust(20)}[/green] {_command_doc(cmd, path)}" for cmd, path in ArchiveBoxGroup.archive_commands.items()
        )
    )

@@ -67,6 +89,7 @@ def help() -> None:
    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration]https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration[/link]
 """)

+    get_config()
    if os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and CONSTANTS.ARCHIVE_DIR.is_dir():
        pretty_out_dir = str(CONSTANTS.DATA_DIR).replace(str(Path("~").expanduser()), "~")
        EXAMPLE_USAGE = f"""
@@ -5,7 +5,6 @@ __package__ = "archivebox.cli"
 import os
 import sys
 from pathlib import Path
-from collections.abc import Mapping

 from rich import print
 import rich_click as click
@@ -13,34 +12,32 @@ import rich_click as click
 from archivebox.misc.util import docstring, enforce_types


-def _normalize_snapshot_record(link_dict: Mapping[str, object]) -> tuple[str, dict[str, object]] | None:
-    url = link_dict.get("url")
-    if not isinstance(url, str) or not url:
-        return None
-
-    record: dict[str, object] = {"url": url}
-    for key in ("timestamp", "title", "tags", "sources"):
-        value = link_dict.get(key)
-        if value is not None:
-            record[key] = value
-    return url, record
+def _display_data_path(path: Path, data_dir: Path) -> str:
+    path = Path(path).resolve()
+    data_dir = Path(data_dir).resolve()
+    try:
+        return f"./{path.relative_to(data_dir)}"
+    except ValueError:
+        return str(path)


@enforce_types
 def init(force: bool = False, quick: bool = False, install: bool = False) -> None:
    """Initialize a new ArchiveBox collection in the current directory"""

-    from archivebox.config import CONSTANTS, VERSION, DATA_DIR
-    from archivebox.config.common import SERVER_CONFIG
+    from archivebox.config import CONSTANTS, VERSION
+    from archivebox.config.common import get_config
    from archivebox.config.collection import write_config_file
-    from archivebox.misc.legacy import parse_json_main_index, parse_json_links_details
    from archivebox.misc.db import apply_migrations
+    from archivebox.misc.checks import check_migrations
+
+    config = get_config()

    # if os.access(out_dir / CONSTANTS.JSON_INDEX_FILENAME, os.F_OK):
    #     print("[red]:warning: This folder contains a JSON index. It is deprecated, and will no longer be kept up to date automatically.[/red]", file=sys.stderr)
    #     print("[red]    You can run `archivebox list --json --with-headers > static_index.json` to manually generate it.[/red]", file=sys.stderr)

-    is_empty = not len(set(os.listdir(DATA_DIR)) - CONSTANTS.ALLOWED_IN_DATA_DIR)
+    is_empty = not len(set(os.listdir(CONSTANTS.DATA_DIR)) - CONSTANTS.ALLOWED_IN_DATA_DIR)
    existing_index = os.path.isfile(CONSTANTS.DATABASE_FILE)
    if is_empty and not existing_index:
        print(f"[turquoise4][+] Initializing a new ArchiveBox v{VERSION} collection...[/turquoise4]")
@@ -68,22 +65,26 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
    else:
        print("\n[green][+] Building archive folder structure...[/green]")

-    print(
-        f"    + ./{CONSTANTS.ARCHIVE_DIR.relative_to(DATA_DIR)}, ./{CONSTANTS.SOURCES_DIR.relative_to(DATA_DIR)}, ./{CONSTANTS.LOGS_DIR.relative_to(DATA_DIR)}...",
-    )
+    archive_path = _display_data_path(CONSTANTS.ARCHIVE_DIR, CONSTANTS.DATA_DIR)
+    sources_path = _display_data_path(CONSTANTS.SOURCES_DIR, CONSTANTS.DATA_DIR)
+    logs_path = _display_data_path(CONSTANTS.LOGS_DIR, CONSTANTS.DATA_DIR)
+    print(f"    + {archive_path}, {sources_path}, {logs_path}...")
    Path(CONSTANTS.SOURCES_DIR).mkdir(exist_ok=True)
-    Path(CONSTANTS.ARCHIVE_DIR).mkdir(exist_ok=True)
+    CONSTANTS.ARCHIVE_DIR.mkdir(parents=True, exist_ok=True)
+    CONSTANTS.USERS_DIR.mkdir(parents=True, exist_ok=True)
    Path(CONSTANTS.LOGS_DIR).mkdir(exist_ok=True)
+    for path in (Path(CONSTANTS.SOURCES_DIR), CONSTANTS.ARCHIVE_DIR, CONSTANTS.USERS_DIR, Path(CONSTANTS.LOGS_DIR)):
+        path.chmod(int(config.OUTPUT_PERMISSIONS, base=8) | 0o111)

-    print(f"    + ./{CONSTANTS.CONFIG_FILE.relative_to(DATA_DIR)}...")
+    print(f"    + {_display_data_path(CONSTANTS.CONFIG_FILE, CONSTANTS.DATA_DIR)}...")

    # create the .archivebox_id file with a unique ID for this collection
    from archivebox.config.paths import _get_collection_id

-    _get_collection_id(DATA_DIR, force_create=True)
+    _get_collection_id(CONSTANTS.DATA_DIR, force_create=True)

    # create the ArchiveBox.conf file
-    write_config_file({"SECRET_KEY": SERVER_CONFIG.SECRET_KEY})
+    write_config_file({"SECRET_KEY": config.SECRET_KEY})

    if os.access(CONSTANTS.DATABASE_FILE, os.F_OK):
        print("\n[green][*] Verifying main SQL index and running any migrations needed...[/green]")
@@ -93,17 +94,24 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
    from archivebox.config.django import setup_django

    setup_django()
+    previous_wants_init = os.environ.get("ARCHIVEBOX_WANTS_INIT")
+    os.environ["ARCHIVEBOX_WANTS_INIT"] = "1"
+    try:
+        check_migrations(blocking=True, auto_apply=False)

-    for migration_line in apply_migrations(DATA_DIR):
-        sys.stdout.write(f"    {migration_line}\n")
+        for migration_line in apply_migrations(CONSTANTS.DATA_DIR):
+            sys.stdout.write(f"    {migration_line}\n")
+    finally:
+        if previous_wants_init is None:
+            os.environ.pop("ARCHIVEBOX_WANTS_INIT", None)
+        else:
+            os.environ["ARCHIVEBOX_WANTS_INIT"] = previous_wants_init

    assert os.path.isfile(CONSTANTS.DATABASE_FILE) and os.access(CONSTANTS.DATABASE_FILE, os.R_OK)
    print()
-    print(f"    √ ./{CONSTANTS.DATABASE_FILE.relative_to(DATA_DIR)}")
+    print(f"    √ {_display_data_path(CONSTANTS.DATABASE_FILE, CONSTANTS.DATA_DIR)}")

    # from django.contrib.auth.models import User
-    # if SHELL_CONFIG.IS_TTY and not User.objects.filter(is_superuser=True).exclude(username='system').exists():
-    #     print('{green}[+] Creating admin user account...{reset}'.format(**SHELL_CONFIG.ANSI))
    #     call_command("createsuperuser", interactive=True)

    print()
@@ -111,86 +119,42 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non

    from archivebox.core.models import Snapshot

-    all_links = Snapshot.objects.none()
-    pending_links: dict[str, dict[str, object]] = {}
+    snapshot_count = 0

    if existing_index:
-        all_links = Snapshot.objects.all()
-        print(f"    √ Loaded {all_links.count()} links from existing main index.")
+        snapshot_count = Snapshot.objects.count()
+        print(f"    √ Loaded {snapshot_count} links from existing main index.")

-    if quick:
-        print("    > Skipping orphan snapshot import (quick mode)")
-    else:
-        try:
-            # Import orphaned links from legacy JSON indexes
-            orphaned_json_links: dict[str, dict[str, object]] = {}
-            for link_dict in parse_json_main_index(DATA_DIR):
-                normalized = _normalize_snapshot_record(link_dict)
-                if normalized is None:
-                    continue
-                url, record = normalized
-                if not all_links.filter(url=url).exists():
-                    orphaned_json_links[url] = record
-            if orphaned_json_links:
-                pending_links.update(orphaned_json_links)
-                print(f"    [yellow]√ Added {len(orphaned_json_links)} orphaned links from existing JSON index...[/yellow]")
-
-            orphaned_data_dir_links: dict[str, dict[str, object]] = {}
-            for link_dict in parse_json_links_details(DATA_DIR):
-                normalized = _normalize_snapshot_record(link_dict)
-                if normalized is None:
-                    continue
-                url, record = normalized
-                if not all_links.filter(url=url).exists():
-                    orphaned_data_dir_links[url] = record
-            if orphaned_data_dir_links:
-                pending_links.update(orphaned_data_dir_links)
-                print(f"    [yellow]√ Added {len(orphaned_data_dir_links)} orphaned links from existing archive directories.[/yellow]")
-
-            if pending_links:
-                for link_dict in pending_links.values():
-                    Snapshot.from_json(link_dict)
-
-            # Hint for orphaned snapshot directories
-            print()
-            print("    [violet]Hint:[/violet] To import orphaned snapshot directories and reconcile filesystem state, run:")
-            print("        archivebox update")
-
-        except (KeyboardInterrupt, SystemExit):
-            print(file=sys.stderr)
-            print("[yellow]:stop_sign: Stopped checking archive directories due to Ctrl-C/SIGTERM[/yellow]", file=sys.stderr)
-            print("    Your archive data is safe, but you should re-run `archivebox init` to finish the process later.", file=sys.stderr)
-            print(file=sys.stderr)
-            print("    [violet]Hint:[/violet] In the future you can run a quick init without checking dirs like so:", file=sys.stderr)
-            print("        archivebox init --quick", file=sys.stderr)
-            raise SystemExit(1)
+    print("    > Skipping orphan snapshot import during init.")
+    print()
+    print("    [violet]Hint:[/violet] To import orphaned snapshot directories and reconcile filesystem state, run:")
+    print("        archivebox update")

    print("\n[green]----------------------------------------------------------------------[/green]")

    from django.contrib.auth.models import User

-    if (SERVER_CONFIG.ADMIN_USERNAME and SERVER_CONFIG.ADMIN_PASSWORD) and not User.objects.filter(
-        username=SERVER_CONFIG.ADMIN_USERNAME,
+    config = get_config()
+    if (config.ADMIN_USERNAME and config.ADMIN_PASSWORD) and not User.objects.filter(
+        username=config.ADMIN_USERNAME,
    ).exists():
        print("[green][+] Found ADMIN_USERNAME and ADMIN_PASSWORD configuration options, creating new admin user.[/green]")
-        User.objects.create_superuser(username=SERVER_CONFIG.ADMIN_USERNAME, password=SERVER_CONFIG.ADMIN_PASSWORD)
+        User.objects.create_superuser(username=config.ADMIN_USERNAME, password=config.ADMIN_PASSWORD)

    if existing_index:
        print("[green][√] Done. Verified and updated the existing ArchiveBox collection.[/green]")
    else:
-        print(f"[green][√] Done. A new ArchiveBox collection was initialized ({len(all_links) + len(pending_links)} links).[/green]")
+        print(f"[green][√] Done. A new ArchiveBox collection was initialized ({snapshot_count} links).[/green]")

    CONSTANTS.PERSONAS_DIR.mkdir(parents=True, exist_ok=True)
    CONSTANTS.DEFAULT_TMP_DIR.mkdir(parents=True, exist_ok=True)
-    CONSTANTS.DEFAULT_LIB_DIR.mkdir(parents=True, exist_ok=True)
-    (CONSTANTS.DEFAULT_LIB_DIR / "bin").mkdir(parents=True, exist_ok=True)

-    from archivebox.config.common import STORAGE_CONFIG
    from archivebox.config.paths import get_or_create_working_tmp_dir, get_or_create_working_lib_dir

-    STORAGE_CONFIG.TMP_DIR.mkdir(parents=True, exist_ok=True)
-    STORAGE_CONFIG.LIB_DIR.mkdir(parents=True, exist_ok=True)
-    (STORAGE_CONFIG.LIB_DIR / "bin").mkdir(parents=True, exist_ok=True)
+    config = get_config()
+    config.TMP_DIR.mkdir(parents=True, exist_ok=True)
+    config.LIB_DIR.mkdir(parents=True, exist_ok=True)
+    (config.LIB_DIR / "bin").mkdir(parents=True, exist_ok=True)

    working_tmp_dir = get_or_create_working_tmp_dir(autofix=True, quiet=True)
    if working_tmp_dir:
@@ -22,11 +22,17 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo
    """

    from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
-    from archivebox.config.paths import ARCHIVE_DIR
+    from archivebox.config import CONSTANTS
    from archivebox.misc.logging import stderr
    from archivebox.cli.archivebox_init import init

-    if not (os.access(ARCHIVE_DIR, os.R_OK) and ARCHIVE_DIR.is_dir()):
+    archive_dir = CONSTANTS.ARCHIVE_DIR
+
+    if dry_run:
+        print("[dim]Dry run - would detect ArchiveBox dependencies and run the abx-dl install flow[/dim]")
+        return
+
+    if not (os.access(archive_dir, os.R_OK) and archive_dir.is_dir()):
        init()  # must init full index because we need a db to store Binary entries in

    # Show what we're installing
@@ -45,10 +51,6 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo
        print(f"    DATA_DIR will be owned by [blue]{ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}[/blue].")
        print()

-    if dry_run:
-        print("[dim]Dry run - would run the abx-dl install flow[/dim]")
-        return
-
    # Set up Django
    from archivebox.config.django import setup_django

@@ -7,50 +7,15 @@ import sys

 import rich_click as click

-from archivebox.cli.archivebox_snapshot import list_snapshots
+from archivebox.cli.archivebox_snapshot import list_snapshots, snapshot_filter_options, snapshot_output_options


@click.command()
-@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
-@click.option("--url__icontains", help="Filter by URL contains")
-@click.option("--url__istartswith", help="Filter by URL starts with")
-@click.option("--tag", "-t", help="Filter by tag name")
-@click.option("--crawl-id", help="Filter by crawl ID")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title")
-@click.option("--with-headers", is_flag=True, help="Include column headers in structured output")
-@click.option("--search", type=click.Choice(["meta", "content", "contents", "deep"]), help="Search mode to use for the query")
-@click.argument("query", nargs=-1)
-def main(
-    status: str | None,
-    url__icontains: str | None,
-    url__istartswith: str | None,
-    tag: str | None,
-    crawl_id: str | None,
-    limit: int | None,
-    sort: str | None,
-    csv: str | None,
-    with_headers: bool,
-    search: str | None,
-    query: tuple[str, ...],
-) -> None:
+@snapshot_output_options
+@snapshot_filter_options(default_filter_type="substring")
+def main(**kwargs) -> None:
    """List Snapshots."""
-    sys.exit(
-        list_snapshots(
-            status=status,
-            url__icontains=url__icontains,
-            url__istartswith=url__istartswith,
-            tag=tag,
-            crawl_id=crawl_id,
-            limit=limit,
-            sort=sort,
-            csv=csv,
-            with_headers=with_headers,
-            search=search,
-            query=" ".join(query),
-        ),
-    )
+    sys.exit(list_snapshots(**kwargs))


 if __name__ == "__main__":
@@ -27,7 +27,7 @@ import sys
 import rich_click as click
 from rich import print as rprint

-from archivebox.cli.cli_utils import apply_filters
+from archivebox.cli.cli_util import apply_filters


 # =============================================================================
@@ -10,10 +10,11 @@ from archivebox.misc.util import docstring, enforce_types
 def manage(args: list[str] | None = None) -> None:
    """Run an ArchiveBox Django management command"""

-    from archivebox.config.common import SHELL_CONFIG
+    from archivebox.config.common import get_config
    from archivebox.misc.logging import stderr

-    if (args and "createsuperuser" in args) and (SHELL_CONFIG.IN_DOCKER and not SHELL_CONFIG.IS_TTY):
+    config = get_config()
+    if (args and "createsuperuser" in args) and (config.IN_DOCKER and not config.IS_TTY):
        stderr("[!] Warning: you need to pass -it to use interactive commands in docker", color="lightyellow")
        stderr("    docker run -it archivebox manage {}".format(" ".join(args or ["..."])), color="lightyellow")
        stderr("")
@@ -31,17 +31,13 @@ import os
 import sys
 import shutil
 import platform
-import subprocess
-import tempfile
-import json
 from pathlib import Path
 from collections.abc import Iterable
-from collections import OrderedDict

 import rich_click as click
 from rich import print as rprint

-from archivebox.cli.cli_utils import apply_filters
+from archivebox.cli.cli_util import apply_filters
 from archivebox.personas import importers as persona_importers


@@ -214,136 +210,6 @@ CHROMIUM_BROWSERS = {"chrome", "chromium", "brave", "edge"}
 # Cookie Extraction via CDP
 # =============================================================================

-NETSCAPE_COOKIE_HEADER = [
-    "# Netscape HTTP Cookie File",
-    "# https://curl.se/docs/http-cookies.html",
-    "# This file was generated by ArchiveBox persona cookie extraction",
-    "#",
-    "# Format: domain\\tincludeSubdomains\\tpath\\tsecure\\texpiry\\tname\\tvalue",
-    "",
-]
-
-
-def _parse_netscape_cookies(path: Path) -> "OrderedDict[tuple[str, str, str], tuple[str, str, str, str, str, str, str]]":
-    cookies = OrderedDict()
-    if not path.exists():
-        return cookies
-
-    for line in path.read_text().splitlines():
-        if not line or line.startswith("#"):
-            continue
-        parts = line.split("\t")
-        if len(parts) < 7:
-            continue
-        domain, include_subdomains, cookie_path, secure, expiry, name, value = parts[:7]
-        key = (domain, cookie_path, name)
-        cookies[key] = (domain, include_subdomains, cookie_path, secure, expiry, name, value)
-    return cookies
-
-
-def _write_netscape_cookies(path: Path, cookies: "OrderedDict[tuple[str, str, str], tuple[str, str, str, str, str, str, str]]") -> None:
-    lines = list(NETSCAPE_COOKIE_HEADER)
-    for cookie in cookies.values():
-        lines.append("\t".join(cookie))
-    path.write_text("\n".join(lines) + "\n")
-
-
-def _merge_netscape_cookies(existing_file: Path, new_file: Path) -> None:
-    existing = _parse_netscape_cookies(existing_file)
-    new = _parse_netscape_cookies(new_file)
-    for key, cookie in new.items():
-        existing[key] = cookie
-    _write_netscape_cookies(existing_file, existing)
-
-
-def extract_cookies_via_cdp(
-    user_data_dir: Path,
-    output_file: Path,
-    profile_dir: str | None = None,
-    chrome_binary: str | None = None,
-) -> bool:
-    """
-    Launch Chrome with the given user data dir and extract cookies via CDP.
-
-    Returns True if successful, False otherwise.
-    """
-    from archivebox.config.common import STORAGE_CONFIG
-
-    # Find the cookie extraction script
-    chrome_plugin_dir = Path(__file__).parent.parent / "plugins" / "chrome"
-    extract_script = chrome_plugin_dir / "extract_cookies.js"
-
-    if not extract_script.exists():
-        rprint(f"[yellow]Cookie extraction script not found at {extract_script}[/yellow]", file=sys.stderr)
-        return False
-
-    # Get node modules dir
-    node_modules_dir = STORAGE_CONFIG.LIB_DIR / "npm" / "node_modules"
-
-    # Set up environment
-    env = os.environ.copy()
-    env["NODE_MODULES_DIR"] = str(node_modules_dir)
-    env["CHROME_USER_DATA_DIR"] = str(user_data_dir)
-    env["CHROME_HEADLESS"] = "true"
-    if chrome_binary:
-        env["CHROME_BINARY"] = str(chrome_binary)
-    output_path = output_file
-    temp_output = None
-    temp_dir = None
-    if output_file.exists():
-        temp_dir = Path(tempfile.mkdtemp(prefix="ab_cookies_"))
-        temp_output = temp_dir / "cookies.txt"
-        output_path = temp_output
-    if profile_dir:
-        extra_arg = f"--profile-directory={profile_dir}"
-        existing_extra = env.get("CHROME_ARGS_EXTRA", "").strip()
-        args_list = []
-        if existing_extra:
-            if existing_extra.startswith("["):
-                try:
-                    parsed = json.loads(existing_extra)
-                    if isinstance(parsed, list):
-                        args_list.extend(str(x) for x in parsed)
-                except Exception:
-                    args_list.extend([s.strip() for s in existing_extra.split(",") if s.strip()])
-            else:
-                args_list.extend([s.strip() for s in existing_extra.split(",") if s.strip()])
-        args_list.append(extra_arg)
-        env["CHROME_ARGS_EXTRA"] = json.dumps(args_list)
-
-    env["COOKIES_OUTPUT_FILE"] = str(output_path)
-
-    try:
-        result = subprocess.run(
-            ["node", str(extract_script)],
-            env=env,
-            capture_output=True,
-            text=True,
-            timeout=60,
-        )
-
-        if result.returncode == 0:
-            if temp_output and temp_output.exists():
-                _merge_netscape_cookies(output_file, temp_output)
-            return True
-        else:
-            rprint(f"[yellow]Cookie extraction failed: {result.stderr}[/yellow]", file=sys.stderr)
-            return False
-
-    except subprocess.TimeoutExpired:
-        rprint("[yellow]Cookie extraction timed out[/yellow]", file=sys.stderr)
-        return False
-    except FileNotFoundError:
-        rprint("[yellow]Node.js not found. Cannot extract cookies.[/yellow]", file=sys.stderr)
-        return False
-    except Exception as e:
-        rprint(f"[yellow]Cookie extraction error: {e}[/yellow]", file=sys.stderr)
-        return False
-    finally:
-        if temp_dir and temp_dir.exists():
-            shutil.rmtree(temp_dir, ignore_errors=True)
-
-
 # =============================================================================
 # Validation Helpers
 # =============================================================================
@@ -506,7 +372,7 @@ def create_personas(
        if not is_tty:
            write_record(
                {
-                    "id": str(persona.id) if hasattr(persona, "id") else None,
+                    "id": str(persona.id),
                    "name": persona.name,
                    "path": str(persona.path),
                    "CHROME_USER_DATA_DIR": persona.CHROME_USER_DATA_DIR,
@@ -558,7 +424,7 @@ def list_personas(
        else:
            write_record(
                {
-                    "id": str(persona.id) if hasattr(persona, "id") else None,
+                    "id": str(persona.id),
                    "name": persona.name,
                    "path": str(persona.path),
                    "CHROME_USER_DATA_DIR": persona.CHROME_USER_DATA_DIR,
@@ -634,7 +500,7 @@ def update_personas(name: str | None = None) -> int:
            if not is_tty:
                write_record(
                    {
-                        "id": str(persona.id) if hasattr(persona, "id") else None,
+                        "id": str(persona.id),
                        "name": persona.name,
                        "path": str(persona.path),
                    },
@@ -8,127 +8,32 @@ import rich_click as click
 from archivebox.misc.util import docstring, enforce_types


-# State Machine ASCII Art Diagrams
-CRAWL_MACHINE_DIAGRAM = """
+EVENT_FLOW_DIAGRAM = """
 ┌─────────────────────────────────────────────────────────────────────────────┐
-│                              CrawlMachine                                   │
+│                           ArchiveBox / abx-dl Flow                         │
 ├─────────────────────────────────────────────────────────────────────────────┤
 │                                                                             │
-│   ┌─────────────┐                                                           │
-│   │   QUEUED    │◄────────────────┐                                         │
-│   │  (initial)  │                 │                                         │
-│   └──────┬──────┘                 │                                         │
-│          │                        │ tick() unless can_start()               │
-│          │ tick() when            │                                         │
-│          │ can_start()            │                                         │
-│          ▼                        │                                         │
-│   ┌─────────────┐                 │                                         │
-│   │   STARTED   │─────────────────┘                                         │
-│   │             │◄────────────────┐                                         │
-│   │ enter:      │                 │                                         │
-│   │  crawl.run()│                 │ tick() unless is_finished()             │
-│   │  (discover  │                 │                                         │
-│   │   Crawl     │─────────────────┘                                         │
-│   │   hooks)    │                                                           │
-│   └──────┬──────┘                                                           │
-│          │                                                                  │
-│          │ tick() when is_finished()                                        │
-│          ▼                                                                  │
-│   ┌─────────────┐                                                           │
-│   │   SEALED    │                                                           │
-│   │   (final)   │                                                           │
-│   │             │                                                           │
-│   │ enter:      │                                                           │
-│   │  cleanup()  │                                                           │
-│   └─────────────┘                                                           │
+│  InstallEvent                                                               │
+│    └─ config.json > required_binaries                                       │
+│         └─ BinaryRequestEvent                                               │
+│              └─ abxpkg BinaryService builtin providers                      │
+│                   └─ BinaryEvent                                            │
+│                        └─ BinaryCacheService / project cache backend        │
 │                                                                             │
-│   Hooks triggered: on_Crawl__* (during STARTED.enter via crawl.run())       │
-│                    on_CrawlEnd__* (during SEALED.enter via cleanup())       │
-└─────────────────────────────────────────────────────────────────────────────┘
-"""
-
-SNAPSHOT_MACHINE_DIAGRAM = """
-┌─────────────────────────────────────────────────────────────────────────────┐
-│                            SnapshotMachine                                  │
-├─────────────────────────────────────────────────────────────────────────────┤
+│  CrawlEvent                                                                 │
+│    └─ CrawlSetupEvent                                                       │
+│         └─ on_CrawlSetup__*                                                 │
 │                                                                             │
-│   ┌─────────────┐                                                           │
-│   │   QUEUED    │◄────────────────┐                                         │
-│   │  (initial)  │                 │                                         │
-│   └──────┬──────┘                 │                                         │
-│          │                        │ tick() unless can_start()               │
-│          │ tick() when            │                                         │
-│          │ can_start()            │                                         │
-│          ▼                        │                                         │
-│   ┌─────────────┐                 │                                         │
-│   │   STARTED   │─────────────────┘                                         │
-│   │             │◄────────────────┐                                         │
-│   │ enter:      │                 │                                         │
-│   │ snapshot    │                 │ tick() unless is_finished()             │
-│   │  .run()     │                 │                                         │
-│   │ (discover   │─────────────────┘                                         │
-│   │  Snapshot   │                                                           │
-│   │  hooks,     │                                                           │
-│   │  create     │                                                           │
-│   │  pending    │                                                           │
-│   │  results)   │                                                           │
-│   └──────┬──────┘                                                           │
-│          │                                                                  │
-│          │ tick() when is_finished()                                        │
-│          ▼                                                                  │
-│   ┌─────────────┐                                                           │
-│   │   SEALED    │                                                           │
-│   │   (final)   │                                                           │
-│   │             │                                                           │
-│   │ enter:      │                                                           │
-│   │  cleanup()  │                                                           │
-│   └─────────────┘                                                           │
+│  CrawlStartEvent                                                            │
+│    └─ SnapshotEvent                                                         │
+│         └─ on_Snapshot__*                                                   │
+│              └─ ArchiveResult / Snapshot / Tag                              │
 │                                                                             │
-│   Hooks triggered: on_Snapshot__* (creates ArchiveResults in STARTED.enter) │
-└─────────────────────────────────────────────────────────────────────────────┘
-"""
-
-BINARY_MACHINE_DIAGRAM = """
-┌─────────────────────────────────────────────────────────────────────────────┐
-│                             BinaryMachine                                   │
-├─────────────────────────────────────────────────────────────────────────────┤
+│  SnapshotCleanupEvent  -> internal cleanup, no direct hook family           │
+│  CrawlCleanupEvent     -> internal cleanup, no direct hook family           │
 │                                                                             │
-│   ┌─────────────┐                                                           │
-│   │   QUEUED    │◄────────────────┐                                         │
-│   │  (initial)  │                 │                                         │
-│   └──────┬──────┘                 │                                         │
-│          │                        │ tick() unless can_install()             │
-│          │                        │ (stays queued if failed)                │
-│          │ tick() when            │                                         │
-│          │ can_install()          │                                         │
-│          │                        │                                         │
-│          │ on_install() runs      │                                         │
-│          │ during transition:     │                                         │
-│          │  • binary.run()        │                                         │
-│          │    (discover Binary    │                                         │
-│          │     hooks, try each    │                                         │
-│          │     provider until     │                                         │
-│          │     one succeeds)      │                                         │
-│          │  • Sets abspath,       │                                         │
-│          │    version, sha256     │                                         │
-│          │                        │                                         │
-│          │ If install fails:      │                                         │
-│          │  raises exception──────┘                                         │
-│          │  (retry_at bumped)                                               │
-│          │                                                                  │
-│          ▼                                                                  │
-│   ┌─────────────┐                                                           │
-│   │  INSTALLED  │                                                           │
-│   │   (final)   │                                                           │
-│   │             │                                                           │
-│   │ Binary is   │                                                           │
-│   │ ready to    │                                                           │
-│   │ use         │                                                           │
-│   └─────────────┘                                                           │
-│                                                                             │
-│   Hooks triggered: on_Binary__* (provider hooks during transition)          │
-│   Providers tried in sequence until one succeeds: apt, brew, pip, npm, etc. │
-│   Installation is synchronous - no intermediate STARTED state               │
+│  ArchiveBox projects bus events into the DB; it no longer drives plugin     │
+│  execution through the old queued model executor.                           │
 └─────────────────────────────────────────────────────────────────────────────┘
 """

@@ -136,24 +41,28 @@ BINARY_MACHINE_DIAGRAM = """
@enforce_types
 def pluginmap(
    show_disabled: bool = False,
-    model: str | None = None,
+    event: str | None = None,
    quiet: bool = False,
 ) -> dict:
    """
-    Show a map of all state machines and their associated plugin hooks.
+    Show the current abx-dl event phases and their associated plugin hooks.

-    Displays ASCII art diagrams of the core queued model state machines (Crawl,
-    Snapshot, Binary) and lists all auto-detected on_Modelname_xyz hooks
-    that will run for each model's transitions.
+    This command reflects the new bus-driven runtime, not the legacy ArchiveBox
+    state-machine executor. Event names are normalized to hook prefixes by
+    stripping a trailing `Event`, then ArchiveBox checks whether any matching
+    `on_{EventFamily}__*` scripts actually exist.
    """
    from rich.console import Console
    from rich.table import Table
    from rich.panel import Panel
    from rich import box

-    from archivebox.hooks import (
+    from archivebox.plugins.hooks import (
        discover_hooks,
        is_background_hook,
+        normalize_hook_event_name,
+    )
+    from archivebox.plugins.discovery import (
        BUILTIN_PLUGINS_DIR,
        USER_PLUGINS_DIR,
    )
@@ -161,40 +70,57 @@ def pluginmap(
    console = Console()
    prnt = console.print

-    # Model event types that can have hooks
-    model_events = {
-        "Crawl": {
-            "description": "Hooks run when a Crawl starts (QUEUED→STARTED)",
-            "machine": "CrawlMachine",
-            "diagram": CRAWL_MACHINE_DIAGRAM,
+    event_phases = {
+        "InstallEvent": {
+            "description": "Pre-run dependency phase. Enabled plugins emit BinaryRequest events from config.json required_binaries.",
+            "emits": ["BinaryRequestEvent", "BinaryEvent", "ProcessEvent"],
        },
-        "CrawlEnd": {
-            "description": "Hooks run when a Crawl finishes (STARTED→SEALED)",
-            "machine": "CrawlMachine",
-            "diagram": None,  # Part of CrawlMachine
+        "BinaryRequestEvent": {
+            "description": "Binary resolution phase. abxpkg BinaryService resolves or installs requested binaries using built-in providers.",
+            "emits": ["BinaryEvent", "ProcessEvent"],
+            "direct_hooks": False,
        },
-        "Snapshot": {
-            "description": "Hooks run for each Snapshot (creates ArchiveResults)",
-            "machine": "SnapshotMachine",
-            "diagram": SNAPSHOT_MACHINE_DIAGRAM,
+        "BinaryEvent": {
+            "description": "Resolved binary metadata event. ArchiveBoxBinaryService projects it into the ArchiveBox DB binary cache.",
+            "emits": [],
+            "direct_hooks": False,
        },
-        "Binary": {
-            "description": "Hooks for installing binary dependencies (providers)",
-            "machine": "BinaryMachine",
-            "diagram": BINARY_MACHINE_DIAGRAM,
+        "CrawlEvent": {
+            "description": "Root crawl lifecycle event emitted by the runner.",
+            "emits": ["CrawlSetupEvent", "CrawlStartEvent", "CrawlCleanupEvent", "CrawlCompletedEvent"],
+        },
+        "CrawlSetupEvent": {
+            "description": "Crawl-scoped setup phase. on_CrawlSetup hooks launch/configure shared daemons and runtime state.",
+            "emits": ["ProcessEvent"],
+        },
+        "SnapshotEvent": {
+            "description": "Per-snapshot extraction phase. on_Snapshot hooks emit ArchiveResult, Snapshot, and Tag records.",
+            "emits": ["ArchiveResultEvent", "SnapshotEvent", "TagEvent", "ProcessEvent"],
+        },
+        "SnapshotCleanupEvent": {
+            "description": "Internal snapshot cleanup phase.",
+            "emits": ["ProcessKillEvent"],
+        },
+        "CrawlCleanupEvent": {
+            "description": "Internal crawl cleanup phase.",
+            "emits": ["ProcessKillEvent"],
        },
    }

-    # Filter to specific model if requested
-    if model:
-        model = model.title()
-        if model not in model_events:
-            prnt(f'[red]Error: Unknown model "{model}". Available: {", ".join(model_events.keys())}[/red]')
-            return {}
-        model_events = {model: model_events[model]}
+    if event:
+        requested = str(event).strip()
+        if requested in event_phases:
+            event_phases = {requested: event_phases[requested]}
+        else:
+            normalized_requested = normalize_hook_event_name(requested)
+            matched_name = next((name for name in event_phases if normalize_hook_event_name(name) == normalized_requested), None)
+            if matched_name is None:
+                prnt(f'[red]Error: Unknown event "{requested}". Available: {", ".join(event_phases.keys())}[/red]')
+                return {}
+            event_phases = {matched_name: event_phases[matched_name]}

    result = {
-        "models": {},
+        "events": {},
        "plugins_dir": str(BUILTIN_PLUGINS_DIR),
        "user_plugins_dir": str(USER_PLUGINS_DIR),
    }
@@ -205,88 +131,86 @@ def pluginmap(
        prnt(f"[dim]Built-in plugins: {BUILTIN_PLUGINS_DIR}[/dim]")
        prnt(f"[dim]User plugins: {USER_PLUGINS_DIR}[/dim]")
        prnt()
+        prnt(
+            Panel(
+                EVENT_FLOW_DIAGRAM,
+                title="[bold green]Event Flow[/bold green]",
+                border_style="green",
+                expand=False,
+            ),
+        )
+        prnt()

-    for event_name, info in model_events.items():
-        # Discover hooks for this event
-        hooks = discover_hooks(event_name, filter_disabled=not show_disabled)
+    for event_name, info in event_phases.items():
+        hook_event = normalize_hook_event_name(event_name)
+        has_direct_hooks = info.get("direct_hooks", True)
+        hooks = discover_hooks(event_name, filter_disabled=not show_disabled) if has_direct_hooks else []

-        # Build hook info list
        hook_infos = []
        for hook_path in hooks:
-            # Get plugin name from parent directory (e.g., 'wget' from 'plugins/wget/on_Snapshot__06_wget.bg.py')
            plugin_name = hook_path.parent.name
-            is_bg = is_background_hook(hook_path.name)
-
            hook_infos.append(
                {
                    "path": str(hook_path),
                    "name": hook_path.name,
                    "plugin": plugin_name,
-                    "is_background": is_bg,
+                    "is_background": is_background_hook(hook_path.name),
                    "extension": hook_path.suffix,
                },
            )

-        result["models"][event_name] = {
+        result["events"][event_name] = {
            "description": info["description"],
-            "machine": info["machine"],
+            "hook_event": hook_event,
+            "emits": info["emits"],
            "hooks": hook_infos,
            "hook_count": len(hook_infos),
        }

-        if not quiet:
-            # Show diagram if this model has one
-            if info.get("diagram"):
-                assert info["diagram"] is not None
-                prnt(
-                    Panel(
-                        info["diagram"],
-                        title=f"[bold green]{info['machine']}[/bold green]",
-                        border_style="green",
-                        expand=False,
-                    ),
-                )
-                prnt()
+        if quiet:
+            continue

-            # Create hooks table
-            table = Table(
-                title=f"[bold yellow]on_{event_name}__* Hooks[/bold yellow] ({len(hooks)} found)",
-                box=box.ROUNDED,
-                show_header=True,
-                header_style="bold magenta",
-            )
-            table.add_column("Plugin", style="cyan", width=20)
-            table.add_column("Hook Name", style="green")
-            table.add_column("BG", justify="center", width=4)
-            table.add_column("Type", justify="center", width=5)
+        title_suffix = f" -> on_{hook_event}__*" if hook_infos else ""
+        table = Table(
+            title=f"[bold yellow]{event_name}[/bold yellow]{title_suffix} ({len(hooks)} hooks)",
+            box=box.ROUNDED,
+            show_header=True,
+            header_style="bold magenta",
+        )
+        table.add_column("Plugin", style="cyan", width=20)
+        table.add_column("Hook Name", style="green")
+        table.add_column("BG", justify="center", width=4)
+        table.add_column("Type", justify="center", width=5)

-            # Sort lexicographically by hook name
-            sorted_hooks = sorted(hook_infos, key=lambda h: h["name"])
-
-            for hook in sorted_hooks:
+        if hook_infos:
+            for hook in sorted(hook_infos, key=lambda h: h["name"]):
                bg_marker = "[yellow]bg[/yellow]" if hook["is_background"] else ""
-                ext = hook["extension"].lstrip(".")
                table.add_row(
                    hook["plugin"],
                    hook["name"],
                    bg_marker,
-                    ext,
+                    hook["extension"].lstrip("."),
                )
+        else:
+            table.add_row("[dim]-[/dim]", "[dim]No direct hooks[/dim]", "", "")

-            prnt(table)
-            prnt()
-            prnt(f"[dim]{info['description']}[/dim]")
-            prnt()
+        prnt(table)
+        prnt(f"[dim]{info['description']}[/dim]")
+        if info["emits"]:
+            prnt(f"[dim]Emits: {', '.join(info['emits'])}[/dim]")
+        if not hook_infos and has_direct_hooks:
+            prnt(f"[dim]No direct on_{hook_event}__* scripts are currently defined for this event family.[/dim]")
+        elif not has_direct_hooks:
+            prnt("[dim]No direct plugin hook family. This event is handled by services.[/dim]")
+        prnt()

-    # Summary
    if not quiet:
-        total_hooks = sum(m["hook_count"] for m in result["models"].values())
+        total_hooks = sum(event_info["hook_count"] for event_info in result["events"].values())
        prnt(f"[bold]Total hooks discovered: {total_hooks}[/bold]")
        prnt()
-        prnt("[dim]Hook naming convention: on_{Model}__{XX}_{description}[.bg].{ext}[/dim]")
-        prnt("[dim]  - XX: Two-digit lexicographic order (00-99)[/dim]")
-        prnt("[dim]  - .bg: Background hook (non-blocking)[/dim]")
-        prnt("[dim]  - ext: py, sh, or js[/dim]")
+        prnt("[dim]Hook naming convention: on_{EventFamily}__{XX}_{description}[.bg].{ext}[/dim]")
+        prnt("[dim]Event names are normalized with a simple `Event` suffix strip before hook discovery.[/dim]")
+        prnt("[dim]If no `on_{EventFamily}__*` scripts exist, the event is shown as having no direct hooks.[/dim]")
        prnt()

    return result
@@ -294,8 +218,8 @@ def pluginmap(

@click.command()
@click.option("--show-disabled", "-a", is_flag=True, help="Show hooks from disabled plugins too")
-@click.option("--model", "-m", type=str, default=None, help="Filter to specific model (Crawl, Snapshot, Binary, CrawlEnd)")
-@click.option("--quiet", "-q", is_flag=True, help="Output JSON only, no ASCII diagrams")
+@click.option("--event", "-e", type=str, default=None, help="Filter to specific event (e.g. InstallEvent, SnapshotEvent)")
+@click.option("--quiet", "-q", is_flag=True, help="Output JSON only, no tables")
@docstring(pluginmap.__doc__)
 def main(**kwargs):
    import json
@@ -30,7 +30,7 @@ import sys
 import rich_click as click
 from rich import print as rprint

-from archivebox.cli.cli_utils import apply_filters
+from archivebox.cli.cli_util import apply_filters


 # =============================================================================
@@ -54,7 +54,7 @@ def list_processes(

    is_tty = sys.stdout.isatty()

-    queryset = Process.objects.all().select_related("binary", "machine").order_by("-start_ts")
+    queryset = Process.objects.all().select_related("binary", "machine").order_by("-started_at", "-created_at")

    # Apply filters
    filter_kwargs = {}
@@ -3,16 +3,16 @@
 __package__ = "archivebox.cli"
 __command__ = "archivebox remove"

-import shutil
+import time
 from pathlib import Path
 from collections.abc import Iterable

 import rich_click as click

+from django.db import OperationalError
 from django.db.models import QuerySet

-from archivebox.config import DATA_DIR
-from archivebox.config.constants import CONSTANTS
+from archivebox.config import CONSTANTS
 from archivebox.config.django import setup_django
 from archivebox.misc.util import enforce_types, docstring
 from archivebox.misc.checks import check_data_folder
@@ -23,6 +23,7 @@ from archivebox.misc.logging_util import (
    log_removal_finished,
    TimedProgress,
 )
+from archivebox.cli.archivebox_snapshot import snapshot_filter_options


@enforce_types
@@ -33,79 +34,122 @@ def remove(
    after: float | None = None,
    before: float | None = None,
    yes: bool = False,
-    delete: bool = False,
-    out_dir: Path = DATA_DIR,
-) -> QuerySet:
+    out_dir: Path = CONSTANTS.DATA_DIR,
+    timeout: float | None = None,
+    **kwargs,
+) -> dict[str, object]:
    """Remove the specified URLs from the archive"""

    setup_django()
    check_data_folder()
+    timeout = float(timeout) if timeout is not None else None

-    from archivebox.cli.archivebox_search import get_snapshots
+    from archivebox.core.models import Snapshot

+    filter_kwargs = {
+        **kwargs,
+        "filter_patterns": filter_patterns,
+        "filter_type": filter_type,
+        "after": after,
+        "before": before,
+    }
    pattern_list = list(filter_patterns)

    log_list_started(pattern_list or None, filter_type)
    timer = TimedProgress(360, prefix="      ")
    try:
-        snapshots = get_snapshots(
-            snapshots=snapshots,
-            filter_patterns=pattern_list or None,
-            filter_type=filter_type,
-            after=after,
-            before=before,
-        )
+        if snapshots is None:
+            snapshots = Snapshot.objects.order_by("-created_at").search(**filter_kwargs)
+        # Freeze the target set up-front so a concurrent daemon writing new
+        # snapshots can't extend the deletion under us, and so the cursor isn't
+        # held open across the per-row deletes below.
+        snapshot_pks = list(snapshots.values_list("pk", flat=True))
    finally:
        timer.end()

-    if not snapshots.exists():
+    if not snapshot_pks:
        log_removal_finished(0, 0)
        raise SystemExit(1)

-    log_list_finished(snapshots)
-    log_removal_started(snapshots, yes=yes, delete=delete)
+    if not yes:
+        log_list_finished(snapshots)
+        log_removal_started(snapshots, yes=False)

-    timer = TimedProgress(360, prefix="      ")
-    try:
-        for snapshot in snapshots:
-            if delete:
-                shutil.rmtree(snapshot.output_dir, ignore_errors=True)
-                legacy_path = CONSTANTS.ARCHIVE_DIR / snapshot.timestamp
-                if legacy_path.is_symlink():
-                    legacy_path.unlink(missing_ok=True)
-    finally:
-        timer.end()
+    from archivebox.search.query import flush_search_index

-    to_remove = snapshots.count()
-
-    from archivebox.search import flush_search_index
-    from archivebox.core.models import Snapshot
+    started_at = time.monotonic()
+    deadline = started_at + timeout if timeout is not None else None

+    # Search-index flush touches a separate backend (FTS / sonic), not the
+    # main index.sqlite3 writer lock, so it's safe to do once up front.
    flush_search_index(snapshots=snapshots)
-    snapshots.delete()
-    all_snapshots = Snapshot.objects.all()
-    log_removal_finished(all_snapshots.count(), to_remove)

-    return all_snapshots
+    # Delete one snapshot at a time. Each ``.delete()`` is its own short
+    # Django-atomic block, so the writer lock is released between rows and
+    # an in-flight daemon transaction can interleave instead of deadlocking.
+    # Filesystem cleanup for each row is scheduled via ``transaction.on_commit``
+    # in ``base_models/models.py`` and runs AFTER its row's tx commits — so
+    # rmtree doesn't hold the lock either.
+    #
+    # The SQLite retry wrapper in core/sqlite_backend/base.py re-raises lock
+    # errors when called inside an atomic block (because it can't safely
+    # release+reacquire a transaction), so we wrap each row's delete in our
+    # own retry loop at this outer (non-atomic) level. Each attempt is a
+    # fresh atomic; an exception cleanly rolls it back before we sleep.
+    retry_interval = 1.0
+    deleted_snapshot_pks = []
+    timed_out = False
+    timeout_error = ""
+    for index, pk in enumerate(snapshot_pks):
+        if deadline is not None and time.monotonic() >= deadline:
+            timed_out = True
+            timeout_error = f"Remove timed out after {timeout:g}s with {len(snapshot_pks) - index} snapshots remaining."
+            break
+        while True:
+            try:
+                deleted_count, _ = Snapshot.objects.filter(pk=pk).delete()
+                if deleted_count:
+                    deleted_snapshot_pks.append(pk)
+                break
+            except OperationalError as err:
+                if "database is locked" not in str(err):
+                    raise
+                remaining_time = deadline - time.monotonic() if deadline is not None else None
+                if remaining_time is not None and remaining_time <= 0:
+                    timed_out = True
+                    timeout_error = f"Remove timed out after {timeout:g}s while waiting for the database lock."
+                    break
+                time.sleep(min(retry_interval, remaining_time) if remaining_time is not None else retry_interval)
+        if timed_out:
+            break
+
+    all_snapshots = Snapshot.objects.all()
+    remaining_count = all_snapshots.count()
+    deleted_snapshot_id_set = set(deleted_snapshot_pks)
+    remaining_snapshot_pks = [snapshot_id for snapshot_id in snapshot_pks if snapshot_id not in deleted_snapshot_id_set]
+    log_removal_finished(remaining_count, len(deleted_snapshot_pks))
+
+    return {
+        "removed_count": len(deleted_snapshot_pks),
+        "removed_snapshot_ids": [str(snapshot_id) for snapshot_id in deleted_snapshot_pks],
+        "not_removed_count": len(remaining_snapshot_pks),
+        "not_removed_snapshot_ids": [str(snapshot_id) for snapshot_id in remaining_snapshot_pks],
+        "success": not timed_out,
+        "error": timeout_error,
+        "timeout": timeout,
+    }


@click.command()
@click.option("--yes", is_flag=True, help="Remove links instantly without prompting to confirm")
-@click.option("--delete", is_flag=True, help="Delete the archived content and metadata folder in addition to removing from index")
-@click.option("--before", type=float, help="Remove only URLs bookmarked before timestamp")
-@click.option("--after", type=float, help="Remove only URLs bookmarked after timestamp")
-@click.option(
-    "--filter-type",
-    "-f",
-    type=click.Choice(("exact", "substring", "domain", "regex", "tag")),
-    default="exact",
-    help="Type of pattern matching to use when filtering URLs",
-)
-@click.argument("filter_patterns", nargs=-1)
+@click.option("--timeout", type=float, default=None, help="Maximum seconds to spend deleting snapshots")
+@snapshot_filter_options(default_filter_type="exact")
@docstring(remove.__doc__)
 def main(**kwargs):
    """Remove the specified URLs from the archive"""
-    remove(**kwargs)
+    result = remove(**kwargs)
+    if not result["success"]:
+        raise SystemExit(124)


 if __name__ == "__main__":
@@ -10,7 +10,7 @@ Modes:
    - Without stdin (TTY): Run the background runner in foreground until killed
    - --crawl-id: Run the crawl runner for a specific crawl only
    - --snapshot-id: Run a specific snapshot through its parent crawl
-    - --binary-id: Emit a BinaryEvent for a specific Binary row
+    - --binary-id: Emit a BinaryRequestEvent for a specific Binary row

 Examples:
    # Run the background runner in foreground
@@ -40,6 +40,9 @@ Examples:
 __package__ = "archivebox.cli"
 __command__ = "archivebox run"

+import asyncio
+import os
+import signal
 import sys
 from collections import defaultdict

@@ -47,6 +50,13 @@ import rich_click as click
 from rich import print as rprint


+RUNNER_DAEMON_ENV = "ARCHIVEBOX_RUNNER_DAEMON"
+
+
+def _exit_daemon_runner_on_signal(sig: signal.Signals) -> None:
+    os._exit(128 + int(sig))
+
+
 def process_stdin_records() -> int:
    """
    Process JSONL records from stdin.
@@ -64,10 +74,19 @@ def process_stdin_records() -> int:
    """
    from django.utils import timezone

-    from archivebox.misc.jsonl import read_stdin, write_record, TYPE_CRAWL, TYPE_SNAPSHOT, TYPE_ARCHIVERESULT, TYPE_BINARY
+    from archivebox.misc.jsonl import (
+        read_stdin,
+        write_record,
+        TYPE_CRAWL,
+        TYPE_SNAPSHOT,
+        TYPE_ARCHIVERESULT,
+        TYPE_BINARYREQUEST,
+        TYPE_BINARY,
+    )
    from archivebox.base_models.models import get_or_create_system_user_pk
    from archivebox.core.models import Snapshot, ArchiveResult
    from archivebox.crawls.models import Crawl
+    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
    from archivebox.machine.models import Binary
    from archivebox.services.runner import run_binary, run_crawl

@@ -103,10 +122,10 @@ def process_stdin_records() -> int:
                    crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})

                if crawl:
-                    crawl.retry_at = timezone.now()
-                    if crawl.status not in [Crawl.StatusChoices.SEALED]:
-                        crawl.status = Crawl.StatusChoices.QUEUED
-                    crawl.save()
+                    crawl.update_and_requeue(
+                        status=Crawl.StatusChoices.QUEUED,
+                        retry_at=timezone.now(),
+                    )
                    full_crawl_ids.add(str(crawl.id))
                    run_all_plugins_for_crawl.add(str(crawl.id))
                    output_records.append(crawl.to_json())
@@ -124,15 +143,7 @@ def process_stdin_records() -> int:
                    snapshot = Snapshot.from_json(record, overrides={"created_by_id": created_by_id})

                if snapshot:
-                    snapshot.retry_at = timezone.now()
-                    if snapshot.status not in [Snapshot.StatusChoices.SEALED]:
-                        snapshot.status = Snapshot.StatusChoices.QUEUED
-                    snapshot.save()
-                    crawl = snapshot.crawl
-                    crawl.retry_at = timezone.now()
-                    if crawl.status != Crawl.StatusChoices.STARTED:
-                        crawl.status = Crawl.StatusChoices.QUEUED
-                    crawl.save(update_fields=["status", "retry_at", "modified_at"])
+                    snapshot.queue_for_extraction()
                    crawl_id = str(snapshot.crawl_id)
                    snapshot_ids_by_crawl[crawl_id].add(str(snapshot.id))
                    run_all_plugins_for_crawl.add(crawl_id)
@@ -169,15 +180,7 @@ def process_stdin_records() -> int:
                        snapshot = None

                if snapshot:
-                    snapshot.retry_at = timezone.now()
-                    if snapshot.status != Snapshot.StatusChoices.STARTED:
-                        snapshot.status = Snapshot.StatusChoices.QUEUED
-                    snapshot.save(update_fields=["status", "retry_at", "modified_at"])
-                    crawl = snapshot.crawl
-                    crawl.retry_at = timezone.now()
-                    if crawl.status != Crawl.StatusChoices.STARTED:
-                        crawl.status = Crawl.StatusChoices.QUEUED
-                    crawl.save(update_fields=["status", "retry_at", "modified_at"])
+                    snapshot.queue_for_extraction()
                    crawl_id = str(snapshot.crawl_id)
                    snapshot_ids_by_crawl[crawl_id].add(str(snapshot.id))
                    if plugin_name:
@@ -185,7 +188,7 @@ def process_stdin_records() -> int:
                    output_records.append(record if not archiveresult else archiveresult.to_json())
                    queued_count += 1

-            elif record_type == TYPE_BINARY:
+            elif record_type in {TYPE_BINARYREQUEST, TYPE_BINARY}:
                if record_id:
                    try:
                        binary = Binary.objects.get(id=record_id)
@@ -228,15 +231,24 @@ def process_stdin_records() -> int:
    targeted_crawl_ids = full_crawl_ids | set(snapshot_ids_by_crawl)
    if targeted_crawl_ids:
        for crawl_id in sorted(targeted_crawl_ids):
-            run_crawl(
-                crawl_id,
-                snapshot_ids=None if crawl_id in full_crawl_ids else sorted(snapshot_ids_by_crawl[crawl_id]),
-                selected_plugins=None if crawl_id in run_all_plugins_for_crawl else sorted(plugin_names_by_crawl[crawl_id]),
-            )
+            try:
+                crawl = Crawl.objects.get(id=crawl_id)
+            except Crawl.DoesNotExist:
+                continue
+            if not crawl.claim_processing_lock(lock_seconds=10):
+                rprint(f"[yellow]Crawl {crawl_id} is already owned by another runner[/yellow]", file=sys.stderr)
+                return 1
+            with foreground_shutdown_signals(), foreground_parent_watchdog():
+                run_crawl(
+                    crawl_id,
+                    snapshot_ids=None if crawl_id in full_crawl_ids else sorted(snapshot_ids_by_crawl[crawl_id]),
+                    selected_plugins=None if crawl_id in run_all_plugins_for_crawl else sorted(plugin_names_by_crawl[crawl_id]),
+                    selected_plugins_are_explicit=False,
+                )
    return 0


-def run_runner(daemon: bool = False) -> int:
+def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_only: bool = False) -> int:
    """
    Run the background runner loop.

@@ -245,34 +257,84 @@ def run_runner(daemon: bool = False) -> int:

    Returns exit code (0 = success, 1 = error).
    """
-    from django.utils import timezone
+    from archivebox.config import CONSTANTS
+    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
    from archivebox.machine.models import Machine, Process
-    from archivebox.services.runner import recover_orphaned_crawls, recover_orphaned_snapshots, run_pending_crawls
+    from archivebox.core.takeover_util import enter_single_runner_gate, standby_until_foreground_runner_needed
+    from archivebox.core.recovery_util import recover_orchestrator_state
+    from archivebox.services.runner import run_pending_crawls

-    Process.cleanup_stale_running()
-    Process.cleanup_orphaned_workers()
-    recover_orphaned_snapshots()
-    recover_orphaned_crawls()
    Machine.current()
    current = Process.current()
-    if current.process_type != Process.TypeChoices.ORCHESTRATOR:
-        current.process_type = Process.TypeChoices.ORCHESTRATOR
-        current.save(update_fields=["process_type", "modified_at"])
+    root_command = current.root
+    if daemon and root_command.process_type in (
+        Process.TypeChoices.SERVER,
+        Process.TypeChoices.ADD,
+        Process.TypeChoices.UPDATE,
+    ):
+        # Server-owned daemon runners are persistent supervisor workers, but
+        # foreground add/update commands are allowed to borrow runner/sonic
+        # leadership without taking down Daphne. Waiting here keeps the worker
+        # on the normal runner path while preventing a server restart loop from
+        # immediately stealing the single-runner gate back from the newer CLI.
+        standby_until_foreground_runner_needed(root_command, data_dir=CONSTANTS.DATA_DIR)
+    if not enter_single_runner_gate(current, data_dir=CONSTANTS.DATA_DIR):
+        current.mark_exited()
+        return 0

+    recover_orchestrator_state(include_chrome=True)
+    if crawl_id:
+        from django.utils import timezone
+        from archivebox.crawls.models import Crawl
+
+        crawl = Crawl.objects.filter(id=crawl_id, status__in=Crawl.RUNNABLE_STATES).first()
+        now = timezone.now()
+        # Only re-lease when the row is unscheduled (retry_at IS NULL) or its
+        # existing lease has already expired. A future retry_at means another
+        # worker is already scheduled — don't clobber.
+        if crawl is not None and (crawl.retry_at is None or crawl.retry_at <= now):
+            # extra_filter pins the read-time retry_at so a concurrent worker
+            # that grabbed the lease between our SELECT and UPDATE wins.
+            crawl.safe_update(
+                {"retry_at": now},
+                refresh=False,
+                extra_filter={"retry_at": crawl.retry_at},
+            )
+    # Only a foreground `archivebox add` gets the interactive "abort current
+    # hook, continue/retry, second Ctrl+C exits" flow. Server/update/run owned
+    # orchestrators should shut down immediately and cleanly on the first signal.
+    interactive_interrupts = current.root.process_type == Process.TypeChoices.ADD
+    if daemon:
+        os.environ[RUNNER_DAEMON_ENV] = "1"
    try:
-        run_pending_crawls(daemon=daemon)
+        with (
+            foreground_shutdown_signals(
+                on_signal=_exit_daemon_runner_on_signal if daemon else None,
+                raise_on_first_signal=not daemon,
+            ),
+            foreground_parent_watchdog(enabled=not daemon),
+        ):
+            run_pending_crawls(
+                daemon=daemon,
+                crawl_id=crawl_id,
+                maintenance_only=maintenance_only,
+                interactive_interrupts=interactive_interrupts,
+            )
        return 0
    except KeyboardInterrupt:
        return 0
+    except asyncio.CancelledError as e:
+        if daemon:
+            rprint(f"[red]Runner cancelled unexpectedly: {type(e).__name__}: {e}[/red]", file=sys.stderr)
+            return 1
+        return 0
    except Exception as e:
        rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
        return 1
    finally:
        current.refresh_from_db()
        if current.status != Process.StatusChoices.EXITED:
-            current.status = Process.StatusChoices.EXITED
-            current.ended_at = current.ended_at or timezone.now()
-            current.save(update_fields=["status", "ended_at", "modified_at"])
+            current.mark_exited()


@click.command()
@@ -280,7 +342,9 @@ def run_runner(daemon: bool = False) -> int:
@click.option("--crawl-id", help="Run the crawl runner for a specific crawl only")
@click.option("--snapshot-id", help="Run one snapshot through its crawl")
@click.option("--binary-id", help="Run one queued binary install directly on the bus")
-def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str):
+@click.option("--maintenance-only", is_flag=True, help="Only process due maintenance ticks on sealed/paused snapshots")
+@click.option("--no-stdin", is_flag=True, hidden=True, help="Run the scheduler even when stdin is not a TTY")
+def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintenance_only: bool, no_stdin: bool):
    """
    Process queued work.

@@ -291,61 +355,81 @@ def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str):
    - --snapshot-id: Run one snapshot through its crawl only
    - --binary-id: Run one queued binary install directly on the bus
    """
-    if snapshot_id:
-        sys.exit(run_snapshot_worker(snapshot_id))
+    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals

-    if binary_id:
+    if daemon and not snapshot_id and not binary_id and not crawl_id:
        try:
-            from archivebox.services.runner import run_binary
-
-            run_binary(binary_id)
-            sys.exit(0)
+            os.environ[RUNNER_DAEMON_ENV] = "1"
+            with (
+                foreground_shutdown_signals(
+                    on_signal=_exit_daemon_runner_on_signal,
+                    raise_on_first_signal=False,
+                ),
+                foreground_parent_watchdog(enabled=False),
+            ):
+                sys.exit(run_runner(daemon=True, maintenance_only=maintenance_only))
        except KeyboardInterrupt:
            sys.exit(0)
-        except Exception as e:
-            rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
-            import traceback

-            traceback.print_exc()
-            sys.exit(1)
+    with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=not daemon):
+        if snapshot_id:
+            sys.exit(run_snapshot_worker(snapshot_id))

-    if crawl_id:
-        try:
-            from archivebox.services.runner import run_crawl
+        if binary_id:
+            try:
+                from archivebox.services.runner import run_binary

-            run_crawl(crawl_id)
-            sys.exit(0)
-        except KeyboardInterrupt:
-            sys.exit(0)
-        except Exception as e:
-            rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
-            import traceback
+                run_binary(binary_id)
+                sys.exit(0)
+            except KeyboardInterrupt:
+                sys.exit(0)
+            except Exception as e:
+                rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
+                import traceback

-            traceback.print_exc()
-            sys.exit(1)
+                traceback.print_exc()
+                sys.exit(1)

-    if daemon:
-        if not sys.stdin.isatty():
-            exit_code = process_stdin_records()
-            if exit_code != 0:
-                sys.exit(exit_code)
-        sys.exit(run_runner(daemon=True))
+        if crawl_id:
+            sys.exit(run_runner(daemon=False, crawl_id=crawl_id, maintenance_only=maintenance_only))

-    if not sys.stdin.isatty():
-        sys.exit(process_stdin_records())
-    else:
-        sys.exit(run_runner(daemon=daemon))
+        if maintenance_only:
+            sys.exit(run_runner(daemon=daemon, maintenance_only=True))
+
+        if not no_stdin and not sys.stdin.isatty():
+            sys.exit(process_stdin_records())
+        else:
+            sys.exit(run_runner(daemon=daemon, maintenance_only=maintenance_only))


 def run_snapshot_worker(snapshot_id: str) -> int:
+    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
    from archivebox.core.models import Snapshot
-    from archivebox.services.runner import run_crawl
+    from archivebox.services.runner import run_due_snapshot
+    from django.utils import timezone

+    snapshot = None
    try:
-        snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
-        run_crawl(str(snapshot.crawl_id), snapshot_ids=[str(snapshot.id)])
+        with foreground_shutdown_signals(), foreground_parent_watchdog():
+            for _ in range(10):
+                snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
+                if snapshot.retry_at is None:
+                    snapshot.update_and_requeue(retry_at=timezone.now())
+                elif snapshot.retry_at > timezone.now():
+                    break
+                if not run_due_snapshot(snapshot, lock_seconds=60):
+                    break
        return 0
    except KeyboardInterrupt:
+        try:
+            if snapshot is not None:
+                snapshot.refresh_from_db()
+            else:
+                snapshot = Snapshot.objects.filter(id=snapshot_id).first()
+            if snapshot is not None and snapshot.status != Snapshot.StatusChoices.SEALED:
+                snapshot.update_and_requeue(retry_at=timezone.now())
+        except Exception:
+            pass
        return 0
    except Exception as e:
        rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
@@ -6,7 +6,6 @@ import rich_click as click
 from rich import print

 from archivebox.misc.util import enforce_types, docstring
-from archivebox.config.common import ARCHIVING_CONFIG


@enforce_types
@@ -20,9 +19,8 @@ def schedule(
    every: str | None = None,
    tag: str = "",
    depth: int | str = 0,
-    overwrite: bool = False,
-    update: bool = not ARCHIVING_CONFIG.ONLY_NEW,
    import_path: str | None = None,
+    config: dict[str, object] | None = None,
 ):
    """Manage database-backed scheduled crawls processed by the crawl runner."""

@@ -30,9 +28,10 @@ def schedule(

    from archivebox.base_models.models import get_or_create_system_user_pk
    from archivebox.crawls.models import Crawl, CrawlSchedule
-    from archivebox.crawls.schedule_utils import validate_schedule
+    from archivebox.crawls.schedule_util import validate_schedule
    from archivebox.services.runner import run_pending_crawls

+    config_overrides = dict(config or {})
    depth = int(depth)
    result: dict[str, object] = {
        "created_schedule_ids": [],
@@ -76,10 +75,12 @@ def schedule(
            status=Crawl.StatusChoices.SEALED,
            retry_at=None,
            config={
-                "ONLY_NEW": not update,
-                "OVERWRITE": overwrite,
                "DEPTH": 0 if is_update_schedule else depth,
                "SCHEDULE_KIND": "update" if is_update_schedule else "crawl",
+                # Caller-supplied overrides (e.g. {"ONLY_NEW": False}) win over the
+                # template defaults. Anything left unset falls through to the
+                # standard config stack at crawl-resolution time.
+                **config_overrides,
            },
        )
        crawl_schedule = CrawlSchedule.objects.create(
@@ -160,8 +161,13 @@ def schedule(
    default="0",
    help="Recursively archive linked pages up to N hops away",
 )
-@click.option("--overwrite", is_flag=True, help="Overwrite existing data if URLs have been archived previously")
-@click.option("--update", is_flag=True, help="Retry previously failed/skipped URLs when scheduled crawls run")
+@click.option(
+    "--only-new/--no-only-new",
+    "only_new",
+    default=None,
+    help="Skip URLs that already have a snapshot (default: inherit from ONLY_NEW config). "
+    "Pass --no-only-new to force re-archive on each scheduled run.",
+)
@click.option("--clear", is_flag=True, help="Disable all currently enabled schedules")
@click.option("--show", is_flag=True, help="Print all currently enabled schedules")
@click.option("--foreground", "-f", is_flag=True, help="Run the global crawl runner in the foreground (no crontab required)")
@@ -170,6 +176,9 @@ def schedule(
@docstring(schedule.__doc__)
 def main(**kwargs):
    """Manage database-backed scheduled crawls processed by the crawl runner."""
+    only_new = kwargs.pop("only_new", None)
+    if only_new is not None:
+        kwargs["config"] = {"ONLY_NEW": bool(only_new)}
    schedule(**kwargs)


@@ -3,254 +3,7 @@
 __package__ = "archivebox.cli"
 __command__ = "archivebox search"

-import sys
-from pathlib import Path
-from typing import TYPE_CHECKING
-from collections.abc import Callable
-
-import rich_click as click
-
-from django.db.models import Q, QuerySet
-
-from archivebox.config import DATA_DIR
-from archivebox.misc.logging import stderr
-from archivebox.misc.util import enforce_types, docstring
-
-if TYPE_CHECKING:
-    from archivebox.core.models import Snapshot
-
-# Filter types for URL matching
-LINK_FILTERS: dict[str, Callable[[str], Q]] = {
-    "exact": lambda pattern: Q(url=pattern),
-    "substring": lambda pattern: Q(url__icontains=pattern),
-    "regex": lambda pattern: Q(url__iregex=pattern),
-    "domain": lambda pattern: (
-        Q(url__istartswith=f"http://{pattern}") | Q(url__istartswith=f"https://{pattern}") | Q(url__istartswith=f"ftp://{pattern}")
-    ),
-    "tag": lambda pattern: Q(tags__name=pattern),
-    "timestamp": lambda pattern: Q(timestamp=pattern),
-}
-
-STATUS_CHOICES = ["indexed", "archived", "unarchived"]
-
-
-def _apply_pattern_filters(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    filter_patterns: list[str],
-    filter_type: str,
-) -> QuerySet["Snapshot", "Snapshot"]:
-    filter_builder = LINK_FILTERS.get(filter_type)
-    if filter_builder is None:
-        stderr()
-        stderr(f"[X] Got invalid pattern for --filter-type={filter_type}", color="red")
-        raise SystemExit(2)
-
-    query = Q()
-    for pattern in filter_patterns:
-        query |= filter_builder(pattern)
-    return snapshots.filter(query)
-
-
-def _snapshots_to_json(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    *,
-    with_headers: bool,
-) -> str:
-    from datetime import datetime, timezone as tz
-
-    from archivebox.config import VERSION
-    from archivebox.config.common import SERVER_CONFIG
-    from archivebox.misc.util import to_json
-
-    main_index_header = (
-        {
-            "info": "This is an index of site data archived by ArchiveBox: The self-hosted web archive.",
-            "schema": "archivebox.index.json",
-            "copyright_info": SERVER_CONFIG.FOOTER_INFO,
-            "meta": {
-                "project": "ArchiveBox",
-                "version": VERSION,
-                "git_sha": VERSION,
-                "website": "https://ArchiveBox.io",
-                "docs": "https://github.com/ArchiveBox/ArchiveBox/wiki",
-                "source": "https://github.com/ArchiveBox/ArchiveBox",
-                "issues": "https://github.com/ArchiveBox/ArchiveBox/issues",
-                "dependencies": {},
-            },
-        }
-        if with_headers
-        else {}
-    )
-
-    snapshot_dicts = [snapshot.to_dict(extended=True) for snapshot in snapshots.iterator(chunk_size=500)]
-    output: dict[str, object] | list[dict[str, object]]
-    if with_headers:
-        output = {
-            **main_index_header,
-            "num_links": len(snapshot_dicts),
-            "updated": datetime.now(tz.utc),
-            "last_run_cmd": sys.argv,
-            "links": snapshot_dicts,
-        }
-    else:
-        output = snapshot_dicts
-
-    return to_json(output, indent=4, sort_keys=True)
-
-
-def _snapshots_to_csv(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    *,
-    cols: list[str],
-    with_headers: bool,
-) -> str:
-    header = ",".join(cols) if with_headers else ""
-    rows = [snapshot.to_csv(cols=cols, separator=",") for snapshot in snapshots.iterator(chunk_size=500)]
-    return "\n".join((header, *rows))
-
-
-def _snapshots_to_html(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    *,
-    with_headers: bool,
-) -> str:
-    from datetime import datetime, timezone as tz
-
-    from django.template.loader import render_to_string
-
-    from archivebox.config import VERSION
-    from archivebox.config.common import SERVER_CONFIG
-    from archivebox.config.version import get_COMMIT_HASH
-
-    template = "static_index.html" if with_headers else "minimal_index.html"
-    snapshot_list = list(snapshots.iterator(chunk_size=500))
-
-    return render_to_string(
-        template,
-        {
-            "version": VERSION,
-            "git_sha": get_COMMIT_HASH() or VERSION,
-            "num_links": str(len(snapshot_list)),
-            "date_updated": datetime.now(tz.utc).strftime("%Y-%m-%d"),
-            "time_updated": datetime.now(tz.utc).strftime("%Y-%m-%d %H:%M"),
-            "links": snapshot_list,
-            "FOOTER_INFO": SERVER_CONFIG.FOOTER_INFO,
-        },
-    )
-
-
-def get_snapshots(
-    snapshots: QuerySet["Snapshot", "Snapshot"] | None = None,
-    filter_patterns: list[str] | None = None,
-    filter_type: str = "substring",
-    after: float | None = None,
-    before: float | None = None,
-    out_dir: Path = DATA_DIR,
-) -> QuerySet["Snapshot", "Snapshot"]:
-    """Filter and return Snapshots matching the given criteria."""
-    from archivebox.core.models import Snapshot
-
-    if snapshots is not None:
-        result = snapshots
-    else:
-        result = Snapshot.objects.all()
-
-    if after is not None:
-        result = result.filter(timestamp__gte=after)
-    if before is not None:
-        result = result.filter(timestamp__lt=before)
-    if filter_patterns:
-        result = _apply_pattern_filters(result, filter_patterns, filter_type)
-
-    # Prefetch crawl relationship to avoid N+1 queries when accessing output_dir
-    result = result.select_related("crawl", "crawl__created_by")
-
-    if not result.exists():
-        stderr("[!] No Snapshots matched your filters:", filter_patterns, f"({filter_type})", color="lightyellow")
-
-    return result
-
-
-@enforce_types
-def search(
-    filter_patterns: list[str] | None = None,
-    filter_type: str = "substring",
-    status: str = "indexed",
-    before: float | None = None,
-    after: float | None = None,
-    sort: str | None = None,
-    json: bool = False,
-    html: bool = False,
-    csv: str | None = None,
-    with_headers: bool = False,
-):
-    """List, filter, and export information about archive entries"""
-
-    if with_headers and not (json or html or csv):
-        stderr("[X] --with-headers requires --json, --html or --csv\n", color="red")
-        raise SystemExit(2)
-
-    # Query DB directly - no filesystem scanning
-    snapshots = get_snapshots(
-        filter_patterns=list(filter_patterns) if filter_patterns else None,
-        filter_type=filter_type,
-        before=before,
-        after=after,
-    )
-
-    # Apply status filter
-    if status == "archived":
-        snapshots = snapshots.filter(downloaded_at__isnull=False)
-    elif status == "unarchived":
-        snapshots = snapshots.filter(downloaded_at__isnull=True)
-    # 'indexed' = all snapshots (no filter)
-
-    if sort:
-        snapshots = snapshots.order_by(sort)
-
-    # Export to requested format
-    if json:
-        output = _snapshots_to_json(snapshots, with_headers=with_headers)
-    elif html:
-        output = _snapshots_to_html(snapshots, with_headers=with_headers)
-    elif csv:
-        output = _snapshots_to_csv(snapshots, cols=csv.split(","), with_headers=with_headers)
-    else:
-        from archivebox.misc.logging_util import printable_folders
-
-        # Convert to dict for printable_folders
-        folders: dict[str, Snapshot | None] = {str(snapshot.output_dir): snapshot for snapshot in snapshots}
-        output = printable_folders(folders, with_headers)
-
-    # Structured exports must be written directly to stdout.
-    # rich.print() reflows long lines to console width, which corrupts JSON/CSV/HTML output.
-    sys.stdout.write(output)
-    if not output.endswith("\n"):
-        sys.stdout.write("\n")
-    return output
-
-
-@click.command()
-@click.option(
-    "--filter-type",
-    "-f",
-    type=click.Choice(["search", *LINK_FILTERS.keys()]),
-    default="substring",
-    help="Pattern matching type for filtering URLs",
-)
-@click.option("--status", "-s", type=click.Choice(STATUS_CHOICES), default="indexed", help="List snapshots with the given status")
-@click.option("--before", "-b", type=float, help="List snapshots bookmarked before the given UNIX timestamp")
-@click.option("--after", "-a", type=float, help="List snapshots bookmarked after the given UNIX timestamp")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--json", "-J", is_flag=True, help="Print output in JSON format")
-@click.option("--html", "-M", is_flag=True, help="Print output in HTML format (suitable for viewing statically without a server)")
-@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: created_at,url,title")
-@click.option("--with-headers", "-H", is_flag=True, help="Include extra CSV/HTML headers in the output")
-@click.help_option("--help", "-h")
-@click.argument("filter_patterns", nargs=-1)
-@docstring(search.__doc__)
-def main(**kwargs):
-    return search(**kwargs)
+from archivebox.cli.archivebox_list import main


 if __name__ == "__main__":
@@ -2,125 +2,245 @@

 __package__ = "archivebox.cli"

-from collections.abc import Iterable
 import sys
+import os
+import socket
+import subprocess
+import time
+from collections.abc import Iterable

 import rich_click as click
 from rich import print

+from archivebox.config import CONSTANTS
 from archivebox.misc.util import docstring, enforce_types
-from archivebox.config.common import SERVER_CONFIG


-def stop_existing_background_runner(*, machine, process_model, supervisor=None, stop_worker_fn=None, log=print) -> int:
-    """Stop any existing orchestrator process so the server can take ownership."""
-    process_model.cleanup_stale_running(machine=machine)
-    process_model.cleanup_orphaned_workers()
+import re as _re

-    running_runners = list(
-        process_model.objects.filter(
-            machine=machine,
-            status=process_model.StatusChoices.RUNNING,
-            process_type=process_model.TypeChoices.ORCHESTRATOR,
-        ).order_by("created_at"),
-    )
-
-    if not running_runners:
-        return 0
-
-    log("[yellow][*] Stopping existing ArchiveBox background runner...[/yellow]")
-
-    if supervisor is not None and stop_worker_fn is not None:
-        for worker_name in ("worker_runner", "worker_runner_watch"):
-            try:
-                stop_worker_fn(supervisor, worker_name)
-            except Exception:
-                pass
-
-    for proc in running_runners:
-        try:
-            proc.kill_tree(graceful_timeout=2.0)
-        except Exception:
-            try:
-                proc.terminate(graceful_timeout=2.0)
-            except Exception:
-                pass
-
-    process_model.cleanup_stale_running(machine=machine)
-    return len(running_runners)
+_IPV4_RE = _re.compile(r"^\d{1,3}(?:\.\d{1,3}){3}$")
+_IPV6_CHARS_RE = _re.compile(r"^[0-9a-fA-F:.]+$")
+_LOCAL_BIND_HOSTS = frozenset({"0.0.0.0", "::", "::0", "127.0.0.1", "::1"})


-def _read_supervisor_worker_command(worker_name: str) -> str:
-    from archivebox.workers.supervisord_util import WORKERS_DIR_NAME, get_sock_file
-
-    worker_conf = get_sock_file().parent / WORKERS_DIR_NAME / f"{worker_name}.conf"
-    if not worker_conf.exists():
-        return ""
-
-    for line in worker_conf.read_text().splitlines():
-        if line.startswith("command="):
-            return line.removeprefix("command=").strip()
-    return ""
+def _is_ipv4_literal(host: str) -> bool:
+    return bool(_IPV4_RE.match(host))


-def _worker_command_matches_bind(command: str, host: str, port: str) -> bool:
-    if not command:
+def _is_ipv6_literal(host: str) -> bool:
+    # Bracketed (e.g. ``[2001:db8::1]``) or bare form. Require at least two
+    # colons so we don't catch random strings with one ``:``.
+    stripped = host.strip("[]")
+    return stripped.count(":") >= 2 and bool(_IPV6_CHARS_RE.match(stripped))
+
+
+def _bind_host_looks_like_ip(host: str) -> bool:
+    if not host or host in _LOCAL_BIND_HOSTS:
        return False
-    return f"{host}:{port}" in command or (f"--bind={host}" in command and f"--port={port}" in command)
+    return _is_ipv4_literal(host) or _is_ipv6_literal(host)


-def stop_existing_server_workers(*, supervisor, stop_worker_fn, host: str, port: str, log=print) -> int:
-    """Stop existing ArchiveBox web workers if they already own the requested bind."""
-    stopped = 0
+def _split_bind_spec(spec: str) -> tuple[str, str]:
+    """Split a ``host:port`` / ``host`` / ``port`` spec into ``(host, port)``.
+
+    The empty strings stand in for "not provided"; the caller fills in
+    defaults. Bracketed IPv6 literals like ``[::1]:8000`` are handled.
+    """
+    spec = (spec or "").strip()
+    if not spec:
+        return "", ""
+    if spec.startswith("["):
+        # Bracketed IPv6: ``[::1]`` or ``[::1]:8000``
+        end = spec.find("]")
+        if end == -1:
+            return spec, ""  # malformed; let validator reject it
+        host = spec[: end + 1]
+        rest = spec[end + 1 :]
+        if rest.startswith(":"):
+            return host, rest[1:]
+        return host, ""
+    if ":" in spec:
+        host, _, port = spec.rpartition(":")
+        return host, port
+    # Bare token: digits = port, anything else = host
+    if spec.isdigit():
+        return "", spec
+    return spec, ""
+
+
+def _parse_and_validate_bind_spec(spec: str) -> tuple[str, str]:
+    """Resolve a CLI/config bind spec to ``(host, port)`` or hard-error.
+
+    Accepts only IP literals (v4 or v6) or the special string ``localhost``
+    (normalized to ``127.0.0.1``). Bare hostnames are rejected because the
+    bind address feeds Daphne, which has to listen on a numeric address;
+    public hostnames belong in ``BASE_URL`` instead. Empty values fall back
+    to ``127.0.0.1`` / ``8000``.
+    """
+    raw_host, raw_port = _split_bind_spec(spec)
+    host = raw_host.strip()
+    port = (raw_port or "").strip() or "8000"
+
+    if host == "" or host.lower() == "localhost":
+        host = "127.0.0.1"
+    elif _is_ipv4_literal(host) or _is_ipv6_literal(host):
+        pass
+    else:
+        print(
+            f"[red][X] Invalid BIND_ADDR host {host!r}: must be an IP literal or 'localhost'.[/red]",
+        )
+        print(
+            "[red]    Hostnames like archive.example.com are not valid bind addresses — Daphne[/red]",
+        )
+        print(
+            "[red]    listens on numeric addresses only. Bind to 0.0.0.0 and set BASE_URL instead:[/red]",
+        )
+        print(
+            f"[red]      BASE_URL=https://{host} archivebox server 0.0.0.0:{port}[/red]",
+        )
+        sys.exit(1)
+
+    try:
+        port_int = int(port)
+    except ValueError:
+        print(f"[red][X] Invalid BIND_ADDR port {port!r}: must be an integer 1-65535.[/red]")
+        sys.exit(1)
+    if not (0 < port_int < 65536):
+        print(f"[red][X] Invalid BIND_ADDR port {port_int}: must be 1-65535.[/red]")
+        sys.exit(1)
+
+    return host, port
+
+
+def _print_server_startup_warnings(config, host: str, port: str) -> None:
+    """Print startup-time security / routing warnings for the server command.
+
+    Runs only from ``archivebox server`` so other entry points (manage shell,
+    plugin lookups, etc.) don't repeat this banner on every config load.
+    """
+    if config.IS_LOWER_SECURITY_MODE:
+        print(
+            f"[yellow][!] WARNING: ArchiveBox is running with SERVER_SECURITY_MODE={config.SERVER_SECURITY_MODE}[/yellow]",
+        )
+        print("[yellow]    Archived pages may share an origin with privileged app routes in this mode.[/yellow]")
+        print("[yellow]    To switch to the safer isolated setup:[/yellow]")
+        print("[yellow]      1. Set SERVER_SECURITY_MODE=safe-subdomains-fullreplay[/yellow]")
+        print("[yellow]      2. Point *.archivebox.localhost (or your chosen base domain) at this server[/yellow]")
+        print(
+            "[yellow]      3. Configure wildcard DNS/TLS or your reverse proxy so admin., web., api., and snapshot subdomains resolve[/yellow]",
+        )
+        print()
+
+    # ``config.BASE_URL`` is the merged value (env > Machine.config > file >
+    # default), which is what the running server will actually use. Earlier we
+    # gated the "BASE_URL not set" warning on ``os.environ["BASE_URL"]`` alone,
+    # which fired noisily when the user pinned BASE_URL via Machine.config /
+    # ArchiveBox.conf instead of via env.
+    base_url = (config.BASE_URL or "").strip()
+    if base_url:
+        # BASE_URL is pinned. The only thing left to surface is a port
+        # mismatch — bind port ≠ BASE_URL's explicit port usually means the
+        # operator started the server with the wrong ``archivebox server PORT``
+        # argument (or forgot to update one side after moving the listener).
+        # A reverse-proxy setup typically omits the port in BASE_URL
+        # (``https://archive.example.com``), so we only warn when BASE_URL
+        # carries an explicit port — otherwise we'd nag every proxy deployment.
+        from urllib.parse import urlparse

-    for worker_name in ("worker_runserver", "worker_daphne"):
        try:
-            proc = supervisor.getProcessInfo(worker_name) if supervisor else None
-        except Exception:
-            proc = None
-        if not isinstance(proc, dict) or proc.get("statename") != "RUNNING":
-            continue
+            base_port = urlparse(base_url).port
+        except (ValueError, TypeError):
+            base_port = None
+        if base_port is not None and str(base_port) != str(port):
+            print(
+                f"[yellow][!] BASE_URL ({base_url}) port {base_port} does not match the port the server is running on ({port}). "
+                "Make sure this is intentional![/yellow]",
+            )
+            print()
+        return

-        command = _read_supervisor_worker_command(worker_name)
-        if not _worker_command_matches_bind(command, host, port):
-            continue
+    # If the user is upgrading from 0.7.3 and already had
+    # CSRF_TRUSTED_ORIGINS set, get_base_url() will silently use that as the
+    # implicit BASE_URL. Surface what we picked so the user knows where their
+    # links / redirects are going — and tell them how to make it explicit.
+    from archivebox.core.routes_util import derive_base_url_from_csrf

-        if stopped == 0:
-            log("[yellow][*] Taking over existing ArchiveBox web server on same port...[/yellow]")
-        stop_worker_fn(supervisor, worker_name)
-        stopped += 1
+    csrf_derived = derive_base_url_from_csrf(config)
+    if csrf_derived:
+        print(
+            f"[yellow][!] BASE_URL is not set; auto-derived [bold]{csrf_derived}[/bold] from a single CSRF_TRUSTED_ORIGINS entry.[/yellow]",
+        )
+        print(
+            "[yellow]    Links / redirects / cookies will use that origin. To silence this hint, set BASE_URL[/yellow]",
+        )
+        print(
+            f"[yellow]    explicitly: [bold]BASE_URL={csrf_derived}[/bold] (matches your existing CSRF_TRUSTED_ORIGINS).[/yellow]",
+        )
+        print()
+        return

-    return stopped
+    # BASE_URL was not set explicitly. The routes_util derivation gives one of
+    # three results, with very different risk profiles — show a tailored hint
+    # so new users coming from the 0.7.x single-domain world know whether the
+    # default is fine for them or needs attention.
+    if _bind_host_looks_like_ip(host):
+        # Real IP literal: subdomain routing can't work, URLs leak the IP.
+        # This is the most urgent case.
+        print(
+            f"[yellow][!] WARNING: BASE_URL is not set and BIND_ADDR resolves to an IP literal ({host}).[/yellow]",
+        )
+        print(
+            "[yellow]    Snapshot / admin / api URLs will be generated with the IP, and subdomain[/yellow]",
+        )
+        print(
+            "[yellow]    routing cannot work against an IP address. Set BASE_URL explicitly, e.g.[/yellow]",
+        )
+        print(
+            "[yellow]      BASE_URL=https://archive.example.com archivebox server 0.0.0.0:8000[/yellow]",
+        )
+        if config.USES_SUBDOMAIN_ROUTING:
+            print(
+                "[yellow]    Or switch SERVER_SECURITY_MODE to a one-domain mode if you can't run a hostname.[/yellow]",
+            )
+        print()
+    else:
+        # Loopback / wildcard bind. The routes_util default of
+        # http://archivebox.localhost:PORT works in a browser on the same
+        # machine, but anything else (reverse proxy, k8s ingress, LAN client)
+        # needs BASE_URL set. (Real hostnames can't reach this branch — the
+        # bind validator rejects them upfront.)
+        print(
+            "[yellow][!] BASE_URL is not set. Generated URLs will fall back to http://archivebox.localhost:<port>.[/yellow]",
+        )
+        print(
+            "[yellow]    That's fine for local browsing on this machine. Set BASE_URL when running behind[/yellow]",
+        )
+        print(
+            "[yellow]    a reverse proxy / ingress / public hostname, e.g.[/yellow]",
+        )
+        print(
+            "[yellow]      BASE_URL=https://archive.example.com archivebox server 0.0.0.0:8000[/yellow]",
+        )
+        print()


@enforce_types
 def server(
-    runserver_args: Iterable[str] = (SERVER_CONFIG.BIND_ADDR,),
+    runserver_args: Iterable[str] | None = None,
    reload: bool = False,
-    init: bool = False,
    debug: bool = False,
    daemonize: bool = False,
    nothreading: bool = False,
 ) -> None:
    """Run the ArchiveBox HTTP server"""
+    from archivebox.config.common import get_config

-    runserver_args = list(runserver_args)
+    config = get_config()
+    runserver_args = list(runserver_args or (config.BIND_ADDR,))

-    if init:
-        from archivebox.cli.archivebox_init import init as archivebox_init
-
-        archivebox_init(quick=True)
-        print()
-
-    from archivebox.misc.checks import check_data_folder
-
-    check_data_folder()
-
-    from archivebox.config.common import SHELL_CONFIG
-
-    run_in_debug = SHELL_CONFIG.DEBUG or debug or reload
+    run_in_debug = config.DEBUG or debug or reload
    if debug or reload:
-        SHELL_CONFIG.DEBUG = True
+        os.environ["DEBUG"] = "True"

    from django.contrib.auth.models import User

@@ -132,90 +252,149 @@ def server(
        print("      [green]archivebox manage createsuperuser[/green]")
        print()

-    host = "127.0.0.1"
-    port = "8000"
+    # First non-empty positional arg is the bind spec; otherwise inherit from
+    # config (which defaults to "127.0.0.1:8000"). _parse_and_validate_bind_spec
+    # hard-errors on hostnames so the rest of the server can assume a numeric
+    # bind host.
+    bind_spec = next((arg for arg in runserver_args if arg), "")
+    host, port = _parse_and_validate_bind_spec(bind_spec)

-    try:
-        host_and_port = [arg for arg in runserver_args if arg.replace(".", "").replace(":", "").isdigit()][0]
-        if ":" in host_and_port:
-            host, port = host_and_port.split(":")
+    if daemonize and os.environ.get("ARCHIVEBOX_SERVER_DAEMON_CHILD") != "1":
+        log_path = CONSTANTS.LOGS_DIR / "server.log"
+        log_path.parent.mkdir(parents=True, exist_ok=True)
+        daemon_env = os.environ.copy()
+        daemon_env["ARCHIVEBOX_SERVER_DAEMON_CHILD"] = "1"
+        daemon_cmd = [sys.executable, "-m", "archivebox", "server"]
+        if debug:
+            daemon_cmd.append("--debug")
+        if reload:
+            daemon_cmd.append("--reload")
+        if nothreading:
+            daemon_cmd.append("--nothreading")
+        daemon_cmd.extend(runserver_args)
+        with log_path.open("a", encoding="utf-8") as log_file:
+            proc = subprocess.Popen(
+                daemon_cmd,
+                cwd=os.getcwd(),
+                env=daemon_env,
+                stdin=subprocess.DEVNULL,
+                stdout=log_file,
+                stderr=log_file,
+                start_new_session=True,
+            )
+        deadline = time.monotonic() + 30
+        while time.monotonic() < deadline:
+            if proc.poll() is not None:
+                print(f"[red][X] ArchiveBox daemon server exited early with code {proc.returncode}. See {log_path}[/red]")
+                sys.exit(proc.returncode or 1)
+            try:
+                with socket.create_connection((host, int(port)), timeout=0.25):
+                    break
+            except OSError:
+                time.sleep(0.1)
        else:
-            if "." in host_and_port:
-                host = host_and_port
-            else:
-                port = host_and_port
-    except IndexError:
-        pass
+            print(f"[yellow][!] ArchiveBox daemon server pid={proc.pid} is still starting. See {log_path}[/yellow]")
+        return
+
+    os.environ["BIND_ADDR"] = f"{host}:{port}"
+    from archivebox.core.routes_util import get_base_url
+
+    base_url = get_base_url().rstrip("/")
+    admin_url = f"{base_url}/admin/"

    from archivebox.workers.supervisord_util import (
-        get_existing_supervisord_process,
-        get_worker,
-        stop_worker,
+        active_supervisord_runtime_components,
+        format_runtime_components,
        start_server_workers,
+        stop_existing_supervisord_process,
        is_port_in_use,
    )
-    from archivebox.machine.models import Machine, Process
-
-    machine = Machine.current()
-    supervisor = get_existing_supervisord_process()
-    stop_existing_background_runner(
-        machine=machine,
-        process_model=Process,
-        supervisor=supervisor,
-        stop_worker_fn=stop_worker,
+    from archivebox.machine.models import Process
+    from archivebox.core.takeover_util import (
+        command_owns_runtime_stack,
+        current_command,
+        foreground_runner_owner,
+        runtime_stack_owner,
+        standby_until_runtime_stack_needed,
    )
-    if supervisor:
-        stop_existing_server_workers(
-            supervisor=supervisor,
-            stop_worker_fn=stop_worker,
-            host=host,
-            port=port,
-        )
-
-    # Check if port is already in use
-    if is_port_in_use(host, int(port)):
-        print(f"[red][X] Error: Port {port} is already in use[/red]")
-        print(f"    Another process (possibly daphne or runserver) is already listening on {host}:{port}")
-        print("    Stop the conflicting process or choose a different port")
-        sys.exit(1)
-
-    supervisor = get_existing_supervisord_process()
-    if supervisor:
-        server_worker_name = "worker_runserver" if run_in_debug else "worker_daphne"
-        server_proc = get_worker(supervisor, server_worker_name)
-        server_state = server_proc.get("statename") if isinstance(server_proc, dict) else None
-        if server_state == "RUNNING":
-            runner_proc = get_worker(supervisor, "worker_runner")
-            runner_watch_proc = get_worker(supervisor, "worker_runner_watch")
-            runner_state = runner_proc.get("statename") if isinstance(runner_proc, dict) else None
-            runner_watch_state = runner_watch_proc.get("statename") if isinstance(runner_watch_proc, dict) else None
-            print("[red][X] Error: ArchiveBox server is already running[/red]")
-            print(
-                f"    [green]√[/green] Web server ({server_worker_name}) is RUNNING on [deep_sky_blue4][link=http://{host}:{port}]http://{host}:{port}[/link][/deep_sky_blue4]",
-            )
-            if runner_state == "RUNNING":
-                print("    [green]√[/green] Background runner (worker_runner) is RUNNING")
-            if runner_watch_state == "RUNNING":
-                print("    [green]√[/green] Reload watcher (worker_runner_watch) is RUNNING")
-            print()
-            print("[yellow]To stop the existing server, run:[/yellow]")
-            print('    pkill -f "archivebox server"')
-            print("    pkill -f supervisord")
-            sys.exit(1)
+    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals

    if run_in_debug:
        print("[green][+] Starting ArchiveBox webserver in DEBUG mode...[/green]")
    else:
        print("[green][+] Starting ArchiveBox webserver...[/green]")
    print(
-        f"    [blink][green]>[/green][/blink] Starting ArchiveBox webserver on [deep_sky_blue4][link=http://{host}:{port}]http://{host}:{port}[/link][/deep_sky_blue4]",
+        f"    [blink][green]>[/green][/blink] Starting ArchiveBox webserver on [dim]BIND_ADDR[/dim] [deep_sky_blue4][link=http://{host}:{port}]http://{host}:{port}[/link][/deep_sky_blue4]",
    )
    print(
-        f"    [green]>[/green] Log in to ArchiveBox Admin UI on [deep_sky_blue3][link=http://{host}:{port}/admin]http://{host}:{port}/admin[/link][/deep_sky_blue3]",
+        f"    [green]>[/green] Log in to ArchiveBox Admin UI on [dim]BASE_URL [/dim] [deep_sky_blue3][link={admin_url}]{admin_url}[/link][/deep_sky_blue3]",
    )
    print("    > Writing ArchiveBox error log to ./logs/errors.log")
    print()
-    start_server_workers(host=host, port=port, daemonize=daemonize, debug=run_in_debug, reload=reload, nothreading=nothreading)
+
+    # Reload config after we've set os.environ["BIND_ADDR"] above so the
+    # security-mode + base-url warnings see the effective values.
+    runtime_config = get_config()
+    _print_server_startup_warnings(runtime_config, host, port)
+    bind_url = f"http://{host}:{port}"
+    command = current_command(Process.TypeChoices.SERVER, data_dir=CONSTANTS.DATA_DIR, url=bind_url)
+
+    def still_owns_runtime_stack() -> bool:
+        from django.db import connections
+
+        try:
+            return command_owns_runtime_stack(command, data_dir=CONSTANTS.DATA_DIR)
+        finally:
+            connections.close_all()
+
+    shutdown_state = None
+    try:
+        with (
+            foreground_shutdown_signals() as shutdown_state,
+            foreground_parent_watchdog(enabled=os.environ.get("ARCHIVEBOX_SERVER_DAEMON_CHILD") != "1"),
+        ):
+            while True:
+                standby_result = standby_until_runtime_stack_needed(command, data_dir=CONSTANTS.DATA_DIR)
+                older_owner = runtime_stack_owner(data_dir=CONSTANTS.DATA_DIR, exclude_id=command.id) or foreground_runner_owner(
+                    data_dir=CONSTANTS.DATA_DIR,
+                    exclude_id=command.id,
+                )
+                takeover_components = active_supervisord_runtime_components(config=config)
+                if older_owner and takeover_components:
+                    print(
+                        "[yellow][*] Taking over "
+                        f"{format_runtime_components(takeover_components)} from older existing archivebox process (pid={older_owner.pid}).[/yellow]",
+                    )
+                stop_existing_supervisord_process()
+                if is_port_in_use(host, int(port)):
+                    print(f"[red][X] Error: Port {port} is already in use[/red]")
+                    print(f"    Another process outside this ArchiveBox runtime is listening on {host}:{port}")
+                    sys.exit(1)
+
+                result = start_server_workers(
+                    host=host,
+                    port=port,
+                    daemonize=False,
+                    debug=run_in_debug,
+                    reload=reload,
+                    nothreading=nothreading,
+                    keep_running=still_owns_runtime_stack,
+                    should_stop_supervisord=still_owns_runtime_stack,
+                    resumed_from_pid=standby_result.get("previous_owner_pid") if standby_result.get("resumed") else None,
+                )
+                if result == "interrupted":
+                    break
+                if not still_owns_runtime_stack():
+                    continue
+                if result == "exited":
+                    print("[yellow][*] Runtime stack exited while this parent is still leader; restarting...[/yellow]")
+                    continue
+                break
+    except KeyboardInterrupt:
+        pass
+    finally:
+        if not shutdown_state or not shutdown_state.signal_name:
+            command.mark_exited()
    print("\n[i][green][🟩] ArchiveBox server shut down gracefully.[/green][/i]")


@@ -224,7 +403,6 @@ def server(
@click.option("--reload", is_flag=True, help="Enable auto-reloading when code or templates change")
@click.option("--debug", is_flag=True, help="Enable DEBUG=True mode with more verbose errors")
@click.option("--nothreading", is_flag=True, help="Force runserver to run in single-threaded mode")
-@click.option("--init", is_flag=True, help="Run a full archivebox init/upgrade before starting the server")
@click.option("--daemonize", is_flag=True, help="Run the server in the background as a daemon")
@docstring(server.__doc__)
 def main(**kwargs):
@@ -12,9 +12,10 @@ from archivebox.misc.util import docstring
 def shell(args: Iterable[str] = ()) -> None:
    """Enter an interactive ArchiveBox Django shell"""

-    from django.core.management import call_command
+    from django.core.management import call_command, get_commands

-    call_command("shell_plus", *args)
+    shell_command = "shell_plus" if "shell_plus" in get_commands() else "shell"
+    call_command(shell_command, *args)


@click.command(add_help_option=False, context_settings=dict(ignore_unknown_options=True))
@@ -35,10 +35,10 @@ from collections.abc import Iterable

 import rich_click as click
 from rich import print as rprint
-from django.db.models import Q, Sum
-from django.db.models.functions import Coalesce
+from django.db.models import QuerySet

-from archivebox.cli.cli_utils import apply_filters
+SNAPSHOT_FILTER_TYPE_CHOICES = ("exact", "substring", "regex", "domain", "tag", "timestamp")
+SNAPSHOT_LIST_CHUNK_SIZE = 100


 # =============================================================================
@@ -67,6 +67,7 @@ def create_snapshots(
        TYPE_SNAPSHOT,
        TYPE_CRAWL,
    )
+    from archivebox.misc.util import validate_url_length
    from archivebox.base_models.models import get_or_create_system_user_pk
    from archivebox.core.models import Snapshot
    from archivebox.crawls.models import Crawl
@@ -110,6 +111,11 @@ def create_snapshots(

                # Create snapshots for each URL in the crawl
                for url in crawl.get_urls_list():
+                    try:
+                        validate_url_length(url)
+                    except ValueError as err:
+                        rprint(f"[red]Error creating snapshot: {err}[/red]", file=sys.stderr)
+                        continue
                    merged_tags = crawl.tags_str
                    if tag:
                        merged_tags = f"{merged_tags},{tag}" if merged_tags else tag
@@ -128,6 +134,8 @@ def create_snapshots(

            elif record_type == TYPE_SNAPSHOT or record.get("url"):
                # Input is a Snapshot or plain URL
+                if record.get("url"):
+                    validate_url_length(str(record["url"]))
                if tag and not record.get("tags"):
                    record["tags"] = tag
                if status:
@@ -171,18 +179,63 @@ def create_snapshots(
 # =============================================================================


+def snapshot_filter_options(*, default_filter_type: str):
+    def decorate(func):
+        for decorator in reversed(
+            (
+                click.option("--status", "-s", help="Filter by status (queued, started, sealed)"),
+                click.option("--url__icontains", help="Filter by URL contains"),
+                click.option("--url__istartswith", help="Filter by URL starts with"),
+                click.option("--tag", "-t", help="Filter by tag name"),
+                click.option("--crawl-id", help="Filter by crawl ID"),
+                click.option("--limit", "-n", type=int, help="Limit number of results"),
+                click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at"),
+                click.option("--search", help="Search mode to use for positional query"),
+                click.option("--before", type=float, help="Only snapshots bookmarked before timestamp"),
+                click.option("--after", type=float, help="Only snapshots bookmarked after timestamp"),
+                click.option(
+                    "--filter-type",
+                    "-f",
+                    type=click.Choice(SNAPSHOT_FILTER_TYPE_CHOICES),
+                    default=default_filter_type,
+                    help="Type of pattern matching to use for positional filters",
+                ),
+                click.argument("filter_patterns", nargs=-1),
+            ),
+        ):
+            func = decorator(func)
+        return func
+
+    return decorate
+
+
+def snapshot_output_options(func):
+    for decorator in reversed(
+        (
+            click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title"),
+            click.option("--json", "as_json", is_flag=True, help="Print output as a JSON array"),
+            click.option("--html", "as_html", is_flag=True, help="Print output as HTML"),
+            click.option("--with-headers", is_flag=True, help="Include column headers in structured output"),
+        ),
+    ):
+        func = decorator(func)
+    return func
+
+
+def build_snapshot_queryset(
+    **kwargs,
+) -> QuerySet:
+    from archivebox.core.models import Snapshot
+
+    return Snapshot.objects.order_by("-created_at").search(**kwargs)
+
+
 def list_snapshots(
-    status: str | None = None,
-    url__icontains: str | None = None,
-    url__istartswith: str | None = None,
-    tag: str | None = None,
-    crawl_id: str | None = None,
-    limit: int | None = None,
-    sort: str | None = None,
    csv: str | None = None,
+    as_json: bool = False,
+    as_html: bool = False,
    with_headers: bool = False,
-    search: str | None = None,
-    query: str | None = None,
+    **kwargs,
 ) -> int:
    """
    List Snapshots as JSONL with optional filters.
@@ -191,104 +244,71 @@ def list_snapshots(
        0: Success (even if no results)
    """
    from archivebox.misc.jsonl import write_record
-    from archivebox.core.models import Snapshot
-    from archivebox.search import (
-        get_default_search_mode,
-        get_search_mode,
-        prioritize_metadata_matches,
-        query_search_index,
-    )

-    if with_headers and not csv:
-        rprint("[red]--with-headers requires --csv[/red]", file=sys.stderr)
+    output_formats = sum(bool(output_format) for output_format in (csv, as_json, as_html))
+    if output_formats > 1:
+        rprint("[red]Choose only one output format: --csv, --json, or --html[/red]", file=sys.stderr)
+        return 2
+    if with_headers and not output_formats:
+        rprint("[red]--with-headers requires --csv, --json, or --html[/red]", file=sys.stderr)
        return 2

-    is_tty = sys.stdout.isatty() and not csv
+    is_tty = sys.stdout.isatty() and not output_formats

-    queryset = Snapshot.objects.annotate(output_size_sum=Coalesce(Sum("archiveresult__output_size"), 0)).order_by("-created_at")
-
-    # Apply filters
-    filter_kwargs = {
-        "status": status,
-        "url__icontains": url__icontains,
-        "url__istartswith": url__istartswith,
-        "crawl_id": crawl_id,
-    }
-    queryset = apply_filters(queryset, filter_kwargs)
-
-    # Tag filter requires special handling (M2M)
-    if tag:
-        queryset = queryset.filter(tags__name__iexact=tag)
-
-    query = (query or "").strip()
-    if query:
-        metadata_qs = queryset.filter(
-            Q(title__icontains=query) | Q(url__icontains=query) | Q(timestamp__icontains=query) | Q(tags__name__icontains=query),
-        )
-        requested_search_mode = (search or "").strip().lower()
-        if requested_search_mode == "content":
-            requested_search_mode = "contents"
-        search_mode = get_default_search_mode() if not requested_search_mode else get_search_mode(requested_search_mode)
-
-        if search_mode == "meta":
-            queryset = metadata_qs
-        else:
-            try:
-                deep_qsearch = None
-                if search_mode == "deep":
-                    qsearch = query_search_index(query, search_mode="contents")
-                    deep_qsearch = query_search_index(query, search_mode="deep")
-                else:
-                    qsearch = query_search_index(query, search_mode=search_mode)
-                queryset = prioritize_metadata_matches(
-                    queryset,
-                    metadata_qs,
-                    qsearch,
-                    deep_queryset=deep_qsearch,
-                    ordering=("-created_at",) if not sort else None,
-                )
-            except Exception as err:
-                rprint(
-                    f"[yellow]Search backend error, falling back to metadata search: {err}[/yellow]",
-                    file=sys.stderr,
-                )
-                queryset = metadata_qs
-
-    if sort:
-        queryset = queryset.order_by(sort)
-    if limit:
-        queryset = queryset[:limit]
+    try:
+        queryset = build_snapshot_queryset(**kwargs)
+    except ValueError as err:
+        rprint(f"[red]{err}[/red]", file=sys.stderr)
+        return 2

    count = 0
+    if as_json:
+        queryset = queryset.prefetch_related("tags")
+        output = queryset.to_json(with_headers=with_headers)
+        sys.stdout.write(output)
+        if output and not output.endswith("\n"):
+            sys.stdout.write("\n")
+        rprint(f"[dim]Listed {queryset.count()} snapshots[/dim]", file=sys.stderr)
+        return 0
+
+    if as_html:
+        queryset = queryset.prefetch_related("tags")
+        output = queryset.to_html(with_headers=with_headers)
+        sys.stdout.write(output)
+        if output and not output.endswith("\n"):
+            sys.stdout.write("\n")
+        rprint(f"[dim]Listed {queryset.count()} snapshots[/dim]", file=sys.stderr)
+        return 0
+
    if csv:
        cols = [col.strip() for col in csv.split(",") if col.strip()]
        if not cols:
            rprint("[red]No CSV columns provided[/red]", file=sys.stderr)
            return 2
-        rows: list[str] = []
        if with_headers:
-            rows.append(",".join(cols))
-        for snapshot in queryset.iterator(chunk_size=500):
-            rows.append(snapshot.to_csv(cols=cols, separator=","))
+            sys.stdout.write(",".join(cols))
+            sys.stdout.write("\n")
+        for snapshot in queryset.prefetch_related("tags").iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE):
+            sys.stdout.write(snapshot.to_csv(cols=cols, separator=","))
+            sys.stdout.write("\n")
            count += 1
-        output = "\n".join(rows)
-        if output:
-            sys.stdout.write(output)
-            if not output.endswith("\n"):
-                sys.stdout.write("\n")
        rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
        return 0

-    for snapshot in queryset:
-        if is_tty:
-            status_color = {
-                "queued": "yellow",
-                "started": "blue",
-                "sealed": "green",
-            }.get(snapshot.status, "dim")
-            rprint(f"[{status_color}]{snapshot.status:8}[/{status_color}] [dim]{snapshot.id}[/dim] {snapshot.url[:60]}")
-        else:
+    if not is_tty:
+        for snapshot in queryset.prefetch_related("tags").iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE):
            write_record(snapshot.to_json())
+            count += 1
+        rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
+        return 0
+
+    for snapshot in queryset.iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE):
+        status_color = {
+            "queued": "yellow",
+            "started": "blue",
+            "sealed": "green",
+        }.get(snapshot.status, "dim")
+        rprint(f"[{status_color}]{snapshot.status:8}[/{status_color}] [dim]{snapshot.id}[/dim] {snapshot.url[:60]}")
        count += 1

    rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
@@ -335,22 +355,34 @@ def update_snapshots(
        try:
            snapshot = Snapshot.objects.get(id=snapshot_id)

-            # Apply updates from CLI flags (override stdin values)
            if status:
-                snapshot.status = status
-                snapshot.retry_at = timezone.now()
+                if status not in Snapshot.StatusChoices.values:
+                    rprint(f"[red]Invalid snapshot status: {status}[/red]", file=sys.stderr)
+                    continue
+                if status == Snapshot.StatusChoices.SEALED:
+                    snapshot.cancel()
+                elif status == Snapshot.StatusChoices.PAUSED:
+                    snapshot.pause()
+                elif status == Snapshot.StatusChoices.QUEUED:
+                    if snapshot.status == Snapshot.StatusChoices.PAUSED:
+                        snapshot.resume()
+                    else:
+                        snapshot.update_and_requeue(status=Snapshot.StatusChoices.QUEUED, retry_at=timezone.now())
+                elif status == Snapshot.StatusChoices.STARTED:
+                    snapshot.update_and_requeue(status=Snapshot.StatusChoices.STARTED, retry_at=timezone.now())
            if tag:
-                # Add tag to existing tags
-                snapshot.save()  # Ensure saved before M2M
                from archivebox.core.models import Tag

                tag_obj, _ = Tag.objects.get_or_create(name=tag)
                snapshot.tags.add(tag_obj)
+                snapshot.safe_update({"modified_at": timezone.now()}, refresh=False)

-            snapshot.save()
+            if not status and not tag:
+                snapshot.safe_update({"modified_at": timezone.now()}, refresh=False)
            updated_count += 1

            if not is_tty:
+                snapshot.refresh_from_db()
                write_record(snapshot.to_json())

        except Snapshot.DoesNotExist:
@@ -435,46 +467,11 @@ def create_cmd(urls: tuple, tag: str, status: str, depth: int):


@main.command("list")
-@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
-@click.option("--url__icontains", help="Filter by URL contains")
-@click.option("--url__istartswith", help="Filter by URL starts with")
-@click.option("--tag", "-t", help="Filter by tag name")
-@click.option("--crawl-id", help="Filter by crawl ID")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title")
-@click.option("--with-headers", is_flag=True, help="Include column headers in structured output")
-@click.option("--search", type=click.Choice(["meta", "content", "contents", "deep"]), help="Search mode to use for the query")
-@click.argument("query", nargs=-1)
-def list_cmd(
-    status: str | None,
-    url__icontains: str | None,
-    url__istartswith: str | None,
-    tag: str | None,
-    crawl_id: str | None,
-    limit: int | None,
-    sort: str | None,
-    csv: str | None,
-    with_headers: bool,
-    search: str | None,
-    query: tuple[str, ...],
-):
+@snapshot_output_options
+@snapshot_filter_options(default_filter_type="substring")
+def list_cmd(**kwargs):
    """List Snapshots as JSONL."""
-    sys.exit(
-        list_snapshots(
-            status=status,
-            url__icontains=url__icontains,
-            url__istartswith=url__istartswith,
-            tag=tag,
-            crawl_id=crawl_id,
-            limit=limit,
-            sort=sort,
-            csv=csv,
-            with_headers=with_headers,
-            search=search,
-            query=" ".join(query),
-        ),
-    )
+    sys.exit(list_snapshots(**kwargs))


@main.command("update")
@@ -1,24 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox snapshot"
-
-import sys
-
-import rich_click as click
-
-from archivebox.cli.archivebox_snapshot import create_snapshots
-
-
-@click.command(context_settings={"ignore_unknown_options": True})
-@click.option("--tag", "-t", default="", help="Comma-separated tags to add")
-@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
-@click.option("--depth", "-d", type=int, default=0, help="Crawl depth (default: 0)")
-@click.argument("urls", nargs=-1)
-def main(tag: str, status: str, depth: int, urls: tuple[str, ...]):
-    """Backwards-compatible `archivebox snapshot URL...` entrypoint."""
-    sys.exit(create_snapshots(urls, tag=tag, status=status, depth=depth))
-
-
-if __name__ == "__main__":
-    main()
@@ -8,22 +8,25 @@ import rich_click as click
 from rich import print

 from archivebox.misc.util import enforce_types, docstring
-from archivebox.config import DATA_DIR, CONSTANTS, ARCHIVE_DIR
-from archivebox.config.common import SHELL_CONFIG
-from archivebox.misc.legacy import parse_json_links_details
+from archivebox.config import CONSTANTS
+from archivebox.config.common import get_config
 from archivebox.misc.system import get_dir_size
 from archivebox.misc.logging_util import printable_filesize


+MAX_STATUS_FS_DIR_SCAN = 5000
+
+
@enforce_types
-def status(out_dir: Path = DATA_DIR) -> None:
+def status(out_dir: Path = CONSTANTS.DATA_DIR) -> None:
    """Print out some info and statistics about the archive collection"""

    from django.contrib.auth import get_user_model
    from django.db.models import Sum
    from django.db.models.functions import Coalesce
-    from archivebox.core.models import Snapshot
+    from archivebox.core.models import ArchiveResult, Snapshot

+    config = get_config()
    User = get_user_model()

    print("[green]\\[*] Scanning archive main index...[/green]")
@@ -33,47 +36,66 @@ def status(out_dir: Path = DATA_DIR) -> None:
    print(f"    Index size: {size} across {num_files} files")
    print()

-    links = list(Snapshot.objects.annotate(output_size_sum=Coalesce(Sum("archiveresult__output_size"), 0)))
-    num_sql_links = len(links)
-    num_link_details = sum(1 for link in parse_json_links_details(out_dir=out_dir))
+    snapshots_qs = Snapshot.objects.all()
+    num_sql_links = snapshots_qs.count()
+    archive_dir = CONSTANTS.ARCHIVE_DIR
+    legacy_snapshot_dirs = []
+    if archive_dir.exists():
+        legacy_snapshot_dirs = [
+            entry for entry in archive_dir.iterdir() if entry.is_dir() and not entry.is_symlink() and Snapshot.is_legacy_archive_dir(entry)
+        ]
    print(f"    > SQL Main Index: {num_sql_links} links".ljust(36), f"(found in {CONSTANTS.SQL_INDEX_FILENAME})")
-    print(f"    > JSON Link Details: {num_link_details} links".ljust(36), f"(found in {ARCHIVE_DIR.name}/*/index.json)")
+    print(f"    > JSON Link Details: {len(legacy_snapshot_dirs)} links".ljust(36), f"(found in {archive_dir.name}/*/index.json)")
    print()
    print("[green]\\[*] Scanning archive data directories...[/green]")
-    users_dir = out_dir / "users"
-    scan_roots = [root for root in (ARCHIVE_DIR, users_dir) if root.exists()]
-    scan_roots_display = ", ".join(str(root) for root in scan_roots) if scan_roots else str(ARCHIVE_DIR)
+    users_dir = CONSTANTS.USERS_DIR
+    scan_roots = [root for root in (archive_dir, users_dir) if root.exists()]
+    scan_roots_display = ", ".join(str(root) for root in scan_roots) if scan_roots else str(archive_dir)
    print(f"[yellow]   {scan_roots_display}[/yellow]")
-    num_bytes = num_dirs = num_files = 0
-    for root in scan_roots:
-        root_bytes, root_dirs, root_files = get_dir_size(root)
-        num_bytes += root_bytes
-        num_dirs += root_dirs
-        num_files += root_files
+    do_precise_fs_scan = num_sql_links <= MAX_STATUS_FS_DIR_SCAN
+    if do_precise_fs_scan:
+        num_bytes = num_dirs = num_files = 0
+        for root in scan_roots:
+            root_bytes, root_dirs, root_files = get_dir_size(root)
+            num_bytes += root_bytes
+            num_dirs += root_dirs
+            num_files += root_files
+    else:
+        num_bytes = snapshots_qs.aggregate(total=Coalesce(Sum("output_size"), 0))["total"] or 0
+        num_dirs = 0
+        num_files = ArchiveResult.objects.exclude(output_files__in=["", "{}"]).count()
    size = printable_filesize(num_bytes)
-    print(f"    Size: {size} across {num_files} files in {num_dirs} directories")
+    if do_precise_fs_scan:
+        print(f"    Size: {size} across {num_files} files in {num_dirs} directories")
+    else:
+        print(f"    Size: {size} across {num_files} DB-tracked output records")

    # Use DB as source of truth for snapshot status
-    num_indexed = len(links)
-    num_archived = sum(1 for snapshot in links if snapshot.is_archived)
+    num_indexed = num_sql_links
+    num_archived = snapshots_qs.filter(status=Snapshot.StatusChoices.SEALED).count()
    num_unarchived = max(num_indexed - num_archived, 0)
    print(f"    > indexed: {num_indexed}".ljust(36), "(total snapshots in DB)")
    print(f"      > archived: {num_archived}".ljust(36), "(snapshots with archived content)")
    print(f"      > unarchived: {num_unarchived}".ljust(36), "(snapshots pending archiving)")

    # Count snapshot directories on filesystem across both legacy and current layouts.
-    expected_snapshot_dirs = {str(Path(snapshot.output_dir).resolve()) for snapshot in links if Path(snapshot.output_dir).exists()}
-    discovered_snapshot_dirs = set()
+    if do_precise_fs_scan:
+        links = list(snapshots_qs)
+        expected_snapshot_dirs = {str(Path(snapshot.output_dir).resolve()) for snapshot in links if Path(snapshot.output_dir).exists()}
+        discovered_snapshot_dirs = {str(entry.resolve()) for entry in legacy_snapshot_dirs}

-    if ARCHIVE_DIR.exists():
-        discovered_snapshot_dirs.update(str(entry.resolve()) for entry in ARCHIVE_DIR.iterdir() if entry.is_dir())
+        if users_dir.exists():
+            discovered_snapshot_dirs.update(
+                str(entry.resolve()) for entry in users_dir.glob(f"*/{CONSTANTS.SNAPSHOTS_DIR_NAME}/*/*/*") if entry.is_dir()
+            )

-    if users_dir.exists():
-        discovered_snapshot_dirs.update(str(entry.resolve()) for entry in users_dir.glob("*/snapshots/*/*/*") if entry.is_dir())
-
-    orphaned_dirs = sorted(discovered_snapshot_dirs - expected_snapshot_dirs)
-    num_present = len(discovered_snapshot_dirs)
-    num_valid = len(discovered_snapshot_dirs & expected_snapshot_dirs)
+        orphaned_dirs = sorted(discovered_snapshot_dirs - expected_snapshot_dirs)
+        num_present = len(discovered_snapshot_dirs)
+        num_valid = len(discovered_snapshot_dirs & expected_snapshot_dirs)
+    else:
+        orphaned_dirs = []
+        num_present = num_archived
+        num_valid = num_archived
    print()
    print(f"    > present: {num_present}".ljust(36), "(snapshot directories on disk)")
    print(f"      > [green]valid:[/green] {num_valid}".ljust(36), "               (directories with matching DB entry)")
@@ -83,7 +105,7 @@ def status(out_dir: Path = DATA_DIR) -> None:

    if num_indexed:
        print("    [violet]Hint:[/violet] You can list snapshots by status like so:")
-        print("        [green]archivebox list --status=<status>  (e.g. archived, queued, etc.)[/green]")
+        print("        [green]archivebox list --status=<status>  (e.g. sealed, queued, etc.)[/green]")

    if orphaned_dirs:
        print("    [violet]Hint:[/violet] To automatically import orphaned data directories into the main index, run:")
@@ -108,10 +130,9 @@ def status(out_dir: Path = DATA_DIR) -> None:
        print("        [green]archivebox manage createsuperuser[/green]")

    print()
-    recent_snapshots = sorted(
-        links,
-        key=lambda snapshot: snapshot.downloaded_at or snapshot.modified_at or snapshot.created_at,
-        reverse=True,
+    recent_snapshots = snapshots_qs.order_by(
+        "-downloaded_at",
+        "-modified_at",
    )[:10]
    for snapshot in recent_snapshots:
        if not snapshot.downloaded_at:
@@ -120,10 +141,10 @@ def status(out_dir: Path = DATA_DIR) -> None:
            (
                "[grey53] "
                f"   > {str(snapshot.downloaded_at)[:16]} "
-                f"[{snapshot.num_outputs} {('X', '√')[snapshot.is_archived]} {printable_filesize(snapshot.archive_size)}] "
+                f"[{snapshot.num_outputs} {('X', '√')[snapshot.status == Snapshot.StatusChoices.SEALED]} {printable_filesize(snapshot.output_size or 0)}] "
                f'"{snapshot.title}": {snapshot.url}'
                "[/grey53]"
-            )[: SHELL_CONFIG.TERM_WIDTH],
+            )[: config.TERM_WIDTH],
        )
    print("[grey53]   ...")

@@ -36,7 +36,7 @@ from collections.abc import Iterable
 import rich_click as click
 from rich import print as rprint

-from archivebox.cli.cli_utils import apply_filters
+from archivebox.cli.cli_util import apply_filters


 # =============================================================================
@@ -13,6 +13,71 @@ import rich_click as click
 from archivebox.misc.util import docstring, enforce_types


+def _format_binary_abspath(
+    abspath: str,
+    *,
+    pwd: Path,
+    lib_dir: Path,
+    personas_dir: Path,
+    home: Path,
+) -> str:
+    path = Path(abspath).expanduser()
+    try:
+        normalized = path.resolve(strict=False)
+    except Exception:
+        normalized = path
+
+    candidate_bases: tuple[tuple[Path, str], ...] = (
+        (pwd, "./"),
+        (lib_dir, "LIB_DIR/"),
+        (Path(os.environ.get("LIB_DIR", "")), "LIB_DIR/") if os.environ.get("LIB_DIR") else (Path(), ""),
+        (personas_dir, "PERSONAS_DIR/"),
+        (home, "~/"),
+    )
+
+    for base, prefix in candidate_bases:
+        if not prefix:
+            continue
+        for candidate in (base, base.resolve(strict=False)):
+            try:
+                relative = normalized.relative_to(candidate)
+            except ValueError:
+                continue
+
+            relative_str = relative.as_posix()
+            if prefix == "./":
+                return "." if not relative_str else f"./{relative_str}"
+            if prefix == "LIB_DIR/":
+                return "LIB_DIR" if not relative_str else f"LIB_DIR/{relative_str}"
+            if prefix == "PERSONAS_DIR/":
+                return "PERSONAS_DIR" if not relative_str else f"PERSONAS_DIR/{relative_str}"
+            return "~" if not relative_str else f"~/{relative_str}"
+
+    return normalized.as_posix()
+
+
+def _render_binary_abspath(abspath: str):
+    from rich.text import Text
+
+    if abspath.startswith("LIB_DIR/"):
+        return Text.assemble(("LIB_DIR", "bright_blue"), (abspath.removeprefix("LIB_DIR"), "green"))
+    if abspath == "LIB_DIR":
+        return Text("LIB_DIR", style="bright_blue")
+    if abspath.startswith("PERSONAS_DIR/"):
+        return Text.assemble(("PERSONAS_DIR", "medium_purple"), (abspath.removeprefix("PERSONAS_DIR"), "green"))
+    if abspath == "PERSONAS_DIR":
+        return Text("PERSONAS_DIR", style="medium_purple")
+    if abspath.startswith("~/"):
+        return Text.assemble(("~", "cyan"), (abspath.removeprefix("~"), "green"))
+    if abspath == "~":
+        return Text("~", style="cyan")
+    if abspath.startswith("./"):
+        return Text.assemble((".", "cyan"), (abspath.removeprefix("."), "green"))
+    if abspath == ".":
+        return Text(".", style="cyan")
+    return Text(abspath, style="green")
+
+
@enforce_types
 def version(
    quiet: bool = False,
@@ -30,13 +95,12 @@ def version(
    from rich.panel import Panel
    from rich.console import Console

-    from archivebox.config import CONSTANTS, DATA_DIR
+    from archivebox.config import CONSTANTS
    from archivebox.config.version import get_COMMIT_HASH, get_BUILD_TIME
    from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, RUNNING_AS_UID, RUNNING_AS_GID, IN_DOCKER
    from archivebox.config.paths import get_data_locations, get_code_locations
-    from archivebox.config.common import SHELL_CONFIG, STORAGE_CONFIG, SEARCH_BACKEND_CONFIG
    from archivebox.misc.logging_util import printable_folder_status
-    from archivebox.config.configset import get_config
+    from archivebox.config.common import get_config

    console = Console()
    prnt = console.print
@@ -54,7 +118,7 @@ def version(
    )
    prnt(
        f"IN_DOCKER={IN_DOCKER}",
-        f"IN_QEMU={SHELL_CONFIG.IN_QEMU}",
+        f"IN_QEMU={config.IN_QEMU}",
        f"ARCH={p.machine}",
        f"OS={p.system}",
        f"PLATFORM={platform.platform()}",
@@ -71,8 +135,8 @@ def version(
        prnt(
            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} PUID={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
            f"FS_UID={DATA_DIR_STAT.st_uid}:{DATA_DIR_STAT.st_gid}",
-            f"FS_PERMS={STORAGE_CONFIG.OUTPUT_PERMISSIONS}",
-            f"FS_ATOMIC={STORAGE_CONFIG.ENFORCE_ATOMIC_WRITES}",
+            f"FS_PERMS={config.OUTPUT_PERMISSIONS}",
+            f"FS_ATOMIC={config.ENFORCE_ATOMIC_WRITES}",
            f"FS_REMOTE={OUTPUT_IS_REMOTE_FS}",
        )
    except Exception:
@@ -81,11 +145,11 @@ def version(
        )

    prnt(
-        f"DEBUG={SHELL_CONFIG.DEBUG}",
-        f"IS_TTY={SHELL_CONFIG.IS_TTY}",
+        f"DEBUG={config.DEBUG}",
+        f"IS_TTY={config.IS_TTY}",
        f"SUDO={CONSTANTS.IS_ROOT}",
        f"ID={CONSTANTS.MACHINE_ID}:{CONSTANTS.COLLECTION_ID}",
-        f"SEARCH_BACKEND={SEARCH_BACKEND_CONFIG.SEARCH_BACKEND_ENGINE}",
+        f"SEARCH_BACKEND={config.SEARCH_BACKEND_ENGINE}",
        f"LDAP={LDAP_ENABLED}",
    )
    prnt()
@@ -126,42 +190,65 @@ def version(

        machine = Machine.current()

-        # Get all binaries from the database with timeout protection
-        all_installed = (
-            Binary.objects.filter(
-                machine=machine,
-            )
-            .exclude(abspath="")
-            .exclude(abspath__isnull=True)
-            .order_by("name")
-        )
+        if isinstance(binaries, str):
+            requested_names = {name.strip() for name in binaries.split(",") if name.strip()}
+        else:
+            requested_names = {name for name in (binaries or ()) if name}

-        if not all_installed.exists():
+        db_binaries: dict[str, Binary] = {}
+        for binary in Binary.objects.filter(machine=machine).order_by("name", "-modified_at"):
+            db_binaries.setdefault(binary.name, binary)
+
+        all_binary_names = sorted(requested_names or set(db_binaries.keys()))
+
+        if not all_binary_names:
            prnt("", "[grey53]No binaries detected. Run [green]archivebox install[/green] to detect dependencies.[/grey53]")
        else:
-            for installed in all_installed:
-                # Skip if user specified specific binaries and this isn't one
-                if binaries and installed.name not in binaries:
+            any_available = False
+            compact_paths = console.is_terminal
+            for name in all_binary_names:
+                if requested_names and name not in requested_names:
                    continue

-                if installed.is_valid:
-                    display_path = installed.abspath.replace(str(DATA_DIR), ".").replace(str(Path("~").expanduser()), "~")
+                installed = db_binaries.get(name)
+                if installed and installed.is_valid:
+                    display_name = Path(name).expanduser().name if ("/" in name or name.startswith("~")) else name
+                    display_path = (
+                        _format_binary_abspath(
+                            installed.abspath,
+                            pwd=Path.cwd(),
+                            lib_dir=config.LIB_DIR,
+                            personas_dir=CONSTANTS.PERSONAS_DIR,
+                            home=Path.home(),
+                        )
+                        if compact_paths
+                        else installed.abspath
+                    )
+                    rendered_path = _render_binary_abspath(display_path) if compact_paths else display_path
                    version_str = (installed.version or "unknown")[:15]
                    provider = (installed.binprovider or "env")[:8]
                    prnt(
                        "",
                        "[green]√[/green]",
                        "",
-                        installed.name.ljust(18),
+                        display_name.ljust(18),
                        version_str.ljust(16),
                        provider.ljust(8),
-                        display_path,
+                        rendered_path,
                        overflow="ignore",
                        crop=False,
                    )
-                else:
-                    prnt("", "[red]X[/red]", "", installed.name.ljust(18), "[grey53]not installed[/grey53]", overflow="ignore", crop=False)
-                    failures.append(installed.name)
+                    any_available = True
+                    continue
+
+                status = (
+                    "[grey53]not recorded[/grey53]" if name in requested_names and installed is None else "[grey53]not installed[/grey53]"
+                )
+                prnt("", "[red]X[/red]", "", name.ljust(18), status, overflow="ignore", crop=False)
+                failures.append(name)
+
+            if not any_available:
+                prnt("", "[grey53]No binaries detected. Run [green]archivebox install[/green] to detect dependencies.[/grey53]")

        # Show hint if no binaries are installed yet
        has_any_installed = Binary.objects.filter(machine=machine).exclude(abspath="").exists()
@@ -1,105 +1,23 @@
-"""
-ArchiveBox config exports.
-
-This module provides backwards-compatible config exports for extractors
-and other modules that expect to import config values directly.
-"""
+"""Minimal import-time config exports."""

 __package__ = "archivebox.config"
 __order__ = 200

-from .paths import (
-    PACKAGE_DIR,
-    DATA_DIR,
-    ARCHIVE_DIR,
-)
-from .constants import CONSTANTS, CONSTANTS_CONFIG, PACKAGE_DIR, DATA_DIR, ARCHIVE_DIR  # noqa
-from .version import VERSION  # noqa
-
-
-###############################################################################
-# Config value exports for extractors
-# These provide backwards compatibility with extractors that import from ..config
-###############################################################################
-
-
-def _get_config():
-    """Lazy import to avoid circular imports."""
-    from .common import ARCHIVING_CONFIG, STORAGE_CONFIG
-
-    return ARCHIVING_CONFIG, STORAGE_CONFIG
-
-
-# Direct exports (evaluated at import time for backwards compat)
-# These are recalculated each time the module attribute is accessed
-

 def __getattr__(name: str):
-    """
-    Module-level __getattr__ for lazy config loading.
+    if name in ("CONSTANTS", "CONSTANTS_CONFIG"):
+        from .constants import CONSTANTS, CONSTANTS_CONFIG

-    Only provides backwards compatibility for GENERIC/SHARED config.
-    Plugin-specific config (binaries, args, toggles) should come from plugin config.json files.
-    """
+        return {"CONSTANTS": CONSTANTS, "CONSTANTS_CONFIG": CONSTANTS_CONFIG}[name]
+    if name in ("PACKAGE_DIR", "DATA_DIR"):
+        from .paths import PACKAGE_DIR, DATA_DIR

-    # Generic timeout settings (used by multiple plugins)
-    if name == "TIMEOUT":
-        cfg, _ = _get_config()
-        return cfg.TIMEOUT
+        return {"PACKAGE_DIR": PACKAGE_DIR, "DATA_DIR": DATA_DIR}[name]
+    if name == "VERSION":
+        from .version import VERSION

-    # Generic SSL/Security settings (used by multiple plugins)
-    if name == "CHECK_SSL_VALIDITY":
-        cfg, _ = _get_config()
-        return cfg.CHECK_SSL_VALIDITY
-
-    # Generic storage settings (used by multiple plugins)
-    if name == "RESTRICT_FILE_NAMES":
-        _, storage = _get_config()
-        return storage.RESTRICT_FILE_NAMES
-
-    # Generic user agent / cookies (used by multiple plugins)
-    if name == "COOKIES_FILE":
-        cfg, _ = _get_config()
-        return cfg.COOKIES_FILE
-    if name == "USER_AGENT":
-        cfg, _ = _get_config()
-        return cfg.USER_AGENT
-
-    # Generic resolution settings (used by multiple plugins)
-    if name == "RESOLUTION":
-        cfg, _ = _get_config()
-        return cfg.RESOLUTION
-
-    # Allowlist/Denylist patterns (compiled regexes)
-    if name == "SAVE_ALLOWLIST_PTN":
-        cfg, _ = _get_config()
-        return cfg.SAVE_ALLOWLIST_PTNS
-    if name == "SAVE_DENYLIST_PTN":
-        cfg, _ = _get_config()
-        return cfg.SAVE_DENYLIST_PTNS
-
-    raise AttributeError(f"module 'archivebox.config' has no attribute '{name}'")
+        return VERSION
+    raise AttributeError(name)


-# Re-export common config classes for direct imports
-def get_CONFIG():
-    """Get all config sections as a dict."""
-    from .common import (
-        SHELL_CONFIG,
-        STORAGE_CONFIG,
-        GENERAL_CONFIG,
-        SERVER_CONFIG,
-        ARCHIVING_CONFIG,
-        SEARCH_BACKEND_CONFIG,
-    )
-    from .ldap import LDAP_CONFIG
-
-    return {
-        "SHELL_CONFIG": SHELL_CONFIG,
-        "STORAGE_CONFIG": STORAGE_CONFIG,
-        "GENERAL_CONFIG": GENERAL_CONFIG,
-        "SERVER_CONFIG": SERVER_CONFIG,
-        "ARCHIVING_CONFIG": ARCHIVING_CONFIG,
-        "SEARCHBACKEND_CONFIG": SEARCH_BACKEND_CONFIG,
-        "LDAP_CONFIG": LDAP_CONFIG,
-    }
+__all__ = ("CONSTANTS", "CONSTANTS_CONFIG", "PACKAGE_DIR", "DATA_DIR", "VERSION")
@@ -1,195 +1,297 @@
 __package__ = "archivebox.config"

-import os
+import io
 import json
+import os
+from collections.abc import Mapping
 from typing import Any

-from pathlib import Path
-from configparser import ConfigParser
-
-from benedict import benedict
-
-
 from archivebox.config.constants import CONSTANTS
-
-from archivebox.misc.logging import stderr
+from archivebox.config.configset import CaseConfigParser
+from archivebox.misc.logging import AttrDict


-class CaseConfigParser(ConfigParser):
-    def optionxform(self, optionstr: str) -> str:
-        return optionstr
+CONFIG_FILE_HEADER = (
+    "# This is the config file for your ArchiveBox collection.\n"
+    "#\n"
+    "# You can add options here manually in INI format, or automatically by running:\n"
+    "#    archivebox config --set KEY=VALUE\n"
+    "#\n"
+    "# This file is kept in sync 1:1 with Machine.config in the index DB —\n"
+    "# editing either side propagates to the other. ``archivebox init`` reads\n"
+    "# this file on startup; the admin Machine.config editor writes both.\n"
+    "#\n"
+    "# Full reference:\n"
+    "#    https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration\n"
+    "\n"
+)


-def get_real_name(key: str) -> str:
-    """get the up-to-date canonical name for a given old alias or current key"""
-    # Config aliases are no longer used with the simplified config system
-    # Just return the key as-is since we no longer have a complex alias mapping
-    return key
+# Recursion guard for the bidirectional file<->DB mirror. Bidirectional sync
+# means a write on one side always triggers an update on the other; without
+# this flag, ``Machine.save -> mirror to file -> write_config_file -> mirror
+# back to Machine -> Machine.save -> ...`` would loop forever. Module-level
+# scalar is fine here — daphne handles each request on its own thread and the
+# admin/CLI write paths are inherently serialized through the DB.
+_MIRROR_IN_PROGRESS: bool = False
+# One-time-per-process startup sync, gated so subsequent ``Machine.current()``
+# calls collapse to a single boolean check.
+_INITIAL_SYNC_DONE: bool = False


-def load_config_val(
-    key: str,
-    default: Any = None,
-    type: type | None = None,
-    aliases: tuple[str, ...] | None = None,
-    config: benedict | None = None,
-    env_vars: os._Environ | None = None,
-    config_file_vars: dict[str, str] | None = None,
-) -> Any:
-    """parse bool, int, and str key=value pairs from env"""
+def _coerce_to_str_dict(config: Any) -> dict[str, str]:
+    """Project an arbitrary config payload to flat ``{UPPER_KEY: str}`` form.

-    assert isinstance(config, dict)
+    INI files only round-trip strings, so we normalize Machine.config values
+    to strings on the way out and accept the same shape on the way back.
+    Pydantic re-coerces types at read time inside ``get_config``.

-    is_read_only = type is None
-    if is_read_only:
-        if callable(default):
-            return default(config)
-        return default
-
-    # get value from environment variables or config files
-    config_keys_to_check = (key, *(aliases or ()))
-    val = None
-    for key in config_keys_to_check:
-        if env_vars:
-            val = env_vars.get(key)
-            if val:
-                break
-
-        if config_file_vars:
-            val = config_file_vars.get(key)
-            if val:
-                break
-
-    is_unset = val is None
-    if is_unset:
-        if callable(default):
-            return default(config)
-        return default
-
-    assert isinstance(val, str)
-
-    # calculate value based on expected type
-    BOOL_TRUEIES = ("true", "yes", "1")
-    BOOL_FALSEIES = ("false", "no", "0")
-
-    if type is bool:
-        if val.lower() in BOOL_TRUEIES:
-            return True
-        elif val.lower() in BOOL_FALSEIES:
-            return False
+    Composite values (``dict`` / ``list`` / ``tuple``) are JSON-encoded so
+    they round-trip back through pydantic-settings — ``str(some_dict)``
+    would produce Python's repr (``{'k': 'v'}`` with single quotes), and
+    pydantic-settings refuses to parse that as a ``dict`` field, causing
+    e.g. ``ABX_INSTALL_CACHE`` to crash with ``ValidationError: Input
+    should be a valid dictionary``.
+    """
+    if not config:
+        return {}
+    if not isinstance(config, Mapping):
+        return {}
+    flat: dict[str, str] = {}
+    for key, value in config.items():
+        upper_key = str(key).upper()
+        if value is None:
+            flat[upper_key] = ""
+        elif isinstance(value, (dict, list, tuple)):
+            flat[upper_key] = json.dumps(value, default=str)
        else:
-            raise ValueError(f"Invalid configuration option {key}={val} (expected a boolean: True/False)")
-
-    elif type is str:
-        if val.lower() in (*BOOL_TRUEIES, *BOOL_FALSEIES):
-            raise ValueError(f"Invalid configuration option {key}={val} (expected a string, but value looks like a boolean)")
-        return val.strip()
-
-    elif type is int:
-        if not val.strip().isdigit():
-            raise ValueError(f"Invalid configuration option {key}={val} (expected an integer)")
-        return int(val.strip())
-
-    elif type is list or type is dict:
-        return json.loads(val)
-
-    elif type is Path:
-        return Path(val)
-
-    raise Exception("Config values can only be str, bool, int, or json")
+            flat[upper_key] = str(value)
+    return flat


-def load_config_file() -> benedict | None:
-    """load the ini-formatted config file from DATA_DIR/Archivebox.conf"""
-
+def _load_file_config_dict() -> tuple[dict[str, str], float | None]:
+    """Return ``(flat_dict, mtime)`` for ``ArchiveBox.conf`` (``({}, None)`` if missing)."""
    config_path = CONSTANTS.CONFIG_FILE
-    if os.access(config_path, os.R_OK):
-        config_file = CaseConfigParser()
-        config_file.read(config_path)
-        # flatten into one namespace
-        config_file_vars = benedict({key.upper(): val for section, options in config_file.items() for key, val in options.items()})
-        # print('[i] Loaded config file', os.path.abspath(config_path))
-        # print(config_file_vars)
-        return config_file_vars
-    return None
+    try:
+        mtime = config_path.stat().st_mtime
+    except FileNotFoundError:
+        return {}, None
+    parser = CaseConfigParser()
+    parser.read(config_path)
+    flat = {key.upper(): value for section in parser.sections() for key, value in parser.items(section)}
+    return flat, mtime


-class PluginConfigSection:
-    """Pseudo-section for all plugin config keys written to [PLUGINS] section in ArchiveBox.conf"""
-
-    toml_section_header = "PLUGINS"
-
-    def __init__(self, key: str):
-        self._key = key
-
-    def __getattr__(self, name: str) -> Any:
-        # Allow hasattr checks to pass for the key
-        if name == self._key:
-            return None
-        raise AttributeError(f"PluginConfigSection has no attribute '{name}'")
-
-    def update_in_place(self, warn: bool = True, persist: bool = False, **kwargs):
-        """No-op update since plugins read config dynamically via get_config()."""
-        pass
-
-
-def section_for_key(key: str) -> Any:
-    """Find the config section containing a given key."""
-    from archivebox.config.common import (
-        SHELL_CONFIG,
-        STORAGE_CONFIG,
-        GENERAL_CONFIG,
-        SERVER_CONFIG,
-        ARCHIVING_CONFIG,
-        SEARCH_BACKEND_CONFIG,
-    )
-
-    # First check core config sections
-    for section in [
-        SHELL_CONFIG,
-        STORAGE_CONFIG,
-        GENERAL_CONFIG,
-        SERVER_CONFIG,
-        ARCHIVING_CONFIG,
-        SEARCH_BACKEND_CONFIG,
-    ]:
-        if hasattr(section, key):
-            return section
-
-    # Check if this is a plugin config key
-    from archivebox.hooks import discover_plugin_configs
-
-    plugin_configs = discover_plugin_configs()
-    for plugin_name, schema in plugin_configs.items():
+def _resolve_section_for_key(key: str, config_sections, plugin_configs) -> str:
+    for section in config_sections.values():
+        if key in type(section).model_fields:
+            return section.toml_section_header
+    for schema in plugin_configs.values():
        if "properties" in schema and key in schema["properties"]:
-            # All plugin config goes to [PLUGINS] section
-            return PluginConfigSection(key)
-
-    raise ValueError(f"No config section found for key: {key}")
+            return "PLUGINS"
+    # Unknown / user-defined keys land in SERVER_CONFIG so we never lose them
+    # (the previous code raised here, which was fine for the CLI-only path but
+    # would break the mirror as soon as anyone added a plugin-tunable that
+    # this process hasn't loaded a schema for).
+    return "SERVER_CONFIG"


-def write_config_file(config: dict[str, str]) -> benedict:
-    """load the ini-formatted config file from DATA_DIR/Archivebox.conf"""
+def _render_config_file_content(config: dict[str, str]) -> str:
+    """Render a flat config dict to INI text, grouped by inferred section."""
+    from archivebox.config.common import get_all_configs
+    from archivebox.plugins.discovery import discover_plugin_configs

+    config_sections = get_all_configs()
+    plugin_configs = discover_plugin_configs()
+
+    parser = CaseConfigParser()
+    for key, val in sorted(config.items()):
+        section = _resolve_section_for_key(key, config_sections, plugin_configs)
+        if section not in parser:
+            parser[section] = {}
+        parser[section][key] = "" if val is None else str(val)
+
+    buf = io.StringIO()
+    buf.write(CONFIG_FILE_HEADER)
+    parser.write(buf)
+    return buf.getvalue()
+
+
+def _write_file_if_changed(content: str) -> bool:
+    """Atomic-write ``ArchiveBox.conf`` only when contents actually differ.
+
+    Skipping unchanged writes is the difference between every Machine.save in
+    a hot autodetection loop costing one disk write vs. zero.
+    """
    from archivebox.misc.system import atomic_write

-    CONFIG_HEADER = """# This is the config file for your ArchiveBox collection.
-    #
-    # You can add options here manually in INI format, or automatically by running:
-    #    archivebox config --set KEY=VALUE
-    #
-    # If you modify this file manually, make sure to update your archive after by running:
-    #    archivebox init
-    #
-    # A list of all possible config with documentation and examples can be found here:
-    #    https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration
+    config_path = CONSTANTS.CONFIG_FILE
+    try:
+        existing = config_path.read_text(encoding="utf-8") if config_path.exists() else None
+    except OSError:
+        existing = None
+    if existing == content:
+        return False
+    atomic_write(config_path, content)
+    return True

+
+def mirror_machine_config_to_file(config: Any) -> None:
+    """Rewrite ``ArchiveBox.conf`` so it mirrors ``Machine.config`` exactly.
+
+    Called from ``Machine.save`` after the row is committed. Recursion-guarded
+    so the matching write_config_file -> Machine.save bounce doesn't loop.
    """
+    global _MIRROR_IN_PROGRESS
+    if _MIRROR_IN_PROGRESS:
+        return
+    _MIRROR_IN_PROGRESS = True
+    try:
+        flat = _coerce_to_str_dict(config)
+        _write_file_if_changed(_render_config_file_content(flat))
+    finally:
+        _MIRROR_IN_PROGRESS = False
+
+
+def _coerce_from_str_dict(file_config: dict[str, str]) -> dict[str, Any]:
+    """Inverse of ``_coerce_to_str_dict``: decode complex INI values to native.
+
+    ``mirror_machine_config_to_file`` JSON-encodes ``dict`` / ``list`` values
+    so they round-trip through INI's string-only storage. When reading the
+    file back into ``Machine.config`` (a JSONField that holds native types)
+    those strings have to be decoded — otherwise downstream consumers like
+    ``_emit_machine_config`` → ``MachineEvent`` → abx-dl see a JSON string
+    where they expect a dict and raise ``TypeError``.
+    Declared fields go through pydantic-settings' own ``field_is_complex`` /
+    ``prepare_field_value`` so they're decoded per annotation. Undeclared
+    keys (e.g. ``ABX_INSTALL_CACHE``, written dynamically by abx-dl) are
+    JSON-decoded when their string starts with ``{`` or ``[`` — the same
+    shape ``_coerce_to_str_dict`` writes them as.
+    """
+    from archivebox.config.common import ArchiveBoxConfig
+    from archivebox.config.configset import IniConfigSettingsSource
+
+    decoder = IniConfigSettingsSource(ArchiveBoxConfig)
+    decoded: dict[str, Any] = dict(file_config)
+    declared_fields = set(ArchiveBoxConfig.model_fields)
+    for field_name, field in ArchiveBoxConfig.model_fields.items():
+        if field_name not in decoded:
+            continue
+        raw = decoded[field_name]
+        if not isinstance(raw, str) or not raw:
+            continue
+        if decoder.field_is_complex(field):
+            decoded[field_name] = decoder.prepare_field_value(field_name, field, raw, True)
+    for key, raw in list(decoded.items()):
+        if key in declared_fields:
+            continue
+        if not isinstance(raw, str) or not raw:
+            continue
+        first = raw[:1]
+        if first not in ("{", "["):
+            continue
+        try:
+            decoded[key] = json.loads(raw)
+        except (TypeError, ValueError):
+            continue
+    return decoded
+
+
+def _mirror_file_to_machine_config(file_config: dict[str, str]) -> None:
+    """Copy ``ArchiveBox.conf`` contents into ``Machine.config``.
+
+    Internal helper used by ``write_config_file`` and the startup sync —
+    callers must hold the ``_MIRROR_IN_PROGRESS`` guard around it.
+    """
+    from archivebox.machine.models import Machine
+
+    machine = Machine.current()
+    if _coerce_to_str_dict(machine.config) == file_config:
+        return
+    machine.config = _coerce_from_str_dict(file_config)
+    machine.save(update_fields=["config", "modified_at"])
+
+
+def sync_machine_and_file(machine: Any = None) -> None:
+    """One-time-per-process reconciliation between the two stores.
+
+    Cheap on the common case where they already agree (single ``stat`` + dict
+    compare ≈ 1ms). When the two sides diverge we merge them: each side's
+    unique keys are preserved, and for keys present on both we let the newer
+    side win (file mtime vs. ``Machine.modified_at``). After the merge both
+    stores hold the union, so every subsequent write keeps them in lockstep
+    via the full-replace mirror functions.
+
+    Pass ``machine`` when the caller already has a current ``Machine``
+    instance (e.g. from ``Machine.current()``) to skip the 10–15ms
+    ``get_host_guid()`` round-trip on the cold path.
+    """
+    global _INITIAL_SYNC_DONE, _MIRROR_IN_PROGRESS
+    if _INITIAL_SYNC_DONE:
+        return
+    _INITIAL_SYNC_DONE = True
+    if _MIRROR_IN_PROGRESS:
+        return
+    _MIRROR_IN_PROGRESS = True
+    try:
+        if machine is None:
+            from archivebox.machine.detect import get_host_guid
+            from archivebox.machine.models import Machine
+
+            try:
+                machine = Machine.objects.filter(guid=get_host_guid()).first()
+            except Exception:
+                return
+            if machine is None:
+                return
+
+        file_config, file_mtime = _load_file_config_dict()
+        machine_config = _coerce_to_str_dict(machine.config)
+        if machine_config == file_config:
+            return
+
+        db_mtime = machine.modified_at.timestamp() if machine.modified_at else 0.0
+        file_is_newer = file_mtime is not None and file_mtime > db_mtime
+
+        merged: dict[str, str] = {}
+        all_keys = set(machine_config) | set(file_config)
+        for key in all_keys:
+            in_file = key in file_config
+            in_db = key in machine_config
+            if in_file and in_db:
+                if file_config[key] == machine_config[key]:
+                    merged[key] = file_config[key]
+                else:
+                    merged[key] = file_config[key] if file_is_newer else machine_config[key]
+            elif in_file:
+                merged[key] = file_config[key]
+            else:
+                merged[key] = machine_config[key]
+
+        if merged != file_config:
+            _write_file_if_changed(_render_config_file_content(merged))
+        if merged != machine_config:
+            machine.config = _coerce_from_str_dict(merged)
+            machine.save(update_fields=["config", "modified_at"])
+    finally:
+        _MIRROR_IN_PROGRESS = False
+
+
+def write_config_file(config: dict[str, str]) -> AttrDict:
+    """Merge ``config`` into ``ArchiveBox.conf``, validate, then mirror to Machine.config.
+
+    Backwards-compatible signature: callers (CLI ``archivebox config --set``
+    and the init flow) pass a partial dict of keys to upsert.
+    """
+
+    from archivebox.config.common import get_all_configs
+    from archivebox.plugins.discovery import discover_plugin_configs
+    from archivebox.misc.system import atomic_write

    config_path = CONSTANTS.CONFIG_FILE

    if not os.access(config_path, os.F_OK):
-        atomic_write(config_path, CONFIG_HEADER)
+        atomic_write(config_path, CONFIG_FILE_HEADER)

    config_file = CaseConfigParser()
    config_file.read(config_path)
@@ -197,23 +299,18 @@ def write_config_file(config: dict[str, str]) -> benedict:
    with open(config_path, encoding="utf-8") as old:
        atomic_write(f"{config_path}.bak", old.read())

+    config_sections = get_all_configs()
+    plugin_configs = discover_plugin_configs()
+
    # Set up sections in empty config file
    for key, val in config.items():
-        section = section_for_key(key)
-        assert section is not None
-
-        if not hasattr(section, "toml_section_header"):
-            raise ValueError(f"{key} is read-only (defined in {type(section).__module__}.{type(section).__name__}). Refusing to set.")
-
-        section_name = section.toml_section_header
-
+        section_name = _resolve_section_for_key(key, config_sections, plugin_configs)
        if section_name in config_file:
            existing_config = dict(config_file[section_name])
        else:
            existing_config = {}

-        config_file[section_name] = benedict({**existing_config, key: val})
-        section.update_in_place(warn=False, persist=False, **{key: val})
+        config_file[section_name] = AttrDict({**existing_config, key: val})

    with open(config_path, "w+", encoding="utf-8") as new:
        config_file.write(new)
@@ -221,9 +318,9 @@ def write_config_file(config: dict[str, str]) -> benedict:
    updated_config = {}
    try:
        # validate the updated_config by attempting to re-parse it
-        from archivebox.config.configset import get_flat_config
+        from archivebox.config.common import get_config

-        updated_config = {**load_all_config(), **get_flat_config()}
+        updated_config = get_config().as_dict()
    except BaseException:  # lgtm [py/catch-base-exception]
        # something went horribly wrong, revert to the previous version
        with open(f"{config_path}.bak", encoding="utf-8") as old:
@@ -234,72 +331,18 @@ def write_config_file(config: dict[str, str]) -> benedict:
    if os.access(f"{config_path}.bak", os.F_OK):
        os.remove(f"{config_path}.bak")

-    return benedict({key.upper(): updated_config.get(key.upper()) for key in config.keys()})
-
-
-def load_config(
-    defaults: dict[str, Any],
-    config: benedict | None = None,
-    out_dir: str | None = None,
-    env_vars: os._Environ | None = None,
-    config_file_vars: dict[str, str] | None = None,
-) -> benedict:
-
-    env_vars = env_vars or os.environ
-    config_file_vars = config_file_vars or load_config_file()
-
-    extended_config = benedict(config.copy() if config else {})
-    for key, default in defaults.items():
+    # Mirror the post-write file state into Machine.config so the DB stays
+    # 1:1 with the on-disk file. Recursion-guarded so Machine.save's own
+    # mirror-back doesn't loop us.
+    global _MIRROR_IN_PROGRESS
+    if not _MIRROR_IN_PROGRESS:
+        _MIRROR_IN_PROGRESS = True
        try:
-            # print('LOADING CONFIG KEY:', key, 'DEFAULT=', default)
-            extended_config[key] = load_config_val(
-                key,
-                default=default["default"],
-                type=default.get("type"),
-                aliases=default.get("aliases"),
-                config=extended_config,
-                env_vars=env_vars,
-                config_file_vars=config_file_vars,
-            )
-        except KeyboardInterrupt:
-            raise SystemExit(0)
-        except Exception as e:
-            stderr()
-            stderr(f"[X] Error while loading configuration value: {key}", color="red", config=extended_config)
-            stderr(f"    {e.__class__.__name__}: {e}")
-            stderr()
-            stderr("    Check your config for mistakes and try again (your archive data is unaffected).")
-            stderr()
-            stderr("    For config documentation and examples see:")
-            stderr("        https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration")
-            stderr()
-            # raise
-            # raise SystemExit(2)
+            flat, _mtime = _load_file_config_dict()
+            _mirror_file_to_machine_config(flat)
+        except Exception:
+            pass
+        finally:
+            _MIRROR_IN_PROGRESS = False

-    return benedict(extended_config)
-
-
-def load_all_config():
-    """Load all config sections and return as a flat dict."""
-    from archivebox.config.common import (
-        SHELL_CONFIG,
-        STORAGE_CONFIG,
-        GENERAL_CONFIG,
-        SERVER_CONFIG,
-        ARCHIVING_CONFIG,
-        SEARCH_BACKEND_CONFIG,
-    )
-
-    flat_config = benedict()
-
-    for config_section in [
-        SHELL_CONFIG,
-        STORAGE_CONFIG,
-        GENERAL_CONFIG,
-        SERVER_CONFIG,
-        ARCHIVING_CONFIG,
-        SEARCH_BACKEND_CONFIG,
-    ]:
-        flat_config.update(dict(config_section))
-
-    return flat_config
+    return AttrDict({key.upper(): updated_config.get(key.upper()) for key in config.keys()})
@@ -1,27 +1,62 @@
-"""
-Simplified config system for ArchiveBox.
-
-This replaces the complex abx_spec_config/base_configset.py with a simpler
-approach that still supports environment variables, config files, and
-per-object overrides.
-"""
+"""Pydantic-backed config loading for ArchiveBox."""

 __package__ = "archivebox.config"

-import os
-import json
 from pathlib import Path
-from typing import Any
+from typing import Any, ClassVar
 from configparser import ConfigParser

 from pydantic_settings import BaseSettings, PydanticBaseSettingsSource, SettingsConfigDict

+COMPUTED_CONFIG_KEYS = (
+    "TERM_WIDTH",
+    "COMMIT_HASH",
+    "BUILD_TIME",
+    "USES_SUBDOMAIN_ROUTING",
+    "ENABLES_FULL_JS_REPLAY",
+    "CONTROL_PLANE_ENABLED",
+    "BLOCK_UNSAFE_METHODS",
+    "SHOULD_NEUTER_RISKY_REPLAY",
+    "IS_UNSAFE_MODE",
+    "IS_DANGEROUS_MODE",
+    "IS_LOWER_SECURITY_MODE",
+    "URL_ALLOWLIST_PTN",
+    "URL_DENYLIST_PTN",
+)
+

 class CaseConfigParser(ConfigParser):
    def optionxform(self, optionstr: str) -> str:
        return optionstr


+# ``IniConfigSettingsSource.get_field_value`` is called once per pydantic field
+# (hundreds of fields across the GlobalConfig + plugin configs). Without
+# caching, every field re-opens and re-parses ``ArchiveBox.conf`` from disk —
+# that's the dominant cost of ``get_config()`` (each call was ~130ms; profile
+# showed 632 ``parser.read(config_path)`` invocations per call). The cache is
+# keyed on (path, mtime) so external edits to the file still get picked up.
+_INI_CACHE: dict[tuple[str, float], dict[str, Any]] = {}
+
+
+def _read_ini_config_cached(config_path_str: str) -> dict[str, Any]:
+    config_path = Path(config_path_str)
+    try:
+        mtime = config_path.stat().st_mtime
+    except FileNotFoundError:
+        return {}
+    cache_key = (str(config_path), mtime)
+    cached = _INI_CACHE.get(cache_key)
+    if cached is not None:
+        return cached
+    parser = CaseConfigParser()
+    parser.read(config_path)
+    flat = {key.upper(): value for section in parser.sections() for key, value in parser.items(section)}
+    _INI_CACHE.clear()
+    _INI_CACHE[cache_key] = flat
+    return flat
+
+
 class IniConfigSettingsSource(PydanticBaseSettingsSource):
    """
    Custom settings source that reads from ArchiveBox.conf (INI format).
@@ -31,10 +66,27 @@ class IniConfigSettingsSource(PydanticBaseSettingsSource):
    def get_field_value(self, field: Any, field_name: str) -> tuple[Any, str, bool]:
        config_vals = self._load_config_file()
        field_value = config_vals.get(field_name.upper())
-        return field_value, field_name, False
+        # Mark complex-typed fields (``dict``/``list``) so the parent
+        # ``prepare_field_value`` JSON-decodes the INI string before pydantic
+        # validates against the dict/list annotation. Without this, e.g.
+        # ``ABX_INSTALL_CACHE`` arrives as a raw JSON string and pydantic
+        # rejects it with ``Input should be a valid dictionary``.
+        value_is_complex = bool(field_value is not None and self.field_is_complex(field))
+        return field_value, field_name, value_is_complex

    def __call__(self) -> dict[str, Any]:
-        return self._load_config_file()
+        # Use the per-field path (``get_field_value`` + ``prepare_field_value``)
+        # so complex types get JSON-decoded. The previous flat-dict return
+        # skipped pydantic-settings' complex-value handling entirely.
+        result: dict[str, Any] = {}
+        for field_name, field in self.settings_cls.model_fields.items():
+            value, key, value_is_complex = self.get_field_value(field, field_name)
+            if value is None:
+                continue
+            prepared = self.prepare_field_value(field_name, field, value, value_is_complex)
+            if prepared is not None:
+                result[key] = prepared
+        return result

    def _load_config_file(self) -> dict[str, Any]:
        try:
@@ -44,14 +96,7 @@ class IniConfigSettingsSource(PydanticBaseSettingsSource):
        except ImportError:
            return {}

-        if not config_path.exists():
-            return {}
-
-        parser = CaseConfigParser()
-        parser.read(config_path)
-
-        # Flatten all sections into single namespace (ignore section headers)
-        return {key.upper(): value for section in parser.sections() for key, value in parser.items(section)}
+        return _read_ini_config_cached(str(config_path))


 class BaseConfigSet(BaseSettings):
@@ -74,7 +119,9 @@ class BaseConfigSet(BaseSettings):
        env_prefix="",
        extra="ignore",
        validate_default=True,
+        populate_by_name=True,
    )
+    computed_config_keys: ClassVar[tuple[str, ...]] = ()

    @classmethod
    def settings_customise_sources(
@@ -108,321 +155,57 @@ class BaseConfigSet(BaseSettings):
        # Flatten all sections into single namespace
        return {key.upper(): value for section in parser.sections() for key, value in parser.items(section)}

-    def update_in_place(self, warn: bool = True, persist: bool = False, **kwargs) -> None:
-        """
-        Update config values in place.
+    def __getitem__(self, key: str) -> Any:
+        if key in type(self).model_fields:
+            return getattr(self, key)
+        if self.__pydantic_extra__ and key in self.__pydantic_extra__:
+            return self.__pydantic_extra__[key]
+        if key in self.computed_config_keys:
+            return getattr(self, key)
+        raise KeyError(key)

-        This allows runtime updates to config without reloading.
-        """
-        for key, value in kwargs.items():
-            if hasattr(self, key):
-                # Use object.__setattr__ to bypass pydantic's frozen model
-                object.__setattr__(self, key, value)
+    def __setitem__(self, key: str, value: Any) -> None:
+        if key in type(self).model_fields:
+            object.__setattr__(self, key, value)
+            return
+        if key in self.computed_config_keys:
+            raise KeyError(f"{key} is computed and cannot be set")
+        if self.model_config.get("extra") != "allow":
+            raise KeyError(f"Unknown config key: {key}")
+        extra = self.__pydantic_extra__
+        if extra is None:
+            extra = {}
+            object.__setattr__(self, "__pydantic_extra__", extra)
+        extra[key] = value

+    def update(self, *args, **kwargs) -> None:
+        values = dict(*args, **kwargs)
+        for key, value in values.items():
+            if key in self.computed_config_keys:
+                continue
+            self[key] = value

-def get_config(
-    defaults: dict | None = None,
-    persona: Any = None,
-    user: Any = None,
-    crawl: Any = None,
-    snapshot: Any = None,
-    archiveresult: Any = None,
-    machine: Any = None,
-) -> dict[str, Any]:
-    """
-    Get merged config from all sources.
+    def __contains__(self, key: str) -> bool:
+        return (
+            key in type(self).model_fields
+            or bool(self.__pydantic_extra__ and key in self.__pydantic_extra__)
+            or key in self.computed_config_keys
+        )

-    Priority (highest to lowest):
-    1. Per-snapshot config (snapshot.config JSON field)
-    2. Per-crawl config (crawl.config JSON field)
-    3. Per-user config (user.config JSON field)
-    4. Per-persona config (persona.get_derived_config() - includes CHROME_USER_DATA_DIR etc.)
-    5. Environment variables
-    6. Per-machine config (machine.config JSON field - resolved binary paths)
-    7. Config file (ArchiveBox.conf)
-    8. Plugin schema defaults (config.json)
-    9. Core config defaults
+    def get(self, key: str, default: Any = None) -> Any:
+        return self[key] if key in self else default

-    Args:
-        defaults: Default values to start with
-        persona: Persona object (provides derived paths like CHROME_USER_DATA_DIR)
-        user: User object with config JSON field
-        crawl: Crawl object with config JSON field
-        snapshot: Snapshot object with config JSON field
-        archiveresult: ArchiveResult object (auto-fetches snapshot)
-        machine: Machine object with config JSON field (defaults to Machine.current())
+    def as_dict(self) -> dict[str, Any]:
+        data = self.model_dump()
+        for key in self.computed_config_keys:
+            data[key] = getattr(self, key)
+        return data

-    Note: Objects are auto-fetched from relationships if not provided:
-        - snapshot auto-fetched from archiveresult.snapshot
-        - crawl auto-fetched from snapshot.crawl
-        - user auto-fetched from crawl.created_by
+    def items(self):
+        return self.as_dict().items()

-    Returns:
-        Merged config dict
-    """
-    # Auto-fetch related objects from relationships
-    if snapshot is None and archiveresult and hasattr(archiveresult, "snapshot"):
-        snapshot = archiveresult.snapshot
+    def keys(self):
+        return self.as_dict().keys()

-    if crawl is None and snapshot and hasattr(snapshot, "crawl"):
-        crawl = snapshot.crawl
-
-    if user is None and crawl and hasattr(crawl, "created_by"):
-        user = crawl.created_by
-
-    if persona is None and crawl is not None:
-        from archivebox.personas.models import Persona
-
-        persona_id = getattr(crawl, "persona_id", None)
-        if persona_id:
-            persona = Persona.objects.filter(id=persona_id).first()
-            if persona is None:
-                raise Persona.DoesNotExist(f"Crawl {getattr(crawl, 'id', None)} references missing Persona {persona_id}")
-
-        if persona is None:
-            crawl_config = getattr(crawl, "config", None) or {}
-            default_persona_name = str(crawl_config.get("DEFAULT_PERSONA") or "").strip()
-            if default_persona_name:
-                persona, _ = Persona.objects.get_or_create(name=default_persona_name or "Default")
-                persona.ensure_dirs()
-    from archivebox.config.constants import CONSTANTS
-    from archivebox.config.common import (
-        SHELL_CONFIG,
-        STORAGE_CONFIG,
-        GENERAL_CONFIG,
-        SERVER_CONFIG,
-        ARCHIVING_CONFIG,
-        SEARCH_BACKEND_CONFIG,
-    )
-
-    # Start with defaults
-    config = dict(defaults or {})
-
-    # Add plugin config defaults from JSONSchema config.json files
-    try:
-        from archivebox.hooks import get_config_defaults_from_plugins
-
-        plugin_defaults = get_config_defaults_from_plugins()
-        config.update(plugin_defaults)
-    except ImportError:
-        pass  # hooks not available yet during early startup
-
-    # Add all core config sections
-    config.update(dict(SHELL_CONFIG))
-    config.update(dict(STORAGE_CONFIG))
-    config.update(dict(GENERAL_CONFIG))
-    config.update(dict(SERVER_CONFIG))
-    config.update(dict(ARCHIVING_CONFIG))
-    config.update(dict(SEARCH_BACKEND_CONFIG))
-
-    # Load from archivebox.config.file
-    config_file = CONSTANTS.CONFIG_FILE
-    if config_file.exists():
-        file_config = BaseConfigSet.load_from_file(config_file)
-        config.update(file_config)
-
-    # Apply machine config overrides (cached binary paths, etc.)
-    if machine is None:
-        # Default to current machine if not provided
-        try:
-            from archivebox.machine.models import Machine
-
-            machine = Machine.current()
-        except Exception:
-            pass  # Machine might not be available during early init
-
-    if machine and hasattr(machine, "config") and machine.config:
-        config.update(machine.config)
-
-    # Override with environment variables (for keys that exist in config)
-    for key in config:
-        env_val = os.environ.get(key)
-        if env_val is not None:
-            config[key] = _parse_env_value(env_val, config.get(key))
-
-    # Also add NEW environment variables (not yet in config)
-    # This is important for worker subprocesses that receive config via Process.env
-    for key, value in os.environ.items():
-        if key.isupper() and key not in config:  # Only uppercase keys (config convention)
-            config[key] = _parse_env_value(value, None)
-
-    # Also check plugin config aliases in environment
-    try:
-        from archivebox.hooks import discover_plugin_configs
-
-        plugin_configs = discover_plugin_configs()
-        for plugin_name, schema in plugin_configs.items():
-            for key, prop_schema in schema.get("properties", {}).items():
-                # Check x-aliases
-                for alias in prop_schema.get("x-aliases", []):
-                    if alias in os.environ and key not in os.environ:
-                        config[key] = _parse_env_value(os.environ[alias], config.get(key))
-                        break
-                # Check x-fallback
-                fallback = prop_schema.get("x-fallback")
-                if fallback and fallback in config and key not in config:
-                    config[key] = config[fallback]
-    except ImportError:
-        pass
-
-    # Apply persona config overrides (includes derived paths like CHROME_USER_DATA_DIR)
-    if persona and hasattr(persona, "get_derived_config"):
-        config.update(persona.get_derived_config())
-
-    # Apply user config overrides
-    if user and hasattr(user, "config") and user.config:
-        config.update(user.config)
-
-    # Apply crawl config overrides
-    if crawl and hasattr(crawl, "config") and crawl.config:
-        config.update(crawl.config)
-
-    # Add crawl path aliases for hooks that need shared crawl state.
-    if crawl and hasattr(crawl, "output_dir"):
-        config["CRAWL_OUTPUT_DIR"] = str(crawl.output_dir)
-        config["CRAWL_DIR"] = str(crawl.output_dir)
-        config["CRAWL_ID"] = str(getattr(crawl, "id", "")) if getattr(crawl, "id", None) else config.get("CRAWL_ID")
-
-    # Apply snapshot config overrides (highest priority)
-    if snapshot and hasattr(snapshot, "config") and snapshot.config:
-        config.update(snapshot.config)
-
-    if snapshot:
-        config["SNAPSHOT_ID"] = str(getattr(snapshot, "id", "")) if getattr(snapshot, "id", None) else config.get("SNAPSHOT_ID")
-        config["SNAPSHOT_DEPTH"] = int(getattr(snapshot, "depth", 0) or 0)
-        if hasattr(snapshot, "output_dir"):
-            config["SNAP_DIR"] = str(snapshot.output_dir)
-        if getattr(snapshot, "crawl_id", None):
-            config["CRAWL_ID"] = str(snapshot.crawl_id)
-
-    # Normalize all aliases to canonical names (after all sources merged)
-    # This handles aliases that came from user/crawl/snapshot configs, not just env
-    try:
-        from archivebox.hooks import discover_plugin_configs
-
-        plugin_configs = discover_plugin_configs()
-        aliases_to_normalize = {}  # {alias_key: canonical_key}
-
-        # Build alias mapping from all plugin schemas
-        for plugin_name, schema in plugin_configs.items():
-            for canonical_key, prop_schema in schema.get("properties", {}).items():
-                for alias in prop_schema.get("x-aliases", []):
-                    aliases_to_normalize[alias] = canonical_key
-
-        # Normalize: copy alias values to canonical keys (aliases take precedence)
-        for alias_key, canonical_key in aliases_to_normalize.items():
-            if alias_key in config:
-                # Alias exists - copy to canonical key (overwriting any default)
-                config[canonical_key] = config[alias_key]
-                # Remove alias from config to keep it clean
-                del config[alias_key]
-    except ImportError:
-        pass
-
-    if not config.get("DATA_DIR"):
-        config["DATA_DIR"] = str(CONSTANTS.DATA_DIR)
-    config["ABX_RUNTIME"] = "archivebox"
-
-    return config
-
-
-def get_flat_config() -> dict[str, Any]:
-    """
-    Get a flat dictionary of all config values.
-
-    Replaces abx.pm.hook.get_FLAT_CONFIG()
-    """
-    return get_config()
-
-
-def get_all_configs() -> dict[str, BaseConfigSet]:
-    """
-    Get all config section objects as a dictionary.
-
-    Replaces abx.pm.hook.get_CONFIGS()
-    """
-    from archivebox.config.common import (
-        SHELL_CONFIG,
-        SERVER_CONFIG,
-        ARCHIVING_CONFIG,
-        SEARCH_BACKEND_CONFIG,
-    )
-
-    return {
-        "SHELL_CONFIG": SHELL_CONFIG,
-        "SERVER_CONFIG": SERVER_CONFIG,
-        "ARCHIVING_CONFIG": ARCHIVING_CONFIG,
-        "SEARCH_BACKEND_CONFIG": SEARCH_BACKEND_CONFIG,
-    }
-
-
-def _parse_env_value(value: str, default: Any = None) -> Any:
-    """Parse an environment variable value based on expected type."""
-    if default is None:
-        # Try to guess the type
-        if value.lower() in ("true", "false", "yes", "no", "1", "0"):
-            return value.lower() in ("true", "yes", "1")
-        try:
-            return int(value)
-        except ValueError:
-            pass
-        try:
-            return json.loads(value)
-        except (json.JSONDecodeError, ValueError):
-            pass
-        return value
-
-    # Parse based on default's type
-    if isinstance(default, bool):
-        return value.lower() in ("true", "yes", "1")
-    elif isinstance(default, int):
-        return int(value)
-    elif isinstance(default, float):
-        return float(value)
-    elif isinstance(default, (list, dict)):
-        return json.loads(value)
-    elif isinstance(default, Path):
-        return Path(value)
-    else:
-        return value
-
-
-# Default worker concurrency settings
-DEFAULT_WORKER_CONCURRENCY = {
-    "crawl": 2,
-    "snapshot": 3,
-    "wget": 2,
-    "ytdlp": 2,
-    "screenshot": 3,
-    "singlefile": 2,
-    "title": 5,
-    "favicon": 5,
-    "headers": 5,
-    "archivedotorg": 2,
-    "readability": 3,
-    "mercury": 3,
-    "git": 2,
-    "pdf": 2,
-    "dom": 3,
-}
-
-
-def get_worker_concurrency() -> dict[str, int]:
-    """
-    Get worker concurrency settings.
-
-    Can be configured via WORKER_CONCURRENCY env var as JSON dict.
-    """
-    config = get_config()
-
-    # Start with defaults
-    concurrency = DEFAULT_WORKER_CONCURRENCY.copy()
-
-    # Override with config
-    if "WORKER_CONCURRENCY" in config:
-        custom = config["WORKER_CONCURRENCY"]
-        if isinstance(custom, str):
-            custom = json.loads(custom)
-        concurrency.update(custom)
-
-    return concurrency
+    def values(self):
+        return self.as_dict().values()
@@ -17,14 +17,16 @@ import sys

 from pathlib import Path

-from benedict import benedict
+from platformdirs import user_config_path

-from archivebox.misc.logging import DEFAULT_CLI_COLORS
+from archivebox.misc.logging import AttrDict, DEFAULT_CLI_COLORS

 from .paths import (
    PACKAGE_DIR,
    DATA_DIR,
    ARCHIVE_DIR,
+    USERS_DIR,
+    _env_path,
    get_collection_id,
    get_machine_id,
    get_machine_type,
@@ -48,6 +50,7 @@ class ConstantsDict:
    PACKAGE_DIR: Path = PACKAGE_DIR
    DATA_DIR: Path = DATA_DIR
    ARCHIVE_DIR: Path = ARCHIVE_DIR
+    USERS_DIR: Path = USERS_DIR

    MACHINE_TYPE: str = get_machine_type()
    MACHINE_ID: str = get_machine_id()
@@ -76,13 +79,17 @@ class ConstantsDict:

    # Data dirs
    ARCHIVE_DIR_NAME: str = "archive"
+    USERS_DIR_NAME: str = "users"
+    SNAPSHOTS_DIR_NAME: str = "snapshots"
+    CRAWLS_DIR_NAME: str = "crawls"
    SOURCES_DIR_NAME: str = "sources"
    PERSONAS_DIR_NAME: str = "personas"
    CACHE_DIR_NAME: str = "cache"
    LOGS_DIR_NAME: str = "logs"
    CUSTOM_PLUGINS_DIR_NAME: str = "custom_plugins"
    CUSTOM_TEMPLATES_DIR_NAME: str = "custom_templates"
-    ARCHIVE_DIR: Path = DATA_DIR / ARCHIVE_DIR_NAME
+    ARCHIVE_DIR: Path = ARCHIVE_DIR
+    USERS_DIR: Path = USERS_DIR
    SOURCES_DIR: Path = DATA_DIR / SOURCES_DIR_NAME
    PERSONAS_DIR: Path = DATA_DIR / PERSONAS_DIR_NAME
    LOGS_DIR: Path = DATA_DIR / LOGS_DIR_NAME
@@ -107,13 +114,26 @@ class ConstantsDict:
    DEFAULT_TMP_DIR: Path = DATA_DIR / TMP_DIR_NAME / MACHINE_ID  # ./data/tmp/abc3244323

    LIB_DIR_NAME: str = "lib"
-    DEFAULT_LIB_DIR: Path = DATA_DIR / LIB_DIR_NAME / MACHINE_TYPE  # ./data/lib/arm64-linux-docker
-    DEFAULT_LIB_BIN_DIR: Path = DEFAULT_LIB_DIR / "bin"  # ./data/lib/arm64-linux-docker/bin
+    DEFAULT_LIB_DIR: Path = _env_path(
+        "LIB_DIR",
+        user_config_path("abx") / LIB_DIR_NAME,
+    )
+    DEFAULT_LIB_BIN_DIR: Path = _env_path("LIB_BIN_DIR", DEFAULT_LIB_DIR / "bin")
+
+    RESERVED_ARCHIVE_DIR_NAMES: frozenset[str] = frozenset(
+        (
+            USERS_DIR_NAME,
+            SNAPSHOTS_DIR_NAME,
+            CRAWLS_DIR_NAME,
+            "invalid",
+            ".DS_Store",
+        ),
+    )

    # Config constants
    TIMEZONE: str = "UTC"
    DEFAULT_CLI_COLORS: dict[str, str] = DEFAULT_CLI_COLORS
-    DISABLED_CLI_COLORS: dict[str, str] = benedict({k: "" for k in DEFAULT_CLI_COLORS})
+    DISABLED_CLI_COLORS: dict[str, str] = AttrDict({k: "" for k in DEFAULT_CLI_COLORS})

    # Hard safety limits (seconds)
    MAX_HOOK_RUNTIME_SECONDS: int = 60 * 60 * 12  # 12 hours
@@ -233,7 +253,7 @@ class ConstantsDict:
            "user_plugins",  # old name for USER_PLUGINS_DIR (now 'plugins')
            "user_templates",  # old name for CUSTOM_TEMPLATES_DIR (now 'templates')
            "static",  # created by old static exports <v0.6.0
-            "sonic",  # created by docker bind mount / sonic FTS process
+            "sonic",  # created by plugin-managed Sonic FTS worker
            ".git",
            ".svn",
            ### Files:
@@ -270,14 +290,9 @@ class ConstantsDict:
        # so it behaves like a dict[key] == dict.key or object attr
        return getattr(cls, key)

-    @classmethod
-    def __benedict__(cls):
-        # when casting to benedict, only include uppercase keys that don't start with an underscore
-        return benedict({key: value for key, value in cls.__dict__.items() if key.isupper() and not key.startswith("_")})
-

 CONSTANTS = ConstantsDict
-CONSTANTS_CONFIG = CONSTANTS.__benedict__()
+CONSTANTS_CONFIG = AttrDict({key: value for key, value in CONSTANTS.__dict__.items() if key.isupper() and not key.startswith("_")})

 # add all key: values to globals() for easier importing, e.g.:
 # from archivebox.config.constants import IS_ROOT, PERSONAS_DIR, ...
@@ -2,6 +2,7 @@ __package__ = "archivebox.config"

 import os
 import sys
+import subprocess

 from datetime import datetime, timezone

@@ -12,29 +13,21 @@ import django.db

 from archivebox.misc import logging

-from . import CONSTANTS
-from .common import SHELL_CONFIG
+from .constants import CONSTANTS
+from .common import get_config

+CONFIG = get_config()

-if not SHELL_CONFIG.USE_COLOR:
+if not CONFIG.USE_COLOR:
    os.environ["NO_COLOR"] = "1"
-if not SHELL_CONFIG.SHOW_PROGRESS:
+if not CONFIG.SHOW_PROGRESS:
    os.environ["TERM"] = "dumb"

-# recreate rich console obj based on new config values
 STDOUT = CONSOLE = Console()
 STDERR = Console(stderr=True)
 logging.CONSOLE = CONSOLE


-def setup_django_minimal():
-    # sys.path.append(str(CONSTANTS.PACKAGE_DIR))
-    # os.environ.setdefault('ARCHIVEBOX_DATA_DIR', str(CONSTANTS.DATA_DIR))
-    # os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings')
-    # django.setup()
-    raise Exception("dont use this anymore")
-
-
 DJANGO_SET_UP = False


@@ -48,6 +41,11 @@ def setup_django(check_db=False, in_memory_db=False) -> None:
        # TODO: figure out why CLI entrypoints with init_pending are running this twice sometimes
        return

+    # Third-party patches are only needed once Django/apps are about to load.
+    # Keeping them out of archivebox.__init__ avoids paying Django/Daphne setup
+    # cost for cheap CLI startup paths like `archivebox <cmd> --help`.
+    import archivebox.misc.monkey_patches  # noqa: F401
+
    from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, SudoPermission

    # if running as root, chown the data dir to the archivebox user to make sure it's accessible to the archivebox user
@@ -55,8 +53,10 @@ def setup_django(check_db=False, in_memory_db=False) -> None:
        with SudoPermission(uid=0):
            # running as root is a special case where it's ok to be a bit slower
            # make sure data dir is always owned by the correct user
-            os.system(f'chown {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP} "{CONSTANTS.DATA_DIR}" 2>/dev/null')
-            os.system(f'chown {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP} "{CONSTANTS.DATA_DIR}"/* 2>/dev/null')
+            subprocess.run(["chown", f"{ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}", str(CONSTANTS.DATA_DIR)], stderr=subprocess.DEVNULL)
+            if CONSTANTS.DATA_DIR.exists():
+                for child in CONSTANTS.DATA_DIR.iterdir():
+                    subprocess.run(["chown", f"{ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}", str(child)], stderr=subprocess.DEVNULL)

    # Suppress the "database access during app initialization" warning
    # This warning can be triggered during django.setup() but is safe to ignore
@@ -107,15 +107,15 @@ def setup_django(check_db=False, in_memory_db=False) -> None:
                    traceback.print_exc()
                return

-        from django.conf import settings
        from archivebox.core.settings_logging import ERROR_LOG as DEFAULT_ERROR_LOG

        # log startup message to the error log
-        error_log = getattr(settings, "ERROR_LOG", DEFAULT_ERROR_LOG)
+        error_log = DEFAULT_ERROR_LOG
        with open(error_log, "a", encoding="utf-8") as f:
            command = " ".join(sys.argv)
            ts = datetime.now(timezone.utc).strftime("%Y-%m-%d__%H:%M:%S")
-            f.write(f"\n> {command}; TS={ts} VERSION={CONSTANTS.VERSION} IN_DOCKER={SHELL_CONFIG.IN_DOCKER} IS_TTY={SHELL_CONFIG.IS_TTY}\n")
+            config = get_config()
+            f.write(f"\n> {command}; TS={ts} VERSION={CONSTANTS.VERSION} IN_DOCKER={config.IN_DOCKER} IS_TTY={config.IS_TTY}\n")

        if check_db:
            # make sure the data dir is owned by a non-root user
@@ -162,6 +162,6 @@ def setup_django(check_db=False, in_memory_db=False) -> None:
            #     logfire.info(f'Started ArchiveBox v{CONSTANTS.VERSION}', argv=sys.argv)

    except KeyboardInterrupt:
-        raise SystemExit(2)
+        raise

    DJANGO_SET_UP = True
@@ -1,6 +1,6 @@
 __package__ = "archivebox.config"

-from pydantic import Field
+from pydantic import Field, PrivateAttr

 from archivebox.config.configset import BaseConfigSet

@@ -14,6 +14,7 @@ class LDAPConfig(BaseConfigSet):
    """

    toml_section_header: str = "LDAP_CONFIG"
+    _scope: str = PrivateAttr(default="server")

    LDAP_ENABLED: bool = Field(default=False)
    LDAP_SERVER_URI: str | None = Field(default=None)
@@ -50,7 +51,3 @@ class LDAPConfig(BaseConfigSet):
            return False, f"LDAP_* config options must all be set if LDAP_ENABLED=True\nMissing: {', '.join(missing)}"

        return True, ""
-
-
-# Singleton instance
-LDAP_CONFIG = LDAPConfig()
@@ -5,19 +5,32 @@ import socket
 import hashlib
 import tempfile
 import platform
+import subprocess
 from pathlib import Path
 from functools import cache
 from datetime import datetime
+from typing import TYPE_CHECKING

-from benedict import benedict
+from .permissions import SudoPermission, IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP

-from .permissions import SudoPermission, IS_ROOT, ARCHIVEBOX_USER
+if TYPE_CHECKING:
+    from archivebox.config.common import ArchiveBoxConfig

 #############################################################################################

 PACKAGE_DIR: Path = Path(__file__).resolve().parent.parent  # archivebox source code dir
-DATA_DIR: Path = Path(os.environ.get("DATA_DIR", os.getcwd())).resolve()  # archivebox user data dir
+DATA_DIR: Path = Path(os.getcwd()).resolve()  # archivebox user data dir
+
+
+def _env_path(key: str, default: Path) -> Path:
+    path = Path(os.environ.get(key) or default).expanduser()
+    if not path.is_absolute():
+        path = DATA_DIR / path
+    return path.resolve()
+
+
 ARCHIVE_DIR: Path = DATA_DIR / "archive"  # archivebox snapshot data dir
+USERS_DIR: Path = ARCHIVE_DIR / "users"  # archivebox user-scoped crawl/snapshot data dir

 IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes")

@@ -54,9 +67,9 @@ def _get_collection_id(DATA_DIR=DATA_DIR, force_create=False) -> str:
            if IS_ROOT:
                with SudoPermission(uid=0):
                    if ARCHIVEBOX_USER == 0:
-                        os.system(f'chmod 777 "{collection_id_file}"')
+                        subprocess.run(["chmod", "777", str(collection_id_file)])
                    else:
-                        os.system(f'chown {ARCHIVEBOX_USER} "{collection_id_file}"')
+                        subprocess.run(["chown", str(ARCHIVEBOX_USER), str(collection_id_file)])
    except (OSError, FileNotFoundError, PermissionError):
        pass
    return collection_id
@@ -114,7 +127,7 @@ def dir_is_writable(dir_path: Path, uid: int | None = None, gid: int | None = No
        if chown:
            # try fixing it using sudo permissions
            with SudoPermission(uid=uid, fallback=fallback):
-                os.system(f'chown {uid}:{gid} "{dir_path}" 2>/dev/null')
+                subprocess.run(["chown", f"{uid}:{gid}", str(dir_path)], stderr=subprocess.DEVNULL)
            return dir_is_writable(dir_path, uid=uid, gid=gid, fallback=fallback, chown=False)
    return False

@@ -143,10 +156,23 @@ def assert_dir_can_contain_unix_sockets(dir_path: Path) -> bool:


 def create_and_chown_dir(dir_path: Path) -> None:
+    """Create a required runtime dir and fix only that dir's ownership when needed."""
+    dir_existed = dir_path.exists()
+    dir_path.mkdir(parents=True, exist_ok=True)
+
+    try:
+        stat = dir_path.stat()
+    except OSError:
+        return
+
+    if dir_existed and stat.st_uid == ARCHIVEBOX_USER and stat.st_gid == ARCHIVEBOX_GROUP:
+        return
+
    with SudoPermission(uid=0, fallback=True):
-        dir_path.mkdir(parents=True, exist_ok=True)
-        os.system(f'chown {ARCHIVEBOX_USER} "{dir_path}" 2>/dev/null')
-        os.system(f'chown {ARCHIVEBOX_USER} "{dir_path}"/* 2>/dev/null &')
+        try:
+            os.chown(dir_path, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP)
+        except (OSError, PermissionError):
+            pass


 def tmp_dir_socket_path_is_short_enough(dir_path: Path) -> bool:
@@ -154,15 +180,15 @@ def tmp_dir_socket_path_is_short_enough(dir_path: Path) -> bool:
    return len(f"file://{socket_file}") <= 96


-@cache
-def get_or_create_working_tmp_dir(autofix=True, quiet=True):
-    from archivebox import CONSTANTS
-    from archivebox.config.common import STORAGE_CONFIG
+def get_or_create_working_tmp_dir(autofix=True, quiet=True, config: "ArchiveBoxConfig | None" = None, **config_kwargs):
+    from archivebox.config.constants import CONSTANTS
+    from archivebox.config.common import get_config
    from archivebox.misc.checks import check_tmp_dir

+    config = config or get_config(**config_kwargs)
    # try a few potential directories in order of preference
    CANDIDATES = [
-        STORAGE_CONFIG.TMP_DIR,  # <user-specified>
+        config.TMP_DIR,  # <user-specified>
        CONSTANTS.DEFAULT_TMP_DIR,  # ./data/tmp/<machine_id>
        Path("/var/run/archivebox") / get_collection_id(),  # /var/run/archivebox/abc5d8512
        Path("/tmp") / "archivebox" / get_collection_id(),  # /tmp/archivebox/abc5d8512
@@ -182,8 +208,8 @@ def get_or_create_working_tmp_dir(autofix=True, quiet=True):
        except Exception:
            pass
        if check_tmp_dir(candidate, throw=False, quiet=True, must_exist=True):
-            if autofix and STORAGE_CONFIG.TMP_DIR != candidate:
-                STORAGE_CONFIG.update_in_place(TMP_DIR=candidate)
+            if autofix and config.TMP_DIR != candidate:
+                os.environ["TMP_DIR"] = str(candidate)
            return candidate
        try:
            if (
@@ -200,30 +226,22 @@ def get_or_create_working_tmp_dir(autofix=True, quiet=True):
    # Fall back to the shortest writable path so read-only CLI commands can still run,
    # and let later permission checks surface the missing socket support if needed.
    if fallback_candidate:
-        if autofix and STORAGE_CONFIG.TMP_DIR != fallback_candidate:
-            STORAGE_CONFIG.update_in_place(TMP_DIR=fallback_candidate)
+        if autofix and config.TMP_DIR != fallback_candidate:
+            os.environ["TMP_DIR"] = str(fallback_candidate)
        return fallback_candidate

    if not quiet:
        raise OSError(f"ArchiveBox is unable to find a writable TMP_DIR, tried {CANDIDATES}!")


-@cache
-def get_or_create_working_lib_dir(autofix=True, quiet=False):
-    from archivebox import CONSTANTS
-    from archivebox.config.common import STORAGE_CONFIG
+def get_or_create_working_lib_dir(autofix=True, quiet=False, config: "ArchiveBoxConfig | None" = None, **config_kwargs):
+    from archivebox.config.common import get_config
    from archivebox.misc.checks import check_lib_dir

-    # try a few potential directories in order of preference
-    CANDIDATES = [
-        STORAGE_CONFIG.LIB_DIR,  # <user-specified>
-        CONSTANTS.DEFAULT_LIB_DIR,  # ./data/lib/arm64-linux-docker
-        Path("/usr/local/share/archivebox") / get_collection_id(),  # /usr/local/share/archivebox/abc5
-        *(
-            [Path("/opt/homebrew/share/archivebox") / get_collection_id()] if os.path.isfile("/opt/homebrew/bin/archivebox") else []
-        ),  # /opt/homebrew/share/archivebox/abc5
-        Path("~/.local/share/archivebox").expanduser() / get_collection_id(),  # ~/.local/share/archivebox/abc5
-    ]
+    config = config or get_config(**config_kwargs)
+
+    # LIB_DIR is either the shared platformdirs default or an explicit env/config override.
+    CANDIDATES = [config.LIB_DIR]

    for candidate in CANDIDATES:
        try:
@@ -231,25 +249,26 @@ def get_or_create_working_lib_dir(autofix=True, quiet=False):
        except Exception:
            pass
        if check_lib_dir(candidate, throw=False, quiet=True, must_exist=True):
-            if autofix and STORAGE_CONFIG.LIB_DIR != candidate:
-                STORAGE_CONFIG.update_in_place(LIB_DIR=candidate)
+            if autofix and config.LIB_DIR != candidate:
+                os.environ["LIB_DIR"] = str(candidate)
            return candidate

    if not quiet:
        raise OSError(f"ArchiveBox is unable to find a writable LIB_DIR, tried {CANDIDATES}!")


-@cache
-def get_data_locations():
-    from archivebox.config import CONSTANTS
-    from archivebox.config.common import STORAGE_CONFIG
+def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs):
+    from archivebox.config.constants import CONSTANTS
+    from archivebox.config.common import get_config
+    from archivebox.misc.logging import AttrDict

+    config = config or get_config(**config_kwargs)
    try:
-        tmp_dir = get_or_create_working_tmp_dir(autofix=True, quiet=True) or STORAGE_CONFIG.TMP_DIR
+        tmp_dir = get_or_create_working_tmp_dir(autofix=True, quiet=True, config=config) or config.TMP_DIR
    except Exception:
-        tmp_dir = STORAGE_CONFIG.TMP_DIR
+        tmp_dir = config.TMP_DIR

-    return benedict(
+    return AttrDict(
        {
            "DATA_DIR": {
                "path": DATA_DIR.resolve(),
@@ -271,10 +290,20 @@ def get_data_locations():
                "is_mount": os.path.ismount(DATABASE_FILE.resolve()),
            },
            "ARCHIVE_DIR": {
-                "path": ARCHIVE_DIR.resolve(),
+                "path": CONSTANTS.ARCHIVE_DIR.resolve(),
                "enabled": True,
-                "is_valid": os.path.isdir(ARCHIVE_DIR) and os.access(ARCHIVE_DIR, os.R_OK) and os.access(ARCHIVE_DIR, os.W_OK),
-                "is_mount": os.path.ismount(ARCHIVE_DIR.resolve()),
+                "is_valid": os.path.isdir(CONSTANTS.ARCHIVE_DIR)
+                and os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK)
+                and os.access(CONSTANTS.ARCHIVE_DIR, os.W_OK),
+                "is_mount": os.path.ismount(CONSTANTS.ARCHIVE_DIR.resolve()),
+            },
+            "USERS_DIR": {
+                "path": CONSTANTS.USERS_DIR.resolve(),
+                "enabled": os.path.isdir(CONSTANTS.USERS_DIR),
+                "is_valid": os.path.isdir(CONSTANTS.USERS_DIR)
+                and os.access(CONSTANTS.USERS_DIR, os.R_OK)
+                and os.access(CONSTANTS.USERS_DIR, os.W_OK),
+                "is_mount": os.path.ismount(CONSTANTS.USERS_DIR.resolve()),
            },
            "SOURCES_DIR": {
                "path": CONSTANTS.SOURCES_DIR.resolve(),
@@ -311,19 +340,20 @@ def get_data_locations():
    )


-@cache
-def get_code_locations():
-    from archivebox.config import CONSTANTS
-    from archivebox.config.common import STORAGE_CONFIG
+def get_code_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs):
+    from archivebox.config.constants import CONSTANTS
+    from archivebox.config.common import get_config
+    from archivebox.misc.logging import AttrDict

+    config = config or get_config(**config_kwargs)
    try:
-        lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True) or STORAGE_CONFIG.LIB_DIR
+        lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True, config=config) or config.LIB_DIR
    except Exception:
-        lib_dir = STORAGE_CONFIG.LIB_DIR
+        lib_dir = config.LIB_DIR

    lib_bin_dir = lib_dir / "bin"

-    return benedict(
+    return AttrDict(
        {
            "PACKAGE_DIR": {
                "path": (PACKAGE_DIR).resolve(),
@@ -336,10 +366,9 @@ def get_code_locations():
                "is_valid": os.access(CONSTANTS.STATIC_DIR, os.R_OK) and os.access(CONSTANTS.STATIC_DIR, os.X_OK),  # read + list
            },
            "CUSTOM_TEMPLATES_DIR": {
-                "path": STORAGE_CONFIG.CUSTOM_TEMPLATES_DIR.resolve(),
-                "enabled": os.path.isdir(STORAGE_CONFIG.CUSTOM_TEMPLATES_DIR),
-                "is_valid": os.path.isdir(STORAGE_CONFIG.CUSTOM_TEMPLATES_DIR)
-                and os.access(STORAGE_CONFIG.CUSTOM_TEMPLATES_DIR, os.R_OK),  # read
+                "path": CONSTANTS.CUSTOM_TEMPLATES_DIR.resolve(),
+                "enabled": os.path.isdir(CONSTANTS.CUSTOM_TEMPLATES_DIR),
+                "is_valid": os.path.isdir(CONSTANTS.CUSTOM_TEMPLATES_DIR) and os.access(CONSTANTS.CUSTOM_TEMPLATES_DIR, os.R_OK),  # read
            },
            "USER_PLUGINS_DIR": {
                "path": CONSTANTS.USER_PLUGINS_DIR.resolve(),
@@ -360,117 +389,3 @@ def get_code_locations():
            },
        },
    )
-
-
-# @cache
-# def get_LIB_DIR():
-#     """
-#     - should be shared with other collections on the same host
-#     - must be scoped by CPU architecture, OS family, and archivebox version
-#     - should not be shared with other hosts/archivebox versions
-#     - must be writable by any archivebox user
-#     - should be persistent across reboots
-#     - can be on a docker bin mount but probably shouldnt be
-#     - ok to have a long path (doesnt contain SOCKETS)
-#     """
-#     from .version import detect_installed_version
-
-#     HOST_DIRS = PlatformDirs(appname='archivebox', appauthor='ArchiveBox', version=detect_installed_version(), opinion=True, ensure_exists=False)
-
-#     lib_dir = tempfile.gettempdir()
-#     try:
-#         if 'SYSTEM_LIB_DIR' in os.environ:
-#             lib_dir = Path(os.environ['SYSTEM_LIB_DIR'])
-#         else:
-#             with SudoPermission(uid=ARCHIVEBOX_USER, fallback=True):
-#                 lib_dir = HOST_DIRS.site_data_path
-
-#         # Docker: /usr/local/share/archivebox/0.8.5
-#         # Ubuntu: /usr/local/share/archivebox/0.8.5
-#         # macOS: /Library/Application Support/archivebox
-#         try:
-#             with SudoPermission(uid=0, fallback=True):
-#                 lib_dir.mkdir(parents=True, exist_ok=True)
-#         except PermissionError:
-#             # our user cannot
-#             lib_dir = HOST_DIRS.user_data_path
-#             lib_dir.mkdir(parents=True, exist_ok=True)
-
-#         if IS_ROOT or not dir_is_writable(lib_dir, uid=ARCHIVEBOX_USER):
-#             if IS_ROOT:
-#                 # make sure lib dir is owned by the archivebox user, not root
-#                 with SudoPermission(uid=0):
-#                     if ARCHIVEBOX_USER == 0:
-#                         # print(f'[yellow]:warning:  Warning: Creating SYSTEM_LIB_DIR {lib_dir} with mode 777 so that non-root archivebox users can share it.[/yellow] (caches shared libs used by archivebox for performance)', file=sys.stderr)
-#                         os.system(f'chmod -R 777 "{lib_dir}"')
-#                     else:
-#                         os.system(f'chown {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP} "{lib_dir}"')
-#             else:
-#                 raise PermissionError()
-#     except (PermissionError, AssertionError):
-#         # raise PermissionError(f'SYSTEM_LIB_DIR {lib_dir} is not writable by archivebox user {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}')
-#         print(f'[red]:cross_mark:  ERROR: SYSTEM_LIB_DIR {lib_dir} is not writable by archivebox user {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}[/red]', file=sys.stderr)
-
-#     return lib_dir
-
-# @cache
-# def get_TMP_DIR():
-#     """
-#     - must NOT be inside DATA_DIR / inside a docker volume bind mount
-#     - must NOT have a long PATH (UNIX socket path length restrictions)
-#     - must NOT be shared with other collections/hosts
-#     - must be writable by archivebox user & root
-#     - must be cleared on every boot / not persisted
-#     - must be cleared on every archivebox version upgrade
-#     """
-#     from .version import detect_installed_version
-
-#     HOST_DIRS = PlatformDirs(appname='archivebox', appauthor='ArchiveBox', version=detect_installed_version(), opinion=True, ensure_exists=False)
-
-#     # print('DATA_DIR OWNED BY:', ARCHIVEBOX_USER, ARCHIVEBOX_GROUP)
-#     # print('RUNNING AS:', self.PUID, self.PGID)
-#     run_dir = tempfile.gettempdir()
-#     try:
-#         if 'SYSTEM_TMP_DIR' in os.environ:
-#             run_dir = Path(os.environ['SYSTEM_TMP_DIR']).resolve() / get_collection_id(DATA_DIR=DATA_DIR)
-#             with SudoPermission(uid=0, fallback=True):
-#                 run_dir.mkdir(parents=True, exist_ok=True)
-#             if not dir_is_writable(run_dir, uid=ARCHIVEBOX_USER):
-#                 if IS_ROOT:
-#                     with SudoPermission(uid=0, fallback=False):
-#                         if ARCHIVEBOX_USER == 0:
-#                             # print(f'[yellow]:warning:  Warning: Creating SYSTEM_TMP_DIR {run_dir} with mode 777 so that non-root archivebox users can access it.[/yellow]', file=sys.stderr)
-#                             os.system(f'chmod -R 777 "{run_dir}"')
-#                         else:
-#                             os.system(f'chown {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP} "{run_dir}"')
-#                 else:
-#                     raise PermissionError()
-#             assert len(str(run_dir / 'supervisord.conf')) < 95, 'SYSTEM_TMP_DIR path is too long, please set SYSTEM_TMP_DIR env variable to a shorter path (unfortunately unix requires socket paths be < 108 chars)'
-#             return run_dir
-
-#         run_dir = (HOST_DIRS.site_runtime_path / get_collection_id(DATA_DIR=DATA_DIR)).resolve()
-#         try:
-#             assert len(str(run_dir)) + len('/supervisord.sock') < 95
-#         except AssertionError:
-#             run_dir = Path(tempfile.gettempdir()).resolve() / 'archivebox' / get_collection_id(DATA_DIR=DATA_DIR)
-#             assert len(str(run_dir)) + len('/supervisord.sock') < 95, 'SYSTEM_TMP_DIR path is too long, please set SYSTEM_TMP_DIR env variable to a shorter path (unfortunately unix requires socket paths be < 108 chars)'
-
-#         with SudoPermission(uid=0, fallback=True):
-#             run_dir.mkdir(parents=True, exist_ok=True)
-
-#         if IS_ROOT or not dir_is_writable(run_dir, uid=ARCHIVEBOX_USER):
-#             if IS_ROOT:
-#                 with SudoPermission(uid=0):
-#                     if ARCHIVEBOX_USER == 0:
-#                         # print(f'[yellow]:warning:  Warning: Creating SYSTEM_TMP_DIR {run_dir} with mode 777 so that non-root archivebox users can access it.[/yellow]', file=sys.stderr)
-#                         os.system(f'chmod -R 777 "{run_dir}"')
-#                     else:
-#                         os.system(f'chown {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP} "{run_dir}"')
-#             else:
-#                 raise PermissionError()
-
-#     except (PermissionError, AssertionError):
-#         # raise PermissionError(f'SYSTEM_TMP_DIR {run_dir} is not writable by archivebox user {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}')
-#         print(f'[red]:cross_mark:  ERROR: SYSTEM_TMP_DIR {run_dir} is not writable by archivebox user {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}[/red]', file=sys.stderr)
-
-#     return run_dir
@@ -6,14 +6,13 @@ import importlib.metadata
 from pathlib import Path
 from functools import cache
 from datetime import datetime
+import re

 #############################################################################################

 IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes")

 PACKAGE_DIR: Path = Path(__file__).resolve().parent.parent  # archivebox source code dir
-DATA_DIR: Path = Path(os.environ.get("DATA_DIR", os.getcwd())).resolve()  # archivebox user data dir
-ARCHIVE_DIR: Path = DATA_DIR / "archive"  # archivebox snapshot data dir

 #############################################################################################

@@ -43,11 +42,57 @@ def detect_installed_version(PACKAGE_DIR: Path = PACKAGE_DIR):

@cache
 def get_COMMIT_HASH() -> str | None:
+    for env_var in ("ARCHIVEBOX_COMMIT_HASH", "COMMIT_HASH"):
+        env_commit_hash = os.environ.get(env_var, "").strip()
+        if re.fullmatch(r"[0-9a-fA-F]{40}", env_commit_hash):
+            return env_commit_hash
+
+    if IN_DOCKER:
+        try:
+            version_txt = Path("/VERSION.txt").read_text()
+            docker_commit_hashes = re.findall(r"COMMIT_HASH=([0-9a-fA-F]{40})", version_txt)
+            if docker_commit_hashes:
+                return docker_commit_hashes[-1]
+        except Exception:
+            pass
+
+    def _read_git_file(git_dir: Path, ref: str) -> str | None:
+        try:
+            return git_dir.joinpath(ref).read_text().strip()
+        except Exception:
+            pass
+
+        try:
+            packed_refs = git_dir.joinpath("packed-refs").read_text().splitlines()
+        except Exception:
+            return None
+
+        for line in packed_refs:
+            if line.startswith("#") or line.startswith("^") or not line.strip():
+                continue
+            commit_hash, packed_ref = line.split(" ", 1)
+            if packed_ref == ref:
+                return commit_hash.strip()
+
+        return None
+
    try:
        git_dir = PACKAGE_DIR.parent / ".git"
-        ref = (git_dir / "HEAD").read_text().strip().split(" ")[-1]
-        commit_hash = git_dir.joinpath(ref).read_text().strip()
-        return commit_hash
+        if git_dir.is_file():
+            gitdir_line = git_dir.read_text().strip()
+            gitdir_path = gitdir_line.removeprefix("gitdir:").strip()
+            git_dir = Path(gitdir_path)
+            if not git_dir.is_absolute():
+                git_dir = PACKAGE_DIR.parent / git_dir
+
+        head = (git_dir / "HEAD").read_text().strip()
+        if re.fullmatch(r"[0-9a-fA-F]{40}", head):
+            return head
+
+        ref = head.removeprefix("ref:").strip()
+        commit_hash = _read_git_file(git_dir, ref)
+        if commit_hash:
+            return commit_hash
    except Exception:
        pass

@@ -1,13 +1,9 @@
 __package__ = "archivebox.config"

-import html
-import json
 import os
 import inspect
-import re
 from pathlib import Path
 from typing import Any
-from collections.abc import Callable
 from urllib.parse import quote, urlencode
 from django.http import HttpRequest
 from django.utils import timezone
@@ -22,48 +18,13 @@ from archivebox.misc.util import parse_date

 from archivebox.machine.models import Binary

-ABX_PLUGINS_DOCS_BASE_URL = "https://archivebox.github.io/abx-plugins/"
-ABX_PLUGINS_GITHUB_BASE_URL = "https://github.com/ArchiveBox/abx-plugins/tree/main/abx_plugins/plugins/"
 LIVE_CONFIG_BASE_URL = "/admin/environment/config/"
 ENVIRONMENT_BINARIES_BASE_URL = "/admin/environment/binaries/"
 INSTALLED_BINARIES_BASE_URL = "/admin/machine/binary/"


-# Common binaries to check for
-KNOWN_BINARIES = [
-    "wget",
-    "curl",
-    "chromium",
-    "chrome",
-    "google-chrome",
-    "google-chrome-stable",
-    "node",
-    "npm",
-    "npx",
-    "yt-dlp",
-    "git",
-    "singlefile",
-    "readability-extractor",
-    "mercury-parser",
-    "python3",
-    "python",
-    "bash",
-    "zsh",
-    "ffmpeg",
-    "ripgrep",
-    "rg",
-    "sonic",
-    "archivebox",
-]
-
-CANONICAL_BINARY_ALIASES = {
-    "youtube-dl": "yt-dlp",
-    "ytdlp": "yt-dlp",
-}
-
-
 def is_superuser(request: HttpRequest) -> bool:
-    return bool(getattr(request.user, "is_superuser", False))
+    return bool(request.user.is_superuser)


 def format_parsed_datetime(value: object) -> str:
@@ -71,55 +32,6 @@ def format_parsed_datetime(value: object) -> str:
    return parsed.strftime("%Y-%m-%d %H:%M:%S") if parsed else ""


-JSON_TOKEN_RE = re.compile(
-    r'(?P<key>"(?:\\u[a-fA-F0-9]{4}|\\[^u]|[^\\"])*")(?=\s*:)'
-    r'|(?P<string>"(?:\\u[a-fA-F0-9]{4}|\\[^u]|[^\\"])*")'
-    r"|(?P<boolean>\btrue\b|\bfalse\b)"
-    r"|(?P<null>\bnull\b)"
-    r"|(?P<number>-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?)",
-)
-
-
-def render_code_block(text: str, *, highlighted: bool = False) -> str:
-    code = html.escape(text, quote=False)
-
-    if highlighted:
-
-        def _wrap_token(match: re.Match[str]) -> str:
-            styles = {
-                "key": "color: #0550ae;",
-                "string": "color: #0a7f45;",
-                "boolean": "color: #8250df; font-weight: 600;",
-                "null": "color: #6e7781; font-style: italic;",
-                "number": "color: #b35900;",
-            }
-            token_type = next(name for name, value in match.groupdict().items() if value is not None)
-            return f'<span style="{styles[token_type]}">{match.group(0)}</span>'
-
-        code = JSON_TOKEN_RE.sub(_wrap_token, code)
-
-    return (
-        '<pre style="max-height: 600px; overflow: auto; background: #f6f8fa; '
-        'border: 1px solid #d0d7de; border-radius: 6px; padding: 12px; margin: 0;">'
-        '<code style="font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, '
-        "'Liberation Mono', monospace; white-space: pre; line-height: 1.5;\">"
-        f"{code}"
-        "</code></pre>"
-    )
-
-
-def render_highlighted_json_block(value: Any) -> str:
-    return render_code_block(json.dumps(value, indent=2, ensure_ascii=False), highlighted=True)
-
-
-def get_plugin_docs_url(plugin_name: str) -> str:
-    return f"{ABX_PLUGINS_DOCS_BASE_URL}#{plugin_name}"
-
-
-def get_plugin_hook_source_url(plugin_name: str, hook_name: str) -> str:
-    return f"{ABX_PLUGINS_GITHUB_BASE_URL}{quote(plugin_name)}/{quote(hook_name)}"
-
-
 def get_live_config_url(key: str) -> str:
    return f"{LIVE_CONFIG_BASE_URL}{quote(key)}/"

@@ -128,201 +40,15 @@ def get_environment_binary_url(name: str) -> str:
    return f"{ENVIRONMENT_BINARIES_BASE_URL}{quote(name)}/"


-def get_installed_binary_change_url(name: str, binary: Any) -> str | None:
-    binary_id = getattr(binary, "id", None)
-    if not binary_id:
+def get_installed_binary_change_url(name: str, binary: Binary | None) -> str | None:
+    if binary is None or not binary.id:
        return None

-    base_url = getattr(binary, "admin_change_url", None) or f"{INSTALLED_BINARIES_BASE_URL}{binary_id}/change/"
-    changelist_filters = urlencode({"q": canonical_binary_name(name)})
+    base_url = binary.admin_change_url
+    changelist_filters = urlencode({"q": name})
    return f"{base_url}?{urlencode({'_changelist_filters': changelist_filters})}"


-def get_machine_admin_url() -> str | None:
-    try:
-        from archivebox.machine.models import Machine
-
-        return Machine.current().admin_change_url
-    except Exception:
-        return None
-
-
-def render_code_tag_list(values: list[str]) -> str:
-    if not values:
-        return '<span style="color: #6e7781;">(none)</span>'
-
-    tags = "".join(
-        str(
-            format_html(
-                '<code style="display: inline-block; margin: 0 6px 6px 0; padding: 2px 6px; '
-                'background: #f6f8fa; border: 1px solid #d0d7de; border-radius: 999px;">{}</code>',
-                value,
-            ),
-        )
-        for value in values
-    )
-    return f'<div style="display: flex; flex-wrap: wrap;">{tags}</div>'
-
-
-def render_plugin_metadata_html(config: dict[str, Any]) -> str:
-    rows = (
-        ("Title", config.get("title") or "(none)"),
-        ("Description", config.get("description") or "(none)"),
-        ("Required Plugins", mark_safe(render_link_tag_list(config.get("required_plugins") or [], get_plugin_docs_url))),
-        ("Required Binaries", mark_safe(render_link_tag_list(config.get("required_binaries") or [], get_environment_binary_url))),
-        ("Output MIME Types", mark_safe(render_code_tag_list(config.get("output_mimetypes") or []))),
-    )
-
-    rendered_rows = "".join(
-        str(
-            format_html(
-                '<div style="margin: 0 0 14px 0;"><div style="font-weight: 600; margin-bottom: 4px;">{}</div><div>{}</div></div>',
-                label,
-                value,
-            ),
-        )
-        for label, value in rows
-    )
-    return f'<div style="margin: 4px 0 0 0;">{rendered_rows}</div>'
-
-
-def render_link_tag_list(values: list[str], url_resolver: Callable[[str], str] | None = None) -> str:
-    if not values:
-        return '<span style="color: #6e7781;">(none)</span>'
-
-    tags = []
-    for value in values:
-        if url_resolver is None:
-            tags.append(
-                str(
-                    format_html(
-                        '<code style="display: inline-block; margin: 0 6px 6px 0; padding: 2px 6px; '
-                        'background: #f6f8fa; border: 1px solid #d0d7de; border-radius: 999px;">{}</code>',
-                        value,
-                    ),
-                ),
-            )
-        else:
-            tags.append(
-                str(
-                    format_html(
-                        '<a href="{}" style="text-decoration: none;">'
-                        '<code style="display: inline-block; margin: 0 6px 6px 0; padding: 2px 6px; '
-                        'background: #f6f8fa; border: 1px solid #d0d7de; border-radius: 999px;">{}</code>'
-                        "</a>",
-                        url_resolver(value),
-                        value,
-                    ),
-                ),
-            )
-    return f'<div style="display: flex; flex-wrap: wrap;">{"".join(tags)}</div>'
-
-
-def render_property_links(prop_name: str, prop_info: dict[str, Any], machine_admin_url: str | None) -> str:
-    links = [
-        str(format_html('<a href="{}">Computed value</a>', get_live_config_url(prop_name))),
-    ]
-    if machine_admin_url:
-        links.append(str(format_html('<a href="{}">Edit override</a>', machine_admin_url)))
-
-    fallback = prop_info.get("x-fallback")
-    if isinstance(fallback, str) and fallback:
-        links.append(str(format_html('<a href="{}">Fallback: <code>{}</code></a>', get_live_config_url(fallback), fallback)))
-
-    aliases = prop_info.get("x-aliases") or []
-    if isinstance(aliases, list):
-        for alias in aliases:
-            if isinstance(alias, str) and alias:
-                links.append(str(format_html('<a href="{}">Alias: <code>{}</code></a>', get_live_config_url(alias), alias)))
-
-    default = prop_info.get("default")
-    if prop_name.endswith("_BINARY") and isinstance(default, str) and default:
-        links.append(str(format_html('<a href="{}">Binary: <code>{}</code></a>', get_environment_binary_url(default), default)))
-
-    return " &nbsp; ".join(links)
-
-
-def render_config_properties_html(properties: dict[str, Any], machine_admin_url: str | None) -> str:
-    header_links = [
-        str(format_html('<a href="{}">Dependencies</a>', ENVIRONMENT_BINARIES_BASE_URL)),
-        str(format_html('<a href="{}">Installed Binaries</a>', INSTALLED_BINARIES_BASE_URL)),
-    ]
-    if machine_admin_url:
-        header_links.insert(0, str(format_html('<a href="{}">Machine Config Editor</a>', machine_admin_url)))
-
-    cards = [
-        f'<div style="margin: 0 0 16px 0;">{" &nbsp; | &nbsp; ".join(header_links)}</div>',
-    ]
-
-    for prop_name, prop_info in properties.items():
-        prop_type = prop_info.get("type", "unknown")
-        if isinstance(prop_type, list):
-            prop_type = " | ".join(str(type_name) for type_name in prop_type)
-        prop_desc = prop_info.get("description", "")
-
-        default_html = ""
-        if "default" in prop_info:
-            default_html = str(
-                format_html(
-                    '<div style="margin-top: 6px;"><b>Default:</b> <code>{}</code></div>',
-                    prop_info["default"],
-                ),
-            )
-
-        description_html = prop_desc or mark_safe('<span style="color: #6e7781;">(no description)</span>')
-        cards.append(
-            str(
-                format_html(
-                    '<div style="margin: 0 0 14px 0; padding: 12px; background: #f6f8fa; border: 1px solid #d0d7de; border-radius: 6px;">'
-                    '<div style="margin-bottom: 6px;">'
-                    '<a href="{}" style="font-weight: 600;"><code>{}</code></a>'
-                    ' <span style="color: #6e7781;">({})</span>'
-                    "</div>"
-                    '<div style="margin-bottom: 6px;">{}</div>'
-                    '<div style="font-size: 0.95em;">{}</div>'
-                    "{}"
-                    "</div>",
-                    get_live_config_url(prop_name),
-                    prop_name,
-                    prop_type,
-                    description_html,
-                    mark_safe(render_property_links(prop_name, prop_info, machine_admin_url)),
-                    mark_safe(default_html),
-                ),
-            ),
-        )
-
-    return "".join(cards)
-
-
-def render_hook_links_html(plugin_name: str, hooks: list[str], source: str) -> str:
-    if not hooks:
-        return '<span style="color: #6e7781;">(none)</span>'
-
-    items = []
-    for hook_name in hooks:
-        if source == "builtin":
-            items.append(
-                str(
-                    format_html(
-                        '<div style="margin: 0 0 8px 0;"><a href="{}" target="_blank" rel="noopener noreferrer"><code>{}</code></a></div>',
-                        get_plugin_hook_source_url(plugin_name, hook_name),
-                        hook_name,
-                    ),
-                ),
-            )
-        else:
-            items.append(
-                str(
-                    format_html(
-                        '<div style="margin: 0 0 8px 0;"><code>{}</code></div>',
-                        hook_name,
-                    ),
-                ),
-            )
-    return "".join(items)
-
-
 def render_binary_detail_description(name: str, merged: dict[str, Any], db_binary: Any) -> str:
    installed_binary_url = get_installed_binary_change_url(name, db_binary)

@@ -380,10 +106,6 @@ def obj_to_yaml(obj: Any, indent: int = 0) -> str:
        return f" {str(obj)}"


-def canonical_binary_name(name: str) -> str:
-    return CANONICAL_BINARY_ALIASES.get(name, name)
-
-
 def _binary_sort_key(binary: Binary) -> tuple[int, int, int, Any]:
    return (
        int(binary.status == Binary.StatusChoices.INSTALLED),
@@ -394,68 +116,32 @@ def _binary_sort_key(binary: Binary) -> tuple[int, int, int, Any]:


 def get_db_binaries_by_name() -> dict[str, Binary]:
+    """Group Binary rows by a URL-safe canonical name.
+
+    Hooks occasionally emit ``BinaryEvent.name`` carrying an abspath rather
+    than a short binary name (see ``services/binary_service.py``). That used
+    to leak ``name='/Users/.../bin/foo'`` rows into the DB, which then broke
+    ``/admin/environment/binaries`` because the admin URL regex is
+    ``(?P<key>[^/]+)``. Canonicalize at the keying step so duplicates fold
+    into the real binary and the admin link key stays slash-free regardless
+    of legacy DB state.
+    """
+    from archivebox.machine.models import _canonical_binary_name
+
    grouped: dict[str, list[Binary]] = {}
+    binary_name_aliases = {
+        "youtube-dl": "yt-dlp",
+    }
    for binary in Binary.objects.all():
-        grouped.setdefault(canonical_binary_name(binary.name), []).append(binary)
+        canonical_name = _canonical_binary_name(binary.name)
+        canonical_name = binary_name_aliases.get(canonical_name, canonical_name)
+        if not canonical_name:
+            continue
+        grouped.setdefault(canonical_name, []).append(binary)

    return {name: max(records, key=_binary_sort_key) for name, records in grouped.items()}


-def serialize_binary_record(name: str, binary: Binary | None) -> dict[str, Any]:
-    is_installed = bool(binary and binary.status == Binary.StatusChoices.INSTALLED)
-    return {
-        "name": canonical_binary_name(name),
-        "version": str(getattr(binary, "version", "") or ""),
-        "binprovider": str(getattr(binary, "binprovider", "") or ""),
-        "abspath": str(getattr(binary, "abspath", "") or ""),
-        "sha256": str(getattr(binary, "sha256", "") or ""),
-        "status": str(getattr(binary, "status", "") or ""),
-        "is_available": is_installed and bool(getattr(binary, "abspath", "") or ""),
-    }
-
-
-def get_filesystem_plugins() -> dict[str, dict[str, Any]]:
-    """Discover plugins from filesystem directories."""
-    import json
-    from archivebox.hooks import BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR
-
-    plugins = {}
-
-    for base_dir, source in [(BUILTIN_PLUGINS_DIR, "builtin"), (USER_PLUGINS_DIR, "user")]:
-        if not base_dir.exists():
-            continue
-
-        for plugin_dir in base_dir.iterdir():
-            if plugin_dir.is_dir() and not plugin_dir.name.startswith("_"):
-                plugin_id = f"{source}.{plugin_dir.name}"
-
-                # Find hook scripts
-                hooks = []
-                for ext in ("sh", "py", "js"):
-                    hooks.extend(plugin_dir.glob(f"on_*__*.{ext}"))
-
-                # Load config.json if it exists
-                config_file = plugin_dir / "config.json"
-                config_data = None
-                if config_file.exists():
-                    try:
-                        with open(config_file) as f:
-                            config_data = json.load(f)
-                    except (json.JSONDecodeError, OSError):
-                        config_data = None
-
-                plugins[plugin_id] = {
-                    "id": plugin_id,
-                    "name": plugin_dir.name,
-                    "path": str(plugin_dir),
-                    "source": source,
-                    "hooks": [str(h.name) for h in hooks],
-                    "config": config_data,
-                }
-
-    return plugins
-
-
@render_with_table_view
 def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
    assert is_superuser(request), "Must be a superuser to view configuration settings."
@@ -471,14 +157,15 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
    all_binary_names = sorted(db_binaries.keys())

    for name in all_binary_names:
-        merged = serialize_binary_record(name, db_binaries.get(name))
+        binary = db_binaries.get(name)
+        binary_is_valid = bool(binary and binary.is_valid)

        rows["Binary Name"].append(ItemLink(name, key=name))

-        if merged["is_available"]:
-            rows["Found Version"].append(f"✅ {merged['version']}" if merged["version"] else "✅ found")
-            rows["Provided By"].append(merged["binprovider"] or "-")
-            rows["Found Abspath"].append(merged["abspath"] or "-")
+        if binary_is_valid:
+            rows["Found Version"].append(f"✅ {binary.version}" if binary.version else "✅ found")
+            rows["Provided By"].append(binary.binprovider or "-")
+            rows["Found Abspath"].append(binary.abspath or "-")
        else:
            rows["Found Version"].append("❌ missing")
            rows["Provided By"].append("-")
@@ -493,22 +180,24 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
@render_with_item_view
 def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
    assert is_superuser(request), "Must be a superuser to view configuration settings."
-    key = canonical_binary_name(key)

+    key = {
+        "youtube-dl": "yt-dlp",
+    }.get(key, key)
    db_binary = get_db_binaries_by_name().get(key)
-    merged = serialize_binary_record(key, db_binary)
-
-    if merged["is_available"]:
+    binary_is_valid = bool(db_binary and db_binary.is_valid)
+    if binary_is_valid:
+        binary_data = db_binary.to_json()
        section: SectionData = {
            "name": key,
-            "description": mark_safe(render_binary_detail_description(key, merged, db_binary)),
+            "description": mark_safe(render_binary_detail_description(key, binary_data, db_binary)),
            "fields": {
                "name": key,
-                "binprovider": merged["binprovider"] or "-",
-                "abspath": merged["abspath"] or "not found",
-                "version": merged["version"] or "unknown",
-                "sha256": merged["sha256"],
-                "status": merged["status"],
+                "binprovider": db_binary.binprovider or "-",
+                "abspath": db_binary.abspath or "not found",
+                "version": db_binary.version or "unknown",
+                "sha256": db_binary.sha256,
+                "status": db_binary.status,
            },
            "help_texts": {},
        }
@@ -523,10 +212,10 @@ def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
        "description": "No persisted Binary record found",
        "fields": {
            "name": key,
-            "binprovider": merged["binprovider"] or "not recorded",
-            "abspath": merged["abspath"] or "not recorded",
-            "version": merged["version"] or "N/A",
-            "status": merged["status"] or "unrecorded",
+            "binprovider": db_binary.binprovider if db_binary else "not recorded",
+            "abspath": db_binary.abspath if db_binary else "not recorded",
+            "version": db_binary.version if db_binary else "N/A",
+            "status": db_binary.status if db_binary else "unrecorded",
        },
        "help_texts": {},
    }
@@ -537,137 +226,13 @@ def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
    )


-@render_with_table_view
-def plugins_list_view(request: HttpRequest, **kwargs) -> TableContext:
-    assert is_superuser(request), "Must be a superuser to view configuration settings."
-
-    rows = {
-        "Name": [],
-        "Source": [],
-        "Path": [],
-        "Hooks": [],
-        "Config": [],
-    }
-
-    plugins = get_filesystem_plugins()
-
-    for plugin_id, plugin in plugins.items():
-        rows["Name"].append(ItemLink(plugin["name"], key=plugin_id))
-        rows["Source"].append(plugin["source"])
-        rows["Path"].append(format_html("<code>{}</code>", plugin["path"]))
-        rows["Hooks"].append(", ".join(plugin["hooks"]) or "(none)")
-
-        # Show config status
-        if plugin.get("config"):
-            config_properties = plugin["config"].get("properties", {})
-            config_count = len(config_properties)
-            rows["Config"].append(f"✅ {config_count} properties" if config_count > 0 else "✅ present")
-        else:
-            rows["Config"].append("❌ none")
-
-    if not plugins:
-        # Show a helpful message when no plugins found
-        rows["Name"].append("(no plugins found)")
-        rows["Source"].append("-")
-        rows["Path"].append(mark_safe("<code>abx_plugins/plugins/</code> or <code>data/custom_plugins/</code>"))
-        rows["Hooks"].append("-")
-        rows["Config"].append("-")
-
-    return TableContext(
-        title="Installed plugins",
-        table=rows,
-    )
-
-
-@render_with_item_view
-def plugin_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
-    assert is_superuser(request), "Must be a superuser to view configuration settings."
-
-    plugins = get_filesystem_plugins()
-
-    plugin = plugins.get(key)
-    if not plugin:
-        return ItemContext(
-            slug=key,
-            title=f"Plugin not found: {key}",
-            data=[],
-        )
-
-    # Base fields that all plugins have
-    docs_url = get_plugin_docs_url(plugin["name"])
-    machine_admin_url = get_machine_admin_url()
-    fields = {
-        "id": plugin["id"],
-        "name": plugin["name"],
-        "source": plugin["source"],
-    }
-
-    sections: list[SectionData] = [
-        {
-            "name": plugin["name"],
-            "description": format_html(
-                '<code>{}</code><br/><a href="{}" target="_blank" rel="noopener noreferrer">ABX Plugin Docs</a>',
-                plugin["path"],
-                docs_url,
-            ),
-            "fields": fields,
-            "help_texts": {},
-        },
-    ]
-
-    if plugin["hooks"]:
-        sections.append(
-            {
-                "name": "Hooks",
-                "description": mark_safe(render_hook_links_html(plugin["name"], plugin["hooks"], plugin["source"])),
-                "fields": {},
-                "help_texts": {},
-            },
-        )
-
-    if plugin.get("config"):
-        sections.append(
-            {
-                "name": "Plugin Metadata",
-                "description": mark_safe(render_plugin_metadata_html(plugin["config"])),
-                "fields": {},
-                "help_texts": {},
-            },
-        )
-
-        sections.append(
-            {
-                "name": "config.json",
-                "description": mark_safe(render_highlighted_json_block(plugin["config"])),
-                "fields": {},
-                "help_texts": {},
-            },
-        )
-
-        config_properties = plugin["config"].get("properties", {})
-        if config_properties:
-            sections.append(
-                {
-                    "name": "Config Properties",
-                    "description": mark_safe(render_config_properties_html(config_properties, machine_admin_url)),
-                    "fields": {},
-                    "help_texts": {},
-                },
-            )
-
-    return ItemContext(
-        slug=key,
-        title=plugin["name"],
-        data=sections,
-    )
-
-
@render_with_table_view
 def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
    assert is_superuser(request), "Must be a superuser to view configuration settings."

    rows = {
        "Name": [],
+        "Type": [],
        "State": [],
        "PID": [],
        "Started": [],
@@ -697,12 +262,115 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
            continue
        all_config[config_name] = config_data

-    # Add top row for supervisord process manager
+    # Collect every PID we plan to show so we can resolve them to Process rows
+    # in a single query. supervisord's per-worker description carries the pid
+    # in the form ``pid 12345, uptime 0:01:23`` (or just the bare ``pid``
+    # placeholder when stopped); we ignore non-numeric values.
+    process_items = supervisor.getAllProcessInfo()
+    if not isinstance(process_items, list):
+        process_items = []
+
+    def _parse_worker_pid_and_uptime(description: str) -> tuple[int | None, str]:
+        body = description.replace("pid ", "", 1)
+        pid_part, _, uptime_part = body.partition(", uptime ")
+        try:
+            return int(pid_part.strip()), uptime_part.strip()
+        except ValueError:
+            return None, ""
+
+    pids: set[int] = set()
+    supervisor_pid = supervisor.getPID()
+    if isinstance(supervisor_pid, int):
+        pids.add(supervisor_pid)
+    for proc_data in process_items:
+        if not isinstance(proc_data, dict):
+            continue
+        pid_int, _ = _parse_worker_pid_and_uptime(str(proc_data.get("description") or ""))
+        if pid_int is not None:
+            pids.add(pid_int)
+
+    pid_to_process_id: dict[int, str] = {}
+    pid_to_process_type: dict[int, str] = {}
+    if pids:
+        try:
+            from archivebox.machine.models import Machine, Process
+
+            for row in (
+                Process.objects.filter(machine=Machine.current(), pid__in=pids)
+                .order_by("pid", "-started_at", "-created_at")
+                .only("id", "pid", "process_type")
+            ):
+                if row.pid in pid_to_process_id:
+                    continue  # keep the most recent row per PID
+                pid_to_process_id[row.pid] = str(row.id)
+                pid_to_process_type[row.pid] = row.process_type
+
+        except Exception:
+            pass
+
+    def _pid_cell(pid_value: int | None, uptime_str: str = ""):
+        if pid_value is None:
+            return ""
+        pid_text = str(pid_value)
+        process_id = pid_to_process_id.get(pid_value)
+        if process_id:
+            link = format_html('<a href="/admin/machine/process/{}/change/">{}</a>', process_id, pid_text)
+        else:
+            link = format_html("{}", pid_text)
+        if uptime_str:
+            return format_html("{}, uptime {}", link, uptime_str)
+        return link
+
+    # Add top row for supervisord process manager. supervisord exposes its
+    # state + pid over XML-RPC but not its own start time / exit status / uptime,
+    # so we read those from the OS process (or fall back to the Process row
+    # recorded in _record_supervisord_process). Exit status stays blank while
+    # it's RUNNING — supervisord wouldn't be answering RPC if it had exited.
    rows["Name"].append(ItemLink("supervisord", key="supervisord"))
+    rows["Type"].append("supervisord")
    supervisor_state = supervisor.getState()
-    rows["State"].append(str(supervisor_state.get("statename") if isinstance(supervisor_state, dict) else ""))
-    rows["PID"].append(str(supervisor.getPID()))
-    rows["Started"].append("-")
+    state_name = str(supervisor_state.get("statename") if isinstance(supervisor_state, dict) else "")
+    rows["State"].append(state_name)
+
+    supervisor_started = ""
+    supervisor_uptime = ""
+    try:
+        import time as _time
+
+        import psutil
+
+        ps_proc = psutil.Process(supervisor_pid)
+        create_time = ps_proc.create_time()
+        supervisor_started = format_parsed_datetime(create_time)
+        seconds = max(int(_time.time() - create_time), 0)
+        hours, remainder = divmod(seconds, 3600)
+        minutes, secs = divmod(remainder, 60)
+        supervisor_uptime = f"{hours}:{minutes:02d}:{secs:02d}"
+    except Exception:
+        try:
+            from archivebox.machine.models import Machine, Process
+
+            row = (
+                Process.objects.filter(
+                    machine=Machine.current(),
+                    process_type=Process.TypeChoices.SUPERVISORD,
+                    pid=supervisor_pid,
+                )
+                .order_by("-started_at")
+                .first()
+            )
+            if row and row.started_at:
+                supervisor_started = row.started_at.strftime("%Y-%m-%d %H:%M:%S")
+                seconds = max(int((timezone.now() - row.started_at).total_seconds()), 0)
+                hours, remainder = divmod(seconds, 3600)
+                minutes, secs = divmod(remainder, 60)
+                supervisor_uptime = f"{hours}:{minutes:02d}:{secs:02d}"
+        except Exception:
+            pass
+
+    rows["PID"].append(_pid_cell(supervisor_pid if isinstance(supervisor_pid, int) else None, supervisor_uptime))
+    rows["Started"].append(supervisor_started or "-")
+
    rows["Command"].append("supervisord --configuration=tmp/supervisord.conf")
    rows["Logfile"].append(
        format_html(
@@ -711,12 +379,9 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
            "logs/supervisord.log",
        ),
    )
-    rows["Exit Status"].append("0")
+    rows["Exit Status"].append("" if state_name == "RUNNING" else "-")

    # Add a row for each worker process managed by supervisord
-    process_items = supervisor.getAllProcessInfo()
-    if not isinstance(process_items, list):
-        process_items = []
    for proc_data in process_items:
        if not isinstance(proc_data, dict):
            continue
@@ -725,10 +390,15 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
        proc_start = proc_data.get("start")
        proc_logfile = str(proc_data.get("stdout_logfile") or "")
        proc_config = all_config.get(proc_name, {})
+        pid_int, uptime_str = _parse_worker_pid_and_uptime(proc_description)

        rows["Name"].append(ItemLink(proc_name, key=proc_name))
+        # Prefer the Process row's process_type when we have one (e.g. "worker",
+        # "hook"); otherwise fall back to the generic "worker" label since
+        # everything in this loop is supervisord-managed.
+        rows["Type"].append(pid_to_process_type.get(pid_int, "worker") if pid_int else "worker")
        rows["State"].append(str(proc_data.get("statename") or ""))
-        rows["PID"].append(proc_description.replace("pid ", ""))
+        rows["PID"].append(_pid_cell(pid_int, uptime_str))
        rows["Started"].append(format_parsed_datetime(proc_start))
        rows["Command"].append(str(proc_config.get("command") or ""))
        rows["Logfile"].append(
@@ -851,7 +521,7 @@ def log_list_view(request: HttpRequest, **kwargs) -> TableContext:
                f.seek(0)
            last_lines = f.read().decode("utf-8", errors="replace").split("\n")
            non_empty_lines = [line for line in last_lines if line.strip()]
-            rows["Most Recent Lines"].append(non_empty_lines[-1])
+            rows["Most Recent Lines"].append(non_empty_lines[-1] if non_empty_lines else "")

    return TableContext(
        title="Debug Log files",
@@ -7,23 +7,3 @@ def register_admin(admin_site):
    from archivebox.core.admin import register_admin as do_register

    do_register(admin_site)
-
-
-def get_CONFIG():
-    from archivebox.config.common import (
-        SHELL_CONFIG,
-        STORAGE_CONFIG,
-        GENERAL_CONFIG,
-        SERVER_CONFIG,
-        ARCHIVING_CONFIG,
-        SEARCH_BACKEND_CONFIG,
-    )
-
-    return {
-        "SHELL_CONFIG": SHELL_CONFIG,
-        "STORAGE_CONFIG": STORAGE_CONFIG,
-        "GENERAL_CONFIG": GENERAL_CONFIG,
-        "SERVER_CONFIG": SERVER_CONFIG,
-        "ARCHIVING_CONFIG": ARCHIVING_CONFIG,
-        "SEARCHBACKEND_CONFIG": SEARCH_BACKEND_CONFIG,
-    }
@@ -10,7 +10,7 @@ from functools import reduce
 from operator import and_

 from django.contrib import admin
-from django.db.models import Min, Q, TextField
+from django.db.models import Min, Prefetch, Q, TextField
 from django.db.models.functions import Cast
 from django.utils.html import format_html
 from django.utils.safestring import mark_safe
@@ -19,51 +19,48 @@ from django.urls import reverse, resolve
 from django.utils import timezone
 from django.utils.text import smart_split

-from archivebox.config import DATA_DIR
-from archivebox.config.common import SERVER_CONFIG
 from archivebox.misc.paginators import AcceleratedPaginator
 from archivebox.base_models.admin import BaseModelAdmin
-from archivebox.hooks import get_plugin_icon
-from archivebox.core.host_utils import build_snapshot_url
+from archivebox.plugins.discovery import get_plugin_icon
+from archivebox.plugins.views import LIVE_PLUGIN_BASE_URL
+from archivebox.core.routes_util import build_snapshot_url
 from archivebox.core.widgets import InlineTagEditorWidget
-from archivebox.core.views import LIVE_PLUGIN_BASE_URL
-from archivebox.machine.env_utils import env_to_shell_exports
+from archivebox.machine.env_util import env_to_shell_exports


 from archivebox.core.models import ArchiveResult, Snapshot


-def _quote_shell_string(value: str) -> str:
-    return "'" + str(value).replace("'", "'\"'\"'") + "'"
-
-
 def _get_replay_source_url(result: ArchiveResult) -> str:
-    process_env = getattr(getattr(result, "process", None), "env", None) or {}
-    return str(process_env.get("SOURCE_URL") or result.snapshot.url or "")
+    process = result.process_record
+    return str((process.url if process else None) or result.snapshot.url or "")


 def build_abx_dl_display_command(result: ArchiveResult) -> str:
    source_url = _get_replay_source_url(result)
    plugin_name = str(result.plugin or "").strip()
-    if not plugin_name and not source_url:
-        return "abx-dl"
-    if not source_url:
-        return f"abx-dl --plugins={plugin_name}"
-    return f"abx-dl --plugins={plugin_name} {_quote_shell_string(source_url)}"
+    cmd = ["abx-dl"]
+    if plugin_name:
+        cmd.append(f"--plugins={plugin_name}")
+    if source_url:
+        cmd.append(source_url)
+    return shlex.join(cmd)


-def build_abx_dl_replay_command(result: ArchiveResult) -> str:
+def build_abx_dl_replay_command(result: ArchiveResult, config=None) -> str:
    display_command = build_abx_dl_display_command(result)
-    process = getattr(result, "process", None)
-    env_items = env_to_shell_exports(getattr(process, "env", None) or {})
-    snapshot_dir = shlex.quote(str(result.snapshot_dir))
+    process = result.process
+    env_items = env_to_shell_exports(process.env if process else {})
+    if config is not None:
+        result.snapshot._runtime_config = config
+    snapshot_dir = shlex.quote(str(result.pwd or result.snapshot_dir))
    if env_items:
        return f"cd {snapshot_dir}; env {env_items} {display_command}"
    return f"cd {snapshot_dir}; {display_command}"


 def get_plugin_admin_url(plugin_name: str) -> str:
-    from archivebox.hooks import BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR, iter_plugin_dirs
+    from archivebox.plugins.discovery import BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR, iter_plugin_dirs

    plugin_dir = next((path.resolve() for path in iter_plugin_dirs() if path.name == plugin_name), None)
    if plugin_dir:
@@ -78,7 +75,13 @@ def get_plugin_admin_url(plugin_name: str) -> str:
    return f"{LIVE_PLUGIN_BASE_URL}builtin.{quote(plugin_name)}/"


-def render_archiveresults_list(archiveresults_qs, limit=50):
+def get_process_link_label(process) -> str:
+    if process.pid:
+        return str(process.pid)
+    return str(process.id)[-8:]
+
+
+def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
    """Render a nice inline list view of archive results with status, plugin, output, and actions."""

    result_ids = list(archiveresults_qs.order_by("plugin").values_list("pk", flat=True)[:limit])
@@ -87,7 +90,13 @@ def render_archiveresults_list(archiveresults_qs, limit=50):

    results_by_id = {
        result.pk: result
-        for result in ArchiveResult.objects.filter(pk__in=result_ids).select_related("snapshot", "process", "process__machine")
+        for result in ArchiveResult.objects.filter(pk__in=result_ids).select_related(
+            "snapshot",
+            "snapshot__crawl",
+            "snapshot__crawl__created_by",
+            "process",
+            "process__machine",
+        )
    }
    results = [results_by_id[result_id] for result_id in result_ids if result_id in results_by_id]

@@ -100,6 +109,7 @@ def render_archiveresults_list(archiveresults_qs, limit=50):
        "failed": ("#991b1b", "#fee2e2"),  # red
        "queued": ("#6b7280", "#f3f4f6"),  # gray
        "started": ("#92400e", "#fef3c7"),  # amber
+        "paused": ("#1d4ed8", "#dbeafe"),  # blue
        "backoff": ("#92400e", "#fef3c7"),
        "skipped": ("#475569", "#f1f5f9"),
        "noresults": ("#475569", "#f1f5f9"),
@@ -129,40 +139,41 @@ def render_archiveresults_list(archiveresults_qs, limit=50):
        # Format timestamp
        end_time = result.end_ts.strftime("%Y-%m-%d %H:%M:%S") if result.end_ts else "-"

+        process = result.process_record
        process_display = "-"
-        if result.process_id and result.process:
+        if process:
            process_display = f'''
-                <a href="{reverse("admin:machine_process_change", args=[result.process_id])}"
+                <a href="{reverse("admin:machine_process_change", args=[process.id])}"
                   style="color: #2563eb; text-decoration: none; font-family: ui-monospace, monospace; font-size: 12px;"
-                   title="View process">{result.process.pid or "-"}</a>
+                   title="View process">{get_process_link_label(process)}</a>
            '''

        machine_display = "-"
-        if result.process_id and result.process and result.process.machine_id:
+        if process and process.machine_id:
            machine_display = f'''
-                <a href="{reverse("admin:machine_machine_change", args=[result.process.machine_id])}"
+                <a href="{reverse("admin:machine_machine_change", args=[process.machine_id])}"
                   style="color: #2563eb; text-decoration: none; font-size: 12px;"
-                   title="View machine">{result.process.machine.hostname}</a>
+                   title="View machine">{process.machine.hostname}</a>
            '''

        # Truncate output for display
-        full_output = result.output_str or "-"
+        full_output = result.output_str_for_display() or "-"
        output_display = full_output[:60]
        if len(full_output) > 60:
            output_display += "..."

        display_cmd = build_abx_dl_display_command(result)
-        replay_cmd = build_abx_dl_replay_command(result)
+        replay_cmd = build_abx_dl_replay_command(result, config=config)
        cmd_str_escaped = html.escape(display_cmd)
        cmd_attr = html.escape(replay_cmd, quote=True)

        # Build output link - use embed_path() which checks output_files first
-        embed_path = result.embed_path() if hasattr(result, "embed_path") else None
-        snapshot_id = str(getattr(result, "snapshot_id", ""))
+        embed_path = result.embed_path()
+        snapshot_id = str(result.snapshot_id)
        if embed_path and result.status == "succeeded":
-            output_link = build_snapshot_url(snapshot_id, embed_path)
+            output_link = build_snapshot_url(snapshot_id, embed_path, config=config)
        else:
-            output_link = build_snapshot_url(snapshot_id, "")
+            output_link = build_snapshot_url(snapshot_id, "", config=config)

        # Get version - try cmd_version field
        version = result.cmd_version if result.cmd_version else "-"
@@ -347,8 +358,7 @@ class ArchiveResultInline(admin.TabularInline):
    def get_formset(self, request, obj=None, **kwargs):
        formset = super().get_formset(request, obj, **kwargs)
        snapshot = self.get_parent_object_from_request(request)
-        form_class = getattr(formset, "form", None)
-        base_fields = getattr(form_class, "base_fields", {})
+        base_fields = formset.form.base_fields
        snapshot_output_dir = str(snapshot.output_dir) if snapshot else ""

        # import ipdb; ipdb.set_trace()
@@ -380,6 +390,7 @@ class ArchiveResultInline(admin.TabularInline):


 class ArchiveResultAdmin(BaseModelAdmin):
+    list_select_related = ()
    list_display = (
        "details_link",
        "zip_link",
@@ -469,10 +480,11 @@ class ArchiveResultAdmin(BaseModelAdmin):

    list_filter = ("status", "plugin", "start_ts")
    ordering = ["-start_ts"]
-    list_per_page = SERVER_CONFIG.SNAPSHOTS_PER_PAGE
+    list_per_page = 50

    paginator = AcceleratedPaginator
    save_on_top = True
+    show_full_result_count = False

    actions = ["delete_selected"]

@@ -486,16 +498,45 @@ class ArchiveResultAdmin(BaseModelAdmin):

    def changelist_view(self, request, extra_context=None):
        self.request = request
-        return super().changelist_view(request, extra_context)
+        saved_list_per_page = self.list_per_page
+        self.list_per_page = request.archivebox_config.SNAPSHOTS_PER_PAGE
+        try:
+            return super().changelist_view(request, extra_context)
+        finally:
+            self.list_per_page = saved_list_per_page

    def get_queryset(self, request):
-        return (
+        ordering = request.GET.get("o")
+        ordering_fields = set()
+        if ordering:
+            for part in ordering.split("."):
+                if not part:
+                    continue
+                try:
+                    idx = abs(int(part)) - 1
+                except ValueError:
+                    continue
+                if 0 <= idx < len(self.list_display):
+                    ordering_fields.add(self.list_display[idx])
+
+        qs = (
            super()
            .get_queryset(request)
-            .select_related("snapshot", "process")
-            .prefetch_related("snapshot__tags")
-            .annotate(snapshot_first_tag=Min("snapshot__tags__name"))
+            .defer(
+                "notes",
+                "output_json",
+            )
+            .prefetch_related(
+                Prefetch(
+                    "snapshot",
+                    queryset=Snapshot.objects.defer("config", "notes").prefetch_related("crawl__created_by", "tags"),
+                ),
+                "process__machine",
+            )
        )
+        if "tags_inline" in ordering_fields:
+            qs = qs.annotate(snapshot_first_tag=Min("snapshot__tags__name"))
+        return qs

    def get_search_results(self, request, queryset, search_term):
        if not search_term:
@@ -532,16 +573,20 @@ class ArchiveResultAdmin(BaseModelAdmin):
        return queryset.filter(reduce(and_, filters)).distinct(), True

    def get_snapshot_view_url(self, result: ArchiveResult) -> str:
-        return build_snapshot_url(str(result.snapshot_id), request=getattr(self, "request", None))
+        request = self.request
+        return build_snapshot_url(str(result.snapshot_id), request=request, config=request.archivebox_config)

    def get_output_view_url(self, result: ArchiveResult) -> str:
-        output_path = result.embed_path() if hasattr(result, "embed_path") else None
+        request = self.request
+        config = request.archivebox_config
+        output_path = result.embed_path()
        if not output_path:
            output_path = result.plugin or ""
-        return build_snapshot_url(str(result.snapshot_id), output_path, request=getattr(self, "request", None))
+        return build_snapshot_url(str(result.snapshot_id), output_path, request=request, config=config)

    def get_output_files_url(self, result: ArchiveResult) -> str:
-        return f"{build_snapshot_url(str(result.snapshot_id), result.plugin, request=getattr(self, 'request', None))}/?files=1"
+        request = self.request
+        return f"{build_snapshot_url(str(result.snapshot_id), result.plugin, request=request, config=request.archivebox_config)}/?files=1"

    def get_output_zip_url(self, result: ArchiveResult) -> str:
        return f"{self.get_output_files_url(result)}&download=zip"
@@ -567,9 +612,10 @@ class ArchiveResultAdmin(BaseModelAdmin):
    )
    def snapshot_info(self, result):
        snapshot_id = str(result.snapshot_id)
+        request = self.request
        return format_html(
            '<a href="{}"><b><code>[{}]</code></b> &nbsp; {} &nbsp; {}</a><br/>',
-            build_snapshot_url(snapshot_id, "index.html"),
+            build_snapshot_url(snapshot_id, "index.html", request=request, config=request.archivebox_config),
            snapshot_id[:8],
            result.snapshot.bookmarked_at.strftime("%Y-%m-%d %H:%M"),
            result.snapshot.url[:128],
@@ -616,31 +662,33 @@ class ArchiveResultAdmin(BaseModelAdmin):

    @admin.display(description="Process", ordering="process__pid")
    def process_link(self, result):
-        if not result.process_id:
+        process = result.process_record
+        if not process:
            return "-"
-        process_label = result.process.pid if result.process and result.process.pid else "-"
+        process_label = get_process_link_label(process)
        return format_html(
            '<a href="{}"><code>{}</code></a>',
-            reverse("admin:machine_process_change", args=[result.process_id]),
+            reverse("admin:machine_process_change", args=[process.id]),
            process_label,
        )

    @admin.display(description="Machine", ordering="process__machine__hostname")
    def machine_link(self, result):
-        if not result.process_id or not result.process or not result.process.machine_id:
+        process = result.process_record
+        if not process or not process.machine_id:
            return "-"
-        machine = result.process.machine
+        machine = process.machine
        return format_html(
-            '<a href="{}"><code>{}</code> {}</a>',
+            '<a href="{}">{}</a>',
            reverse("admin:machine_machine_change", args=[machine.id]),
-            str(machine.id)[:8],
            machine.hostname,
        )

    @admin.display(description="Command")
    def cmd_str(self, result):
+        request = self.request
        display_cmd = build_abx_dl_display_command(result)
-        replay_cmd = build_abx_dl_replay_command(result)
+        replay_cmd = build_abx_dl_replay_command(result, config=request.archivebox_config)
        return format_html(
            """
            <div style="position: relative; width: 100%; max-width: 100%; overflow: hidden; box-sizing: border-box;">
@@ -661,27 +709,30 @@ class ArchiveResultAdmin(BaseModelAdmin):
        )

    def output_display(self, result):
+        request = self.request
+        config = request.archivebox_config
        # Determine output link path - use embed_path() which checks output_files
-        embed_path = result.embed_path() if hasattr(result, "embed_path") else None
+        embed_path = result.embed_path()
        output_path = embed_path if (result.status == "succeeded" and embed_path) else "index.html"
        snapshot_id = str(result.snapshot_id)
        return format_html(
            '<a href="{}" class="output-link">↗️</a><pre>{}</pre>',
-            build_snapshot_url(snapshot_id, output_path),
-            result.output_str,
+            build_snapshot_url(snapshot_id, output_path, request=request, config=config),
+            result.output_str_for_display(),
        )

    @admin.display(description="Output", ordering="output_str")
    def output_str_display(self, result):
-        output_text = str(result.output_str or "").strip()
+        output_text = str(result.output_str_for_display() or "").strip()
        if not output_text:
            return "-"

-        live_path = result.embed_path() if hasattr(result, "embed_path") else None
+        request = self.request
+        live_path = result.embed_path()
        if live_path:
            return format_html(
                '<a href="{}" title="{}"><code>{}</code></a>',
-                build_snapshot_url(str(result.snapshot_id), live_path),
+                build_snapshot_url(str(result.snapshot_id), live_path, request=request, config=request.archivebox_config),
                output_text,
                output_text,
            )
@@ -733,17 +784,18 @@ class ArchiveResultAdmin(BaseModelAdmin):
        )

    def output_summary(self, result):
-        snapshot_dir = Path(DATA_DIR) / str(result.pwd).split("data/", 1)[-1]
+        snapshot_dir = Path(result.snapshot.output_dir)
        output_html = format_html(
            '<pre style="display: inline-block">{}</pre><br/>',
-            result.output_str,
+            result.output_str_for_display(),
        )
        snapshot_id = str(result.snapshot_id)
+        request = self.request
        output_html += format_html(
            '<a href="{}#all">See result files ...</a><br/><pre><code>',
-            build_snapshot_url(snapshot_id, "index.html"),
+            build_snapshot_url(snapshot_id, "index.html", request=request, config=request.archivebox_config),
        )
-        embed_path = result.embed_path() if hasattr(result, "embed_path") else ""
+        embed_path = result.embed_path() or ""
        path_from_embed = snapshot_dir / (embed_path or "")
        output_html += format_html(
            '<i style="padding: 1px">{}</i><b style="padding-right: 20px">/</b><i>{}</i><br/><hr/>',
@@ -3,12 +3,16 @@ __package__ = "archivebox.core"
 from typing import TYPE_CHECKING, Any

 from django.contrib import admin
+from django.db import DatabaseError, connection
 from admin_data_views.admin import (
    admin_data_index_view as adv_admin_data_index_view,
    get_admin_data_urls as adv_get_admin_data_urls,
    get_app_list as adv_get_app_list,
 )

+from archivebox.config import VERSION
+from archivebox.config.version import get_COMMIT_HASH
+
 if TYPE_CHECKING:
    from django.http import HttpRequest
    from django.template.response import TemplateResponse
@@ -23,6 +27,43 @@ class ArchiveBoxAdmin(admin.AdminSite):
    site_title = "Admin"
    namespace = "admin"

+    def each_context(self, request: "HttpRequest") -> dict[str, Any]:
+        context = super().each_context(request)
+        context["VERSION"] = VERSION
+        context["STATIC_CACHE_KEY"] = (get_COMMIT_HASH() or VERSION or "dev").strip()
+        return context
+
+    @staticmethod
+    def _format_object_count(count: int) -> tuple[int, str, str]:
+        if count >= 1_000_000_000:
+            count_label = f"{count / 1_000_000_000:.1f}B"
+        elif count >= 1_000_000:
+            count_label = f"{count / 1_000_000:.1f}M"
+        elif count >= 1_000:
+            count_label = f"{count / 1_000:.1f}K"
+        else:
+            count_label = f"{count:,}"
+        count_label = count_label.replace(".0", "")
+        return count, count_label, f"Object count: {count:,}"
+
+    def _set_model_object_count(
+        self,
+        models_by_table: dict[str, list[dict[str, Any]]],
+        table: str,
+        count: int,
+        title: str | None = None,
+    ) -> None:
+        models = models_by_table.get(table)
+        if not models:
+            return
+        count, count_label, count_title = self._format_object_count(count)
+        if title:
+            count_title = title
+        for model in models:
+            model["object_count"] = count
+            model["object_count_label"] = count_label
+            model["object_count_title"] = count_title
+
    def get_app_list(self, request: "HttpRequest", app_label: str | None = None) -> list["AppDict"]:
        if app_label is None:
            return adv_get_app_list(self, request)
@@ -31,6 +72,51 @@ class ArchiveBoxAdmin(admin.AdminSite):
    def admin_data_index_view(self, request: "HttpRequest", **kwargs: Any) -> "TemplateResponse":
        return adv_admin_data_index_view(self, request, **kwargs)

+    def index(self, request: "HttpRequest", extra_context: dict[str, Any] | None = None) -> "TemplateResponse":
+        response = super().index(request, extra_context)
+        if connection.vendor != "sqlite":
+            return response
+
+        models_by_table: dict[str, list[dict[str, Any]]] = {}
+        for app in response.context_data.get("app_list", []):
+            for model in app.get("models", []):
+                model_class = model.get("model")
+                if not model_class or not model.get("perms", {}).get("view"):
+                    continue
+                models_by_table.setdefault(model_class._meta.db_table, []).append(model)
+
+        if not models_by_table:
+            return response
+
+        try:
+            with connection.cursor() as cursor:
+                cursor.execute("SELECT tbl, stat FROM sqlite_stat1")
+                for table, stat in cursor.fetchall():
+                    try:
+                        count = int(str(stat).split()[0])
+                    except (IndexError, TypeError, ValueError):
+                        continue
+                    self._set_model_object_count(
+                        models_by_table,
+                        table,
+                        count,
+                        title=f"Approximate count from SQLite stats: {count:,}",
+                    )
+                    models_by_table.pop(table, None)
+        except DatabaseError:
+            pass
+
+        for table in list(models_by_table):
+            try:
+                with connection.cursor() as cursor:
+                    cursor.execute(f"SELECT COUNT(*) FROM {connection.ops.quote_name(table)}")
+                    count = int(cursor.fetchone()[0])
+            except DatabaseError:
+                continue
+            self._set_model_object_count(models_by_table, table, count)
+            models_by_table.pop(table, None)
+        return response
+
    def get_admin_data_urls(self) -> list["URLResolver | URLPattern"]:
        return adv_get_admin_data_urls(self)

@@ -6,13 +6,14 @@ from django import forms
 from django.contrib import admin, messages
 from django.contrib.admin.options import IS_POPUP_VAR
 from django.http import HttpRequest, HttpResponseRedirect
+from django.template.response import TemplateResponse
 from django.urls import reverse
 from django.utils.html import format_html
 from django.utils.safestring import mark_safe

 from archivebox.base_models.admin import BaseModelAdmin
 from archivebox.core.models import SnapshotTag, Tag
-from archivebox.core.tag_utils import (
+from archivebox.core.tag_util import (
    TAG_HAS_SNAPSHOTS_CHOICES,
    TAG_SORT_CHOICES,
    build_tag_cards,
@@ -23,7 +24,7 @@ from archivebox.core.tag_utils import (
    normalize_has_snapshots_filter,
    normalize_tag_sort,
 )
-from archivebox.core.host_utils import build_snapshot_url
+from archivebox.core.routes_util import build_snapshot_url


 class TagInline(admin.TabularInline):
@@ -62,8 +63,8 @@ class TagAdmin(BaseModelAdmin):
    change_form_template = "admin/core/tag/change_form.html"
    list_display = ("name", "num_snapshots", "created_at", "created_by")
    list_filter = ("created_at", "created_by")
-    search_fields = ("id", "name", "slug")
-    readonly_fields = ("slug", "id", "created_at", "modified_at", "snapshots")
+    search_fields = ("id", "name")
+    readonly_fields = ("id", "created_at", "modified_at", "snapshots")
    actions = ["delete_selected"]
    ordering = ["name", "id"]

@@ -71,7 +72,7 @@ class TagAdmin(BaseModelAdmin):
        (
            "Tag",
            {
-                "fields": ("name", "slug"),
+                "fields": ("name",),
                "classes": ("card",),
            },
        ),
@@ -117,8 +118,11 @@ class TagAdmin(BaseModelAdmin):
        created_by = normalize_created_by_filter((request.GET.get("created_by") or "").strip())
        year = normalize_created_year_filter((request.GET.get("year") or "").strip())
        has_snapshots = normalize_has_snapshots_filter((request.GET.get("has_snapshots") or "all").strip())
-        extra_context = {
+        context = {
+            **self.admin_site.each_context(request),
            **(extra_context or {}),
+            "title": "Tags",
+            "opts": self.model._meta,
            "initial_query": query,
            "initial_sort": sort,
            "initial_created_by": created_by,
@@ -131,6 +135,8 @@ class TagAdmin(BaseModelAdmin):
            "initial_tag_cards": build_tag_cards(
                query=query,
                request=request,
+                limit=150,
+                preview_limit=0,
                sort=sort,
                created_by=created_by,
                year=year,
@@ -139,7 +145,7 @@ class TagAdmin(BaseModelAdmin):
            "tag_search_api_url": reverse("api-1:search_tags"),
            "tag_create_api_url": reverse("api-1:tags_create"),
        }
-        return super().changelist_view(request, extra_context=extra_context)
+        return TemplateResponse(request, self.change_list_template, context)

    def render_change_form(self, request, context, add=False, change=False, form_url="", obj=None):
        current_name = (request.POST.get("name") or "").strip()
@@ -147,7 +153,9 @@ class TagAdmin(BaseModelAdmin):
            current_name = obj.name

        similar_tag_cards = (
-            build_tag_cards(query=current_name, request=request, limit=12) if current_name else build_tag_cards(request=request, limit=12)
+            build_tag_cards(query=current_name, request=request, limit=12, preview_limit=0)
+            if current_name
+            else build_tag_cards(request=request, limit=12, preview_limit=0)
        )
        if obj:
            similar_tag_cards = [card for card in similar_tag_cards if card["id"] != obj.pk]
@@ -223,7 +231,9 @@ class TagAdmin(BaseModelAdmin):

    @admin.display(description="Snapshots", ordering="num_snapshots")
    def num_snapshots(self, tag: Tag):
-        count = getattr(tag, "num_snapshots", tag.snapshot_set.count())
+        count = tag.__dict__.get("num_snapshots")
+        if count is None:
+            count = tag.snapshot_set.count()
        return format_html(
            '<a href="/admin/core/snapshot/?tags__id__exact={}">{} total</a>',
            tag.id,
@@ -1,8 +1,11 @@
 __package__ = "archivebox.core"

+from urllib.parse import urlencode
+
 from django.contrib import admin
 from django.contrib.auth.admin import UserAdmin
 from django.contrib.auth import get_user_model
+from django.db.models import Count
 from django.utils.html import format_html
 from django.utils.safestring import mark_safe

@@ -11,6 +14,7 @@ class CustomUserAdmin(UserAdmin):
    sort_fields = ["id", "email", "username", "is_superuser", "last_login", "date_joined"]
    list_display = ["username", "id", "email", "is_superuser", "last_login", "date_joined"]
    readonly_fields = ("snapshot_set", "archiveresult_set", "tag_set", "apitoken_set", "outboundwebhook_set")
+    change_form_template = "admin/auth/user/change_form.html"

    # Preserve Django's default user creation form and fieldsets
    # This ensures passwords are properly hashed and permissions are set correctly
@@ -19,6 +23,62 @@ class CustomUserAdmin(UserAdmin):
    # Extend fieldsets for change form only (not user creation)
    fieldsets = [*(UserAdmin.fieldsets or ()), ("Data", {"fields": readonly_fields})]

+    def get_queryset(self, request):
+        return super().get_queryset(request).annotate(snapshot_count=Count("crawl__snapshot_set", distinct=True))
+
+    def snapshot_rss_badge(self, obj, api_token: str = ""):
+        params = {"created_by": obj.username, "limit": 50}
+        if api_token:
+            params["api_key"] = api_token
+        rss_url = f"/api/v1/core/snapshots.rss?{urlencode(params)}"
+        return format_html(
+            (
+                '<a href="{}" title="Snapshot RSS feed for {}" '
+                'style="display:inline-flex;align-items:center;gap:5px;padding:3px 8px;border-radius:4px;'
+                "background:#fff3e0;border:1px solid #f59e0b;color:#7c2d12;font-weight:700;"
+                'font-size:12px;line-height:1.2;text-decoration:none;white-space:nowrap;">'
+                '<span aria-hidden="true" style="display:inline-block;width:8px;height:8px;border-radius:50%;'
+                'background:#f97316;box-shadow:0 0 0 3px rgba(249,115,22,.18);"></span>'
+                "RSS</a>"
+            ),
+            rss_url,
+            obj.username,
+        )
+
+    def snapshot_count_badge(self, obj):
+        snapshots_url = f"/admin/core/snapshot/?created_by__id__exact={obj.pk}"
+        snapshot_count = obj.__dict__.get("snapshot_count", 0)
+        snapshot_label = "snapshot" if snapshot_count == 1 else "snapshots"
+        return format_html(
+            (
+                '<a href="{}" title="View snapshots for {}" '
+                'style="display:inline-flex;align-items:center;padding:3px 8px;border-radius:4px;'
+                "background:#f7f8fa;border:1px solid #d0d7de;color:#24292f;font-weight:600;"
+                'font-size:12px;line-height:1.2;text-decoration:none;white-space:nowrap;">'
+                "{} {}</a>"
+            ),
+            snapshots_url,
+            obj.username,
+            snapshot_count,
+            snapshot_label,
+        )
+
+    @admin.display(description="Snapshots", ordering="snapshot_count")
+    def snapshot_count_column(self, obj):
+        return self.snapshot_count_badge(obj)
+
+    def get_list_display(self, request):
+        from archivebox.api.auth import get_or_create_api_token
+
+        api_token = get_or_create_api_token(request.user)
+        token = api_token.token if api_token else ""
+
+        @admin.display(description="RSS Feed")
+        def snapshot_rss_feed(obj):
+            return self.snapshot_rss_badge(obj, api_token=token)
+
+        return ["username", snapshot_rss_feed, "snapshot_count_column", "id", "email", "is_superuser", "last_login", "date_joined"]
+
    @admin.display(description="Snapshots")
    def snapshot_set(self, obj):
        total_count = obj.snapshot_set.count()
@@ -16,24 +16,15 @@ class CoreConfig(AppConfig):
        from archivebox.core.admin_site import register_admin_site

        register_admin_site()
+        from archivebox.base_models.models import ModelWithOutputDir
+
+        ModelWithOutputDir.register_delete_signal()

        # Import models to register state machines with the registry
        # Skip during makemigrations to avoid premature state machine access
        if "makemigrations" not in sys.argv:
            from archivebox.core import models  # noqa: F401

-        pidfile = os.environ.get("ARCHIVEBOX_RUNSERVER_PIDFILE")
-        if pidfile:
-            should_write_pid = True
-            if os.environ.get("ARCHIVEBOX_AUTORELOAD") == "1":
-                should_write_pid = os.environ.get(DJANGO_AUTORELOAD_ENV) == "true"
-            if should_write_pid:
-                try:
-                    with open(pidfile, "w") as handle:
-                        handle.write(str(os.getpid()))
-                except Exception:
-                    pass
-
        def _should_prepare_runtime() -> bool:
            if os.environ.get("ARCHIVEBOX_RUNSERVER") == "1":
                if os.environ.get("ARCHIVEBOX_AUTORELOAD") == "1":
@@ -42,8 +33,13 @@ class CoreConfig(AppConfig):
            return False

        if _should_prepare_runtime():
-            from archivebox.machine.models import Process, Machine
+            from archivebox.config import CONSTANTS
+            from archivebox.machine.models import Process

-            Process.cleanup_stale_running()
-            Process.cleanup_orphaned_workers()
-            Machine.current()
+            Process.current().mark_running(
+                process_type=Process.TypeChoices.WORKER,
+                worker_type="worker_runserver",
+                pwd=str(CONSTANTS.DATA_DIR),
+                url=os.environ.get("ARCHIVEBOX_RUNSERVER_BIND_URL") or "",
+                timeout=CONSTANTS.MAX_HOOK_RUNTIME_SECONDS,
+            )
@@ -12,6 +12,65 @@ from django.core.asgi import get_asgi_application

 setup_django(in_memory_db=False, check_db=True)

+
+def _patch_thread_sensitive_context_shutdown() -> None:
+    """Stop ``ThreadSensitiveContext.__aexit__`` from blocking the daphne loop.
+
+    Django 6.0's ASGIHandler wraps every request in ``async with
+    ThreadSensitiveContext():`` (django/core/handlers/asgi.py:169). On exit
+    asgiref calls ``executor.shutdown()`` with the default ``wait=True``
+    (asgiref/sync.py:148), which is a *synchronous* ``Thread.join()`` inside
+    an async function — so it blocks the daphne event loop until the
+    executor's worker thread exits.
+
+    That's normally fine because the request handler has already awaited
+    every ``sync_to_async`` it submitted, so the worker is idle and dies as
+    soon as the shutdown sentinel reaches it. The blocking turns into a
+    problem when a client disconnects mid-request:
+
+    * ``SyncToAsync.__call__`` shields the executor work with
+      ``await asyncio.shield(exec_coro)`` (asgiref/sync.py:506) so that the
+      sync DB call doesn't get torn down halfway through.
+    * On cancellation it calls ``exec_coro.cancel()`` (line 522) which only
+      flips the asyncio ``Future`` to cancelled — the underlying thread
+      keeps running the SQL query.
+    * Control unwinds to ``__aexit__`` while the orphaned thread is still
+      mid-query. ``shutdown(wait=True)`` then blocks the event loop until
+      that orphan finishes.
+
+    Under heavy SQLite contention (the load-test scenario that surfaced
+    this on cabbage) those orphan threads can take 30 seconds each waiting
+    for write locks, and the daphne loop is single-threaded — so every
+    such orphan stalls every other in-flight request, healthchecks time
+    out, and the container goes ``unhealthy``.
+
+    Switching to ``shutdown(wait=False)`` queues the sentinel and returns
+    immediately; the worker thread still exits cleanly once its current
+    task finishes, and asgiref's ``WeakKeyDictionary`` releases the
+    executor as soon as the request's context is GC'd. No per-request
+    teardown guarantee is lost — there was no caller relying on it.
+    """
+    from asgiref import sync as _asgiref_sync
+
+    original_aexit = _asgiref_sync.ThreadSensitiveContext.__aexit__
+
+    async def __aexit__(self, exc, value, tb):  # type: ignore[no-redef]
+        if not self.token:
+            return
+        executor = _asgiref_sync.SyncToAsync.context_to_thread_executor.pop(self, None)
+        if executor is not None:
+            executor.shutdown(wait=False)
+        _asgiref_sync.SyncToAsync.thread_sensitive_context.reset(self.token)
+
+    # Idempotent: only patch once even if asgi.py is reloaded.
+    if getattr(original_aexit, "_archivebox_patched", False):
+        return
+    __aexit__._archivebox_patched = True  # type: ignore[attr-defined]
+    _asgiref_sync.ThreadSensitiveContext.__aexit__ = __aexit__
+
+
+_patch_thread_sensitive_context_shutdown()
+
 # Standard Django ASGI application (no websockets/channels needed)
 application = get_asgi_application()

@@ -0,0 +1,9 @@
+from archivebox.config import VERSION
+from archivebox.config.version import get_COMMIT_HASH
+
+
+def archivebox_globals(request):
+    return {
+        "VERSION": VERSION,
+        "STATIC_CACHE_KEY": (get_COMMIT_HASH() or VERSION or "dev").strip(),
+    }
@@ -1,15 +1,24 @@
 __package__ = "archivebox.core"

+import re
+from decimal import Decimal, InvalidOperation, ROUND_CEILING
+
 from django import forms
-from django.utils.html import format_html

 from archivebox.misc.util import URL_REGEX, find_all_urls, parse_filesize_to_bytes
 from taggit.utils import edit_string_for_tags, parse_tags
 from archivebox.base_models.admin import KeyValueWidget
-from archivebox.crawls.schedule_utils import validate_schedule
-from archivebox.config.common import SEARCH_BACKEND_CONFIG
+from archivebox.crawls.schedule_util import validate_schedule
+from archivebox.config.common import get_config, parse_delete_after
+from archivebox.core.permissions import PERMISSIONS_CHOICES, PERMISSIONS_PUBLIC, filter_personas_by_permissions, is_admin_user
 from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget
-from archivebox.hooks import get_plugins, discover_plugin_configs, get_plugin_icon
+from archivebox.plugins.discovery import get_plugins
+from archivebox.plugins.forms import (
+    PLUGIN_GROUP_DEFINITIONS,
+    TIMEOUT_INPUT_PATTERN,
+    PluginConfigFormMixin,
+    get_choice_field,
+)
 from archivebox.personas.models import Persona

 DEPTH_CHOICES = (
@@ -21,35 +30,9 @@ DEPTH_CHOICES = (
 )


-def get_plugin_choices():
-    """Get available extractor plugins from discovered hooks."""
-    return [(name, name) for name in get_plugins()]
+class AddLinkForm(PluginConfigFormMixin, forms.Form):
+    allow_crawl_execution_config_fields = False

-
-def get_plugin_choice_label(plugin_name: str, plugin_configs: dict[str, dict]) -> str:
-    schema = plugin_configs.get(plugin_name, {})
-    description = str(schema.get("description") or "").strip()
-    if not description:
-        return plugin_name
-    icon_html = get_plugin_icon(plugin_name)
-
-    return format_html(
-        '<span class="plugin-choice-icon">{}</span><span class="plugin-choice-name">{}</span><a class="plugin-choice-description" href="https://archivebox.github.io/abx-plugins/#{}" target="_blank" rel="noopener noreferrer">{}</a>',
-        icon_html,
-        plugin_name,
-        plugin_name,
-        description,
-    )
-
-
-def get_choice_field(form: forms.Form, name: str) -> forms.ChoiceField:
-    field = form.fields[name]
-    if not isinstance(field, forms.ChoiceField):
-        raise TypeError(f"{name} must be a ChoiceField")
-    return field
-
-
-class AddLinkForm(forms.Form):
    # Basic fields
    url = forms.CharField(
        label="URLs",
@@ -57,6 +40,13 @@ class AddLinkForm(forms.Form):
        widget=forms.Textarea(
            attrs={
                "data-url-regex": URL_REGEX.pattern,
+                "placeholder": (
+                    "Enter URLs to archive, as one per line, CSV, JSON, or embedded in text "
+                    "(e.g. markdown, HTML, etc.). Examples:\n"
+                    "https://example.com\n"
+                    "https://news.ycombinator.com,https://news.google.com\n"
+                    "[ArchiveBox](https://github.com/ArchiveBox/ArchiveBox)"
+                ),
            },
        ),
        required=True,
@@ -74,7 +64,7 @@ class AddLinkForm(forms.Form):
        widget=forms.RadioSelect(attrs={"class": "depth-selection"}),
    )
    max_urls = forms.IntegerField(
-        label="Max URLs",
+        label="Max crawl URLs",
        required=False,
        min_value=0,
        initial=0,
@@ -86,8 +76,8 @@ class AddLinkForm(forms.Form):
            },
        ),
    )
-    max_size = forms.CharField(
-        label="Max size",
+    crawl_max_size = forms.CharField(
+        label="Max crawl size",
        required=False,
        initial="0",
        widget=forms.TextInput(
@@ -96,6 +86,60 @@ class AddLinkForm(forms.Form):
            },
        ),
    )
+    crawl_timeout = forms.CharField(
+        label="Max crawl time",
+        required=False,
+        initial=0,
+        widget=forms.TextInput(
+            attrs={
+                "pattern": TIMEOUT_INPUT_PATTERN,
+                "title": "Use 0, integer seconds, or a duration like 1.5m or 1hr. Non-zero values must be greater than 10 seconds.",
+                "placeholder": "0, 300, 1.5m, or 1hr",
+            },
+        ),
+    )
+    timeout = forms.CharField(
+        label="Max subtask time",
+        required=False,
+        widget=forms.TextInput(
+            attrs={
+                "pattern": TIMEOUT_INPUT_PATTERN,
+                "title": "Use integer seconds or a duration like 1.5m or 1hr. Non-zero values must be greater than 10 seconds.",
+                "placeholder": "60, 1.5m, or 1hr",
+            },
+        ),
+    )
+    snapshot_max_size = forms.CharField(
+        label="Max snapshot size",
+        required=False,
+        initial="0",
+        widget=forms.TextInput(
+            attrs={
+                "placeholder": "0 = unlimited, or e.g. 45mb / 1gb",
+            },
+        ),
+    )
+    delete_after = forms.CharField(
+        label="Delete after",
+        required=False,
+        initial="0",
+        widget=forms.TextInput(
+            attrs={
+                "placeholder": "0 = keep forever, or e.g. 1d / 6mo",
+            },
+        ),
+    )
+    crawl_max_concurrent_snapshots = forms.IntegerField(
+        label="Max in parallel",
+        required=False,
+        min_value=1,
+        widget=forms.NumberInput(
+            attrs={
+                "min": 1,
+                "step": 1,
+            },
+        ),
+    )
    notes = forms.CharField(
        label="Notes",
        strip=True,
@@ -113,38 +157,44 @@ class AddLinkForm(forms.Form):
    )

    # Plugin groups
-    chrome_plugins = forms.MultipleChoiceField(
-        label="Chrome-dependent plugins",
+    main_plugins = forms.MultipleChoiceField(
+        label="Main",
        required=False,
        widget=forms.CheckboxSelectMultiple,
        choices=[],  # populated in __init__
    )
-    archiving_plugins = forms.MultipleChoiceField(
-        label="Archiving",
+    page_setup_plugins = forms.MultipleChoiceField(
+        label="Page Setup",
        required=False,
        widget=forms.CheckboxSelectMultiple,
        choices=[],
    )
-    parsing_plugins = forms.MultipleChoiceField(
-        label="Parsing",
+    media_plugins = forms.MultipleChoiceField(
+        label="Media",
        required=False,
        widget=forms.CheckboxSelectMultiple,
        choices=[],
    )
-    search_plugins = forms.MultipleChoiceField(
-        label="Search",
+    text_plugins = forms.MultipleChoiceField(
+        label="Text",
        required=False,
        widget=forms.CheckboxSelectMultiple,
        choices=[],
    )
-    binary_plugins = forms.MultipleChoiceField(
-        label="Binary providers",
+    metadata_plugins = forms.MultipleChoiceField(
+        label="Metadata",
        required=False,
        widget=forms.CheckboxSelectMultiple,
        choices=[],
    )
-    extension_plugins = forms.MultipleChoiceField(
-        label="Browser extensions",
+    postprocessing_plugins = forms.MultipleChoiceField(
+        label="Postprocessing",
+        required=False,
+        widget=forms.CheckboxSelectMultiple,
+        choices=[],
+    )
+    other_plugins = forms.MultipleChoiceField(
+        label="Other",
        required=False,
        widget=forms.CheckboxSelectMultiple,
        choices=[],
@@ -162,14 +212,20 @@ class AddLinkForm(forms.Form):
        ),
    )
    persona = forms.ModelChoiceField(
-        label="Persona (authentication profile)",
+        label="Persona (configuration profile)",
        required=False,
        queryset=Persona.objects.none(),
        empty_label=None,
        to_field_name="name",
    )
-    index_only = forms.BooleanField(
-        label="Index only dry run (add crawl but don't archive yet)",
+    permissions = forms.ChoiceField(
+        label="Permissions",
+        choices=PERMISSIONS_CHOICES,
+        initial="public",
+        required=True,
+    )
+    start_paused = forms.BooleanField(
+        label="Start paused",
        initial=False,
        required=False,
    )
@@ -181,101 +237,64 @@ class AddLinkForm(forms.Form):
    )

    def __init__(self, *args, **kwargs):
+        self.request = kwargs.pop("request", None)
+        self.can_override_crawl_config = bool(self.request and is_admin_user(self.request))
        super().__init__(*args, **kwargs)

        default_persona = Persona.get_or_create_default()
-        self.fields["persona"].queryset = Persona.objects.order_by("name")
-        self.fields["persona"].initial = default_persona.name
+        persona_queryset = Persona.objects.order_by("name")
+        if not self.can_override_crawl_config:
+            persona_queryset = filter_personas_by_permissions(persona_queryset, {PERMISSIONS_PUBLIC})
+        self.fields["persona"].queryset = persona_queryset

-        # Get all plugins
-        all_plugins = get_plugins()
-        plugin_configs = discover_plugin_configs()
+        selected_persona = persona_queryset.filter(id=default_persona.id).first() or persona_queryset.first()
+        default_config = get_config(persona=selected_persona) if selected_persona else get_config()
+        if selected_persona:
+            self.fields["persona"].initial = selected_persona.name
+        self.fields["permissions"].initial = default_config.PERMISSIONS
+        self.fields["timeout"].initial = default_config.TIMEOUT
+        self.fields["crawl_max_concurrent_snapshots"].initial = default_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS
+        self.fields["delete_after"].initial = default_config.DELETE_AFTER
+        self.fields["url_filters"].initial = {
+            "allowlist": "",
+            "denylist": "",
+            "same_domain_only": False,
+            "subpaths_only": False,
+            "only_new": bool(default_config.ONLY_NEW),
+        }

-        # Define plugin groups
-        chrome_dependent = {
-            "accessibility",
-            "chrome",
-            "consolelog",
-            "dom",
-            "headers",
-            "parse_dom_outlinks",
-            "pdf",
-            "redirects",
-            "responses",
-            "screenshot",
-            "seo",
-            "singlefile",
-            "ssl",
-            "staticfile",
-            "title",
-        }
-        archiving = {
-            "archivedotorg",
-            "defuddle",
-            "favicon",
-            "forumdl",
-            "gallerydl",
-            "git",
-            "htmltotext",
-            "mercury",
-            "papersdl",
-            "readability",
-            "trafilatura",
-            "wget",
-            "ytdlp",
-        }
-        parsing = {
-            "parse_html_urls",
-            "parse_jsonl_urls",
-            "parse_netscape_urls",
-            "parse_rss_urls",
-            "parse_txt_urls",
-        }
-        search = {
-            "search_backend_ripgrep",
-            "search_backend_sonic",
-            "search_backend_sqlite",
-        }
-        binary = {"apt", "brew", "custom", "env", "npm", "pip"}
-        extensions = {"twocaptcha", "istilldontcareaboutcookies", "ublock"}
-
-        # Populate plugin field choices
-        get_choice_field(self, "chrome_plugins").choices = [
-            (p, get_plugin_choice_label(p, plugin_configs)) for p in sorted(all_plugins) if p in chrome_dependent
-        ]
-        get_choice_field(self, "archiving_plugins").choices = [
-            (p, get_plugin_choice_label(p, plugin_configs)) for p in sorted(all_plugins) if p in archiving
-        ]
-        get_choice_field(self, "parsing_plugins").choices = [
-            (p, get_plugin_choice_label(p, plugin_configs)) for p in sorted(all_plugins) if p in parsing
-        ]
-        get_choice_field(self, "search_plugins").choices = [
-            (p, get_plugin_choice_label(p, plugin_configs)) for p in sorted(all_plugins) if p in search
-        ]
-        get_choice_field(self, "binary_plugins").choices = [
-            (p, get_plugin_choice_label(p, plugin_configs)) for p in sorted(all_plugins) if p in binary
-        ]
-        get_choice_field(self, "extension_plugins").choices = [
-            (p, get_plugin_choice_label(p, plugin_configs)) for p in sorted(all_plugins) if p in extensions
-        ]
-
-        required_search_plugin = f"search_backend_{SEARCH_BACKEND_CONFIG.SEARCH_BACKEND_ENGINE}".strip()
-        search_choices = [choice[0] for choice in get_choice_field(self, "search_plugins").choices]
-        if required_search_plugin in search_choices:
-            get_choice_field(self, "search_plugins").initial = [required_search_plugin]
+        if self.is_bound:
+            selected_persona = (
+                persona_queryset.filter(name=str(self.data.get(self.add_prefix("persona")) or "")).first() or selected_persona
+            )
+        if self.can_override_crawl_config:
+            self.build_plugin_groups(get_config(persona=selected_persona) if selected_persona else get_config())
+        else:
+            all_plugins = get_plugins()
+            for field_name, *_rest, plugin_names in PLUGIN_GROUP_DEFINITIONS:
+                get_choice_field(self, field_name).choices = [(p, p) for p in all_plugins if p in plugin_names]
+            get_choice_field(self, "other_plugins").choices = [(p, p) for p in all_plugins]
+            self.plugin_groups = []

    def clean(self):
        cleaned_data = super().clean() or {}

+        if not self.can_override_crawl_config:
+            cleaned_data["plugins"] = []
+            cleaned_data["plugin_config"] = {}
+            cleaned_data["config"] = {}
+            return cleaned_data
+
        # Combine all plugin groups into single list
        all_selected_plugins = []
        for field in [
-            "chrome_plugins",
-            "archiving_plugins",
-            "parsing_plugins",
-            "search_plugins",
-            "binary_plugins",
-            "extension_plugins",
+            "main_plugins",
+            "page_setup_plugins",
+            "media_plugins",
+            "text_plugins",
+            "metadata_plugins",
+            "postprocessing_plugins",
+            "other_plugins",
        ]:
            selected = cleaned_data.get(field)
            if isinstance(selected, list):
@@ -284,14 +303,23 @@ class AddLinkForm(forms.Form):
        # Store combined list for easy access
        cleaned_data["plugins"] = all_selected_plugins

+        plugin_config_overrides = self.clean_plugin_config_overrides(get_config(persona=cleaned_data.get("persona")))
+        custom_config = cleaned_data.get("config") or {}
+        if not isinstance(custom_config, dict):
+            custom_config = {}
+        cleaned_data["plugin_config"] = plugin_config_overrides
+        cleaned_data["config"] = {**plugin_config_overrides, **custom_config}
+
        return cleaned_data

    def clean_url(self):
        value = self.cleaned_data.get("url") or ""
-        urls = "\n".join(find_all_urls(value))
-        if not urls:
+        valid_urls = []
+        for url in find_all_urls(value):
+            valid_urls.append(url)
+        if not valid_urls:
            raise forms.ValidationError("Enter at least one valid URL.")
-        return urls
+        return "\n".join(valid_urls)

    def clean_url_filters(self):
        from archivebox.crawls.models import Crawl
@@ -301,14 +329,16 @@ class AddLinkForm(forms.Form):
            "allowlist": "\n".join(Crawl.split_filter_patterns(value.get("allowlist", ""))),
            "denylist": "\n".join(Crawl.split_filter_patterns(value.get("denylist", ""))),
            "same_domain_only": bool(value.get("same_domain_only")),
+            "subpaths_only": bool(value.get("subpaths_only")),
+            "only_new": bool(value.get("only_new")),
        }

    def clean_max_urls(self):
        value = self.cleaned_data.get("max_urls")
        return int(value or 0)

-    def clean_max_size(self):
-        raw_value = str(self.cleaned_data.get("max_size") or "").strip()
+    def clean_crawl_max_size(self):
+        raw_value = str(self.cleaned_data.get("crawl_max_size") or "").strip()
        if not raw_value:
            return 0
        try:
@@ -316,7 +346,67 @@ class AddLinkForm(forms.Form):
        except ValueError as err:
            raise forms.ValidationError(str(err))
        if value < 0:
-            raise forms.ValidationError("Max size must be 0 or a positive number of bytes.")
+            raise forms.ValidationError("Max crawl size must be 0 or a positive number of bytes.")
+        return value
+
+    def clean_crawl_timeout(self):
+        return self._clean_timeout_seconds(self.cleaned_data.get("crawl_timeout"), "Max crawl time", blank_value=0)
+
+    def clean_timeout(self):
+        return self._clean_timeout_seconds(self.cleaned_data.get("timeout"), "Max subtask time", blank_value=None)
+
+    def _clean_timeout_seconds(self, raw_value, field_label: str, *, blank_value):
+        raw_value = str(raw_value or "").strip().lower()
+        if not raw_value:
+            return blank_value
+        if raw_value.isdigit():
+            value = int(raw_value)
+        else:
+            match = re.fullmatch(r"(\d+(?:\.\d+)?)\s*(s|sec|secs|second|seconds|m|min|mins|minute|minutes|h|hr|hrs|hour|hours)", raw_value)
+            if not match:
+                raise forms.ValidationError(f"{field_label} must be seconds or a duration like 1.5m or 1hr.")
+            amount_str, unit = match.groups()
+            try:
+                amount = Decimal(amount_str)
+            except InvalidOperation as err:
+                raise forms.ValidationError(f"{field_label} must be seconds or a duration like 1.5m or 1hr.") from err
+            multiplier = 1
+            if unit in {"m", "min", "mins", "minute", "minutes"}:
+                multiplier = 60
+            elif unit in {"h", "hr", "hrs", "hour", "hours"}:
+                multiplier = 60 * 60
+            value = int((amount * multiplier).to_integral_value(rounding=ROUND_CEILING))
+        if 0 < value <= 10:
+            raise forms.ValidationError(f"{field_label} must be 0 or greater than 10 seconds.")
+        return value
+
+    def clean_snapshot_max_size(self):
+        raw_value = str(self.cleaned_data.get("snapshot_max_size") or "").strip()
+        if not raw_value:
+            return 0
+        try:
+            value = parse_filesize_to_bytes(raw_value)
+        except ValueError as err:
+            raise forms.ValidationError(str(err))
+        if value < 0:
+            raise forms.ValidationError("Max snapshot size must be 0 or a positive number of bytes.")
+        return value
+
+    def clean_delete_after(self):
+        raw_value = str(self.cleaned_data.get("delete_after") or "0").strip() or "0"
+        try:
+            parse_delete_after(raw_value)
+        except ValueError as err:
+            raise forms.ValidationError(str(err))
+        return raw_value
+
+    def clean_crawl_max_concurrent_snapshots(self):
+        value = self.cleaned_data.get("crawl_max_concurrent_snapshots")
+        if value in (None, ""):
+            value = get_config().CRAWL_MAX_CONCURRENT_SNAPSHOTS
+        value = int(value)
+        if value < 1:
+            raise forms.ValidationError("Max concurrent snapshots must be at least 1.")
        return value

    def clean_schedule(self):
@@ -1,235 +0,0 @@
-from __future__ import annotations
-
-import re
-from urllib.parse import urlparse
-
-from archivebox.config.common import SERVER_CONFIG
-
-
-_SNAPSHOT_ID_RE = re.compile(r"^[0-9a-fA-F-]{8,36}$")
-_SNAPSHOT_SUBDOMAIN_RE = re.compile(r"^snap-(?P<suffix>[0-9a-fA-F]{12})$")
-
-
-def split_host_port(host: str) -> tuple[str, str | None]:
-    parsed = urlparse(f"//{host}")
-    hostname = (parsed.hostname or host or "").lower()
-    port = str(parsed.port) if parsed.port else None
-    return hostname, port
-
-
-def _normalize_base_url(value: str | None) -> str:
-    if not value:
-        return ""
-    base = value.strip()
-    if not base:
-        return ""
-    if "://" not in base:
-        base = f"http://{base}"
-    parsed = urlparse(base)
-    if not parsed.netloc:
-        return ""
-    return f"{parsed.scheme}://{parsed.netloc}"
-
-
-def normalize_base_url(value: str | None) -> str:
-    return _normalize_base_url(value)
-
-
-def get_listen_host() -> str:
-    return (SERVER_CONFIG.LISTEN_HOST or "").strip()
-
-
-def get_listen_parts() -> tuple[str, str | None]:
-    return split_host_port(get_listen_host())
-
-
-def _build_listen_host(subdomain: str | None) -> str:
-    host, port = get_listen_parts()
-    if not host:
-        return ""
-    full_host = f"{subdomain}.{host}" if subdomain else host
-    if port:
-        return f"{full_host}:{port}"
-    return full_host
-
-
-def get_admin_host() -> str:
-    if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING:
-        return get_listen_host().lower()
-    override = _normalize_base_url(SERVER_CONFIG.ADMIN_BASE_URL)
-    if override:
-        return urlparse(override).netloc.lower()
-    return _build_listen_host("admin")
-
-
-def get_web_host() -> str:
-    if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING:
-        return get_listen_host().lower()
-    override = _normalize_base_url(SERVER_CONFIG.ARCHIVE_BASE_URL)
-    if override:
-        return urlparse(override).netloc.lower()
-    return _build_listen_host("web")
-
-
-def get_api_host() -> str:
-    if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING:
-        return get_listen_host().lower()
-    return _build_listen_host("api")
-
-
-def get_public_host() -> str:
-    if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING:
-        return get_listen_host().lower()
-    return _build_listen_host("public")
-
-
-def get_snapshot_subdomain(snapshot_id: str) -> str:
-    normalized = re.sub(r"[^0-9a-fA-F]", "", snapshot_id or "")
-    suffix = (normalized[-12:] if len(normalized) >= 12 else normalized).lower()
-    return f"snap-{suffix}"
-
-
-def get_snapshot_host(snapshot_id: str) -> str:
-    if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING:
-        return get_listen_host().lower()
-    return _build_listen_host(get_snapshot_subdomain(snapshot_id))
-
-
-def get_original_host(domain: str) -> str:
-    if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING:
-        return get_listen_host().lower()
-    return _build_listen_host(domain)
-
-
-def is_snapshot_subdomain(subdomain: str) -> bool:
-    value = (subdomain or "").strip()
-    return bool(_SNAPSHOT_SUBDOMAIN_RE.match(value) or _SNAPSHOT_ID_RE.match(value))
-
-
-def get_snapshot_lookup_key(snapshot_ref: str) -> str:
-    value = (snapshot_ref or "").strip().lower()
-    match = _SNAPSHOT_SUBDOMAIN_RE.match(value)
-    if match:
-        return match.group("suffix")
-    return value
-
-
-def get_listen_subdomain(request_host: str) -> str:
-    if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING:
-        return ""
-    req_host, req_port = split_host_port(request_host)
-    listen_host, listen_port = get_listen_parts()
-    if not listen_host:
-        return ""
-    if listen_port and req_port and listen_port != req_port:
-        return ""
-    if req_host == listen_host:
-        return ""
-    suffix = f".{listen_host}"
-    if req_host.endswith(suffix):
-        return req_host[: -len(suffix)]
-    return ""
-
-
-def host_matches(request_host: str, target_host: str) -> bool:
-    if not request_host or not target_host:
-        return False
-    req_host, req_port = split_host_port(request_host)
-    target_host_only, target_port = split_host_port(target_host)
-    if req_host != target_host_only:
-        return False
-    if target_port and req_port and target_port != req_port:
-        return False
-    return True
-
-
-def _scheme_from_request(request=None) -> str:
-    if request:
-        return request.scheme
-    return "http"
-
-
-def _build_base_url_for_host(host: str, request=None) -> str:
-    if not host:
-        return ""
-    scheme = _scheme_from_request(request)
-    return f"{scheme}://{host}"
-
-
-def get_admin_base_url(request=None) -> str:
-    override = _normalize_base_url(SERVER_CONFIG.ADMIN_BASE_URL)
-    if override:
-        return override
-    if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING:
-        return _build_base_url_for_host(get_listen_host(), request=request)
-    return _build_base_url_for_host(get_admin_host(), request=request)
-
-
-def get_web_base_url(request=None) -> str:
-    override = _normalize_base_url(SERVER_CONFIG.ARCHIVE_BASE_URL)
-    if override:
-        return override
-    if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING:
-        return _build_base_url_for_host(get_listen_host(), request=request)
-    return _build_base_url_for_host(get_web_host(), request=request)
-
-
-def get_api_base_url(request=None) -> str:
-    if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING:
-        return _build_base_url_for_host(get_listen_host(), request=request)
-    return _build_base_url_for_host(get_api_host(), request=request)
-
-
-def get_public_base_url(request=None) -> str:
-    return _build_base_url_for_host(get_public_host(), request=request)
-
-
-# Backwards-compat aliases (archive == web)
-def get_archive_base_url(request=None) -> str:
-    return get_web_base_url(request=request)
-
-
-def get_snapshot_base_url(snapshot_id: str, request=None) -> str:
-    if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING:
-        return _build_url(get_web_base_url(request=request), f"/snapshot/{snapshot_id}")
-    return _build_base_url_for_host(get_snapshot_host(snapshot_id), request=request)
-
-
-def get_original_base_url(domain: str, request=None) -> str:
-    if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING:
-        return _build_url(get_web_base_url(request=request), f"/original/{domain}")
-    return _build_base_url_for_host(get_original_host(domain), request=request)
-
-
-def build_admin_url(path: str = "", request=None) -> str:
-    return _build_url(get_admin_base_url(request), path)
-
-
-def build_web_url(path: str = "", request=None) -> str:
-    return _build_url(get_web_base_url(request), path)
-
-
-def build_api_url(path: str = "", request=None) -> str:
-    return _build_url(get_api_base_url(request), path)
-
-
-def build_archive_url(path: str = "", request=None) -> str:
-    return _build_url(get_archive_base_url(request), path)
-
-
-def build_snapshot_url(snapshot_id: str, path: str = "", request=None) -> str:
-    return _build_url(get_snapshot_base_url(snapshot_id, request=request), path)
-
-
-def build_original_url(domain: str, path: str = "", request=None) -> str:
-    return _build_url(get_original_base_url(domain, request=request), path)
-
-
-def _build_url(base_url: str, path: str) -> str:
-    if not base_url:
-        if not path:
-            return ""
-        return path if path.startswith("/") else f"/{path}"
-    if not path:
-        return base_url
-    return f"{base_url}{path if path.startswith('/') else f'/{path}'}"
@@ -12,15 +12,16 @@ from django.contrib.staticfiles import finders
 from django.utils.http import http_date
 from django.http import HttpResponseForbidden, HttpResponseNotModified

-from archivebox.config.common import SERVER_CONFIG
+from archivebox.config.common import get_config
 from archivebox.config import VERSION
 from archivebox.config.version import get_COMMIT_HASH
-from archivebox.core.host_utils import (
+from archivebox.core.routes_util import (
    build_snapshot_url,
    build_admin_url,
    build_web_url,
    get_api_host,
    get_admin_host,
+    get_base_host,
    get_listen_host,
    get_listen_subdomain,
    get_public_host,
@@ -32,6 +33,39 @@ from archivebox.core.host_utils import (
 from archivebox.core.views import SnapshotHostView, OriginalDomainHostView


+ADMIN_LOGIN_HINT_COOKIE = "archivebox_admin_logged_in"
+
+
+def _admin_login_hint_cookie_domain(config) -> str | None:
+    """Resolve the parent domain to scope the cross-subdomain login hint.
+
+    NOTE: this cookie carries only the single bit "user is logged in on
+    admin somewhere"; it MUST NOT be confused with the session cookie,
+    which stays admin-host-scoped (see core/settings.py
+    SESSION_COOKIE_DOMAIN comment — admin/public is a security boundary).
+
+    Returns the hostname portion of ``get_base_host`` (which respects
+    ``BASE_URL`` and falls back to the local-bind mapping). Strips the
+    port — cookie ``Domain=`` attributes don't include ports. Returns
+    ``None`` when subdomain routing is off, the base host is empty, or
+    the base host is an IP / bare ``localhost`` (browsers reject
+    cross-host cookies for those).
+    """
+    if not config.USES_SUBDOMAIN_ROUTING:
+        return None
+    base_host = get_base_host(config=config)
+    if not base_host:
+        return None
+    hostname, _port = split_host_port(base_host)
+    if not hostname or hostname == "localhost":
+        return None
+    try:
+        ipaddress.ip_address(hostname)
+    except ValueError:
+        return hostname
+    return None
+
+
 def detect_timezone(request, activate: bool = True):
    gmt_offset = (request.COOKIES.get("GMT_OFFSET") or "").strip()
    tz = None
@@ -85,7 +119,11 @@ def CacheControlMiddleware(get_response):

        if "/archive/" in request.path or "/static/" in request.path or snapshot_path_re.match(request.path):
            if not response.get("Cache-Control"):
-                policy = "public" if SERVER_CONFIG.PUBLIC_SNAPSHOTS else "private"
+                config = request.__dict__.get("archivebox_config")
+                if config is None:
+                    config = get_config(resolve_plugins=False)
+                    request.archivebox_config = config
+                policy = "private" if config.PERMISSIONS == "private" else "public"
                response["Cache-Control"] = f"{policy}, max-age=60, stale-while-revalidate=300"
                # print('Set Cache-Control header to', response['Cache-Control'])
        return response
@@ -98,7 +136,12 @@ def ServerSecurityModeMiddleware(get_response):
    allowed_methods = {"GET", "HEAD", "OPTIONS"}

    def middleware(request):
-        if SERVER_CONFIG.CONTROL_PLANE_ENABLED:
+        config = request.__dict__.get("archivebox_config")
+        if config is None:
+            config = get_config(resolve_plugins=False)
+            request.archivebox_config = config
+
+        if config.CONTROL_PLANE_ENABLED:
            return get_response(request)

        request.user = AnonymousUser()
@@ -122,32 +165,55 @@ def HostRoutingMiddleware(get_response):
    )

    def middleware(request):
+        if request.path in {"/health", "/health/"}:
+            return get_response(request)
+
        request_host = (request.get_host() or "").lower()
-        admin_host = get_admin_host()
-        web_host = get_web_host()
-        api_host = get_api_host()
-        public_host = get_public_host()
-        listen_host = get_listen_host()
-        subdomain = get_listen_subdomain(request_host)
+        config = request.__dict__.get("archivebox_config")
+        if config is None:
+            config = get_config(resolve_plugins=False)
+            request.archivebox_config = config
+        admin_host = get_admin_host(config=config)
+        web_host = get_web_host(config=config)
+        api_host = get_api_host(config=config)
+        public_host = get_public_host(config=config)
+        listen_host = get_listen_host(config=config)
+        subdomain = get_listen_subdomain(request_host, config=config)

        # Framework-owned assets must bypass snapshot/original-domain replay routing.
        # Otherwise pages on snapshot subdomains can receive HTML for JS/CSS requests.
        if request.path.startswith("/static/") or request.path in {"/favicon.ico", "/robots.txt"}:
            return get_response(request)

-        if SERVER_CONFIG.USES_SUBDOMAIN_ROUTING and not host_matches(request_host, admin_host):
+        if config.USES_SUBDOMAIN_ROUTING and config.BASE_URL and not host_matches(request_host, admin_host):
+            add_should_redirect = not config.PUBLIC_ADD_VIEW and (request.path == "/add" or request.path.startswith("/add/"))
            if (
                request.path == "/admin"
                or request.path.startswith("/admin/")
                or request.path == "/accounts"
                or request.path.startswith("/accounts/")
+                or add_should_redirect
            ):
                target = build_admin_url(request.path, request=request)
                if request.META.get("QUERY_STRING"):
                    target = f"{target}?{request.META['QUERY_STRING']}"
                return redirect(target)

-        if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING:
+        if subdomain and is_snapshot_subdomain(subdomain):
+            view = SnapshotHostView.as_view()
+            return view(request, snapshot_id=subdomain, path=request.path.lstrip("/"))
+
+        # In subdomain mode with no explicit BASE_URL we can't safely emit
+        # ``admin.``/``web.``/``snap-*.`` redirects: every URL builder uses the
+        # request's own Host (via the request-host fallback in get_base_url),
+        # so prepending ``admin.`` to whatever the client sent produces a
+        # redirect chain of ``admin.admin.admin.<host>``. Pass the request
+        # through; the misconfig banner on the rendered page tells the user
+        # to pin BASE_URL so the redirects can resume.
+        if config.USES_SUBDOMAIN_ROUTING and not config.BASE_URL:
+            return get_response(request)
+
+        if not config.USES_SUBDOMAIN_ROUTING:
            if host_matches(request_host, listen_host):
                return get_response(request)

@@ -164,7 +230,7 @@ def HostRoutingMiddleware(get_response):

        if host_matches(request_host, admin_host):
            snapshot_match = snapshot_path_re.match(request.path)
-            if SERVER_CONFIG.USES_SUBDOMAIN_ROUTING and snapshot_match:
+            if config.USES_SUBDOMAIN_ROUTING and snapshot_match:
                snapshot_id = snapshot_match.group("snapshot_id")
                replay_path = (snapshot_match.group("path") or "").strip("/")
                if replay_path == "index.html":
@@ -173,7 +239,21 @@ def HostRoutingMiddleware(get_response):
                if request.META.get("QUERY_STRING"):
                    target = f"{target}?{request.META['QUERY_STRING']}"
                return redirect(target)
-            return get_response(request)
+            response = get_response(request)
+            hint_cookie_domain = _admin_login_hint_cookie_domain(config)
+            if request.user.is_authenticated and not request.path.startswith("/admin/logout"):
+                response.set_cookie(
+                    ADMIN_LOGIN_HINT_COOKIE,
+                    "1",
+                    max_age=1209600,
+                    domain=hint_cookie_domain,
+                    secure=request.is_secure(),
+                    httponly=True,
+                    samesite="Lax",
+                )
+            else:
+                response.delete_cookie(ADMIN_LOGIN_HINT_COOKIE, domain=hint_cookie_domain, samesite="Lax")
+            return response

        if host_matches(request_host, api_host):
            request.user = AnonymousUser()
@@ -191,17 +271,20 @@ def HostRoutingMiddleware(get_response):
            return get_response(request)

        if host_matches(request_host, web_host):
+            if request.COOKIES.get(ADMIN_LOGIN_HINT_COOKIE) == "1" and (request.path == "/public" or request.path.startswith("/public/")):
+                target = build_admin_url("/admin/core/snapshot/", request=request)
+                return redirect(target)
            request.user = AnonymousUser()
            request._cached_user = request.user
            return get_response(request)

        if host_matches(request_host, public_host):
+            if request.COOKIES.get(ADMIN_LOGIN_HINT_COOKIE) == "1" and (request.path == "/public" or request.path.startswith("/public/")):
+                target = build_admin_url("/admin/core/snapshot/", request=request)
+                return redirect(target)
            return get_response(request)

        if subdomain:
-            if is_snapshot_subdomain(subdomain):
-                view = SnapshotHostView.as_view()
-                return view(request, snapshot_id=subdomain, path=request.path.lstrip("/"))
            view = OriginalDomainHostView.as_view()
            return view(request, domain=subdomain, path=request.path.lstrip("/"))

@@ -211,7 +294,13 @@ def HostRoutingMiddleware(get_response):
                target = f"{target}?{request.META['QUERY_STRING']}"
            return redirect(target)

-        if admin_host or web_host:
+        if (admin_host or web_host) and config.BASE_URL:
+            # Only force a canonical-host redirect when BASE_URL was set
+            # explicitly. If BASE_URL is empty (e.g. 0.7.3 → 0.9.0 upgrade
+            # where the user has CSRF_TRUSTED_ORIGINS but never set BASE_URL),
+            # the subdomain we'd redirect to may not actually resolve in the
+            # user's reverse proxy — serve the request as-is instead and let
+            # the misconfig banner surface the problem in the page.
            target = build_web_url(request.path, request=request)
            if target:
                if request.META.get("QUERY_STRING"):
@@ -224,17 +313,22 @@ def HostRoutingMiddleware(get_response):


 class ReverseProxyAuthMiddleware(RemoteUserMiddleware):
-    header = "HTTP_{normalized}".format(normalized=SERVER_CONFIG.REVERSE_PROXY_USER_HEADER.replace("-", "_").upper())
+    header = "HTTP_REMOTE_USER"

    def process_request(self, request):
-        if SERVER_CONFIG.REVERSE_PROXY_WHITELIST == "":
+        config = request.__dict__.get("archivebox_config")
+        if config is None:
+            config = get_config(resolve_plugins=False)
+            request.archivebox_config = config
+        self.header = "HTTP_{normalized}".format(normalized=config.REVERSE_PROXY_USER_HEADER.replace("-", "_").upper())
+        if config.REVERSE_PROXY_WHITELIST == "":
            return

        ip = request.META.get("REMOTE_ADDR")
        if not isinstance(ip, str):
            return

-        for cidr in SERVER_CONFIG.REVERSE_PROXY_WHITELIST.split(","):
+        for cidr in config.REVERSE_PROXY_WHITELIST.split(","):
            try:
                network = ipaddress.ip_network(cidr)
            except ValueError:
@@ -12,12 +12,7 @@ try:

    ARCHIVE_DIR = CONSTANTS.ARCHIVE_DIR
 except ImportError:
-    try:
-        from archivebox.config import CONFIG
-
-        ARCHIVE_DIR = Path(CONFIG.get("ARCHIVE_DIR", "./archive"))
-    except ImportError:
-        ARCHIVE_DIR = Path("./archive")
+    ARCHIVE_DIR = Path("./archive")

 try:
    from archivebox.misc.util import to_json
@@ -4,6 +4,12 @@

 from django.db import migrations, models, connection
 import django.utils.timezone
+from uuid import UUID
+from datetime import datetime
+import json
+
+
+PROGRESS_EVERY = 10000


 def get_table_columns(table_name):
@@ -13,6 +19,32 @@ def get_table_columns(table_name):
    return {row[1] for row in cursor.fetchall()}


+def normalize_cmd(cmd):
+    if not cmd:
+        return "[]"
+    try:
+        parsed = json.loads(cmd)
+        if isinstance(parsed, list):
+            return json.dumps([str(part) for part in parsed])
+    except (TypeError, json.JSONDecodeError):
+        pass
+    return json.dumps(str(cmd).split())
+
+
+def normalize_status(status):
+    return {
+        "success": "succeeded",
+        "succeded": "succeeded",
+        "succeeded": "succeeded",
+        "failed": "failed",
+        "skipped": "skipped",
+        "noresults": "noresults",
+        "queued": "queued",
+        "started": "started",
+        "backoff": "backoff",
+    }.get(str(status or "").strip().lower(), "failed")
+
+
 def upgrade_core_tables(apps, schema_editor):
    """Upgrade core tables from v0.7.2 or v0.8.6rc0 to v0.9.0."""
    from archivebox.uuid_compat import uuid7
@@ -35,7 +67,9 @@ def upgrade_core_tables(apps, schema_editor):
    has_uuid = "uuid" in archiveresult_cols
    has_abid = "abid" in archiveresult_cols

-    print(f"DEBUG: ArchiveResult row_count={row_count}, has_data={has_data}, has_uuid={has_uuid}, has_abid={has_abid}")
+    if has_data:
+        source_schema = "0.8.x abid" if has_abid and not has_uuid else "0.8.x uuid" if has_uuid else "0.7.x"
+        print(f"    - Rebuilding core tables from {source_schema} schema ({row_count} ArchiveResults)...")

    # ============================================================================
    # PART 1: Upgrade core_archiveresult table
@@ -55,68 +89,152 @@ def upgrade_core_tables(apps, schema_editor):
            status VARCHAR(15) NOT NULL DEFAULT 'queued',
            extractor VARCHAR(32),
            output VARCHAR(1024),
+            created_at DATETIME,
+            modified_at DATETIME,

            FOREIGN KEY (snapshot_id) REFERENCES core_snapshot(id) ON DELETE CASCADE
        );
    """)

    if has_data:
+        has_archiveresult_created_at = "created_at" in archiveresult_cols
+        has_archiveresult_modified_at = "modified_at" in archiveresult_cols
+        archiveresult_select_cols = [
+            "id",
+            "snapshot_id",
+            "cmd",
+            "pwd",
+            "cmd_version",
+            "start_ts",
+            "end_ts",
+            "status",
+            "extractor",
+            "output",
+        ]
+        if has_archiveresult_created_at:
+            archiveresult_select_cols.append("created_at")
+        if has_archiveresult_modified_at:
+            archiveresult_select_cols.append("modified_at")
+
        if has_uuid and not has_abid:
            # Migrating from v0.7.2+ (has uuid column)
-            print("Migrating ArchiveResult from v0.7.2+ schema (with uuid)...")
-            cursor.execute("""
-                INSERT OR IGNORE INTO core_archiveresult_new (
-                    id, uuid, snapshot_id, cmd, pwd, cmd_version,
-                    start_ts, end_ts, status, extractor, output
-                )
-                SELECT
-                    id, uuid, snapshot_id, cmd, pwd, cmd_version,
-                    start_ts, end_ts, status, extractor, output
-                FROM core_archiveresult;
-            """)
-        elif has_abid and not has_uuid:
-            # Migrating from v0.8.6rc0 (has abid instead of uuid)
-            print("Migrating ArchiveResult from v0.8.6rc0 schema...")
-            cursor.execute("""
-                INSERT OR IGNORE INTO core_archiveresult_new (
-                    id, uuid, snapshot_id, cmd, pwd, cmd_version,
-                    start_ts, end_ts, status, extractor, output
-                )
-                SELECT
-                    id, abid as uuid, snapshot_id, cmd, pwd, cmd_version,
-                    start_ts, end_ts, status, extractor, output
-                FROM core_archiveresult;
-            """)
-        else:
-            # Migrating from v0.7.2 (no uuid or abid column - generate fresh UUIDs)
-            print("Migrating ArchiveResult from v0.7.2 schema (no uuid - generating UUIDs)...")
-            cursor.execute(
-                "SELECT id, snapshot_id, cmd, pwd, cmd_version, start_ts, end_ts, status, extractor, output FROM core_archiveresult",
-            )
+            print(f"      copying {row_count} ArchiveResults...")
+            select_cols = ["id", "uuid", "snapshot_id", "cmd", "pwd", "cmd_version", "start_ts", "end_ts", "status", "extractor", "output"]
+            if has_archiveresult_created_at:
+                select_cols.append("created_at")
+            if has_archiveresult_modified_at:
+                select_cols.append("modified_at")
+            cursor.execute(f"SELECT {', '.join(select_cols)} FROM core_archiveresult")
            old_records = cursor.fetchall()
-            for record in old_records:
-                new_uuid = uuid7().hex
+            for i, record in enumerate(old_records, start=1):
+                values = dict(zip(select_cols, record))
+                try:
+                    new_uuid = UUID(str(values["uuid"])).hex
+                except (TypeError, ValueError):
+                    new_uuid = uuid7().hex
+                start_ts = values["start_ts"] or datetime.now().isoformat()
+                end_ts = values["end_ts"] or start_ts
                cursor.execute(
                    """
                    INSERT OR IGNORE INTO core_archiveresult_new (
                        id, uuid, snapshot_id, cmd, pwd, cmd_version,
-                        start_ts, end_ts, status, extractor, output
-                    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                        start_ts, end_ts, status, extractor, output,
+                        created_at, modified_at
+                    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                """,
                    (
-                        record[0],
+                        values["id"],
                        new_uuid,
-                        record[1],
-                        record[2],
-                        record[3],
-                        record[4],
-                        record[5],
-                        record[6],
-                        record[7],
-                        record[8],
-                        record[9],
+                        values["snapshot_id"],
+                        normalize_cmd(values["cmd"]),
+                        values["pwd"] or "",
+                        values["cmd_version"] or "",
+                        start_ts,
+                        end_ts,
+                        normalize_status(values["status"] or "queued"),
+                        values["extractor"] or "",
+                        values["output"] or "",
+                        values.get("created_at") or start_ts,
+                        values.get("modified_at") or end_ts,
                    ),
                )
+                if i % PROGRESS_EVERY == 0:
+                    print(f"        copied {i}/{len(old_records)} ArchiveResults...")
+        elif has_abid and not has_uuid:
+            # Migrating from v0.8.6rc0 (has abid instead of uuid)
+            print(f"      copying {row_count} ArchiveResults...")
+            cursor.execute(f"SELECT {', '.join(archiveresult_select_cols)} FROM core_archiveresult")
+            old_records = cursor.fetchall()
+            for i, record in enumerate(old_records, start=1):
+                values = dict(zip(archiveresult_select_cols, record))
+                try:
+                    new_uuid = UUID(str(values["id"])).hex
+                except (TypeError, ValueError):
+                    new_uuid = uuid7().hex
+                start_ts = values["start_ts"] or datetime.now().isoformat()
+                end_ts = values["end_ts"] or start_ts
+                cursor.execute(
+                    """
+                    INSERT OR IGNORE INTO core_archiveresult_new (
+                        uuid, snapshot_id, cmd, pwd, cmd_version,
+                        start_ts, end_ts, status, extractor, output,
+                        created_at, modified_at
+                    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                """,
+                    (
+                        new_uuid,
+                        values["snapshot_id"],
+                        normalize_cmd(values["cmd"]),
+                        values["pwd"] or "",
+                        values["cmd_version"] or "",
+                        start_ts,
+                        end_ts,
+                        normalize_status(values["status"] or "queued"),
+                        values["extractor"] or "",
+                        values["output"] or "",
+                        values.get("created_at") or start_ts,
+                        values.get("modified_at") or end_ts,
+                    ),
+                )
+                if i % PROGRESS_EVERY == 0:
+                    print(f"        copied {i}/{len(old_records)} ArchiveResults...")
+        else:
+            # Migrating from v0.7.2 (no uuid or abid column - generate fresh UUIDs)
+            print(f"      copying {row_count} ArchiveResults...")
+            cursor.execute(f"SELECT {', '.join(archiveresult_select_cols)} FROM core_archiveresult")
+            old_records = cursor.fetchall()
+            for i, record in enumerate(old_records, start=1):
+                values = dict(zip(archiveresult_select_cols, record))
+                new_uuid = uuid7().hex
+                start_ts = values["start_ts"] or datetime.now().isoformat()
+                end_ts = values["end_ts"] or start_ts
+                cursor.execute(
+                    """
+                    INSERT OR IGNORE INTO core_archiveresult_new (
+                        id, uuid, snapshot_id, cmd, pwd, cmd_version,
+                        start_ts, end_ts, status, extractor, output,
+                        created_at, modified_at
+                    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                """,
+                    (
+                        values["id"],
+                        new_uuid,
+                        values["snapshot_id"],
+                        normalize_cmd(values["cmd"]),
+                        values["pwd"] or "",
+                        values["cmd_version"] or "",
+                        start_ts,
+                        end_ts,
+                        normalize_status(values["status"] or "queued"),
+                        values["extractor"] or "",
+                        values["output"] or "",
+                        values.get("created_at") or start_ts,
+                        values.get("modified_at") or end_ts,
+                    ),
+                )
+                if i % PROGRESS_EVERY == 0:
+                    print(f"        copied {i}/{len(old_records)} ArchiveResults...")
+        print(f"      copied {len(old_records)} ArchiveResults")

    cursor.execute("DROP TABLE IF EXISTS core_archiveresult;")
    cursor.execute("ALTER TABLE core_archiveresult_new RENAME TO core_archiveresult;")
@@ -173,44 +291,90 @@ def upgrade_core_tables(apps, schema_editor):

            if has_added and not has_bookmarked_at:
                # Migrating from v0.7.2 (has added/updated fields)
-                print("Migrating Snapshot from v0.7.2 schema...")
-                # Transform added→bookmarked_at/created_at and updated→modified_at
+                print("      copying Snapshots from 0.7.x schema...")
+                # timestamp is the legacy bookmark/import timestamp and archive/{timestamp} identity.
+                # added is the DB row creation/import time, and updated was renamed to downloaded_at in 0.8.x.
                cursor.execute("""
                    INSERT OR IGNORE INTO core_snapshot_new (
                        id, url, timestamp, title,
-                        bookmarked_at, created_at, modified_at,
+                        bookmarked_at, created_at, modified_at, downloaded_at,
                        status
                    )
                    SELECT
                        id, url, timestamp, title,
-                        COALESCE(added, CURRENT_TIMESTAMP) as bookmarked_at,
+                        COALESCE(
+                            CASE
+                                WHEN CAST(timestamp AS REAL) BETWEEN 788918400 AND 2082758400
+                                THEN datetime(CAST(timestamp AS REAL), 'unixepoch')
+                            END,
+                            added,
+                            CURRENT_TIMESTAMP
+                        ) as bookmarked_at,
                        COALESCE(added, CURRENT_TIMESTAMP) as created_at,
                        COALESCE(updated, added, CURRENT_TIMESTAMP) as modified_at,
-                        'queued' as status
+                        updated as downloaded_at,
+                        CASE
+                            WHEN EXISTS (
+                                SELECT 1 FROM core_archiveresult
+                                WHERE core_archiveresult.snapshot_id = core_snapshot.id
+                                AND core_archiveresult.status IN ('queued', 'started', 'backoff')
+                            )
+                            THEN 'queued'
+                            WHEN EXISTS (
+                                SELECT 1 FROM core_archiveresult
+                                WHERE core_archiveresult.snapshot_id = core_snapshot.id
+                            )
+                            THEN 'sealed'
+                            ELSE 'queued'
+                        END as status
                    FROM core_snapshot;
                """)
+                print(f"      copied {cursor.rowcount} Snapshots")
            elif has_bookmarked_at and not has_added:
                # Migrating from v0.8.6rc0 (already has bookmarked_at/created_at/modified_at)
-                print("Migrating Snapshot from v0.8.6rc0 schema...")
+                print("      copying Snapshots from 0.8.x schema...")
                # Check what fields exist
                has_status = "status" in snapshot_cols
                has_retry_at = "retry_at" in snapshot_cols
                has_crawl_id = "crawl_id" in snapshot_cols

                # Build column list based on what exists
-                cols = ["id", "url", "timestamp", "title", "bookmarked_at", "created_at", "modified_at", "downloaded_at"]
+                insert_cols = ["id", "url", "timestamp", "title", "bookmarked_at", "created_at", "modified_at", "downloaded_at"]
+                select_cols = ["id", "url", "timestamp", "title", "bookmarked_at", "created_at", "modified_at", "downloaded_at"]
                if has_crawl_id:
-                    cols.append("crawl_id")
+                    insert_cols.append("crawl_id")
+                    select_cols.append("REPLACE(crawl_id, '-', '')")
                if has_status:
-                    cols.append("status")
+                    insert_cols.append("status")
+                    select_cols.append(
+                        """
+                        CASE
+                            WHEN status IN ('sealed', 'started', 'paused') THEN status
+                            WHEN EXISTS (
+                                SELECT 1 FROM core_archiveresult
+                                WHERE core_archiveresult.snapshot_id = core_snapshot.id
+                                AND core_archiveresult.status IN ('queued', 'started', 'backoff')
+                            )
+                            THEN status
+                            WHEN EXISTS (
+                                SELECT 1 FROM core_archiveresult
+                                WHERE core_archiveresult.snapshot_id = core_snapshot.id
+                            )
+                            THEN 'sealed'
+                            ELSE status
+                        END
+                        """,
+                    )
                if has_retry_at:
-                    cols.append("retry_at")
+                    insert_cols.append("retry_at")
+                    select_cols.append("retry_at")

                cursor.execute(f"""
-                    INSERT OR IGNORE INTO core_snapshot_new ({", ".join(cols)})
-                    SELECT {", ".join(cols)}
+                    INSERT OR IGNORE INTO core_snapshot_new ({", ".join(insert_cols)})
+                    SELECT {", ".join(select_cols)}
                    FROM core_snapshot;
                """)
+                print(f"      copied {cursor.rowcount} Snapshots")
            else:
                print(f"Warning: Unexpected Snapshot schema - has_added={has_added}, has_bookmarked_at={has_bookmarked_at}")

@@ -261,7 +425,7 @@ def upgrade_core_tables(apps, schema_editor):

            if tag_id_type and "char" in tag_id_type.lower():
                # v0.8.6rc0: Tag IDs are UUIDs, need to convert to INTEGER
-                print("Converting Tag IDs from UUID to INTEGER...")
+                print("      converting Tag IDs from UUID to integers...")

                # Get all tags with their UUIDs
                cursor.execute("SELECT id, name, slug, created_at, modified_at, created_by_id FROM core_tag ORDER BY name")
@@ -280,6 +444,8 @@ def upgrade_core_tables(apps, schema_editor):
                    """,
                        (i, name, slug, created_at, modified_at, created_by_id),
                    )
+                    if i % PROGRESS_EVERY == 0:
+                        print(f"        copied {i}/{len(tags)} Tags...")

                # Update snapshot_tags to use new INTEGER IDs
                cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='core_snapshot_tags'")
@@ -291,7 +457,7 @@ def upgrade_core_tables(apps, schema_editor):
                    cursor.execute("DELETE FROM core_snapshot_tags")

                    # Re-insert with new integer tag IDs
-                    for st_id, snapshot_id, old_tag_id in snapshot_tags:
+                    for i, (st_id, snapshot_id, old_tag_id) in enumerate(snapshot_tags, start=1):
                        new_tag_id = uuid_to_int_map.get(old_tag_id)
                        if new_tag_id:
                            cursor.execute(
@@ -301,14 +467,18 @@ def upgrade_core_tables(apps, schema_editor):
                            """,
                                (st_id, snapshot_id, new_tag_id),
                            )
+                        if i % PROGRESS_EVERY == 0:
+                            print(f"        copied {i}/{len(snapshot_tags)} SnapshotTag rows...")
+                print(f"      copied {len(tags)} Tags")
            else:
                # v0.7.2: Tag IDs are already INTEGER
-                print("Migrating Tag from v0.7.2 schema...")
+                print("      copying Tags from 0.7.x schema...")
                cursor.execute("""
                    INSERT OR IGNORE INTO core_tag_new (id, name, slug)
                    SELECT id, name, slug
                    FROM core_tag;
                """)
+                print(f"      copied {cursor.rowcount} Tags")

    cursor.execute("DROP TABLE IF EXISTS core_tag;")
    cursor.execute("ALTER TABLE core_tag_new RENAME TO core_tag;")
@@ -318,13 +488,13 @@ def upgrade_core_tables(apps, schema_editor):
    cursor.execute("CREATE INDEX IF NOT EXISTS core_tag_created_by_id_idx ON core_tag(created_by_id);")

    if has_data:
-        print("✓ Core tables upgraded to v0.9.0")
+        print("    ✓ Core table rebuild complete")


 class Migration(migrations.Migration):
    dependencies = [
        ("core", "0022_auto_20231023_2008"),
-        ("crawls", "0001_initial"),
+        ("crawls", "0002_upgrade_from_0_8_6"),
        ("auth", "0012_alter_user_first_name_max_length"),
    ]

@@ -359,6 +529,11 @@ class Migration(migrations.Migration):
                    name="modified_at",
                    field=models.DateTimeField(auto_now=True),
                ),
+                migrations.AddField(
+                    model_name="snapshot",
+                    name="downloaded_at",
+                    field=models.DateTimeField(blank=True, db_index=True, default=None, editable=False, null=True),
+                ),
                # Declare fs_version (already created in database with DEFAULT '0.8.0')
                migrations.AddField(
                    model_name="snapshot",
@@ -23,6 +23,9 @@ def create_default_crawl_and_assign_snapshots(apps, schema_editor):
        print("✓ Fresh install or all snapshots already have crawls")
        return

+    cursor.execute("SELECT url FROM core_snapshot WHERE crawl_id IS NULL ORDER BY bookmarked_at, timestamp")
+    crawl_urls = "\n".join(url for (url,) in cursor.fetchall() if url)
+
    # Get or create system user (pk=1)
    cursor.execute("SELECT id FROM auth_user WHERE id = 1")
    if not cursor.fetchone():
@@ -34,22 +37,36 @@ def create_default_crawl_and_assign_snapshots(apps, schema_editor):
            [datetime.now().isoformat()],
        )

-    # Create a default crawl for migrated snapshots
-    # At this point crawls_crawl is guaranteed to have v0.9.0 schema (crawls/0002 ran first)
-    crawl_id = str(uuid_lib.uuid4())
+    # Create a default crawl for migrated snapshots.
+    # Depending on migration graph order, later crawls migrations may already
+    # have removed output_dir by the time this data migration runs.
+    crawl_id = uuid_lib.uuid4().hex
    now = datetime.now().isoformat()
+    cursor.execute("PRAGMA table_info(crawls_crawl)")
+    crawl_columns = {row[1] for row in cursor.fetchall()}
+    default_columns: list[str] = []
+    default_values: list[str] = []
+    if "output_dir" in crawl_columns:
+        default_columns.append("output_dir")
+        default_values.append("''")
+    for column in ("max_size", "max_urls", "crawl_max_size", "snapshot_max_size"):
+        if column in crawl_columns:
+            default_columns.append(column)
+            default_values.append("0")
+    default_columns_sql = f"{', '.join(default_columns)}, " if default_columns else ""
+    default_values_sql = f"{', '.join(default_values)}, " if default_values else ""

    cursor.execute(
-        """
+        f"""
        INSERT INTO crawls_crawl (
            id, created_at, modified_at, num_uses_succeeded, num_uses_failed,
-            urls, max_depth, tags_str, label, notes, output_dir,
+            urls, max_depth, tags_str, label, notes, {default_columns_sql}
            status, retry_at, created_by_id, schedule_id, config, persona_id
-        ) VALUES (?, ?, ?, 0, 0, '', 0, '', 'Migrated from v0.7.2/v0.8.6',
-                  'Auto-created crawl for migrated snapshots', '',
-                  'sealed', ?, 1, NULL, '{}', NULL)
+        ) VALUES (?, ?, ?, 0, 0, ?, 0, '', 'Migrated from v0.7.2/v0.8.6',
+                  'Auto-created crawl for migrated snapshots', {default_values_sql}
+                  'sealed', ?, 1, NULL, '{{}}', NULL)
    """,
-        [crawl_id, now, now, now],
+        [crawl_id, now, now, crawl_urls, now],
    )

    # Assign all snapshots without a crawl to the default crawl
@@ -118,7 +135,7 @@ class Migration(migrations.Migration):
                        SELECT
                            id, url, timestamp, title,
                            bookmarked_at, created_at, modified_at,
-                            crawl_id, parent_snapshot_id,
+                            REPLACE(crawl_id, '-', ''), REPLACE(parent_snapshot_id, '-', ''),
                            downloaded_at, depth, fs_version,
                            COALESCE(config, '{}'), COALESCE(notes, ''),
                            num_uses_succeeded, num_uses_failed,
@@ -25,7 +25,9 @@ def copy_old_fields_to_new(apps, schema_editor):
        # Copy output -> output_str
        cursor.execute("UPDATE core_archiveresult SET output_str = COALESCE(output, '') WHERE output_str = '' OR output_str IS NULL")

-    # Copy timestamps to new timestamp fields if they don't have values yet
+    # Fill missing row metadata for older schemas that did not have it.
+    # 0023 creates/preserves created_at + modified_at before any SQLite table
+    # rebuilds in this migration, so never overwrite existing non-empty values.
    if "start_ts" in cols and "created_at" in cols:
        cursor.execute(
            "UPDATE core_archiveresult SET created_at = COALESCE(start_ts, CURRENT_TIMESTAMP) WHERE created_at IS NULL OR created_at = ''",
@@ -33,17 +35,12 @@ def copy_old_fields_to_new(apps, schema_editor):

    if "end_ts" in cols and "modified_at" in cols:
        cursor.execute(
-            "UPDATE core_archiveresult SET modified_at = COALESCE(end_ts, start_ts, CURRENT_TIMESTAMP) WHERE modified_at IS NULL OR modified_at = ''",
+            "UPDATE core_archiveresult SET modified_at = COALESCE(end_ts, start_ts, created_at, CURRENT_TIMESTAMP) WHERE modified_at IS NULL OR modified_at = ''",
        )

    # NOTE: Snapshot timestamps (added→bookmarked_at, updated→modified_at) were already
    # transformed by migration 0023, so we don't need to copy them here.
-    # NOTE: UUIDs are already populated by migration 0023 for all migration paths
-
-    # Debug: Check Snapshot timestamps at end of RunPython
-    cursor.execute("SELECT id, bookmarked_at, modified_at FROM core_snapshot LIMIT 2")
-    snap_after = cursor.fetchall()
-    print(f"DEBUG 0025: Snapshot timestamps at END of RunPython: {snap_after}")
+    # NOTE: UUIDs are already populated by migration 0023 for all migration paths.


 class Migration(migrations.Migration):
@@ -64,16 +61,29 @@ class Migration(migrations.Migration):
        ),
        # NOTE: RemoveField for cmd, cmd_version, pwd moved to migration 0027
        # to allow data migration to Process records first
+        # NOTE: created_at/modified_at are created by migration 0023 so they can
+        # preserve old ArchiveResult row metadata before SQLite table rebuilds.
+        # Update Django's state here before the AddField operations below.
+        migrations.SeparateDatabaseAndState(
+            database_operations=[],
+            state_operations=[
+                migrations.AddField(
+                    model_name="archiveresult",
+                    name="created_at",
+                    field=models.DateTimeField(db_index=True, default=django.utils.timezone.now),
+                ),
+                migrations.AddField(
+                    model_name="archiveresult",
+                    name="modified_at",
+                    field=models.DateTimeField(auto_now=True),
+                ),
+            ],
+        ),
        migrations.AddField(
            model_name="archiveresult",
            name="config",
            field=models.JSONField(blank=True, default=dict, null=True),
        ),
-        migrations.AddField(
-            model_name="archiveresult",
-            name="created_at",
-            field=models.DateTimeField(db_index=True, default=django.utils.timezone.now),
-        ),
        migrations.AddField(
            model_name="archiveresult",
            name="hook_name",
@@ -85,11 +95,6 @@ class Migration(migrations.Migration):
                max_length=255,
            ),
        ),
-        migrations.AddField(
-            model_name="archiveresult",
-            name="modified_at",
-            field=models.DateTimeField(auto_now=True),
-        ),
        migrations.AddField(
            model_name="archiveresult",
            name="notes",
@@ -140,74 +145,78 @@ class Migration(migrations.Migration):
            name="retry_at",
            field=models.DateTimeField(blank=True, db_index=True, default=django.utils.timezone.now, null=True),
        ),
-        # NOTE: bookmarked_at and created_at already added by migration 0023
-        migrations.AddField(
-            model_name="snapshot",
-            name="config",
-            field=models.JSONField(default=dict),
-        ),
-        migrations.AddField(
-            model_name="snapshot",
-            name="current_step",
-            field=models.PositiveSmallIntegerField(
-                db_index=True,
-                default=0,
-                help_text="Current hook step being executed (0-9). Used for sequential hook execution.",
-            ),
-        ),
-        migrations.AddField(
-            model_name="snapshot",
-            name="depth",
-            field=models.PositiveSmallIntegerField(db_index=True, default=0),
-        ),
-        migrations.AddField(
-            model_name="snapshot",
-            name="downloaded_at",
-            field=models.DateTimeField(blank=True, db_index=True, default=None, editable=False, null=True),
-        ),
-        # NOTE: fs_version already added by migration 0023 with default='0.8.0'
-        # NOTE: modified_at already added by migration 0023
-        migrations.AddField(
-            model_name="snapshot",
-            name="notes",
-            field=models.TextField(blank=True, default=""),
-        ),
-        migrations.AddField(
-            model_name="snapshot",
-            name="num_uses_failed",
-            field=models.PositiveIntegerField(default=0),
-        ),
-        migrations.AddField(
-            model_name="snapshot",
-            name="num_uses_succeeded",
-            field=models.PositiveIntegerField(default=0),
-        ),
-        migrations.AddField(
-            model_name="snapshot",
-            name="parent_snapshot",
-            field=models.ForeignKey(
-                blank=True,
-                help_text="Parent snapshot that discovered this URL (for recursive crawling)",
-                null=True,
-                on_delete=django.db.models.deletion.SET_NULL,
-                related_name="child_snapshots",
-                to="core.snapshot",
-            ),
-        ),
-        migrations.AddField(
-            model_name="snapshot",
-            name="retry_at",
-            field=models.DateTimeField(blank=True, db_index=True, default=django.utils.timezone.now, null=True),
-        ),
-        migrations.AddField(
-            model_name="snapshot",
-            name="status",
-            field=models.CharField(
-                choices=[("queued", "Queued"), ("started", "Started"), ("sealed", "Sealed")],
-                db_index=True,
-                default="queued",
-                max_length=15,
-            ),
+        # NOTE: bookmarked_at, created_at, and downloaded_at already added by migration 0023
+        # 0023 rebuilds core_snapshot with these columns already present so it
+        # can preserve legacy status/retry_at/config data in one tight SQLite
+        # table copy. Keep this migration state-only: running normal AddField
+        # operations here causes SQLite to rebuild the table from the pre-0025
+        # state and repopulate existing columns with defaults, reopening sealed
+        # migrated rows as queued.
+        migrations.SeparateDatabaseAndState(
+            database_operations=[],
+            state_operations=[
+                migrations.AddField(
+                    model_name="snapshot",
+                    name="config",
+                    field=models.JSONField(default=dict),
+                ),
+                migrations.AddField(
+                    model_name="snapshot",
+                    name="current_step",
+                    field=models.PositiveSmallIntegerField(
+                        db_index=True,
+                        default=0,
+                        help_text="Current hook step being executed (0-9). Used for sequential hook execution.",
+                    ),
+                ),
+                migrations.AddField(
+                    model_name="snapshot",
+                    name="depth",
+                    field=models.PositiveSmallIntegerField(db_index=True, default=0),
+                ),
+                migrations.AddField(
+                    model_name="snapshot",
+                    name="notes",
+                    field=models.TextField(blank=True, default=""),
+                ),
+                migrations.AddField(
+                    model_name="snapshot",
+                    name="num_uses_failed",
+                    field=models.PositiveIntegerField(default=0),
+                ),
+                migrations.AddField(
+                    model_name="snapshot",
+                    name="num_uses_succeeded",
+                    field=models.PositiveIntegerField(default=0),
+                ),
+                migrations.AddField(
+                    model_name="snapshot",
+                    name="parent_snapshot",
+                    field=models.ForeignKey(
+                        blank=True,
+                        help_text="Parent snapshot that discovered this URL (for recursive crawling)",
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name="child_snapshots",
+                        to="core.snapshot",
+                    ),
+                ),
+                migrations.AddField(
+                    model_name="snapshot",
+                    name="retry_at",
+                    field=models.DateTimeField(blank=True, db_index=True, default=django.utils.timezone.now, null=True),
+                ),
+                migrations.AddField(
+                    model_name="snapshot",
+                    name="status",
+                    field=models.CharField(
+                        choices=[("queued", "Queued"), ("started", "Started"), ("sealed", "Sealed")],
+                        db_index=True,
+                        default="queued",
+                        max_length=15,
+                    ),
+                ),
+            ],
        ),
        migrations.AddField(
            model_name="tag",
@@ -7,6 +7,9 @@ from pathlib import Path
 from archivebox.uuid_compat import uuid7


+PROGRESS_EVERY = 10000
+
+
 def parse_cmd_field(cmd_raw):
    """
    Parse cmd field which could be:
@@ -140,35 +143,33 @@ def get_or_create_binary(cursor, machine_id, name, abspath, version):
    cursor.execute("PRAGMA table_info(machine_binary)")
    binary_cols = {row[1] for row in cursor.fetchall()}

-    # Use only columns that exist in current schema
-    # 0.8.x schema: id, created_at, modified_at, machine_id, name, binprovider, abspath, version, sha256, num_uses_failed, num_uses_succeeded
-    # 0.9.x schema adds: binproviders, overrides, status, retry_at, output_dir
-    if "binproviders" in binary_cols:
-        # 0.9.x schema
-        cursor.execute(
-            """
-            INSERT INTO machine_binary (
-                id, created_at, modified_at, machine_id,
-                name, binproviders, overrides, binprovider, abspath, version, sha256,
-                status, retry_at, output_dir,
-                num_uses_failed, num_uses_succeeded
-            ) VALUES (?, ?, ?, ?, ?, 'env', '{}', 'env', ?, ?, '',
-                      'succeeded', NULL, '', 0, 0)
+    values_by_col = {
+        "id": binary_id,
+        "created_at": now,
+        "modified_at": now,
+        "machine_id": machine_id,
+        "name": name,
+        "binproviders": "env",
+        "overrides": "{}",
+        "binprovider": "env",
+        "abspath": abspath,
+        "version": version,
+        "sha256": "",
+        "status": "installed",
+        "retry_at": None,
+        "output_dir": "",
+        "num_uses_failed": 0,
+        "num_uses_succeeded": 0,
+    }
+    insert_cols = [col for col in values_by_col if col in binary_cols]
+    placeholders = ", ".join(["?"] * len(insert_cols))
+    cursor.execute(
+        f"""
+        INSERT INTO machine_binary ({", ".join(insert_cols)})
+        VALUES ({placeholders})
        """,
-            [binary_id, now, now, machine_id, name, abspath, version],
-        )
-    else:
-        # 0.8.x schema (simpler)
-        cursor.execute(
-            """
-            INSERT INTO machine_binary (
-                id, created_at, modified_at, machine_id,
-                name, binprovider, abspath, version, sha256,
-                num_uses_failed, num_uses_succeeded
-            ) VALUES (?, ?, ?, ?, ?, 'env', ?, ?, '', 0, 0)
-        """,
-            [binary_id, now, now, machine_id, name, abspath, version],
-        )
+        [values_by_col[col] for col in insert_cols],
+    )

    return binary_id

@@ -214,36 +215,42 @@ def create_process(cursor, machine_id, pwd, cmd, status, exit_code, started_at,
    # Set retry_at to now for queued processes, NULL otherwise
    retry_at = now if status == "queued" else None

+    cursor.execute("PRAGMA table_info(machine_process)")
+    process_cols = {row[1] for row in cursor.fetchall()}
+    values_by_col = {
+        "id": process_id,
+        "created_at": now,
+        "modified_at": now,
+        "machine_id": machine_id,
+        "parent_id": None,
+        "process_type": "hook",
+        "worker_type": "",
+        "pwd": pwd,
+        "cmd": cmd_json,
+        "env": "{}",
+        "timeout": 120,
+        "pid": None,
+        "exit_code": exit_code,
+        "stdout": "",
+        "stderr": "",
+        "started_at": started_at,
+        "ended_at": ended_at,
+        "binary_id": binary_id,
+        "iface_id": None,
+        "url": None,
+        "status": status,
+        "retry_at": retry_at,
+        "num_uses_failed": 0,
+        "num_uses_succeeded": 0,
+    }
+    insert_cols = [col for col in values_by_col if col in process_cols]
+    placeholders = ", ".join(["?"] * len(insert_cols))
    cursor.execute(
-        """
-        INSERT INTO machine_process (
-            id, created_at, modified_at, machine_id, parent_id, process_type,
-            pwd, cmd, env, timeout,
-            pid, exit_code, stdout, stderr,
-            started_at, ended_at,
-            binary_id, iface_id, url,
-            status, retry_at
-        ) VALUES (?, ?, ?, ?, NULL, 'cli',
-                  ?, ?, '{}', 120,
-                  NULL, ?, '', '',
-                  ?, ?,
-                  ?, NULL, NULL,
-                  ?, ?)
-    """,
-        [
-            process_id,
-            now,
-            now,
-            machine_id,
-            pwd,
-            cmd_json,
-            exit_code,
-            started_at,
-            ended_at,
-            binary_id,
-            status,
-            retry_at,
-        ],
+        f"""
+        INSERT INTO machine_process ({", ".join(insert_cols)})
+        VALUES ({placeholders})
+        """,
+        [values_by_col[col] for col in insert_cols],
    )

    return process_id
@@ -274,13 +281,8 @@ def copy_archiveresult_data_to_process(apps, schema_editor):
    cursor.execute("PRAGMA table_info(core_archiveresult)")
    cols = {row[1] for row in cursor.fetchall()}

-    print(f"DEBUG 0027: Columns found: {sorted(cols)}")
-    print(
-        f"DEBUG 0027: Has cmd={('cmd' in cols)}, pwd={('pwd' in cols)}, cmd_version={('cmd_version' in cols)}, process_id={('process_id' in cols)}",
-    )
-
    if "cmd" not in cols or "pwd" not in cols or "cmd_version" not in cols:
-        print("✓ Fresh install or fields already removed - skipping data copy")
+        print("    ✓ ArchiveResult process metadata already migrated")
        return

    # Check if process_id field exists (should exist from 0026)
@@ -304,10 +306,10 @@ def copy_archiveresult_data_to_process(apps, schema_editor):
    results = cursor.fetchall()

    if not results:
-        print("✓ No ArchiveResults need Process migration")
+        print("    ✓ No ArchiveResults need Process migration")
        return

-    print(f"Migrating {len(results)} ArchiveResults to Process records...")
+    print(f"    - Migrating {len(results)} ArchiveResults to Process rows...")

    migrated_count = 0
    skipped_count = 0
@@ -316,16 +318,10 @@ def copy_archiveresult_data_to_process(apps, schema_editor):
    for i, row in enumerate(results):
        ar_id, snapshot_id, plugin, cmd_raw, pwd, cmd_version, status, start_ts, end_ts, created_at = row

-        if i == 0:
-            print(f"DEBUG 0027: First row: ar_id={ar_id}, plugin={plugin}, cmd={cmd_raw[:50] if cmd_raw else None}, status={status}")
-
        try:
            # Parse cmd field
            cmd_array = parse_cmd_field(cmd_raw)

-            if i == 0:
-                print(f"DEBUG 0027: Parsed cmd: {cmd_array}")
-
            # Extract binary info from cmd[0] if available
            binary_id = None
            if cmd_array and cmd_array[0]:
@@ -342,9 +338,6 @@ def copy_archiveresult_data_to_process(apps, schema_editor):
                    binary_version,
                )

-                if i == 0:
-                    print(f"DEBUG 0027: Created Binary: id={binary_id}, name={binary_name}")
-
            # Map status
            process_status, exit_code = map_status(status)

@@ -365,9 +358,6 @@ def copy_archiveresult_data_to_process(apps, schema_editor):
                binary_id=binary_id,
            )

-            if i == 0:
-                print(f"DEBUG 0027: Created Process: id={process_id}")
-
            # Link ArchiveResult to Process
            cursor.execute(
                "UPDATE core_archiveresult SET process_id = ? WHERE id = ?",
@@ -375,9 +365,8 @@ def copy_archiveresult_data_to_process(apps, schema_editor):
            )

            migrated_count += 1
-
-            if i == 0:
-                print("DEBUG 0027: Linked ArchiveResult to Process")
+            if migrated_count % PROGRESS_EVERY == 0:
+                print(f"      migrated {migrated_count}/{len(results)} ArchiveResults...")

        except Exception as e:
            print(f"✗ Error migrating ArchiveResult {ar_id}: {e}")
@@ -387,7 +376,7 @@ def copy_archiveresult_data_to_process(apps, schema_editor):
            error_count += 1
            continue

-    print(f"✓ Migration complete: {migrated_count} migrated, {skipped_count} skipped, {error_count} errors")
+    print(f"    ✓ Process migration complete: {migrated_count} migrated, {skipped_count} skipped, {error_count} errors")


 class Migration(migrations.Migration):
@@ -6,6 +6,9 @@ from uuid import UUID
 from archivebox.uuid_compat import uuid7


+PROGRESS_EVERY = 10000
+
+
 def migrate_archiveresult_id_to_uuid(apps, schema_editor):
    """
    Migrate ArchiveResult from integer PK to UUID PK (clean one-step migration).
@@ -28,7 +31,7 @@ def migrate_archiveresult_id_to_uuid(apps, schema_editor):
    # Check if table exists and has data
    cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='core_archiveresult'")
    if not cursor.fetchone():
-        print("ArchiveResult table does not exist, skipping migration")
+        print("    ✓ ArchiveResult table does not exist, skipping UUID PK migration")
        return

    cursor.execute("SELECT COUNT(*) FROM core_archiveresult")
@@ -38,16 +41,16 @@ def migrate_archiveresult_id_to_uuid(apps, schema_editor):
    # (fresh installs create table with uuid from 0025, but model expects no uuid after 0029)

    if row_count == 0:
-        print("[0029] Recreating ArchiveResult table schema (integer→UUID PK, removing uuid column)...")
+        print("    - Rebuilding empty ArchiveResult table with UUID primary keys...")
    else:
-        print(f"[0029] Migrating {row_count} ArchiveResult records from integer PK to UUID PK...")
+        print(f"    - Migrating {row_count} ArchiveResults from integer IDs to UUID primary keys...")

    # Step 0: Check if machine_process table exists, if not NULL out process_id values
    cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='machine_process'")
    machine_process_exists = cursor.fetchone() is not None

    if not machine_process_exists:
-        print("machine_process table does not exist yet, setting process_id to NULL")
+        print("      machine_process is unavailable; clearing process_id references...")
        cursor.execute("UPDATE core_archiveresult SET process_id = NULL WHERE process_id IS NOT NULL")

    # Step 1: Create new table with UUID as primary key (clean - no old_id or uuid columns)
@@ -149,10 +152,6 @@ def migrate_archiveresult_id_to_uuid(apps, schema_editor):
        # Build INSERT statement (only copy fields that exist in source)
        existing_fields = [f for f in fields_to_copy if f in values]

-        if i == 0:
-            print(f"[0029] Source columns: {col_names}")
-            print(f"[0029] Copying fields: {existing_fields}")
-
        placeholders = ", ".join(["?"] * (len(existing_fields) + 1))  # +1 for id
        field_list = "id, " + ", ".join(existing_fields)

@@ -164,13 +163,16 @@ def migrate_archiveresult_id_to_uuid(apps, schema_editor):
                insert_values,
            )
            inserted_count += 1
+            if inserted_count % PROGRESS_EVERY == 0:
+                print(f"      copied {inserted_count}/{len(old_records)} ArchiveResults...")
        except Exception as e:
            print(f"[0029] ERROR inserting record {old_id}: {e}")
            if i == 0:
                print(f"[0029] First record values: {insert_values[:5]}...")
                raise

-    print(f"[0029] Inserted {inserted_count}/{len(old_records)} records")
+    if old_records:
+        print(f"      copied {inserted_count}/{len(old_records)} ArchiveResults")

    # Step 4: Replace old table with new table
    cursor.execute("DROP TABLE core_archiveresult")
@@ -185,7 +187,7 @@ def migrate_archiveresult_id_to_uuid(apps, schema_editor):
    cursor.execute("CREATE INDEX core_archiveresult_hook_name_idx ON core_archiveresult(hook_name)")
    cursor.execute("CREATE INDEX core_archiveresult_process_id_idx ON core_archiveresult(process_id)")

-    print(f"✓ Migrated {row_count} ArchiveResult records to UUID primary key")
+    print(f"    ✓ ArchiveResult UUID primary key migration complete ({row_count} records)")


 class Migration(migrations.Migration):
@@ -0,0 +1,14 @@
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0033_alter_archiveresult_status"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="tag",
+            name="slug",
+        ),
+    ]
@@ -0,0 +1,17 @@
+# Generated by Django 6.0.5 on 2026-05-24 10:26
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0034_remove_tag_slug"),
+        ("machine", "0013_alter_machine_config"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="archiveresult",
+            index=models.Index(fields=["-start_ts", "-id"], name="archiveresult_start_idx"),
+        ),
+    ]
@@ -0,0 +1,17 @@
+# Generated by Django 6.0.5 on 2026-05-24 10:28
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0035_archiveresult_archiveresult_start_idx"),
+        ("crawls", "0007_remove_crawl_crawl_admin_order_idx_and_more"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="snapshot",
+            index=models.Index(fields=["-bookmarked_at", "-created_at"], name="snapshot_public_order_idx"),
+        ),
+    ]
@@ -0,0 +1,15 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0036_snapshot_snapshot_public_order_idx"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="snapshot",
+            name="url",
+            field=models.CharField(db_index=True, max_length=65535),
+        ),
+    ]
@@ -0,0 +1,16 @@
+# Generated by ArchiveBox on 2026-05-27
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0037_alter_snapshot_url_max_length"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="snapshot",
+            index=models.Index(fields=["crawl", "status", "modified_at"], name="snapshot_progress_idx"),
+        ),
+    ]