release: archivebox 0.9.34rc63

release: archivebox 0.9.34rc62
release: archivebox 0.9.34rc61
2026-06-06 23:27:54 -07:00 · 2026-06-06 23:19:13 -07:00 · 2026-06-06 23:09:44 -07:00 · 2026-06-06 20:41:21 -07:00 · 2026-06-06 17:28:42 -07:00 · 2026-06-06 03:11:53 -04:00
276 changed files with 16398 additions and 11779 deletions
@@ -3,18 +3,29 @@
 /.heartbeat.json
 /package.json
 /package-lock.json
+__pycache__
+**/__pycache__
 *.pyc
-__pycache__/
+*.pyo
+*.py[cod]
+*$py.class
 .mypy_cache/
 .pytest_cache/
+.ruff_cache/
+.uv-cache/
 .github/
 .pdm-build/
 .pdm-python
 .eggs/
 .git/
+!.git/
+.git/*
 .vscode/
 !.git/HEAD
-!.git/refs/heads/*
+!.git/packed-refs
+!.git/refs/
+!.git/refs/heads/
+!.git/refs/heads/**

 venv/
 .venv/
@@ -1,214 +0,0 @@
-name: Build Debian package
-
-on:
-  workflow_dispatch:
-  workflow_call:
-  push:
-    branches: ['**']
-    paths:
-      - 'pkg/debian/**'
-      - 'bin/build_deb.sh'
-      - 'bin/release_deb.sh'
-      - '.github/workflows/debian.yml'
-      - 'pyproject.toml'
-  # release trigger is handled by release.yml to avoid double-runs
-
-permissions:
-  contents: write
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build:
-    runs-on: ubuntu-24.04
-    strategy:
-      matrix:
-        arch: [amd64, arm64]
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Get version
-        id: version
-        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
-
-      - name: Install nfpm
-        env:
-          NFPM_VERSION: "2.45.1"
-        run: |
-          curl -fsSL "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${NFPM_VERSION}_Linux_x86_64.tar.gz" \
-            | sudo tar -xz -C /usr/local/bin nfpm
-          nfpm --version
-
-      - name: Build .deb package
-        run: |
-          export VERSION="${{ steps.version.outputs.version }}"
-          export ARCH="${{ matrix.arch }}"
-          ./bin/build_deb.sh
-
-      - name: Verify .deb package contents
-        run: |
-          DEB_FILE="$(ls dist/archivebox*.deb | head -1)"
-          echo "=== Package info ==="
-          dpkg-deb --info "$DEB_FILE"
-          echo ""
-          echo "=== Package contents ==="
-          dpkg-deb --contents "$DEB_FILE"
-          echo ""
-          echo "=== Control fields ==="
-          dpkg-deb --field "$DEB_FILE"
-
-      - name: Upload .deb artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: archivebox-${{ steps.version.outputs.version }}-${{ matrix.arch }}.deb
-          path: dist/*.deb
-
-  test:
-    needs: build
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - arch: amd64
-            runner: ubuntu-24.04
-          - arch: arm64
-            runner: ubuntu-24.04-arm
-    runs-on: ${{ matrix.runner }}
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v4
-        with:
-          enable-cache: false
-
-      - name: Install build dependencies
-        uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0
-        with:
-          packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1
-          version: 1.0
-
-      - name: Build local wheel
-        run: |
-          uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
-          uv build --wheel --out-dir /tmp/wheels/
-
-      - name: Download .deb artifact
-        uses: actions/download-artifact@v4
-        with:
-          pattern: archivebox-*-${{ matrix.arch }}.deb
-          merge-multiple: true
-
-      - name: Install system dependencies
-        run: |
-          sudo apt-get install -y python3.13 python3.13-venv python3-pip git curl wget
-
-      - name: Pre-seed virtualenv with local wheel before dpkg install
-        run: |
-          # CI-only: pre-seed the venv with a local wheel so we test the
-          # unreleased code, not whatever is on PyPI. We explicitly use python3.13
-          # here (matching the system dep installed above) to ensure the venv is
-          # created with the correct Python version. On real installs, install.sh
-          # handles this by preferring python3.13 and failing if < 3.13.
-          # When postinstall.sh runs during dpkg -i, it finds the venv already
-          # populated and just upgrades deps.
-          sudo mkdir -p /opt/archivebox
-          sudo python3.13 -m venv /opt/archivebox/venv
-          sudo /opt/archivebox/venv/bin/python3 -m pip install --quiet --upgrade pip setuptools
-          sudo /opt/archivebox/venv/bin/pip install --quiet /tmp/wheels/archivebox-*.whl
-          echo "[√] Pre-seeded /opt/archivebox/venv with local wheel"
-
-      - name: Install .deb package
-        run: |
-          # dpkg install will run postinstall which creates the user,
-          # sets up systemd, and tries pip install (which finds it already installed)
-          sudo dpkg -i archivebox*.deb || sudo apt-get install -f -y
-
-      - name: Verify archivebox is installed
-        run: |
-          which archivebox
-          archivebox version
-
-      - name: Verify wrapper script works
-        run: |
-          /usr/bin/archivebox version
-          /usr/bin/archivebox --help | head -5
-
-      - name: Test archivebox init as archivebox user
-        run: |
-          # The postinstall should have created the user
-          id archivebox
-          sudo mkdir -p /tmp/archivebox-test
-          sudo chown archivebox:archivebox /tmp/archivebox-test
-          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox init'
-
-      - name: Test archivebox status
-        run: |
-          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox status'
-
-      - name: Test archivebox add
-        run: |
-          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox add "https://example.com"'
-
-      - name: Verify systemd service file exists
-        run: |
-          test -f /usr/lib/systemd/system/archivebox.service
-          cat /usr/lib/systemd/system/archivebox.service
-
-  # Upload .deb to GitHub Release only after tests pass
-  release:
-    if: github.event_name == 'release' || github.event_name == 'workflow_call'
-    needs: [build, test]
-    runs-on: ubuntu-24.04
-    permissions:
-      contents: write
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Determine release tag
-        id: tag
-        run: |
-          TAG="${{ github.event.release.tag_name }}"
-          if [ -z "$TAG" ]; then
-            TAG="v$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')"
-            echo "[i] No release tag in event context, using version from pyproject.toml: $TAG"
-          fi
-          echo "tag=$TAG" >> "$GITHUB_OUTPUT"
-
-      - name: Download all .deb artifacts
-        uses: actions/download-artifact@v4
-        with:
-          pattern: archivebox-*.deb
-          merge-multiple: true
-
-      - name: Upload .deb to GitHub Release
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          TAG="${{ steps.tag.outputs.tag }}"
-          # Verify the release exists before uploading
-          if ! gh release view "$TAG" >/dev/null 2>&1; then
-            echo "[!] No GitHub Release found for tag $TAG."
-            if [ -n "${{ github.event.release.tag_name }}" ]; then
-              echo "[X] This was triggered by a release event — the release should exist. Failing."
-              exit 1
-            fi
-            echo "[i] Skipping upload (workflow_dispatch without a release)."
-            echo "    Create a release first or trigger via the release event."
-            exit 0
-          fi
-          gh release upload "$TAG" *.deb --clobber
@@ -5,7 +5,7 @@ on:
  workflow_call:
  push:
    branches:
-      - dev
+      - "**"
    tags:
      - 'v*'
  # pull_request:
@@ -13,13 +13,14 @@ on:
 env:
  DOCKERHUB_IMAGE: archivebox/archivebox
  GHCR_IMAGE: ghcr.io/archivebox/archivebox
+  ABX_DL_IMAGE: archivebox/abx-dl:latest

 permissions:
  contents: read
  packages: write

 concurrency:
-  group: docker-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

 jobs:
@@ -45,6 +46,44 @@ jobs:
        #   submodules: true
        #   fetch-depth: 1

+      - name: Wait for released ArchiveBox deps on PyPI
+        run: |
+          python3 - <<'PY'
+          import json
+          import re
+          import sys
+          import time
+          import tomllib
+          import urllib.request
+
+          watched = {"abxbus", "abxpkg", "abx-plugins", "abx-dl"}
+          deps = tomllib.loads(open("pyproject.toml", "rb").read().decode())["project"]["dependencies"]
+          required = {}
+          for dep in deps:
+              for name in watched:
+                  match = re.match(rf"{re.escape(name)}\s*(==|>=)\s*([^,;\s]+)", dep)
+                  if match:
+                      required[name] = match.group(2)
+
+          deadline = time.monotonic() + 300
+          missing = required.copy()
+          while missing and time.monotonic() < deadline:
+              for name, version in list(missing.items()):
+                  with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/json", timeout=20) as resp:
+                      releases = json.load(resp)["releases"]
+                  if version in releases:
+                      print(f"{name} {version} is available on PyPI")
+                      missing.pop(name)
+                  else:
+                      print(f"{name} {version} is not available on PyPI yet")
+              if missing:
+                  time.sleep(10)
+
+          if missing:
+              print(f"Missing PyPI releases after wait: {missing}", file=sys.stderr)
+              sys.exit(1)
+          PY
+
      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
@@ -58,6 +97,24 @@ jobs:
      - name: Available platforms
        run: echo ${{ steps.buildx.outputs.platforms }}

+      - name: Wait for published abx-dl image
+        id: abx_dl_image
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          deadline=$((SECONDS + 1800))
+          until docker buildx imagetools inspect "${ABX_DL_IMAGE}" >/tmp/abx-dl-image.json; do
+            if (( SECONDS >= deadline )); then
+              echo "Timed out waiting for published ${ABX_DL_IMAGE}" >&2
+              exit 1
+            fi
+            echo "${ABX_DL_IMAGE} is not published yet; waiting..."
+            sleep 30
+          done
+
+          echo "image=${ABX_DL_IMAGE}" >> "$GITHUB_OUTPUT"
+          docker buildx imagetools inspect "${ABX_DL_IMAGE}"
+
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        if: github.event_name != 'pull_request'
@@ -78,13 +135,10 @@ jobs:
        run: |
          set -Eeuo pipefail
          VERSION="$(python3 - <<'PY'
-          from pathlib import Path
-          import re
+          import tomllib

-          match = re.search(r'^version = "([^"]+)"$', Path("pyproject.toml").read_text(), re.MULTILINE)
-          if not match:
-              raise SystemExit("Failed to read version from pyproject.toml")
-          print(match.group(1))
+          with open("pyproject.toml", "rb") as f:
+              print(tomllib.load(f)["project"]["version"])
          PY
          )"

@@ -93,11 +147,12 @@ jobs:
            echo "org.opencontainers.image.version=${VERSION}"
            echo "org.opencontainers.image.revision=${GITHUB_SHA}"
            echo 'org.opencontainers.image.source=https://github.com/ArchiveBox/ArchiveBox'
+            echo "io.archivebox.abx-dl.image=${{ steps.abx_dl_image.outputs.image }}"
            echo 'EOF'
            echo "version=${VERSION}"
          } >> "$GITHUB_OUTPUT"

-          echo "[+] Building ${{ matrix.platform }} for ${VERSION}"
+          echo "[+] Building ${{ matrix.platform }} for ${VERSION} using ${{ steps.abx_dl_image.outputs.image }}"

      - name: Build and push digest
        id: docker_build
@@ -111,6 +166,8 @@ jobs:
            ${{ env.DOCKERHUB_IMAGE }}
            ${{ env.GHCR_IMAGE }}
          labels: ${{ steps.docker_meta.outputs.labels }}
+          build-args: |
+            ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
          cache-from: type=gha,scope=${{ matrix.cache_scope }}
          cache-to: type=gha,mode=max,scope=${{ matrix.cache_scope }}
          platforms: ${{ matrix.platform }}
@@ -177,36 +234,49 @@ jobs:
        run: |
          set -Eeuo pipefail
          VERSION="$(python3 - <<'PY'
-          from pathlib import Path
-          import re
+          import tomllib

-          match = re.search(r'^version = "([^"]+)"$', Path("pyproject.toml").read_text(), re.MULTILINE)
-          if not match:
-              raise SystemExit("Failed to read version from pyproject.toml")
-          print(match.group(1))
+          with open("pyproject.toml", "rb") as f:
+              print(tomllib.load(f)["project"]["version"])
          PY
          )"
-          SHORT_SHA="${GITHUB_SHA::8}"
+          BRANCH_TAG="$(printf '%s' "${GITHUB_REF_NAME}" | tr -c 'A-Za-z0-9_.-' '-' | sed -E 's/^-+//; s/-+$//; s/-+/-/g' | cut -c1-128)"
+          SHORT_SHA="${GITHUB_SHA::12}"
+          test -n "$BRANCH_TAG"
+          test -n "$SHORT_SHA"

          {
            echo 'dockerhub_tags<<EOF'
-            echo "${DOCKERHUB_IMAGE}:dev"
+            if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
+              echo "${DOCKERHUB_IMAGE}:latest"
+            fi
+            echo "${DOCKERHUB_IMAGE}:${BRANCH_TAG}"
            echo "${DOCKERHUB_IMAGE}:${VERSION}"
            echo "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
            echo 'EOF'
            echo 'ghcr_tags<<EOF'
-            echo "${GHCR_IMAGE}:dev"
+            if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
+              echo "${GHCR_IMAGE}:latest"
+            fi
+            echo "${GHCR_IMAGE}:${BRANCH_TAG}"
            echo "${GHCR_IMAGE}:${VERSION}"
            echo "${GHCR_IMAGE}:sha-${SHORT_SHA}"
            echo 'EOF'
            echo "version=${VERSION}"
+            echo "branch_tag=${BRANCH_TAG}"
            echo "short_sha=${SHORT_SHA}"
          } >> "$GITHUB_OUTPUT"

          echo "[+] Publishing Docker Hub tags:"
-          printf '%s\n' "${DOCKERHUB_IMAGE}:dev" "${DOCKERHUB_IMAGE}:${VERSION}" "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
+          if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
+            printf '%s\n' "${DOCKERHUB_IMAGE}:latest"
+          fi
+          printf '%s\n' "${DOCKERHUB_IMAGE}:${BRANCH_TAG}" "${DOCKERHUB_IMAGE}:${VERSION}" "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
          echo "[+] Publishing GHCR tags:"
-          printf '%s\n' "${GHCR_IMAGE}:dev" "${GHCR_IMAGE}:${VERSION}" "${GHCR_IMAGE}:sha-${SHORT_SHA}"
+          if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
+            printf '%s\n' "${GHCR_IMAGE}:latest"
+          fi
+          printf '%s\n' "${GHCR_IMAGE}:${BRANCH_TAG}" "${GHCR_IMAGE}:${VERSION}" "${GHCR_IMAGE}:sha-${SHORT_SHA}"

      - name: Create Docker Hub manifest
        shell: bash
@@ -1,233 +0,0 @@
-name: Build Homebrew formula
-
-on:
-  workflow_dispatch:
-  workflow_call:
-  push:
-    branches: ['**']
-    paths:
-      - 'brew_dist/**'
-      - 'bin/build_brew.sh'
-      - 'bin/release_brew.sh'
-      - '.github/workflows/homebrew.yml'
-      - 'pyproject.toml'
-  # release trigger is handled by release.yml to avoid double-runs
-
-permissions:
-  contents: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build-and-test:
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [macos-latest, ubuntu-24.04]
-    runs-on: ${{ matrix.os }}
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Get version
-        id: version
-        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
-
-      - name: Validate formula template syntax
-        run: ruby -c brew_dist/archivebox.rb
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v4
-        with:
-          enable-cache: false
-
-      - name: Install build dependencies (Linux)
-        if: runner.os == 'Linux'
-        uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0
-        with:
-          packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1
-          version: 1.0
-
-      - name: Build local sdist
-        run: |
-          uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
-          uv build --sdist --out-dir /tmp/sdist/
-
-      - name: Generate formula from local sdist
-        run: |
-          VERSION="${{ steps.version.outputs.version }}"
-          SDIST_PATH="$(ls /tmp/sdist/archivebox-*.tar.gz | head -1)"
-          SDIST_SHA256="$(shasum -a 256 "$SDIST_PATH" | awk '{print $1}')"
-
-          # Install archivebox + poet into a temp venv to generate resource stanzas
-          python3 -m venv /tmp/poet-venv
-          source /tmp/poet-venv/bin/activate
-          pip install --quiet "$SDIST_PATH" homebrew-pypi-poet
-          echo "[+] Generating resource stanzas with homebrew-pypi-poet..."
-          RESOURCES="$(poet archivebox)"
-          deactivate
-
-          # For CI: use file:// URL pointing to local sdist
-          # For release: this gets overridden with the PyPI URL
-          SDIST_URL="file://${SDIST_PATH}"
-
-          cat > /tmp/archivebox.rb << RUBY
-class Archivebox < Formula
-  include Language::Python::Virtualenv
-
-  desc "Self-hosted internet archiving solution"
-  homepage "https://github.com/ArchiveBox/ArchiveBox"
-  url "${SDIST_URL}"
-  sha256 "${SDIST_SHA256}"
-  license "MIT"
-
-  depends_on "python@3.13"
-
-${RESOURCES}
-
-  def install
-    virtualenv_install_with_resources
-  end
-
-  def post_install
-    data_dir = var/"archivebox"
-    data_dir.mkpath
-    ENV["DATA_DIR"] = data_dir.to_s
-    system bin/"archivebox", "init"
-  end
-
-  def caveats
-    <<~EOS
-      ArchiveBox data is stored in:
-        #{var}/archivebox
-
-      To start archiving, run:
-        cd #{var}/archivebox && archivebox add 'https://example.com'
-
-      To start the web UI:
-        cd #{var}/archivebox && archivebox server 0.0.0.0:8000
-    EOS
-  end
-
-  test do
-    assert_match version.to_s, shell_output("#{bin}/archivebox version")
-  end
-end
-RUBY
-
-          echo "[√] Generated formula:"
-          ruby -c /tmp/archivebox.rb
-          cat /tmp/archivebox.rb
-
-      - name: Install Homebrew (Linux only)
-        if: runner.os == 'Linux'
-        run: |
-          NONINTERACTIVE=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
-          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
-          echo "HOMEBREW_PREFIX=$HOMEBREW_PREFIX" >> "$GITHUB_ENV"
-          echo "HOMEBREW_CELLAR=$HOMEBREW_CELLAR" >> "$GITHUB_ENV"
-          echo "HOMEBREW_REPOSITORY=$HOMEBREW_REPOSITORY" >> "$GITHUB_ENV"
-          echo "$HOMEBREW_PREFIX/bin" >> "$GITHUB_PATH"
-          echo "$HOMEBREW_PREFIX/sbin" >> "$GITHUB_PATH"
-
-      - name: Install brew dependencies
-        run: brew install python@3.13
-
-      - name: Install archivebox via brew from local formula
-        run: |
-          brew install --build-from-source --verbose /tmp/archivebox.rb
-
-      - name: Verify archivebox version
-        run: archivebox version
-
-      - name: Test archivebox init
-        run: |
-          mkdir -p /tmp/archivebox-test && cd /tmp/archivebox-test
-          archivebox init
-
-      - name: Test archivebox status
-        run: |
-          cd /tmp/archivebox-test
-          archivebox status
-
-  # On release only: generate the real formula with PyPI URL and push to tap
-  release:
-    if: github.event_name == 'release' || github.event_name == 'workflow_call'
-    needs: build-and-test
-    runs-on: macos-latest
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Get version
-        id: version
-        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - name: Wait for PyPI package availability
-        run: |
-          VERSION="${{ steps.version.outputs.version }}"
-          echo "[+] Waiting for archivebox==${VERSION} to be available on PyPI..."
-          for i in $(seq 1 30); do
-            if pip index versions archivebox 2>/dev/null | grep -q "$VERSION"; then
-              echo "[√] archivebox==${VERSION} is available on PyPI"
-              break
-            fi
-            if [ "$i" -eq 30 ]; then
-              echo "[!] Timed out waiting for PyPI. Trying to install anyway..."
-              break
-            fi
-            echo "    Attempt $i/30 - not yet available, waiting 30s..."
-            sleep 30
-          done
-
-      - name: Generate release formula via build_brew.sh
-        run: ./bin/build_brew.sh
-
-      - name: Test formula install
-        run: |
-          brew install --build-from-source brew_dist/archivebox.rb
-          archivebox version
-
-      - name: Upload formula artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: archivebox.rb
-          path: brew_dist/archivebox.rb
-
-      - name: Push to homebrew-archivebox tap
-        env:
-          GH_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }}
-        run: |
-          VERSION="${{ steps.version.outputs.version }}"
-          git clone "https://x-access-token:${GH_TOKEN}@github.com/ArchiveBox/homebrew-archivebox.git" /tmp/tap
-
-          cp brew_dist/archivebox.rb /tmp/tap/Formula/archivebox.rb 2>/dev/null || \
-            cp brew_dist/archivebox.rb /tmp/tap/archivebox.rb
-
-          cd /tmp/tap
-          git config user.name "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-          git add -A
-          if git diff --cached --quiet; then
-            echo "[i] No changes to formula, skipping push."
-          else
-            git commit -m "Update archivebox to v${VERSION}"
-            git push origin HEAD
-            echo "[√] Formula pushed to homebrew-archivebox tap"
-          fi
@@ -43,8 +43,8 @@ jobs:

      - name: Install dependencies with uv
        run: |
-          uv sync --extra sonic --extra debug --all-groups --no-sources --no-cache
-          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
+          uv venv
+          uv pip install --all-groups -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[sonic,debug]"

      - name: Run prek
        run: uv run --no-sync --no-sources prek run --all-files
@@ -22,6 +22,7 @@ jobs:
      id-token: write

    runs-on: ubuntu-24.04
+    environment: pypi
    steps:
      - uses: actions/checkout@v4

@@ -42,6 +43,44 @@ jobs:
          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
          version: 1.0

+      - name: Wait for released ArchiveBox deps on PyPI
+        run: |
+          python - <<'PY'
+          import json
+          import re
+          import sys
+          import time
+          import tomllib
+          import urllib.request
+
+          watched = {"abxbus", "abxpkg", "abx-plugins", "abx-dl"}
+          deps = tomllib.loads(open("pyproject.toml", "rb").read().decode())["project"]["dependencies"]
+          required = {}
+          for dep in deps:
+              for name in watched:
+                  match = re.match(rf"{re.escape(name)}\s*(==|>=)\s*([^,;\s]+)", dep)
+                  if match:
+                      required[name] = match.group(2)
+
+          deadline = time.monotonic() + 300
+          missing = required.copy()
+          while missing and time.monotonic() < deadline:
+              for name, version in list(missing.items()):
+                  with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/json", timeout=20) as resp:
+                      releases = json.load(resp)["releases"]
+                  if version in releases:
+                      print(f"{name} {version} is available on PyPI")
+                      missing.pop(name)
+                  else:
+                      print(f"{name} {version} is not available on PyPI yet")
+              if missing:
+                  time.sleep(10)
+
+          if missing:
+              print(f"Missing PyPI releases after wait: {missing}", file=sys.stderr)
+              sys.exit(1)
+          PY
+
      - name: UV install archivebox dev + run sub-dependencies
        run: uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache

@@ -2,8 +2,8 @@ name: Release

 # Orchestrates the full release pipeline:
 #   1. Build and publish pip package to PyPI
-#   2. Build .deb packages and Homebrew formula (in parallel, after pip)
-#   3. Build Docker images (in parallel with deb/brew)
+#   2. Build Homebrew formula after pip
+#   3. Build Docker images in parallel
 #
 # Individual workflows also run on push for CI (see their own triggers).
 # Manual fallback for the hosted release pipeline. The normal prerelease loop
@@ -28,12 +28,6 @@ jobs:
    uses: ./.github/workflows/pip.yml
    secrets: inherit

-  debian:
-    name: Build .deb packages
-    needs: pip
-    uses: ./.github/workflows/debian.yml
-    secrets: inherit
-
  homebrew:
    name: Update Homebrew formula
    needs: pip
@@ -49,8 +49,12 @@ jobs:

            # Extract a display name for the test
            name="main/$(basename $test_file .py | sed 's/^test_//')"
+            needs_chromium=false
+            if grep -Eiq '(chrom|archivewebpage|PLUGINS=.*title|--plugins=.*title|SAVE_TITLE.*[Tt]rue)' "$test_file"; then
+              needs_chromium=true
+            fi

-            json_array+="{\"path\":\"$test_file\",\"name\":\"$name\"}"
+            json_array+="{\"path\":\"$test_file\",\"name\":\"$name\",\"needs_chromium\":$needs_chromium}"
          done
          json_array+="]"

@@ -151,13 +155,54 @@ jobs:

      - name: Install dependencies with uv
        run: |
-          uv sync --dev --all-extras --no-sources --no-cache
-          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
+          uv venv
+          uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
+
+      - name: Install Chrome through ArchiveBox
+        if: ${{ matrix.test.needs_chromium }}
+        env:
+          MIN_CHROMIUM_MAJOR: "149"
+          LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_INSTALL_TIMEOUT: "900"
+          ABX_CI_DATA_DIR: ${{ runner.temp }}/archivebox-ci-data
+        run: |
+          set -euo pipefail
+          mkdir -p "$LIB_DIR" "$ABX_CI_DATA_DIR"
+          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
+          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
+
+          uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox install chrome
+
+          candidate="$(uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="chromium", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
+          if [ ! -x "$candidate" ]; then
+            echo "ArchiveBox did not install an executable Chromium binary: ${candidate:-not found}" >&2
+            exit 1
+          fi
+          version="$("$candidate" --version || true)"
+          major="$(printf '%s\n' "$version" | sed -E 's/.* ([0-9]+)\..*/\1/' | head -1)"
+          case "$major" in
+            ''|*[!0-9]*) major=0 ;;
+          esac
+          if [ "$major" -lt "$MIN_CHROMIUM_MAJOR" ]; then
+            echo "ArchiveBox installed Chromium is too old: ${version:-unknown}" >&2
+            exit 1
+          fi
+          echo "Using ArchiveBox-installed Chromium: $candidate ($version)"
+          echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV"

      - name: Run test - ${{ matrix.test.name }}
        run: |
          mkdir -p tests/out
+          set +e
          uv run --no-sync --no-sources pytest -xvs "${{ matrix.test.path }}" --basetemp=tests/out --ignore=archivebox/pkgs
+          status=$?
+          set -e
+          if [ "$status" -eq 5 ]; then
+            echo "No tests collected from ${{ matrix.test.path }}; treating as an empty test module."
+            exit 0
+          fi
+          exit "$status"

  plugin-tests:
    name: ${{ matrix.plugin.name }}
@@ -221,58 +266,28 @@ jobs:

      - name: Install dependencies with uv
        run: |
-          uv sync --dev --all-extras --no-sources --no-cache
-          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
+          uv venv
+          uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"

-      - name: Select Chromium for extension-capable CDP
+      - name: Install Chrome through ArchiveBox
        if: ${{ matrix.plugin.needs_chromium }}
        env:
          MIN_CHROMIUM_MAJOR: "149"
          LIB_DIR: ${{ runner.temp }}/abx-lib
          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_INSTALL_TIMEOUT: "900"
+          ABX_CI_DATA_DIR: ${{ runner.temp }}/archivebox-ci-data
        run: |
          set -euo pipefail
-          mkdir -p "$LIB_DIR"
+          mkdir -p "$LIB_DIR" "$ABX_CI_DATA_DIR"
          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"

-          candidate="/usr/bin/chromium"
-          version=""
-          major=0
-          if [ -x "$candidate" ]; then
-            version="$("$candidate" --version || true)"
-            major="$(printf '%s\n' "$version" | sed -E 's/.* ([0-9]+)\..*/\1/' | head -1)"
-            case "$major" in
-              ''|*[!0-9]*) major=0 ;;
-            esac
-          fi
+          uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox install chrome

-          if [ "$major" -ge "$MIN_CHROMIUM_MAJOR" ]; then
-            echo "Using runner Chromium: $candidate ($version)"
-            echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV"
-            exit 0
-          fi
-
-          echo "Runner Chromium is unavailable or too old: ${version:-not found}"
-          echo "Installing Playwright Chromium into $LIB_DIR..."
-          uv run --no-sync --no-sources abxpkg install \
-            --binproviders=npm \
-            --lib="$LIB_DIR" \
-            --postinstall-scripts=false \
-            --min-release-age=0 \
-            --no-cache \
-            --overrides='{"npm":{"install_args":["playwright@next"]}}' \
-            playwright
-          uv run --no-sync --no-sources abxpkg install \
-            --binproviders=playwright \
-            --lib="$LIB_DIR" \
-            --bin-dir="$LIB_DIR/env/bin" \
-            --min-version="$MIN_CHROMIUM_MAJOR" \
-            chromium
-
-          candidate="$LIB_DIR/env/bin/chromium"
+          candidate="$(uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="chromium", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
          if [ ! -x "$candidate" ]; then
-            echo "Playwright Chromium shim was not created at $candidate" >&2
+            echo "ArchiveBox did not install an executable Chromium binary: ${candidate:-not found}" >&2
            exit 1
          fi
          version="$("$candidate" --version || true)"
@@ -281,10 +296,10 @@ jobs:
            ''|*[!0-9]*) major=0 ;;
          esac
          if [ "$major" -lt "$MIN_CHROMIUM_MAJOR" ]; then
-            echo "Playwright Chromium is still too old: ${version:-unknown}" >&2
+            echo "ArchiveBox installed Chromium is too old: ${version:-unknown}" >&2
            exit 1
          fi
-          echo "Using Playwright Chromium: $candidate ($version)"
+          echo "Using ArchiveBox-installed Chromium: $candidate ($version)"
          echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV"

      - name: Run plugin tests - ${{ matrix.plugin.name }}
@@ -72,8 +72,8 @@ jobs:

      - name: Install dependencies with uv
        run: |
-          uv sync --dev --all-extras --no-sources --no-cache
-          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
+          uv venv
+          uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"

      ### Run the tests
      - name: Directory listing for debugging
@@ -0,0 +1,35 @@
+name: Update Homebrew tap
+
+on:
+  push:
+    branches: [dev]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: update-homebrew-tap-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  dispatch:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Dispatch tap update
+        env:
+          HOMEBREW_TAP_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }}
+        run: |
+          if [ -z "$HOMEBREW_TAP_TOKEN" ]; then
+            echo "HOMEBREW_TAP_TOKEN is not configured; tap hourly fallback will update the formula."
+            exit 0
+          fi
+
+          curl --fail-with-body \
+            -X POST \
+            -H "Accept: application/vnd.github+json" \
+            -H "Authorization: Bearer ${HOMEBREW_TAP_TOKEN}" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            https://api.github.com/repos/ArchiveBox/homebrew-archivebox/dispatches \
+            -d '{"event_type":"archivebox-dev-updated"}'
@@ -1,38 +1,152 @@
-# This is the Dockerfile for ArchiveBox, it bundles the following main dependencies:
-#     python3.13, uv, python3-ldap
-#     curl, wget, git, dig, ping, tree, nano
-#     node, npm
-#     ArchiveBox and plugin runtime dependencies installed by archivebox init --install
-# Usage:
-#     git clone https://github.com/ArchiveBox/ArchiveBox && cd ArchiveBox
-#     docker build . -t archivebox
-#     docker run -v "$PWD/data":/data archivebox init
-#     docker run -v "$PWD/data":/data archivebox add 'https://example.com'
-#     docker run -v "$PWD/data":/data -it archivebox manage createsuperuser
-#     docker run -v "$PWD/data":/data -p 8000:8000 archivebox server
-#     docker buildx build . --platform=linux/amd64,linux/arm64 --push -t archivebox/archivebox:dev -t archivebox/archivebox:sha-abc123
-# Read more here: https://github.com/ArchiveBox/ArchiveBox#archivebox-development
+# syntax=docker/dockerfile:1.7

+# Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image.
+# abx-dl owns Python, Node, Chromium, and downloader plugin runtimes.
+# ArchiveBox owns ripgrep, sonic, supervisor, Django, and the app runtime.
+# Build abx-dl first, then point this file at it:
+#   docker buildx build ../abx-dl -f ../abx-dl/Dockerfile \
+#       --build-context abxbus=../abxbus \
+#       --build-context abxpkg=../abxpkg \
+#       --build-context abx-plugins=../abx-plugins \
+#       -t archivebox/abx-dl:dev
+#   docker buildx build . -f Dockerfile \
+#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:latest \
+#       -t archivebox:multistage

-#########################################################################################
+ARG ABX_DL_IMAGE=archivebox/abx-dl:latest

-### Example: Using ArchiveBox in your own project's Dockerfile ########
-
-# FROM python:3.13-slim
-# WORKDIR /data
-# RUN pip install archivebox>=0.9.0   # use latest release here
-# RUN archivebox install
-# RUN useradd -ms /bin/bash archivebox && chown -R archivebox /data
-
-#########################################################################################
+FROM archivebox/sonic:1.4.9 AS sonic
+FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base

 ARG TARGETPLATFORM
 ARG TARGETOS
 ARG TARGETARCH
-ARG TARGETVARIANT=
+ARG TARGETVARIANT

-FROM archivebox/sonic:1.4.9 AS sonic
-FROM ubuntu:24.04
+ENV TZ=UTC \
+    LANGUAGE=en_US:en \
+    LC_ALL=C.UTF-8 \
+    LANG=C.UTF-8 \
+    DEBIAN_FRONTEND=noninteractive \
+    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
+    PYTHONIOENCODING=UTF-8 \
+    PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    PIP_NO_COMPILE=1 \
+    PIP_ONLY_BINARY=aiohttp \
+    npm_config_loglevel=error
+
+ENV PYTHON_VERSION=3.13 \
+    NODE_VERSION=24
+
+ENV ARCHIVEBOX_USER=archivebox \
+    DEFAULT_PUID=911 \
+    DEFAULT_PGID=911 \
+    IN_DOCKER=True
+
+ENV CODE_DIR=/app \
+    DATA_DIR=/data \
+    LIB_DIR=/opt/archivebox/lib \
+    ABXPKG_LIB_DIR=/opt/archivebox/lib \
+    PLAYWRIGHT_BROWSERS_PATH=/opt/archivebox/lib/playwright/cache \
+    PERSONAS_DIR=/data/personas \
+    CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \
+    CHROME_HEADLESS=true \
+    CHROME_SANDBOX=false \
+    CHROME_ISOLATION=crawl \
+    CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
+
+ENV TMP_DIR=/tmp/archivebox \
+    PIP_VENV_PYTHON=/venv/bin/python3 \
+    GOOGLE_API_KEY=no \
+    GOOGLE_DEFAULT_CLIENT_ID=no \
+    GOOGLE_DEFAULT_CLIENT_SECRET=no
+
+ENV HOME=/home/archivebox \
+    XDG_CONFIG_HOME=/home/archivebox/.config \
+    XDG_CACHE_HOME=/home/archivebox/.cache \
+    ABXPKG_INSTALL_TIMEOUT=600 \
+    ABXPKG_POSTINSTALL_SCRIPTS=True \
+    ABXPKG_MIN_RELEASE_AGE=0 \
+    TIMEOUT=600
+
+ENV UV_COMPILE_BYTECODE=false \
+    UV_PYTHON_PREFERENCE=managed \
+    UV_PYTHON_INSTALL_DIR=/opt/uv/python \
+    ARCHIVEBOX_PYTHON_INSTALL_DIR=/opt/archivebox/python \
+    UV_LINK_MODE=copy \
+    UV_PROJECT_ENVIRONMENT=/venv \
+    VIRTUAL_ENV=/venv \
+    PATH="/venv/bin:/opt/node/bin:/opt/archivebox/lib/bin:$PATH"
+
+SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
+WORKDIR "$CODE_DIR"
+
+RUN cp /VERSION.txt /ABX-DL-VERSION.txt \
+    && (echo "[i] Docker build for ArchiveBox multistage starting..." \
+    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
+    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
+    && uname -a \
+    && sed -n '1,7p' /etc/os-release \
+    && which node && node --version \
+    && which uv && uv self version \
+    ) | tee -a /VERSION.txt
+
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    echo "[+] APT Installing ArchiveBox search dependency ripgrep for $TARGETPLATFORM..." \
+    && apt-get update -qq \
+    && apt-get install -qq -y --no-install-recommends ripgrep \
+    && rm -rf /var/lib/apt/lists/*
+
+FROM archivebox-runtime-base AS archivebox-builder
+
+WORKDIR "$CODE_DIR"
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
+    echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..." \
+    && export UV_PYTHON_INSTALL_DIR="$ARCHIVEBOX_PYTHON_INSTALL_DIR" \
+    && echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
+    && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends \
+    && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests \
+    && rm -f /etc/apt/apt.conf.d/docker-clean \
+    && apt-get update -qq \
+    && apt-get install -qq -y --no-install-recommends \
+        build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
+    && /usr/bin/uv venv --clear /venv --python "${PYTHON_VERSION}" \
+    && /usr/bin/uv pip install setuptools pip wheel \
+    && /usr/bin/uv sync \
+        --refresh \
+        --no-dev \
+        --inexact \
+        --all-extras \
+        --no-install-project \
+        --no-install-workspace \
+        --no-sources \
+    && apt-get purge -y build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
+    && apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
+RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
+    && COMMIT_HASH="$( \
+        if [[ -f "$CODE_DIR/.git/HEAD" ]]; then \
+            HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"; \
+            if [[ "$HEAD_REF" =~ ^[0-9a-fA-F]{40}$ ]]; then \
+                echo "$HEAD_REF"; \
+            elif [[ "$HEAD_REF" == ref:\ * ]]; then \
+                REF_PATH="${HEAD_REF#ref: }"; \
+                cat "$CODE_DIR/.git/$REF_PATH" 2>/dev/null || awk -v ref="$REF_PATH" '$2 == ref {print $1}' "$CODE_DIR/.git/packed-refs" 2>/dev/null || true; \
+            fi; \
+        fi)" \
+    && if [[ "$COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt; fi \
+    && /usr/bin/uv pip install --no-deps "$CODE_DIR" \
+    && (/usr/bin/uv pip show archivebox && which archivebox) | tee -a /VERSION.txt \
+    && rm -rf "$CODE_DIR/.git"
+
+FROM archivebox-runtime-base

 LABEL name="archivebox" \
    maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
@@ -43,356 +157,64 @@ LABEL name="archivebox" \
    org.opencontainers.image.vendor="ArchiveBox" \
    org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \
    org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \
-    com.docker.image.source.entrypoint="Dockerfile" \
-    # TODO: release ArchiveBox as a Docker Desktop extension (requires these labels):
-    # https://docs.docker.com/desktop/extensions-sdk/architecture/metadata/
-    com.docker.desktop.extension.api.version=">= 1.4.7" \
-    com.docker.desktop.extension.icon="https://archivebox.io/icon.png" \
-    com.docker.extension.publisher-url="https://archivebox.io" \
-    com.docker.extension.screenshots='[{"alt": "Screenshot of Admin UI", "url": "https://github.com/ArchiveBox/ArchiveBox/assets/511499/e8e0b6f8-8fdf-4b7f-8124-c10d8699bdb2"}]' \
-    com.docker.extension.detailed-description='See here for detailed documentation: https://wiki.archivebox.io' \
-    com.docker.extension.changelog='See here for release notes: https://github.com/ArchiveBox/ArchiveBox/releases' \
-    com.docker.extension.categories='database,utility-tools'
+    com.docker.image.source.entrypoint="Dockerfile"

-ARG TARGETPLATFORM
-ARG TARGETOS
-ARG TARGETARCH
-ARG TARGETVARIANT
-######### Environment Variables #################################
-
-# Global build-time and runtime environment constants + default pkg manager config
-ENV TZ=UTC \
-    LANGUAGE=en_US:en \
-    LC_ALL=C.UTF-8 \
-    LANG=C.UTF-8 \
-    DEBIAN_FRONTEND=noninteractive \
-    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
-    PYTHONIOENCODING=UTF-8 \
-    PYTHONUNBUFFERED=1 \
-    PIP_DISABLE_PIP_VERSION_CHECK=1 \
-    npm_config_loglevel=error
-
-# Language Version config
-ENV PYTHON_VERSION=3.13 \
-    NODE_VERSION=22.22.3
-
-# Non-root User config
-ENV ARCHIVEBOX_USER="archivebox" \
-    DEFAULT_PUID=911 \
-    DEFAULT_PGID=911 \
-    IN_DOCKER=True
-
-# ArchiveBox Source Code + Lib + Data paths
-ENV CODE_DIR=/app \
-    DATA_DIR=/data \
-    LIB_DIR=/opt/archivebox/lib \
-    ABXPKG_LIB_DIR=/opt/archivebox/lib \
-    SONIC_BINARY=/opt/archivebox/lib/env/bin/sonic \
-    PLAYWRIGHT_BROWSERS_PATH=/browsers
-
-# Bash SHELL config
-# http://redsymbol.net/articles/unofficial-bash-strict-mode/
-SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] 
-
-######### System Environment ####################################
-
-# Detect ArchiveBox version number by reading pyproject.toml (also serves to invalidate the entire build cache whenever pyproject.toml changes)
-WORKDIR "$CODE_DIR"
-
-# Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up back-to-back Docker builds)
-RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
-    && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-intall-recommends \
-    && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-intall-suggests \
-    && rm -f /etc/apt/apt.conf.d/docker-clean
-
-# Print debug info about build and save it to disk, for human eyes only, not used by anything else
-RUN (echo "[i] Docker build for ArchiveBox starting..." \
-    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})" \
-    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
-    && echo \
-    && echo "PYTHON=${PYTHON_VERSION} NODE=${NODE_VERSION} PATH=${PATH}" \
-    && echo "CODE_DIR=${CODE_DIR} DATA_DIR=${DATA_DIR}" \
-    && echo \
-    && uname -a \
-    && sed -n '1,7p' /etc/os-release \
-    && which bash && bash --version | sed -n '1p' \
-    && which dpkg && dpkg --version | sed -n '1p' \
-    && echo -e '\n\n' && env && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
-# Create non-privileged user for archivebox and chrome
-RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
-    && groupadd --system $ARCHIVEBOX_USER \
-    && useradd --system --create-home --gid $ARCHIVEBOX_USER --groups audio,video $ARCHIVEBOX_USER \
-    && usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
-    && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
-    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u $ARCHIVEBOX_USER) PGID=$(id -g $ARCHIVEBOX_USER)\n\n" \
-    | tee -a /VERSION.txt
-    # DEFAULT_PUID and DEFAULT_PID are overridden by PUID and PGID in /bin/docker_entrypoint.sh at runtime
-    # https://docs.linuxserver.io/general/understanding-puid-and-pgid
-
-# Install system apt dependencies (adding backports to access more recent apt updates)
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    echo "[+] APT Installing base system dependencies for $TARGETPLATFORM..." \
-    && mkdir -p /etc/apt/keyrings \
-    && apt-get update -qq \
-    && apt-get install -qq -y \
-        # 1. packaging dependencies
-        apt-transport-https ca-certificates apt-utils gnupg2 curl wget \
-        # 2. docker and init system dependencies
-        zlib1g-dev dumb-init gosu cron unzip grep dnsutils git ripgrep python3.12-venv default-jre-headless \
-        # 3. frivolous CLI helpers to make debugging failed archiving easier
-        tree nano iputils-ping \
-        # nano iputils-ping dnsutils htop procps jq yq
-    && rm -rf /var/lib/apt/lists/*
-
-# Install sonic search backend
 COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
 COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
-RUN (which sonic && sonic --version) | tee -a /VERSION.txt

-######### Language Environments ####################################
+COPY --from=archivebox-builder /opt/archivebox/python /opt/archivebox/python
+COPY --from=archivebox-builder /venv /venv
+COPY --from=archivebox-builder /app /app
+COPY --from=archivebox-builder /VERSION.txt /VERSION.txt

-# Set up Python environment
-# NOT NEEDED because we're using a pre-built python image, keeping this here in case we switch back to custom-building our own:
-#RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-#    --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-# RUN echo "[+] APT Installing PYTHON $PYTHON_VERSION for $TARGETPLATFORM (skipped, provided by base image)..." \
-    # && apt-get update -qq \
-    # && apt-get install -qq -y --no-upgrade \
-    #     python${PYTHON_VERSION} python${PYTHON_VERSION}-minimal python3-pip python${PYTHON_VERSION}-venv pipx \
-    # && rm -rf /var/lib/apt/lists/* \
-    # tell PDM to allow using global system python site packages
-    # && rm /usr/lib/python3*/EXTERNALLY-MANAGED \
-    # && ln -s "$(which python${PYTHON_VERSION})" /usr/bin/python \
-    # create global virtual environment GLOBAL_VENV to use (better than using pip install --global)
-    # && python3 -m venv --system-site-packages --symlinks $GLOBAL_VENV \
-    # && python3 -m venv --system-site-packages $GLOBAL_VENV \
-    # && python3 -m venv $GLOBAL_VENV \
-    # install global dependencies / python build dependencies in GLOBAL_VENV
-    # && pip install --upgrade pip setuptools wheel \
-    # Save version info
-    # && ( \
-    #     which python3 && python3 --version | grep " $PYTHON_VERSION" \
-    #     && which pip && pip --version \
-    #     # && which pdm && pdm --version \
-    #     && echo -e '\n\n' \
-    # ) | tee -a /VERSION.txt
+RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
+    && ln -sf /venv/bin/archivebox /usr/local/bin/archivebox \
+    && getent group "$ARCHIVEBOX_USER" >/dev/null || groupadd --system "$ARCHIVEBOX_USER" \
+    && id -u "$ARCHIVEBOX_USER" >/dev/null 2>&1 || useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \
+    && usermod --append --groups audio,video "$ARCHIVEBOX_USER" \
+    && [[ "$(id -u "$ARCHIVEBOX_USER")" == "$DEFAULT_PUID" ]] || usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
+    && [[ "$(id -g "$ARCHIVEBOX_USER")" == "$DEFAULT_PGID" ]] || groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
+    && (which sonic && sonic --version) | tee -a /VERSION.txt \
+    && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
+    && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER" "/home/$ARCHIVEBOX_USER/.config" "/home/$ARCHIVEBOX_USER/.cache" \
+    && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER/.config/abx" "/home/$ARCHIVEBOX_USER/.cache/abxbus" "/home/$ARCHIVEBOX_USER/.cache/pnpm" "/home/$ARCHIVEBOX_USER/.cache/uv" \
+    && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
+    && openssl rand -hex 16 > /etc/machine-id \
+    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u "$ARCHIVEBOX_USER") PGID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
+    && echo -e "TMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt

-
-# Set up Node environment from the official platform tarball. This avoids
-# NodeSource apt dependencies pulling Ubuntu's python3-minimal postinst into
-# emulated Docker builds.
-RUN --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
-    case "$TARGETARCH" in \
-        amd64) NODE_DIST_ARCH="x64" ;; \
-        arm64) NODE_DIST_ARCH="arm64" ;; \
-        *) echo "Unsupported TARGETARCH=$TARGETARCH for Node binary install" >&2; exit 1 ;; \
-    esac \
-    && NODE_TARBALL="node-v${NODE_VERSION}-linux-${NODE_DIST_ARCH}.tar.gz" \
-    && echo "[+] Installing NODE $NODE_VERSION for linux/${NODE_DIST_ARCH}..." \
-    && curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TARBALL}" \
-    && curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/SHASUMS256.txt" \
-    && grep "  ${NODE_TARBALL}$" SHASUMS256.txt | sha256sum -c - \
-    && tar -xzf "$NODE_TARBALL" -C /usr/local --strip-components=1 --no-same-owner \
-    && rm "$NODE_TARBALL" SHASUMS256.txt \
-    # Save version info
-    && ( \
-        which node && node --version \
-        && which npm && npm --version \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
-
-# Set up uv and main app /venv
-RUN curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR=/bin sh
-ENV UV_COMPILE_BYTECODE=0 \
-    UV_PYTHON_PREFERENCE=managed \
-    UV_PYTHON_INSTALL_DIR=/opt/uv/python \
-    UV_LINK_MODE=copy \
-    UV_PROJECT_ENVIRONMENT=/venv
-WORKDIR "$CODE_DIR"
-# COPY --chown=root:root --chmod=755 pyproject.toml "$CODE_DIR/"
-RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[+] UV Creating /venv using python ${PYTHON_VERSION} for ${TARGETPLATFORM}..." \
-    && uv venv /venv --python ${PYTHON_VERSION}
-ENV VIRTUAL_ENV=/venv PATH="/venv/bin:$PATH"
-RUN uv pip install setuptools pip \
-    && ( \
-        which python3 && python3 --version \
-        && which uv && uv self version \
-        && uv python find --system && uv python find \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
-
-######### ArchiveBox & Extractor Dependencies ##################################
-
-# Install ArchiveBox C-compiled/apt-installed Python dependencies in app /venv (currently only used for python-ldap)
-WORKDIR "$CODE_DIR"
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    #--mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-    echo "[+] APT Installing + Compiling python3-ldap for PIP archivebox[ldap] on ${TARGETPLATFORM}..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends \
-        build-essential gcc \
-        python3-dev libssl-dev libldap2-dev libsasl2-dev python3-ldap \
-        python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \
-    && uv pip install \
-        "python-ldap>=3.4.3" \
-    && apt-get purge -y \
-        python3-dev build-essential gcc \
-    && apt-get autoremove -y \
-    && rm -rf /var/lib/apt/lists/*
-
-
-# Runtime config used by plugin hooks. Plugin binaries and npm packages are
-# installed into LIB_DIR below by archivebox init --install and resolved from
-# LIB_DIR by ArchiveBox/abxpkg, not by mutating the container PATH.
-ENV PERSONAS_DIR=/data/personas \
-    CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \
-    CHROME_HEADLESS=true \
-    CHROME_SANDBOX=false \
-    CHROME_ISOLATION=crawl \
-    CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
-
-######### Build Dependencies ####################################
-
-
-# Install ArchiveBox Python venv dependencies from pyproject.toml.
-RUN --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
-    --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[+] PIP Installing ArchiveBox dependencies from pyproject.toml..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends build-essential gcc python3-dev \
-    && uv --no-cache sync \
-        --refresh \
-        --no-dev \
-        --inexact \
-        --all-extras \
-        --no-install-project \
-        --no-install-workspace \
-        --no-sources \
-    && apt-get purge -y python3-dev build-essential gcc \
-    && apt-get autoremove -y \
-    && find /venv -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && rm -rf /var/lib/apt/lists/*
-    # installs the pip packages that archivebox depends on, defined in pyproject.toml dependencies
-
-# Setup ArchiveBox runtime config
-ENV TMP_DIR=/tmp/archivebox \
-    PIP_VENV_PYTHON=/usr/bin/python3.12 \
-    GOOGLE_API_KEY=no \
-    GOOGLE_DEFAULT_CLIENT_ID=no \
-    GOOGLE_DEFAULT_CLIENT_SECRET=no
-
-WORKDIR "$DATA_DIR"
-RUN openssl rand -hex 16 > /etc/machine-id \
-    && mkdir -p "$DATA_DIR" \
-    && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" \
-    && mkdir -p "$TMP_DIR" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$TMP_DIR" \
-    && mkdir -p "$LIB_DIR" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
-    && mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \
-    && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && echo -e "\nTMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
-
-# Pre-bake plugin-managed runtime dependencies using the same abx-dl installer
-# path users run later, before copying ArchiveBox source so source-only edits do
-# not invalidate the heavy browser/plugin dependency layer.
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/puppeteer,sharing=locked,id=puppeteer-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing plugin runtime dependencies into $LIB_DIR..." \
-    && export PERSONAS_DIR="$LIB_DIR/personas" \
-    && export CHROME_USER_DATA_DIR="$LIB_DIR/chrome_profile" \
-    && mkdir -p "$LIB_DIR" \
-    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends build-essential \
-    && if [ "$TARGETARCH" = "arm64" ]; then \
-        abxpkg install --binproviders=npm --overrides='{"npm":{"install_args":["playwright@next"]}}' playwright; \
-        abxpkg install --no-cache --install-timeout=600 --binproviders=playwright --bin-dir="$LIB_DIR/env/bin" chromium; \
-    fi \
-    && TIMEOUT=600 PUID=0 PGID=0 abx-dl plugins --install \
-    && abxpkg install --no-cache --binproviders=chromewebstore --overrides='{"chromewebstore":{"install_args":["fpeoodllldobpkbkabpblcfaogecpndd","--name=archivewebpage"]}}' archivewebpage \
-    && test -f "$LIB_DIR/chromewebstore/extensions/fpeoodllldobpkbkabpblcfaogecpndd__archivewebpage/manifest.json" \
-    && mkdir -p "$LIB_DIR/env/bin" \
-    && ln -sf "$(command -v node)" "$LIB_DIR/env/bin/node" \
-    && ln -sf "$(command -v npm)" "$LIB_DIR/env/bin/npm" \
-    && ln -sf "$(command -v java)" "$LIB_DIR/env/bin/java" \
-    && ln -sf "$(command -v rg)" "$LIB_DIR/env/bin/rg" \
-    && ln -sf "$(command -v sonic)" "$LIB_DIR/env/bin/sonic" \
-    && find "$LIB_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find "$LIB_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && rm -rf "$LIB_DIR/personas" "$LIB_DIR/chrome_profile" /opt/archivebox/lib-layer \
-    && mkdir -p /opt/archivebox/lib-layer \
-    && cp -a "$LIB_DIR"/. /opt/archivebox/lib-layer/ \
-    && apt-get purge -y build-essential \
-    && apt-get autoremove -y \
-    && rm -rf /var/lib/apt/lists/* \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" /opt/archivebox/lib-layer
-
-RUN rm -rf "$LIB_DIR" \
-    && mv /opt/archivebox/lib-layer "$LIB_DIR" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR"
-
-# Install ArchiveBox Python package from the checked-out source.
-WORKDIR "$CODE_DIR"
-COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
-RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
-    && pip install \
-        --no-deps \
-        "$CODE_DIR" \
-    && ( \
-        pip show archivebox \
-        && which archivebox \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt \
-    && find /venv "$CODE_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete
-    # installs archivebox itself, and any other vendored packages in pkgs/*, defined in pyproject.toml workspaces
-
-# Initialize an empty image collection without rerunning dependency installs.
 WORKDIR "$DATA_DIR"
 RUN echo "[+] Initializing image collection..." \
+    && find "$DATA_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} + \
    && PUID=0 PGID=0 archivebox init \
-    && find "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
    && (chown "$DEFAULT_PUID:$DEFAULT_PGID" \
        "$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \
        "$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \
        "$DATA_DIR"/archive "$DATA_DIR"/archive/users "$DATA_DIR"/personas \
        "$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \
-        2>/dev/null || true)
+        "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
+        2>/dev/null || true) \
+    && find "$TMP_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} +

-# Print version for nice docker finish summary
-RUN (echo -e "\n\n[√] Finished Docker build successfully. Saving build summary in: /VERSION.txt" \
-    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})\n" \
+RUN chmod +x "$CODE_DIR"/bin/*.sh \
+    && chmod g+w "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"
+
+RUN "$LIB_DIR/bin/chromium" --version | tee -a /VERSION.txt \
+    && "$LIB_DIR/uv/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \
+    && /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \
+    && /usr/local/bin/sonic --version | tee -a /VERSION.txt \
+    && /venv/bin/supervisord --version | tee -a /VERSION.txt \
+    && for forbidden_bin in gcc g++ make; do ! command -v "$forbidden_bin" || (echo "Unexpected build tool in runtime: $forbidden_bin=$(command -v "$forbidden_bin")" >&2 && exit 1); done \
+    && stat -c "%U:%G %a %n" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$LIB_DIR" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \
+    && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
+
+RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully." \
+    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
    && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
    ) | tee -a /VERSION.txt

-# Verify ArchiveBox is installed and write full version/dependency info.
-RUN chmod +x "$CODE_DIR"/bin/*.sh \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
-    && chmod g+w "$TMP_DIR" "$LIB_DIR" "$LIB_DIR"/bin "$PLAYWRIGHT_BROWSERS_PATH" \
-    && TIMEOUT=600 gosu "$ARCHIVEBOX_USER" archivebox install 2>&1 | tee -a /VERSION.txt \
-    && TIMEOUT=600 gosu "$ARCHIVEBOX_USER" archivebox install archivewebpage search_backend_sonic opendataloader search_backend_ripgrep 2>&1 | tee -a /VERSION.txt \
-    && gosu "$ARCHIVEBOX_USER" archivebox version 2>&1 | tee -a /VERSION.txt \
-    && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
-
-####################################################
-
-# Expose ArchiveBox's main interfaces to the outside world
 WORKDIR "$DATA_DIR"
 VOLUME "$DATA_DIR"
 EXPOSE 8000
@@ -9,24 +9,18 @@
 #       --build-context abxpkg=../abxpkg \
 #       --build-context abx-plugins=../abx-plugins \
 #       -t archivebox/abx-dl:dev
-#   docker buildx build . -f Dockerfile.multistage \
-#       --build-context abx-dl=docker-image://archivebox/abx-dl:dev \
-#       --build-arg ABX_DL_IMAGE=abx-dl \
+#   docker buildx build . -f Dockerfile \
+#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:latest \
 #       -t archivebox:multistage

-ARG TARGETPLATFORM=linux/amd64
-ARG TARGETOS=linux
-ARG TARGETARCH=amd64
-ARG TARGETVARIANT=
-ARG ABX_DL_IMAGE=abx-dl
+ARG ABX_DL_IMAGE=archivebox/abx-dl:latest

-FROM ${ABX_DL_IMAGE} AS abx-dl
 FROM archivebox/sonic:1.4.9 AS sonic
-FROM ubuntu:24.04 AS archivebox-runtime-base
+FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base

-ARG TARGETPLATFORM=linux/amd64
-ARG TARGETOS=linux
-ARG TARGETARCH=amd64
+ARG TARGETPLATFORM
+ARG TARGETOS
+ARG TARGETARCH
 ARG TARGETVARIANT

 ENV TZ=UTC \
@@ -37,7 +31,10 @@ ENV TZ=UTC \
    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
    PYTHONIOENCODING=UTF-8 \
    PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    PIP_NO_COMPILE=1 \
+    PIP_ONLY_BINARY=aiohttp \
    npm_config_loglevel=error

 ENV PYTHON_VERSION=3.13 \
@@ -66,55 +63,41 @@ ENV TMP_DIR=/tmp/archivebox \
    GOOGLE_DEFAULT_CLIENT_ID=no \
    GOOGLE_DEFAULT_CLIENT_SECRET=no

-ENV UV_COMPILE_BYTECODE=0 \
+ENV HOME=/home/archivebox \
+    XDG_CONFIG_HOME=/home/archivebox/.config \
+    XDG_CACHE_HOME=/home/archivebox/.cache \
+    ABXPKG_INSTALL_TIMEOUT=600 \
+    ABXPKG_POSTINSTALL_SCRIPTS=True \
+    ABXPKG_MIN_RELEASE_AGE=0 \
+    TIMEOUT=600
+
+ENV UV_COMPILE_BYTECODE=false \
    UV_PYTHON_PREFERENCE=managed \
    UV_PYTHON_INSTALL_DIR=/opt/uv/python \
+    ARCHIVEBOX_PYTHON_INSTALL_DIR=/opt/archivebox/python \
    UV_LINK_MODE=copy \
    UV_PROJECT_ENVIRONMENT=/venv \
    VIRTUAL_ENV=/venv \
-    PATH="/venv/bin:/opt/node/bin:$PATH"
+    PATH="/venv/bin:/opt/node/bin:/opt/archivebox/lib/bin:$PATH"

 SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
 WORKDIR "$CODE_DIR"

-RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
-    && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends \
-    && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests \
-    && rm -f /etc/apt/apt.conf.d/docker-clean
-
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    echo "[+] APT Installing ArchiveBox base runtime dependencies for $TARGETPLATFORM..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y \
-        apt-transport-https apt-utils ca-certificates curl wget gnupg2 \
-        dumb-init gosu unzip git grep ripgrep dnsutils iputils-ping procps tree nano \
-        cron openssl xz-utils zlib1g libldap2 libsasl2-2 libssl3 libsqlite3-0 \
-        libasound2t64 libatk-bridge2.0-0 libatk1.0-0 libcairo2 libcups2 \
-        libdbus-1-3 libdrm2 libgbm1 libglib2.0-0 libgtk-3-0 libnspr4 libnss3 \
-        libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxdamage1 \
-        libxext6 libxfixes3 libxkbcommon0 libxrandr2 libxshmfence1 \
-        fonts-liberation fonts-noto-color-emoji xdg-utils \
-        ffmpeg imagemagick tesseract-ocr openjdk-21-jre-headless \
-    && rm -rf /var/lib/apt/lists/*
-
-# Runtime-owned layers copied from the abx-dl image.
-COPY --from=abx-dl /bin/uv /bin/uv
-COPY --from=abx-dl /opt/uv/python /opt/uv/python
-COPY --from=abx-dl /opt/node /opt/node
-COPY --from=abx-dl /venv /venv
-COPY --from=abx-dl /VERSION.txt /ABX-DL-VERSION.txt
-
-RUN (echo "[i] Docker build for ArchiveBox multistage starting..." \
+RUN cp /VERSION.txt /ABX-DL-VERSION.txt \
+    && (echo "[i] Docker build for ArchiveBox multistage starting..." \
    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
    && uname -a \
    && sed -n '1,7p' /etc/os-release \
-    && which python3 && python3 --version \
    && which node && node --version \
    && which uv && uv self version \
    ) | tee -a /VERSION.txt

-ENV PYTHONDONTWRITEBYTECODE=1
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    echo "[+] APT Installing ArchiveBox search dependency ripgrep for $TARGETPLATFORM..." \
+    && apt-get update -qq \
+    && apt-get install -qq -y --no-install-recommends ripgrep \
+    && rm -rf /var/lib/apt/lists/*

 FROM archivebox-runtime-base AS archivebox-builder

@@ -123,10 +106,17 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
    --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
    echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..." \
+    && export UV_PYTHON_INSTALL_DIR="$ARCHIVEBOX_PYTHON_INSTALL_DIR" \
+    && echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
+    && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends \
+    && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests \
+    && rm -f /etc/apt/apt.conf.d/docker-clean \
    && apt-get update -qq \
    && apt-get install -qq -y --no-install-recommends \
        build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
-    && uv sync \
+    && /usr/bin/uv venv --clear /venv --python "${PYTHON_VERSION}" \
+    && /usr/bin/uv pip install setuptools pip wheel \
+    && /usr/bin/uv sync \
        --refresh \
        --no-dev \
        --inexact \
@@ -136,17 +126,25 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
        --no-sources \
    && apt-get purge -y build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
    && apt-get autoremove -y \
-    && find /venv -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
    && rm -rf /var/lib/apt/lists/*

 COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
 RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
    echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
-    && uv pip install --no-deps "$CODE_DIR" \
-    && (uv pip show archivebox && which archivebox) | tee -a /VERSION.txt \
-    && find /venv "$CODE_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete
+    && COMMIT_HASH="$( \
+        if [[ -f "$CODE_DIR/.git/HEAD" ]]; then \
+            HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"; \
+            if [[ "$HEAD_REF" =~ ^[0-9a-fA-F]{40}$ ]]; then \
+                echo "$HEAD_REF"; \
+            elif [[ "$HEAD_REF" == ref:\ * ]]; then \
+                REF_PATH="${HEAD_REF#ref: }"; \
+                cat "$CODE_DIR/.git/$REF_PATH" 2>/dev/null || awk -v ref="$REF_PATH" '$2 == ref {print $1}' "$CODE_DIR/.git/packed-refs" 2>/dev/null || true; \
+            fi; \
+        fi)" \
+    && if [[ "$COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt; fi \
+    && /usr/bin/uv pip install --no-deps "$CODE_DIR" \
+    && (/usr/bin/uv pip show archivebox && which archivebox) | tee -a /VERSION.txt \
+    && rm -rf "$CODE_DIR/.git"

 FROM archivebox-runtime-base

@@ -159,56 +157,55 @@ LABEL name="archivebox" \
    org.opencontainers.image.vendor="ArchiveBox" \
    org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \
    org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \
-    com.docker.image.source.entrypoint="Dockerfile.multistage"
+    com.docker.image.source.entrypoint="Dockerfile"

 COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
 COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg

+COPY --from=archivebox-builder /opt/archivebox/python /opt/archivebox/python
 COPY --from=archivebox-builder /venv /venv
 COPY --from=archivebox-builder /app /app
 COPY --from=archivebox-builder /VERSION.txt /VERSION.txt
-COPY --from=abx-dl --chown=911:911 /opt/archivebox/lib /opt/archivebox/lib

 RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
-    && groupadd --system "$ARCHIVEBOX_USER" \
-    && useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \
-    && usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
-    && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
+    && getent group "$ARCHIVEBOX_USER" >/dev/null || groupadd --system "$ARCHIVEBOX_USER" \
+    && id -u "$ARCHIVEBOX_USER" >/dev/null 2>&1 || useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \
+    && usermod --append --groups audio,video "$ARCHIVEBOX_USER" \
+    && [[ "$(id -u "$ARCHIVEBOX_USER")" == "$DEFAULT_PUID" ]] || usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
+    && [[ "$(id -g "$ARCHIVEBOX_USER")" == "$DEFAULT_PGID" ]] || groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
    && (which sonic && sonic --version) | tee -a /VERSION.txt \
-    && mkdir -p "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
+    && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
+    && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER" "/home/$ARCHIVEBOX_USER/.config" "/home/$ARCHIVEBOX_USER/.cache" \
+    && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER/.config/abx" "/home/$ARCHIVEBOX_USER/.cache/abxbus" "/home/$ARCHIVEBOX_USER/.cache/pnpm" "/home/$ARCHIVEBOX_USER/.cache/uv" \
+    && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
    && openssl rand -hex 16 > /etc/machine-id \
    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u "$ARCHIVEBOX_USER") PGID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
    && echo -e "TMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt

 WORKDIR "$DATA_DIR"
 RUN echo "[+] Initializing image collection..." \
+    && find "$DATA_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} + \
    && PUID=0 PGID=0 archivebox init \
-    && find "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
    && (chown "$DEFAULT_PUID:$DEFAULT_PGID" \
        "$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \
        "$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \
        "$DATA_DIR"/archive "$DATA_DIR"/archive/users "$DATA_DIR"/personas \
        "$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \
+        "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
        2>/dev/null || true)

 RUN chmod +x "$CODE_DIR"/bin/*.sh \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
-    && chmod g+w "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && TIMEOUT=600 gosu "$ARCHIVEBOX_USER" archivebox install search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \
-    && "$LIB_DIR/env/bin/chromium" --version | tee -a /VERSION.txt \
-    && "$LIB_DIR/pip/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \
+    && chmod g+w "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"
+
+RUN "$LIB_DIR/bin/chromium" --version | tee -a /VERSION.txt \
+    && "$LIB_DIR/uv/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \
    && /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \
    && /usr/local/bin/sonic --version | tee -a /VERSION.txt \
    && /venv/bin/supervisord --version | tee -a /VERSION.txt \
-    && ! command -v gcc \
-    && ! command -v g++ \
-    && ! command -v make \
-    && gosu "$ARCHIVEBOX_USER" archivebox version 2>&1 | tee -a /VERSION.txt \
-    && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
+    && for forbidden_bin in gcc g++ make; do ! command -v "$forbidden_bin" || (echo "Unexpected build tool in runtime: $forbidden_bin=$(command -v "$forbidden_bin")" >&2 && exit 1); done \
+    && stat -c "%U:%G %a %n" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$LIB_DIR" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \
    && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*

 RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully." \
@@ -295,12 +295,10 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <summary><b><img src="https://user-images.githubusercontent.com/511499/117448075-49597580-af0c-11eb-91ba-f34fff10096b.png" alt="aptitude" height="28px" align="top"/> <code>apt</code></b> (Ubuntu/Debian/etc.)</summary>
 <br/>
 <ol>
-<li>Download and install the <code>.deb</code> package from the <a href="https://github.com/ArchiveBox/ArchiveBox/releases">latest release</a>.
-<pre lang="bash"><code style="white-space: pre-line"># download the .deb for your architecture (amd64 or arm64)
-ARCH="$(dpkg --print-architecture)"
-VERSION="$(curl -fsSL https://api.github.com/repos/ArchiveBox/ArchiveBox/releases/latest | python3 -c "import sys,json; print(json.load(sys.stdin)['tag_name'].lstrip('v'))")"
-curl -fsSL "https://github.com/ArchiveBox/ArchiveBox/releases/latest/download/archivebox_${VERSION}_${ARCH}.deb" -o /tmp/archivebox.deb
-sudo apt install /tmp/archivebox.deb
+<li>Add the third-party ArchiveBox apt repo and install <code>archivebox</code>.
+<pre lang="bash"><code style="white-space: pre-line">echo 'deb [trusted=yes] https://archivebox.github.io/debian-archivebox dev main' | sudo tee /etc/apt/sources.list.d/archivebox.list
+sudo apt update
+sudo apt install archivebox
 archivebox version                         # make sure all dependencies are installed
 </code></pre>
 </li>
@@ -79,16 +79,10 @@ def __getattr__(name: str):
        return VERSION
    if name in ("BUILTIN_PLUGINS_DIR", "USER_PLUGINS_DIR", "ALL_PLUGINS", "LOADED_PLUGINS"):
        from abx_plugins import get_plugins_dir
+        from .config.constants import CONSTANTS

        builtin_plugins_dir = Path(get_plugins_dir()).resolve()
-        user_plugins_dir = (
-            Path(
-                os.environ.get("ARCHIVEBOX_USER_PLUGINS_DIR")
-                or os.environ.get("USER_PLUGINS_DIR")
-                or os.environ.get("DATA_DIR", os.getcwd()),
-            )
-            / "custom_plugins"
-        )
+        user_plugins_dir = CONSTANTS.USER_PLUGINS_DIR
        plugins = {
            "builtin": builtin_plugins_dir,
            "user": user_plugins_dir,
@@ -17,4 +17,4 @@ ASCII_LOGO_MINI = r"""
 """

 if __name__ == "__main__":
-    main(args=sys.argv[1:], stdin=sys.stdin)
+    main(args=sys.argv[1:])
@@ -51,6 +51,27 @@ def auth_using_token(token: str | None, request: HttpRequest | None = None) -> U
    return user


+def token_from_request(request: HttpRequest) -> str:
+    token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key") or ""
+    auth_header = request.headers.get("Authorization", "")
+    if not token and auth_header.lower().startswith("bearer "):
+        token = auth_header.split(None, 1)[1].strip()
+    return token
+
+
+def authenticated_user_from_request(request: HttpRequest) -> User | None:
+    user = request.user
+    if user.is_authenticated and user.is_active:
+        return user
+
+    token = token_from_request(request)
+    token_user = auth_using_token(token=token, request=request) if token else None
+    if token_user and token_user.is_active:
+        request.user = token_user
+        return token_user
+    return None
+
+
 def auth_using_password(username: str | None, password: str | None, request: HttpRequest | None = None) -> User | None:
    """Given a username and password, check if they are valid and return the corresponding user"""
    user: User | None = None
@@ -68,14 +68,14 @@ class NinjaAPIWithIOCapture(NinjaAPI):
        # response['X-ArchiveBox-View'] = self.get_openapi_operation_id(request) or 'Unknown'

        # Add Auth Headers to response
-        api_token_attr = getattr(request, "_api_token", None)
+        api_token_attr = request.__dict__.get("_api_token")
        api_token = api_token_attr if isinstance(api_token_attr, APIToken) else None
        token_expiry = api_token.expires.isoformat() if api_token and api_token.expires else "Never"

-        response["X-ArchiveBox-Auth-Method"] = str(getattr(request, "_api_auth_method", "None"))
+        response["X-ArchiveBox-Auth-Method"] = str(request.__dict__.get("_api_auth_method", "None"))
        response["X-ArchiveBox-Auth-Expires"] = token_expiry
        response["X-ArchiveBox-Auth-Token-Id"] = str(api_token.id) if api_token else "None"
-        response["X-ArchiveBox-Auth-User-Id"] = str(request.user.pk) if getattr(request.user, "pk", None) else "None"
+        response["X-ArchiveBox-Auth-User-Id"] = str(request.user.pk) if request.user.pk else "None"
        response["X-ArchiveBox-Auth-User-Username"] = request.user.username if isinstance(request.user, User) else "None"

        # import ipdb; ipdb.set_trace()
@@ -8,8 +8,11 @@ from enum import Enum
 from django.http import HttpRequest

 from ninja import Router, Schema
+from ninja.errors import HttpError
+from pydantic import Field

 from archivebox.misc.util import ansi_to_html
+from archivebox.core.models import SnapshotQuerySet


 # from .auth import API_AUTH_METHODS
@@ -21,6 +24,7 @@ router = Router(tags=["ArchiveBox CLI Sub-Commands"])
 # Schemas

 JSONType = list[Any] | dict[str, Any] | bool | int | str | None
+FILTER_PATTERNS_EXAMPLES = [["https://example.com"]]


 class CLICommandResponseSchema(Schema):
@@ -32,26 +36,11 @@ class CLICommandResponseSchema(Schema):
    stderr: str


-class FilterTypeChoices(str, Enum):
-    exact = "exact"
-    substring = "substring"
-    regex = "regex"
-    domain = "domain"
-    tag = "tag"
-    timestamp = "timestamp"
-
-
-class StatusChoices(str, Enum):
-    indexed = "indexed"
-    archived = "archived"
-    unarchived = "unarchived"
-    present = "present"
-    valid = "valid"
-    invalid = "invalid"
-    duplicate = "duplicate"
-    orphaned = "orphaned"
-    corrupted = "corrupted"
-    unrecognized = "unrecognized"
+FilterTypeChoices = Enum(
+    "FilterTypeChoices",
+    {filter_type: filter_type for filter_type in SnapshotQuerySet.FILTER_TYPE_CHOICES},
+    type=str,
+)


 class AddCommandSchema(Schema):
@@ -71,14 +60,27 @@ class AddCommandSchema(Schema):
    index_only: bool = False


-class UpdateCommandSchema(Schema):
-    resume: str | None = None
+class SnapshotFilterCommandSchema(Schema):
    after: float | None = 0
-    before: float | None = 999999999999999
+    before: float | None = None
    filter_type: str | None = FilterTypeChoices.substring
-    filter_patterns: list[str] | None = ["https://example.com"]
+    filter_patterns: list[str] | None = Field(default=None, examples=FILTER_PATTERNS_EXAMPLES)
+    status: str | None = None
+    url__icontains: str | None = None
+    url__istartswith: str | None = None
+    tag: str | None = None
+    crawl_id: str | None = None
+    limit: int | None = None
+    sort: str | None = None
+    search: str | None = None
+
+
+class UpdateCommandSchema(SnapshotFilterCommandSchema):
+    resume: str | None = None
    batch_size: int = 100
    continuous: bool = False
+    index_only: bool = False
+    migrate_only: bool = False


 class ScheduleCommandSchema(Schema):
@@ -97,24 +99,23 @@ class ScheduleCommandSchema(Schema):
    clear: bool = False


-class ListCommandSchema(Schema):
-    filter_patterns: list[str] | None = ["https://example.com"]
-    filter_type: str = FilterTypeChoices.substring
-    status: StatusChoices = StatusChoices.indexed
-    after: float | None = 0
-    before: float | None = 999999999999999
-    sort: str = "bookmarked_at"
+class ListCommandSchema(SnapshotFilterCommandSchema):
    as_json: bool = True
    as_html: bool = False
    as_csv: str | None = "timestamp,url"
    with_headers: bool = False


-class RemoveCommandSchema(Schema):
-    after: float | None = 0
-    before: float | None = 999999999999999
+class RemoveCommandSchema(SnapshotFilterCommandSchema):
    filter_type: str = FilterTypeChoices.exact
-    filter_patterns: list[str] | None = ["https://example.com"]
+    timeout: float = 60.0
+
+
+def snapshot_filter_kwargs(args: SnapshotFilterCommandSchema, *, default_filter_type: str) -> dict[str, Any]:
+    kwargs = args.dict()
+    kwargs["filter_patterns"] = kwargs.get("filter_patterns") or []
+    kwargs["filter_type"] = kwargs.get("filter_type") or default_filter_type
+    return kwargs


@router.post("/add", response=CLICommandResponseSchema, summary="archivebox add [args] [urls]")
@@ -150,8 +151,8 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
        "snapshot_ids": snapshot_ids,
        "queued_urls": args.urls,
    }
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")

    return {
        "success": True,
@@ -165,24 +166,39 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):

@router.post("/update", response=CLICommandResponseSchema, summary="archivebox update [args] [filter_patterns]")
 def cli_update(request: HttpRequest, args: UpdateCommandSchema):
-    from archivebox.cli.archivebox_update import update
+    from archivebox.cli.archivebox_update import _build_filtered_snapshots_queryset, update
+    from archivebox.core.snapshot_status import normalize_snapshot_status

-    result = update(
-        filter_patterns=args.filter_patterns or [],
-        filter_type=args.filter_type or FilterTypeChoices.substring,
-        after=args.after,
-        before=args.before,
-        resume=args.resume,
-        batch_size=args.batch_size,
-        continuous=args.continuous,
-        stop_daemon_stack=False,
+    try:
+        status = normalize_snapshot_status(args.status)
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
+
+    update_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.substring)
+    update_kwargs["status"] = status
+    update_kwargs["stop_daemon_stack"] = False
+
+    is_filtered_update = any(
+        (update_kwargs.get(key) for key in (*SnapshotQuerySet.FILTER_ARG_KEYS, "resume") if key != "filter_type"),
    )
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
+    matched_snapshot_ids = []
+    if is_filtered_update:
+        matched_snapshot_ids = [
+            str(snapshot_id) for snapshot_id in _build_filtered_snapshots_queryset(**update_kwargs).values_list("id", flat=True)
+        ]
+
+    update(**update_kwargs)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
    return {
        "success": True,
        "errors": [],
-        "result": result,
+        "result": {
+            "matched_count": len(matched_snapshot_ids),
+            "snapshot_ids": matched_snapshot_ids,
+        }
+        if is_filtered_update
+        else None,
        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
    }
@@ -211,8 +227,8 @@ def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
        config=config_overrides or None,
    )

-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
    return {
        "success": True,
        "errors": [],
@@ -225,32 +241,38 @@ def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):

@router.post("/search", response=CLICommandResponseSchema, summary="archivebox search [args] [filter_patterns]")
 def cli_search(request: HttpRequest, args: ListCommandSchema):
-    from archivebox.cli.archivebox_search import search
+    from archivebox.cli.archivebox_snapshot import build_snapshot_queryset

-    result = search(
-        filter_patterns=args.filter_patterns,
-        filter_type=args.filter_type,
-        status=args.status,
-        after=args.after,
-        before=args.before,
-        sort=args.sort,
-        csv=args.as_csv,
-        json=args.as_json,
-        html=args.as_html,
-        with_headers=args.with_headers,
-    )
+    search_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.substring)
+    as_json = search_kwargs.pop("as_json")
+    as_html = search_kwargs.pop("as_html")
+    as_csv = search_kwargs.pop("as_csv")
+    with_headers = search_kwargs.pop("with_headers")
+    try:
+        snapshots = build_snapshot_queryset(**search_kwargs).select_related("crawl", "crawl__created_by")
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err

    result_format = "txt"
-    if args.as_json:
+    if as_json:
        result_format = "json"
-        result = json.loads(result)
-    elif args.as_html:
+        result = [
+            json.loads(json.dumps(snapshot.to_dict(extended=True), default=str))
+            for snapshot in snapshots.prefetch_related("tags").iterator(chunk_size=500)
+        ]
+    elif as_html:
        result_format = "html"
-    elif args.as_csv:
+        result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500))
+    elif as_csv:
        result_format = "csv"
+        cols = [col.strip() for col in as_csv.split(",") if col.strip()]
+        rows = [snapshot.to_csv(cols=cols, separator=",") for snapshot in snapshots.prefetch_related("tags").iterator(chunk_size=500)]
+        result = "\n".join((",".join(cols), *rows) if with_headers else rows)
+    else:
+        result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500))

-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
    return {
        "success": True,
        "errors": [],
@@ -264,37 +286,26 @@ def cli_search(request: HttpRequest, args: ListCommandSchema):
@router.post("/remove", response=CLICommandResponseSchema, summary="archivebox remove [args] [filter_patterns]")
 def cli_remove(request: HttpRequest, args: RemoveCommandSchema):
    from archivebox.cli.archivebox_remove import remove
-    from archivebox.cli.archivebox_search import get_snapshots
    from archivebox.core.models import Snapshot

-    filter_patterns = args.filter_patterns or []
-    snapshots_to_remove = get_snapshots(
-        filter_patterns=filter_patterns,
-        filter_type=args.filter_type,
-        after=args.after,
-        before=args.before,
-    )
-    removed_snapshot_ids = [str(snapshot_id) for snapshot_id in snapshots_to_remove.values_list("id", flat=True)]
+    remove_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.exact)
+    timeout_arg = remove_kwargs.pop("timeout")
+    timeout = min(float(timeout_arg if timeout_arg is not None else 60.0), 60.0)
+    try:
+        snapshots_to_remove = Snapshot.objects.order_by("-created_at").search(**remove_kwargs)
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err

-    remove(
+    result = remove(
        yes=True,  # no way to interactively ask for confirmation via API, so we force yes
        snapshots=snapshots_to_remove,
-        before=args.before,
-        after=args.after,
-        filter_type=args.filter_type,
-        filter_patterns=filter_patterns,
+        timeout=timeout,
    )
-
-    result = {
-        "removed_count": len(removed_snapshot_ids),
-        "removed_snapshot_ids": removed_snapshot_ids,
-        "remaining_snapshots": Snapshot.objects.count(),
-    }
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
    return {
-        "success": True,
-        "errors": [],
+        "success": bool(result["success"]),
+        "errors": [str(result["error"])] if result["error"] else [],
        "result": result,
        "result_format": "json",
        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
@@ -29,7 +29,7 @@ from ninja.errors import HttpError

 from archivebox.core.models import Snapshot, ArchiveResult, Tag
 from archivebox.core.permissions import public_snapshots_queryset
-from archivebox.api.auth import auth_using_token
+from archivebox.api.auth import authenticated_user_from_request
 from archivebox.config.common import get_config
 from archivebox.core.routes_util import build_web_url
 from archivebox.misc.util import filter_queryset_by_uuid_substring, validate_url_length
@@ -49,9 +49,10 @@ from archivebox.core.tag_util import (
    rename_tag as rename_tag_record,
 )
 from archivebox.crawls.models import Crawl
-from archivebox.api.v1_crawls import CrawlSchema
+from archivebox.api.v1_crawls import CrawlSchema, get_crawl_by_ref
 from archivebox.search.config import get_search_mode, get_search_mode_backend
 from archivebox.search.query import apply_snapshot_search
+from archivebox.core.snapshot_status import filter_snapshots_by_status, normalize_snapshot_status


 router = Router(tags=["Core Models"])
@@ -79,7 +80,7 @@ class CustomPagination(PaginationBase):
    def paginate_queryset(self, queryset, pagination: Input, request: HttpRequest, **params):
        limit = min(pagination.limit, 500)
        offset = pagination.offset or (pagination.page * limit)
-        total = queryset.count()
+        total = queryset.values("pk").distinct().count() if queryset.query.distinct else queryset.count()
        total_pages = math.ceil(total / limit)
        current_page = math.ceil(offset / (limit + 1))
        items = queryset[offset : offset + limit]
@@ -212,7 +213,10 @@ class ArchiveResultFilterSchema(FilterSchema):
@paginate(CustomPagination)
 def get_archiveresults(request: HttpRequest, filters: Query[ArchiveResultFilterSchema]):
    """List all ArchiveResult entries matching these filters."""
-    return filters.filter(ArchiveResult.objects.all()).distinct()
+    queryset = filters.filter(ArchiveResult.objects.all())
+    if filters.search or filters.snapshot_tag:
+        return queryset.distinct()
+    return queryset


 def _uuid_ref_query(field_name: str, ref: str) -> Q:
@@ -273,7 +277,7 @@ def _parse_archiveresult_output_json(output_json: str | None) -> dict[str, Any]


 def _get_archiveresult_upload_data(request: HttpRequest):
-    cached = getattr(request, "_archiveresult_upload_data", None)
+    cached = request.__dict__.get("_archiveresult_upload_data")
    if cached is not None:
        return cached

@@ -468,12 +472,7 @@ def _write_archiveresult_files(
            )

        guessed_mime = mimetypes.guess_type(relative_output_path)[0]
-        output_mime_type = (
-            (mime_types[0] if mime_types else "")
-            or getattr(uploaded_file, "content_type", None)
-            or guessed_mime
-            or "application/octet-stream"
-        )
+        output_mime_type = (mime_types[0] if mime_types else "") or uploaded_file.content_type or guessed_mime or "application/octet-stream"
        output_files[relative_output_path] = {
            "extension": PurePosixPath(relative_output_path).suffix.lower().lstrip("."),
            "mimetype": output_mime_type,
@@ -501,7 +500,7 @@ def _write_archiveresult_files(
        guessed_mime = mimetypes.guess_type(saved_output_path)[0]
        output_mime_type = (
            (mime_types[index] if index < len(mime_types) else "")
-            or getattr(uploaded_file, "content_type", None)
+            or uploaded_file.content_type
            or guessed_mime
            or "application/octet-stream"
        )
@@ -702,7 +701,7 @@ class SnapshotSchema(Schema):

    @staticmethod
    def resolve_archiveresults(obj, context):
-        if bool(getattr(context["request"], "with_archiveresults", False)):
+        if bool(context["request"].__dict__.get("with_archiveresults", False)):
            return obj.archiveresult_set.all().distinct()
        return ArchiveResult.objects.none()

@@ -859,6 +858,7 @@ class SnapshotFilterSchema(FilterSchema):
    modified_at__lt: Annotated[datetime | None, FilterLookup("modified_at__lt")] = None
    search: str | None = None
    search_mode: str | None = None
+    status: str | None = None
    url: Annotated[str | None, FilterLookup("url")] = None
    tag: Annotated[str | None, FilterLookup("tags__name")] = None
    title: Annotated[str | None, FilterLookup("title__icontains")] = None
@@ -872,18 +872,25 @@ class SnapshotFilterSchema(FilterSchema):
    def filter_search_mode(self, value: str | None) -> Q:
        return Q()

+    def filter_status(self, value: str | None) -> Q:
+        return Q()
+

@router.get("/snapshots", response=list[SnapshotSchema], url_name="get_snapshots")
@paginate(CustomPagination)
 def get_snapshots(request: HttpRequest, filters: Query[SnapshotFilterSchema], with_archiveresults: bool = False):
    """List all Snapshot entries matching these filters."""
    setattr(request, "with_archiveresults", with_archiveresults)
-    queryset = filters.filter(Snapshot.objects.all()).distinct()
+    try:
+        queryset = filter_snapshots_by_status(Snapshot.objects.all(), filters.status)
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
+    queryset = filters.filter(queryset).distinct()
    query = (filters.search or "").strip()
    if not query:
        return queryset

-    runtime_config = getattr(request, "archivebox_config", None)
+    runtime_config = request.archivebox_config
    search_mode = get_search_mode(filters.search_mode, config=runtime_config)
    try:
        return apply_snapshot_search(
@@ -921,18 +928,16 @@ def get_snapshots_rss(
 def get_snapshot(request: HttpRequest, snapshot_id: str, with_archiveresults: bool = True):
    """Get a specific Snapshot by id."""
    setattr(request, "with_archiveresults", with_archiveresults)
-    queryset = Snapshot.objects.all()
-    try:
-        return queryset.get(_uuid_ref_query("id", snapshot_id) | Q(timestamp__startswith=snapshot_id))
-    except Snapshot.DoesNotExist:
-        return queryset.get(_uuid_ref_query("id", snapshot_id))
+    return _get_snapshot_by_ref(snapshot_id)


@router.post("/snapshots", response=SnapshotSchema, url_name="create_snapshot")
 def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
    tags = normalize_tag_list(data.tags)
-    if data.status is not None and data.status not in Snapshot.StatusChoices.values:
-        raise HttpError(400, f"Invalid status: {data.status}")
+    try:
+        status = normalize_snapshot_status(data.status)
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
    if not data.url.strip():
        raise HttpError(400, "URL is required")
    try:
@@ -943,7 +948,7 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
        raise HttpError(400, "depth must be between 0 and 4")

    if data.crawl_id:
-        crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), data.crawl_id).get()
+        crawl = get_crawl_by_ref(data.crawl_id)
        crawl_tags = normalize_tag_list(crawl.tags_str.split(","))
        tags = tags or crawl_tags
    else:
@@ -960,7 +965,7 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
        "depth": data.depth,
        "title": data.title,
        "timestamp": str(timezone.now().timestamp()),
-        "status": data.status or Snapshot.StatusChoices.QUEUED,
+        "status": status or Snapshot.StatusChoices.QUEUED,
        "retry_at": timezone.now(),
    }
    snapshot, _ = Snapshot.objects.get_or_create(
@@ -973,10 +978,8 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
    if data.title is not None and snapshot.title != data.title:
        snapshot.title = data.title
        update_fields.append("title")
-    if data.status is not None and snapshot.status != data.status:
-        if data.status not in Snapshot.StatusChoices.values:
-            raise HttpError(400, f"Invalid status: {data.status}")
-        snapshot.status = data.status
+    if status is not None and snapshot.status != status:
+        snapshot.status = status
        update_fields.append("status")
    if update_fields:
        update_fields.append("modified_at")
@@ -997,10 +1000,7 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
@router.patch("/snapshot/{snapshot_id}", response=SnapshotSchema, url_name="patch_snapshot")
 def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateSchema):
    """Update a snapshot (e.g., set status=sealed to cancel queued work)."""
-    try:
-        snapshot = Snapshot.objects.get(Q(id__startswith=snapshot_id) | Q(timestamp__startswith=snapshot_id))
-    except Snapshot.DoesNotExist:
-        snapshot = filter_queryset_by_uuid_substring(Snapshot.objects.all(), snapshot_id).get()
+    snapshot = _get_snapshot_by_ref(snapshot_id)

    payload = data.dict(exclude_unset=True)
    update_fields = ["modified_at"]
@@ -1023,9 +1023,10 @@ def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateS
        raise HttpError(400, f"Invalid action: {action}")

    if "status" in payload:
-        if payload["status"] not in Snapshot.StatusChoices.values:
-            raise HttpError(400, f"Invalid status: {payload['status']}")
-        snapshot.status = payload["status"]
+        try:
+            snapshot.status = normalize_snapshot_status(payload["status"])
+        except ValueError as err:
+            raise HttpError(400, str(err)) from err
        if snapshot.status == Snapshot.StatusChoices.SEALED and "retry_at" not in payload:
            snapshot.retry_at = None
        update_fields.append("status")
@@ -1081,7 +1082,7 @@ class TagSchema(Schema):
    def resolve_created_by_username(obj):
        user_model = get_user_model()
        user = user_model.objects.get(id=obj.created_by_id)
-        username = getattr(user, "username", None)
+        username = user.username
        return username if isinstance(username, str) else str(user)

    @staticmethod
@@ -1090,7 +1091,7 @@ class TagSchema(Schema):

    @staticmethod
    def resolve_snapshots(obj, context):
-        if bool(getattr(context["request"], "with_snapshots", False)):
+        if bool(context["request"].__dict__.get("with_snapshots", False)):
            return obj.snapshot_set.all().distinct()
        return Snapshot.objects.none()

@@ -1294,16 +1295,7 @@ def _public_tag_listing_enabled() -> bool:


 def _request_has_tag_autocomplete_access(request: HttpRequest) -> bool:
-    user = getattr(request, "user", None)
-    if getattr(user, "is_authenticated", False):
-        return True
-
-    token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key")
-    auth_header = request.headers.get("Authorization", "")
-    if not token and auth_header.lower().startswith("bearer "):
-        token = auth_header.split(None, 1)[1].strip()
-
-    if token and auth_using_token(token=token, request=request):
+    if authenticated_user_from_request(request):
        return True

    return _public_tag_listing_enabled()
@@ -1315,7 +1307,7 @@ def tags_autocomplete(request: HttpRequest, q: str = ""):
    if not _request_has_tag_autocomplete_access(request):
        raise HttpError(401, "Authentication required")

-    public_only = not getattr(request.user, "is_authenticated", False) and not getattr(request, "_api_token", None)
+    public_only = not request.user.is_authenticated and not request.__dict__.get("_api_token")
    queryset = get_matching_tags(q)
    public_snapshots = public_snapshots_queryset(Snapshot.objects.all())
    if public_only:
@@ -1324,7 +1316,7 @@ def tags_autocomplete(request: HttpRequest, q: str = ""):
    add_snapshot_counts(tags, snapshot_queryset=public_snapshots if public_only else None)

    return {
-        "tags": [{"id": tag.pk, "name": tag.name, "num_snapshots": getattr(tag, "num_snapshots", 0)} for tag in tags],
+        "tags": [{"id": tag.pk, "name": tag.name, "num_snapshots": tag.__dict__.get("num_snapshots", 0)} for tag in tags],
    }


@@ -24,7 +24,7 @@ from archivebox.config.common import get_config
 from archivebox.crawls.models import Crawl
 from archivebox.misc.util import filter_queryset_by_uuid_substring

-from .auth import API_AUTH_METHODS, auth_using_token
+from .auth import API_AUTH_METHODS, authenticated_user_from_request

 router = Router(tags=["Crawl Models"], auth=API_AUTH_METHODS)

@@ -58,7 +58,7 @@ class CrawlSchema(Schema):
    def resolve_created_by_username(obj):
        user_model = get_user_model()
        user = user_model.objects.get(id=obj.created_by_id)
-        username = getattr(user, "username", None)
+        username = user.username
        return username if isinstance(username, str) else str(user)

    @staticmethod
@@ -71,7 +71,7 @@ class CrawlSchema(Schema):

    @staticmethod
    def resolve_snapshots(obj, context):
-        if bool(getattr(context["request"], "with_snapshots", False)):
+        if bool(context["request"].__dict__.get("with_snapshots", False)):
            return obj.snapshot_set.all().distinct()
        return Snapshot.objects.none()

@@ -122,8 +122,7 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema):

    tags = normalize_tag_list(data.tags, data.tags_str)
    config = dict(data.config or {})
-    request_user = request.user if request.user.is_authenticated else None
-    config.setdefault("PERMISSIONS", str(get_config(user=request_user).PERMISSIONS))
+    config.setdefault("PERMISSIONS", str(get_config().PERMISSIONS))
    crawl = Crawl.objects.create(
        urls="\n".join(urls),
        max_depth=data.max_depth,
@@ -138,12 +137,16 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
    return crawl


+def get_crawl_by_ref(crawl_id: str):
+    return filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
+
+
@router.get("/crawl/{crawl_id}", response=CrawlSchema, url_name="get_crawl")
 def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_snapshots: bool = False, with_archiveresults: bool = False):
    """Get a specific Crawl by id."""
    setattr(request, "with_snapshots", with_snapshots)
    setattr(request, "with_archiveresults", with_archiveresults)
-    crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
+    crawl = get_crawl_by_ref(crawl_id)

    if crawl and as_rss:
        query = request.GET.copy()
@@ -157,33 +160,18 @@ def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_sn
 def crawl_file(request: HttpRequest, crawl_id: str, path: str):
    # Try to resolve the crawl first; if it doesn't exist, return 404.
    try:
-        crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
+        crawl = get_crawl_by_ref(crawl_id)
    except Crawl.DoesNotExist:
        raise HttpError(404, "Crawl not found")

-    # Determine the effective viewer: session user takes precedence, otherwise
-    # fall back to an API token passed via ?api_key=, X-ArchiveBox-API-Key, or
-    # Authorization: Bearer ... (so that programmatic clients still work).
-    user = getattr(request, "user", None)
-    is_authenticated = bool(getattr(user, "is_authenticated", False) and getattr(user, "is_active", False))
-    if not is_authenticated:
-        token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key")
-        auth_header = request.headers.get("Authorization", "")
-        if not token and auth_header.lower().startswith("bearer "):
-            token = auth_header.split(None, 1)[1].strip()
-        token_user = auth_using_token(token=token, request=request) if token else None
-        if token_user and token_user.is_active:
-            user = token_user
-            is_authenticated = True
-            # Re-bind so is_admin_user() / ownership checks below see the token user.
-            setattr(request, "user", token_user)
+    user = authenticated_user_from_request(request)

    # Gate access using the same model as SnapshotView/can_view_snapshot:
    # admins always pass; owners can see their own crawls; otherwise the crawl
    # must be PUBLIC or UNLISTED. Don't disclose existence of private crawls.
    if not is_admin_user(request):
        permissions = normalize_permissions(crawl.permissions)
-        is_owner = bool(is_authenticated and getattr(crawl, "created_by_id", None) == getattr(user, "id", None))
+        is_owner = bool(user and crawl.created_by_id == user.id)
        if not is_owner and permissions not in {PERMISSIONS_PUBLIC, PERMISSIONS_UNLISTED}:
            raise HttpError(404, "Crawl not found")

@@ -218,7 +206,7 @@ def crawl_file_nested_2(request: HttpRequest, crawl_id: str, folder: str, subfol
@router.patch("/crawl/{crawl_id}", response=CrawlSchema, url_name="patch_crawl")
 def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
    """Update a crawl (e.g., set status=sealed to cancel queued work)."""
-    crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
+    crawl = get_crawl_by_ref(crawl_id)
    payload = data.dict(exclude_unset=True)
    update_fields = ["modified_at"]

@@ -260,7 +248,7 @@ def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):

@router.delete("/crawl/{crawl_id}", response=CrawlDeleteResponseSchema, url_name="delete_crawl")
 def delete_crawl(request: HttpRequest, crawl_id: str):
-    crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
+    crawl = get_crawl_by_ref(crawl_id)
    crawl_id_str = str(crawl.id)
    snapshot_count = crawl.snapshot_set.count()
    deleted_count, _ = crawl.delete()
@@ -8,8 +8,10 @@ from uuid import UUID
 from django.db.models import Q
 from django.http import HttpRequest
 from ninja import Router, Schema
+from ninja.pagination import paginate
 from pydantic import Field

+from archivebox.api.v1_core import CustomPagination
 from archivebox.personas.importers import validate_persona_name
 from archivebox.personas.models import Persona

@@ -113,6 +115,7 @@ def find_persona(extension_persona_id: str, name: str) -> Persona | None:


@router.get("/personas", response=list[PersonaSchema], url_name="get_personas")
+@paginate(CustomPagination)
 def get_personas(request: HttpRequest):
    """List personas available on this ArchiveBox server."""
    return Persona.objects.all().order_by("name")
@@ -136,7 +139,7 @@ def sync_persona(request: HttpRequest, payload: PersonaSyncSchema):
    created = persona is None
    if persona is None:
        persona = Persona(name=name)
-        if getattr(request.user, "is_authenticated", False):
+        if request.user.is_authenticated:
            persona.created_by = request.user

    persona.config = {
@@ -5,7 +5,7 @@ __package__ = "archivebox.base_models"
 import json
 import uuid
 from collections.abc import Mapping
-from typing import NotRequired, TypedDict
+from typing import NotRequired, TypedDict, cast

 from django import forms
 from django.contrib import admin
@@ -76,39 +76,23 @@ class KeyValueWidget(forms.Widget):
    def _get_config_options(self) -> dict[str, ConfigOption]:
        """Get available config options from plugins."""
        try:
-            from archivebox.config.common import ArchiveBoxConfig
-            from archivebox.plugins.discovery import discover_plugin_configs
+            from archivebox.config.common import config_field_metadata

            options: dict[str, ConfigOption] = {}
-            skipped_core_keys = {"ABX_RUNTIME", "DATA_DIR", "CRAWL_DIR", "SNAP_DIR"}
-            for key, field in ArchiveBoxConfig.model_fields.items():
-                if key in skipped_core_keys or key in ArchiveBoxConfig.computed_config_keys:
-                    continue
-                default = field.default
-                try:
-                    json.dumps(default)
-                except TypeError:
-                    default = str(default)
-                options[key] = {
-                    "plugin": "archivebox",
-                    "type": str(field.annotation),
-                    "default": default,
-                    "description": field.description or "",
+            for key, metadata in config_field_metadata().items():
+                option_type = metadata.get("type", "string")
+                option: ConfigOption = {
+                    "plugin": str(metadata.get("plugin", "archivebox")),
+                    "type": cast(str | list[str], option_type if isinstance(option_type, (str, list)) else str(option_type)),
+                    "default": metadata.get("default", ""),
+                    "description": str(metadata.get("description", "")),
                }
-
-            plugin_configs = discover_plugin_configs()
-            for plugin_name, schema in plugin_configs.items():
-                for key, prop in schema.get("properties", {}).items():
-                    option: ConfigOption = {
-                        "plugin": plugin_name,
-                        "type": prop.get("type", "string"),
-                        "default": prop.get("default", ""),
-                        "description": prop.get("description", ""),
-                    }
+                schema = metadata.get("schema")
+                if isinstance(schema, Mapping):
                    for schema_key in ("enum", "pattern", "minimum", "maximum"):
-                        if schema_key in prop:
-                            option[schema_key] = prop[schema_key]
-                    options[key] = option
+                        if schema_key in schema:
+                            option[schema_key] = schema[schema_key]
+                options[key] = option
            return options
        except Exception:
            return {}
@@ -839,7 +823,7 @@ class ConfigEditorMixin(admin.ModelAdmin):
        """
        from archivebox.config.common import is_sensitive_config_key

-        if change and obj.pk and getattr(obj, "config", None) is not None:
+        if change and obj.pk and obj.config is not None:
            try:
                stored = type(obj).objects.filter(pk=obj.pk).values_list("config", flat=True).first() or {}
            except Exception:
@@ -20,6 +20,8 @@ from django.conf import settings

 from django_stubs_ext.db.models import TypedModelMeta

+from archivebox.config import CONSTANTS
+

 def normalize_config_json_values(config: Any) -> Any:
    if not isinstance(config, dict):
@@ -113,6 +115,10 @@ class ModelWithHealthStats(models.Model):
    class Meta(TypedModelMeta):
        abstract = True

+    @property
+    def admin_change_url(self) -> str:
+        return f"/admin/{self._meta.app_label}/{self._meta.model_name}/{self.pk}/change/"
+
    @property
    def health(self) -> int:
        total = max(self.num_uses_failed + self.num_uses_succeeded, 1)
@@ -165,7 +171,7 @@ class ModelWithDeleteAfter(models.Model):
    def get_delete_after_config_value(self):
        from archivebox.config.common import get_config

-        return get_config(include_machine=False).DELETE_AFTER
+        return get_config(include_machine=False, resolve_plugins=False).DELETE_AFTER

    def set_delete_at_from_config(self, config_value=None) -> bool:
        if self.delete_at is not None:
@@ -253,9 +259,7 @@ class ModelWithOutputDir(ModelWithUUID):

    @classmethod
    def validate_output_paths_for_delete(cls, paths) -> tuple[Path, ...]:
-        from archivebox.config.common import get_config
-
-        data_dir = get_config().DATA_DIR.resolve()
+        data_dir = CONSTANTS.DATA_DIR.resolve()
        safe_paths = []
        for raw_path in paths:
            path = Path(raw_path)
@@ -66,10 +66,6 @@ class ArchiveBoxGroup(click.Group):
        # Introspection commands
        "pluginmap": "archivebox.cli.archivebox_pluginmap.main",
    }
-    legacy_model_commands = {
-        "crawl": "archivebox.cli.archivebox_crawl_compat.main",
-        "snapshot": "archivebox.cli.archivebox_snapshot_compat.main",
-    }
    all_subcommands = {
        **meta_commands,
        **setup_commands,
@@ -81,34 +77,29 @@ class ArchiveBoxGroup(click.Group):
        "import": "add",
        "archive": "add",
    }
-    legacy_model_subcommands = {
-        "crawl": {"create", "list", "update", "delete"},
-        "snapshot": {"create", "list", "update", "delete"},
-    }

    @classmethod
    def get_canonical_name(cls, cmd_name):
        return cls.renamed_commands.get(cmd_name, cmd_name)

    @classmethod
-    def _should_use_legacy_model_command(cls, cmd_name: str) -> bool:
-        if cmd_name not in cls.legacy_model_commands:
-            return False
+    def _needs_django_for_lazy_import(cls, cmd_name: str) -> bool:
+        wants_help = any(arg in ("-h", "--help", "--version") for arg in sys.argv[1:])
+        return not wants_help and (cmd_name in cls.archive_commands or cmd_name in cls.model_commands)

-        try:
-            arg_idx = sys.argv.index(cmd_name)
-        except ValueError:
-            return False
+    @classmethod
+    def _setup_django_for_lazy_import(cls, cmd_name: str) -> None:
+        if not cls._needs_django_for_lazy_import(cmd_name):
+            return

-        remaining_args = sys.argv[arg_idx + 1 :]
-        if not remaining_args:
-            return False
+        from django.apps import apps

-        first_arg = remaining_args[0]
-        if first_arg in ("-h", "--help"):
-            return False
+        if apps.ready:
+            return

-        return first_arg not in cls.legacy_model_subcommands[cmd_name]
+        from archivebox.config.django import setup_django
+
+        setup_django()

    def get_command(self, ctx, cmd_name):
        # handle renamed commands
@@ -120,11 +111,9 @@ class ArchiveBoxGroup(click.Group):
            cmd_name = new_name
            ctx.invoked_subcommand = cmd_name

-        if self._should_use_legacy_model_command(cmd_name):
-            return self._lazy_load(self.legacy_model_commands[cmd_name])
-
        # handle lazy loading of commands
        if cmd_name in self.all_subcommands:
+            self._setup_django_for_lazy_import(cmd_name)
            return self._lazy_load(cmd_name)

        # fall-back to using click's default command lookup
@@ -139,9 +128,9 @@ class ArchiveBoxGroup(click.Group):

        # print(f'LAZY LOADING {import_path}')
        mod = import_module(modname)
-        func = getattr(mod, funcname)
+        func = vars(mod)[funcname]

-        if not hasattr(func, "__doc__"):
+        if func.__doc__ is None:
            raise ValueError(f"lazy loading of {import_path} failed - no docstring found on method")

        # if not isinstance(cmd, click.BaseCommand):
@@ -199,15 +188,12 @@ def cli(ctx, help=False):
                raise


-def main(args=None, prog_name=None, stdin=None):
+def main(args=None, prog_name=None):
    # show `docker run archivebox xyz` in help messages if running in docker
    IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes")
    IS_TTY = sys.stdin.isatty()
    prog_name = prog_name or (f"docker compose run{'' if IS_TTY else ' -T'} archivebox" if IN_DOCKER else "archivebox")

-    # stdin param allows passing input data from caller (used by __main__.py)
-    # currently not used by click-based CLI, but kept for backwards compatibility
-
    previous_unraisablehook = sys.unraisablehook

    def ignore_shutdown_unraisable(unraisable):
@@ -108,16 +108,15 @@ def add(

    # import models once django is set up
    from archivebox.crawls.models import Crawl
+    from archivebox.core.models import Snapshot
    from archivebox.base_models.models import get_or_create_system_user_pk
    from archivebox.personas.models import Persona
    from archivebox.misc.logging_util import printable_filesize
    from archivebox.misc.system import get_dir_size
    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
-    from django.contrib.auth import get_user_model
    from django.utils import timezone

    created_by_id = created_by_id or get_or_create_system_user_pk()
-    created_by = get_user_model().objects.filter(pk=created_by_id).first()
    started_at = timezone.now()

    if isinstance(urls, str):
@@ -126,16 +125,28 @@ def add(
        url_list = [str(url).strip() for url in urls if str(url).strip()]
    if snapshot_ids and len(snapshot_ids) != len(url_list):
        raise ValueError("snapshot_ids length must match urls length")
+    admitted_urls: list[str] = []
+    admitted_snapshot_ids: list[str] | None = [] if snapshot_ids else None
+    for index, url in enumerate(url_list):
+        if Snapshot.is_archivebox_internal_url(url):
+            print(f"[yellow][!] Skipping internal ArchiveBox URL: {url}[/yellow]")
+            continue
+        admitted_urls.append(url)
+        if admitted_snapshot_ids is not None and snapshot_ids:
+            admitted_snapshot_ids.append(snapshot_ids[index])

    # 1. Save the provided URLs to sources/2024-11-05__23-59-59__cli_add.txt
    sources_file = CONSTANTS.SOURCES_DIR / f"{timezone.now().strftime('%Y-%m-%d__%H-%M-%S')}__cli_add.txt"
    sources_file.parent.mkdir(parents=True, exist_ok=True)
-    if snapshot_ids:
+    if admitted_snapshot_ids is not None:
        sources_file.write_text(
-            "\n".join(json.dumps({"url": url, "id": snapshot_ids[index], "tags": tag, "depth": 0}) for index, url in enumerate(url_list)),
+            "\n".join(
+                json.dumps({"url": url, "id": admitted_snapshot_ids[index], "tags": tag, "depth": 0})
+                for index, url in enumerate(admitted_urls)
+            ),
        )
    else:
-        sources_file.write_text("\n".join(url_list))
+        sources_file.write_text("\n".join(admitted_urls))

    # 2. Create a new Crawl with inline URLs
    cli_args = [*sys.argv]
@@ -145,13 +156,12 @@ def add(

    timestamp = timezone.now().strftime("%Y-%m-%d__%H-%M-%S")

-    # Read URLs directly into crawl
    urls_content = sources_file.read_text()
    persona_name = (persona or "Default").strip() or "Default"
    plugins = plugins or ""
    persona_obj, _ = Persona.objects.get_or_create(name=persona_name)
    persona_obj.ensure_dirs()
-    effective_persona_config = get_config(persona=persona_obj, user=created_by)
+    effective_persona_config = get_config(persona=persona_obj)

    crawl_config = {
        "PERMISSIONS": str(effective_persona_config.PERMISSIONS),
@@ -172,9 +182,8 @@ def add(
        **({"URL_DENYLIST": url_denylist} if url_denylist else {}),
    }
    # Caller-supplied overrides (e.g. {"ONLY_NEW": False}) are the highest
-    # priority — they win over persona/plugin/env defaults and get stamped
-    # directly onto crawl.config so the runtime resolution and admin UI both
-    # reflect them faithfully.
+    # priority for crawl-frozen keys. Runtime-derived execution keys are
+    # stripped by Crawl.save() and rederived when hooks run.
    crawl_config.update(config_overrides)

    crawl = Crawl.objects.create(
@@ -185,7 +194,7 @@ def add(
        label=f"{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]",
        created_by_id=created_by_id,
        status=Crawl.StatusChoices.QUEUED,
-        retry_at=None if (index_only or bg) else timezone.now(),
+        retry_at=None if index_only else timezone.now(),
        config=crawl_config,
    )

@@ -217,8 +226,8 @@ def add(
        # Foreground mode: run full crawl runner until all work is done
        print("[green]\\[*] Starting crawl runner to process crawl...[/green]")
        from archivebox.machine.models import Process
-        from archivebox.core.takeover_util import command_owns_runtime_stack, current_command, standby_until_runtime_stack_needed
-        from archivebox.workers.supervisord_util import run_runner_worker, stop_own_supervisord_process
+        from archivebox.core.takeover_util import command_owns_foreground_runner, current_command, standby_until_foreground_runner_needed
+        from archivebox.workers.supervisord_util import get_existing_supervisord_process, run_runner_worker, stop_own_supervisord_process

        command = current_command(Process.TypeChoices.ADD, data_dir=CONSTANTS.DATA_DIR, url=first_url)
        exit_code = 0
@@ -226,15 +235,27 @@ def add(
            try:
                with foreground_shutdown_signals(first_signal_message=None), foreground_parent_watchdog():
                    while True:
-                        standby_until_runtime_stack_needed(command, data_dir=CONSTANTS.DATA_DIR)
+                        standby_until_foreground_runner_needed(command, data_dir=CONSTANTS.DATA_DIR)
                        exit_code = run_runner_worker(
                            ["--crawl-id", str(crawl.id)],
                            name=f"worker_runner_add_{os.getpid()}",
                            interactive_interrupts=True,
                        )
+                        crawl.refresh_from_db(fields=["status", "retry_at"])
+                        if exit_code == 0 and crawl.status == crawl.StatusChoices.SEALED:
+                            break
+                        # A shared supervisord can be stopped by another
+                        # foreground owner (for example `archivebox server`
+                        # shutting down) and stop the one-shot add runner while
+                        # this add's crawl is still runnable. Keep the add
+                        # process alive so it can re-enter the normal
+                        # foreground-runner ownership loop and finish its crawl.
+                        supervisor_gone = exit_code == 1 and get_existing_supervisord_process(quiet=True) is None
+                        if crawl.status in crawl.RUNNABLE_STATES and (exit_code in (0, 130, 143) or supervisor_gone):
+                            continue
                        if exit_code == 0:
                            break
-                        if not command_owns_runtime_stack(command, data_dir=CONSTANTS.DATA_DIR):
+                        if not command_owns_foreground_runner(command, data_dir=CONSTANTS.DATA_DIR):
                            continue
                        raise SystemExit(exit_code)
            except KeyboardInterrupt:
@@ -6,6 +6,7 @@ import sys
 import toml
 import rich_click as click
 from rich import print
+from pathlib import Path

 from archivebox.misc.util import docstring, enforce_types
 from archivebox.misc.toml_util import CustomTOMLEncoder
@@ -30,12 +31,17 @@ def config(
    from archivebox.misc.logging_util import printable_config
    from abx_plugins.plugins.base.utils import resolve_alias
    from archivebox.config.collection import write_config_file
+    from archivebox.config import CONSTANTS_CONFIG
    from archivebox.config.common import ArchiveBoxConfig, get_config, get_all_configs
    from archivebox.plugins.discovery import discover_plugin_configs

    check_data_folder()

    FLAT_CONFIG = get_config().as_dict()
+    runtime_derived_keys = ArchiveBoxConfig.runtime_derived_config_keys()
+    readonly_config = {key: val for key, val in CONSTANTS_CONFIG.items() if key.isupper() and isinstance(val, Path)}
+    writable_config = {key: val for key, val in FLAT_CONFIG.items() if key not in runtime_derived_keys and key not in readonly_config}
+    readable_config = {**writable_config, **readonly_config}
    CONFIGS = get_all_configs()
    plugin_schemas = {
        plugin_name: schema.get("properties", {}) for plugin_name, schema in discover_plugin_configs().items() if isinstance(schema, dict)
@@ -56,18 +62,23 @@ def config(
            config_options = [
                core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options
            ]
-            matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
+            matching_config = {key: readable_config[key] for key in config_options if key in readable_config}
            for config_section in CONFIGS.values():
-                aliases = getattr(config_section, "aliases", {})
+                aliases = {str(field.alias): field_name for field_name, field in type(config_section).model_fields.items() if field.alias}

                for search_key in config_options:
                    # search all aliases in the section
                    for alias_key, key in aliases.items():
-                        if search_key.lower() in alias_key.lower():
+                        if key in readable_config and search_key.lower() in alias_key.lower():
                            matching_config[key] = dict(config_section)[key]

                    # search all keys and values in the section
                    for existing_key, value in dict(config_section).items():
+                        if existing_key in readable_config and (
+                            search_key.lower() in existing_key.lower() or search_key.lower() in str(value).lower()
+                        ):
+                            matching_config[existing_key] = value
+                    for existing_key, value in readonly_config.items():
                        if search_key.lower() in existing_key.lower() or search_key.lower() in str(value).lower():
                            matching_config[existing_key] = value

@@ -79,18 +90,18 @@ def config(
            config_options = [
                core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options
            ]
-            matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
-            failed_config = [key for key in config_options if key not in FLAT_CONFIG]
+            matching_config = {key: readable_config[key] for key in config_options if key in readable_config}
+            failed_config = [key for key in config_options if key not in readable_config]
            if failed_config:
                print("\n[red][X] These options failed to get[/red]")
                print("    {}".format("\n    ".join(config_options)))
                raise SystemExit(1)
        else:
-            matching_config = FLAT_CONFIG
+            matching_config = readable_config

        # Display core config sections
        for config_section in CONFIGS.values():
-            section_header = getattr(config_section, "toml_section_header", "")
+            section_header = config_section.toml_section_header
            if isinstance(section_header, str) and section_header:
                print(f"[grey53]\\[{section_header}][/grey53]")
            else:
@@ -100,12 +111,18 @@ def config(
            print(_format_toml(kv_in_section))
            print("[grey53]################################################################[/grey53]")

+        readonly_keys = {key: val for key, val in readonly_config.items() if key in matching_config}
+        if readonly_keys:
+            print("[grey53]\\[CONSTANTS]                                        # (read-only)[/grey53]")
+            print(_format_toml(readonly_keys))
+            print("[grey53]################################################################[/grey53]")
+
        plugin_keys = {}

        # Collect all plugin config keys
        for schema in plugin_schemas.values():
            for key in schema.keys():
-                if key in matching_config:
+                if key in matching_config and key in writable_config:
                    plugin_keys[key] = matching_config[key]

        # Display all plugin config in single [PLUGINS] section
@@ -135,7 +152,7 @@ def config(
                    f"[yellow][i] Note: The config option {raw_key} has been renamed to {key}, please use the new name going forwards.[/yellow]",
                )

-            if key in FLAT_CONFIG:
+            if key in writable_config:
                new_config[key] = val.strip()
            else:
                failed_options.append(line)
@@ -1,27 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox crawl"
-
-import sys
-
-import rich_click as click
-
-from archivebox.cli.archivebox_add import add
-
-
-@click.command(context_settings={"ignore_unknown_options": True})
-@click.option("--depth", "-d", type=int, default=0, help="Max crawl depth (default: 0)")
-@click.option("--tag", "-t", default="", help="Comma-separated tags to add")
-@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
-@click.option("--wait/--no-wait", "wait", default=True, help="Accepted for backwards compatibility")
-@click.argument("urls", nargs=-1)
-def main(depth: int, tag: str, status: str, wait: bool, urls: tuple[str, ...]):
-    """Backwards-compatible `archivebox crawl URL...` entrypoint."""
-    del status, wait
-    add(list(urls), depth=depth, tag=tag, bg=True)
-    sys.exit(0)
-
-
-if __name__ == "__main__":
-    main()
@@ -48,10 +48,11 @@ def process_archiveresult_by_id(archiveresult_id: str) -> int:
    from rich import print as rprint
    from django.utils import timezone
    from archivebox.core.models import ArchiveResult
+    from archivebox.api.v1_core import _uuid_ref_query
    from archivebox.services.runner import run_crawl

    try:
-        archiveresult = ArchiveResult.objects.get(id=archiveresult_id)
+        archiveresult = ArchiveResult.objects.get(_uuid_ref_query("id", archiveresult_id))
    except ArchiveResult.DoesNotExist:
        rprint(f"[red]ArchiveResult {archiveresult_id} not found[/red]", file=sys.stderr)
        return 1
@@ -116,6 +117,7 @@ def run_plugins(
    wait: bool = True,
    emit_results: bool = True,
    show_progress: bool = True,
+    preserve_queued: bool = False,
 ) -> int:
    """
    Run plugins on Snapshots from input.
@@ -209,7 +211,7 @@ def run_plugins(
            if snapshot_id in existing_snapshot_ids
            for plugin_name in plugin_names
        )
-    plugins_by_name = discover_plugins()
+    plugins_by_name = discover_plugins(runtime="archivebox")
    requested_rows: set[tuple[str, str, str]] = set()
    for snapshot_id, plugin_name in requested_pairs:
        plugin = plugins_by_name.get(plugin_name)
@@ -219,6 +221,18 @@ def run_plugins(
        else:
            requested_rows.add((snapshot_id, plugin_name, ""))

+    queued_rows: set[tuple[str, str, str]] = set()
+    if preserve_queued and requested_rows:
+        queued_rows = {
+            (str(snapshot_id), plugin_name, hook_name)
+            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
+                snapshot_id__in=existing_snapshot_ids,
+                plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in requested_rows},
+                status=ArchiveResult.StatusChoices.QUEUED,
+            ).values_list("snapshot_id", "plugin", "hook_name")
+        }
+    rows_to_queue = requested_rows - queued_rows
+
    reset_fields = {
        "status": ArchiveResult.StatusChoices.QUEUED,
        "output_str": "",
@@ -230,23 +244,34 @@ def run_plugins(
        "end_ts": None,
        "modified_at": timezone.now(),
    }
-    if plugins_list:
-        ArchiveResult.objects.filter(snapshot_id__in=existing_snapshot_ids, plugin__in=plugins_list).update(**reset_fields)
-    elif requested_plugins_by_snapshot:
-        snapshot_ids_by_plugin: dict[str, set[str]] = defaultdict(set)
-        for snapshot_id, plugin_names in requested_plugins_by_snapshot.items():
-            if snapshot_id in existing_snapshot_ids:
-                for plugin_name in plugin_names:
-                    snapshot_ids_by_plugin[plugin_name].add(snapshot_id)
-        for plugin_name, plugin_snapshot_ids in snapshot_ids_by_plugin.items():
-            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name).update(**reset_fields)
-    existing_rows = set(
-        ArchiveResult.objects.filter(
-            snapshot_id__in=existing_snapshot_ids,
-            plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in requested_rows},
-        ).values_list("snapshot_id", "plugin", "hook_name"),
+    if rows_to_queue and plugins_list:
+        rows_to_reset_by_hook: dict[tuple[str, str], set[str]] = defaultdict(set)
+        for snapshot_id, plugin_name, hook_name in rows_to_queue:
+            rows_to_reset_by_hook[(plugin_name, hook_name)].add(snapshot_id)
+        for (plugin_name, hook_name), plugin_snapshot_ids in rows_to_reset_by_hook.items():
+            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name, hook_name=hook_name).update(
+                **reset_fields,
+            )
+    elif rows_to_queue and requested_plugins_by_snapshot:
+        snapshot_ids_by_hook: dict[tuple[str, str], set[str]] = defaultdict(set)
+        for snapshot_id, plugin_name, hook_name in rows_to_queue:
+            snapshot_ids_by_hook[(plugin_name, hook_name)].add(snapshot_id)
+        for (plugin_name, hook_name), plugin_snapshot_ids in snapshot_ids_by_hook.items():
+            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name, hook_name=hook_name).update(
+                **reset_fields,
+            )
+    existing_rows = (
+        {
+            (str(snapshot_id), plugin_name, hook_name)
+            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
+                snapshot_id__in=existing_snapshot_ids,
+                plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in rows_to_queue},
+            ).values_list("snapshot_id", "plugin", "hook_name")
+        }
+        if rows_to_queue
+        else set()
    )
-    missing_rows = requested_rows - {(str(snapshot_id), plugin_name, hook_name) for snapshot_id, plugin_name, hook_name in existing_rows}
+    missing_rows = rows_to_queue - existing_rows
    if missing_rows:
        ArchiveResult.objects.bulk_create(
            [
@@ -271,7 +296,23 @@ def run_plugins(
            # also keep status=paused here: `retry_at` only asks the orchestrator
            # to process the queued plugin rows, and run_due_snapshot restores
            # retry_at=MAX afterward instead of resuming the snapshot lifecycle.
-            for snapshot in Snapshot.objects.filter(id__in=existing_snapshot_ids).only("id", "status", "modified_at"):
+            affected_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in rows_to_queue}
+            if preserve_queued and queued_rows:
+                queued_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in queued_rows}
+                affected_snapshot_ids.update(
+                    str(snapshot_id)
+                    for snapshot_id in Snapshot.objects.filter(id__in=queued_snapshot_ids)
+                    .filter(retry_at__gt=queue_at)
+                    .values_list("id", flat=True)
+                )
+                affected_snapshot_ids.update(
+                    str(snapshot_id)
+                    for snapshot_id in Snapshot.objects.filter(id__in=queued_snapshot_ids, retry_at__isnull=True).values_list(
+                        "id",
+                        flat=True,
+                    )
+                )
+            for snapshot in Snapshot.objects.filter(id__in=affected_snapshot_ids).only("id", "status", "modified_at"):
                # Guard the read-time status so we never bump retry_at on a
                # row that's been re-queued / started by a concurrent runner.
                snapshot.safe_update(
@@ -378,16 +419,11 @@ def run_plugins(


 def is_archiveresult_id(value: str) -> bool:
-    """Check if value looks like an ArchiveResult UUID."""
-    import re
-
-    uuid_pattern = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.I)
-    if not uuid_pattern.match(value):
-        return False
-    # Verify it's actually an ArchiveResult (not a Snapshot or other object)
+    """Check if value resolves to an ArchiveResult ID."""
    from archivebox.core.models import ArchiveResult
+    from archivebox.api.v1_core import _uuid_ref_query

-    return ArchiveResult.objects.filter(id=value).exists()
+    return ArchiveResult.objects.filter(_uuid_ref_query("id", value)).exists()


@click.command()
@@ -13,6 +13,10 @@ from rich.panel import Panel


 def _command_doc(cmd: str, import_path: str) -> str:
+    def first_doc_line(docstring: str | None) -> str:
+        lines = (docstring or "").splitlines()
+        return lines[0] if lines else ""
+
    modname, _ = import_path.rsplit(".", 1)
    spec = importlib.util.find_spec(modname)
    if spec is None or spec.origin is None:
@@ -26,9 +30,9 @@ def _command_doc(cmd: str, import_path: str) -> str:
    for name in (cmd, "main"):
        for node in tree.body:
            if isinstance(node, ast.FunctionDef) and node.name == name:
-                return (ast.get_docstring(node) or "").splitlines()[0]
+                return first_doc_line(ast.get_docstring(node))

-    return (ast.get_docstring(tree) or "").splitlines()[0]
+    return first_doc_line(ast.get_docstring(tree))


 def help() -> None:
@@ -85,8 +89,8 @@ def help() -> None:
    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration]https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration[/link]
 """)

-    config = get_config()
-    if os.access(config.ARCHIVE_DIR, os.R_OK) and config.ARCHIVE_DIR.is_dir():
+    get_config()
+    if os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and CONSTANTS.ARCHIVE_DIR.is_dir():
        pretty_out_dir = str(CONSTANTS.DATA_DIR).replace(str(Path("~").expanduser()), "~")
        EXAMPLE_USAGE = f"""
 [light_slate_blue]DATA DIR[/light_slate_blue]: [yellow]{pretty_out_dir}[/yellow]
@@ -25,7 +25,7 @@ def _display_data_path(path: Path, data_dir: Path) -> str:
 def init(force: bool = False, quick: bool = False, install: bool = False) -> None:
    """Initialize a new ArchiveBox collection in the current directory"""

-    from archivebox.config import CONSTANTS, VERSION, DATA_DIR
+    from archivebox.config import CONSTANTS, VERSION
    from archivebox.config.common import get_config
    from archivebox.config.collection import write_config_file
    from archivebox.misc.db import apply_migrations
@@ -37,7 +37,7 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
    #     print("[red]:warning: This folder contains a JSON index. It is deprecated, and will no longer be kept up to date automatically.[/red]", file=sys.stderr)
    #     print("[red]    You can run `archivebox list --json --with-headers > static_index.json` to manually generate it.[/red]", file=sys.stderr)

-    is_empty = not len(set(os.listdir(DATA_DIR)) - CONSTANTS.ALLOWED_IN_DATA_DIR)
+    is_empty = not len(set(os.listdir(CONSTANTS.DATA_DIR)) - CONSTANTS.ALLOWED_IN_DATA_DIR)
    existing_index = os.path.isfile(CONSTANTS.DATABASE_FILE)
    if is_empty and not existing_index:
        print(f"[turquoise4][+] Initializing a new ArchiveBox v{VERSION} collection...[/turquoise4]")
@@ -65,23 +65,23 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
    else:
        print("\n[green][+] Building archive folder structure...[/green]")

-    archive_path = _display_data_path(config.ARCHIVE_DIR, DATA_DIR)
-    sources_path = _display_data_path(CONSTANTS.SOURCES_DIR, DATA_DIR)
-    logs_path = _display_data_path(CONSTANTS.LOGS_DIR, DATA_DIR)
+    archive_path = _display_data_path(CONSTANTS.ARCHIVE_DIR, CONSTANTS.DATA_DIR)
+    sources_path = _display_data_path(CONSTANTS.SOURCES_DIR, CONSTANTS.DATA_DIR)
+    logs_path = _display_data_path(CONSTANTS.LOGS_DIR, CONSTANTS.DATA_DIR)
    print(f"    + {archive_path}, {sources_path}, {logs_path}...")
    Path(CONSTANTS.SOURCES_DIR).mkdir(exist_ok=True)
-    config.ARCHIVE_DIR.mkdir(parents=True, exist_ok=True)
-    config.USERS_DIR.mkdir(parents=True, exist_ok=True)
+    CONSTANTS.ARCHIVE_DIR.mkdir(parents=True, exist_ok=True)
+    CONSTANTS.USERS_DIR.mkdir(parents=True, exist_ok=True)
    Path(CONSTANTS.LOGS_DIR).mkdir(exist_ok=True)
-    for path in (Path(CONSTANTS.SOURCES_DIR), config.ARCHIVE_DIR, config.USERS_DIR, Path(CONSTANTS.LOGS_DIR)):
+    for path in (Path(CONSTANTS.SOURCES_DIR), CONSTANTS.ARCHIVE_DIR, CONSTANTS.USERS_DIR, Path(CONSTANTS.LOGS_DIR)):
        path.chmod(int(config.OUTPUT_PERMISSIONS, base=8) | 0o111)

-    print(f"    + {_display_data_path(CONSTANTS.CONFIG_FILE, DATA_DIR)}...")
+    print(f"    + {_display_data_path(CONSTANTS.CONFIG_FILE, CONSTANTS.DATA_DIR)}...")

    # create the .archivebox_id file with a unique ID for this collection
    from archivebox.config.paths import _get_collection_id

-    _get_collection_id(DATA_DIR, force_create=True)
+    _get_collection_id(CONSTANTS.DATA_DIR, force_create=True)

    # create the ArchiveBox.conf file
    write_config_file({"SECRET_KEY": config.SECRET_KEY})
@@ -94,14 +94,22 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
    from archivebox.config.django import setup_django

    setup_django()
-    check_migrations(blocking=True, auto_apply=False)
+    previous_wants_init = os.environ.get("ARCHIVEBOX_WANTS_INIT")
+    os.environ["ARCHIVEBOX_WANTS_INIT"] = "1"
+    try:
+        check_migrations(blocking=True, auto_apply=False)

-    for migration_line in apply_migrations(DATA_DIR):
-        sys.stdout.write(f"    {migration_line}\n")
+        for migration_line in apply_migrations(CONSTANTS.DATA_DIR):
+            sys.stdout.write(f"    {migration_line}\n")
+    finally:
+        if previous_wants_init is None:
+            os.environ.pop("ARCHIVEBOX_WANTS_INIT", None)
+        else:
+            os.environ["ARCHIVEBOX_WANTS_INIT"] = previous_wants_init

    assert os.path.isfile(CONSTANTS.DATABASE_FILE) and os.access(CONSTANTS.DATABASE_FILE, os.R_OK)
    print()
-    print(f"    √ {_display_data_path(CONSTANTS.DATABASE_FILE, DATA_DIR)}")
+    print(f"    √ {_display_data_path(CONSTANTS.DATABASE_FILE, CONSTANTS.DATA_DIR)}")

    # from django.contrib.auth.models import User
    #     call_command("createsuperuser", interactive=True)
@@ -22,12 +22,11 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo
    """

    from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
-    from archivebox.config.common import get_config
+    from archivebox.config import CONSTANTS
    from archivebox.misc.logging import stderr
    from archivebox.cli.archivebox_init import init

-    config = get_config()
-    archive_dir = config.ARCHIVE_DIR
+    archive_dir = CONSTANTS.ARCHIVE_DIR

    if dry_run:
        print("[dim]Dry run - would detect ArchiveBox dependencies and run the abx-dl install flow[/dim]")
@@ -7,50 +7,15 @@ import sys

 import rich_click as click

-from archivebox.cli.archivebox_snapshot import list_snapshots
+from archivebox.cli.archivebox_snapshot import list_snapshots, snapshot_filter_options, snapshot_output_options


@click.command()
-@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
-@click.option("--url__icontains", help="Filter by URL contains")
-@click.option("--url__istartswith", help="Filter by URL starts with")
-@click.option("--tag", "-t", help="Filter by tag name")
-@click.option("--crawl-id", help="Filter by crawl ID")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title")
-@click.option("--with-headers", is_flag=True, help="Include column headers in structured output")
-@click.option("--search", help="Search mode to use for the query")
-@click.argument("query", nargs=-1)
-def main(
-    status: str | None,
-    url__icontains: str | None,
-    url__istartswith: str | None,
-    tag: str | None,
-    crawl_id: str | None,
-    limit: int | None,
-    sort: str | None,
-    csv: str | None,
-    with_headers: bool,
-    search: str | None,
-    query: tuple[str, ...],
-) -> None:
+@snapshot_output_options
+@snapshot_filter_options(default_filter_type="substring")
+def main(**kwargs) -> None:
    """List Snapshots."""
-    sys.exit(
-        list_snapshots(
-            status=status,
-            url__icontains=url__icontains,
-            url__istartswith=url__istartswith,
-            tag=tag,
-            crawl_id=crawl_id,
-            limit=limit,
-            sort=sort,
-            csv=csv,
-            with_headers=with_headers,
-            search=search,
-            query=" ".join(query),
-        ),
-    )
+    sys.exit(list_snapshots(**kwargs))


 if __name__ == "__main__":
@@ -372,7 +372,7 @@ def create_personas(
        if not is_tty:
            write_record(
                {
-                    "id": str(persona.id) if hasattr(persona, "id") else None,
+                    "id": str(persona.id),
                    "name": persona.name,
                    "path": str(persona.path),
                    "CHROME_USER_DATA_DIR": persona.CHROME_USER_DATA_DIR,
@@ -424,7 +424,7 @@ def list_personas(
        else:
            write_record(
                {
-                    "id": str(persona.id) if hasattr(persona, "id") else None,
+                    "id": str(persona.id),
                    "name": persona.name,
                    "path": str(persona.path),
                    "CHROME_USER_DATA_DIR": persona.CHROME_USER_DATA_DIR,
@@ -500,7 +500,7 @@ def update_personas(name: str | None = None) -> int:
            if not is_tty:
                write_record(
                    {
-                        "id": str(persona.id) if hasattr(persona, "id") else None,
+                        "id": str(persona.id),
                        "name": persona.name,
                        "path": str(persona.path),
                    },
@@ -12,7 +12,7 @@ import rich_click as click
 from django.db import OperationalError
 from django.db.models import QuerySet

-from archivebox.config import DATA_DIR
+from archivebox.config import CONSTANTS
 from archivebox.config.django import setup_django
 from archivebox.misc.util import enforce_types, docstring
 from archivebox.misc.checks import check_data_folder
@@ -23,6 +23,7 @@ from archivebox.misc.logging_util import (
    log_removal_finished,
    TimedProgress,
 )
+from archivebox.cli.archivebox_snapshot import snapshot_filter_options


@enforce_types
@@ -33,49 +34,55 @@ def remove(
    after: float | None = None,
    before: float | None = None,
    yes: bool = False,
-    out_dir: Path = DATA_DIR,
-) -> QuerySet:
+    out_dir: Path = CONSTANTS.DATA_DIR,
+    timeout: float | None = None,
+    **kwargs,
+) -> dict[str, object]:
    """Remove the specified URLs from the archive"""

    setup_django()
    check_data_folder()
+    timeout = float(timeout) if timeout is not None else None

-    from archivebox.cli.archivebox_search import get_snapshots
+    from archivebox.core.models import Snapshot

+    filter_kwargs = {
+        **kwargs,
+        "filter_patterns": filter_patterns,
+        "filter_type": filter_type,
+        "after": after,
+        "before": before,
+    }
    pattern_list = list(filter_patterns)

    log_list_started(pattern_list or None, filter_type)
    timer = TimedProgress(360, prefix="      ")
    try:
-        snapshots = get_snapshots(
-            snapshots=snapshots,
-            filter_patterns=pattern_list or None,
-            filter_type=filter_type,
-            after=after,
-            before=before,
-        )
+        if snapshots is None:
+            snapshots = Snapshot.objects.order_by("-created_at").search(**filter_kwargs)
+        # Freeze the target set up-front so a concurrent daemon writing new
+        # snapshots can't extend the deletion under us, and so the cursor isn't
+        # held open across the per-row deletes below.
+        snapshot_pks = list(snapshots.values_list("pk", flat=True))
    finally:
        timer.end()

-    if not snapshots.exists():
+    if not snapshot_pks:
        log_removal_finished(0, 0)
        raise SystemExit(1)

-    log_list_finished(snapshots)
-    log_removal_started(snapshots, yes=yes)
+    if not yes:
+        log_list_finished(snapshots)
+        log_removal_started(snapshots, yes=False)

-    from archivebox.core.models import Snapshot
    from archivebox.search.query import flush_search_index

-    # Freeze the target set up-front so a concurrent daemon writing new
-    # snapshots can't extend the deletion under us, and so the cursor isn't
-    # held open across the per-row deletes below.
-    snapshot_pks = list(snapshots.values_list("pk", flat=True))
-    to_remove = len(snapshot_pks)
+    started_at = time.monotonic()
+    deadline = started_at + timeout if timeout is not None else None

    # Search-index flush touches a separate backend (FTS / sonic), not the
    # main index.sqlite3 writer lock, so it's safe to do once up front.
-    flush_search_index(snapshots=Snapshot.objects.filter(pk__in=snapshot_pks))
+    flush_search_index(snapshots=snapshots)

    # Delete one snapshot at a time. Each ``.delete()`` is its own short
    # Django-atomic block, so the writer lock is released between rows and
@@ -90,40 +97,59 @@ def remove(
    # own retry loop at this outer (non-atomic) level. Each attempt is a
    # fresh atomic; an exception cleanly rolls it back before we sleep.
    retry_interval = 1.0
-    retry_timeout = 60.0
-    for pk in snapshot_pks:
-        deadline = time.monotonic() + retry_timeout
+    deleted_snapshot_pks = []
+    timed_out = False
+    timeout_error = ""
+    for index, pk in enumerate(snapshot_pks):
+        if deadline is not None and time.monotonic() >= deadline:
+            timed_out = True
+            timeout_error = f"Remove timed out after {timeout:g}s with {len(snapshot_pks) - index} snapshots remaining."
+            break
        while True:
            try:
-                Snapshot.objects.filter(pk=pk).delete()
+                deleted_count, _ = Snapshot.objects.filter(pk=pk).delete()
+                if deleted_count:
+                    deleted_snapshot_pks.append(pk)
                break
            except OperationalError as err:
-                if "database is locked" not in str(err) or time.monotonic() >= deadline:
+                if "database is locked" not in str(err):
                    raise
-                time.sleep(retry_interval)
+                remaining_time = deadline - time.monotonic() if deadline is not None else None
+                if remaining_time is not None and remaining_time <= 0:
+                    timed_out = True
+                    timeout_error = f"Remove timed out after {timeout:g}s while waiting for the database lock."
+                    break
+                time.sleep(min(retry_interval, remaining_time) if remaining_time is not None else retry_interval)
+        if timed_out:
+            break

    all_snapshots = Snapshot.objects.all()
-    log_removal_finished(all_snapshots.count(), to_remove)
+    remaining_count = all_snapshots.count()
+    deleted_snapshot_id_set = set(deleted_snapshot_pks)
+    remaining_snapshot_pks = [snapshot_id for snapshot_id in snapshot_pks if snapshot_id not in deleted_snapshot_id_set]
+    log_removal_finished(remaining_count, len(deleted_snapshot_pks))

-    return all_snapshots
+    return {
+        "removed_count": len(deleted_snapshot_pks),
+        "removed_snapshot_ids": [str(snapshot_id) for snapshot_id in deleted_snapshot_pks],
+        "not_removed_count": len(remaining_snapshot_pks),
+        "not_removed_snapshot_ids": [str(snapshot_id) for snapshot_id in remaining_snapshot_pks],
+        "success": not timed_out,
+        "error": timeout_error,
+        "timeout": timeout,
+    }


@click.command()
@click.option("--yes", is_flag=True, help="Remove links instantly without prompting to confirm")
-@click.option("--before", type=float, help="Remove only URLs bookmarked before timestamp")
-@click.option("--after", type=float, help="Remove only URLs bookmarked after timestamp")
-@click.option(
-    "--filter-type",
-    "-f",
-    type=click.Choice(("exact", "substring", "domain", "regex", "tag")),
-    default="exact",
-    help="Type of pattern matching to use when filtering URLs",
-)
-@click.argument("filter_patterns", nargs=-1)
+@click.option("--timeout", type=float, default=None, help="Maximum seconds to spend deleting snapshots")
+@snapshot_filter_options(default_filter_type="exact")
@docstring(remove.__doc__)
 def main(**kwargs):
    """Remove the specified URLs from the archive"""
-    remove(**kwargs)
+    result = remove(**kwargs)
+    if not result["success"]:
+        raise SystemExit(124)


 if __name__ == "__main__":
@@ -40,14 +40,23 @@ Examples:
 __package__ = "archivebox.cli"
 __command__ = "archivebox run"

-import sys
 import asyncio
+import os
+import signal
+import sys
 from collections import defaultdict

 import rich_click as click
 from rich import print as rprint


+RUNNER_DAEMON_ENV = "ARCHIVEBOX_RUNNER_DAEMON"
+
+
+def _exit_daemon_runner_on_signal(sig: signal.Signals) -> None:
+    os._exit(128 + int(sig))
+
+
 def process_stdin_records() -> int:
    """
    Process JSONL records from stdin.
@@ -234,6 +243,7 @@ def process_stdin_records() -> int:
                    crawl_id,
                    snapshot_ids=None if crawl_id in full_crawl_ids else sorted(snapshot_ids_by_crawl[crawl_id]),
                    selected_plugins=None if crawl_id in run_all_plugins_for_crawl else sorted(plugin_names_by_crawl[crawl_id]),
+                    selected_plugins_are_explicit=False,
                )
    return 0

@@ -250,11 +260,24 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on
    from archivebox.config import CONSTANTS
    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
    from archivebox.machine.models import Machine, Process
-    from archivebox.core.takeover_util import enter_single_runner_gate
-    from archivebox.services.runner import recover_orchestrator_state, run_pending_crawls
+    from archivebox.core.takeover_util import enter_single_runner_gate, standby_until_foreground_runner_needed
+    from archivebox.core.recovery_util import recover_orchestrator_state
+    from archivebox.services.runner import run_pending_crawls

    Machine.current()
    current = Process.current()
+    root_command = current.root
+    if daemon and root_command.process_type in (
+        Process.TypeChoices.SERVER,
+        Process.TypeChoices.ADD,
+        Process.TypeChoices.UPDATE,
+    ):
+        # Server-owned daemon runners are persistent supervisor workers, but
+        # foreground add/update commands are allowed to borrow runner/sonic
+        # leadership without taking down Daphne. Waiting here keeps the worker
+        # on the normal runner path while preventing a server restart loop from
+        # immediately stealing the single-runner gate back from the newer CLI.
+        standby_until_foreground_runner_needed(root_command, data_dir=CONSTANTS.DATA_DIR)
    if not enter_single_runner_gate(current, data_dir=CONSTANTS.DATA_DIR):
        current.mark_exited()
        return 0
@@ -281,8 +304,16 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on
    # hook, continue/retry, second Ctrl+C exits" flow. Server/update/run owned
    # orchestrators should shut down immediately and cleanly on the first signal.
    interactive_interrupts = current.root.process_type == Process.TypeChoices.ADD
+    if daemon:
+        os.environ[RUNNER_DAEMON_ENV] = "1"
    try:
-        with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=not daemon):
+        with (
+            foreground_shutdown_signals(
+                on_signal=_exit_daemon_runner_on_signal if daemon else None,
+                raise_on_first_signal=not daemon,
+            ),
+            foreground_parent_watchdog(enabled=not daemon),
+        ):
            run_pending_crawls(
                daemon=daemon,
                crawl_id=crawl_id,
@@ -290,7 +321,12 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on
                interactive_interrupts=interactive_interrupts,
            )
        return 0
-    except (KeyboardInterrupt, asyncio.CancelledError):
+    except KeyboardInterrupt:
+        return 0
+    except asyncio.CancelledError as e:
+        if daemon:
+            rprint(f"[red]Runner cancelled unexpectedly: {type(e).__name__}: {e}[/red]", file=sys.stderr)
+            return 1
        return 0
    except Exception as e:
        rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
@@ -307,7 +343,8 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on
@click.option("--snapshot-id", help="Run one snapshot through its crawl")
@click.option("--binary-id", help="Run one queued binary install directly on the bus")
@click.option("--maintenance-only", is_flag=True, help="Only process due maintenance ticks on sealed/paused snapshots")
-def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintenance_only: bool):
+@click.option("--no-stdin", is_flag=True, hidden=True, help="Run the scheduler even when stdin is not a TTY")
+def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintenance_only: bool, no_stdin: bool):
    """
    Process queued work.

@@ -322,7 +359,14 @@ def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintena

    if daemon and not snapshot_id and not binary_id and not crawl_id:
        try:
-            with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=False):
+            os.environ[RUNNER_DAEMON_ENV] = "1"
+            with (
+                foreground_shutdown_signals(
+                    on_signal=_exit_daemon_runner_on_signal,
+                    raise_on_first_signal=False,
+                ),
+                foreground_parent_watchdog(enabled=False),
+            ):
                sys.exit(run_runner(daemon=True, maintenance_only=maintenance_only))
        except KeyboardInterrupt:
            sys.exit(0)
@@ -352,7 +396,7 @@ def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintena
        if maintenance_only:
            sys.exit(run_runner(daemon=daemon, maintenance_only=True))

-        if not sys.stdin.isatty():
+        if not no_stdin and not sys.stdin.isatty():
            sys.exit(process_stdin_records())
        else:
            sys.exit(run_runner(daemon=daemon, maintenance_only=maintenance_only))
@@ -367,10 +411,14 @@ def run_snapshot_worker(snapshot_id: str) -> int:
    snapshot = None
    try:
        with foreground_shutdown_signals(), foreground_parent_watchdog():
-            snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
-            if snapshot.retry_at is None:
-                snapshot.update_and_requeue(retry_at=timezone.now())
-            run_due_snapshot(snapshot, lock_seconds=60)
+            for _ in range(10):
+                snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
+                if snapshot.retry_at is None:
+                    snapshot.update_and_requeue(retry_at=timezone.now())
+                elif snapshot.retry_at > timezone.now():
+                    break
+                if not run_due_snapshot(snapshot, lock_seconds=60):
+                    break
        return 0
    except KeyboardInterrupt:
        try:
@@ -3,255 +3,7 @@
 __package__ = "archivebox.cli"
 __command__ = "archivebox search"

-import sys
-from pathlib import Path
-from typing import TYPE_CHECKING
-from collections.abc import Callable
-
-import rich_click as click
-
-from django.db.models import Q, QuerySet
-
-from archivebox.config import DATA_DIR
-from archivebox.config.common import get_config
-from archivebox.misc.logging import stderr
-from archivebox.misc.util import enforce_types, docstring
-
-if TYPE_CHECKING:
-    from archivebox.core.models import Snapshot
-
-# Filter types for URL matching
-LINK_FILTERS: dict[str, Callable[[str], Q]] = {
-    "exact": lambda pattern: Q(url=pattern),
-    "substring": lambda pattern: Q(url__icontains=pattern),
-    "regex": lambda pattern: Q(url__iregex=pattern),
-    "domain": lambda pattern: (
-        Q(url__istartswith=f"http://{pattern}") | Q(url__istartswith=f"https://{pattern}") | Q(url__istartswith=f"ftp://{pattern}")
-    ),
-    "tag": lambda pattern: Q(tags__name=pattern),
-    "timestamp": lambda pattern: Q(timestamp=pattern),
-}
-
-STATUS_CHOICES = ["indexed", "archived", "unarchived"]
-
-
-def _apply_pattern_filters(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    filter_patterns: list[str],
-    filter_type: str,
-) -> QuerySet["Snapshot", "Snapshot"]:
-    filter_builder = LINK_FILTERS.get(filter_type)
-    if filter_builder is None:
-        stderr()
-        stderr(f"[X] Got invalid pattern for --filter-type={filter_type}", color="red")
-        raise SystemExit(2)
-
-    query = Q()
-    for pattern in filter_patterns:
-        query |= filter_builder(pattern)
-    return snapshots.filter(query)
-
-
-def _snapshots_to_json(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    *,
-    with_headers: bool,
-) -> str:
-    from datetime import datetime, timezone as tz
-
-    from archivebox.config import VERSION
-    from archivebox.misc.util import to_json
-
-    config = get_config()
-    main_index_header = (
-        {
-            "info": "This is an index of site data archived by ArchiveBox: The self-hosted web archive.",
-            "schema": "archivebox.index.json",
-            "copyright_info": config.FOOTER_INFO,
-            "meta": {
-                "project": "ArchiveBox",
-                "version": VERSION,
-                "git_sha": VERSION,
-                "website": "https://ArchiveBox.io",
-                "docs": "https://github.com/ArchiveBox/ArchiveBox/wiki",
-                "source": "https://github.com/ArchiveBox/ArchiveBox",
-                "issues": "https://github.com/ArchiveBox/ArchiveBox/issues",
-                "dependencies": {},
-            },
-        }
-        if with_headers
-        else {}
-    )
-
-    snapshot_dicts = [snapshot.to_dict(extended=True) for snapshot in snapshots.iterator(chunk_size=500)]
-    output: dict[str, object] | list[dict[str, object]]
-    if with_headers:
-        output = {
-            **main_index_header,
-            "num_links": len(snapshot_dicts),
-            "updated": datetime.now(tz.utc),
-            "last_run_cmd": sys.argv,
-            "links": snapshot_dicts,
-        }
-    else:
-        output = snapshot_dicts
-
-    return to_json(output, indent=4, sort_keys=True)
-
-
-def _snapshots_to_csv(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    *,
-    cols: list[str],
-    with_headers: bool,
-) -> str:
-    header = ",".join(cols) if with_headers else ""
-    rows = [snapshot.to_csv(cols=cols, separator=",") for snapshot in snapshots.iterator(chunk_size=500)]
-    return "\n".join((header, *rows))
-
-
-def _snapshots_to_html(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    *,
-    with_headers: bool,
-) -> str:
-    from datetime import datetime, timezone as tz
-
-    from django.template.loader import render_to_string
-
-    from archivebox.config import VERSION
-    from archivebox.config.version import get_COMMIT_HASH
-
-    config = get_config()
-    template = "static_index.html" if with_headers else "minimal_index.html"
-    snapshot_list = list(snapshots.iterator(chunk_size=500))
-
-    return render_to_string(
-        template,
-        {
-            "version": VERSION,
-            "git_sha": get_COMMIT_HASH() or VERSION,
-            "num_links": str(len(snapshot_list)),
-            "date_updated": datetime.now(tz.utc).strftime("%Y-%m-%d"),
-            "time_updated": datetime.now(tz.utc).strftime("%Y-%m-%d %H:%M"),
-            "links": snapshot_list,
-            "FOOTER_INFO": config.FOOTER_INFO,
-        },
-    )
-
-
-def get_snapshots(
-    snapshots: QuerySet["Snapshot", "Snapshot"] | None = None,
-    filter_patterns: list[str] | None = None,
-    filter_type: str = "substring",
-    after: float | None = None,
-    before: float | None = None,
-    out_dir: Path = DATA_DIR,
-) -> QuerySet["Snapshot", "Snapshot"]:
-    """Filter and return Snapshots matching the given criteria."""
-    from archivebox.core.models import Snapshot
-
-    if snapshots is not None:
-        result = snapshots
-    else:
-        result = Snapshot.objects.all()
-
-    if after is not None:
-        result = result.filter(timestamp__gte=after)
-    if before is not None:
-        result = result.filter(timestamp__lt=before)
-    if filter_patterns:
-        result = _apply_pattern_filters(result, filter_patterns, filter_type)
-
-    # Prefetch crawl relationship to avoid N+1 queries when accessing output_dir
-    result = result.select_related("crawl", "crawl__created_by")
-
-    if not result.exists():
-        stderr("[!] No Snapshots matched your filters:", filter_patterns, f"({filter_type})", color="lightyellow")
-
-    return result
-
-
-@enforce_types
-def search(
-    filter_patterns: list[str] | None = None,
-    filter_type: str = "substring",
-    status: str = "indexed",
-    before: float | None = None,
-    after: float | None = None,
-    sort: str | None = None,
-    json: bool = False,
-    html: bool = False,
-    csv: str | None = None,
-    with_headers: bool = False,
-):
-    """List, filter, and export information about archive entries"""
-
-    if with_headers and not (json or html or csv):
-        stderr("[X] --with-headers requires --json, --html or --csv\n", color="red")
-        raise SystemExit(2)
-
-    # Query DB directly - no filesystem scanning
-    snapshots = get_snapshots(
-        filter_patterns=list(filter_patterns) if filter_patterns else None,
-        filter_type=filter_type,
-        before=before,
-        after=after,
-    )
-
-    # Apply status filter
-    if status == "archived":
-        snapshots = snapshots.filter(downloaded_at__isnull=False)
-    elif status == "unarchived":
-        snapshots = snapshots.filter(downloaded_at__isnull=True)
-    # 'indexed' = all snapshots (no filter)
-
-    if sort:
-        snapshots = snapshots.order_by(sort)
-
-    # Export to requested format
-    if json:
-        output = _snapshots_to_json(snapshots, with_headers=with_headers)
-    elif html:
-        output = _snapshots_to_html(snapshots, with_headers=with_headers)
-    elif csv:
-        output = _snapshots_to_csv(snapshots, cols=csv.split(","), with_headers=with_headers)
-    else:
-        from archivebox.misc.logging_util import printable_folders
-
-        # Convert to dict for printable_folders
-        folders: dict[str, Snapshot | None] = {str(snapshot.output_dir): snapshot for snapshot in snapshots}
-        output = printable_folders(folders, with_headers)
-
-    # Structured exports must be written directly to stdout.
-    # rich.print() reflows long lines to console width, which corrupts JSON/CSV/HTML output.
-    sys.stdout.write(output)
-    if not output.endswith("\n"):
-        sys.stdout.write("\n")
-    return output
-
-
-@click.command()
-@click.option(
-    "--filter-type",
-    "-f",
-    type=click.Choice(["search", *LINK_FILTERS.keys()]),
-    default="substring",
-    help="Pattern matching type for filtering URLs",
-)
-@click.option("--status", "-s", type=click.Choice(STATUS_CHOICES), default="indexed", help="List snapshots with the given status")
-@click.option("--before", "-b", type=float, help="List snapshots bookmarked before the given UNIX timestamp")
-@click.option("--after", "-a", type=float, help="List snapshots bookmarked after the given UNIX timestamp")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--json", "-J", is_flag=True, help="Print output in JSON format")
-@click.option("--html", "-M", is_flag=True, help="Print output in HTML format (suitable for viewing statically without a server)")
-@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: created_at,url,title")
-@click.option("--with-headers", "-H", is_flag=True, help="Include extra CSV/HTML headers in the output")
-@click.help_option("--help", "-h")
-@click.argument("filter_patterns", nargs=-1)
-@docstring(search.__doc__)
-def main(**kwargs):
-    return search(**kwargs)
+from archivebox.cli.archivebox_list import main


 if __name__ == "__main__":
@@ -12,6 +12,7 @@ from collections.abc import Iterable
 import rich_click as click
 from rich import print

+from archivebox.config import CONSTANTS
 from archivebox.misc.util import docstring, enforce_types


@@ -259,8 +260,6 @@ def server(
    host, port = _parse_and_validate_bind_spec(bind_spec)

    if daemonize and os.environ.get("ARCHIVEBOX_SERVER_DAEMON_CHILD") != "1":
-        from archivebox.config import CONSTANTS
-
        log_path = CONSTANTS.LOGS_DIR / "server.log"
        log_path.parent.mkdir(parents=True, exist_ok=True)
        daemon_env = os.environ.copy()
@@ -314,6 +313,7 @@ def server(
    from archivebox.core.takeover_util import (
        command_owns_runtime_stack,
        current_command,
+        foreground_runner_owner,
        runtime_stack_owner,
        standby_until_runtime_stack_needed,
    )
@@ -337,21 +337,28 @@ def server(
    runtime_config = get_config()
    _print_server_startup_warnings(runtime_config, host, port)
    bind_url = f"http://{host}:{port}"
-    command = current_command(Process.TypeChoices.SERVER, data_dir=config.DATA_DIR, url=bind_url)
+    command = current_command(Process.TypeChoices.SERVER, data_dir=CONSTANTS.DATA_DIR, url=bind_url)

    def still_owns_runtime_stack() -> bool:
        from django.db import connections

        try:
-            return command_owns_runtime_stack(command, data_dir=config.DATA_DIR)
+            return command_owns_runtime_stack(command, data_dir=CONSTANTS.DATA_DIR)
        finally:
            connections.close_all()

+    shutdown_state = None
    try:
-        with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=os.environ.get("ARCHIVEBOX_SERVER_DAEMON_CHILD") != "1"):
+        with (
+            foreground_shutdown_signals() as shutdown_state,
+            foreground_parent_watchdog(enabled=os.environ.get("ARCHIVEBOX_SERVER_DAEMON_CHILD") != "1"),
+        ):
            while True:
-                standby_result = standby_until_runtime_stack_needed(command, data_dir=config.DATA_DIR)
-                older_owner = runtime_stack_owner(data_dir=config.DATA_DIR, exclude_id=command.id)
+                standby_result = standby_until_runtime_stack_needed(command, data_dir=CONSTANTS.DATA_DIR)
+                older_owner = runtime_stack_owner(data_dir=CONSTANTS.DATA_DIR, exclude_id=command.id) or foreground_runner_owner(
+                    data_dir=CONSTANTS.DATA_DIR,
+                    exclude_id=command.id,
+                )
                takeover_components = active_supervisord_runtime_components(config=config)
                if older_owner and takeover_components:
                    print(
@@ -386,7 +393,8 @@ def server(
    except KeyboardInterrupt:
        pass
    finally:
-        command.mark_exited()
+        if not shutdown_state or not shutdown_state.signal_name:
+            command.mark_exited()
    print("\n[i][green][🟩] ArchiveBox server shut down gracefully.[/green][/i]")


@@ -12,9 +12,10 @@ from archivebox.misc.util import docstring
 def shell(args: Iterable[str] = ()) -> None:
    """Enter an interactive ArchiveBox Django shell"""

-    from django.core.management import call_command
+    from django.core.management import call_command, get_commands

-    call_command("shell_plus", *args)
+    shell_command = "shell_plus" if "shell_plus" in get_commands() else "shell"
+    call_command(shell_command, *args)


@click.command(add_help_option=False, context_settings=dict(ignore_unknown_options=True))
@@ -35,9 +35,10 @@ from collections.abc import Iterable

 import rich_click as click
 from rich import print as rprint
-from django.db.models import Case, IntegerField, QuerySet, When
+from django.db.models import QuerySet

-from archivebox.cli.cli_util import apply_filters
+SNAPSHOT_FILTER_TYPE_CHOICES = ("exact", "substring", "regex", "domain", "tag", "timestamp")
+SNAPSHOT_LIST_CHUNK_SIZE = 100


 # =============================================================================
@@ -178,72 +179,63 @@ def create_snapshots(
 # =============================================================================


+def snapshot_filter_options(*, default_filter_type: str):
+    def decorate(func):
+        for decorator in reversed(
+            (
+                click.option("--status", "-s", help="Filter by status (queued, started, sealed)"),
+                click.option("--url__icontains", help="Filter by URL contains"),
+                click.option("--url__istartswith", help="Filter by URL starts with"),
+                click.option("--tag", "-t", help="Filter by tag name"),
+                click.option("--crawl-id", help="Filter by crawl ID"),
+                click.option("--limit", "-n", type=int, help="Limit number of results"),
+                click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at"),
+                click.option("--search", help="Search mode to use for positional query"),
+                click.option("--before", type=float, help="Only snapshots bookmarked before timestamp"),
+                click.option("--after", type=float, help="Only snapshots bookmarked after timestamp"),
+                click.option(
+                    "--filter-type",
+                    "-f",
+                    type=click.Choice(SNAPSHOT_FILTER_TYPE_CHOICES),
+                    default=default_filter_type,
+                    help="Type of pattern matching to use for positional filters",
+                ),
+                click.argument("filter_patterns", nargs=-1),
+            ),
+        ):
+            func = decorator(func)
+        return func
+
+    return decorate
+
+
+def snapshot_output_options(func):
+    for decorator in reversed(
+        (
+            click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title"),
+            click.option("--json", "as_json", is_flag=True, help="Print output as a JSON array"),
+            click.option("--html", "as_html", is_flag=True, help="Print output as HTML"),
+            click.option("--with-headers", is_flag=True, help="Include column headers in structured output"),
+        ),
+    ):
+        func = decorator(func)
+    return func
+
+
 def build_snapshot_queryset(
-    *,
-    status: str | None = None,
-    url__icontains: str | None = None,
-    url__istartswith: str | None = None,
-    tag: str | None = None,
-    crawl_id: str | None = None,
-    sort: str | None = None,
-    search: str | None = None,
-    query: str | None = None,
-    limit: int | None = None,
+    **kwargs,
 ) -> QuerySet:
    from archivebox.core.models import Snapshot
-    from archivebox.search.query import apply_snapshot_search

-    queryset = Snapshot.objects.order_by("-created_at")
-    queryset = apply_filters(
-        queryset,
-        {
-            "status": status,
-            "url__icontains": url__icontains,
-            "url__istartswith": url__istartswith,
-            "crawl_id": crawl_id,
-        },
-    )
-
-    if tag:
-        queryset = queryset.filter(tags__name__iexact=tag)
-
-    query = (query or "").strip()
-    if query:
-        try:
-            queryset = apply_snapshot_search(
-                queryset,
-                query,
-                search_mode=search,
-                ordering=("-created_at",) if not sort else None,
-                max_results=limit,
-                skip_backend_when_metadata_satisfies_limit=True,
-                include_metadata_for_forced_backend=True,
-            )
-        except Exception as err:
-            rprint(
-                f"[yellow]Search backend error, falling back to metadata search: {err}[/yellow]",
-                file=sys.stderr,
-            )
-            queryset = apply_snapshot_search(queryset, query, search_mode="meta")
-
-    if sort:
-        queryset = queryset.order_by(sort)
-
-    return queryset
+    return Snapshot.objects.order_by("-created_at").search(**kwargs)


 def list_snapshots(
-    status: str | None = None,
-    url__icontains: str | None = None,
-    url__istartswith: str | None = None,
-    tag: str | None = None,
-    crawl_id: str | None = None,
-    limit: int | None = None,
-    sort: str | None = None,
    csv: str | None = None,
+    as_json: bool = False,
+    as_html: bool = False,
    with_headers: bool = False,
-    search: str | None = None,
-    query: str | None = None,
+    **kwargs,
 ) -> int:
    """
    List Snapshots as JSONL with optional filters.
@@ -252,96 +244,71 @@ def list_snapshots(
        0: Success (even if no results)
    """
    from archivebox.misc.jsonl import write_record
-    from archivebox.core.models import Snapshot

-    if with_headers and not csv:
-        rprint("[red]--with-headers requires --csv[/red]", file=sys.stderr)
+    output_formats = sum(bool(output_format) for output_format in (csv, as_json, as_html))
+    if output_formats > 1:
+        rprint("[red]Choose only one output format: --csv, --json, or --html[/red]", file=sys.stderr)
+        return 2
+    if with_headers and not output_formats:
+        rprint("[red]--with-headers requires --csv, --json, or --html[/red]", file=sys.stderr)
        return 2

-    is_tty = sys.stdout.isatty() and not csv
+    is_tty = sys.stdout.isatty() and not output_formats

-    queryset = build_snapshot_queryset(
-        status=status,
-        url__icontains=url__icontains,
-        url__istartswith=url__istartswith,
-        tag=tag,
-        crawl_id=crawl_id,
-        sort=sort,
-        search=search,
-        query=query,
-        limit=limit,
-    )
-
-    if not is_tty:
-        if limit:
-            limited_ids = list(queryset.values_list("id", flat=True)[:limit])
-            preserved_order = Case(
-                *(When(id=snapshot_id, then=position) for position, snapshot_id in enumerate(limited_ids)),
-                output_field=IntegerField(),
-            )
-            queryset = Snapshot.objects.filter(id__in=limited_ids).order_by(preserved_order)
-        queryset = queryset.prefetch_related("tags")
-    elif limit:
-        queryset = queryset[:limit]
+    try:
+        queryset = build_snapshot_queryset(**kwargs)
+    except ValueError as err:
+        rprint(f"[red]{err}[/red]", file=sys.stderr)
+        return 2

    count = 0
+    if as_json:
+        queryset = queryset.prefetch_related("tags")
+        output = queryset.to_json(with_headers=with_headers)
+        sys.stdout.write(output)
+        if output and not output.endswith("\n"):
+            sys.stdout.write("\n")
+        rprint(f"[dim]Listed {queryset.count()} snapshots[/dim]", file=sys.stderr)
+        return 0
+
+    if as_html:
+        queryset = queryset.prefetch_related("tags")
+        output = queryset.to_html(with_headers=with_headers)
+        sys.stdout.write(output)
+        if output and not output.endswith("\n"):
+            sys.stdout.write("\n")
+        rprint(f"[dim]Listed {queryset.count()} snapshots[/dim]", file=sys.stderr)
+        return 0
+
    if csv:
        cols = [col.strip() for col in csv.split(",") if col.strip()]
        if not cols:
            rprint("[red]No CSV columns provided[/red]", file=sys.stderr)
            return 2
-        rows: list[str] = []
        if with_headers:
-            rows.append(",".join(cols))
-        simple_cols = {
-            "id",
-            "crawl_id",
-            "url",
-            "title",
-            "timestamp",
-            "depth",
-            "status",
-            "fs_version",
-            "bookmarked_at",
-            "created_at",
-            "modified_at",
-            "retry_at",
-            "downloaded_at",
-        }
-        from archivebox.misc.util import to_json
-
-        for snapshot in queryset.iterator(chunk_size=500):
-            if set(cols).issubset(simple_cols):
-                rows.append(
-                    ",".join(
-                        to_json(
-                            value.isoformat() if hasattr((value := getattr(snapshot, col, "")), "isoformat") else value,
-                            indent=None,
-                        )
-                        for col in cols
-                    ),
-                )
-            else:
-                rows.append(snapshot.to_csv(cols=cols, separator=","))
+            sys.stdout.write(",".join(cols))
+            sys.stdout.write("\n")
+        for snapshot in queryset.prefetch_related("tags").iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE):
+            sys.stdout.write(snapshot.to_csv(cols=cols, separator=","))
+            sys.stdout.write("\n")
            count += 1
-        output = "\n".join(rows)
-        if output:
-            sys.stdout.write(output)
-            if not output.endswith("\n"):
-                sys.stdout.write("\n")
        rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
        return 0

-    for snapshot in queryset:
-        if is_tty:
-            status_color = {
-                "queued": "yellow",
-                "started": "blue",
-                "sealed": "green",
-            }.get(snapshot.status, "dim")
-            rprint(f"[{status_color}]{snapshot.status:8}[/{status_color}] [dim]{snapshot.id}[/dim] {snapshot.url[:60]}")
-        else:
+    if not is_tty:
+        for snapshot in queryset.prefetch_related("tags").iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE):
            write_record(snapshot.to_json())
+            count += 1
+        rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
+        return 0
+
+    for snapshot in queryset.iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE):
+        status_color = {
+            "queued": "yellow",
+            "started": "blue",
+            "sealed": "green",
+        }.get(snapshot.status, "dim")
+        rprint(f"[{status_color}]{snapshot.status:8}[/{status_color}] [dim]{snapshot.id}[/dim] {snapshot.url[:60]}")
        count += 1

    rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
@@ -500,46 +467,11 @@ def create_cmd(urls: tuple, tag: str, status: str, depth: int):


@main.command("list")
-@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
-@click.option("--url__icontains", help="Filter by URL contains")
-@click.option("--url__istartswith", help="Filter by URL starts with")
-@click.option("--tag", "-t", help="Filter by tag name")
-@click.option("--crawl-id", help="Filter by crawl ID")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title")
-@click.option("--with-headers", is_flag=True, help="Include column headers in structured output")
-@click.option("--search", help="Search mode to use for the query")
-@click.argument("query", nargs=-1)
-def list_cmd(
-    status: str | None,
-    url__icontains: str | None,
-    url__istartswith: str | None,
-    tag: str | None,
-    crawl_id: str | None,
-    limit: int | None,
-    sort: str | None,
-    csv: str | None,
-    with_headers: bool,
-    search: str | None,
-    query: tuple[str, ...],
-):
+@snapshot_output_options
+@snapshot_filter_options(default_filter_type="substring")
+def list_cmd(**kwargs):
    """List Snapshots as JSONL."""
-    sys.exit(
-        list_snapshots(
-            status=status,
-            url__icontains=url__icontains,
-            url__istartswith=url__istartswith,
-            tag=tag,
-            crawl_id=crawl_id,
-            limit=limit,
-            sort=sort,
-            csv=csv,
-            with_headers=with_headers,
-            search=search,
-            query=" ".join(query),
-        ),
-    )
+    sys.exit(list_snapshots(**kwargs))


@main.command("update")
@@ -1,24 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox snapshot"
-
-import sys
-
-import rich_click as click
-
-from archivebox.cli.archivebox_snapshot import create_snapshots
-
-
-@click.command(context_settings={"ignore_unknown_options": True})
-@click.option("--tag", "-t", default="", help="Comma-separated tags to add")
-@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
-@click.option("--depth", "-d", type=int, default=0, help="Crawl depth (default: 0)")
-@click.argument("urls", nargs=-1)
-def main(tag: str, status: str, depth: int, urls: tuple[str, ...]):
-    """Backwards-compatible `archivebox snapshot URL...` entrypoint."""
-    sys.exit(create_snapshots(urls, tag=tag, status=status, depth=depth))
-
-
-if __name__ == "__main__":
-    main()
@@ -8,7 +8,7 @@ import rich_click as click
 from rich import print

 from archivebox.misc.util import enforce_types, docstring
-from archivebox.config import DATA_DIR, CONSTANTS
+from archivebox.config import CONSTANTS
 from archivebox.config.common import get_config
 from archivebox.misc.system import get_dir_size
 from archivebox.misc.logging_util import printable_filesize
@@ -18,7 +18,7 @@ MAX_STATUS_FS_DIR_SCAN = 5000


@enforce_types
-def status(out_dir: Path = DATA_DIR) -> None:
+def status(out_dir: Path = CONSTANTS.DATA_DIR) -> None:
    """Print out some info and statistics about the archive collection"""

    from django.contrib.auth import get_user_model
@@ -38,7 +38,7 @@ def status(out_dir: Path = DATA_DIR) -> None:

    snapshots_qs = Snapshot.objects.all()
    num_sql_links = snapshots_qs.count()
-    archive_dir = config.ARCHIVE_DIR
+    archive_dir = CONSTANTS.ARCHIVE_DIR
    legacy_snapshot_dirs = []
    if archive_dir.exists():
        legacy_snapshot_dirs = [
@@ -48,7 +48,7 @@ def status(out_dir: Path = DATA_DIR) -> None:
    print(f"    > JSON Link Details: {len(legacy_snapshot_dirs)} links".ljust(36), f"(found in {archive_dir.name}/*/index.json)")
    print()
    print("[green]\\[*] Scanning archive data directories...[/green]")
-    users_dir = config.USERS_DIR
+    users_dir = CONSTANTS.USERS_DIR
    scan_roots = [root for root in (archive_dir, users_dir) if root.exists()]
    scan_roots_display = ", ".join(str(root) for root in scan_roots) if scan_roots else str(archive_dir)
    print(f"[yellow]   {scan_roots_display}[/yellow]")
@@ -105,7 +105,7 @@ def status(out_dir: Path = DATA_DIR) -> None:

    if num_indexed:
        print("    [violet]Hint:[/violet] You can list snapshots by status like so:")
-        print("        [green]archivebox list --status=<status>  (e.g. archived, queued, etc.)[/green]")
+        print("        [green]archivebox list --status=<status>  (e.g. sealed, queued, etc.)[/green]")

    if orphaned_dirs:
        print("    [violet]Hint:[/violet] To automatically import orphaned data directories into the main index, run:")
@@ -15,6 +15,7 @@ from pathlib import Path
 import rich_click as click

 from archivebox.misc.util import enforce_types, docstring
+from archivebox.cli.archivebox_snapshot import snapshot_filter_options

 if TYPE_CHECKING:
    from django.db.models import QuerySet
@@ -32,66 +33,36 @@ def _get_snapshot_crawl(snapshot: Snapshot) -> Crawl | None:


 def _get_search_indexing_plugins() -> list[str]:
-    from abx_dl.models import discover_plugins
+    from archivebox.config.common import get_config
+    from archivebox.plugins.hooks import discover_hooks
    from archivebox.plugins.discovery import get_search_backends

    available_backends = set(get_search_backends())
-    plugins = discover_plugins()
    return sorted(
        plugin_name
-        for plugin_name, plugin in plugins.items()
-        if plugin_name.startswith("search_backend_")
-        and plugin_name.removeprefix("search_backend_") in available_backends
-        and any("Snapshot" in hook.name and "index" in hook.name.lower() for hook in plugin.hooks)
+        for plugin_name in {
+            hook.parent.name
+            for hook in discover_hooks("Snapshot", config=get_config())
+            if hook.parent.name.startswith("search_backend_") and "index" in hook.name.lower()
+        }
+        if plugin_name.startswith("search_backend_") and plugin_name.removeprefix("search_backend_") in available_backends
    )


 def _build_filtered_snapshots_queryset(
-    *,
-    filter_patterns: Iterable[str],
-    filter_type: str,
-    status: str | None = None,
-    url__icontains: str | None = None,
-    url__istartswith: str | None = None,
-    tag: str | None = None,
-    crawl_id: str | None = None,
-    limit: int | None = None,
-    sort: str | None = None,
-    search: str | None = None,
-    before: float | None = None,
-    after: float | None = None,
-    resume: str | None = None,
+    **kwargs,
 ):
-    from datetime import datetime
+    from archivebox.core.models import Snapshot
    from archivebox.cli.archivebox_snapshot import build_snapshot_queryset

-    filter_patterns = tuple(filter_patterns)
-    snapshots = build_snapshot_queryset(
-        status=status,
-        url__icontains=url__icontains,
-        url__istartswith=url__istartswith,
-        tag=tag,
-        crawl_id=crawl_id,
-        sort=sort,
-        search=search,
-        query=" ".join(filter_patterns) if search else None,
-    )
-
-    if filter_patterns and not search:
-        snapshots = snapshots.filter_by_patterns(list(filter_patterns), filter_type)
-
-    if before:
-        snapshots = snapshots.filter(bookmarked_at__lt=datetime.fromtimestamp(before))
-    if after:
-        snapshots = snapshots.filter(bookmarked_at__gt=datetime.fromtimestamp(after))
-    if resume:
-        snapshots = snapshots.filter(timestamp__lte=resume)
-    if not sort:
-        snapshots = snapshots.order_by("-timestamp")
+    limit = kwargs.pop("limit", None)
+    snapshots = build_snapshot_queryset(**kwargs)
+    if kwargs.get("resume"):
+        snapshots = snapshots.filter(timestamp__lte=kwargs["resume"])
    snapshots = snapshots.select_related("crawl")
-    if limit:
-        limited_ids = list(snapshots.values_list("id", flat=True)[:limit])
-        snapshots = snapshots.model.objects.filter(id__in=limited_ids).select_related("crawl")
+    if limit is not None and limit > 0:
+        snapshot_ids = list(snapshots.values_list("id", flat=True)[:limit])
+        snapshots = Snapshot.objects.filter(id__in=snapshot_ids).select_related("crawl")

    return snapshots

@@ -105,9 +76,17 @@ def reindex_snapshots(
    wait_for_turn=None,
 ) -> dict[str, Any]:
    from archivebox.cli.archivebox_extract import run_plugins
+    from archivebox.core.models import ArchiveResult
+    from abx_dl.models import discover_plugins

-    stats: dict[str, Any] = {"processed": 0, "queued": 0, "reindexed": 0, "snapshot_ids": []}
+    stats: dict[str, Any] = {"processed": 0, "requested": 0, "queued": 0, "skipped_queued": 0, "reindexed": 0, "snapshot_ids": []}
    records: list[dict[str, str]] = []
+    plugins_by_name = discover_plugins(runtime="archivebox")
+    required_hooks_by_plugin = {
+        plugin_name: frozenset(hook.name for hook in plugins_by_name[plugin_name].filter_hooks("Snapshot"))
+        for plugin_name in search_plugins
+        if plugin_name in plugins_by_name
+    }

    total = snapshots.count()
    print(f"[*] Reindexing {total} snapshots with search plugins: {', '.join(search_plugins)}")
@@ -118,6 +97,31 @@ def reindex_snapshots(
        if wait_for_turn:
            wait_for_turn()
        batch_records = list(records)
+        snapshot_ids = {record["snapshot_id"] for record in batch_records}
+        plugin_names = {record["plugin"] for record in batch_records}
+        queued_rows = {
+            (str(snapshot_id), plugin_name, hook_name)
+            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
+                snapshot_id__in=snapshot_ids,
+                plugin__in=plugin_names,
+                status=ArchiveResult.StatusChoices.QUEUED,
+            ).values_list("snapshot_id", "plugin", "hook_name")
+        }
+        records_to_queue = []
+        for record in batch_records:
+            snapshot_id = record["snapshot_id"]
+            plugin_name = record["plugin"]
+            required_hooks = required_hooks_by_plugin.get(plugin_name, frozenset())
+            if required_hooks and all((snapshot_id, plugin_name, hook_name) in queued_rows for hook_name in required_hooks):
+                stats["skipped_queued"] += 1
+                continue
+            records_to_queue.append(record)
+        if not records_to_queue:
+            print(
+                f"    [{stats['processed']}/{total}] Already queued {len(batch_records)} index jobs",
+            )
+            records.clear()
+            return
        # `archivebox update --index-only` intentionally breaks the usual
        # "runner discovers work" rule by inserting synthetic queued
        # ArchiveResult rows for search backends. run_plugins() keeps this as
@@ -127,15 +131,17 @@ def reindex_snapshots(
        # finish.
        exit_code = run_plugins(
            args=(),
-            records=batch_records,
+            records=records_to_queue,
            wait=False,
            emit_results=False,
            show_progress=False,
+            preserve_queued=True,
        )
        if exit_code != 0:
            raise SystemExit(exit_code)
+        stats["queued"] += len(records_to_queue)
        print(
-            f"    [{stats['processed']}/{total}] Queued {len(batch_records)} index jobs for orchestrator",
+            f"    [{stats['processed']}/{total}] Queued {len(records_to_queue)} index jobs for orchestrator",
        )
        records.clear()

@@ -156,7 +162,7 @@ def reindex_snapshots(
                        "plugin": plugin_name,
                    },
                )
-                stats["queued"] += 1
+                stats["requested"] += 1
            if len(records) >= batch_size:
                run_batch()
        except KeyboardInterrupt as err:
@@ -219,27 +225,29 @@ def update(
    from archivebox.machine.models import Process
    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals, raise_if_shutdown_requested
    from archivebox.core.takeover_util import (
-        command_owns_runtime_stack,
+        command_owns_foreground_runner,
        current_command,
        ensure_daemon_stack,
-        standby_until_runtime_stack_needed,
+        standby_until_foreground_runner_needed,
    )
-    from archivebox.workers.supervisord_util import run_runner_worker, stop_existing_supervisord_process, stop_own_supervisord_process
+    from archivebox.workers.supervisord_util import run_runner_worker, stop_own_supervisord_process

    command = current_command(Process.TypeChoices.UPDATE, data_dir=CONSTANTS.DATA_DIR)

    def wait_for_turn() -> None:
        raise_if_shutdown_requested()
-        standby_until_runtime_stack_needed(command, data_dir=CONSTANTS.DATA_DIR)
+        standby_until_foreground_runner_needed(command, data_dir=CONSTANTS.DATA_DIR)
        raise_if_shutdown_requested()

-    def run_scoped_runner(*args: str) -> None:
+    def run_scoped_runner(*args: str, ensure_daemon_reason: str | None = None) -> None:
        while True:
            wait_for_turn()
+            if ensure_daemon_reason:
+                ensure_daemon_stack(reason=ensure_daemon_reason)
            exit_code = run_runner_worker(list(args), name=f"worker_runner_update_{os.getpid()}")
            if exit_code == 0:
                return
-            if not command_owns_runtime_stack(command, data_dir=CONSTANTS.DATA_DIR):
+            if not command_owns_foreground_runner(command, data_dir=CONSTANTS.DATA_DIR):
                continue
            raise SystemExit(exit_code)

@@ -263,8 +271,6 @@ def update(

    try:
        wait_for_turn()
-        if stop_daemon_stack:
-            stop_existing_supervisord_process()

        with foreground_shutdown_signals(), foreground_parent_watchdog():
            while True:
@@ -272,6 +278,9 @@ def update(
                do_index = index_only or not migrate_only
                do_run_until_idle = do_migrate or do_index
                ran_post_migrate_runner = False
+                full_update_empty = False
+                maintenance_work_queued = False
+                runner_work_queued = False

                if do_migrate:
                    if (
@@ -308,6 +317,8 @@ def update(
                        )
                        print_stats(stats)
                        touched_snapshot_ids.update(stats.get("snapshot_ids", []))
+                        maintenance_work_queued = stats.get("queued", 0) > 0
+                        runner_work_queued = runner_work_queued or maintenance_work_queued
                    else:
                        stats_combined = {"phase1": {}, "phase2": {}}

@@ -324,6 +335,17 @@ def update(
                            wait_for_turn=wait_for_turn,
                        )
                        print_combined_stats(stats_combined)
+                        full_update_empty = (
+                            stats_combined["phase1"].get("processed", 0) == 0 and stats_combined["phase2"].get("snapshots", 0) == 0
+                        )
+                        maintenance_work_queued = any(
+                            (
+                                stats_combined["phase1"].get("queued", 0),
+                                stats_combined["phase2"].get("queued", 0),
+                                stats_combined["phase2"].get("crawls_sealed", 0),
+                            ),
+                        )
+                        runner_work_queued = runner_work_queued or maintenance_work_queued

                    if do_run_until_idle:
                        # Filesystem migration is maintenance on existing
@@ -335,8 +357,17 @@ def update(
                        # rows first, runs the targeted plugins, and may leave
                        # the fs_version maintenance tick hidden behind that
                        # plugin work until another update pass.
-                        print("[*] Phase 3: Running filesystem maintenance until idle...")
-                        if is_filtered_update:
+                        if full_update_empty:
+                            print("[*] No snapshots or legacy archive directories found; skipping filesystem maintenance runner.")
+                        elif not maintenance_work_queued:
+                            print("[*] No filesystem maintenance work queued; skipping filesystem maintenance runner.")
+                        else:
+                            print("[*] Phase 3: Running filesystem maintenance until idle...")
+                        if full_update_empty:
+                            pass
+                        elif not maintenance_work_queued:
+                            pass
+                        elif is_filtered_update:
                            if not touched_snapshot_ids:
                                print("[*] No matching snapshots queued work for the runner.")
                            for snapshot_id in sorted(touched_snapshot_ids):
@@ -346,35 +377,80 @@ def update(
                        ran_post_migrate_runner = True

                if do_index:
-                    ensure_daemon_stack(reason="search indexing")
-                    search_plugins = _get_search_indexing_plugins()
-                    if not search_plugins:
-                        print("[*] No search indexing plugins are available, nothing to backfill.")
+                    if full_update_empty:
+                        print("[*] No snapshots found; skipping search indexing backfill.")
                    else:
-                        snapshots = _build_filtered_snapshots_queryset(
-                            filter_patterns=filter_patterns,
-                            filter_type=filter_type,
-                            status=status,
-                            url__icontains=url__icontains,
-                            url__istartswith=url__istartswith,
-                            tag=tag,
-                            crawl_id=crawl_id,
-                            limit=limit,
-                            sort=sort,
-                            search=search,
-                            before=before,
-                            after=after,
-                            resume=resume,
-                        )
-                        stats = reindex_snapshots(
-                            snapshots,
-                            search_plugins=search_plugins,
-                            batch_size=batch_size,
-                            collect_ids=is_filtered_update,
-                            wait_for_turn=wait_for_turn,
-                        )
-                        print_index_stats(stats)
-                        touched_snapshot_ids.update(stats.get("snapshot_ids", []))
+                        ensure_daemon_stack(reason="search indexing")
+                        search_plugins = _get_search_indexing_plugins()
+                        if not search_plugins:
+                            print("[*] No search indexing plugins are available, nothing to backfill.")
+                        else:
+                            snapshots = _build_filtered_snapshots_queryset(
+                                filter_patterns=filter_patterns,
+                                filter_type=filter_type,
+                                status=status,
+                                url__icontains=url__icontains,
+                                url__istartswith=url__istartswith,
+                                tag=tag,
+                                crawl_id=crawl_id,
+                                limit=limit,
+                                sort=sort,
+                                search=search,
+                                before=before,
+                                after=after,
+                                resume=resume,
+                            )
+                            from django.db.models import Exists, OuterRef, Q
+                            from django.utils import timezone
+                            from archivebox.core.models import ArchiveResult, Snapshot
+
+                            scoped_snapshot_ids = snapshots.order_by().values("id") if is_filtered_update else None
+                            queued_index_results = ArchiveResult.objects.filter(
+                                status=ArchiveResult.StatusChoices.QUEUED,
+                                plugin__in=search_plugins,
+                            )
+                            if scoped_snapshot_ids is not None:
+                                queued_index_results = queued_index_results.filter(snapshot_id__in=scoped_snapshot_ids)
+
+                            if queued_index_results.exists():
+                                runner_work_queued = True
+                                now = timezone.now()
+                                queued_result_for_snapshot = queued_index_results.filter(snapshot_id=OuterRef("pk"))
+                                snapshots_to_wake = (
+                                    Snapshot.objects.filter(
+                                        status__in=(Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED),
+                                    )
+                                    .annotate(
+                                        has_queued_index_result=Exists(queued_result_for_snapshot),
+                                    )
+                                    .filter(
+                                        has_queued_index_result=True,
+                                    )
+                                    .filter(
+                                        Q(retry_at__isnull=True) | Q(retry_at__gt=now),
+                                    )
+                                )
+                                if scoped_snapshot_ids is not None:
+                                    snapshots_to_wake = snapshots_to_wake.filter(id__in=scoped_snapshot_ids)
+                                woken_count = snapshots_to_wake.update(
+                                    retry_at=now,
+                                    modified_at=now,
+                                )
+                                print(
+                                    "[*] Existing queued search index jobs found; "
+                                    f"skipping backfill scan and waking {woken_count} snapshot(s) for the runner.",
+                                )
+                            else:
+                                stats = reindex_snapshots(
+                                    snapshots,
+                                    search_plugins=search_plugins,
+                                    batch_size=batch_size,
+                                    collect_ids=True,
+                                    wait_for_turn=wait_for_turn,
+                                )
+                                print_index_stats(stats)
+                                touched_snapshot_ids.update(stats.get("snapshot_ids", []))
+                                runner_work_queued = runner_work_queued or stats["queued"] > 0

                if do_run_until_idle and (do_index or not ran_post_migrate_runner):
                    # Search/index backfill intentionally queues targeted
@@ -384,14 +460,26 @@ def update(
                    # For a normal unfiltered `archivebox update`, keep the
                    # historical final pass broad enough to resume genuinely
                    # queued/interrupted crawl work after maintenance is done.
-                    print("[*] Phase 3: Running queued/interrupted crawl work until idle...")
-                    if is_filtered_update:
+                    if full_update_empty:
+                        print("[*] No snapshots found; skipping queued/interrupted crawl runner.")
+                    elif not runner_work_queued:
+                        print("[*] No queued/interrupted crawl work found; skipping queued/interrupted crawl runner.")
+                    else:
+                        print("[*] Phase 3: Running queued/interrupted crawl work until idle...")
+                    if full_update_empty:
+                        pass
+                    elif not runner_work_queued:
+                        pass
+                    elif touched_snapshot_ids:
                        if not touched_snapshot_ids:
                            print("[*] No matching snapshots queued work for the runner.")
                        for snapshot_id in sorted(touched_snapshot_ids):
                            run_scoped_runner("--snapshot-id", snapshot_id)
                    else:
-                        run_scoped_runner(*(["--maintenance-only"] if index_only or migrate_only else []))
+                        run_scoped_runner(
+                            *(["--maintenance-only"] if index_only or migrate_only else []),
+                            ensure_daemon_reason="search indexing" if do_index else None,
+                        )

                if not continuous:
                    break
@@ -401,7 +489,7 @@ def update(
                resume = None
    except (KeyboardInterrupt, asyncio.CancelledError) as err:
        exit_code = 130
-        exact_resume = getattr(err, "archivebox_resume", None)
+        exact_resume = err.__dict__.get("archivebox_resume")
        resume_cmd = ["archivebox", "update"]
        if migrate_only:
            resume_cmd.append("--migrate-only")
@@ -464,7 +552,7 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500
    1:1 mapping between DB and filesystem.
    """
    from archivebox.core.models import Snapshot
-    from archivebox.config.common import get_config
+    from archivebox.config import CONSTANTS
    from archivebox.crawls.models import Crawl
    from django.utils import timezone

@@ -473,8 +561,7 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500
    crawl_url_sets: dict[str, set[str]] = {}
    dirty_crawl_ids: set[str] = set()

-    runtime_config = get_config()
-    archive_dir = runtime_config.ARCHIVE_DIR
+    archive_dir = CONSTANTS.ARCHIVE_DIR
    if not archive_dir.exists():
        return stats

@@ -639,7 +726,7 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
        "updated_db": 0,
        "queued": 0,
        "sealed": 0,
-        "crawls_queued": 0,
+        "crawls_sealed": 0,
    }
    current_fs_version = Snapshot._fs_current_version()

@@ -647,6 +734,7 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
    if resume:
        queryset = queryset.filter(timestamp__lte=resume)
    total = queryset.count()
+    stats["snapshots"] = total
    print(f"[*] Processing {total} snapshots from database (most recent first)...")

    def update_in_batches(rows, *, label: str, **updates) -> int:
@@ -745,7 +833,10 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
    queue_stale_fs_batch()

    now = timezone.now()
-    stats["crawls_queued"] = (
+    # Crawls with no open child snapshots are already finished. Seal them here
+    # instead of waking the foreground runner; otherwise migration/update can
+    # accidentally re-enter full crawl execution for historical rows.
+    stats["crawls_sealed"] = (
        Crawl.objects.filter(
            status__in=Crawl.RUNNABLE_STATES,
        )
@@ -753,11 +844,12 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
            snapshot_set__status__in=Snapshot.OPEN_STATES,
        )
        .update(
-            retry_at=now,
+            status=Crawl.StatusChoices.SEALED,
+            retry_at=None,
            modified_at=now,
        )
    )
-    stats["updated_db"] += stats["crawls_queued"]
+    stats["updated_db"] += stats["crawls_sealed"]
    return stats


@@ -895,7 +987,7 @@ Phase 2 (Process DB):
  Updated JSON:     {s2.get("updated_json", 0)}
  Updated DB rows:  {s2.get("updated_db", 0)}
  Sealed snapshots: {s2.get("sealed", 0)}
-  Queued crawls:    {s2.get("crawls_queued", 0)}
+  Sealed crawls:    {s2.get("crawls_sealed", 0)}
 """)


@@ -905,28 +997,19 @@ def print_index_stats(stats: dict[str, Any]) -> None:
    print(f"""
 [green]Search Reindex Complete[/green]
  Scanned rows:      {stats["processed"]}
+  Requested jobs:    {stats.get("requested", stats["queued"])}
  Queued index jobs: {stats["queued"]}
+  Already queued:    {stats.get("skipped_queued", 0)}
 """)


@click.command()
@click.option("--resume", type=str, help="Resume from timestamp")
-@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
-@click.option("--url__icontains", help="Filter by URL contains")
-@click.option("--url__istartswith", help="Filter by URL starts with")
-@click.option("--tag", "-t", help="Filter by tag name")
-@click.option("--crawl-id", help="Filter by crawl ID")
-@click.option("--limit", "-n", type=int, help="Limit number of snapshots to update")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--search", help="Search mode to use for positional query")
-@click.option("--before", type=float, help="Only snapshots before timestamp")
-@click.option("--after", type=float, help="Only snapshots after timestamp")
-@click.option("--filter-type", type=click.Choice(["exact", "substring", "regex", "domain", "tag", "timestamp"]), default="exact")
@click.option("--batch-size", type=int, default=500, help="Commit every N records")
@click.option("--continuous", is_flag=True, help="Run continuously as background worker")
@click.option("--index-only", is_flag=True, help="Backfill available search indexes from existing archived content")
@click.option("--migrate-only", is_flag=True, help="Only migrate filesystem and update database/index state")
-@click.argument("filter_patterns", nargs=-1)
+@snapshot_filter_options(default_filter_type="exact")
@docstring(update.__doc__)
 def main(**kwargs):
    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
@@ -934,6 +1017,8 @@ def main(**kwargs):
    try:
        with foreground_shutdown_signals(), foreground_parent_watchdog():
            update(**kwargs)
+    except ValueError as err:
+        raise click.BadParameter(str(err), param_hint="--status") from err
    except KeyboardInterrupt:
        raise SystemExit(130) from None

@@ -32,7 +32,6 @@ def _format_binary_abspath(
        (lib_dir, "LIB_DIR/"),
        (Path(os.environ.get("LIB_DIR", "")), "LIB_DIR/") if os.environ.get("LIB_DIR") else (Path(), ""),
        (personas_dir, "PERSONAS_DIR/"),
-        (Path(os.environ.get("PERSONAS_DIR", "")), "PERSONAS_DIR/") if os.environ.get("PERSONAS_DIR") else (Path(), ""),
        (home, "~/"),
    )

@@ -155,7 +154,7 @@ def version(
    )
    prnt()

-    if not (os.access(config.ARCHIVE_DIR, os.R_OK) and os.access(CONSTANTS.CONFIG_FILE, os.R_OK)):
+    if not (os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and os.access(CONSTANTS.CONFIG_FILE, os.R_OK)):
        PANEL_TEXT = "\n".join(
            (
                "",
@@ -219,7 +218,7 @@ def version(
                            installed.abspath,
                            pwd=Path.cwd(),
                            lib_dir=config.LIB_DIR,
-                            personas_dir=config.PERSONAS_DIR,
+                            personas_dir=CONSTANTS.PERSONAS_DIR,
                            home=Path.home(),
                        )
                        if compact_paths
@@ -275,7 +274,7 @@ def version(
            prnt(f"  [red]Error getting code locations: {e}[/red]")

        prnt()
-        if os.access(config.ARCHIVE_DIR, os.R_OK) or os.access(CONSTANTS.CONFIG_FILE, os.R_OK):
+        if os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) or os.access(CONSTANTS.CONFIG_FILE, os.R_OK):
            prnt("[bright_yellow][i] Data locations:[/bright_yellow]")
            try:
                for name, path in get_data_locations().items():
@@ -3,6 +3,7 @@ __package__ = "archivebox.config"
 import io
 import json
 import os
+from collections.abc import Mapping
 from typing import Any

 from archivebox.config.constants import CONSTANTS
@@ -54,7 +55,7 @@ def _coerce_to_str_dict(config: Any) -> dict[str, str]:
    """
    if not config:
        return {}
-    if not hasattr(config, "items"):
+    if not isinstance(config, Mapping):
        return {}
    flat: dict[str, str] = {}
    for key, value in config.items():
@@ -8,11 +8,13 @@ import re
 import secrets
 import sys
 import shutil
+import inspect
 from functools import lru_cache
 from collections.abc import Mapping
 from datetime import timedelta
 from typing import Any, ClassVar, cast
 from pathlib import Path
+from urllib.parse import quote

 from rich.console import Console
 from pydantic import BaseModel, Field, PrivateAttr, create_model, field_validator, model_validator
@@ -30,6 +32,7 @@ from .permissions import IN_DOCKER
 ConfigOverrides = Mapping[str, object]
 ConfigPayload = dict[str, object]
 PluginSchemaDocuments = dict[str, dict[str, Any]]
+LIVE_CONFIG_BASE_URL = "/admin/environment/config/"

 ###################### Config ##########################

@@ -64,6 +67,16 @@ def permissions_from_legacy_public_flags(raw_config: Mapping[str, object]) -> st
    return None


+def resolve_delete_after_config_value(*configs: Mapping[str, Any] | None) -> str:
+    for config in configs:
+        if config is None:
+            continue
+        value = config.get("DELETE_AFTER")
+        if value:
+            return str(value)
+    return "0"
+
+
 _SENSITIVE_CONFIG_KEY_NEEDLES = ("TOKEN", "SECRET", "API_KEY", "APIKEY", "PASSWORD")
 SENSITIVE_CONFIG_VALUE_REDACTED = "********"
 _SCOPE_CRAWL_FROZEN = "crawl_frozen"
@@ -118,8 +131,15 @@ def redact_sensitive_config(config: Mapping[str, Any] | None) -> dict[str, Any]:
    return redacted


-def normalize_runtime_config(config: BaseConfigSet | Mapping[str, Any] | str | None) -> dict[str, Any]:
-    """Return a JSON-safe config dict suitable for storage or event payloads."""
+def normalize_runtime_config(
+    config: BaseConfigSet | Mapping[str, Any] | str | None,
+    *,
+    only_crawl_execution: bool = False,
+    exclude_runtime_derived: bool = False,
+    exclude_crawl_execution: bool = False,
+    json_safe: bool = True,
+) -> dict[str, Any]:
+    """Return config filtered for runtime/frozen usage, optionally JSON-safe."""
    if config is None:
        return {}
    if isinstance(config, BaseConfigSet):
@@ -128,21 +148,51 @@ def normalize_runtime_config(config: BaseConfigSet | Mapping[str, Any] | str | N
        config = json.loads(config)
    else:
        config = dict(config)
-    return {key: value for key, value in json.loads(json.dumps(config, default=str)).items() if value is not None}
+
+    runtime_derived_keys = ArchiveBoxConfig.runtime_derived_config_keys() if exclude_runtime_derived else frozenset()
+    filtered = {
+        key: value
+        for key, value in config.items()
+        if (
+            value is not None
+            and (not only_crawl_execution or ArchiveBoxConfig.scope_for_key(str(key)) == _SCOPE_CRAWL_EXECUTION)
+            and (not exclude_runtime_derived or str(key) not in runtime_derived_keys)
+            and (not exclude_crawl_execution or ArchiveBoxConfig.scope_for_key(str(key)) != _SCOPE_CRAWL_EXECUTION)
+        )
+    }
+    if not json_safe:
+        return filtered
+    return {key: value for key, value in json.loads(json.dumps(filtered, default=str)).items() if value is not None}


 def build_crawl_config_snapshot(
    *,
-    user: Any = None,
    persona: Any = None,
    overrides: Mapping[str, Any] | None = None,
    base_config: ArchiveBoxBaseConfig | Mapping[str, object] | None = None,
 ) -> dict[str, Any]:
-    """Build the frozen runtime config stored on Crawl.config at creation time."""
-    effective = get_config(user=user, persona=persona, base_config=base_config)
-    frozen = effective.for_crawl_frozen()
+    """Build the frozen crawl config stored on Crawl.config at creation time."""
+    explicit_overrides = set(overrides or {})
+    plugin_owned_keys = set(_plugin_config_properties(PLUGIN_CONFIG_SCHEMAS)) - set(ArchiveBoxBaseConfig.model_fields)
+    effective = get_config(persona=persona, base_config=base_config)
+    frozen = effective.for_crawl_frozen(persona=persona)
+    if persona is not None:
+        persona_config = persona.get_derived_config()
+        for key in plugin_owned_keys - explicit_overrides:
+            if key in persona_config:
+                frozen.pop(key, None)
    if overrides:
-        frozen = get_config(base_config=frozen, overrides=overrides, include_machine=False).for_crawl_frozen()
+        resolved = get_config(base_config=frozen, overrides=overrides, include_machine=False)
+        resolved_payload = normalize_runtime_config(resolved)
+        frozen = resolved.for_crawl_frozen(persona=persona)
+        for key in plugin_owned_keys & explicit_overrides:
+            if ArchiveBoxConfig.scope_for_key(key) == _SCOPE_CRAWL_FROZEN and key in resolved_payload:
+                frozen[key] = resolved_payload[key]
+        if persona is not None:
+            persona_config = persona.get_derived_config()
+            for key in plugin_owned_keys - explicit_overrides:
+                if key in persona_config:
+                    frozen.pop(key, None)
    return frozen


@@ -187,11 +237,6 @@ class StorageConfig(BaseConfigSet):
    toml_section_header: str = "STORAGE_CONFIG"
    _scope: str = PrivateAttr(default=_SCOPE_SERVER)

-    # ARCHIVE_DIR / USERS_DIR are resolved dynamically via get_config().
-    ARCHIVE_DIR: Path = Field(default=CONSTANTS.ARCHIVE_DIR)
-    USERS_DIR: Path = Field(default=CONSTANTS.USERS_DIR)
-    PERSONAS_DIR: Path = Field(default=CONSTANTS.PERSONAS_DIR)
-
    # TMP_DIR must be a local, fast, readable/writable dir by archivebox user,
    # must be a short path due to unix path length restrictions for socket files (<100 chars)
    # must be a local SSD/tmpfs for speed and because bind mounts/network mounts/FUSE dont support unix sockets
@@ -206,10 +251,6 @@ class StorageConfig(BaseConfigSet):
    # Runtime lookup must use provider-specific paths under LIB_DIR instead.
    LIB_BIN_DIR: Path = Field(default=CONSTANTS.DEFAULT_LIB_BIN_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})

-    # CUSTOM_TEMPLATES_DIR allows users to override default templates
-    # defaults to DATA_DIR / 'user_templates' but can be configured
-    CUSTOM_TEMPLATES_DIR: Path = Field(default=CONSTANTS.CUSTOM_TEMPLATES_DIR)
-
    OUTPUT_PERMISSIONS: str = Field(default="644")
    ENFORCE_ATOMIC_WRITES: bool = Field(default=True)
    ALLOW_NO_UNIX_SOCKETS: bool = Field(default=False, alias="ARCHIVEBOX_ALLOW_NO_UNIX_SOCKETS")
@@ -244,8 +285,6 @@ class ServerConfig(BaseConfigSet):
    FOOTER_INFO: str = Field(
        default="Content is hosted for personal archiving purposes only.  Contact server owner for any takedown requests.",
    )
-    # CUSTOM_TEMPLATES_DIR: Path          = Field(default=None)  # this is now a constant
-
    PUBLIC_INDEX: bool = Field(default=True)
    PUBLIC_ADD_VIEW: bool = Field(default=False)

@@ -355,7 +394,7 @@ class ArchivingConfig(BaseConfigSet):
    URL_DENYLIST: str = Field(default=r"\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$", alias="URL_BLACKLIST")
    URL_ALLOWLIST: str | None = Field(default=None, alias="URL_WHITELIST")

-    DEFAULT_PERSONA: str = Field(default="Default")
+    DEFAULT_PERSONA: str = Field(default="Default", json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
    PERMISSIONS: str = Field(
        default="public",
        description="Snapshot visibility: public lists and serves content, unlisted serves direct links only, private requires admin login.",
@@ -449,7 +488,7 @@ class SearchBackendConfig(BaseConfigSet):
    toml_section_header: str = "SEARCH_BACKEND_CONFIG"
    _scope: str = PrivateAttr(default=_SCOPE_SERVER)

-    SEARCH_BACKEND_ENGINE: str = Field(default="ripgrep")
+    SEARCH_BACKEND_ENGINE: str = Field(default="ripgrep", json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})


 def _plugin_user_config_value(value: Any) -> str:
@@ -469,9 +508,7 @@ def _discover_plugin_config_schemas() -> PluginSchemaDocuments:

    schemas: PluginSchemaDocuments = {}
    if BASE_CONFIG_PATH.exists():
-        schemas["base"] = {
-            "properties": json.loads(BASE_CONFIG_PATH.read_text()).get("properties", {}),
-        }
+        schemas["base"] = json.loads(BASE_CONFIG_PATH.read_text())
    schemas.update(discover_plugin_configs())
    return schemas

@@ -524,10 +561,6 @@ class ArchiveBoxBaseConfig(
        populate_by_name=True,
    )

-    DATA_DIR: Path = Field(default=CONSTANTS.DATA_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
-    ABX_RUNTIME: str = Field(default="archivebox", json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
-    CRAWL_DIR: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
-    SNAP_DIR: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
    computed_config_keys: ClassVar[tuple[str, ...]] = COMPUTED_CONFIG_KEYS

    @classmethod
@@ -566,68 +599,164 @@ class ArchiveBoxBaseConfig(

    @classmethod
    def _plugin_field_scope(cls, key: str) -> str | None:
+        scope = None
        for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items():
            properties = schema.get("properties") if isinstance(schema, dict) else None
            if not isinstance(properties, dict) or key not in properties:
                continue
            prop_schema = properties.get(key) or {}
            if isinstance(prop_schema, Mapping) and prop_schema.get("x-scope"):
-                return str(prop_schema["x-scope"])
-            if str(plugin_name).startswith("search_backend_"):
-                return _SCOPE_SERVER
-            return _SCOPE_CRAWL_FROZEN
-        return None
+                scope = str(prop_schema["x-scope"])
+            elif scope is None:
+                scope = _SCOPE_CRAWL_FROZEN
+        return scope

    @classmethod
+    @lru_cache(maxsize=None)
    def scope_for_key(cls, key: str) -> str:
+        for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items():
+            properties = schema.get("properties") if isinstance(schema, dict) else None
+            if isinstance(properties, dict) and key == f"{str(plugin_name).upper()}_ENABLED" and key in properties:
+                return _SCOPE_CRAWL_EXECUTION
+        if key.endswith("_BINARY"):
+            return _SCOPE_CRAWL_EXECUTION
        return cls._core_field_scope(key) or cls._plugin_field_scope(key) or _SCOPE_SERVER

+    @classmethod
+    @lru_cache(maxsize=1)
+    def _scope_by_key(cls) -> dict[str, str]:
+        return {key: cls.scope_for_key(key) for key in cls.model_fields}
+
+    @classmethod
+    @lru_cache(maxsize=1)
+    def _crawl_frozen_keys(cls) -> frozenset[str]:
+        return frozenset(key for key, scope in cls._scope_by_key().items() if scope == _SCOPE_CRAWL_FROZEN)
+
+    @classmethod
+    @lru_cache(maxsize=1)
+    def _crawl_runtime_keys(cls) -> frozenset[str]:
+        return frozenset(key for key, scope in cls._scope_by_key().items() if scope in {_SCOPE_CRAWL_FROZEN, _SCOPE_CRAWL_EXECUTION})
+
+    @classmethod
+    @lru_cache(maxsize=1)
+    def runtime_derived_config_keys(cls) -> frozenset[str]:
+        runtime_derived_keys = {
+            "ABX_INSTALL_CACHE",
+            "ACTIVE_PERSONA",
+            "CHROME_DOWNLOADS_DIR",
+            "CHROME_USER_DATA_DIR",
+            "CRAWL_DIR",
+            "DEFAULT_PERSONA",
+            "EXTRA_CONTEXT",
+            "SNAP_DIR",
+        }
+        return frozenset(
+            key for key, scope in cls._scope_by_key().items() if scope == _SCOPE_CRAWL_EXECUTION and key in runtime_derived_keys
+        )
+
    def _scoped_config(self, *, include_execution: bool) -> dict[str, Any]:
-        allowed_scopes = {_SCOPE_CRAWL_FROZEN}
-        if include_execution:
-            allowed_scopes.add(_SCOPE_CRAWL_EXECUTION)
-        return {key: value for key, value in normalize_runtime_config(self).items() if type(self).scope_for_key(key) in allowed_scopes}
+        keys = type(self)._crawl_runtime_keys() if include_execution else type(self)._crawl_frozen_keys()
+        payload = self.model_dump(mode="json")
+        return {key: payload[key] for key in keys if payload.get(key) is not None}

-    def for_crawl_execution(self) -> dict[str, Any]:
-        """Config safe to pass to crawl/snapshot hook execution."""
-        return self._scoped_config(include_execution=True)
+    def for_crawl(self) -> dict[str, Any]:
+        """Config scoped to crawl execution, without runtime object overlays."""
+        config = self._scoped_config(include_execution=True)
+        for key in type(self).runtime_derived_config_keys():
+            config.pop(key, None)
+        return config

-    def for_crawl_frozen(self) -> dict[str, Any]:
+    def for_crawl_frozen(self, *, persona: Any = None) -> dict[str, Any]:
        """Config safe to persist permanently on Crawl.config."""
-        return self._scoped_config(include_execution=False)
+        frozen = self._scoped_config(include_execution=False)
+        if persona is not None:
+            persona_config = dict(persona.config or {})
+            scope_by_key = type(self)._scope_by_key()
+            for key in persona.get_derived_config():
+                if key not in persona_config and scope_by_key.get(key) == _SCOPE_CRAWL_EXECUTION:
+                    frozen.pop(key, None)
+        return frozen
+
+    def for_crawl_runtime(
+        self,
+        *,
+        crawl: Any = None,
+        snapshot: Any = None,
+        persona: Any = None,
+        runtime_overrides: Mapping[str, Any] | None = None,
+        extra_context: Mapping[str, Any] | None = None,
+        crawl_output_dir: Any = None,
+        snapshot_output_dir: Any = None,
+    ) -> dict[str, Any]:
+        """Config payload safe to pass to crawl/snapshot hook execution."""
+        config = self.for_crawl()
+        config["DATA_DIR"] = str(CONSTANTS.DATA_DIR)
+        scope_by_key = type(self)._scope_by_key()
+        model_fields = type(self).model_fields
+        # ArchiveBox owns SEARCH_BACKEND_ENGINE and uses it during model
+        # validation to derive the selected backend's *_ENABLED flag. Hooks
+        # only receive the backend-local flags, never the selector itself.
+        config.pop("SEARCH_BACKEND_ENGINE", None)
+        if persona is not None:
+            for key, value in persona.get_derived_config().items():
+                if scope_by_key.get(key) == _SCOPE_CRAWL_EXECUTION:
+                    config[key] = value
+
+        if crawl is not None:
+            for key, value in dict(crawl.config or {}).items():
+                if key in model_fields and scope_by_key.get(key) != _SCOPE_CRAWL_EXECUTION:
+                    config[key] = value
+            config["CRAWL_DIR"] = str(crawl_output_dir if crawl_output_dir is not None else crawl.output_dir)
+
+        if snapshot is not None:
+            for key, value in dict(snapshot.config or {}).items():
+                if key in model_fields and scope_by_key.get(key) != _SCOPE_CRAWL_EXECUTION:
+                    config[key] = value
+            config["SNAP_DIR"] = str(snapshot_output_dir if snapshot_output_dir is not None else snapshot.output_dir)
+
+        if runtime_overrides:
+            config.update(normalize_runtime_config(runtime_overrides, json_safe=False))
+
+        if extra_context:
+            context: dict[str, Any] = {}
+            if config.get("EXTRA_CONTEXT"):
+                parsed_extra_context = json.loads(str(config["EXTRA_CONTEXT"]))
+                if not isinstance(parsed_extra_context, dict):
+                    raise TypeError("EXTRA_CONTEXT must decode to an object")
+                context = parsed_extra_context
+            context.update(dict(extra_context))
+            config["EXTRA_CONTEXT"] = json.dumps(context, separators=(",", ":"), sort_keys=True)
+
+        return config

    @model_validator(mode="after")
    def resolve_runtime_paths(self):
-        self.DATA_DIR = self.DATA_DIR.expanduser().resolve()
-
-        archive_dir = self.ARCHIVE_DIR.expanduser()
-        if archive_dir == (CONSTANTS.DATA_DIR / CONSTANTS.ARCHIVE_DIR_NAME) and self.DATA_DIR != CONSTANTS.DATA_DIR:
-            archive_dir = self.DATA_DIR / CONSTANTS.ARCHIVE_DIR_NAME
-        if not archive_dir.is_absolute():
-            archive_dir = self.DATA_DIR / archive_dir
-        self.ARCHIVE_DIR = archive_dir.resolve()
-
-        users_dir = self.USERS_DIR.expanduser()
-        if users_dir == (CONSTANTS.ARCHIVE_DIR / CONSTANTS.USERS_DIR_NAME):
-            users_dir = self.ARCHIVE_DIR / CONSTANTS.USERS_DIR_NAME
-        if not users_dir.is_absolute():
-            users_dir = self.ARCHIVE_DIR / users_dir
-        self.USERS_DIR = users_dir.resolve()
-
        lib_dir = self.LIB_DIR.expanduser()
        if not lib_dir.is_absolute():
-            lib_dir = self.DATA_DIR / lib_dir
+            lib_dir = CONSTANTS.DATA_DIR / lib_dir
        self.LIB_DIR = lib_dir.resolve()

        lib_bin_dir = self.LIB_BIN_DIR.expanduser()
        if lib_bin_dir == CONSTANTS.DEFAULT_LIB_BIN_DIR and self.LIB_DIR != CONSTANTS.DEFAULT_LIB_DIR:
            lib_bin_dir = self.LIB_DIR / "bin"
        elif not lib_bin_dir.is_absolute():
-            lib_bin_dir = self.DATA_DIR / lib_bin_dir
+            lib_bin_dir = CONSTANTS.DATA_DIR / lib_bin_dir
        self.LIB_BIN_DIR = lib_bin_dir.resolve()

        return self

+    @model_validator(mode="after")
+    def derive_plugin_enabled_config(self):
+        plugin_names = _normalize_plugins_config_value(self.PLUGINS)
+        selected_plugins = _plugins_with_required_plugins(plugin_names) if plugin_names else set()
+        search_backend = self.SEARCH_BACKEND_ENGINE.strip().lower()
+        if search_backend:
+            selected_plugins.add(f"search_backend_{search_backend}")
+        for plugin_name, enabled_key in _plugin_enabled_config_keys().items():
+            if plugin_names or plugin_name in selected_plugins:
+                setattr(self, enabled_key, plugin_name in selected_plugins)
+        return self
+

 def _build_archivebox_config_model(plugin_schemas: PluginSchemaDocuments) -> type[ArchiveBoxBaseConfig]:
    core_fields = set(ArchiveBoxBaseConfig.model_fields)
@@ -649,15 +778,192 @@ PLUGIN_CONFIG_SCHEMAS = _discover_plugin_config_schemas()
 ArchiveBoxConfig = _build_archivebox_config_model(PLUGIN_CONFIG_SCHEMAS)


+def _normalize_plugins_config_value(value: Any) -> set[str]:
+    if value is None:
+        return set()
+    if isinstance(value, str):
+        raw = value.strip()
+        if not raw:
+            return set()
+        if raw.startswith("["):
+            try:
+                parsed = json.loads(raw)
+            except json.JSONDecodeError:
+                parsed = None
+            if isinstance(parsed, list):
+                return {str(plugin).strip().lower() for plugin in parsed if str(plugin).strip()}
+        return {plugin.strip().lower() for plugin in raw.split(",") if plugin.strip()}
+    if isinstance(value, (list, tuple, set)):
+        return {str(plugin).strip().lower() for plugin in value if str(plugin).strip()}
+    normalized = str(value).strip().lower()
+    return {normalized} if normalized else set()
+
+
+@lru_cache(maxsize=1)
+def _plugin_enabled_config_keys() -> dict[str, str]:
+    enabled_keys: dict[str, str] = {}
+    for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items():
+        properties = schema.get("properties") if isinstance(schema, dict) else None
+        if not isinstance(properties, dict):
+            continue
+        enabled_key = f"{str(plugin_name).upper()}_ENABLED"
+        if enabled_key in properties and ArchiveBoxConfig.scope_for_key(enabled_key) == _SCOPE_CRAWL_EXECUTION:
+            enabled_keys[str(plugin_name).lower()] = enabled_key
+    return enabled_keys
+
+
+def _plugins_with_required_plugins(plugin_names: set[str]) -> set[str]:
+    selected = set(plugin_names)
+    pending = list(selected)
+    while pending:
+        plugin_name = pending.pop()
+        schema = PLUGIN_CONFIG_SCHEMAS.get(plugin_name, {})
+        required_plugins = schema.get("required_plugins") if isinstance(schema, dict) else None
+        if not isinstance(required_plugins, list):
+            continue
+        for required_plugin in required_plugins:
+            required_plugin_name = str(required_plugin).strip().lower()
+            if required_plugin_name and required_plugin_name not in selected:
+                selected.add(required_plugin_name)
+                pending.append(required_plugin_name)
+    return selected
+
+
+def get_live_config_url(key: str) -> str:
+    return f"{LIVE_CONFIG_BASE_URL}{quote(key)}/"
+
+
+@lru_cache(maxsize=1)
+def config_field_metadata() -> dict[str, dict[str, Any]]:
+    """Return one centralized metadata map for core and plugin config fields."""
+    metadata: dict[str, dict[str, Any]] = {}
+    for key, field in ArchiveBoxConfig.model_fields.items():
+        if ArchiveBoxConfig.scope_for_key(key) == _SCOPE_CRAWL_EXECUTION or key in ArchiveBoxConfig.computed_config_keys:
+            continue
+        default = field.default
+        try:
+            json.dumps(default)
+        except TypeError:
+            default = str(default)
+        metadata[key] = {
+            "plugin": "archivebox",
+            "section": find_config_section(key),
+            "type": config_field_type(key),
+            "default": default,
+            "description": field.description or "",
+            "scope": ArchiveBoxConfig.scope_for_key(key),
+            "sensitive": is_sensitive_config_key(key),
+        }
+    for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items():
+        properties = schema.get("properties") if isinstance(schema, dict) else None
+        if not isinstance(properties, dict):
+            continue
+        for key, prop in properties.items():
+            if not isinstance(prop, Mapping):
+                continue
+            if ArchiveBoxConfig.scope_for_key(key) == _SCOPE_CRAWL_EXECUTION:
+                continue
+            metadata[key] = {
+                **metadata.get(key, {}),
+                "plugin": plugin_name,
+                "section": "PLUGINS",
+                "type": prop.get("type", metadata.get(key, {}).get("type", "string")),
+                "default": prop.get("default", metadata.get(key, {}).get("default", "")),
+                "description": prop.get("description", metadata.get(key, {}).get("description", "")),
+                "scope": ArchiveBoxConfig.scope_for_key(key),
+                "sensitive": bool(prop.get("x-sensitive")) or is_sensitive_config_key(key),
+                "schema": dict(prop),
+            }
+    return metadata
+
+
+def find_config_section(key: str) -> str:
+    from archivebox.config import CONSTANTS_CONFIG
+
+    if key in CONSTANTS_CONFIG:
+        return "CONSTANT"
+    for section_id, section in get_all_configs().items():
+        if key in type(section).model_fields:
+            return section_id
+    if key in _plugin_config_properties(PLUGIN_CONFIG_SCHEMAS):
+        return "PLUGINS"
+    return "DYNAMIC"
+
+
+def find_config_default(key: str) -> str:
+    from archivebox.config import CONSTANTS_CONFIG
+
+    if key in CONSTANTS_CONFIG:
+        return str(CONSTANTS_CONFIG[key])
+
+    field = ArchiveBoxConfig.model_fields.get(key)
+    if field is None:
+        return ""
+    default_val = field.default
+    if callable(default_val):
+        default_val = inspect.getsource(default_val).split("lambda", 1)[-1].split(":", 1)[-1].replace("\n", " ").strip()
+        if default_val.count(")") > default_val.count("("):
+            default_val = default_val[:-1]
+    else:
+        default_val = str(default_val)
+    return default_val
+
+
+def config_field_type(key: str) -> str:
+    field = ArchiveBoxConfig.model_fields.get(key)
+    if field is None:
+        return "str"
+    annotation = field.annotation
+    try:
+        return annotation.__name__
+    except AttributeError:
+        return str(annotation)
+
+
+def find_config_type(key: str) -> str:
+    return config_field_type(key)
+
+
+def find_config_source(key: str, merged_config: Mapping[str, Any]) -> str:
+    """Determine where a config value comes from."""
+    from archivebox.machine.models import Machine
+
+    machine = Machine.current()
+    if machine.config and key in machine.config:
+        return "Machine"
+
+    if key in os.environ:
+        return "Environment"
+
+    file_config = BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE)
+    if key in file_config:
+        return "File"
+
+    if key in _plugin_config_properties(PLUGIN_CONFIG_SCHEMAS):
+        return "Plugin Default"
+
+    return "Default"
+
+
+def get_request_config(request: Any, *, resolve_plugins: bool = False) -> ArchiveBoxBaseConfig:
+    """Return the per-request ArchiveBox config, upgrading to plugin resolution if needed."""
+    request_state = request.__dict__
+    request_config = request_state.get("archivebox_config")
+    request_config_resolves_plugins = bool(request_state.get("_archivebox_config_resolves_plugins", False))
+    if request_config is None or (resolve_plugins and not request_config_resolves_plugins):
+        request_config = get_config(resolve_plugins=resolve_plugins)
+        request.archivebox_config = request_config
+        request._archivebox_config_resolves_plugins = resolve_plugins
+    return request_config
+
+
 def get_config(
    defaults: ConfigOverrides | None = None,
    overrides: ConfigOverrides | None = None,
    base_config: ArchiveBoxBaseConfig | Mapping[str, object] | None = None,
    persona: Any = None,
-    user: Any = None,
    crawl: Any = None,
    snapshot: Any = None,
-    archiveresult: Any = None,
    machine: Any = None,
    include_machine: bool = True,
    resolve_plugins: bool = True,
@@ -666,28 +972,20 @@ def get_config(
    """
    Get merged config from all sources.

-    Priority (highest to lowest):
-    1. Explicit overrides
-    2. Per-ArchiveResult config
-    3. Per-snapshot config and output path
-    4. Frozen per-crawl config and output path
-    5. Per-user config (only when resolving outside a crawl)
-    6. Per-persona derived config (only when resolving outside a crawl)
-    7. Current machine derived config (only when resolving outside a crawl)
-    8. Environment variables (only when resolving outside a crawl)
-    9. Config file (ArchiveBox.conf, only when resolving outside a crawl)
-    10. Plugin schema defaults
-    11. Core config defaults
-    """
-    if snapshot is None and archiveresult is not None:
-        snapshot = archiveresult.snapshot
+    Defaults are hydrated by pydantic from core/plugin defaults,
+    ArchiveBox.conf, and environment variables. Persisted Machine/Persona
+    values then apply for live crawl-execution scope, while Crawl/Snapshot
+    rows apply their frozen crawl-scope values. Explicit overrides win last.

+    Crawl-execution config is not persisted on Crawl.config. It is rederived
+    from current Machine/Persona state and hydrated process defaults each time.
+    """
    if crawl is None and snapshot is not None:
        crawl = snapshot.crawl

    crawl_config_base = crawl is not None and base_config is None

-    if include_machine and machine is None and not crawl_config_base:
+    if include_machine and machine is None:
        try:
            from django.apps import apps

@@ -698,58 +996,68 @@ def get_config(
        except Exception:
            machine = None

-    if persona is None and crawl is not None and not crawl_config_base:
+    if persona is None and crawl is not None:
        persona = crawl.resolve_persona()

    config_data: ConfigPayload = dict(defaults or {})
    base_config_payload: ConfigPayload = {}
    if crawl_config_base:
-        config_data.update(dict(crawl.config or {}))
+        config_data.update(
+            normalize_runtime_config(ArchiveBoxConfig().model_dump(mode="json"), exclude_runtime_derived=True, json_safe=False),
+        )
+        config_data.update(normalize_runtime_config(dict(crawl.config or {}), exclude_crawl_execution=True, json_safe=False))
    elif base_config is not None:
        if isinstance(base_config, ArchiveBoxBaseConfig):
            base_config_payload.update(base_config.model_dump(mode="json"))
        else:
            base_config_payload.update(dict(base_config))
-        config_data.update(base_config_payload)
+        config_data.update(normalize_runtime_config(base_config_payload, exclude_runtime_derived=True, json_safe=False))
    else:
-        config_data.update(ArchiveBoxConfig().model_dump(mode="json"))
+        config_data.update(
+            normalize_runtime_config(ArchiveBoxConfig().model_dump(mode="json"), exclude_runtime_derived=True, json_safe=False),
+        )
        legacy_permissions = permissions_from_legacy_public_flags({**BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE), **os.environ})
        if legacy_permissions:
            config_data["PERMISSIONS"] = legacy_permissions

    scope_overrides: ConfigPayload = {}

-    if not crawl_config_base:
-        if include_machine and machine is not None and machine.config:
-            from archivebox.machine.models import _sanitize_machine_config
+    if include_machine and machine is not None and machine.config:
+        from archivebox.machine.models import _sanitize_machine_config

-            scope_overrides.update(_sanitize_machine_config(machine.config, lib_dir=config_data.get("LIB_DIR")))
+        scope_overrides.update(
+            normalize_runtime_config(
+                _sanitize_machine_config(machine.config, lib_dir=config_data.get("LIB_DIR")),
+                only_crawl_execution=crawl_config_base,
+                exclude_runtime_derived=True,
+                json_safe=False,
+            ),
+        )

-        if persona is not None:
-            scope_overrides.update(persona.get_derived_config())
-
-        user_config = getattr(user, "config", None)
-        if user_config:
-            scope_overrides.update(user_config)
+    if persona is not None:
+        persona_config = normalize_runtime_config(
+            persona.get_derived_config(),
+            exclude_runtime_derived=not crawl_config_base,
+            json_safe=False,
+        )
+        if crawl_config_base:
+            scope_by_key = ArchiveBoxConfig._scope_by_key()
+            crawl_keys = set(dict(crawl.config or {}))
+            persona_config = {
+                key: value
+                for key, value in persona_config.items()
+                if scope_by_key.get(key) == _SCOPE_CRAWL_EXECUTION or key not in crawl_keys
+            }
+        scope_overrides.update(persona_config)

    if crawl is not None and crawl.config and not crawl_config_base:
-        scope_overrides.update(crawl.config)
-
-    if crawl is not None:
-        if not overrides or "CRAWL_DIR" not in overrides:
-            scope_overrides["CRAWL_DIR"] = crawl.output_dir
+        scope_overrides.update(normalize_runtime_config(crawl.config, exclude_crawl_execution=True, json_safe=False))

    if snapshot is not None and snapshot.config:
-        scope_overrides.update(snapshot.config)
-
-    if snapshot is not None:
-        scope_overrides["SNAP_DIR"] = snapshot.output_dir
-
-    if archiveresult is not None and archiveresult.config:
-        scope_overrides.update(archiveresult.config)
+        scope_overrides.update(normalize_runtime_config(snapshot.config, exclude_crawl_execution=True, json_safe=False))

    if overrides:
-        scope_overrides.update(overrides)
+        scope_overrides.update(normalize_runtime_config(overrides, exclude_crawl_execution=True, json_safe=False))

    legacy_scope_permissions = permissions_from_legacy_public_flags(scope_overrides)
    if legacy_scope_permissions:
@@ -759,28 +1067,46 @@ def get_config(
    config_data.update(archivebox_scope_overrides)

    if resolve_plugins:
-        plugin_schemas = {
-            plugin_name: schema.get("properties", {}) for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items() if isinstance(schema, dict)
-        }
+        plugin_schemas = {plugin_name: schema for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items() if isinstance(schema, dict)}
        plugin_global_config = {key: str(value) if isinstance(value, Path) else value for key, value in config_data.items()}
-        plugin_user_config = _plugin_user_config(scope_overrides)
+        plugin_user_config = _plugin_user_config(
+            {
+                **normalize_runtime_config(config_data, only_crawl_execution=True, json_safe=False),
+                **scope_overrides,
+            },
+        )
        if not crawl_config_base:
-            plugin_user_config = {**BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE), **plugin_user_config}
+            plugin_user_config = {
+                **normalize_runtime_config(
+                    BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE),
+                    exclude_runtime_derived=True,
+                    json_safe=False,
+                ),
+                **plugin_user_config,
+            }
        plugin_sections = resolve_plugin_configs(
            plugin_schemas,
            global_config=plugin_global_config,
            user_config=plugin_user_config,
+            environ={},
        )
        for plugin_config in plugin_sections.values():
-            config_data.update(plugin_config)
+            for key, value in plugin_config.items():
+                if key in ArchiveBoxBaseConfig.model_fields and key not in archivebox_scope_overrides and key not in base_config_payload:
+                    continue
+                config_data[key] = value
        if base_config_payload:
-            config_data.update({key: value for key, value in base_config_payload.items() if key in _archivebox_config_input_names()})
+            config_data.update(
+                {
+                    key: value
+                    for key, value in normalize_runtime_config(base_config_payload, exclude_runtime_derived=True, json_safe=False).items()
+                    if key in _archivebox_config_input_names()
+                },
+            )
        if crawl_config_base:
-            config_data.update(dict(crawl.config or {}))
+            config_data.update(normalize_runtime_config(dict(crawl.config or {}), exclude_crawl_execution=True, json_safe=False))
        config_data.update(archivebox_scope_overrides)

-    config_data["ABX_RUNTIME"] = "archivebox"
-
    # Decode JSON-encoded complex values (dict/list fields) that came from
    # string-only sources before validation. ``IniConfigSettingsSource`` does
    # this for the ArchiveBox.conf path, but Machine.config (mirrored from the
@@ -807,7 +1133,7 @@ def get_config(
    config = ArchiveBoxConfig.model_validate(config_data)
    if redact_sensitive:
        for key in type(config).model_fields:
-            value = getattr(config, key, None)
+            value = config[key]
            if is_sensitive_config_key(key) and value not in (None, ""):
                setattr(config, key, SENSITIVE_CONFIG_VALUE_REDACTED)
    os.environ["LIB_DIR"] = str(config.LIB_DIR)
@@ -19,7 +19,7 @@ if TYPE_CHECKING:
 #############################################################################################

 PACKAGE_DIR: Path = Path(__file__).resolve().parent.parent  # archivebox source code dir
-DATA_DIR: Path = Path(os.environ.get("DATA_DIR", os.getcwd())).resolve()  # archivebox user data dir
+DATA_DIR: Path = Path(os.getcwd()).resolve()  # archivebox user data dir


 def _env_path(key: str, default: Path) -> Path:
@@ -29,8 +29,8 @@ def _env_path(key: str, default: Path) -> Path:
    return path.resolve()


-ARCHIVE_DIR: Path = _env_path("ARCHIVE_DIR", DATA_DIR / "archive")  # archivebox snapshot data dir
-USERS_DIR: Path = _env_path("USERS_DIR", ARCHIVE_DIR / "users")  # archivebox user-scoped crawl/snapshot data dir
+ARCHIVE_DIR: Path = DATA_DIR / "archive"  # archivebox snapshot data dir
+USERS_DIR: Path = ARCHIVE_DIR / "users"  # archivebox user-scoped crawl/snapshot data dir

 IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes")

@@ -290,20 +290,20 @@ def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
                "is_mount": os.path.ismount(DATABASE_FILE.resolve()),
            },
            "ARCHIVE_DIR": {
-                "path": config.ARCHIVE_DIR.resolve(),
+                "path": CONSTANTS.ARCHIVE_DIR.resolve(),
                "enabled": True,
-                "is_valid": os.path.isdir(config.ARCHIVE_DIR)
-                and os.access(config.ARCHIVE_DIR, os.R_OK)
-                and os.access(config.ARCHIVE_DIR, os.W_OK),
-                "is_mount": os.path.ismount(config.ARCHIVE_DIR.resolve()),
+                "is_valid": os.path.isdir(CONSTANTS.ARCHIVE_DIR)
+                and os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK)
+                and os.access(CONSTANTS.ARCHIVE_DIR, os.W_OK),
+                "is_mount": os.path.ismount(CONSTANTS.ARCHIVE_DIR.resolve()),
            },
            "USERS_DIR": {
-                "path": config.USERS_DIR.resolve(),
-                "enabled": os.path.isdir(config.USERS_DIR),
-                "is_valid": os.path.isdir(config.USERS_DIR)
-                and os.access(config.USERS_DIR, os.R_OK)
-                and os.access(config.USERS_DIR, os.W_OK),
-                "is_mount": os.path.ismount(config.USERS_DIR.resolve()),
+                "path": CONSTANTS.USERS_DIR.resolve(),
+                "enabled": os.path.isdir(CONSTANTS.USERS_DIR),
+                "is_valid": os.path.isdir(CONSTANTS.USERS_DIR)
+                and os.access(CONSTANTS.USERS_DIR, os.R_OK)
+                and os.access(CONSTANTS.USERS_DIR, os.W_OK),
+                "is_mount": os.path.ismount(CONSTANTS.USERS_DIR.resolve()),
            },
            "SOURCES_DIR": {
                "path": CONSTANTS.SOURCES_DIR.resolve(),
@@ -366,9 +366,9 @@ def get_code_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
                "is_valid": os.access(CONSTANTS.STATIC_DIR, os.R_OK) and os.access(CONSTANTS.STATIC_DIR, os.X_OK),  # read + list
            },
            "CUSTOM_TEMPLATES_DIR": {
-                "path": config.CUSTOM_TEMPLATES_DIR.resolve(),
-                "enabled": os.path.isdir(config.CUSTOM_TEMPLATES_DIR),
-                "is_valid": os.path.isdir(config.CUSTOM_TEMPLATES_DIR) and os.access(config.CUSTOM_TEMPLATES_DIR, os.R_OK),  # read
+                "path": CONSTANTS.CUSTOM_TEMPLATES_DIR.resolve(),
+                "enabled": os.path.isdir(CONSTANTS.CUSTOM_TEMPLATES_DIR),
+                "is_valid": os.path.isdir(CONSTANTS.CUSTOM_TEMPLATES_DIR) and os.access(CONSTANTS.CUSTOM_TEMPLATES_DIR, os.R_OK),  # read
            },
            "USER_PLUGINS_DIR": {
                "path": CONSTANTS.USER_PLUGINS_DIR.resolve(),
@@ -15,8 +15,6 @@ from contextlib import contextmanager
 #############################################################################################

 DATA_DIR = Path(os.getcwd())
-if os.environ.get("DATA_DIR") and Path(os.environ["DATA_DIR"]).resolve() != DATA_DIR.resolve():
-    raise SystemExit(f"[X] DATA_DIR={os.environ['DATA_DIR']} must equal cwd={DATA_DIR}; cd into the data dir before running archivebox")

 try:
    DATA_DIR_STAT = DATA_DIR.stat()
@@ -6,6 +6,7 @@ import importlib.metadata
 from pathlib import Path
 from functools import cache
 from datetime import datetime
+import re

 #############################################################################################

@@ -41,11 +42,57 @@ def detect_installed_version(PACKAGE_DIR: Path = PACKAGE_DIR):

@cache
 def get_COMMIT_HASH() -> str | None:
+    for env_var in ("ARCHIVEBOX_COMMIT_HASH", "COMMIT_HASH"):
+        env_commit_hash = os.environ.get(env_var, "").strip()
+        if re.fullmatch(r"[0-9a-fA-F]{40}", env_commit_hash):
+            return env_commit_hash
+
+    if IN_DOCKER:
+        try:
+            version_txt = Path("/VERSION.txt").read_text()
+            docker_commit_hashes = re.findall(r"COMMIT_HASH=([0-9a-fA-F]{40})", version_txt)
+            if docker_commit_hashes:
+                return docker_commit_hashes[-1]
+        except Exception:
+            pass
+
+    def _read_git_file(git_dir: Path, ref: str) -> str | None:
+        try:
+            return git_dir.joinpath(ref).read_text().strip()
+        except Exception:
+            pass
+
+        try:
+            packed_refs = git_dir.joinpath("packed-refs").read_text().splitlines()
+        except Exception:
+            return None
+
+        for line in packed_refs:
+            if line.startswith("#") or line.startswith("^") or not line.strip():
+                continue
+            commit_hash, packed_ref = line.split(" ", 1)
+            if packed_ref == ref:
+                return commit_hash.strip()
+
+        return None
+
    try:
        git_dir = PACKAGE_DIR.parent / ".git"
-        ref = (git_dir / "HEAD").read_text().strip().split(" ")[-1]
-        commit_hash = git_dir.joinpath(ref).read_text().strip()
-        return commit_hash
+        if git_dir.is_file():
+            gitdir_line = git_dir.read_text().strip()
+            gitdir_path = gitdir_line.removeprefix("gitdir:").strip()
+            git_dir = Path(gitdir_path)
+            if not git_dir.is_absolute():
+                git_dir = PACKAGE_DIR.parent / git_dir
+
+        head = (git_dir / "HEAD").read_text().strip()
+        if re.fullmatch(r"[0-9a-fA-F]{40}", head):
+            return head
+
+        ref = head.removeprefix("ref:").strip()
+        commit_hash = _read_git_file(git_dir, ref)
+        if commit_hash:
+            return commit_hash
    except Exception:
        pass

@@ -24,7 +24,7 @@ INSTALLED_BINARIES_BASE_URL = "/admin/machine/binary/"


 def is_superuser(request: HttpRequest) -> bool:
-    return bool(getattr(request.user, "is_superuser", False))
+    return bool(request.user.is_superuser)


 def format_parsed_datetime(value: object) -> str:
@@ -44,7 +44,7 @@ def get_installed_binary_change_url(name: str, binary: Binary | None) -> str | N
    if binary is None or not binary.id:
        return None

-    base_url = getattr(binary, "admin_change_url", None) or f"{INSTALLED_BINARIES_BASE_URL}{binary.id}/change/"
+    base_url = binary.admin_change_url
    changelist_filters = urlencode({"q": name})
    return f"{base_url}?{urlencode({'_changelist_filters': changelist_filters})}"

@@ -158,7 +158,7 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:

    for name in all_binary_names:
        binary = db_binaries.get(name)
-        binary_is_valid = bool(binary and getattr(binary, "is_valid", getattr(binary, "abspath", None)))
+        binary_is_valid = bool(binary and binary.is_valid)

        rows["Binary Name"].append(ItemLink(name, key=name))

@@ -185,9 +185,9 @@ def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
        "youtube-dl": "yt-dlp",
    }.get(key, key)
    db_binary = get_db_binaries_by_name().get(key)
-    binary_is_valid = bool(db_binary and getattr(db_binary, "is_valid", getattr(db_binary, "abspath", None)))
+    binary_is_valid = bool(db_binary and db_binary.is_valid)
    if binary_is_valid:
-        binary_data = db_binary.to_json() if hasattr(db_binary, "to_json") else db_binary.__dict__
+        binary_data = db_binary.to_json()
        section: SectionData = {
            "name": key,
            "description": mark_safe(render_binary_detail_description(key, binary_data, db_binary)),
@@ -19,8 +19,6 @@ from django.urls import reverse, resolve
 from django.utils import timezone
 from django.utils.text import smart_split

-from archivebox.config import DATA_DIR
-from archivebox.config.common import get_config
 from archivebox.misc.paginators import AcceleratedPaginator
 from archivebox.base_models.admin import BaseModelAdmin
 from archivebox.plugins.discovery import get_plugin_icon
@@ -35,7 +33,7 @@ from archivebox.core.models import ArchiveResult, Snapshot

 def _get_replay_source_url(result: ArchiveResult) -> str:
    process = result.process_record
-    return str(getattr(process, "url", None) or result.snapshot.url or "")
+    return str((process.url if process else None) or result.snapshot.url or "")


 def build_abx_dl_display_command(result: ArchiveResult) -> str:
@@ -51,8 +49,8 @@ def build_abx_dl_display_command(result: ArchiveResult) -> str:

 def build_abx_dl_replay_command(result: ArchiveResult, config=None) -> str:
    display_command = build_abx_dl_display_command(result)
-    process = getattr(result, "process", None)
-    env_items = env_to_shell_exports(getattr(process, "env", None) or {})
+    process = result.process
+    env_items = env_to_shell_exports(process.env if process else {})
    if config is not None:
        result.snapshot._runtime_config = config
    snapshot_dir = shlex.quote(str(result.pwd or result.snapshot_dir))
@@ -77,6 +75,12 @@ def get_plugin_admin_url(plugin_name: str) -> str:
    return f"{LIVE_PLUGIN_BASE_URL}builtin.{quote(plugin_name)}/"


+def get_process_link_label(process) -> str:
+    if process.pid:
+        return str(process.pid)
+    return str(process.id)[-8:]
+
+
 def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
    """Render a nice inline list view of archive results with status, plugin, output, and actions."""

@@ -141,7 +145,7 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
            process_display = f'''
                <a href="{reverse("admin:machine_process_change", args=[process.id])}"
                   style="color: #2563eb; text-decoration: none; font-family: ui-monospace, monospace; font-size: 12px;"
-                   title="View process">{process.pid or "-"}</a>
+                   title="View process">{get_process_link_label(process)}</a>
            '''

        machine_display = "-"
@@ -164,8 +168,8 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
        cmd_attr = html.escape(replay_cmd, quote=True)

        # Build output link - use embed_path() which checks output_files first
-        embed_path = result.embed_path() if hasattr(result, "embed_path") else None
-        snapshot_id = str(getattr(result, "snapshot_id", ""))
+        embed_path = result.embed_path()
+        snapshot_id = str(result.snapshot_id)
        if embed_path and result.status == "succeeded":
            output_link = build_snapshot_url(snapshot_id, embed_path, config=config)
        else:
@@ -354,8 +358,7 @@ class ArchiveResultInline(admin.TabularInline):
    def get_formset(self, request, obj=None, **kwargs):
        formset = super().get_formset(request, obj, **kwargs)
        snapshot = self.get_parent_object_from_request(request)
-        form_class = getattr(formset, "form", None)
-        base_fields = getattr(form_class, "base_fields", {})
+        base_fields = formset.form.base_fields
        snapshot_output_dir = str(snapshot.output_dir) if snapshot else ""

        # import ipdb; ipdb.set_trace()
@@ -491,12 +494,10 @@ class ArchiveResultAdmin(BaseModelAdmin):

    def change_view(self, request, object_id, form_url="", extra_context=None):
        self.request = request
-        request.archivebox_config = getattr(request, "archivebox_config", None) or get_config()
        return super().change_view(request, object_id, form_url, extra_context)

    def changelist_view(self, request, extra_context=None):
        self.request = request
-        request.archivebox_config = getattr(request, "archivebox_config", None) or get_config()
        saved_list_per_page = self.list_per_page
        self.list_per_page = request.archivebox_config.SNAPSHOTS_PER_PAGE
        try:
@@ -522,7 +523,6 @@ class ArchiveResultAdmin(BaseModelAdmin):
            super()
            .get_queryset(request)
            .defer(
-                "config",
                "notes",
                "output_json",
            )
@@ -573,20 +573,20 @@ class ArchiveResultAdmin(BaseModelAdmin):
        return queryset.filter(reduce(and_, filters)).distinct(), True

    def get_snapshot_view_url(self, result: ArchiveResult) -> str:
-        request = getattr(self, "request", None)
-        return build_snapshot_url(str(result.snapshot_id), request=request, config=getattr(request, "archivebox_config", None))
+        request = self.request
+        return build_snapshot_url(str(result.snapshot_id), request=request, config=request.archivebox_config)

    def get_output_view_url(self, result: ArchiveResult) -> str:
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
-        output_path = result.embed_path() if hasattr(result, "embed_path") else None
+        request = self.request
+        config = request.archivebox_config
+        output_path = result.embed_path()
        if not output_path:
            output_path = result.plugin or ""
        return build_snapshot_url(str(result.snapshot_id), output_path, request=request, config=config)

    def get_output_files_url(self, result: ArchiveResult) -> str:
-        request = getattr(self, "request", None)
-        return f"{build_snapshot_url(str(result.snapshot_id), result.plugin, request=request, config=getattr(request, 'archivebox_config', None))}/?files=1"
+        request = self.request
+        return f"{build_snapshot_url(str(result.snapshot_id), result.plugin, request=request, config=request.archivebox_config)}/?files=1"

    def get_output_zip_url(self, result: ArchiveResult) -> str:
        return f"{self.get_output_files_url(result)}&download=zip"
@@ -612,10 +612,10 @@ class ArchiveResultAdmin(BaseModelAdmin):
    )
    def snapshot_info(self, result):
        snapshot_id = str(result.snapshot_id)
-        request = getattr(self, "request", None)
+        request = self.request
        return format_html(
            '<a href="{}"><b><code>[{}]</code></b> &nbsp; {} &nbsp; {}</a><br/>',
-            build_snapshot_url(snapshot_id, "index.html", request=request, config=getattr(request, "archivebox_config", None)),
+            build_snapshot_url(snapshot_id, "index.html", request=request, config=request.archivebox_config),
            snapshot_id[:8],
            result.snapshot.bookmarked_at.strftime("%Y-%m-%d %H:%M"),
            result.snapshot.url[:128],
@@ -665,7 +665,7 @@ class ArchiveResultAdmin(BaseModelAdmin):
        process = result.process_record
        if not process:
            return "-"
-        process_label = process.pid or "-"
+        process_label = get_process_link_label(process)
        return format_html(
            '<a href="{}"><code>{}</code></a>',
            reverse("admin:machine_process_change", args=[process.id]),
@@ -679,17 +679,16 @@ class ArchiveResultAdmin(BaseModelAdmin):
            return "-"
        machine = process.machine
        return format_html(
-            '<a href="{}"><code>{}</code> {}</a>',
+            '<a href="{}">{}</a>',
            reverse("admin:machine_machine_change", args=[machine.id]),
-            str(machine.id)[:8],
            machine.hostname,
        )

    @admin.display(description="Command")
    def cmd_str(self, result):
-        request = getattr(self, "request", None)
+        request = self.request
        display_cmd = build_abx_dl_display_command(result)
-        replay_cmd = build_abx_dl_replay_command(result, config=getattr(request, "archivebox_config", None))
+        replay_cmd = build_abx_dl_replay_command(result, config=request.archivebox_config)
        return format_html(
            """
            <div style="position: relative; width: 100%; max-width: 100%; overflow: hidden; box-sizing: border-box;">
@@ -710,10 +709,10 @@ class ArchiveResultAdmin(BaseModelAdmin):
        )

    def output_display(self, result):
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
        # Determine output link path - use embed_path() which checks output_files
-        embed_path = result.embed_path() if hasattr(result, "embed_path") else None
+        embed_path = result.embed_path()
        output_path = embed_path if (result.status == "succeeded" and embed_path) else "index.html"
        snapshot_id = str(result.snapshot_id)
        return format_html(
@@ -728,12 +727,12 @@ class ArchiveResultAdmin(BaseModelAdmin):
        if not output_text:
            return "-"

-        request = getattr(self, "request", None)
-        live_path = result.embed_path() if hasattr(result, "embed_path") else None
+        request = self.request
+        live_path = result.embed_path()
        if live_path:
            return format_html(
                '<a href="{}" title="{}"><code>{}</code></a>',
-                build_snapshot_url(str(result.snapshot_id), live_path, request=request, config=getattr(request, "archivebox_config", None)),
+                build_snapshot_url(str(result.snapshot_id), live_path, request=request, config=request.archivebox_config),
                output_text,
                output_text,
            )
@@ -785,18 +784,18 @@ class ArchiveResultAdmin(BaseModelAdmin):
        )

    def output_summary(self, result):
-        snapshot_dir = Path(DATA_DIR) / str(result.pwd).split("data/", 1)[-1]
+        snapshot_dir = Path(result.snapshot.output_dir)
        output_html = format_html(
            '<pre style="display: inline-block">{}</pre><br/>',
            result.output_str_for_display(),
        )
        snapshot_id = str(result.snapshot_id)
-        request = getattr(self, "request", None)
+        request = self.request
        output_html += format_html(
            '<a href="{}#all">See result files ...</a><br/><pre><code>',
-            build_snapshot_url(snapshot_id, "index.html", request=request, config=getattr(request, "archivebox_config", None)),
+            build_snapshot_url(snapshot_id, "index.html", request=request, config=request.archivebox_config),
        )
-        embed_path = result.embed_path() if hasattr(result, "embed_path") else ""
+        embed_path = result.embed_path() or ""
        path_from_embed = snapshot_dir / (embed_path or "")
        output_html += format_html(
            '<i style="padding: 1px">{}</i><b style="padding-right: 20px">/</b><i>{}</i><br/><hr/>',
@@ -11,7 +11,7 @@ from django.http import JsonResponse, HttpResponseBadRequest, HttpResponseNotAll
 from django.utils import timezone
 from django.utils.html import format_html, format_html_join
 from django.utils.safestring import mark_safe
-from django.db.models import Q, Count, Exists, OuterRef, Prefetch
+from django.db.models import Q, Count, Exists, F, OuterRef, Prefetch
 from django import forms
 from django.template import Template, RequestContext
 from django.contrib.admin.helpers import ActionForm
@@ -190,6 +190,7 @@ class SnapshotSizeListFilter(admin.SimpleListFilter):
 class SnapshotResultHealthListFilter(admin.SimpleListFilter):
    title = "ArchiveResult status"
    parameter_name = "archiveresult_status"
+    SNAPSHOT_FIRST_VALUES = {"succeeded"}

    def lookups(self, request, model_admin):
        return (
@@ -199,7 +200,6 @@ class SnapshotResultHealthListFilter(admin.SimpleListFilter):
            ("failed", ">50% failed"),
            ("running", ">50% running"),
            ("pending", ">50% queued"),
-            ("backoff", ">50% waiting to retry"),
            ("noresults", ">50% noresults"),
        )

@@ -215,27 +215,64 @@ class SnapshotResultHealthListFilter(admin.SimpleListFilter):
                "succeeded": ArchiveResult.StatusChoices.SUCCEEDED,
                "failed": ArchiveResult.StatusChoices.FAILED,
                "running": ArchiveResult.StatusChoices.STARTED,
-                "pending": ArchiveResult.StatusChoices.QUEUED,
+                "pending": (ArchiveResult.StatusChoices.QUEUED, ArchiveResult.StatusChoices.BACKOFF),
                "backoff": ArchiveResult.StatusChoices.BACKOFF,
                "noresults": ArchiveResult.StatusChoices.NORESULTS,
            }
            if value in status_by_value:
-                # Start from ArchiveResult.status for every majority-status filter.
-                # The (status, snapshot_id) index keeps this plan stable regardless
-                # of which status is most common in a user's collection.
-                return queryset.filter(pk__in=ArchiveResult.snapshot_ids_with_majority_status(status_by_value[value]))
+                status = status_by_value[value]
+                if value in self.SNAPSHOT_FIRST_VALUES:
+                    # "succeeded" is overwhelmingly common in large collections.
+                    # Scan Snapshots in admin order and do indexed per-snapshot
+                    # probes so page 1 can stop after list_per_page matches.
+                    queryset = queryset.alias(
+                        total_results=ArchiveResult.snapshot_count_expr(),
+                        matching_results=ArchiveResult.snapshot_count_expr(status=status),
+                    ).filter(matching_results__gt=F("total_results") / 2)
+                    queryset._archivebox_count_hint = "model_estimate"
+                    queryset.query._archivebox_count_hint = queryset._archivebox_count_hint
+                    return queryset
+
+                # Rare statuses are faster status-first: use the
+                # (status, snapshot_id) index to find candidate snapshots.
+                snapshot_ids = ArchiveResult.cached_snapshot_ids_with_majority_status(status)
+                queryset = queryset.filter(pk__in=snapshot_ids)
+                queryset._archivebox_count_hint = len(snapshot_ids)
+                queryset.query._archivebox_count_hint = queryset._archivebox_count_hint
+                return queryset
        return queryset


 class SnapshotChangeList(SearchResultsChangeList):
    def __init__(self, request, *args, **kwargs):
        super().__init__(request, *args, **kwargs)
-        resolver_name = getattr(getattr(request, "resolver_match", None), "url_name", "")
+        resolver_name = request.resolver_match.url_name
        self.embedded_changelist = request.GET.get("_embedded") == "crawl"
        self.snapshot_is_grid_view = not self.embedded_changelist and (
            resolver_name == "grid" or request.path.rstrip("/").endswith("/grid")
        )

+    def _attach_archiveresult_summaries(self):
+        snapshot_ids = [obj.pk for obj in self.result_list]
+        if not snapshot_ids:
+            return
+
+        results_by_snapshot = {snapshot_id: [] for snapshot_id in snapshot_ids}
+        seen_plugins = {snapshot_id: set() for snapshot_id in snapshot_ids}
+        rows = (
+            ArchiveResult.objects.filter(snapshot_id__in=snapshot_ids, status=ArchiveResult.StatusChoices.SUCCEEDED, output_size__gt=0)
+            .order_by("snapshot_id", "plugin")
+            .values_list("snapshot_id", "plugin", "status", "output_size")
+        )
+        for snapshot_id, plugin, status, output_size in rows.iterator(chunk_size=1000):
+            if plugin in seen_plugins[snapshot_id]:
+                continue
+            seen_plugins[snapshot_id].add(plugin)
+            results_by_snapshot[snapshot_id].append(SimpleNamespace(plugin=plugin, status=status, output_size=output_size))
+
+        for obj in self.result_list:
+            obj.__dict__["_admin_archiveresults"] = results_by_snapshot[obj.pk]
+
    def get_results(self, request):
        super().get_results(request)
        if request.GET.get("_embedded") == "crawl":
@@ -243,28 +280,11 @@ class SnapshotChangeList(SearchResultsChangeList):
        else:
            self.full_result_count = self.model_admin.get_paginator(
                request,
-                self.model._default_manager.all().order_by(),
+                self.model._default_manager.all().order_by("-id"),
                self.list_per_page,
            ).count
        self.show_full_result_count = True
-
-        snapshot_ids = [obj.pk for obj in self.result_list]
-        if snapshot_ids:
-            results_by_snapshot = {snapshot_id: [] for snapshot_id in snapshot_ids}
-            seen_plugins = {snapshot_id: set() for snapshot_id in snapshot_ids}
-            rows = (
-                ArchiveResult.objects.filter(snapshot_id__in=snapshot_ids, status=ArchiveResult.StatusChoices.SUCCEEDED, output_size__gt=0)
-                .order_by("snapshot_id", "plugin")
-                .values_list("snapshot_id", "plugin", "status", "output_size")
-            )
-            for snapshot_id, plugin, status, output_size in rows.iterator(chunk_size=1000):
-                if plugin in seen_plugins[snapshot_id]:
-                    continue
-                seen_plugins[snapshot_id].add(plugin)
-                results_by_snapshot[snapshot_id].append(SimpleNamespace(plugin=plugin, status=status, output_size=output_size))
-
-            for obj in self.result_list:
-                obj.__dict__["_admin_archiveresults"] = results_by_snapshot[obj.pk]
+        self._attach_archiveresult_summaries()


 class SnapshotAdminForm(forms.ModelForm):
@@ -309,9 +329,7 @@ class SnapshotAdminForm(forms.ModelForm):
        # Handle tags_editor field
        if commit:
            instance.save()
-            save_m2m = getattr(self, "_save_m2m", None)
-            if callable(save_m2m):
-                save_m2m()
+            self._save_m2m()

            # Parse and save tags from tags_editor
            tags_str = self.cleaned_data.get("tags_editor", "")
@@ -447,7 +465,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
        ),
    )

-    ordering = ["-timestamp"]
+    ordering = ["-created_at"]
    actions = [
        "add_tags",
        "remove_tags",
@@ -475,7 +493,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
        return super().get_ordering(request)

    def change_view(self, request, object_id, form_url="", extra_context=None):
-        request.archivebox_config = getattr(request, "archivebox_config", None) or get_config()
+        self.request = request
        extra_context = extra_context or {}
        extra_context["CONFIG"] = request.archivebox_config
        snapshot = self.get_object(request, object_id)
@@ -490,7 +508,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):

    def changelist_view(self, request, extra_context=None):
        self.request = request
-        request.archivebox_config = getattr(request, "archivebox_config", None) or get_config()
        saved_list_per_page = self.list_per_page
        embedded_changelist = request.GET.get("_embedded") == "crawl"
        if embedded_changelist:
@@ -526,12 +543,12 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
        return super().lookup_allowed(lookup, value, request=request)

    def get_snapshot_view_url(self, obj: Snapshot) -> str:
-        request = getattr(self, "request", None)
-        return build_snapshot_url(str(obj.id), request=request, config=getattr(request, "archivebox_config", None))
+        request = self.request
+        return build_snapshot_url(str(obj.id), request=request, config=request.archivebox_config)

    def get_snapshot_files_url(self, obj: Snapshot) -> str:
-        request = getattr(self, "request", None)
-        return f"{build_snapshot_url(str(obj.id), request=request, config=getattr(request, 'archivebox_config', None))}/?files=1"
+        request = self.request
+        return f"{build_snapshot_url(str(obj.id), request=request, config=request.archivebox_config)}/?files=1"

    def get_snapshot_zip_url(self, obj: Snapshot) -> str:
        return f"{self.get_snapshot_files_url(obj)}&download=zip"
@@ -636,7 +653,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
        ordering_fields = self._get_ordering_fields(request)
        needs_files_sort = "files" in ordering_fields
        needs_tags_sort = "tags_inline" in ordering_fields
-        is_change_view = getattr(getattr(request, "resolver_match", None), "url_name", "") == "core_snapshot_change"
+        is_change_view = request.resolver_match.url_name == "core_snapshot_change"
        prefetches = ["tags"]
        if is_change_view:
            prefetches.append(
@@ -688,7 +705,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):

    @admin.display(description="👁", ordering="permissions")
    def permissions_badge(self, obj):
-        permissions = getattr(obj, "snapshot_permissions", None)
+        permissions = obj.__dict__.get("snapshot_permissions")
        if permissions is None:
            if obj.permissions:
                permissions = obj.permissions
@@ -858,8 +875,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
        )

    def status_info(self, obj):
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
        favicon_url = build_snapshot_url(str(obj.id), "favicon.ico", request=request, config=config)
        return format_html(
            """
@@ -876,16 +893,16 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):

    @admin.display(description="Archive Results")
    def archiveresults_list(self, obj):
-        request = getattr(self, "request", None)
-        return render_archiveresults_list(obj.archiveresult_set.all(), limit=8, config=getattr(request, "archivebox_config", None))
+        request = self.request
+        return render_archiveresults_list(obj.archiveresult_set.all(), limit=8, config=request.archivebox_config)

    @admin.display(
        description="Title",
        ordering="title",
    )
    def title_str(self, obj):
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
        title_raw = (obj.title or "").strip()
        url_raw = (obj.url or "").strip()
        title_normalized = title_raw.lower()
@@ -938,8 +955,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
        return mark_safe(f'<span class="tags-inline-editor">{tags_html}</span>')

    def _get_preview_data(self, obj):
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
        results = self._get_prefetched_results(obj)
        if results is not None:
            has_screenshot = any(r.plugin == "screenshot" for r in results)
@@ -991,8 +1008,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
        if not preview:
            return ""

-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
        favicon_url = build_snapshot_url(str(obj.id), "favicon/favicon.ico", request=request, config=config)
        fallback_list = ",".join([build_snapshot_url(str(obj.id), "favicon.ico", request=request, config=config)])
        onerror_js = (
@@ -1030,8 +1047,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):

    @admin.display(description=" ", empty_value="")
    def snapshot_summary(self, obj):
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
        preview = self._get_preview_data(obj)
        stats = self._get_progress_stats(obj)
        archive_size = stats["output_size"] or 0
@@ -1095,8 +1112,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
        )
        visible_results = sorted_results[:14]
        output = []
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
        for result in visible_results:
            icon = mark_safe(get_plugin_icon(result.plugin))
            if not icon.strip():
@@ -1127,8 +1144,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
        # ordering='archiveresult_count'
    )
    def size(self, obj):
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
        archive_size = self._get_progress_stats(obj)["output_size"] or 0
        if archive_size:
            size_txt = printable_filesize(archive_size)
@@ -1242,7 +1259,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
        )

    def _get_progress_stats(self, obj):
-        cached_stats = getattr(obj, "_admin_progress_stats", None)
+        cached_stats = obj.__dict__.get("_admin_progress_stats")
        if cached_stats is not None:
            return cached_stats

@@ -1288,31 +1305,31 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
    def _get_prefetched_results(self, obj):
        if "_admin_archiveresults" in obj.__dict__:
            return obj.__dict__["_admin_archiveresults"]
-        if hasattr(obj, "_prefetched_objects_cache") and "archiveresult_set" in obj._prefetched_objects_cache:
+        if "archiveresult_set" in obj.__dict__.get("_prefetched_objects_cache", {}):
            return obj.archiveresult_set.all()
        return None

    def _get_expected_hook_total(self, obj) -> int:
        try:
-            request = getattr(self, "request", None)
-            if getattr(getattr(request, "resolver_match", None), "url_name", "") in {"core_snapshot_changelist", "core_snapshot_change"}:
+            request = self.request
+            if request.resolver_match.url_name in {"core_snapshot_changelist", "core_snapshot_change"}:
                return 0

-            crawl = getattr(obj, "crawl", None)
-            snapshot_config = getattr(obj, "config", None) or {}
-            crawl_config = getattr(crawl, "config", None) or {}
+            crawl = obj.crawl
+            snapshot_config = obj.config or {}
+            crawl_config = crawl.config or {}
            has_scoped_config = bool(snapshot_config or crawl_config)

            if request is not None and not has_scoped_config:
-                cached_total = getattr(request, "archivebox_expected_snapshot_hook_total", None)
+                cached_total = request.__dict__.get("archivebox_expected_snapshot_hook_total")
                if cached_total is None:
-                    config = getattr(request, "archivebox_config", None) or get_config()
+                    config = request.archivebox_config
                    cached_total = len(discover_hooks("Snapshot", config=config))
                    request.archivebox_expected_snapshot_hook_total = cached_total
                return cached_total

            if request is not None:
-                scoped_cache = getattr(request, "archivebox_expected_snapshot_hook_totals_by_scope", None)
+                scoped_cache = request.__dict__.get("archivebox_expected_snapshot_hook_totals_by_scope")
                if scoped_cache is None:
                    scoped_cache = {}
                    request.archivebox_expected_snapshot_hook_totals_by_scope = scoped_cache
@@ -1332,8 +1349,9 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
            return 0

    def _get_prefetched_tags(self, obj):
-        if hasattr(obj, "_prefetched_objects_cache") and "tags" in obj._prefetched_objects_cache:
-            return list(obj._prefetched_objects_cache["tags"])
+        prefetched_cache = obj.__dict__.get("_prefetched_objects_cache", {})
+        if "tags" in prefetched_cache:
+            return list(prefetched_cache["tags"])
        return None

    def _get_ordering_fields(self, request):
@@ -117,7 +117,7 @@ class TagAdmin(BaseModelAdmin):
        sort = normalize_tag_sort((request.GET.get("sort") or "created_desc").strip())
        created_by = normalize_created_by_filter((request.GET.get("created_by") or "").strip())
        year = normalize_created_year_filter((request.GET.get("year") or "").strip())
-        has_snapshots = normalize_has_snapshots_filter((request.GET.get("has_snapshots") or "yes").strip())
+        has_snapshots = normalize_has_snapshots_filter((request.GET.get("has_snapshots") or "all").strip())
        context = {
            **self.admin_site.each_context(request),
            **(extra_context or {}),
@@ -231,7 +231,7 @@ class TagAdmin(BaseModelAdmin):

    @admin.display(description="Snapshots", ordering="num_snapshots")
    def num_snapshots(self, tag: Tag):
-        count = getattr(tag, "num_snapshots", None)
+        count = tag.__dict__.get("num_snapshots")
        if count is None:
            count = tag.snapshot_set.count()
        return format_html(
@@ -47,7 +47,7 @@ class CustomUserAdmin(UserAdmin):

    def snapshot_count_badge(self, obj):
        snapshots_url = f"/admin/core/snapshot/?created_by__id__exact={obj.pk}"
-        snapshot_count = getattr(obj, "snapshot_count", 0)
+        snapshot_count = obj.__dict__.get("snapshot_count", 0)
        snapshot_label = "snapshot" if snapshot_count == 1 else "snapshots"
        return format_html(
            (
@@ -31,6 +31,8 @@ DEPTH_CHOICES = (


 class AddLinkForm(PluginConfigFormMixin, forms.Form):
+    allow_crawl_execution_config_fields = False
+
    # Basic fields
    url = forms.CharField(
        label="URLs",
@@ -119,7 +119,7 @@ def CacheControlMiddleware(get_response):

        if "/archive/" in request.path or "/static/" in request.path or snapshot_path_re.match(request.path):
            if not response.get("Cache-Control"):
-                config = getattr(request, "archivebox_config", None)
+                config = request.__dict__.get("archivebox_config")
                if config is None:
                    config = get_config(resolve_plugins=False)
                    request.archivebox_config = config
@@ -136,7 +136,7 @@ def ServerSecurityModeMiddleware(get_response):
    allowed_methods = {"GET", "HEAD", "OPTIONS"}

    def middleware(request):
-        config = getattr(request, "archivebox_config", None)
+        config = request.__dict__.get("archivebox_config")
        if config is None:
            config = get_config(resolve_plugins=False)
            request.archivebox_config = config
@@ -169,7 +169,7 @@ def HostRoutingMiddleware(get_response):
            return get_response(request)

        request_host = (request.get_host() or "").lower()
-        config = getattr(request, "archivebox_config", None)
+        config = request.__dict__.get("archivebox_config")
        if config is None:
            config = get_config(resolve_plugins=False)
            request.archivebox_config = config
@@ -185,25 +185,7 @@ def HostRoutingMiddleware(get_response):
        if request.path.startswith("/static/") or request.path in {"/favicon.ico", "/robots.txt"}:
            return get_response(request)

-        # In subdomain mode with no explicit BASE_URL we can't safely emit
-        # ``admin.``/``web.``/``snap-*.`` redirects: every URL builder uses the
-        # request's own Host (via the request-host fallback in get_base_url),
-        # so prepending ``admin.`` to whatever the client sent produces a
-        # redirect chain of ``admin.admin.admin.<host>``. Pass the request
-        # through; the misconfig banner on the rendered page tells the user
-        # to pin BASE_URL so the redirects can resume.
-        if config.USES_SUBDOMAIN_ROUTING and not config.BASE_URL:
-            return get_response(request)
-
-        if config.USES_SUBDOMAIN_ROUTING and not host_matches(request_host, admin_host):
-            # ``/add`` is admin-only unless ``PUBLIC_ADD_VIEW`` is on. Without
-            # this redirect, hitting it on public.* falls into AddView's
-            # auth check, bounces through ``/accounts/login/?next=/add/`` →
-            # ``/admin/login/?next=/add/``, and Django admin's LoginView
-            # silently drops ``next`` when the user already has an admin
-            # session — dumping the user on the admin homepage instead of
-            # the add form. Routing the request to admin.* directly lets
-            # AddView run on the host where the session lives.
+        if config.USES_SUBDOMAIN_ROUTING and config.BASE_URL and not host_matches(request_host, admin_host):
            add_should_redirect = not config.PUBLIC_ADD_VIEW and (request.path == "/add" or request.path.startswith("/add/"))
            if (
                request.path == "/admin"
@@ -217,6 +199,20 @@ def HostRoutingMiddleware(get_response):
                    target = f"{target}?{request.META['QUERY_STRING']}"
                return redirect(target)

+        if subdomain and is_snapshot_subdomain(subdomain):
+            view = SnapshotHostView.as_view()
+            return view(request, snapshot_id=subdomain, path=request.path.lstrip("/"))
+
+        # In subdomain mode with no explicit BASE_URL we can't safely emit
+        # ``admin.``/``web.``/``snap-*.`` redirects: every URL builder uses the
+        # request's own Host (via the request-host fallback in get_base_url),
+        # so prepending ``admin.`` to whatever the client sent produces a
+        # redirect chain of ``admin.admin.admin.<host>``. Pass the request
+        # through; the misconfig banner on the rendered page tells the user
+        # to pin BASE_URL so the redirects can resume.
+        if config.USES_SUBDOMAIN_ROUTING and not config.BASE_URL:
+            return get_response(request)
+
        if not config.USES_SUBDOMAIN_ROUTING:
            if host_matches(request_host, listen_host):
                return get_response(request)
@@ -289,9 +285,6 @@ def HostRoutingMiddleware(get_response):
            return get_response(request)

        if subdomain:
-            if is_snapshot_subdomain(subdomain):
-                view = SnapshotHostView.as_view()
-                return view(request, snapshot_id=subdomain, path=request.path.lstrip("/"))
            view = OriginalDomainHostView.as_view()
            return view(request, domain=subdomain, path=request.path.lstrip("/"))

@@ -323,7 +316,7 @@ class ReverseProxyAuthMiddleware(RemoteUserMiddleware):
    header = "HTTP_REMOTE_USER"

    def process_request(self, request):
-        config = getattr(request, "archivebox_config", None)
+        config = request.__dict__.get("archivebox_config")
        if config is None:
            config = get_config(resolve_plugins=False)
            request.archivebox_config = config
@@ -0,0 +1,14 @@
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0047_archiveresult_status_snapshot_index"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="archiveresult",
+            name="config",
+        ),
+    ]
@@ -0,0 +1,17 @@
+# Generated by Django 6.0.6 on 2026-06-04 22:16
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0048_remove_archiveresult_config"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="snapshot",
+            name="url",
+            field=models.TextField(db_index=True),
+        ),
+    ]
@@ -14,7 +14,7 @@ from urllib.parse import urlparse
 from statemachine import State, registry

 from django.db import models, transaction
-from django.db.models import Case, Q, QuerySet, Sum, Value, When
+from django.db.models import Case, F, Q, QuerySet, Sum, Value, When
 from django.db.models.functions import Coalesce, Concat
 from django.db.models.fields.json import KT
 from django.utils.functional import cached_property
@@ -31,7 +31,6 @@ from archivebox.config import CONSTANTS
 from archivebox.config.common import get_config, rprint
 from archivebox.misc.system import atomic_write
 from archivebox.misc.util import (
-    MAX_URL_LENGTH,
    parse_date,
    domain as url_domain,
    to_json,
@@ -63,6 +62,13 @@ if TYPE_CHECKING:
    from archivebox.config.common import ArchiveBoxBaseConfig


+class UngroupedSubquery(models.Subquery):
+    """Scalar subquery that should not be copied into the outer GROUP BY."""
+
+    def get_group_by_cols(self):
+        return []
+
+
 class Tag(ModelWithUUID):
    # Keep AutoField for compatibility with main branch migrations
    # Don't use UUIDField here - requires complex FK transformation
@@ -203,7 +209,7 @@ class SnapshotQuerySet(models.QuerySet):
                offset += chunk_size
            return

-        unique_field_names = {pk_field, *(field.name for field in self.model._meta.fields if getattr(field, "unique", False))}
+        unique_field_names = {pk_field, *(field.name for field in self.model._meta.fields if field.unique)}
        if not any(field_name in unique_field_names for field_name in ordered_field_names):
            offset = 0
            while True:
@@ -260,6 +266,22 @@ class SnapshotQuerySet(models.QuerySet):
        "tag": lambda pattern: models.Q(tags__name=pattern),
        "timestamp": lambda pattern: models.Q(timestamp=pattern),
    }
+    FILTER_TYPE_CHOICES = tuple(FILTER_TYPES)
+    FILTER_ARG_KEYS = (
+        "after",
+        "before",
+        "filter_type",
+        "filter_patterns",
+        "status",
+        "url__icontains",
+        "url__istartswith",
+        "tag",
+        "crawl_id",
+        "limit",
+        "sort",
+        "search",
+    )
+    SPECIAL_FILTER_ARG_KEYS = frozenset({"filter_patterns", "filter_type", "query", "search", "tag", "before", "after", "limit", "sort"})

    def filter_by_patterns(self, patterns: list[str], filter_type: str = "exact") -> "SnapshotQuerySet":
        """Filter snapshots by URL patterns using specified filter type"""
@@ -276,6 +298,62 @@ class SnapshotQuerySet(models.QuerySet):
                raise SystemExit(2)
        return self.filter(q_filter)

+    def search(self, **kwargs) -> "SnapshotQuerySet":
+        from datetime import timezone as dt_timezone
+
+        from archivebox.core.snapshot_status import filter_snapshots_by_status
+        from archivebox.search.query import apply_snapshot_search
+
+        queryset = self
+        filter_patterns = tuple(str(pattern) for pattern in kwargs.get("filter_patterns") or ())
+        filter_type = kwargs.get("filter_type") or "substring"
+        query = kwargs.get("query")
+        if isinstance(query, (list, tuple)):
+            query = " ".join(str(part) for part in query)
+        query = (query or (" ".join(filter_patterns) if kwargs.get("search") else "")).strip()
+
+        field_names = {field.name for field in self.model._meta.get_fields()}
+        field_names.update(field.attname for field in self.model._meta.fields)
+        field_filters = {
+            key: value
+            for key, value in kwargs.items()
+            if value is not None and key not in self.SPECIAL_FILTER_ARG_KEYS and key.split("__", 1)[0] in field_names
+        }
+        status = field_filters.pop("status", None)
+        queryset = filter_snapshots_by_status(queryset, status)
+        if field_filters:
+            queryset = queryset.filter(**field_filters)
+        if kwargs.get("tag"):
+            queryset = queryset.filter(tags__name__iexact=kwargs["tag"])
+        if kwargs.get("before") is not None:
+            queryset = queryset.filter(bookmarked_at__lt=datetime.fromtimestamp(float(kwargs["before"]), tz=dt_timezone.utc))
+        if kwargs.get("after") is not None:
+            queryset = queryset.filter(bookmarked_at__gt=datetime.fromtimestamp(float(kwargs["after"]), tz=dt_timezone.utc))
+
+        if query:
+            queryset = apply_snapshot_search(
+                queryset,
+                query,
+                search_mode=kwargs.get("search"),
+                ordering=("-created_at",) if not kwargs.get("sort") else None,
+                max_results=kwargs.get("limit"),
+                skip_backend_when_metadata_satisfies_limit=True,
+                include_metadata_for_forced_backend=True,
+            )
+        elif filter_patterns:
+            queryset = queryset.filter_by_patterns(list(filter_patterns), filter_type)
+
+        if kwargs.get("sort"):
+            queryset = queryset.order_by(kwargs["sort"])
+        elif not queryset.query.order_by:
+            queryset = queryset.order_by("-created_at")
+
+        limit = kwargs.get("limit")
+        if limit is not None and limit > 0:
+            queryset = queryset[:limit]
+
+        return queryset
+
    # =========================================================================
    # Export Methods
    # =========================================================================
@@ -389,7 +467,10 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
    created_at = models.DateTimeField(default=timezone.now, db_index=True)
    modified_at = models.DateTimeField(auto_now=True)

-    url = models.CharField(max_length=MAX_URL_LENGTH, unique=False, db_index=True)  # URLs can appear in multiple crawls
+    # Stored as a variable-length TextField so short URLs don't reserve space and very long
+    # URLs (up to MAX_URL_LENGTH chars, enforced in save()) are supported, while keeping a
+    # normal index so exact, prefix, and substring lookups all stay fast.
+    url = models.TextField(db_index=True)  # URLs can appear in multiple crawls
    timestamp = models.CharField(max_length=32, unique=True, db_index=True, editable=False)
    bookmarked_at = models.DateTimeField(default=timezone.now, db_index=True)
    crawl: Crawl = models.ForeignKey(Crawl, on_delete=models.CASCADE, null=False, related_name="snapshot_set", db_index=True)  # type: ignore[assignment]
@@ -457,6 +538,11 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
    objects = SnapshotManager()
    archiveresult_set: models.Manager["ArchiveResult"]

+    if TYPE_CHECKING:
+
+        @property
+        def sm(self) -> "SnapshotMachine": ...
+
    class Meta(
        ModelWithDeleteAfter.Meta,
        ModelWithOutputDir.Meta,
@@ -659,7 +745,9 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
            self.sm.seal()

    def get_delete_after_config_value(self):
-        return get_config(snapshot=self).DELETE_AFTER
+        from archivebox.config.common import resolve_delete_after_config_value
+
+        return resolve_delete_after_config_value(self.config, self.crawl.config)

    @classmethod
    def missing_delete_at_candidates(cls):
@@ -676,6 +764,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
        from archivebox.core.routes_util import (
            get_admin_host,
            get_api_host,
+            get_base_host,
            get_listen_host,
            get_public_host,
            get_web_host,
@@ -690,6 +779,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
        protected_roots: set[tuple[str, str | None]] = set()
        for host_value in (
            get_listen_host(config=config),
+            get_base_host(config=config),
            get_admin_host(config=config),
            get_web_host(config=config),
            get_api_host(config=config),
@@ -818,7 +908,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
        # work and crawl maintenance so SQLite commits before touching the disk.
        transaction.on_commit(finish_snapshot_save)

-        migration_cleanup = getattr(self, "_pending_fs_migration_cleanup", None)
+        migration_cleanup = self.__dict__.get("_pending_fs_migration_cleanup")
        if migration_cleanup:
            old_dir, new_dir = migration_cleanup
            transaction.on_commit(lambda: self._cleanup_old_migration_dir(old_dir, new_dir))
@@ -890,9 +980,9 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
        runtime_config = config or get_config()

        if source_dir and current == target:
-            current_dir = self.get_storage_path_for_version(target, config=runtime_config)
+            current_dir = self.get_storage_path_for_version(target)
            cleanup = self._fs_migrate_legacy_to_0_9_0(source_dir=source_dir, target_dir=current_dir)
-            crawl_dir = self.crawl.output_dir_for_config(runtime_config)
+            crawl_dir = self.crawl.output_dir
            old_crawl_dir = crawl_dir.with_name(str(uuid.UUID(hex=self.crawl.id.hex)))
            if old_crawl_dir.exists() and not crawl_dir.exists() and not old_crawl_dir.is_symlink():
                crawl_dir.parent.mkdir(parents=True, exist_ok=True)
@@ -932,9 +1022,9 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW

    def _fs_migrate_from_0_9_0_to_0_9_4(self, source_dir: Path | None = None, config: "ArchiveBoxBaseConfig | None" = None):
        runtime_config = config or get_config()
-        target_dir = self.get_storage_path_for_version("0.9.4", config=runtime_config)
+        target_dir = self.get_storage_path_for_version("0.9.4")
        cleanup = self._fs_migrate_legacy_to_0_9_0(source_dir=source_dir or self.output_dir, target_dir=target_dir, config=runtime_config)
-        crawl_dir = self.crawl.output_dir_for_config(runtime_config)
+        crawl_dir = self.crawl.output_dir
        old_crawl_dir = crawl_dir.with_name(str(uuid.UUID(hex=self.crawl.id.hex)))
        if old_crawl_dir.exists() and not crawl_dir.exists() and not old_crawl_dir.is_symlink():
            crawl_dir.parent.mkdir(parents=True, exist_ok=True)
@@ -956,8 +1046,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
        import filecmp
        import shutil

-        old_dir = Path(source_dir) if source_dir else self.get_storage_path_for_version("0.8.0", config=config)
-        new_dir = Path(target_dir) if target_dir else self.get_storage_path_for_version("0.9.0", config=config)
+        old_dir = Path(source_dir) if source_dir else self.get_storage_path_for_version("0.8.0")
+        new_dir = Path(target_dir) if target_dir else self.get_storage_path_for_version("0.9.0")

        if old_dir == new_dir:
            return None
@@ -1087,7 +1177,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
        except Exception:
            return "unknown"

-    def get_storage_path_for_version(self, version: str, config: "ArchiveBoxBaseConfig | None" = None) -> Path:
+    def get_storage_path_for_version(self, version: str) -> Path:
        """
        Calculate storage path for specific filesystem version.
        Centralizes path logic so it's reusable.
@@ -1095,10 +1185,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
        0.7.x/0.8.x: archive/{timestamp}
        0.9.x: archive/users/{username}/snapshots/YYYYMMDD/{domain}/{uuid}/
        """
-        runtime_config = config or get_config()
-
        if version in ("0.7.0", "0.8.0"):
-            return runtime_config.ARCHIVE_DIR / self.timestamp
+            return CONSTANTS.ARCHIVE_DIR / self.timestamp

        elif version in ("0.9.0", "0.9.1", "0.9.2", "0.9.3", "0.9.4", "1.0.0"):
            username = self.created_by.username
@@ -1108,10 +1196,10 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW

            domain = self.extract_domain_from_url(self.url)

-            return runtime_config.USERS_DIR / username / CONSTANTS.SNAPSHOTS_DIR_NAME / date_str / domain / str(self.id)
+            return CONSTANTS.USERS_DIR / username / CONSTANTS.SNAPSHOTS_DIR_NAME / date_str / domain / str(self.id)
        else:
            # Unknown version - use current
-            return self.get_storage_path_for_version(self._fs_current_version(), config=runtime_config)
+            return self.get_storage_path_for_version(self._fs_current_version())

    # =========================================================================
    # Loading and Creation from Filesystem (Used by archivebox update ONLY)
@@ -1702,7 +1790,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
                    f.write(json.dumps(process.to_json()) + "\n")

                # Write ArchiveResult record
-                f.write(json.dumps(ar.to_json()) + "\n")
+                f.write(json.dumps(ar.to_json(snapshot_output_dir=output_dir)) + "\n")
        os.replace(tmp_index_path, index_path)

    def read_index_jsonl(self, output_dir: Path | None = None) -> dict:
@@ -1978,7 +2066,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
    @admin.display(description="Tags")
    def tags_str(self, nocache=True) -> str | None:
        calc_tags_str = lambda: ",".join(sorted(tag.name for tag in self.tags.all()))
-        prefetched_cache = getattr(self, "_prefetched_objects_cache", {})
+        prefetched_cache = self.__dict__.get("_prefetched_objects_cache", {})
        if "tags" in prefetched_cache:
            return calc_tags_str()
        cache_key = f"{self.pk}-tags"
@@ -1988,12 +2076,12 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
        """Generate HTML icons showing which extractor plugins have succeeded for this snapshot"""
        from django.utils.html import format_html

-        compact_icons = getattr(self, "_icons_compact", False)
+        compact_icons = self.__dict__.get("_icons_compact", False)
        cache_key = f"result_icons:{self.pk}:{'compact' if compact_icons else 'full'}:{(self.downloaded_at or self.modified_at or self.created_at or self.bookmarked_at).timestamp()}"

        def calc_icons():
            if compact_icons and self.status == self.StatusChoices.STARTED:
-                progress_stats = getattr(self, "_icons_progress_stats", None) or self.get_progress_stats()
+                progress_stats = self.__dict__.get("_icons_progress_stats") or self.get_progress_stats()
                total = int(progress_stats.get("total") or 0)
                succeeded = int(progress_stats.get("succeeded") or 0)
                failed = int(progress_stats.get("failed") or 0)
@@ -2025,8 +2113,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
                    running,
                )

-            precomputed_archive_results = getattr(self, "_icons_archive_results", None)
-            prefetched_cache = getattr(self, "_prefetched_objects_cache", {})
+            precomputed_archive_results = self.__dict__.get("_icons_archive_results")
+            prefetched_cache = self.__dict__.get("_prefetched_objects_cache", {})
            if precomputed_archive_results is not None and compact_icons:
                archive_results = {plugin: True for plugin in precomputed_archive_results}
            elif "archiveresult_set" in prefetched_cache:
@@ -2049,7 +2137,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
            output_template = '<a href="/{}/{}" class="exists-{}" title="{}">{}</a>'

            # Get all plugins from hooks system (sorted by numeric prefix)
-            all_plugins = getattr(self, "_icons_plugin_names", None)
+            all_plugins = self.__dict__.get("_icons_plugin_names")
            if all_plugins is None and not compact_icons:
                all_plugins = [get_plugin_name(e) for e in get_plugins()]
            elif all_plugins is None:
@@ -2200,8 +2288,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
        """The filesystem path to the snapshot's output directory."""
        import os

-        runtime_config = getattr(self, "_runtime_config", None) or get_config()
-        current_path = self.get_storage_path_for_version(self.fs_version, config=runtime_config)
+        current_path = self.get_storage_path_for_version(self.fs_version)

        if current_path.exists():
            return current_path
@@ -2212,7 +2299,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
                return hyphen_path

        # Check for backwards-compat symlink
-        old_path = runtime_config.ARCHIVE_DIR / self.timestamp
+        old_path = CONSTANTS.ARCHIVE_DIR / self.timestamp
        if old_path.is_symlink():
            link_target = Path(os.readlink(old_path))
            return (old_path.parent / link_target).resolve() if not link_target.is_absolute() else link_target.resolve()
@@ -2225,7 +2312,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
        """Ensure the legacy archive/<timestamp> path resolves to this snapshot."""
        import os

-        legacy_path = get_config().ARCHIVE_DIR / self.timestamp
+        legacy_path = CONSTANTS.ARCHIVE_DIR / self.timestamp
        target = Path(self.get_storage_path_for_version(self._fs_current_version()))

        if target == legacy_path:
@@ -2301,8 +2388,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW

        if self.fs_version in ("0.9.0", "0.9.1", "0.9.2", "0.9.3", "0.9.4", "1.0.0"):
            username = "web"
-            crawl = getattr(self, "crawl", None)
-            if crawl and getattr(crawl, "created_by_id", None):
+            crawl = self.crawl if self.crawl_id else None
+            if crawl and crawl.created_by_id:
                username = crawl.created_by.username
            if username == "system":
                username = "web"
@@ -2326,7 +2413,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW

        output_dir = Path(self.output_dir).resolve()
        try:
-            rel_users_path = output_dir.relative_to(get_config().USERS_DIR)
+            rel_users_path = output_dir.relative_to(CONSTANTS.USERS_DIR)
        except Exception:
            rel_users_path = None

@@ -2652,7 +2739,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW

        return snapshot

-    def create_pending_archiveresults(self) -> list["ArchiveResult"]:
+    def create_pending_archiveresults(self, hooks: Iterable[tuple[str, str]] | None = None) -> list["ArchiveResult"]:
        """
        Create ArchiveResult records for all enabled hooks.

@@ -2663,18 +2750,17 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
        Creates one ArchiveResult per hook (not per plugin), with hook_name set.
        This enables step-based execution where all hooks in a step can run in parallel.
        """
-        from archivebox.plugins.hooks import discover_hooks
-        from archivebox.config.common import get_config
+        if hooks is None:
+            from archivebox.plugins.hooks import discover_hooks
+            from archivebox.config.common import get_config

-        # Get merged config with crawl-specific PLUGINS filter
-        config = get_config(crawl=self.crawl, snapshot=self)
-        hooks = discover_hooks("Snapshot", config=config)
+            # Compatibility path for direct model callers. The runner passes its
+            # abx-dl hook inventory explicitly so queued rows match execution.
+            config = get_config(crawl=self.crawl, snapshot=self)
+            hooks = ((hook_path.parent.name, hook_path.stem) for hook_path in discover_hooks("Snapshot", config=config))
        archiveresults = []

-        for hook_path in hooks:
-            hook_name = hook_path.stem  # e.g., 'on_Snapshot__50_wget'
-            plugin = hook_path.parent.name  # e.g., 'wget'
-
+        for plugin, hook_name in hooks:
            # ArchiveResult output is one filesystem directory per plugin hook, so
            # retries must update this row in place instead of creating siblings.
            archiveresult, _created = ArchiveResult.objects.get_or_create(
@@ -2836,7 +2922,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW

    @cached_property
    def is_archived(self) -> bool:
-        cached_is_archived = getattr(self, "_is_archived_cached", None)
+        cached_is_archived = self.__dict__.get("_is_archived_cached")
        if cached_is_archived is not None:
            return bool(cached_is_archived)

@@ -2890,10 +2976,10 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW

    @cached_property
    def num_outputs(self) -> int:
-        if hasattr(self, "num_outputs_cached"):
-            return int(self.num_outputs_cached or 0)
+        if "num_outputs_cached" in self.__dict__:
+            return int(self.__dict__["num_outputs_cached"] or 0)

-        prefetched_cache = getattr(self, "_prefetched_objects_cache", {})
+        prefetched_cache = self.__dict__.get("_prefetched_objects_cache", {})
        if "archiveresult_set" in prefetched_cache:
            return sum(1 for result in self.archiveresult_set.all() if result.status == "succeeded")

@@ -2901,10 +2987,10 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW

    @cached_property
    def num_failures(self) -> int:
-        if hasattr(self, "num_failures_cached"):
-            return int(self.num_failures_cached or 0)
+        if "num_failures_cached" in self.__dict__:
+            return int(self.__dict__["num_failures_cached"] or 0)

-        prefetched_cache = getattr(self, "_prefetched_objects_cache", {})
+        prefetched_cache = self.__dict__.get("_prefetched_objects_cache", {})
        if "archiveresult_set" in prefetched_cache:
            return sum(1 for result in self.archiveresult_set.all() if result.status == "failed")

@@ -3162,7 +3248,11 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
        """Convert to CSV string"""
        data = self.to_dict()
        cols = cols or ["timestamp", "is_archived", "url"]
-        return separator.join(to_json(data.get(col, ""), indent=None).ljust(ljust) for col in cols)
+        invalid_cols = [col for col in dict.fromkeys(cols) if col not in data]
+        if invalid_cols:
+            supported_cols = ", ".join(sorted(data))
+            raise ValueError(f"Invalid CSV field(s): {', '.join(invalid_cols)}\nSupported CSV fields: {supported_cols}")
+        return separator.join(to_json(data[col], indent=None).ljust(ljust) for col in cols)

    def write_json_details(self, out_dir: Path | str | None = None) -> None:
        """Write JSON index file for this snapshot to its output directory"""
@@ -3458,7 +3548,7 @@ class SnapshotMachine(BaseStateMachine):
        # and the runner still needs to enqueue those child snapshots.


-class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWithNotes):
+class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
    class StatusChoices(models.TextChoices):
        QUEUED = "queued", "Queued"
        STARTED = "started", "Started"
@@ -3523,7 +3613,17 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
        qs = cls.objects.filter(snapshot_id=models.OuterRef(outer_ref))
        if status is not None:
            qs = qs.filter(status=status)
-        return qs.order_by().values("snapshot_id").annotate(count=models.Count("pk")).values("count")
+        return qs.order_by().values("snapshot_id").annotate(count=models.Count("*")).values("count")
+
+    @classmethod
+    def snapshot_half_count_subquery(cls, *, outer_ref: str = "snapshot_id") -> QuerySet:
+        return (
+            cls.objects.filter(snapshot_id=models.OuterRef(outer_ref))
+            .order_by()
+            .values("snapshot_id")
+            .annotate(half=models.Count("*") / models.Value(2))
+            .values("half")
+        )

    @classmethod
    def snapshot_count_expr(cls, *, status: str | None = None, outer_ref: str = "pk"):
@@ -3539,28 +3639,50 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
        return {status: qs.filter(status=status).count() for status in (statuses or cls.StatusChoices.values)}

    @classmethod
-    def snapshot_ids_with_majority_status(cls, status: str) -> QuerySet:
+    def snapshot_ids_with_majority_status(cls, status: str | Iterable[str]) -> QuerySet:
        """Return Snapshot IDs where more than half of ArchiveResults have ``status``.

        Start from ArchiveResult.status for every majority-status filter. The
        ``(status, snapshot_id)`` index keeps the plan predictable even when a
        user's collection has an unusual status distribution.
        """
+        statuses = tuple(status) if not isinstance(status, str) else (status,)
+        total_half = UngroupedSubquery(cls.snapshot_half_count_subquery(outer_ref="snapshot_id"), output_field=models.IntegerField())
        return (
-            cls.objects.filter(status=status)
+            cls.objects.filter(status__in=statuses)
            .order_by()
            .values("snapshot_id")
            .annotate(
-                matching_results=models.Count("pk"),
-                total_results=models.Subquery(
-                    cls.snapshot_count_subquery(outer_ref="snapshot_id"),
-                    output_field=models.IntegerField(),
-                ),
+                matching_results=models.Count("*"),
+                total_half=total_half,
            )
-            .filter(matching_results__gt=models.F("total_results") / 2)
+            .filter(matching_results__gt=models.F("total_half"))
            .values("snapshot_id")
        )

+    @classmethod
+    def cached_snapshot_ids_with_majority_status(cls, status: str | Iterable[str], *, timeout: int = 60) -> tuple[str, ...]:
+        statuses = tuple(status) if not isinstance(status, str) else (status,)
+        cache_key = f"archivebox:archiveresult:majority_status:{':'.join(sorted(statuses))}"
+        cached_ids = cache.get(cache_key)
+        if cached_ids is not None:
+            return tuple(cached_ids)
+
+        snapshot_ids = tuple(
+            str(snapshot_id) for snapshot_id in cls.snapshot_ids_with_majority_status(statuses).values_list("snapshot_id", flat=True)
+        )
+        cache.set(cache_key, snapshot_ids, timeout=timeout)
+        return snapshot_ids
+
+    @classmethod
+    def clear_majority_status_cache(cls) -> None:
+        cache.delete_many(
+            [
+                *(f"archivebox:archiveresult:majority_status:{status}" for status in cls.StatusChoices.values),
+                f"archivebox:archiveresult:majority_status:{':'.join(sorted((cls.StatusChoices.BACKOFF, cls.StatusChoices.QUEUED)))}",
+            ],
+        )
+
    # UUID primary key (migrated from integer in 0029)
    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
    created_at = models.DateTimeField(default=timezone.now, db_index=True)
@@ -3609,7 +3731,6 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
    class Meta(
        ModelWithDeleteAfter.Meta,
        ModelWithOutputDir.Meta,
-        ModelWithConfig.Meta,
        ModelWithNotes.Meta,
    ):
        app_label = "core"
@@ -3644,14 +3765,15 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
        return "\n".join(self._format_output_line_for_display(line) for line in str(self.output_str or "").splitlines())

    def get_delete_after_config_value(self):
-        return get_config(archiveresult=self).DELETE_AFTER
+        snapshot = self.snapshot
+        from archivebox.config.common import resolve_delete_after_config_value
+
+        return resolve_delete_after_config_value(snapshot.config, snapshot.crawl.config)

    @classmethod
    def missing_delete_at_candidates(cls):
        return cls.objects.filter(delete_at__isnull=True).filter(
-            Q(config__has_key="DELETE_AFTER")
-            | Q(snapshot__config__has_key="DELETE_AFTER")
-            | Q(snapshot__crawl__config__has_key="DELETE_AFTER"),
+            Q(snapshot__config__has_key="DELETE_AFTER") | Q(snapshot__crawl__config__has_key="DELETE_AFTER"),
        )

    @property
@@ -3659,12 +3781,21 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
        """Convenience property to access the user who created this archive result via its snapshot's crawl."""
        return self.snapshot.crawl.created_by

-    def to_json(self) -> dict:
+    def to_json(self, *, snapshot_output_dir: Path | None = None) -> dict:
        """
        Convert ArchiveResult model instance to a JSON-serializable dict.
        """
        from archivebox.config import VERSION

+        process = self.process_record
+        pwd = (
+            process.pwd
+            if process and process.pwd
+            else str((snapshot_output_dir / self.plugin) if snapshot_output_dir is not None else self.output_dir)
+        )
+        cmd = process.cmd if process else []
+        cmd_version = process.cmd_version if process else ""
+
        record = {
            "type": "ArchiveResult",
            "schema_version": VERSION,
@@ -3686,13 +3817,12 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
            record["output_size"] = self.output_size
        if self.output_mimetypes:
            record["output_mimetypes"] = self.output_mimetypes
-        if self.pwd:
-            record["pwd"] = self.pwd
-        if self.cmd:
-            record["cmd"] = self.cmd
-        if self.cmd_version:
-            record["cmd_version"] = self.cmd_version
-        process = self.process_record
+        if pwd:
+            record["pwd"] = pwd
+        if cmd:
+            record["cmd"] = cmd
+        if cmd_version:
+            record["cmd_version"] = cmd_version
        if process:
            record["process_id"] = str(process.id)
        return record
@@ -3752,21 +3882,51 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
            or "snapshot_id" in update_fields
        )
        old_snapshot_id = None
+        old_output_size = 0
        if refresh_snapshot_size and not is_new:
-            old_snapshot_id = type(self).objects.filter(pk=self.pk).values_list("snapshot_id", flat=True).first()
-
-        update_fields = kwargs.get("update_fields")
-        if self.delete_at is None:
-            self.set_delete_at_from_config()
-            if self.delete_at is not None and update_fields is not None:
-                kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "delete_at"]))
+            old_values = type(self).objects.filter(pk=self.pk).values("snapshot_id", "output_size").first()
+            if old_values:
+                old_snapshot_id = old_values["snapshot_id"]
+                old_output_size = int(old_values["output_size"] or 0)

+        # ArchiveResult rows are updated on every plugin event. Resolving
+        # DELETE_AFTER here is deceptively expensive because the effective
+        # value lives on Snapshot/Crawl config, so a save of an already-loaded
+        # result can still materialize parent objects and parse config. The
+        # orchestrator owns the repair pass for these rows instead: it fills
+        # missing delete_at values from fresh Snapshot/Crawl config when the
+        # queue is idle, outside the hook-result write hot path.
        # Skip ModelWithOutputDir.save() to avoid creating index.json in plugin directories
        # Call the Django Model.save() directly instead
        models.Model.save(self, *args, **kwargs)
        if refresh_snapshot_size:
-            snapshot_ids = {snapshot_id for snapshot_id in (old_snapshot_id, self.snapshot_id) if snapshot_id}
-            transaction.on_commit(lambda: type(self).refresh_snapshot_output_sizes(snapshot_ids))
+            current_snapshot_id = self.snapshot_id
+            snapshot_ids = {snapshot_id for snapshot_id in (old_snapshot_id, current_snapshot_id) if snapshot_id}
+            current_output_size = int(self.output_size or 0)
+            if len(snapshot_ids) > 1:
+                # Moving an ArchiveResult between Snapshots is rare and cannot
+                # be represented as a single delta on one parent row. Keep the
+                # conservative aggregate fallback for that shape.
+                transaction.on_commit(lambda: type(self).refresh_snapshot_output_sizes(snapshot_ids))
+            elif current_snapshot_id:
+                # Hook-result projection updates ArchiveResult rows at very
+                # high frequency during indexing. Re-aggregating every sibling
+                # row for the parent Snapshot on each save turns those short
+                # writes into a table-scan hot path. For the common case where
+                # the result stays attached to the same Snapshot, the persisted
+                # parent total is exactly the old total plus this row's size
+                # delta; F() keeps that update atomic with concurrent result
+                # saves for other plugins on the same Snapshot.
+                size_delta = current_output_size if is_new else current_output_size - old_output_size
+                if size_delta:
+                    transaction.on_commit(
+                        lambda: Snapshot.objects.filter(pk=current_snapshot_id).update(
+                            output_size=F("output_size") + size_delta,
+                            modified_at=timezone.now(),
+                        ),
+                    )
+        if is_new or update_fields is None or "status" in update_fields or "snapshot" in update_fields or "snapshot_id" in update_fields:
+            transaction.on_commit(type(self).clear_majority_status_cache)

        # if is_new:
        #     from archivebox.misc.logging_util import log_worker_event
@@ -3788,6 +3948,7 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
        deleted = super().delete(*args, **kwargs)
        if snapshot_id:
            transaction.on_commit(lambda: type(self).refresh_snapshot_output_sizes({snapshot_id}))
+            transaction.on_commit(type(self).clear_majority_status_cache)
        return deleted

    @staticmethod
@@ -4495,7 +4656,8 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
    def _url_passes_filters(self, url: str) -> bool:
        """Check if URL passes URL_ALLOWLIST and URL_DENYLIST config filters.

-        Uses proper config hierarchy: defaults -> file -> env -> machine -> user -> crawl -> snapshot
+        Uses the centralized config resolver so frozen crawl/snapshot values
+        and live Machine/Persona execution values apply in their scoped order.
        """
        return self.snapshot.crawl.url_passes_filters(url, snapshot=self.snapshot)

@@ -1,13 +1,21 @@
 from __future__ import annotations

+from pathlib import Path
+
 from django.utils import timezone
 from rich.console import Console


+def _is_signal_interrupted_exit(exit_code: int | None) -> bool:
+    return exit_code is not None and (exit_code < 0 or exit_code >= 128)
+
+
 def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int]:
    from archivebox.crawls.models import Crawl
    from archivebox.core.models import ArchiveResult, Snapshot
+    from archivebox.services.archive_result_service import _collect_output_metadata
    from archivebox.machine.models import Process
+    from django.core.exceptions import ValidationError
    from django.db.models import Exists, OuterRef, Q, Subquery, Value
    from django.db.models.functions import Coalesce

@@ -20,7 +28,9 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
        "crawls_queued_without_retry_at": 0,
        "snapshots_queued_without_retry_at": 0,
        "archiveresults_backoff": 0,
+        "snapshots_queued_plugin_rows_waiting_on_stale_lease": 0,
        "archiveresults_started_without_running_process": 0,
+        "archiveresults_missing_for_orphaned_hook_processes": 0,
        "snapshots_started_without_running_results": 0,
        "crawls_started_with_due_snapshots": 0,
        "crawls_started_waiting_on_future_snapshots": 0,
@@ -73,6 +83,24 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
        retry_at__isnull=True,
    ).update(retry_at=now, modified_at=now)
    cleaned["archiveresults_backoff"] = backoff_results.update(status=ArchiveResult.StatusChoices.QUEUED, modified_at=now)
+    # Targeted plugin rows on final/paused Snapshots are scheduled through the
+    # parent Snapshot.retry_at. retry_at=NULL is the normal idle marker for a
+    # sealed Snapshot and must not be interpreted as queued work just because
+    # old/synthetic ArchiveResult rows exist. If takeover kills the runner
+    # after it leases the Snapshot but before queued ArchiveResult rows finish,
+    # the rows remain QUEUED while retry_at sits in the future. Recovery runs
+    # only after this runner has won the single-runner gate, so it can safely
+    # unlock those stale plugin leases for immediate processing instead of
+    # waiting out the previous owner's full lock timeout.
+    queued_plugin_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.QUEUED)
+    cleaned["snapshots_queued_plugin_rows_waiting_on_stale_lease"] = (
+        Snapshot.objects.filter(
+            id__in=queued_plugin_results.values("snapshot_id"),
+            status__in=[Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED],
+        )
+        .filter(retry_at__gt=now)
+        .update(retry_at=now, modified_at=now)
+    )
    # Impossible state repair: STARTED ArchiveResults without a live Process
    # have no owner left to emit completion. Requeue only the result row; the
    # snapshot/crawl schedulers will pick up normal retry processing.
@@ -81,6 +109,76 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
        process=None,
        modified_at=now,
    )
+    orphaned_hook_processes = Process.objects.filter(
+        process_type=Process.TypeChoices.HOOK,
+        archiveresult__isnull=True,
+    ).exclude(status=Process.StatusChoices.RUNNING)
+    for process in orphaned_hook_processes.only("id", "pwd", "cmd", "process_type", "status"):
+        hook_script_name = process.hook_script_name
+        if not hook_script_name or not process.pwd:
+            continue
+        plugin_dir = Path(process.pwd)
+        try:
+            # Old or synthetic hook Process rows can point at arbitrary paths.
+            # Only paths whose parent directory is a valid Snapshot id can be
+            # reconstructed into ArchiveResult rows.
+            snapshot = Snapshot.objects.filter(id=plugin_dir.parent.name).first()
+        except ValidationError:
+            continue
+        if snapshot is None:
+            continue
+        result, created = ArchiveResult.objects.get_or_create(
+            snapshot=snapshot,
+            plugin=plugin_dir.name,
+            hook_name=Path(hook_script_name).stem,
+            defaults={
+                "status": ArchiveResult.StatusChoices.QUEUED,
+            },
+        )
+        if result.status == ArchiveResult.StatusChoices.QUEUED:
+            requeue_snapshot = False
+            # A runner can die after the hook Process exits but before the
+            # ProcessCompletedEvent projector links/finalizes ArchiveResult.
+            # Reconstruct only that exact hook row from the durable Process row.
+            output_files, output_size, output_mimetypes = _collect_output_metadata(plugin_dir)
+            result.process = process
+            if _is_signal_interrupted_exit(process.exit_code):
+                # The owning runner died or was asked to stop while the hook was
+                # still active. Keep the work item queued so takeover retries the
+                # same hook; treating an unknown signal exit as success would
+                # silently skip unfinished side effects.
+                result.output_files = {}
+                result.output_size = 0
+                result.output_mimetypes = ""
+                result.output_str = ""
+                result.status = ArchiveResult.StatusChoices.QUEUED
+                requeue_snapshot = True
+            else:
+                result.output_files = output_files
+                result.output_size = output_size
+                result.output_mimetypes = output_mimetypes
+                result.output_str = process.stderr if process.exit_code not in (0, None) else ""
+                result.status = (
+                    ArchiveResult.StatusChoices.FAILED
+                    if process.exit_code not in (0, None)
+                    else (ArchiveResult.StatusChoices.SUCCEEDED if output_files else ArchiveResult.StatusChoices.NORESULTS)
+                )
+            result.save(
+                update_fields=[
+                    "process",
+                    "output_files",
+                    "output_size",
+                    "output_mimetypes",
+                    "output_str",
+                    "status",
+                    "modified_at",
+                ],
+            )
+            if requeue_snapshot:
+                Snapshot.objects.filter(id=snapshot.id).update(retry_at=now, modified_at=now)
+        if created:
+            cleaned["archiveresults_missing_for_orphaned_hook_processes"] += 1
+            Snapshot.objects.filter(id=snapshot.id).update(retry_at=now, modified_at=now)
    started_snapshots = Snapshot.objects.filter(status=Snapshot.StatusChoices.STARTED).filter(
        Q(retry_at__isnull=True) | Q(retry_at__gt=now),
    )
@@ -72,11 +72,14 @@ INSTALLED_APPS = [
    "archivebox.crawls",  # handles Crawl and CrawlSchedule models and management (depends on core)
    "archivebox.progressmonitor",  # live progress endpoint and admin monitor template
    "archivebox.api",  # Django-Ninja-based Rest API interfaces, config, APIToken model, etc.
+    "abx_plugins.plugins.opencode",
    # 3rd-party apps from PyPI that need to be loaded last
    "admin_data_views",  # handles rendering some convenient automatic read-only views of data in Django admin
    "django_extensions",  # provides Django Debug Toolbar (and other non-debug helpers)
 ]

+DJANGO_OBJECT_ACTIONS_DEFAULT_HTTP_METHOD = "POST"
+

 MIDDLEWARE = [
    "archivebox.core.middleware.TimezoneMiddleware",
@@ -174,9 +177,9 @@ except ImportError:

 STATIC_URL = "/static/"
 TEMPLATES_DIR_NAME = "templates"
-CUSTOM_TEMPLATES_ENABLED = os.path.isdir(CONFIG.CUSTOM_TEMPLATES_DIR) and os.access(CONFIG.CUSTOM_TEMPLATES_DIR, os.R_OK)
+CUSTOM_TEMPLATES_ENABLED = os.path.isdir(CONSTANTS.CUSTOM_TEMPLATES_DIR) and os.access(CONSTANTS.CUSTOM_TEMPLATES_DIR, os.R_OK)
 STATICFILES_DIRS = [
-    *([str(CONFIG.CUSTOM_TEMPLATES_DIR / "static")] if CUSTOM_TEMPLATES_ENABLED else []),
+    *([str(CONSTANTS.CUSTOM_TEMPLATES_DIR / "static")] if CUSTOM_TEMPLATES_ENABLED else []),
    # *[
    #     str(plugin_dir / 'static')
    #     for plugin_dir in PLUGIN_DIRS.values()
@@ -187,7 +190,7 @@ STATICFILES_DIRS = [
 ]

 TEMPLATE_DIRS = [
-    *([str(CONFIG.CUSTOM_TEMPLATES_DIR)] if CUSTOM_TEMPLATES_ENABLED else []),
+    *([str(CONSTANTS.CUSTOM_TEMPLATES_DIR)] if CUSTOM_TEMPLATES_ENABLED else []),
    # *[
    #     str(plugin_dir / 'templates')
    #     for plugin_dir in PLUGIN_DIRS.values()
@@ -334,7 +337,7 @@ STORAGES = {
        "BACKEND": "django.core.files.storage.FileSystemStorage",
        "OPTIONS": {
            "base_url": "/archive/",
-            "location": CONFIG.ARCHIVE_DIR,
+            "location": CONSTANTS.ARCHIVE_DIR,
        },
    },
    # "snapshots": {
@@ -182,6 +182,7 @@ SETTINGS_LOGGING = {
            "formatter": "rich",
            "level": "DEBUG",
            "markup": False,
+            "enable_link_path": False,
            "rich_tracebacks": False,  # Use standard Python tracebacks (no frame/box)
            "console": STDERR,
            "filters": ["noisyrequestsfilter", "daphneclosetimeout", "asynciocancelledshield", "stripansi"],
@@ -198,6 +199,7 @@ SETTINGS_LOGGING = {
        "outbound_webhooks": {
            "class": "rich.logging.RichHandler",
            "markup": False,
+            "enable_link_path": False,
            "rich_tracebacks": False,  # Use standard Python tracebacks (no frame/box)
            "formatter": "outbound_webhooks",
        },
@@ -134,7 +134,10 @@ def foreground_shutdown_signals(
        # signal for hard foreground-command shutdown. Server/update/run and
        # other non-interactive commands raise immediately so their finally
        # blocks can stop owned children without prompting.
-        if raise_on_first_signal or already_requested:
+        if already_requested:
+            os.write(sys.stdout.fileno(), f"\n[🛑] Got {sig.name} again, exiting immediately.\n".encode())
+            os._exit(130)
+        if raise_on_first_signal:
            raise KeyboardInterrupt

    try:
@@ -143,6 +146,8 @@ def foreground_shutdown_signals(
            signal.signal(sig, raise_keyboard_interrupt)
        yield state
    finally:
+        if state.signal_name and previous_active_state is not None and not previous_active_state.signal_name:
+            previous_active_state.signal_name = state.signal_name
        _active_shutdown_state = previous_active_state
        for sig, previous_handler in previous_handlers.items():
            signal.signal(sig, previous_handler)
@@ -0,0 +1,25 @@
+__package__ = "archivebox.core"
+
+from django.db.models import QuerySet
+
+
+def snapshot_status_values() -> tuple[str, ...]:
+    from archivebox.core.models import Snapshot
+
+    return tuple(Snapshot.StatusChoices.values)
+
+
+def normalize_snapshot_status(status: str | None) -> str | None:
+    value = str(status or "").strip().lower()
+    if not value:
+        return None
+
+    valid_statuses = snapshot_status_values()
+    if value not in valid_statuses:
+        raise ValueError(f"Invalid snapshot status: {status}. Expected one of: {', '.join(valid_statuses)}")
+    return value
+
+
+def filter_snapshots_by_status(queryset: QuerySet, status: str | None) -> QuerySet:
+    value = normalize_snapshot_status(status)
+    return queryset.filter(status=value) if value else queryset
@@ -114,9 +114,9 @@ def add_snapshot_counts(tags: list[Tag], snapshot_queryset: QuerySet[Snapshot] |
    queryset = SnapshotTag.objects.filter(tag_id__in=tag_ids)
    if snapshot_queryset is not None:
        queryset = queryset.filter(snapshot_id__in=snapshot_queryset.values("id"))
-    counts = {row["tag_id"]: row["num_snapshots"] for row in queryset.values("tag_id").annotate(num_snapshots=Count("snapshot_id"))}
+    counts = {int(row["tag_id"]): row["num_snapshots"] for row in queryset.values("tag_id").annotate(num_snapshots=Count("snapshot_id"))}
    for tag in tags:
-        tag.num_snapshots = counts.get(tag.pk, 0)
+        tag.num_snapshots = counts.get(int(tag.pk), 0)


 def get_tag_creator_choices() -> list[tuple[str, str]]:
@@ -246,7 +246,7 @@ def _build_snapshot_preview_map(


 def build_tag_card(tag: Tag, snapshot_previews: list[dict[str, Any]] | None = None) -> dict[str, Any]:
-    count = getattr(tag, "num_snapshots", None)
+    count = tag.__dict__.get("num_snapshots")
    if count is None:
        count = tag.snapshot_set.count()
    return {
@@ -18,10 +18,18 @@ def runtime_stack_owner_types():
    from archivebox.machine.models import Process

    return (
-        Process.TypeChoices.UPDATE,
        Process.TypeChoices.SERVER,
        Process.TypeChoices.ORCHESTRATOR,
+    )
+
+
+def foreground_runner_owner_types():
+    from archivebox.machine.models import Process
+
+    return (
+        Process.TypeChoices.SERVER,
        Process.TypeChoices.ADD,
+        Process.TypeChoices.UPDATE,
    )


@@ -70,15 +78,13 @@ def runtime_stack_owner(*, data_dir: str | Path, exclude_id=None):
    if exclude_id is not None:
        base_qs = base_qs.exclude(id=exclude_id)

-    top_level_types = (
-        Process.TypeChoices.UPDATE,
-        Process.TypeChoices.SERVER,
-        Process.TypeChoices.ADD,
-    )
    for qs in (
-        base_qs.filter(process_type__in=top_level_types),
+        # Only server parents own HTTP runtime leadership. Foreground add/update
+        # commands can own runner/sonic components, but server startup must never
+        # wait behind them before binding Daphne.
+        base_qs.filter(process_type=Process.TypeChoices.SERVER),
        # A foreground `archivebox run` process is allowed to own the runtime
-        # stack when no server/update/add parent is alive. A runner launched by
+        # stack when no server/add parent is alive. A runner launched by
        # supervisord is only a child worker; after its parent is killed it must
        # not keep stealing leadership from the next foreground command.
        base_qs.filter(process_type=Process.TypeChoices.ORCHESTRATOR).exclude(parent__process_type=Process.TypeChoices.SUPERVISORD),
@@ -95,6 +101,30 @@ def command_owns_runtime_stack(command, *, data_dir: str | Path) -> bool:
    return bool(owner and owner.id == command.id)


+def foreground_runner_owner(*, data_dir: str | Path, exclude_id=None):
+    from archivebox.machine.models import Machine, Process
+
+    machine = Machine.current()
+    qs = Process.objects.filter(
+        machine=machine,
+        status=Process.StatusChoices.RUNNING,
+        pwd=str(data_dir),
+        process_type__in=foreground_runner_owner_types(),
+    )
+    if exclude_id is not None:
+        qs = qs.exclude(id=exclude_id)
+    for proc in qs.order_by("-created_at", "-modified_at").iterator(chunk_size=50):
+        if proc.is_running:
+            return proc
+        proc.mark_exited(exit_code=proc.exit_code if proc.exit_code is not None else 0)
+    return None
+
+
+def command_owns_foreground_runner(command, *, data_dir: str | Path) -> bool:
+    owner = foreground_runner_owner(data_dir=data_dir)
+    return bool(owner and owner.id == command.id)
+
+
 def runtime_stack_component_label(*, owner=None, data_dir: str | Path) -> str:
    try:
        from archivebox.config.common import get_config
@@ -108,9 +138,9 @@ def runtime_stack_component_label(*, owner=None, data_dir: str | Path) -> str:
    if not names and owner is not None:
        from archivebox.machine.models import Process

-        if owner.process_type in {Process.TypeChoices.SERVER, Process.TypeChoices.ADD}:
+        if owner.process_type == Process.TypeChoices.SERVER:
            names = ["orchestrator", "server"]
-        elif owner.process_type in {Process.TypeChoices.UPDATE, Process.TypeChoices.ORCHESTRATOR}:
+        elif owner.process_type == Process.TypeChoices.ORCHESTRATOR:
            names = ["orchestrator"]

    return ", ".join(dict.fromkeys(names)) or "runtime stack"
@@ -292,3 +322,26 @@ def standby_until_runtime_stack_needed(command, *, data_dir: str | Path, interva
    command.modified_at = timezone.now()
    command.save(update_fields=["modified_at"])
    return {"resumed": announced, "previous_owner_pid": previous_owner_pid}
+
+
+def standby_until_foreground_runner_needed(command, *, data_dir: str | Path, interval: float = 2.0) -> dict[str, object]:
+    from archivebox.workers.supervisord_util import reap_foreground_supervisord_process
+
+    announced = False
+    previous_owner_pid = None
+    while not command_owns_foreground_runner(command, data_dir=data_dir):
+        reap_foreground_supervisord_process()
+        if not announced:
+            owner = foreground_runner_owner(data_dir=data_dir)
+            owner_pid = owner.pid if owner else "unknown"
+            previous_owner_pid = owner_pid
+            rprint(
+                f"[yellow][*] A newer archivebox process took over the orchestrator, sonic "
+                f"(pid={owner_pid}). Work will continue there, and will resume here if that process exits and work still remains.[/yellow]",
+                file=sys.stderr,
+            )
+            announced = True
+        time.sleep(interval)
+    command.modified_at = timezone.now()
+    command.save(update_fields=["modified_at"])
+    return {"resumed": announced, "previous_owner_pid": previous_owner_pid}
@@ -76,6 +76,12 @@ def _normalize_output_files(output_files: Any) -> dict[str, dict[str, Any]]:
    return {}


+def _snapshot_id(value: Any) -> Any:
+    from archivebox.core.models import Snapshot
+
+    return value.id if isinstance(value, Snapshot) else value
+
+
 def _coerce_output_file_size(value: Any) -> int | None:
    try:
        return max(int(value or 0), 0)
@@ -85,7 +91,7 @@ def _coerce_output_file_size(value: Any) -> int | None:

 def _count_media_files(result) -> int:
    try:
-        output_files = _normalize_output_files(getattr(result, "output_files", None) or {})
+        output_files = _normalize_output_files(result.output_files or {})
    except Exception:
        output_files = {}

@@ -121,7 +127,7 @@ def _list_media_files(result) -> list[dict]:
    except Exception:
        return media_files

-    output_files = _normalize_output_files(getattr(result, "output_files", None) or {})
+    output_files = _normalize_output_files(result.output_files or {})
    candidates: list[tuple[Path, int | None]] = []
    if output_files:
        for path, metadata in output_files.items():
@@ -300,7 +306,7 @@ def result_list(context, cl):
    """
    num_sorted_fields = 0
    request = context.get("request")
-    config = getattr(request, "archivebox_config", None) or context.get("CONFIG")
+    config = request.__dict__.get("archivebox_config") if request is not None else context.get("CONFIG")
    results = cl.result_list
    if config is not None:
        for obj in results:
@@ -443,7 +449,7 @@ def _unconfigured_banner_context(request) -> dict:
    # operator's clipboard already aligned with subdomain routing. The config
    # parser strips the leading ``*.`` so users can paste it verbatim.
    suggested_base_url = f"{scheme}://*.{canonical_host}" if canonical_host else ""
-    user = getattr(request, "user", None)
+    user = request.user
    is_superuser = bool(user and user.is_authenticated and user.is_superuser)
    machine_admin_url = ""
    if is_superuser:
@@ -496,19 +502,19 @@ def public_base_url(context) -> str:

@register.simple_tag(takes_context=True)
 def snapshot_base_url(context, snapshot) -> str:
-    snapshot_id = getattr(snapshot, "id", snapshot)
+    snapshot_id = _snapshot_id(snapshot)
    return get_snapshot_base_url(str(snapshot_id), request=context.get("request"), config=context.get("CONFIG"))


@register.simple_tag(takes_context=True)
 def snapshot_url(context, snapshot, path: str = "") -> str:
-    snapshot_id = getattr(snapshot, "id", snapshot)
+    snapshot_id = _snapshot_id(snapshot)
    return build_snapshot_url(str(snapshot_id), path, request=context.get("request"), config=context.get("CONFIG"))


@register.simple_tag(takes_context=True)
 def snapshot_preview_url(context, snapshot, path: str = "") -> str:
-    snapshot_id = getattr(snapshot, "id", snapshot)
+    snapshot_id = _snapshot_id(snapshot)
    return _build_snapshot_preview_url(str(snapshot_id), path, request=context.get("request"), config=context.get("CONFIG"))


@@ -538,16 +544,18 @@ def plugin_card(context, result) -> str:
        - output_path: Path to output relative to snapshot dir (from embed_path())
        - plugin: Plugin base name
    """
-    if result is None or not hasattr(result, "plugin"):
+    from archivebox.core.models import ArchiveResult
+
+    if result is None or not isinstance(result, ArchiveResult):
        return ""

    plugin = get_plugin_name(result.plugin)
    template_str = get_plugin_template(plugin, "card")

    # Use embed_path() for the display path
-    raw_output_path = result.embed_path() if hasattr(result, "embed_path") else ""
+    raw_output_path = result.embed_path() or ""
    output_url = build_snapshot_url(
-        str(getattr(result, "snapshot_id", "")),
+        str(result.snapshot_id),
        raw_output_path or "",
        request=context.get("request"),
        config=context.get("CONFIG"),
@@ -558,7 +566,7 @@ def plugin_card(context, result) -> str:
    media_file_count = _count_media_files(result) if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl") else 0
    media_files = _list_media_files(result) if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl") else []
    if media_files:
-        snapshot_id = str(getattr(result, "snapshot_id", ""))
+        snapshot_id = str(result.snapshot_id)
        request = context.get("request")
        config = context.get("CONFIG")
        for item in media_files:
@@ -592,7 +600,7 @@ def plugin_card(context, result) -> str:
        pass

    if force_text_preview:
-        preview = _render_text_file_preview(getattr(result, "snapshot_dir", None), raw_output_path, plugin, icon_html)
+        preview = _render_text_file_preview(result.snapshot_dir, raw_output_path, plugin, icon_html)
        if preview:
            return mark_safe(preview)

@@ -608,7 +616,7 @@ def plugin_card(context, result) -> str:
 def output_card(snapshot, output_path: str, plugin: str) -> str:
    plugin_name = get_plugin_name(plugin)
    icon_html = get_plugin_icon(plugin_name)
-    preview = _render_text_file_preview(getattr(snapshot, "output_dir", None), output_path, plugin_name, icon_html)
+    preview = _render_text_file_preview(snapshot.output_dir, output_path, plugin_name, icon_html)
    if preview:
        return mark_safe(preview)

@@ -624,7 +632,9 @@ def plugin_full(context, result) -> str:

    Usage: {% plugin_full result %}
    """
-    if result is None or not hasattr(result, "plugin"):
+    from archivebox.core.models import ArchiveResult
+
+    if result is None or not isinstance(result, ArchiveResult):
        return ""

    plugin = get_plugin_name(result.plugin)
@@ -634,14 +644,13 @@ def plugin_full(context, result) -> str:
        return ""

    raw_output_path = ""
-    if hasattr(result, "embed_path_db"):
-        raw_output_path = result.embed_path_db() or ""
-    if not raw_output_path and hasattr(result, "embed_path"):
+    raw_output_path = result.embed_path_db() or ""
+    if not raw_output_path:
        raw_output_path = result.embed_path() or ""
    if _is_root_snapshot_output_path(raw_output_path):
        return ""
    output_url = build_snapshot_url(
-        str(getattr(result, "snapshot_id", "")),
+        str(result.snapshot_id),
        raw_output_path,
        request=context.get("request"),
        config=context.get("CONFIG"),
@@ -685,7 +694,7 @@ def api_token(context) -> str:
    from archivebox.api.auth import get_or_create_api_token

    request = context.get("request")
-    user = getattr(request, "user", None)
+    user = request.user
    if not user or not user.is_authenticated:
        return ""

@@ -19,6 +19,7 @@ from archivebox.core.views import (
    SnapshotView,
    SnapshotPathView,
    SnapshotReplayView,
+    SnapshotReplayAuthView,
    OriginalDomainReplayView,
    PublicIndexView,
    AddView,
@@ -26,6 +27,7 @@ from archivebox.core.views import (
    HealthCheckView,
 )
 from archivebox.progressmonitor.views import live_progress_view
+from abx_plugins.plugins.opencode.views import opencode_proxy_view


 # GLOBAL_CONTEXT doesn't work as-is, disabled for now: https://github.com/ArchiveBox/ArchiveBox/discussions/1306
@@ -41,6 +43,8 @@ urlpatterns = [
    path("robots.txt", static.serve, {"document_root": CONSTANTS.STATIC_DIR, "path": "robots.txt"}),
    path("favicon.ico", static.serve, {"document_root": CONSTANTS.STATIC_DIR, "path": "favicon.ico"}),
    path("docs/", RedirectView.as_view(url="https://github.com/ArchiveBox/ArchiveBox/wiki"), name="Docs"),
+    re_path(r"^admin/agent/?(?=$|opencode)", include("abx_plugins.plugins.opencode.urls")),
+    re_path(r"^(?P<path>assets/.*)$", opencode_proxy_view, name="opencode-assets"),
    path("public/", PublicIndexView.as_view(), name="public-index"),
    path("public.html", RedirectView.as_view(url="/public/"), name="public-index-html"),
    path("archive/", RedirectView.as_view(url="/")),
@@ -65,6 +69,7 @@ urlpatterns = [
        name="snapshot-path-nodate",
    ),
    path("admin/core/snapshot/add/", RedirectView.as_view(url="/add/")),
+    path("admin/core/snapshot/replay-auth/", SnapshotReplayAuthView.as_view(), name="snapshot-replay-auth"),
    path("add/", AddView.as_view(), name="add"),
    # ``query_string=True`` preserves the ``?next=…`` param that Django's
    # auth/login mixins append, so e.g. ``UserPassesTestMixin`` redirecting
@@ -5,9 +5,7 @@ import os
 import posixpath
 from glob import glob, escape
 from django.utils import timezone
-import inspect
 from typing import cast
-from collections.abc import Callable
 from pathlib import Path
 from urllib.parse import quote, urlparse

@@ -20,7 +18,11 @@ from django.views.generic.list import ListView
 from django.views.generic import FormView
 from django.db.models import Q
 from django.contrib import messages
+from django.conf import settings
+from django.contrib.auth import HASH_SESSION_KEY, SESSION_KEY, get_user_model
 from django.contrib.auth.mixins import UserPassesTestMixin
+from django.contrib.sessions.models import Session
+from django.core import signing
 from django.views.decorators.csrf import csrf_exempt
 from django.utils.decorators import method_decorator

@@ -30,7 +32,19 @@ from admin_data_views.utils import render_with_table_view, render_with_item_view
 from abx_plugins.plugins.archivewebpage import replay_preview as archivewebpage_replay

 from archivebox.config import CONSTANTS, CONSTANTS_CONFIG, VERSION
-from archivebox.config.common import SENSITIVE_CONFIG_VALUE_REDACTED, get_config, get_all_configs, redact_sensitive_config
+from archivebox.config.common import (
+    SENSITIVE_CONFIG_VALUE_REDACTED,
+    find_config_default,
+    find_config_section,
+    find_config_source,
+    find_config_type,
+    get_config,
+    get_all_configs,
+    get_request_config,
+    _plugin_config_properties,
+    redact_sensitive_config,
+)
+from archivebox.config.common import PLUGIN_CONFIG_SCHEMAS
 from archivebox.config.configset import BaseConfigSet
 from archivebox.misc.paginators import CountlessPaginator
 from archivebox.misc.util import (
@@ -80,16 +94,6 @@ from archivebox.plugins.views import get_config_definition_link
 from archivebox.progressmonitor.views import live_progress_view, progress_endpoint


-def _get_request_config(request: HttpRequest, *, resolve_plugins: bool = False):
-    request_config = getattr(request, "archivebox_config", None)
-    request_config_resolves_plugins = bool(getattr(request, "_archivebox_config_resolves_plugins", False))
-    if request_config is None or (resolve_plugins and not request_config_resolves_plugins):
-        request_config = get_config(resolve_plugins=resolve_plugins)
-        request.archivebox_config = request_config
-        request._archivebox_config_resolves_plugins = resolve_plugins
-    return request_config
-
-
 def _files_index_target(snapshot: Snapshot, archivefile: str | None) -> str:
    target = archivefile or ""
    if target == "index.html":
@@ -122,14 +126,133 @@ def _find_snapshot_by_ref(snapshot_ref: str) -> Snapshot | None:


 def _admin_login_redirect_or_forbidden(request: HttpRequest):
-    if _get_request_config(request).CONTROL_PLANE_ENABLED:
+    if get_request_config(request).CONTROL_PLANE_ENABLED:
        return redirect(f"/admin/login/?next={request.path}")
    return HttpResponseForbidden("ArchiveBox is running with the control plane disabled in this security mode.")


+REPLAY_AUTH_SALT = "archivebox.private-snapshot-replay"
+REPLAY_COOKIE_PREFIX = f"archivebox_replay_{CONSTANTS.COLLECTION_ID}_"
+REPLAY_GRANT_MAX_AGE = 60
+
+
+def _replay_cookie_name(snapshot: Snapshot) -> str:
+    return f"{REPLAY_COOKIE_PREFIX}{str(snapshot.id).replace('-', '')[-12:]}"
+
+
+def _clean_replay_next(path: str | None) -> str:
+    """Only allow same-snap relative replay paths; grants must never redirect off-host."""
+    path = f"/{(path or 'index.html').lstrip('/')}"
+    parsed = urlparse(path)
+    if parsed.scheme or parsed.netloc or path.startswith("//"):
+        return "/index.html"
+    return path
+
+
+def _replay_payload_is_valid(payload: dict, snapshot: Snapshot) -> bool:
+    """A replay cookie is not its own auth source; it must point at a live admin session.
+
+    Replayed pages can execute hostile JS, so admin cookies stay host-only on admin.*.
+    The snap host gets only this host-only HttpOnly cookie, and every request checks
+    that the original Django session still exists and still belongs to an active staff user.
+    Logout, session expiry, user deletion/deactivation, or password auth-hash rotation all
+    make the replay cookie inert without needing admin.* to delete a cookie on snap-*.
+    """
+    if payload.get("snapshot_id") != str(snapshot.id):
+        return False
+    try:
+        session = Session.objects.get(session_key=str(payload.get("session_key") or ""))
+        session_data = session.get_decoded()
+        user_id = str(session_data.get(SESSION_KEY) or "")
+        auth_hash = str(session_data.get(HASH_SESSION_KEY) or "")
+        user = get_user_model().objects.get(pk=user_id)
+    except Exception:
+        return False
+    return (
+        str(payload.get("user_id")) == user_id
+        and str(payload.get("auth_hash") or "") == auth_hash
+        and user.is_active
+        and user.is_staff
+        and auth_hash == user.get_session_auth_hash()
+    )
+
+
+def _has_replay_cookie(request: HttpRequest, snapshot: Snapshot) -> bool:
+    value = request.COOKIES.get(_replay_cookie_name(snapshot))
+    if not value:
+        return False
+    try:
+        payload = signing.loads(value, salt=REPLAY_AUTH_SALT, max_age=settings.SESSION_COOKIE_AGE)
+    except signing.BadSignature:
+        return False
+    return isinstance(payload, dict) and _replay_payload_is_valid(payload, snapshot)
+
+
+def _private_snapshot_auth_redirect(request: HttpRequest, snapshot: Snapshot, path: str = "", *, preserve_query: bool = True):
+    next_path = _clean_replay_next(path or "index.html")
+    if preserve_query and request.META.get("QUERY_STRING"):
+        next_path = f"{next_path}?{request.META['QUERY_STRING']}"
+    target = build_admin_url(
+        f"/admin/core/snapshot/replay-auth/?snapshot={snapshot.id}&next={quote(next_path, safe='')}",
+        request=request,
+    )
+    return redirect(target)
+
+
+def _replay_auth_response(request: HttpRequest, snapshot: Snapshot):
+    try:
+        payload = signing.loads(str(request.GET.get("grant") or ""), salt=REPLAY_AUTH_SALT, max_age=REPLAY_GRANT_MAX_AGE)
+    except signing.BadSignature:
+        return _private_snapshot_auth_redirect(request, snapshot, "index.html", preserve_query=False)
+
+    if not isinstance(payload, dict) or not _replay_payload_is_valid(payload, snapshot):
+        return _private_snapshot_auth_redirect(request, snapshot, "index.html", preserve_query=False)
+
+    cookie_value = signing.dumps(payload, salt=REPLAY_AUTH_SALT)
+    response = redirect(_clean_replay_next(request.GET.get("next")))
+    response.set_cookie(
+        _replay_cookie_name(snapshot),
+        cookie_value,
+        max_age=settings.SESSION_COOKIE_AGE,
+        secure=request.is_secure(),
+        httponly=True,
+        samesite="Lax",
+    )
+    return response
+
+
+class SnapshotReplayAuthView(View):
+    """Admin-only handoff that lets a snap host mint its own replay cookie.
+
+    admin.* cannot set a host-only cookie for snap-* (browsers forbid that), and
+    widening the real Django session cookie to *.archivebox.localhost would let XSS
+    in replayed pages hit the admin UI. Instead admin.* proves the user is logged in
+    with a short URL grant, then snap-* validates it and sets a snap-host-only cookie.
+    """
+
+    def get(self, request: HttpRequest):
+        if not is_admin_user(request):
+            return redirect(f"{build_admin_url('/admin/login/', request=request)}?next={quote(request.get_full_path(), safe='')}")
+
+        snapshot = _find_snapshot_by_ref(str(request.GET.get("snapshot") or ""))
+        if not snapshot:
+            raise Http404
+
+        payload = {
+            "snapshot_id": str(snapshot.id),
+            "user_id": str(request.user.pk),
+            "session_key": request.session.session_key,
+            "auth_hash": request.user.get_session_auth_hash(),
+        }
+        grant = signing.dumps(payload, salt=REPLAY_AUTH_SALT)
+        next_path = _clean_replay_next(request.GET.get("next"))
+        target = build_snapshot_url(str(snapshot.id), "_auth", request=request, config=get_request_config(request))
+        return redirect(f"{target}?grant={quote(grant, safe='')}&next={quote(next_path, safe='')}")
+
+
 class HomepageView(View):
    def get(self, request):
-        request_config = _get_request_config(request)
+        request_config = get_request_config(request)
        if request.user.is_authenticated and request_config.CONTROL_PLANE_ENABLED:
            return redirect("/admin/core/snapshot/")

@@ -200,13 +323,13 @@ class SnapshotView(View):

        # Reuse the middleware-attached config; never re-bootstrap from env + plugin
        # schemas just to render a snapshot page (that pays ~30ms for no reason).
-        runtime_config = _get_request_config(request)
+        runtime_config = get_request_config(request)
        snapshot._runtime_config = runtime_config
        snapshot_permissions = get_snapshot_permissions(snapshot)
        hidden_card_plugins = {"archivedotorg", "favicon", "title"}
        outputs = [
            out
-            for out in snapshot.discover_outputs(include_filesystem_fallback=False)
+            for out in snapshot.discover_outputs(include_filesystem_fallback=True)
            if (out.get("size") or 0) > 0 and out.get("name") not in hidden_card_plugins
        ]
        archiveresults = {out["name"]: out for out in outputs}
@@ -364,7 +487,7 @@ class SnapshotView(View):
                try:
                    snapshot = Snapshot.objects.get(Q(timestamp=slug) | Q(id__startswith=slug))
                    if not can_view_snapshot(request, snapshot):
-                        return _admin_login_redirect_or_forbidden(request)
+                        return _private_snapshot_auth_redirect(request, snapshot, archivefile or "index.html")
                    canonical_base = snapshot.url_path
                    if canonical_base != snapshot.legacy_archive_path:
                        target_path = f"/{canonical_base}/{archivefile or 'index.html'}"
@@ -573,7 +696,7 @@ class SnapshotPathView(View):
        if snapshot_id:
            snapshot = _find_snapshot_by_ref(snapshot_id)
            if snapshot and not can_view_snapshot(request, snapshot):
-                return _admin_login_redirect_or_forbidden(request)
+                return _private_snapshot_auth_redirect(request, snapshot, path or "index.html")
        else:
            # fuzzy lookup by date + domain/url (most recent)
            username_lookup = "system" if username == "web" else username
@@ -732,9 +855,9 @@ def _replay_path_visible(request: HttpRequest, path: Path) -> bool:
    if not snapshot_id:
        return False
    snapshot = Snapshot.objects.filter(id=snapshot_id).select_related("crawl", "crawl__created_by").first()
-    if not snapshot or not can_view_snapshot(request, snapshot):
+    if not snapshot or (not can_view_snapshot(request, snapshot) and not _has_replay_cookie(request, snapshot)):
        return False
-    request.archivebox_config = _get_request_config(request, resolve_plugins=False)
+    request.archivebox_config = get_request_config(request, resolve_plugins=False)
    return True


@@ -845,11 +968,14 @@ def _serve_responses_path(request, responses_root: Path, rel_path: str, show_ind


 def _serve_snapshot_replay(request: HttpRequest, snapshot: Snapshot, path: str = ""):
-    request_config = _get_request_config(request, resolve_plugins=False)
+    rel_path = path or ""
+    request_config = get_request_config(
+        request,
+        resolve_plugins=rel_path.startswith("replay/") or rel_path == "replay",
+    )
    request.archivebox_config = request_config
    request.archivebox_snapshot_url = snapshot.url
    snapshot._runtime_config = request_config
-    rel_path = path or ""

    if rel_path.startswith("replay/") or rel_path == "replay":
        response = archivewebpage_replay.serve_replay_asset_response(rel_path, request_config, HttpResponse)
@@ -899,7 +1025,8 @@ def _serve_snapshot_replay(request: HttpRequest, snapshot: Snapshot, path: str =


 def _serve_original_domain_replay(request: HttpRequest, domain: str, path: str = ""):
-    request_config = _get_request_config(request)
+    request_config = get_request_config(request, resolve_plugins=False)
+    request.archivebox_config = request_config
    requested_root_index = path in ("", "index.html") or path.endswith("/")
    rel_path = path or ""
    if not rel_path or rel_path.endswith("/"):
@@ -909,13 +1036,13 @@ def _serve_original_domain_replay(request: HttpRequest, domain: str, path: str =
        raise Http404

    domain = domain.lower()
-    match = _latest_response_match(request, domain, rel_path, data_root=request_config.USERS_DIR)
+    match = _latest_response_match(request, domain, rel_path, data_root=CONSTANTS.USERS_DIR)
    if not match and "." not in Path(rel_path).name:
        index_path = f"{rel_path.rstrip('/')}/index.html"
-        match = _latest_response_match(request, domain, index_path, data_root=request_config.USERS_DIR)
+        match = _latest_response_match(request, domain, index_path, data_root=CONSTANTS.USERS_DIR)
    if not match and "." not in Path(rel_path).name:
        html_path = f"{rel_path}.html"
-        match = _latest_response_match(request, domain, html_path, data_root=request_config.USERS_DIR)
+        match = _latest_response_match(request, domain, html_path, data_root=CONSTANTS.USERS_DIR)

    show_indexes = bool(request.GET.get("files"))
    if match:
@@ -924,7 +1051,7 @@ def _serve_original_domain_replay(request: HttpRequest, domain: str, path: str =
        if response is not None:
            return response

-    responses_root = _latest_responses_root(request, domain, data_root=request_config.USERS_DIR)
+    responses_root = _latest_responses_root(request, domain, data_root=CONSTANTS.USERS_DIR)
    if responses_root:
        response = _serve_responses_path(request, responses_root, rel_path, show_indexes)
        if response is not None:
@@ -946,13 +1073,15 @@ class SnapshotHostView(View):
    """Serve snapshot directory contents on <snapshot-subdomain>.<listen_host>/<path>."""

    def get(self, request, snapshot_id: str, path: str = ""):
-        request_config = _get_request_config(request)
+        request_config = get_request_config(request)
        snapshot = _find_snapshot_by_ref(snapshot_id)

        if not snapshot:
            raise Http404
-        if not can_view_snapshot(request, snapshot):
-            return _admin_login_redirect_or_forbidden(request)
+        if path == "_auth":
+            return _replay_auth_response(request, snapshot)
+        if not can_view_snapshot(request, snapshot) and not _has_replay_cookie(request, snapshot):
+            return _private_snapshot_auth_redirect(request, snapshot, path)

        canonical_host = get_snapshot_host(str(snapshot.id), config=request_config)
        if not host_matches(request.get_host(), canonical_host):
@@ -971,8 +1100,10 @@ class SnapshotReplayView(View):
        snapshot = _find_snapshot_by_ref(snapshot_id)
        if not snapshot:
            raise Http404
-        if not can_view_snapshot(request, snapshot):
-            return _admin_login_redirect_or_forbidden(request)
+        if path == "_auth":
+            return _replay_auth_response(request, snapshot)
+        if not can_view_snapshot(request, snapshot) and not _has_replay_cookie(request, snapshot):
+            return _private_snapshot_auth_redirect(request, snapshot, path)

        return _serve_snapshot_replay(request, snapshot, path)

@@ -998,15 +1129,15 @@ class PublicIndexView(ListView):
    paginator_class = CountlessPaginator

    def get_paginate_by(self, queryset):
-        runtime_config = getattr(self, "runtime_config", None)
+        runtime_config = self.__dict__.get("runtime_config")
        if runtime_config is None:
-            self.runtime_config = runtime_config = _get_request_config(self.request, resolve_plugins=False)
+            self.runtime_config = runtime_config = get_request_config(self.request, resolve_plugins=False)
        return runtime_config.SNAPSHOTS_PER_PAGE

    def get_context_data(self, **kwargs):
-        runtime_config = getattr(self, "runtime_config", None)
+        runtime_config = self.__dict__.get("runtime_config")
        if runtime_config is None:
-            self.runtime_config = runtime_config = _get_request_config(self.request, resolve_plugins=False)
+            self.runtime_config = runtime_config = get_request_config(self.request, resolve_plugins=False)
        search_mode = get_search_mode(self.request.GET.get("search_mode"), config=runtime_config)
        search_mode_backend = get_search_mode_backend(search_mode, config=runtime_config)
        context = {
@@ -1023,7 +1154,7 @@ class PublicIndexView(ListView):
            self.request.GET.get("q")
            and get_search_mode_base(search_mode, config=runtime_config) == "deep"
            and search_mode_backend
-            and getattr(context.get("paginator"), "count", 0) == 0,
+            and context["paginator"].count == 0,
        )
        snapshots = list(context.get("object_list") or ())
        icons_by_snapshot: dict[str, set[str]] = {str(snapshot.id): set() for snapshot in snapshots}
@@ -1086,7 +1217,7 @@ class PublicIndexView(ListView):
        if not query:
            return qs

-        runtime_config = getattr(self, "runtime_config", None)
+        runtime_config = self.__dict__.get("runtime_config")
        search_mode = get_search_mode(self.request.GET.get("search_mode"), config=runtime_config)
        try:
            return apply_snapshot_search(
@@ -1105,7 +1236,7 @@ class PublicIndexView(ListView):
    def get(self, *args, **kwargs):
        if self.request.user.is_authenticated:
            return redirect("/admin/core/snapshot/")
-        if _get_request_config(self.request).PUBLIC_INDEX:
+        if get_request_config(self.request).PUBLIC_INDEX:
            response = super().get(*args, **kwargs)
            return response
        else:
@@ -1132,7 +1263,7 @@ class AddView(UserPassesTestMixin, FormView):
        return kwargs

    def test_func(self):
-        return _get_request_config(self.request).PUBLIC_ADD_VIEW or self.request.user.is_authenticated
+        return get_request_config(self.request).PUBLIC_ADD_VIEW or self.request.user.is_authenticated

    def _can_override_crawl_config(self) -> bool:
        return is_admin_user(self.request)
@@ -1146,11 +1277,11 @@ class AddView(UserPassesTestMixin, FormView):
        if not self._can_override_crawl_config():
            return {}

-        return custom_config
+        return {str(key): value for key, value in custom_config.items() if not str(key).endswith("_BINARY")}

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
-        request_config = _get_request_config(self.request, resolve_plugins=True)
+        request_config = get_request_config(self.request, resolve_plugins=True)
        required_search_plugin = f"search_backend_{request_config.SEARCH_BACKEND_ENGINE}".strip()
        can_override_crawl_config = self._can_override_crawl_config()
        plugin_configs = discover_plugin_configs() if can_override_crawl_config else {}
@@ -1252,7 +1383,7 @@ class AddView(UserPassesTestMixin, FormView):

                created_by_id = get_or_create_system_user_pk()

-        created_by_name = getattr(self.request.user, "username", "web") if self.request.user.is_authenticated else "web"
+        created_by_name = self.request.user.username if self.request.user.is_authenticated else "web"

        # 1. save the provided urls to sources/2024-11-05__23-59-59__web_ui_add_by_user_<user_pk>.txt
        sources_file = CONSTANTS.SOURCES_DIR / f"{timezone.now().strftime('%Y-%m-%d__%H-%M-%S')}__web_ui_add_by_user_{created_by_id}.txt"
@@ -1267,8 +1398,7 @@ class AddView(UserPassesTestMixin, FormView):
        config = {}
        if plugins:
            config["PLUGINS"] = plugins
-        request_user = self.request.user if self.request.user.is_authenticated else None
-        effective_config = get_config(persona=persona, user=request_user) if persona else get_config(user=request_user)
+        effective_config = get_config(persona=persona) if persona else get_config()
        if crawl_max_concurrent_snapshots != int(effective_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS):
            config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] = crawl_max_concurrent_snapshots
        if delete_after != str(effective_config.DELETE_AFTER):
@@ -1378,11 +1508,7 @@ class WebAddView(AddView):
                return redirect(f"/{snapshot.url_path}")

        request_host = (request.get_host() or "").lower()
-        if (
-            request.user.is_authenticated
-            and not _get_request_config(request).PUBLIC_ADD_VIEW
-            and host_matches(request_host, get_web_host())
-        ):
+        if request.user.is_authenticated and not get_request_config(request).PUBLIC_ADD_VIEW and host_matches(request_host, get_web_host()):
            return redirect(build_admin_url(request.get_full_path(), request=request))

        if not self.test_func():
@@ -1461,79 +1587,11 @@ class HealthCheckView(View):
        return HttpResponse("OK", content_type="text/plain", status=200)


-def find_config_section(key: str) -> str:
-    CONFIGS = get_all_configs()
-
-    if key in CONSTANTS_CONFIG:
-        return "CONSTANT"
-    matching_sections = [section_id for section_id, section in CONFIGS.items() if key in dict(section)]
-    section = matching_sections[0] if matching_sections else "DYNAMIC"
-    return section
-
-
-def find_config_default(key: str) -> str:
-    CONFIGS = get_all_configs()
-
-    if key in CONSTANTS_CONFIG:
-        return str(CONSTANTS_CONFIG[key])
-
-    default_val = None
-
-    for config in CONFIGS.values():
-        if key in dict(config):
-            default_val = type(config).model_fields[key].default
-            break
-
-    if isinstance(default_val, Callable):
-        default_val = inspect.getsource(default_val).split("lambda", 1)[-1].split(":", 1)[-1].replace("\n", " ").strip()
-        if default_val.count(")") > default_val.count("("):
-            default_val = default_val[:-1]
-    else:
-        default_val = str(default_val)
-
-    return default_val
-
-
-def find_config_type(key: str) -> str:
-    CONFIGS = get_all_configs()
-
-    for config in CONFIGS.values():
-        if key in type(config).model_fields:
-            annotation = type(config).model_fields[key].annotation
-            return getattr(annotation, "__name__", str(annotation))
-    return "str"
-
-
-def find_config_source(key: str, merged_config: dict) -> str:
-    """Determine where a config value comes from."""
-    from archivebox.machine.models import Machine
-
-    # Environment variables override all persistent config sources.
-    if key in os.environ:
-        return "Environment"
-
-    # Machine.config overrides ArchiveBox.conf.
-    try:
-        machine = Machine.current()
-        if machine.config and key in machine.config:
-            return "Machine"
-    except Exception:
-        pass
-
-    # Check if it's from archivebox.config.file
-    file_config = BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE)
-    if key in file_config:
-        return "Config File"
-
-    # Otherwise it's using the default
-    return "Default"
-
-
@render_with_table_view
 def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
    CONFIGS = get_all_configs()

-    assert getattr(request.user, "is_superuser", False), "Must be a superuser to view configuration settings."
+    assert request.user.is_superuser, "Must be a superuser to view configuration settings."

    merged_config = get_config(redact_sensitive=True)

@@ -1555,12 +1613,12 @@ def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
            rows["Type"].append(format_html("<code>{}</code>", find_config_type(key)))

            # Use merged config value (includes machine overrides)
-            actual_value = merged_config.get(key, getattr(section, key, None))
+            actual_value = merged_config.get(key, dict(section)[key])
            rows["Value"].append(mark_safe(f"<code>{actual_value}</code>"))

            # Show where the value comes from
            source = find_config_source(key, merged_config)
-            source_colors = {"Machine": "purple", "Environment": "blue", "Config File": "green", "Default": "gray"}
+            source_colors = {"Machine": "purple", "Environment": "blue", "File": "green", "Plugin Default": "teal", "Default": "gray"}
            rows["Source"].append(format_html('<code style="color: {}">{}</code>', source_colors.get(source, "gray"), source))

            rows["Default"].append(
@@ -1575,7 +1633,7 @@ def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
    for key in CONSTANTS_CONFIG.keys():
        rows["Section"].append(section)  # section.replace('_', ' ').title().replace(' Config', '')
        rows["Key"].append(ItemLink(key, key=key))
-        rows["Type"].append(format_html("<code>{}</code>", getattr(type(CONSTANTS_CONFIG[key]), "__name__", str(CONSTANTS_CONFIG[key]))))
+        rows["Type"].append(format_html("<code>{}</code>", type(CONSTANTS_CONFIG[key]).__name__))
        rows["Value"].append(format_html("<code>{}</code>", redact_sensitive_config(CONSTANTS_CONFIG).get(key)))
        rows["Source"].append(mark_safe('<code style="color: gray">Constant</code>'))
        rows["Default"].append(
@@ -1598,44 +1656,39 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont

    CONFIGS = get_all_configs()

-    assert getattr(request.user, "is_superuser", False), "Must be a superuser to view configuration settings."
+    assert request.user.is_superuser, "Must be a superuser to view configuration settings."

    merged_config = get_config(redact_sensitive=True)

    # Determine all sources for this config value
    sources_info = []

+    # Machine config
+    machine = Machine.current()
+    machine_admin_url = machine.admin_change_url
+    if machine.config and key in machine.config:
+        sources_info.append(("Machine", redact_sensitive_config(machine.config).get(key), "purple"))
+
    # Environment variable
    if key in os.environ:
        sources_info.append(("Environment", redact_sensitive_config(os.environ).get(key), "blue"))

-    # Machine config
-    machine = None
-    machine_admin_url = None
-    try:
-        machine = Machine.current()
-        machine_admin_url = f"/admin/machine/machine/{machine.id}/change/"
-        if machine.config and key in machine.config:
-            sources_info.append(("Machine", redact_sensitive_config(machine.config).get(key), "purple"))
-    except Exception:
-        pass
-
    # Config file value
    if CONSTANTS.CONFIG_FILE.exists():
        file_config = BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE)
        if key in file_config:
-            sources_info.append(("Config File", redact_sensitive_config(file_config).get(key), "green"))
+            sources_info.append(("File", redact_sensitive_config(file_config).get(key), "green"))

    # Default value
    default_val = find_config_default(key)
-    if default_val:
+    if key in _plugin_config_properties(PLUGIN_CONFIG_SCHEMAS):
+        sources_info.append(("Plugin Default", default_val, "gray"))
+    elif default_val:
        sources_info.append(("Default", default_val, "gray"))

    # Final computed value
    config_source = find_config_source(key, merged_config)
    final_value = merged_config.get(key, CONFIGS.get(key, None))
-    if config_source == "Environment":
-        final_value = get_config(include_machine=False, redact_sensitive=True).model_dump(mode="json").get(key, CONFIGS.get(key, None))
    is_redacted = final_value == SENSITIVE_CONFIG_VALUE_REDACTED

    # Build sources display
@@ -1705,11 +1758,12 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont
                <br/><br/>
                Priority order (highest to lowest):
                <ol>
-                    <li><b style="color: blue">Environment</b> - Environment variables</li>
                    <li><b style="color: purple">Machine</b> - Machine-specific overrides
                        {f'<br/><a href="{machine_admin_url}">→ Edit <code>{key}</code> in Machine.config for this server</a>' if machine_admin_url else ""}
                    </li>
-                    <li><b style="color: green">Config File</b> - data/ArchiveBox.conf</li>
+                    <li><b style="color: blue">Environment</b> - process defaults from environment variables</li>
+                    <li><b style="color: green">File</b> - data/ArchiveBox.conf</li>
+                    <li><b style="color: gray">Plugin Default</b> - Default value from plugin config.json</li>
                    <li><b style="color: gray">Default</b> - Default value from code</li>
                </ol>
                {f'<br/><b>Tip:</b> To override <code>{key}</code> on this machine, <a href="{machine_admin_url}">edit the Machine.config field</a> and add:<br/><code>{{"\\"{key}\\": "your_value_here"}}</code>' if machine_admin_url and key not in CONSTANTS_CONFIG else ""}
@@ -4,6 +4,8 @@ import json
 import re
 import hashlib
 from django import forms
+from django.db.models.manager import BaseManager
+from django.db.models.query import QuerySet
 from django.utils.html import escape
 from django.utils.safestring import mark_safe

@@ -65,10 +67,12 @@ class TagEditorWidget(forms.Widget):
        # Parse value to get list of tag names
        tags = []
        if value:
-            if hasattr(value, "all"):  # QuerySet
+            if isinstance(value, (BaseManager, QuerySet)):
                tags = sorted([tag.name for tag in value.all()])
            elif isinstance(value, (list, tuple)):
-                if value and hasattr(value[0], "name"):  # List of Tag objects
+                from archivebox.core.models import Tag
+
+                if value and isinstance(value[0], Tag):  # List of Tag objects
                    tags = sorted([tag.name for tag in value])
                else:  # List of strings or IDs
                    # Could be tag IDs from form submission
@@ -675,12 +679,14 @@ class InlineTagEditorWidget(TagEditorWidget):
        # Parse value to get list of tag dicts with id and name
        tag_data = []
        if value:
-            if hasattr(value, "all"):  # QuerySet
+            if isinstance(value, (BaseManager, QuerySet)):
                for tag in value.all():
                    tag_data.append({"id": tag.pk, "name": tag.name})
                tag_data.sort(key=lambda x: x["name"].lower())
            elif isinstance(value, (list, tuple)):
-                if value and hasattr(value[0], "name"):
+                from archivebox.core.models import Tag
+
+                if value and isinstance(value[0], Tag):
                    for tag in value:
                        tag_data.append({"id": tag.pk, "name": tag.name})
                    tag_data.sort(key=lambda x: x["name"].lower())
@@ -8,6 +8,7 @@ from django import forms
 from django.core.paginator import Paginator
 from django.http import JsonResponse, HttpRequest, HttpResponseBadRequest, HttpResponseNotAllowed
 from django.shortcuts import get_object_or_404, redirect
+from django.template.response import TemplateResponse
 from django.template.loader import render_to_string
 from django.urls import path, reverse
 from django.utils.html import escape, format_html, format_html_join
@@ -605,9 +606,7 @@ class CrawlAdminForm(forms.ModelForm):
        if commit:
            instance.save()
            instance.apply_crawl_config_filters()
-            save_m2m = getattr(self, "_save_m2m", None)
-            if callable(save_m2m):
-                save_m2m()
+            self._save_m2m()
        return instance


@@ -729,7 +728,6 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
        super().__init__(model, admin_site)
        self.crawl_admin_base_config = None
        self.stop_reason_cache = {}
-        self.persona_limit_config_cache = {}

    class Media:
        css = {"all": ("admin/crawls/crawl_change.css",)}
@@ -739,9 +737,10 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
        self.request = request
        self.crawl_admin_base_config = request.archivebox_config
        self.stop_reason_cache = {}
-        self.persona_limit_config_cache = {}
        response = super().changelist_view(request, extra_context)
-        cl = getattr(response, "context_data", {}).get("cl") if hasattr(response, "context_data") else None
+        if not isinstance(response, TemplateResponse):
+            return response
+        cl = response.context_data.get("cl")
        if cl is not None and not self.should_annotate_snapshot_counts(request):
            self.hydrate_visible_snapshot_counts(cl.result_list)
        return response
@@ -790,7 +789,6 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
        self.request = request
        self.crawl_admin_base_config = request.archivebox_config
        self.stop_reason_cache = {}
-        self.persona_limit_config_cache = {}
        crawl = self.get_object(request, object_id)
        if crawl:
            self.hydrate_visible_snapshot_counts([crawl])
@@ -945,7 +943,7 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
            updated += len(batch)
        return updated

-    @action(label="Recrawl", description="Create a new crawl with the same settings")
+    @action(label="Recrawl", description="Create a new crawl with the same settings", methods=("POST",))
    def recrawl(self, request, obj):
        """Duplicate this crawl as a new crawl with the same URLs and settings."""

@@ -982,7 +980,7 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
        if obj.pk in self.stop_reason_cache:
            return self.stop_reason_cache[obj.pk]

-        output_dir = obj.output_dir_for_config(self.crawl_admin_base_config)
+        output_dir = obj.output_dir
        config = self.limit_config_for_crawl(obj, output_dir)
        reason = obj.stop_reason(
            config=config,
@@ -994,22 +992,13 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
        return reason

    def limit_config_for_crawl(self, obj, output_dir):
-        config = {
-            "CRAWL_DIR": str(output_dir),
-            "CRAWL_MAX_URLS": self.crawl_admin_base_config.CRAWL_MAX_URLS,
-            "CRAWL_MAX_SIZE": self.crawl_admin_base_config.CRAWL_MAX_SIZE,
-            "CRAWL_TIMEOUT": self.crawl_admin_base_config.CRAWL_TIMEOUT,
-            "SNAPSHOT_MAX_SIZE": self.crawl_admin_base_config.SNAPSHOT_MAX_SIZE,
-        }
-        if obj.persona_id:
-            if obj.persona_id not in self.persona_limit_config_cache:
-                self.persona_limit_config_cache[obj.persona_id] = {
-                    key: value for key, value in obj.persona.get_derived_config().items() if key in config
-                }
-            config.update(self.persona_limit_config_cache[obj.persona_id])
-        if obj.config:
-            config.update({key: value for key, value in obj.config.items() if key in config})
-        return config
+        from archivebox.config.common import get_config
+
+        return get_config(crawl=obj).for_crawl_runtime(
+            crawl=obj,
+            persona=obj.resolve_persona(),
+            crawl_output_dir=output_dir,
+        )

    @admin.display(description="Status", ordering="status")
    def status_with_stop_reason(self, obj):
@@ -1106,12 +1095,10 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):

    @admin.display(description="Snapshots")
    def snapshots_changelist(self, obj):
-        request = getattr(self, "request", None)
+        request = self.request
        snapshot_changelist = reverse("admin:core_snapshot_changelist")
        scoped_params = {"crawl_id": str(obj.pk)}
        full_url = f"{snapshot_changelist}?{urlencode(scoped_params)}"
-        if request is None:
-            return format_html('<a class="button" href="{}">Open snapshots changelist</a>', full_url)

        snapshot_admin = self.admin_site._registry[Snapshot]
        changelist_request = copy(request)
@@ -1305,7 +1292,7 @@ class CrawlScheduleAdmin(BaseModelAdmin):
        return self.fieldsets

    def save_model(self, request, obj, form, change):
-        if not obj.created_by_id and getattr(request, "user", None) and request.user.is_authenticated:
+        if not obj.created_by_id and request.user.is_authenticated:
            obj.created_by = request.user
        super().save_model(request, obj, form, change)

@@ -1315,14 +1302,14 @@ class CrawlScheduleAdmin(BaseModelAdmin):

    @admin.display(description="# Crawls", ordering="crawl_count")
    def num_crawls(self, obj):
-        count = getattr(obj, "crawl_count", None)
+        count = obj.__dict__.get("crawl_count")
        if count is None:
            count = obj.crawl_set.count()
        return count

    @admin.display(description="# Snapshots", ordering="snapshot_count")
    def num_snapshots(self, obj):
-        count = getattr(obj, "snapshot_count", None)
+        count = obj.__dict__.get("snapshot_count")
        if count is None:
            count = Snapshot.objects.filter(crawl__schedule=obj).count()
        return count
@@ -1338,7 +1325,7 @@ class CrawlScheduleAdmin(BaseModelAdmin):
        crawl_ids = obj.crawl_set.values_list("pk", flat=True)
        return render_snapshots_list(
            Snapshot.objects.filter(crawl_id__in=crawl_ids),
-            request=getattr(self, "request", None),
+            request=self.request,
            prefix="schedule_snapshots",
        )

@@ -1,5 +1,6 @@
 import os
 import json
+import uuid

 from django.db import migrations
 from django.db.models import Q
@@ -9,31 +10,6 @@ VALID_PERMISSIONS = {"public", "unlisted", "private"}
 BATCH_SIZE = 1000


-def legacy_bool(value):
-    if value is None:
-        return None
-    normalized = str(value).strip().lower()
-    if normalized in {"1", "true", "yes", "on"}:
-        return True
-    if normalized in {"0", "false", "no", "off"}:
-        return False
-    return None
-
-
-def permissions_from_legacy_public_flags(config):
-    if str(config.get("PERMISSIONS") or "").strip():
-        return None
-    public_snapshots = legacy_bool(config.get("PUBLIC_SNAPSHOTS"))
-    public_index = legacy_bool(config.get("PUBLIC_INDEX"))
-    if public_snapshots is False:
-        return "private"
-    if public_index is False:
-        return "unlisted"
-    if public_snapshots is True or public_index is True:
-        return "public"
-    return None
-
-
 def normalize_permissions(value, default):
    value = str(value or "").strip().lower()
    return value if value in VALID_PERMISSIONS else default
@@ -59,22 +35,19 @@ def raw_base_config(apps):


 def resolve_permissions(config, default):
+    from archivebox.config.common import permissions_from_legacy_public_flags
+
    explicit = str(config.get("PERMISSIONS") or "").strip().lower()
    if explicit in VALID_PERMISSIONS:
        return explicit
    return permissions_from_legacy_public_flags(config) or default


-def model_has_config(model):
-    try:
-        model._meta.get_field("config")
-    except Exception:
-        return False
-    return True
-
-
 def id_values(pk):
-    return str(pk), getattr(pk, "hex", str(pk).replace("-", ""))
+    if isinstance(pk, uuid.UUID):
+        return str(pk), pk.hex
+    pk_str = str(pk)
+    return pk_str, pk_str.replace("-", "")


 def flush_batch(cursor, table_name, batch):
@@ -125,8 +98,6 @@ def _ensure_permissions_column(cursor):

 def hydrate_crawl_permissions(apps, schema_editor):
    Crawl = apps.get_model("crawls", "Crawl")
-    User = apps.get_model("auth", "User")
-    user_has_config = model_has_config(User)
    base_config = raw_base_config(apps)
    default_permissions = resolve_permissions(base_config, "public")
    table_name = schema_editor.quote_name(Crawl._meta.db_table)
@@ -135,17 +106,13 @@ def hydrate_crawl_permissions(apps, schema_editor):
    batch = []
    missing_permissions = Q(permissions__isnull=True) | (Q(permissions__isnull=False) & ~Q(permissions__in=VALID_PERMISSIONS))

-    for crawl in Crawl.objects.filter(missing_permissions).select_related("persona", "created_by").iterator(chunk_size=BATCH_SIZE):
+    for crawl in Crawl.objects.filter(missing_permissions).select_related("persona").iterator(chunk_size=BATCH_SIZE):
        config = dict(crawl.config or {})
        resolved = dict(base_config)
        if crawl.persona_id:
            persona_config = crawl.persona.config or {}
            if isinstance(persona_config, dict):
                resolved.update(persona_config)
-        if user_has_config:
-            user_config = crawl.created_by.config or {}
-            if isinstance(user_config, dict):
-                resolved.update(user_config)
        resolved.update(config)
        config["PERMISSIONS"] = resolve_permissions(resolved, default_permissions)
        batch.append((crawl.id, config))
@@ -1,6 +1,5 @@
 import json

-from django.conf import settings
 from django.db import migrations


@@ -19,26 +18,31 @@ def _flush_updates(Crawl, db_alias, pending):

 def freeze_existing_crawl_configs(apps, schema_editor):
    from archivebox.config.common import build_crawl_config_snapshot
-    from archivebox.personas.models import Persona
+    from archivebox.config.constants import CONSTANTS
+    from archivebox.personas.models import derive_persona_config
+
+    class PersonaConfigSnapshot:
+        def __init__(self, persona):
+            self.name = persona.name
+            self.config = dict(persona.config or {})
+
+        def get_derived_config(self):
+            return derive_persona_config(name=self.name, config=self.config, persona_dir=CONSTANTS.PERSONAS_DIR / self.name)

    Crawl = apps.get_model("crawls", "Crawl")
-    auth_app_label, auth_model_name = settings.AUTH_USER_MODEL.split(".", 1)
-    User = apps.get_model(auth_app_label, auth_model_name)
+    Persona = apps.get_model("personas", "Persona")
    db_alias = schema_editor.connection.alias
-    rows = Crawl.objects.using(db_alias).values_list("id", "persona_id", "created_by_id", "config")
-    persona_ids = {persona_id for _, persona_id, _, _ in rows if persona_id}
-    user_ids = {user_id for _, _, user_id, _ in rows if user_id}
-    personas = {persona.pk: persona for persona in Persona.objects.using(db_alias).filter(pk__in=persona_ids)}
-    users = {user.pk: user for user in User.objects.using(db_alias).filter(pk__in=user_ids)}
+    rows = Crawl.objects.using(db_alias).values_list("id", "persona_id", "config")
+    persona_ids = {persona_id for _, persona_id, _ in rows if persona_id}
+    personas = {persona.pk: PersonaConfigSnapshot(persona) for persona in Persona.objects.using(db_alias).filter(pk__in=persona_ids)}

    frozen_cache = {}
    pending = []
-    for crawl_id, persona_id, user_id, current_config in rows.iterator(chunk_size=BATCH_SIZE):
+    for crawl_id, persona_id, current_config in rows.iterator(chunk_size=BATCH_SIZE):
        current_config = dict(current_config or {})
-        cache_key = (persona_id, user_id, _config_cache_key(current_config))
+        cache_key = (persona_id, _config_cache_key(current_config))
        if cache_key not in frozen_cache:
            frozen_cache[cache_key] = build_crawl_config_snapshot(
-                user=users.get(user_id),
                persona=personas.get(persona_id),
                overrides=current_config,
            )
@@ -109,11 +109,10 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes):
        template = self.template
        label = template.label or self.label
        persona = template.persona if template.persona_id else None
-        user = template.created_by if template.created_by_id else None

        return Crawl.objects.create(
            urls=template.urls,
-            config=build_crawl_config_snapshot(user=user, persona=persona, overrides=self.config or {}),
+            config=build_crawl_config_snapshot(persona=persona, overrides=self.config or {}),
            max_depth=template.max_depth,
            tags_str=template.tags_str,
            persona_id=template.persona_id,
@@ -171,10 +170,14 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
    INACTIVE_STATES = (StatusChoices.PAUSED, StatusChoices.SEALED)

    schedule_id: uuid.UUID | None
-    sm: "CrawlMachine"

    snapshot_set: models.Manager["Snapshot"]

+    if TYPE_CHECKING:
+
+        @property
+        def sm(self) -> "CrawlMachine": ...
+
    class Meta(
        ModelWithDeleteAfter.Meta,
        ModelWithOutputDir.Meta,
@@ -197,9 +200,9 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
        return f"[...{short_id}] {first_url[:120]}"

    def get_delete_after_config_value(self):
-        from archivebox.config.common import get_config
+        from archivebox.config.common import resolve_delete_after_config_value

-        return get_config(crawl=self).DELETE_AFTER
+        return resolve_delete_after_config_value(self.config)

    def pause(self, *, save: bool = True) -> bool:
        return super().pause(save=save)
@@ -286,15 +289,14 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
        config = dict(self.config or {})
        is_new = self._state.adding or old_crawl is None
        persona = self.persona if self.persona_id else None
-        user = self.created_by if self.created_by_id else None
        if is_new:
            from archivebox.config.common import build_crawl_config_snapshot

-            config = build_crawl_config_snapshot(user=user, persona=persona, overrides=config)
+            config = build_crawl_config_snapshot(persona=persona, overrides=config)
        if str(config.get("PERMISSIONS") or "").strip().lower() not in PERMISSIONS_VALUES:
            from archivebox.config.common import get_config

-            config["PERMISSIONS"] = normalize_permissions(get_config(persona=persona, user=user, include_machine=True).PERMISSIONS)
+            config["PERMISSIONS"] = normalize_permissions(get_config(persona=persona, include_machine=True).PERMISSIONS)
        if "CRAWL_MAX_CONCURRENT_SNAPSHOTS" in config:
            raw_concurrency = config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"]
            if raw_concurrency in (None, ""):
@@ -469,11 +471,8 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
        )
        return crawl

-    def output_dir_for_config(self, runtime_config: Mapping[str, Any] | Any) -> Path:
-        """
-        Construct output directory: archive/users/{username}/crawls/{YYYYMMDD}/{domain}/{crawl-id}
-        Domain is extracted from the first URL in the crawl.
-        """
+    @property
+    def output_dir(self) -> Path:
        from archivebox.config import CONSTANTS
        from archivebox.core.models import Snapshot

@@ -486,14 +485,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
                break
        domain = Snapshot.extract_domain_from_url(first_url) if first_url else "unknown"

-        users_dir = Path(runtime_config["USERS_DIR"]) if isinstance(runtime_config, Mapping) else runtime_config.USERS_DIR
-        return users_dir / self.created_by.username / CONSTANTS.CRAWLS_DIR_NAME / date_str / domain / str(self.id)
-
-    @property
-    def output_dir(self) -> Path:
-        from archivebox.config.common import get_config
-
-        output_dir = self.output_dir_for_config(get_config(resolve_plugins=False))
+        output_dir = CONSTANTS.USERS_DIR / self.created_by.username / CONSTANTS.CRAWLS_DIR_NAME / date_str / domain / str(self.id)
        hyphen_dir = output_dir.with_name(str(uuid.UUID(hex=self.id.hex)))
        return output_dir if output_dir.exists() or not hyphen_dir.exists() else hyphen_dir

@@ -783,7 +775,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
    def _config_value(config: Mapping[str, Any] | Any, key: str, default: Any = None) -> Any:
        if isinstance(config, Mapping):
            return config.get(key, default)
-        return getattr(config, key, default)
+        return config[key] if key in config else default

    @classmethod
    def create_scheduler_row(cls, **kwargs) -> "Crawl":
@@ -794,17 +786,12 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
        kwargs.setdefault("created_at", now)
        kwargs.setdefault("modified_at", now)
        config = normalize_config_json_values(kwargs.get("config") or {})
-        user = kwargs.get("created_by")
        persona = kwargs.get("persona")
-        if user is None and kwargs.get("created_by_id"):
-            from django.contrib.auth import get_user_model
-
-            user = get_user_model().objects.filter(pk=kwargs["created_by_id"]).first()
        if persona is None and kwargs.get("persona_id"):
            from archivebox.personas.models import Persona

            persona = Persona.objects.filter(pk=kwargs["persona_id"]).first()
-        kwargs["config"] = build_crawl_config_snapshot(user=user, persona=persona, overrides=config)
+        kwargs["config"] = build_crawl_config_snapshot(persona=persona, overrides=config)
        crawl = cls(**kwargs)
        if crawl.delete_at is None:
            crawl.set_delete_at_from_config()
@@ -820,18 +807,20 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
    ) -> str:
        from abx_dl.limits import CrawlLimitState

+        if output_dir is None:
+            output_dir = self.output_dir
        if config is None:
            from archivebox.config.common import get_config

-            config = get_config(crawl=self, include_machine=False)
-        if output_dir is None:
-            output_dir = self.output_dir
+            config = get_config(crawl=self, include_machine=False).for_crawl_runtime(
+                crawl=self,
+                persona=self.resolve_persona(),
+                crawl_output_dir=output_dir,
+            )

        limits_path = output_dir / ".abx-dl" / "limits.json"
        if limits_path.exists():
-            config_with_crawl_dir = {**dict(config.items())} if isinstance(config, Mapping) else config
-            config_with_crawl_dir["CRAWL_DIR"] = str(output_dir)
-            stop_reason = CrawlLimitState.from_config(config_with_crawl_dir).get_stop_reason()
+            stop_reason = CrawlLimitState.from_config(config).get_stop_reason()
            if stop_reason:
                return stop_reason

@@ -1302,10 +1291,11 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
        from archivebox.machine.models import Binary, Machine

        def get_runtime_config():
-            config = get_config(crawl=self)
-            if persona_runtime_overrides:
-                config.update(persona_runtime_overrides)
-            return config
+            return get_config(crawl=self).for_crawl_runtime(
+                crawl=self,
+                persona=persona,
+                runtime_overrides=persona_runtime_overrides,
+            )

        system_task = self.get_system_task()
        if system_task == "archivebox://update":
@@ -434,6 +434,9 @@ class ProcessAdmin(BaseModelAdmin):
        "crawl_link",
        "cmd_display",
        "env_display",
+        "stdout_display",
+        "stderr_display",
+        "archiveresult_output_display",
        "timeout",
        "pid",
        "exit_code",
@@ -475,7 +478,7 @@ class ProcessAdmin(BaseModelAdmin):
        (
            "Output",
            {
-                "fields": ("stdout", "stderr"),
+                "fields": ("stdout_display", "stderr_display", "archiveresult_output_display"),
                "classes": ("card", "wide", "collapse"),
            },
        ),
@@ -548,6 +551,7 @@ class ProcessAdmin(BaseModelAdmin):
        label="Kill",
        description="Kill this process if it is still running",
        attrs={"class": "deletelink"},
+        methods=("POST",),
    )
    def kill_process(self, request, obj):
        self._terminate_processes(request, [obj])
@@ -590,9 +594,10 @@ class ProcessAdmin(BaseModelAdmin):

    @admin.display(description="ArchiveResult", ordering="archiveresult__plugin")
    def archiveresult_link(self, process):
-        if not hasattr(process, "archiveresult"):
+        try:
+            ar = process.archiveresult
+        except Process.archiveresult.RelatedObjectDoesNotExist:
            return "-"
-        ar = process.archiveresult
        return format_html(
            '<a href="/admin/core/archiveresult/{}/change">{} ← <code>{}</code></a>',
            ar.id,
@@ -602,9 +607,9 @@ class ProcessAdmin(BaseModelAdmin):

    @admin.display(description="Snapshot", ordering="archiveresult__snapshot__id")
    def snapshot_link(self, process):
-        ar = getattr(process, "archiveresult", None)
-        snapshot = getattr(ar, "snapshot", None)
-        if not snapshot:
+        try:
+            snapshot = process.archiveresult.snapshot
+        except Process.archiveresult.RelatedObjectDoesNotExist:
            return "-"
        return format_html(
            '<a href="/admin/core/snapshot/{}/change"><code>{}</code></a>',
@@ -614,10 +619,9 @@ class ProcessAdmin(BaseModelAdmin):

    @admin.display(description="Crawl", ordering="archiveresult__snapshot__crawl__id")
    def crawl_link(self, process):
-        ar = getattr(process, "archiveresult", None)
-        snapshot = getattr(ar, "snapshot", None)
-        crawl = getattr(snapshot, "crawl", None)
-        if not crawl:
+        try:
+            crawl = process.archiveresult.snapshot.crawl
+        except Process.archiveresult.RelatedObjectDoesNotExist:
            return "-"
        return format_html(
            '<a href="/admin/crawls/crawl/{}/change"><code>{}</code></a>',
@@ -682,7 +686,10 @@ class ProcessAdmin(BaseModelAdmin):

    @admin.display(description="Output", ordering="archiveresult__output_size")
    def output_summary(self, process):
-        output_files = getattr(getattr(process, "archiveresult", None), "output_files", {}) or {}
+        try:
+            output_files = process.archiveresult.output_files or {}
+        except Process.archiveresult.RelatedObjectDoesNotExist:
+            output_files = {}

        if isinstance(output_files, str):
            try:
@@ -736,6 +743,28 @@ class ProcessAdmin(BaseModelAdmin):
            return "-"
        return _render_copy_block(env_text, multiline=True)

+    @admin.display(description="Stdout")
+    def stdout_display(self, process):
+        if not process.stdout:
+            return "-"
+        return _render_copy_block(process.stdout, multiline=True)
+
+    @admin.display(description="Stderr")
+    def stderr_display(self, process):
+        if not process.stderr:
+            return "-"
+        return _render_copy_block(process.stderr, multiline=True)
+
+    @admin.display(description="ArchiveResult Output")
+    def archiveresult_output_display(self, process):
+        try:
+            output = process.archiveresult.output_str
+        except Process.archiveresult.RelatedObjectDoesNotExist:
+            return "-"
+        if not output:
+            return "-"
+        return _render_copy_block(output, multiline=True)
+

 def register_admin(admin_site):
    admin_site.register(Machine, MachineAdmin)
@@ -0,0 +1,130 @@
+import json
+from pathlib import Path
+
+from django.db import migrations
+from django.utils import timezone
+
+
+def _cmd_array(value):
+    if isinstance(value, list):
+        return value
+    if isinstance(value, str):
+        try:
+            parsed = json.loads(value)
+        except json.JSONDecodeError:
+            return [value] if value else []
+        return parsed if isinstance(parsed, list) else []
+    return []
+
+
+def _ensure_placeholder_iface(NetworkInterface, machine_id, hostname):
+    iface = NetworkInterface.objects.filter(machine_id=machine_id).order_by("-modified_at", "-created_at").first()
+    if iface is not None:
+        return iface
+
+    now = timezone.now()
+    return NetworkInterface.objects.create(
+        machine_id=machine_id,
+        created_at=now,
+        modified_at=now,
+        mac_address="00:00:00:00:00:00",
+        ip_public="0.0.0.0",
+        ip_local="0.0.0.0",
+        dns_server="0.0.0.0",
+        hostname=(hostname or "unknown")[:63],
+        iface="unknown",
+        isp="",
+        city="",
+        region="",
+        country="",
+    )
+
+
+def _get_or_create_binary(Binary, machine_id, reference):
+    reference = str(reference or "").strip()
+    if not reference:
+        return None
+
+    name = Path(reference).name or reference
+    qs = Binary.objects.filter(machine_id=machine_id)
+    binary = qs.filter(abspath=reference).order_by("-modified_at", "-created_at").first()
+    if binary is None:
+        binary = qs.filter(name=name).order_by("-modified_at", "-created_at").first()
+    if binary is not None:
+        return binary
+
+    now = timezone.now()
+    return Binary.objects.create(
+        machine_id=machine_id,
+        created_at=now,
+        modified_at=now,
+        name=name[:63],
+        binproviders="env",
+        overrides={},
+        binprovider="env",
+        abspath=reference[:255],
+        version="",
+        sha256="",
+        status="installed",
+        retry_at=None,
+    )
+
+
+def repair_process_binary_iface_links(apps, schema_editor):
+    Binary = apps.get_model("machine", "Binary")
+    Machine = apps.get_model("machine", "Machine")
+    NetworkInterface = apps.get_model("machine", "NetworkInterface")
+    Process = apps.get_model("machine", "Process")
+
+    machines = {machine.id: machine for machine in Machine.objects.only("id", "hostname").iterator(chunk_size=100)}
+    iface_by_machine = {}
+    binary_by_key = {}
+
+    qs = Process.objects.filter(machine_id__isnull=False).filter(binary_id__isnull=True) | Process.objects.filter(
+        machine_id__isnull=False,
+        iface_id__isnull=True,
+    )
+    for process in qs.distinct().only("id", "machine_id", "binary_id", "iface_id", "cmd").iterator(chunk_size=500):
+        update_fields = []
+        machine = machines.get(process.machine_id)
+
+        if process.iface_id is None:
+            iface = iface_by_machine.get(process.machine_id)
+            if iface is None:
+                iface = _ensure_placeholder_iface(
+                    NetworkInterface,
+                    process.machine_id,
+                    machine.hostname if machine is not None else "",
+                )
+                iface_by_machine[process.machine_id] = iface
+            process.iface_id = iface.id
+            update_fields.append("iface_id")
+
+        if process.binary_id is None:
+            cmd = _cmd_array(process.cmd)
+            reference = str(cmd[0]).strip() if cmd else ""
+            if reference:
+                key = (process.machine_id, reference)
+                binary = binary_by_key.get(key)
+                if binary is None:
+                    binary = _get_or_create_binary(Binary, process.machine_id, reference)
+                    binary_by_key[key] = binary
+                if binary is not None:
+                    process.binary_id = binary.id
+                    update_fields.append("binary_id")
+
+        if update_fields:
+            process.modified_at = timezone.now()
+            process.save(update_fields=[*update_fields, "modified_at"])
+
+
+class Migration(migrations.Migration):
+    atomic = False
+
+    dependencies = [
+        ("machine", "0019_single_active_runner_constraint"),
+    ]
+
+    operations = [
+        migrations.RunPython(repair_process_binary_iface_links, migrations.RunPython.noop),
+    ]
@@ -3,6 +3,7 @@ from __future__ import annotations
 __package__ = "archivebox.machine"

 import os
+import signal
 import sys
 import uuid
 import socket
@@ -19,6 +20,7 @@ from django.db.models import Q, QuerySet
 from django.utils import timezone
 from django.utils.functional import cached_property

+from archivebox.config import CONSTANTS
 from archivebox.config.common import rprint
 from archivebox.base_models.models import ModelWithDeleteAfter, ModelWithHealthStats, normalize_config_json_values
 from archivebox.workers.models import BaseStateMachine, ModelWithStateMachine
@@ -54,6 +56,13 @@ PROCESS_TIMEOUT_GRACE = timedelta(seconds=30)  # Extra margin before force-clean
 START_TIME_TOLERANCE = 5.0  # Seconds tolerance for start time matching


+def _default_exit_code_for_unowned_process(process_type: str) -> int:
+    # Hooks are externally visible work items. If their owning runner disappeared
+    # before recording the real exit code, retrying is safer than converting an
+    # unknown interrupted extraction into a durable success/no-result row.
+    return 128 + signal.SIGTERM if process_type == Process.TypeChoices.HOOK else 0
+
+
 def _find_existing_binary_for_reference(machine: Machine, reference: str) -> Binary | None:
    reference = str(reference or "").strip()
    if not reference:
@@ -411,14 +420,25 @@ class NetworkInterface(ModelWithHealthStats):
    def current(cls, refresh: bool = False) -> NetworkInterface:
        global _CURRENT_INTERFACE
        machine = Machine.current(refresh=refresh)
-        if _CURRENT_INTERFACE:
-            if (
-                not refresh
-                and _CURRENT_INTERFACE.machine_id == machine.id
-                and timezone.now() < _CURRENT_INTERFACE.modified_at + timedelta(seconds=NETWORK_INTERFACE_RECHECK_INTERVAL)
-            ):
+        if _CURRENT_INTERFACE and _CURRENT_INTERFACE.machine_id == machine.id:
+            if not refresh:
+                # Callers that pass refresh=False are asking for attribution to
+                # the currently known interface, not for public-IP/ISP probing.
+                # Maintenance paths create many short-lived services per run;
+                # expiring this in-memory object by age forced every crawl to
+                # hit external network APIs even though Process rows only need
+                # a stable existing FK. Active downloading paths opt into live
+                # detection with refresh=True.
                return _CURRENT_INTERFACE
-            _CURRENT_INTERFACE = None
+            if timezone.now() < _CURRENT_INTERFACE.modified_at + timedelta(seconds=NETWORK_INTERFACE_RECHECK_INTERVAL):
+                return _CURRENT_INTERFACE
+        _CURRENT_INTERFACE = None
+
+        if not refresh:
+            _CURRENT_INTERFACE = cls.objects.filter(machine=machine).order_by("-modified_at", "-created_at").first()
+            if _CURRENT_INTERFACE is not None:
+                return _CURRENT_INTERFACE
+
        net_info = get_host_network()
        lookup = dict(
            machine=machine,
@@ -442,7 +462,7 @@ class NetworkInterface(ModelWithHealthStats):
            if refresh or timezone.now() >= _CURRENT_INTERFACE.modified_at + timedelta(seconds=NETWORK_INTERFACE_RECHECK_INTERVAL):
                updates = ["modified_at"]
                for key, value in net_info.items():
-                    if getattr(_CURRENT_INTERFACE, key) != value:
+                    if _CURRENT_INTERFACE.__dict__.get(key) != value:
                        setattr(_CURRENT_INTERFACE, key, value)
                        updates.append(key)
                if len(updates) > 1:
@@ -547,9 +567,15 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):

    state_machine_name: str | None = "archivebox.machine.models.BinaryMachine"
    active_state: str = StatusChoices.QUEUED
+    warn_on_save_outside_runner = False

    objects = BinaryManager()  # pyright: ignore[reportIncompatibleVariableOverride]

+    if TYPE_CHECKING:
+
+        @property
+        def sm(self) -> BinaryMachine: ...
+
    class Meta(ModelWithHealthStats.Meta, ModelWithStateMachine.Meta):
        app_label = "machine"
        verbose_name = "Binary"
@@ -581,10 +607,7 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
        Get output directory for this binary's hook logs.
        Path: data/machines/{machine_uuid}/binaries/{binary_name}/{binary_uuid}
        """
-        from archivebox.config.common import get_config
-
-        data_dir = get_config().DATA_DIR
-        return data_dir / "machines" / str(self.machine_id) / "binaries" / self.name / str(self.id)
+        return CONSTANTS.DATA_DIR / "machines" / str(self.machine_id) / "binaries" / self.name / str(self.id)

    def to_json(self) -> dict:
        """
@@ -635,22 +658,6 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
        binary_overrides = record.get("overrides", {})
        normalized_overrides = binary_overrides if isinstance(binary_overrides, dict) else {}

-        # abx-plugins currently emits a GitHub install URL for readability-extractor,
-        # but the package is published on npm. Prefer the registry package to avoid
-        # long git-based installs in CI while still using canonical install_args.
-        if (
-            name == "readability-extractor"
-            and isinstance(normalized_overrides.get("npm"), dict)
-            and normalized_overrides["npm"].get("install_args") == ["https://github.com/ArchiveBox/readability-extractor"]
-        ):
-            normalized_overrides = {
-                **normalized_overrides,
-                "npm": {
-                    **normalized_overrides["npm"],
-                    "install_args": ["readability-extractor"],
-                },
-            }
-
        # Case 1: Already installed (from on_Crawl hooks) - has abspath AND binproviders
        # This happens when on_Crawl hooks detect already-installed binaries
        abspath = record.get("abspath")
@@ -1144,15 +1151,13 @@ class Process(ModelWithDeleteAfter, models.Model):
        return f"Process[{self.id}] {cmd_str} ({self.status})"

    def get_delete_after_config_value(self):
-        from archivebox.config.common import get_config
-
        value = self.env.get("DELETE_AFTER")
        if value not in (None, ""):
            return value
        value = (self.machine.config or {}).get("DELETE_AFTER")
        if value not in (None, ""):
            return value
-        return get_config(include_machine=False).DELETE_AFTER
+        return "0"

    @classmethod
    def missing_delete_at_candidates(cls):
@@ -1174,17 +1179,18 @@ class Process(ModelWithDeleteAfter, models.Model):
    @property
    def plugin(self) -> str:
        """Get plugin name from associated ArchiveResult (if any)."""
-        if hasattr(self, "archiveresult"):
-            # Inline import to avoid circular dependency
+        try:
            return self.archiveresult.plugin
-        return ""
+        except Process.archiveresult.RelatedObjectDoesNotExist:
+            return ""

    @property
    def hook_name(self) -> str:
        """Get hook name from associated ArchiveResult (if any)."""
-        if hasattr(self, "archiveresult"):
+        try:
            return self.archiveresult.hook_name
-        return ""
+        except Process.archiveresult.RelatedObjectDoesNotExist:
+            return ""

    def to_json(self) -> dict:
        """
@@ -1610,7 +1616,9 @@ class Process(ModelWithDeleteAfter, models.Model):
                    is_stale = True  # Process no longer exists

            if is_stale:
-                proc.mark_exited(exit_code=proc.exit_code if proc.exit_code is not None else 0)
+                proc.mark_exited(
+                    exit_code=proc.exit_code if proc.exit_code is not None else _default_exit_code_for_unowned_process(proc.process_type),
+                )
                cleaned += 1

        return cleaned
@@ -2140,8 +2148,7 @@ class Process(ModelWithDeleteAfter, models.Model):
                # TODO: Uncomment to cleanup (keeping for debugging for now)
                # self.stderr_file.unlink(missing_ok=True)

-            # Try to get exit code from proc or default to unknown
-            self.exit_code = self.exit_code if self.exit_code is not None else 0
+            self.exit_code = self.exit_code if self.exit_code is not None else _default_exit_code_for_unowned_process(self.process_type)
            if self.exit_code == -1:
                self.exit_code = 137
            self.ended_at = timezone.now()
@@ -2449,16 +2456,14 @@ class Process(ModelWithDeleteAfter, models.Model):
        """
        import subprocess
        from importlib.resources import files
-        from archivebox.config.common import get_config

        chrome_utils = files("abx_plugins.plugins.chrome").joinpath("chrome_utils.js")
        if not chrome_utils.exists():
            return 0

-        config = get_config()
        crawl_roots = [
            crawls_dir
-            for user_dir in config.USERS_DIR.iterdir()
+            for user_dir in CONSTANTS.USERS_DIR.iterdir()
            if user_dir.is_dir()
            for crawls_dir in [user_dir / "crawls"]
            if crawls_dir.is_dir()
@@ -2507,7 +2512,9 @@ class Process(ModelWithDeleteAfter, models.Model):
        # orphaned Process backlog cannot be materialized in memory at once.
        for proc in running_children.iterator(chunk_size=100):
            if not proc.is_running:
-                proc.mark_exited(exit_code=proc.exit_code if proc.exit_code is not None else 0)
+                proc.mark_exited(
+                    exit_code=proc.exit_code if proc.exit_code is not None else _default_exit_code_for_unowned_process(proc.process_type),
+                )
                cleaned += 1
                continue

@@ -2531,7 +2538,9 @@ class Process(ModelWithDeleteAfter, models.Model):
            ):
                continue

-            proc.mark_exited(exit_code=proc.exit_code if proc.exit_code is not None else 0)
+            proc.mark_exited(
+                exit_code=proc.exit_code if proc.exit_code is not None else _default_exit_code_for_unowned_process(proc.process_type),
+            )
            cleaned += 1

        if cleaned:
@@ -66,7 +66,7 @@ def check_data_folder(config=None, **config_kwargs) -> None:
    from archivebox.config.paths import create_and_chown_dir, get_or_create_working_tmp_dir, get_or_create_working_lib_dir

    config = config or get_config(**config_kwargs)
-    archive_dir = config.ARCHIVE_DIR
+    archive_dir = CONSTANTS.ARCHIVE_DIR
    archive_dir_exists = os.path.isdir(archive_dir)
    if not archive_dir_exists:
        print("[red][X] No archivebox index found in the current directory.[/red]", file=sys.stderr)
@@ -82,7 +82,7 @@ def check_data_folder(config=None, **config_kwargs) -> None:

    # Create data dir subdirs
    create_and_chown_dir(CONSTANTS.SOURCES_DIR)
-    create_and_chown_dir(config.USERS_DIR)
+    create_and_chown_dir(CONSTANTS.USERS_DIR)
    create_and_chown_dir(CONSTANTS.PERSONAS_DIR / "Default")
    create_and_chown_dir(CONSTANTS.LOGS_DIR)
    # create_and_chown_dir(CONSTANTS.CACHE_DIR)
@@ -211,11 +211,10 @@ def check_not_inside_source_dir():
    """Prevent running ArchiveBox from inside its source directory (would pollute repo with data files)."""
    cwd = Path(os.getcwd()).resolve()
    is_source_dir = (cwd / "archivebox" / "__init__.py").exists() and (cwd / "pyproject.toml").exists()
-    data_dir_set_elsewhere = os.environ.get("DATA_DIR", "").strip() and Path(os.environ["DATA_DIR"]).resolve() != cwd
    is_testing = "pytest" in sys.modules or "unittest" in sys.modules

-    if is_source_dir and not data_dir_set_elsewhere and not is_testing:
-        raise SystemExit("[!] Cannot run from source dir, set DATA_DIR or cd to a data folder first")
+    if is_source_dir and not is_testing:
+        raise SystemExit("[!] Cannot run from source dir, cd to a data folder first")


 def check_data_dir_permissions(config=None, **config_kwargs):
@@ -1,7 +1,7 @@
 """
 Database utility functions for ArchiveBox.

-Post-bootstrap: requires archivebox.config (DATA_DIR) and uses Django lazily
+Post-bootstrap: requires archivebox.config constants and uses Django lazily
 (``from django.db import ...`` inside functions). Not safe to import pre-bootstrap.
 """

@@ -17,7 +17,7 @@ from collections.abc import Callable
 from contextlib import contextmanager
 from sqlite3 import OperationalError as SQLiteOperationalError

-from archivebox.config import DATA_DIR
+from archivebox.config import CONSTANTS
 from archivebox.misc.util import enforce_types


@@ -79,7 +79,7 @@ def run_db_analyze_batch(
        return []

    next_table, *rest = remaining
-    raw_conn = getattr(connection, "connection", None)
+    raw_conn = connection.connection
    progress_handler_set = False
    if raw_conn is not None and max_seconds_per_table > 0:
        deadline = time.monotonic() + max_seconds_per_table
@@ -117,7 +117,7 @@ def compact_command(cmdline: list[str] | None, fallback: str = "") -> str:
    return " ".join([Path(parts[0]).name, *parts[1:]])[:220]


-def sqlite_lock_holders(db_path: Path = DATA_DIR / "index.sqlite3") -> list[str]:
+def sqlite_lock_holders(db_path: Path = CONSTANTS.DATABASE_FILE) -> list[str]:
    import psutil

    db_path = db_path.resolve()
@@ -146,7 +146,7 @@ def sqlite_lock_holders(db_path: Path = DATA_DIR / "index.sqlite3") -> list[str]
    return holders


-def log_sqlite_lock_holders(console: Any, *, db_path: Path = DATA_DIR / "index.sqlite3", limit: int = 8) -> None:
+def log_sqlite_lock_holders(console: Any, *, db_path: Path = CONSTANTS.DATABASE_FILE, limit: int = 8) -> None:
    holders = sqlite_lock_holders(db_path)
    if holders:
        console.print("[yellow]    DB holders:[/yellow]")
@@ -346,7 +346,7 @@ HISTORICAL_GHOST_MIGRATIONS: frozenset[tuple[str, str]] = frozenset(


@enforce_types
-def migration_state(out_dir: Path = DATA_DIR) -> tuple[list[str], list[str], dict[str, str]]:
+def migration_state(out_dir: Path = CONSTANTS.DATA_DIR) -> tuple[list[str], list[str], dict[str, str]]:
    """Cheaply compare migration files to django_migrations without invoking migrate."""
    from django.apps import apps
    from django.db import connection
@@ -373,7 +373,7 @@ def migration_state(out_dir: Path = DATA_DIR) -> tuple[list[str], list[str], dic
    loader.load_disk()
    for (app_label, migration_name), migration in loader.disk_migrations.items():
        disk_migrations.add((app_label, migration_name))
-        for replaced_app, replaced_name in getattr(migration, "replaces", ()) or ():
+        for replaced_app, replaced_name in migration.replaces or ():
            squashed_replaced.add((replaced_app, replaced_name))

    applied = {(app, name) for app, name in applied if app in app_labels}
@@ -392,14 +392,19 @@ def migration_state(out_dir: Path = DATA_DIR) -> tuple[list[str], list[str], dic


@enforce_types
-def pending_migrations(out_dir: Path = DATA_DIR) -> list[str]:
+def pending_migrations(out_dir: Path = CONSTANTS.DATA_DIR) -> list[str]:
    """Return migration files on disk that have not been applied yet."""
    pending, _missing_from_code, _rollback_targets = migration_state(out_dir=out_dir)
    return pending


@enforce_types
-def apply_migrations(out_dir: Path = DATA_DIR, stdout: TextIO | None = None, stderr: TextIO | None = None, verbosity: int = 1) -> list[str]:
+def apply_migrations(
+    out_dir: Path = CONSTANTS.DATA_DIR,
+    stdout: TextIO | None = None,
+    stderr: TextIO | None = None,
+    verbosity: int = 1,
+) -> list[str]:
    """Apply pending Django migrations"""
    from django.core.management import call_command

@@ -27,7 +27,6 @@ __package__ = "archivebox.misc"

 import sys
 import json
-import select
 from typing import Any, TextIO
 from collections.abc import Iterable, Iterator
 from pathlib import Path
@@ -112,14 +111,6 @@ def read_stdin(stream: TextIO | None = None) -> Iterator[dict[str, Any]]:
    if active_stream.isatty():
        return

-    try:
-        ready, _, _ = select.select([active_stream], [], [], 0)
-    except (OSError, ValueError):
-        ready = [active_stream]
-
-    if not ready:
-        return
-
    for line in active_stream:
        record = parse_line(line)
        if record:
@@ -1,7 +1,7 @@
 __package__ = "archivebox"

 # Post-bootstrap CLI logging helpers (event loggers, progress bars, formatters).
-# Requires archivebox.config to be loaded — imports DATA_DIR, get_config, and
+# Requires archivebox.config to be loaded — imports CONSTANTS/get_config and
 # references Django ORM types. For pre-bootstrap logging primitives use
 # misc/logging.py, which has no archivebox or Django dependencies.

@@ -24,7 +24,7 @@ if TYPE_CHECKING:
 from rich import print
 from rich.panel import Panel

-from archivebox.config import DATA_DIR, VERSION
+from archivebox.config import CONSTANTS, VERSION
 from archivebox.config.common import get_config
 from archivebox.misc.util import enforce_types
 from archivebox.misc.logging import ANSI
@@ -171,7 +171,9 @@ def log_list_finished(snapshots):


 def log_removal_started(snapshots, yes: bool):
-    count = snapshots.count() if hasattr(snapshots, "count") else len(snapshots)
+    from django.db.models import QuerySet
+
+    count = snapshots.count() if isinstance(snapshots, QuerySet) else len(snapshots)
    print(f"[yellow3][i] Found {count} matching URLs to remove.[/]")
    file_counts = [s.num_outputs for s in snapshots if os.access(s.output_dir, os.R_OK)]
    print(
@@ -203,7 +205,7 @@ def log_removal_finished(remaining_links: int, removed_links: int):


@enforce_types
-def pretty_path(path: Path | str, pwd: Path | str = DATA_DIR, color: bool = True) -> str:
+def pretty_path(path: Path | str, pwd: Path | str = CONSTANTS.DATA_DIR, color: bool = True) -> str:
    """convert paths like .../ArchiveBox/archivebox/../output/abc into output/abc"""
    pwd = str(Path(pwd))  # .resolve()
    path = str(path)
@@ -4,6 +4,7 @@ from django.core.paginator import Paginator
 from django.core.paginator import Page
 from django.core.paginator import EmptyPage
 from django.db import connection
+from django.db.models import QuerySet
 from django.utils.functional import cached_property


@@ -19,9 +20,13 @@ class CountlessPage(Page):
 class CountlessPaginator(Paginator):
    has_exact_count = False

+    def __init__(self, *args, **kwargs):
+        self._count_hint = 0
+        super().__init__(*args, **kwargs)
+
    @cached_property
    def count(self):
-        return getattr(self, "_count_hint", 0)
+        return self._count_hint

    @cached_property
    def num_pages(self):
@@ -56,16 +61,29 @@ class AcceleratedPaginator(Paginator):

    @cached_property
    def count(self):
-        query = getattr(self.object_list, "query", None)
-        if query is not None and (getattr(query, "distinct", False) or getattr(getattr(query, "where", None), "children", None)):
+        if not isinstance(self.object_list, QuerySet):
+            return super().count
+
+        query = self.object_list.query
+        count_hint = self.object_list.__dict__.get("_archivebox_count_hint")
+        if count_hint is None:
+            count_hint = query.__dict__.get("_archivebox_count_hint")
+        if count_hint is not None:
+            model = self.object_list.model
+            if count_hint == "model_estimate":
+                return self._model_count_estimate(model)
+            if callable(count_hint):
+                return count_hint()
+            return count_hint
+
+        if query.distinct or query.where.children:
            # fallback to normal count method on filtered queryset
            return super().count

-        model = getattr(self.object_list, "model", None)
-        if model is None:
-            return super().count
-
        # otherwise count total rows in a separate fast query
+        return self._model_count_estimate(self.object_list.model)
+
+    def _model_count_estimate(self, model):
        if connection.vendor == "sqlite":
            table_name = model._meta.db_table
            with connection.cursor() as cursor:
@@ -2,6 +2,7 @@ import html
 import json
 import re
 import os
+import sys
 import stat
 import asyncio
 import posixpath
@@ -17,6 +18,7 @@ from pathlib import Path
 from urllib.parse import urlencode

 from django import template
+from django.core.handlers.asgi import ASGIRequest
 from django.contrib.staticfiles import finders
 from django.template import TemplateDoesNotExist, loader
 from django.views import static
@@ -117,7 +119,7 @@ def _render_mhtml_preview_document(filename: str, output_path: str) -> str:


 def _format_direntry_timestamp(stat_result: os.stat_result) -> str:
-    timestamp = getattr(stat_result, "st_birthtime", None) or stat_result.st_mtime
+    timestamp = stat_result.st_birthtime if sys.platform == "darwin" else stat_result.st_mtime
    return datetime.fromtimestamp(timestamp).strftime("%Y-%m-%d %H:%M")


@@ -333,7 +335,7 @@ mimetypes.add_type("multipart/related", ".mhtml")
 mimetypes.add_type("multipart/related", ".mht")

 try:
-    _markdown = getattr(importlib.import_module("markdown"), "markdown")
+    _markdown = importlib.import_module("markdown").markdown
 except ImportError:
    _markdown: Callable[..., str] | None = None

@@ -709,6 +711,10 @@ def _apply_archive_replay_headers(
    return response


+def _is_asgi_request(request) -> bool:
+    return isinstance(request, ASGIRequest) or "scope" in request.__dict__
+
+
 def serve_static_with_byterange_support(request, path, document_root=None, show_indexes=False, is_archive_replay: bool = False):
    """
    Overrides Django's built-in django.views.static.serve function to support byte range requests.
@@ -716,10 +722,9 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
    https://github.com/satchamo/django/commit/2ce75c5c4bee2a858c0214d136bfcd351fcde11d
    """
    assert document_root
-    config = getattr(request, "archivebox_config", None)
+    config = request.__dict__.get("archivebox_config")
    if config is None:
        config = get_config(resolve_plugins=False)
-        request.archivebox_config = config
    fullpath, path = _resolve_archive_path(document_root, path)
    if os.access(fullpath, os.R_OK) and fullpath.is_dir():
        if request.GET.get("download") == "zip" and show_indexes:
@@ -727,7 +732,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
                fullpath,
                path,
                is_archive_replay=is_archive_replay,
-                use_async_stream=hasattr(request, "scope"),
+                use_async_stream=_is_asgi_request(request),
                config=config,
            )
        if show_indexes:
@@ -872,7 +877,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
                fullpath.name,
                raw_output_path,
                wacz_path=fullpath,
-                fallback_url=getattr(request, "archivebox_snapshot_url", "") or "",
+                fallback_url=request.archivebox_snapshot_url or "",
                last_modified=http_date(statobj.st_mtime),
                etag=etag or "",
                cache_control=(
@@ -909,7 +914,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
            response.headers["Content-Security-Policy"] = (
                "default-src 'self' data: blob:; "
                "script-src 'unsafe-inline'; "
-                "style-src 'unsafe-inline'; "
+                "style-src 'unsafe-inline' data: blob:; "
                "connect-src 'self'; "
                "frame-src 'self' data: blob:; "
                "object-src 'none'; "
@@ -978,7 +983,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
    # setup response object
    ranged_file = RangedFileReader(open(fullpath, "rb"))
    response = StreamingHttpResponse(
-        _stream_ranged_file_async(ranged_file) if hasattr(request, "scope") else ranged_file,
+        _stream_ranged_file_async(ranged_file) if _is_asgi_request(request) else ranged_file,
        content_type=content_type,
    )
    response.headers["Last-Modified"] = http_date(statobj.st_mtime)
@@ -96,7 +96,7 @@ class JSONSchemaWithLambdas(GenerateJsonSchema):

 def better_toml_dump_str(val: Any) -> str:
    try:
-        dump_str = cast(Callable[[Any], str], getattr(toml.encoder, "_dump_str"))
+        dump_str = cast(Callable[[Any], str], toml.encoder._dump_str)
        return dump_str(val)
    except Exception:
        # if we hit any of toml's numerous encoding bugs,
@@ -86,6 +86,11 @@ URL_REGEX = re.compile(
    re.IGNORECASE | re.UNICODE,
 )

+# Maximum supported URL length. Very long URLs are rare but must be supported correctly
+# (e.g. data: URLs, deeply nested query strings). The Snapshot.url column is stored as a
+# variable-length TextField (so short URLs don't reserve space and very long URLs still
+# fit) while keeping a normal index on the field so exact, prefix, and substring lookups
+# all keep working.
 MAX_URL_LENGTH = 65535

 QUOTE_DELIMITERS = (
@@ -474,7 +479,7 @@ class ExtendedEncoder(pyjson.JSONEncoder):
    def default(self, o):
        cls_name = o.__class__.__name__

-        if hasattr(o, "_asdict"):
+        if isinstance(o, tuple) and "_asdict" in vars(type(o)):
            return o._asdict()

        elif isinstance(o, bytes):
@@ -612,7 +612,7 @@ def export_browser_state(
    if not state_script.exists():
        return False, None, f"Browser state export script not found at {state_script}"

-    node_modules_dir = get_config().LIB_DIR / "npm" / "node_modules"
+    node_modules_dir = get_config().LIB_DIR / "pnpm" / "packages" / "chrome" / "node_modules"
    chrome_plugin_dir = Path(get_plugins_dir()).resolve()

    env = os.environ.copy()
@@ -1,5 +1,6 @@
 import os
 import json
+import uuid

 from django.db import migrations
 from django.db.models import Q
@@ -9,31 +10,6 @@ VALID_PERMISSIONS = {"public", "unlisted", "private"}
 BATCH_SIZE = 1000


-def legacy_bool(value):
-    if value is None:
-        return None
-    normalized = str(value).strip().lower()
-    if normalized in {"1", "true", "yes", "on"}:
-        return True
-    if normalized in {"0", "false", "no", "off"}:
-        return False
-    return None
-
-
-def permissions_from_legacy_public_flags(config):
-    if str(config.get("PERMISSIONS") or "").strip():
-        return None
-    public_snapshots = legacy_bool(config.get("PUBLIC_SNAPSHOTS"))
-    public_index = legacy_bool(config.get("PUBLIC_INDEX"))
-    if public_snapshots is False:
-        return "private"
-    if public_index is False:
-        return "unlisted"
-    if public_snapshots is True or public_index is True:
-        return "public"
-    return None
-
-
 def normalize_permissions(value, default):
    value = str(value or "").strip().lower()
    return value if value in VALID_PERMISSIONS else default
@@ -59,22 +35,19 @@ def raw_base_config(apps):


 def resolve_permissions(config, default):
+    from archivebox.config.common import permissions_from_legacy_public_flags
+
    explicit = str(config.get("PERMISSIONS") or "").strip().lower()
    if explicit in VALID_PERMISSIONS:
        return explicit
    return permissions_from_legacy_public_flags(config) or default


-def model_has_config(model):
-    try:
-        model._meta.get_field("config")
-    except Exception:
-        return False
-    return True
-
-
 def id_values(pk):
-    return str(pk), getattr(pk, "hex", str(pk).replace("-", ""))
+    if isinstance(pk, uuid.UUID):
+        return str(pk), pk.hex
+    pk_str = str(pk)
+    return pk_str, pk_str.replace("-", "")


 def flush_batch(cursor, table_name, batch):
@@ -120,8 +93,6 @@ def _ensure_permissions_column(cursor):

 def hydrate_persona_permissions(apps, schema_editor):
    Persona = apps.get_model("personas", "Persona")
-    User = apps.get_model("auth", "User")
-    user_has_config = model_has_config(User)
    base_config = raw_base_config(apps)
    default_permissions = resolve_permissions(base_config, "public")
    table_name = schema_editor.quote_name(Persona._meta.db_table)
@@ -130,13 +101,9 @@ def hydrate_persona_permissions(apps, schema_editor):
    batch = []
    missing_permissions = Q(permissions__isnull=True) | (Q(permissions__isnull=False) & ~Q(permissions__in=VALID_PERMISSIONS))

-    for persona in Persona.objects.filter(missing_permissions).select_related("created_by").iterator(chunk_size=BATCH_SIZE):
+    for persona in Persona.objects.filter(missing_permissions).iterator(chunk_size=BATCH_SIZE):
        config = dict(persona.config or {})
        resolved = dict(base_config)
-        if user_has_config:
-            user_config = persona.created_by.config or {}
-            if isinstance(user_config, dict):
-                resolved.update(user_config)
        resolved.update(config)
        config["PERMISSIONS"] = resolve_permissions(resolved, default_permissions)
        batch.append((persona.id, config))
@@ -16,6 +16,7 @@ import sys
 from contextlib import contextmanager
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
+from collections.abc import Mapping

 from django.db import models
 from django.db.models.fields.json import KT
@@ -59,6 +60,26 @@ VOLATILE_PROFILE_FILE_NAMES = {
 }


+def derive_persona_config(*, name: str, config: Mapping[str, Any] | None, persona_dir: Path) -> dict[str, Any]:
+    derived = dict(config or {})
+
+    if "CHROME_USER_DATA_DIR" not in derived:
+        derived["CHROME_USER_DATA_DIR"] = str(persona_dir / "chrome_profile")
+    if "CHROME_DOWNLOADS_DIR" not in derived:
+        derived["CHROME_DOWNLOADS_DIR"] = str(persona_dir / "chrome_downloads")
+
+    cookies_path = persona_dir / "cookies.txt"
+    if "COOKIES_FILE" not in derived and cookies_path.exists():
+        derived["COOKIES_FILE"] = str(cookies_path)
+
+    auth_path = persona_dir / "auth.json"
+    if "AUTH_STORAGE_FILE" not in derived and auth_path.exists():
+        derived["AUTH_STORAGE_FILE"] = str(auth_path)
+
+    derived["ACTIVE_PERSONA"] = name
+    return derived
+
+
 class Persona(ModelWithConfig):
    """
    Browser persona/profile for archiving sessions.
@@ -99,8 +120,7 @@ class Persona(ModelWithConfig):
        if str(config.get("PERMISSIONS") or "").strip().lower() not in PERMISSIONS_VALUES:
            from archivebox.config.common import get_config

-            user = self.created_by if self.created_by_id else None
-            config["PERMISSIONS"] = normalize_permissions(get_config(user=user, include_machine=True).PERMISSIONS)
+            config["PERMISSIONS"] = normalize_permissions(get_config(include_machine=True).PERMISSIONS)
            self.config = config
            update_fields = kwargs.get("update_fields")
            if update_fields is not None:
@@ -151,22 +171,7 @@ class Persona(ModelWithConfig):
        - AUTH_STORAGE_FILE (derived from persona path, if file exists)
        - ACTIVE_PERSONA (set to this persona's name)
        """
-        derived = dict(self.config or {})
-
-        # Add derived paths (don't override if explicitly set in config)
-        if "CHROME_USER_DATA_DIR" not in derived:
-            derived["CHROME_USER_DATA_DIR"] = self.CHROME_USER_DATA_DIR
-        if "CHROME_DOWNLOADS_DIR" not in derived:
-            derived["CHROME_DOWNLOADS_DIR"] = self.CHROME_DOWNLOADS_DIR
-        if "COOKIES_FILE" not in derived and self.COOKIES_FILE:
-            derived["COOKIES_FILE"] = self.COOKIES_FILE
-        if "AUTH_STORAGE_FILE" not in derived and self.AUTH_STORAGE_FILE:
-            derived["AUTH_STORAGE_FILE"] = self.AUTH_STORAGE_FILE
-
-        # Always set ACTIVE_PERSONA to this persona's name
-        derived["ACTIVE_PERSONA"] = self.name
-
-        return derived
+        return derive_persona_config(name=self.name, config=self.config, persona_dir=self.path)

    def ensure_dirs(self) -> None:
        """Create persona directories if they don't exist."""
@@ -232,6 +237,15 @@ class Persona(ModelWithConfig):
    def runtime_downloads_dir_for_crawl(self, crawl) -> Path:
        return self.runtime_root_for_crawl(crawl) / "chrome_downloads"

+    def runtime_root_for_snapshot(self, snapshot) -> Path:
+        return Path(snapshot.output_dir) / ".persona" / self.name
+
+    def runtime_profile_dir_for_snapshot(self, snapshot) -> Path:
+        return self.runtime_root_for_snapshot(snapshot) / "chrome_profile"
+
+    def runtime_downloads_dir_for_snapshot(self, snapshot) -> Path:
+        return self.runtime_root_for_snapshot(snapshot) / "chrome_downloads"
+
    def copy_chrome_profile(self, source_dir: Path, destination_dir: Path) -> None:
        destination_dir.parent.mkdir(parents=True, exist_ok=True)
        shutil.rmtree(destination_dir, ignore_errors=True)
@@ -284,6 +298,33 @@ class Persona(ModelWithConfig):
            "CHROME_DOWNLOADS_DIR": str(runtime_downloads_dir),
        }

+    def prepare_runtime_for_snapshot(self, snapshot, chrome_binary: str = "") -> dict[str, str]:
+        crawl_runtime_profile_dir = self.runtime_profile_dir_for_crawl(snapshot.crawl)
+        template_dir = crawl_runtime_profile_dir if crawl_runtime_profile_dir.exists() else Path(self.CHROME_USER_DATA_DIR)
+        runtime_root = self.runtime_root_for_snapshot(snapshot)
+        runtime_profile_dir = self.runtime_profile_dir_for_snapshot(snapshot)
+        runtime_downloads_dir = self.runtime_downloads_dir_for_snapshot(snapshot)
+
+        if runtime_root.exists():
+            shutil.rmtree(runtime_root, ignore_errors=True)
+        if template_dir.exists() and any(template_dir.iterdir()):
+            self.copy_chrome_profile(template_dir, runtime_profile_dir)
+        else:
+            runtime_profile_dir.mkdir(parents=True, exist_ok=True)
+
+        runtime_downloads_dir.mkdir(parents=True, exist_ok=True)
+        self.cleanup_chrome_profile(runtime_profile_dir)
+
+        (runtime_root / "persona_name.txt").write_text(self.name)
+        (runtime_root / "template_dir.txt").write_text(str(template_dir))
+        if chrome_binary:
+            (runtime_root / "chrome_binary.txt").write_text(chrome_binary)
+
+        return {
+            "CHROME_USER_DATA_DIR": str(runtime_profile_dir),
+            "CHROME_DOWNLOADS_DIR": str(runtime_downloads_dir),
+        }
+
    def cleanup_runtime_for_crawl(self, crawl) -> None:
        shutil.rmtree(Path(crawl.output_dir) / ".persona", ignore_errors=True)

@@ -1,7 +1,6 @@
 __package__ = "archivebox.plugins"

 import json
-import os
 from collections.abc import Iterable
 from functools import lru_cache
 from pathlib import Path
@@ -26,9 +25,7 @@ class PluginSpecialConfig(TypedDict):


 BUILTIN_PLUGINS_DIR = Path(get_plugins_dir()).resolve()
-USER_PLUGINS_DIR = Path(
-    os.environ.get("ARCHIVEBOX_USER_PLUGINS_DIR") or str(CONSTANTS.USER_PLUGINS_DIR),
-).expanduser()
+USER_PLUGINS_DIR = CONSTANTS.USER_PLUGINS_DIR


 def iter_plugin_dirs() -> list[Path]:
@@ -167,7 +164,7 @@ def discover_plugins_that_provide_interface(
                module = importlib.util.module_from_spec(spec)
                spec.loader.exec_module(module)

-                if not all(hasattr(module, attr) for attr in required_attrs):
+                if not all(attr in vars(module) for attr in required_attrs):
                    continue

                if plugin_prefix:
@@ -198,12 +195,15 @@ def get_search_backends() -> dict[str, Any]:
    )


+@lru_cache(maxsize=1)
 def discover_plugin_configs() -> dict[str, dict[str, Any]]:
    """
    Discover all plugin config.json schemas.

    Each plugin can define a config.json file with JSONSchema defining
-    its configuration options.
+    its configuration options. This is intentionally cached because these
+    schemas are plugin package metadata, not live user config; runtime values
+    still come from env/db config at each callsite.
    """
    configs = {}

@@ -9,7 +9,8 @@ from typing import Any
 from django import forms
 from django.utils.html import format_html

-from archivebox.config.common import get_config
+from archivebox.config import CONSTANTS_CONFIG
+from archivebox.config.common import ArchiveBoxConfig, get_config
 from archivebox.plugins.discovery import discover_plugin_configs, get_plugin_icon, get_plugins


@@ -129,6 +130,7 @@ HIDDEN_PLUGIN_CONFIG_UI_PLUGINS = {
    "env",
    "media",
    "npm",
+    "opencode",
    "pip",
    "puppeteer",
    "search_backend_ripgrep",
@@ -250,6 +252,7 @@ def _coerce_plugin_config_value(raw_value: Any, schema: Mapping[str, Any]) -> An

 class PluginConfigFormMixin:
    plugin_groups: list[dict[str, Any]]
+    allow_crawl_execution_config_fields = True

    def build_plugin_groups(self, runtime_config: Mapping[str, Any] | None = None) -> None:
        all_plugins = get_plugins()
@@ -314,7 +317,11 @@ class PluginConfigFormMixin:
            config_fields = [
                self._build_plugin_config_field(str(plugin_name), str(config_key), prop_schema, runtime_config)
                for config_key, prop_schema in properties.items()
-                if isinstance(prop_schema, dict)
+                if (
+                    isinstance(prop_schema, dict)
+                    and str(config_key) not in CONSTANTS_CONFIG
+                    and (self.allow_crawl_execution_config_fields or ArchiveBoxConfig.scope_for_key(str(config_key)) == "crawl_frozen")
+                )
            ]
            cards.append(
                {
@@ -427,6 +434,10 @@ class PluginConfigFormMixin:
                input_name = _plugin_config_input_name(plugin_name, config_key)
                if input_name not in self.data:
                    continue
+                if str(config_key) in CONSTANTS_CONFIG:
+                    continue
+                if not self.allow_crawl_execution_config_fields and ArchiveBoxConfig.scope_for_key(str(config_key)) != "crawl_frozen":
+                    continue

                raw_value: Any = self.data.get(input_name)
                if "array" in _schema_types(prop_schema) and isinstance(prop_schema.get("enum"), list):
@@ -595,5 +606,6 @@ def get_plugin_config_binary_urls(runtime_config: Mapping[str, Any]) -> dict[str
            )
        if binary is None and name != value:
            binary = Binary.objects.get_valid_binary(name, machine=machine)
-        urls[key] = get_installed_binary_change_url(getattr(binary, "name", name), binary) or get_environment_binary_url(name)
+        binary_name = binary.name if binary is not None else name
+        urls[key] = get_installed_binary_change_url(binary_name, binary) or get_environment_binary_url(name)
    return urls
@@ -46,7 +46,7 @@ import json
 import os
 from collections.abc import Mapping
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Optional, Protocol, TypeGuard
+from typing import TYPE_CHECKING, Any, Optional, Protocol, TypeGuard, runtime_checkable

 from archivebox.config.constants import CONSTANTS
 from archivebox.config.version import VERSION
@@ -62,12 +62,13 @@ if TYPE_CHECKING:
    from archivebox.machine.models import Process


+@runtime_checkable
 class ConfigDump(Protocol):
    def as_dict(self) -> dict[str, Any]: ...


 def _has_config_dump(config: object) -> TypeGuard[ConfigDump]:
-    return callable(getattr(config, "as_dict", None))
+    return isinstance(config, ConfigDump)


 def _config_to_overrides(config: ConfigLookup | Mapping[str, Any] | None) -> dict[str, Any]:
@@ -259,13 +260,13 @@ def run_hook(
    This is the low-level hook executor that creates a Process record and
    uses Process.launch() for subprocess management.

-    Config is passed to hooks via environment variables. Caller MUST use
-    get_config() to merge all sources (file, env, machine, crawl, snapshot).
+    Config is passed to hooks via environment variables. Crawl/snapshot callers
+    should pass the runtime config produced by for_crawl_runtime().

    Args:
        script: Path to the hook script (.sh, .py, or .js)
        output_dir: Working directory for the script (where output files go)
-        config: Optional pre-merged config dict from get_config(crawl=..., snapshot=...).
+        config: Optional runtime config dict from for_crawl_runtime().
                If omitted, pass scope/override args using kwargs prefixed with config_.
        timeout: Maximum execution time in seconds
                 If None, auto-detects from PLUGINNAME_TIMEOUT config (fallback to TIMEOUT, default 300)
@@ -277,18 +278,21 @@ def run_hook(

    Example:
        from archivebox.config.common import get_config
-        config = get_config(crawl=my_crawl, snapshot=my_snapshot)
+        config = get_config(crawl=my_crawl, snapshot=my_snapshot).for_crawl_runtime(crawl=my_crawl, snapshot=my_snapshot)
        process = run_hook(hook_path, output_dir, config=config, url=url, snapshot_id=id)
        if process.status == 'exited':
            records = process.get_records()  # Get parsed JSONL output
    """
    from archivebox.machine.models import Process, Machine, NetworkInterface
-    from archivebox.config.common import get_config
-    import sys
+    from archivebox.config.common import get_config, normalize_runtime_config

    config_scope = {key.removeprefix("config_"): kwargs.pop(key) for key in list(kwargs) if key.startswith("config_")}
-    resolved_config = get_config(overrides=_config_to_overrides(config), **config_scope)
-    hook_config = resolved_config.for_crawl_execution()
+    config_overrides = _config_to_overrides(config)
+    resolved_config = get_config(overrides=config_overrides, **config_scope)
+    hook_config = normalize_runtime_config(
+        config_overrides if config is not None else resolved_config.for_crawl(),
+        json_safe=False,
+    )

    # Auto-detect timeout from plugin config if not explicitly provided
    if timeout is None:
@@ -328,12 +332,15 @@ def run_hook(
        )
        return process

-    # Determine the interpreter based on file extension
+    # Python hooks carry their runtime contract in the shebang
+    # (usually `abxpkg run --script python3`), so execute them directly.
+    # For shell/JS hooks we still dispatch through the conventional
+    # interpreter because those hooks do not need per-script Python env setup.
    ext = script.suffix.lower()
    if ext == ".sh":
        cmd = ["bash", str(script)]
    elif ext == ".py":
-        cmd = [sys.executable, str(script)]
+        cmd = [str(script)]
    elif ext == ".js":
        cmd = ["node", str(script)]
    else:
@@ -363,44 +370,29 @@ def run_hook(

    # Set up environment with base paths
    env = os.environ.copy()
-    env["DATA_DIR"] = str(resolved_config.DATA_DIR)
-    env["ARCHIVE_DIR"] = str(resolved_config.ARCHIVE_DIR)
-    env["ABX_RUNTIME"] = "archivebox"
+    env["DATA_DIR"] = str(CONSTANTS.DATA_DIR)
    env["LIBRARY_VERSION"] = VERSION
    env.setdefault("MACHINE_ID", os.environ.get("MACHINE_ID", CONSTANTS.MACHINE_ID))
-
-    resolved_output_dir = output_dir.resolve()
-    snap_dir = _model_output_dir_from_child_path(resolved_output_dir, CONSTANTS.SNAPSHOTS_DIR_NAME)
-    crawl_dir = _model_output_dir_from_child_path(resolved_output_dir, CONSTANTS.CRAWLS_DIR_NAME)
+    snap_dir = hook_config.get("SNAP_DIR") or _model_output_dir_from_child_path(output_dir, CONSTANTS.SNAPSHOTS_DIR_NAME)
+    crawl_dir = hook_config.get("CRAWL_DIR") or _model_output_dir_from_child_path(output_dir, CONSTANTS.CRAWLS_DIR_NAME)
    if snap_dir:
        env["SNAP_DIR"] = str(snap_dir)
    if crawl_dir:
        env["CRAWL_DIR"] = str(crawl_dir)

-    crawl_id = kwargs.get("_crawl_id") or kwargs.get("crawl_id")
-    if crawl_id:
-        try:
-            from archivebox.crawls.models import Crawl
-
-            crawl = Crawl.objects.filter(id=crawl_id).first()
-            if crawl:
-                env["CRAWL_DIR"] = str(crawl.output_dir)
-        except Exception:
-            pass
-
    # Export runtime library roots; abx-dl/abxpkg own executable lookup env.
-    lib_dir = resolved_config.LIB_DIR
+    lib_dir = hook_config.get("LIB_DIR")
    if lib_dir:
        env["LIB_DIR"] = str(lib_dir)
        env["ABXPKG_LIB_DIR"] = str(lib_dir)

    # Set Node.js module resolution paths.
    # NODE_PATH may be a path list, but NODE_MODULES_DIR is a single canonical directory.
-    node_modules_dir = resolved_config.get("NODE_MODULES_DIR")
+    node_modules_dir = hook_config.get("NODE_MODULES_DIR")
    if not node_modules_dir and lib_dir:
-        node_modules_dir = Path(lib_dir) / "npm" / "node_modules"
+        node_modules_dir = Path(lib_dir) / "pnpm" / "packages" / "chrome" / "node_modules"

-    node_path_parts = [part for part in str(resolved_config.get("NODE_PATH") or "").split(os.pathsep) if part]
+    node_path_parts = [part for part in str(hook_config.get("NODE_PATH") or "").split(os.pathsep) if part]
    if node_modules_dir:
        node_modules_dir = Path(node_modules_dir)
        node_modules_dir.mkdir(parents=True, exist_ok=True)
@@ -423,7 +415,6 @@ def run_hook(
        "NODE_MODULES_DIR",
        "NODE_MODULE_DIR",
        "DATA_DIR",
-        "ARCHIVE_DIR",
        "MACHINE_ID",
        "SNAP_DIR",
        "CRAWL_DIR",
@@ -78,13 +78,9 @@ def get_live_config_url(key: str) -> str:


 def get_machine_admin_url() -> str | None:
-    try:
-        from archivebox.machine.models import Machine
+    from archivebox.machine.models import Machine

-        machine = Machine.current()
-        return getattr(machine, "admin_change_url", None) or f"/admin/machine/machine/{machine.id}/change/"
-    except Exception:
-        return None
+    return Machine.current().admin_change_url


 def render_code_tag_list(values: list[str]) -> str:
@@ -35,6 +35,9 @@
        top: 0;
        z-index: 10;
    }
+    #progress-monitor.collapsed .header-bar {
+        cursor: pointer;
+    }
    #progress-monitor .header-left {
        display: flex;
        align-items: center;
@@ -71,7 +74,12 @@
        animation: idle-pulse 5s infinite;
    }
    #progress-monitor .status-dot.stopped {
-        background: #6e7681;
+        background: #d29922;
+        box-shadow: 0 0 4px #d29922;
+    }
+    #progress-monitor .status-dot.error {
+        background: #f85149;
+        box-shadow: 0 0 8px #f85149;
    }
    #progress-monitor .status-dot.flash {
        animation: flash 0.3s ease-out;
@@ -967,19 +975,19 @@
            <div class="stats">
                <div class="stat">
                    <span class="stat-label">Crawls</span>
-                    <span class="stat-value compact info"><span id="crawls-active">0</span> active · <span id="crawls-queued">0</span> queued</span>
+                    <span class="stat-value compact info"><span id="crawls-active-segment"><span id="crawls-active">0</span> active · </span><span id="crawls-queued">0</span> queued</span>
                </div>
                <div class="stat">
                    <span class="stat-label">Snapshots</span>
-                    <span class="stat-value compact info"><span id="snapshots-active">0</span> active · <span id="snapshots-queued">0</span> queued</span>
+                    <span class="stat-value compact info"><span id="snapshots-active-segment"><span id="snapshots-active">0</span> active · </span><span id="snapshots-queued">0</span> queued</span>
                </div>
                <div class="stat">
                    <span class="stat-label">Downloads</span>
-                    <span class="stat-value compact warning"><span id="downloads-active">0</span> active · <span id="downloads-queued">0</span> queued</span>
+                    <span class="stat-value compact warning"><span id="downloads-active-segment"><span id="downloads-active">0</span> active · </span><span id="downloads-queued">0</span> queued</span>
                </div>
                <div class="stat">
                    <span class="stat-label">Indexing</span>
-                    <span class="stat-value compact success"><span id="indexing-active">0</span> active · <span id="indexing-queued">0</span> queued</span>
+                    <span class="stat-value compact success"><span id="indexing-active-segment"><span id="indexing-active">0</span> active · </span><span id="indexing-queued">0</span> queued</span>
                </div>
            </div>
        </div>
@@ -1682,17 +1690,24 @@
        `;
    }

+    function setOrchestratorState(state, label) {
+        const dot = document.getElementById('orchestrator-dot');
+        dot.classList.remove('stopped', 'idle', 'running', 'error');
+        dot.classList.add(state);
+        document.getElementById('orchestrator-text').textContent = label;
+        return dot;
+    }
+
    function updateProgress(data) {
+        idleMessage.style.color = '';
        function setCount(id, value) {
            const el = document.getElementById(id);
            if (el) el.textContent = Number(value || 0).toLocaleString();
        }
-        function setOrchestratorState(state, label) {
-            const dot = document.getElementById('orchestrator-dot');
-            dot.classList.remove('stopped', 'idle', 'running');
-            dot.classList.add(state);
-            document.getElementById('orchestrator-text').textContent = label;
-            return dot;
+        function setActiveCount(id, value) {
+            setCount(id, value);
+            const segment = document.getElementById(`${id}-segment`);
+            if (segment) segment.style.display = Number(value || 0) > 0 ? '' : 'none';
        }

        // Calculate if there's activity
@@ -1715,17 +1730,15 @@
        // Update orchestrator status - show "Running" only when there are active workers.
        const pidEl = document.getElementById('orchestrator-pid');
        const hasWorkers = data.total_workers > 0;
-        const hasBlockedCrawl = (data.active_crawls || []).some(c => c.worker_state === 'crashed');
        let dot = null;

        if (hasWorkers) {
            dot = setOrchestratorState('running', 'Running');
-        } else if (hasActivity && hasBlockedCrawl && !data.orchestrator_running) {
+        } else if (!data.orchestrator_running) {
            dot = setOrchestratorState('stopped', 'Runner stopped');
        } else if (hasActivity) {
-            dot = setOrchestratorState('idle', data.orchestrator_running ? 'Idle' : 'Waiting');
+            dot = setOrchestratorState('idle', 'Idle');
        } else {
-            // No activity - show as idle (whether orchestrator process exists or not)
            dot = setOrchestratorState('idle', 'Idle');
        }

@@ -1742,15 +1755,17 @@
        setTimeout(() => dot.classList.remove('flash'), 300);

        [
-            ['crawls-active', data.crawls_active],
            ['crawls-queued', data.crawls_queued],
-            ['snapshots-active', data.snapshots_active],
            ['snapshots-queued', data.snapshots_queued],
-            ['downloads-active', data.downloads_active],
            ['downloads-queued', data.downloads_queued],
-            ['indexing-active', data.indexing_active],
            ['indexing-queued', data.indexing_queued],
        ].forEach(([id, value]) => setCount(id, value));
+        [
+            ['crawls-active', data.crawls_active],
+            ['snapshots-active', data.snapshots_active],
+            ['downloads-active', data.downloads_active],
+            ['indexing-active', data.indexing_active],
+        ].forEach(([id, value]) => setActiveCount(id, value));
        updateScreencastPanel(data);

        // Render crawl tree
@@ -1784,18 +1799,24 @@

    function fetchProgress() {
        fetch(progressEndpoint, { credentials: 'same-origin' })
-            .then(response => response.json())
+            .then(response => {
+                if (!response.ok) throw new Error(`HTTP ${response.status}`);
+                return response.json();
+            })
            .then(data => {
                if (data.error) {
                    console.error('Progress API error:', data.error, data.traceback);
+                    setOrchestratorState('error', 'Backend not responding');
                    idleMessage.textContent = 'API Error: ' + data.error;
                    idleMessage.style.color = '#f85149';
+                    return;
                }
                monitor.classList.toggle('is-guest', data.is_admin === false);
                updateProgress(data);
            })
            .catch(error => {
                console.error('Progress fetch error:', error);
+                setOrchestratorState('error', 'Backend not responding');
                idleMessage.textContent = 'Fetch Error: ' + error.message;
                idleMessage.style.color = '#f85149';
            });
@@ -1866,9 +1887,14 @@
    }

    // Collapse toggle
-    collapseBtn.addEventListener('click', function() {
+    collapseBtn.addEventListener('click', function(event) {
+        event.stopPropagation();
        setCollapsedState(!isCollapsed);
    });
+    monitor.querySelector('.header-bar').addEventListener('click', function(event) {
+        if (event.target.closest('button, a, input, select, textarea, [role="button"]')) return;
+        if (isCollapsed) setCollapsedState(false);
+    });

    crawlTree.addEventListener('click', function(event) {
        const actionBtn = event.target.closest('.crawl-action-btn');
@@ -327,8 +327,20 @@ def live_progress_view(request):
            .order_by("-modified_at")[:max_active_crawls],
        )
        paused_crawls = list(
-            crawl_scope.filter(status=Crawl.StatusChoices.PAUSED, created_at__gte=paused_crawl_cutoff)
+            crawl_scope.filter(
+                Q(status=Crawl.StatusChoices.PAUSED, created_at__gte=paused_crawl_cutoff)
+                | Q(
+                    status=Crawl.StatusChoices.PAUSED,
+                    snapshot_set__status__in=Snapshot.RUNNABLE_STATES,
+                    snapshot_set__retry_at__lte=now,
+                )
+                | Q(
+                    status=Crawl.StatusChoices.PAUSED,
+                    snapshot_set__archiveresult__status=ArchiveResult.StatusChoices.QUEUED,
+                ),
+            )
            .values(*active_crawl_fields)
+            .distinct()
            .order_by("-modified_at")[:max_active_crawls],
        )
        queued_crawls = list(
@@ -345,9 +357,7 @@ def live_progress_view(request):
        persona_objects_by_id = {}
        persona_objects_by_name = {}
        persona_ids = {crawl["persona_id"] for crawl in active_crawls_list if crawl["persona_id"]}
-        persona_names = {
-            str((crawl["config"] or {}).get("DEFAULT_PERSONA") or "Default") for crawl in active_crawls_list if not crawl["persona_id"]
-        }
+        persona_names = {"Default"} if any(not crawl["persona_id"] for crawl in active_crawls_list) else set()
        if persona_ids or persona_names:
            from archivebox.personas.models import Persona

@@ -813,7 +823,7 @@ def live_progress_view(request):
            urls_preview = crawl["urls"][:60] if crawl["urls"] else None
            crawl_tags = [tag.strip() for tag in (crawl["tags_str"] or "").replace("\n", ",").split(",") if tag.strip()]
            persona_details = persona_details_by_id.get(str(crawl["persona_id"])) if crawl["persona_id"] else None
-            persona_name = persona_details["name"] if persona_details else str((crawl["config"] or {}).get("DEFAULT_PERSONA") or "Default")
+            persona_name = persona_details["name"] if persona_details else "Default"
            persona_details = persona_details or persona_details_by_name.get(persona_name)
            crawl_output_size = crawl_output_sizes_by_crawl.get(crawl_id, 0)
            avg_snapshot_size = int(crawl_output_size / completed_snapshots) if completed_snapshots else 0
@@ -19,8 +19,8 @@ class SearchResultsChangeList(ChangeList):

    def __init__(self, request, *args, **kwargs):
        """Capture normalized search mode before Django builds results."""
-        self.search_mode = get_search_mode(request.GET.get("search_mode"), config=getattr(request, "archivebox_config", None))
-        self.search_mode_backend = get_search_mode_backend(self.search_mode, config=getattr(request, "archivebox_config", None))
+        self.search_mode = get_search_mode(request.GET.get("search_mode"), config=request.archivebox_config)
+        self.search_mode_backend = get_search_mode_backend(self.search_mode, config=request.archivebox_config)
        super().__init__(request, *args, **kwargs)
        self.embedded_changelist = request.GET.get("_embedded") == "crawl"

@@ -31,7 +31,7 @@ class SearchResultsChangeList(ChangeList):
            self.opts.model_name == "snapshot"
            and self.query
            and self.result_count == 0
-            and get_search_mode_base(self.search_mode, config=getattr(request, "archivebox_config", None)) == "deep"
+            and get_search_mode_base(self.search_mode, config=request.archivebox_config) == "deep"
            and self.search_mode_backend,
        )

@@ -55,13 +55,11 @@ class SearchResultsAdminMixin(admin.ModelAdmin):

    def get_default_search_mode(self):
        """Return the default search mode for the current request config."""
-        request = getattr(self, "request", None)
-        return get_default_search_mode(config=getattr(request, "archivebox_config", None))
+        return get_default_search_mode(config=self.request.archivebox_config)

    def get_search_mode_options(self):
        """Return selector options for the current request config."""
-        request = getattr(self, "request", None)
-        return get_search_mode_options(config=getattr(request, "archivebox_config", None))
+        return get_search_mode_options(config=self.request.archivebox_config)

    def get_search_results(self, request, queryset, search_term: str):
        """Apply admin search semantics to a changelist queryset."""
@@ -69,14 +67,14 @@ class SearchResultsAdminMixin(admin.ModelAdmin):
        search_term = search_term.strip()
        if not search_term:
            return super().get_search_results(request, queryset, search_term)
-        search_mode = get_search_mode(request.GET.get("search_mode"), config=getattr(request, "archivebox_config", None))
+        search_mode = get_search_mode(request.GET.get("search_mode"), config=request.archivebox_config)
        if queryset.model._meta.label_lower == "core.snapshot" and request.GET.get("_embedded") != "crawl":
            cached_ids = get_cached_admin_search_ids(request)
            if cached_ids is not None:
                return queryset.filter(pk__in=cached_ids) if cached_ids else queryset.none(), False
            return queryset.none(), False

-        if get_search_mode_base(search_mode, config=getattr(request, "archivebox_config", None)) == "meta":
+        if get_search_mode_base(search_mode, config=request.archivebox_config) == "meta":
            qs, use_distinct = super().get_search_results(request, queryset, search_term)
            return qs, use_distinct
        if request.GET.get("_embedded") == "crawl":
@@ -85,7 +83,7 @@ class SearchResultsAdminMixin(admin.ModelAdmin):
                    pk__in=query_search_index(
                        search_term,
                        search_mode=search_mode,
-                        config=getattr(request, "archivebox_config", None),
+                        config=request.archivebox_config,
                    ).values("pk"),
                ), False
            except Exception as err:
@@ -12,16 +12,16 @@ _search_backends_cache: dict | None = None

@contextmanager
 def search_backend_env(config: dict[str, Any] | None = None, **config_kwargs: Any):
-    """Temporarily expose resolved config through os.environ for backend code."""
+    """Temporarily expose resolved search config through os.environ for backend code."""
    config = config or get_config(**config_kwargs)
    updates = {}
    for key, value in config.items():
+        if not str(key).startswith("SEARCH_BACKEND_"):
+            continue
        if value is None:
            continue
        if isinstance(value, (str, int, float, bool, os.PathLike)):
            updates[str(key)] = str(value)
-    updates["DATA_DIR"] = str(config.DATA_DIR)
-    updates["SNAP_DIR"] = str(config.USERS_DIR)
    previous = {key: os.environ.get(key) for key in updates}
    os.environ.update(updates)
    try:
@@ -62,7 +62,7 @@ def get_search_mode_options(config: dict[str, Any] | None = None, **config_kwarg
    ]
    options = [
        {"value": "meta", "label": "meta"},
-        {"value": "contents", "label": "contents"},
+        {"value": "contents", "label": "deep"},
    ]
    if backend_names:
        options.extend(
@@ -248,10 +248,8 @@ def iter_query_search_ids(
            backend = backends[backend_name]
            try:
                with search_backend_env(config=config):
-                    if hasattr(backend, "iter_search"):
+                    if backend_name == "ripgrep":
                        ids = backend.iter_search(query, search_mode=search_mode_base)
-                    elif backend_name == "ripgrep":
-                        ids = backend.search(query, search_mode=search_mode_base)
                    else:
                        ids = backend.search(query)
                    for snapshot_id in ids:
@@ -101,7 +101,8 @@ def iter_admin_backend_search_ids(iterator, queryset):
 def admin_snapshot_search_stream_view(model_admin, request):
    """Stream admin Snapshot search progress and cache matching IDs."""
    query = (request.GET.get("q") or "").strip()
-    search_mode = get_search_mode(request.GET.get("search_mode"), config=getattr(request, "archivebox_config", None))
+    config = request.archivebox_config
+    search_mode = get_search_mode(request.GET.get("search_mode"), config=config)
    if not query:
        return StreamingHttpResponse((), content_type="text/plain")

@@ -115,12 +116,12 @@ def admin_snapshot_search_stream_view(model_admin, request):
    filter_request.path = target_url.path or request.path
    filter_request.path_info = target_url.path or request.path_info
    filter_request.GET = target_get
-    filter_request.archivebox_config = getattr(request, "archivebox_config", None)
+    filter_request.archivebox_config = config

    # Build the same filtered base queryset the changelist uses, but with the
    # search params stripped. The stream intersects each wave with this queryset
    # before writing IDs into the short-lived cache consumed by the changelist.
-    current_request = getattr(model_admin, "request", None)
+    current_request = model_admin.__dict__.get("request")
    try:
        base_queryset = model_admin.get_changelist_instance(filter_request).queryset
    finally:
@@ -150,12 +151,12 @@ def admin_snapshot_search_stream_view(model_admin, request):
            nonlocal last_sent
            iterator = None
            try:
-                search_mode_base = get_search_mode_base(search_mode, config=getattr(request, "archivebox_config", None))
+                search_mode_base = get_search_mode_base(search_mode, config=config)
                iterator = (
                    iter_admin_meta_search_ids(query, base_queryset)
                    if search_mode_base == "meta"
                    else iter_admin_backend_search_ids(
-                        iter_query_search_ids(query, search_mode=search_mode, config=getattr(request, "archivebox_config", None)),
+                        iter_query_search_ids(query, search_mode=search_mode, config=config),
                        base_queryset,
                    )
                )
@@ -2,10 +2,15 @@ from __future__ import annotations

 import asyncio
 import json
+import os
+import sys
+import time
 from collections import defaultdict
 from collections.abc import Iterable
+from contextlib import contextmanager
+from functools import wraps
 from pathlib import Path
-from typing import Any
+from typing import Any, Protocol, runtime_checkable

 from asgiref.sync import sync_to_async
 from django.db import IntegrityError
@@ -18,6 +23,57 @@ from abx_dl.services.base import BaseService
 from .process_service import parse_event_datetime


+def _perf_trace(label):
+    def decorator(func):
+        if asyncio.iscoroutinefunction(func):
+
+            @wraps(func)
+            async def async_wrapper(*args, **kwargs):
+                if os.environ.get("ARCHIVEBOX_PERF_TRACE") != "1":
+                    return await func(*args, **kwargs)
+                started_at = time.perf_counter()
+                try:
+                    return await func(*args, **kwargs)
+                finally:
+                    elapsed_ms = (time.perf_counter() - started_at) * 1000
+                    print(f"PERF_TRACE label={label} ms={elapsed_ms:.3f}", file=sys.stderr, flush=True)
+
+            return async_wrapper
+
+        @wraps(func)
+        def sync_wrapper(*args, **kwargs):
+            if os.environ.get("ARCHIVEBOX_PERF_TRACE") != "1":
+                return func(*args, **kwargs)
+            started_at = time.perf_counter()
+            try:
+                return func(*args, **kwargs)
+            finally:
+                elapsed_ms = (time.perf_counter() - started_at) * 1000
+                print(f"PERF_TRACE label={label} ms={elapsed_ms:.3f}", file=sys.stderr, flush=True)
+
+        return sync_wrapper
+
+    return decorator
+
+
+@contextmanager
+def _perf_span(label: str):
+    if os.environ.get("ARCHIVEBOX_PERF_TRACE") != "1":
+        yield
+        return
+    started_at = time.perf_counter()
+    try:
+        yield
+    finally:
+        elapsed_ms = (time.perf_counter() - started_at) * 1000
+        print(f"PERF_TRACE label={label} ms={elapsed_ms:.3f}", file=sys.stderr, flush=True)
+
+
+@runtime_checkable
+class ModelDumpable(Protocol):
+    def model_dump(self) -> dict[str, Any]: ...
+
+
 def _collect_output_metadata(plugin_dir: Path) -> tuple[dict[str, dict], int, str]:
    exclude_names = {"stdout.log", "stderr.log", "process.pid", "hook.pid", "listener.pid"}
    output_files: dict[str, dict] = {}
@@ -97,15 +153,8 @@ def _normalize_output_files(raw_output_files: Any) -> dict[str, dict]:
        if isinstance(item, str):
            normalized[item] = _enrich_metadata(item, {})
            continue
-        if hasattr(item, "model_dump"):
+        if isinstance(item, ModelDumpable):
            item = item.model_dump()
-        elif hasattr(item, "path"):
-            item = {
-                "path": getattr(item, "path", ""),
-                "extension": getattr(item, "extension", ""),
-                "mimetype": getattr(item, "mimetype", ""),
-                "size": getattr(item, "size", 0),
-            }
        if not isinstance(item, dict):
            continue
        path = str(item.get("path") or "").strip()
@@ -191,6 +240,10 @@ def _has_content_files(output_files: Any) -> bool:
    return any(Path(path).suffix not in {".log", ".pid", ".sh"} for path in _normalize_output_files(output_files))


+def _is_signal_interrupted_exit(exit_code: int) -> bool:
+    return exit_code < 0 or (exit_code >= 128 and exit_code != PROCESS_EXIT_SKIPPED)
+
+
 def _iter_archiveresult_records(stdout: str) -> list[dict]:
    records: list[dict] = []
    for raw_line in stdout.splitlines():
@@ -206,34 +259,40 @@ def _iter_archiveresult_records(stdout: str) -> list[dict]:
    return records


-class ArchiveResultService(BaseService):
-    LISTENS_TO = [ArchiveResultEvent, ProcessCompletedEvent]
-    EMITS = []
+@_perf_trace("archivebox.ArchiveResultService._save_archiveresult_event_sync")
+def _save_archiveresult_event_to_db(
+    event: ArchiveResultEvent,
+    process_started: ProcessStartedEvent | None,
+) -> None:
+    """Project one ArchiveResultEvent with a single thread-sensitive ORM hop.

-    def __init__(self, bus):
-        self._completed_process_event_ids: set[str] = set()
-        self._save_locks: dict[tuple[str, str, str], asyncio.Lock] = {}
-        super().__init__(bus)
-        self.bus.on(ArchiveResultEvent, self.on_ArchiveResultEvent__save_to_db)
-        self.bus.on(ProcessCompletedEvent, self.on_ProcessCompletedEvent__save_to_db)
+    Django's async ORM still delegates each query to sync Django work. The hot
+    search/index maintenance path was paying that handoff separately for
+    Snapshot lookup, Process lookup, ArchiveResult lookup, update, and title
+    checks. Keep the public ArchiveResultEvent path intact, but run the DB
+    projection as one short synchronous block so SQLite sees the same indexed
+    reads/writes without per-query asyncio/threadpool churn.
+    """
+    from archivebox.core.models import ArchiveResult, Snapshot
+    from archivebox.machine.models import Process

-    async def on_ArchiveResultEvent__save_to_db(self, event: ArchiveResultEvent) -> None:
-        from archivebox.core.models import ArchiveResult, Snapshot
-        from archivebox.machine.models import Process
+    with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.snapshot_lookup"):
+        snapshot = Snapshot.objects.filter(id=event.snapshot_id).select_related("crawl", "crawl__created_by").first()
+    if snapshot is None:
+        return

-        snapshot = await Snapshot.objects.filter(id=event.snapshot_id).select_related("crawl", "crawl__created_by").afirst()
-        if snapshot is None:
-            return
-        plugin_dir = Path(snapshot.output_dir) / event.plugin
-        output_files, output_size, output_mimetypes = await sync_to_async(_resolve_output_metadata)(event.output_files, plugin_dir)
-        process_started = await self.bus.find(
-            ProcessStartedEvent,
-            past=True,
-            future=False,
-            where=lambda candidate: self.bus.event_is_child_of(event, candidate),
+    with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.plugin_dir"):
+        plugin_dir = (
+            Path(process_started.output_dir)
+            if process_started is not None and process_started.output_dir
+            else Path(snapshot.output_dir) / event.plugin
        )
-        process = None
-        if process_started is not None:
+    with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.resolve_output_metadata"):
+        output_files, output_size, output_mimetypes = _resolve_output_metadata(event.output_files, plugin_dir)
+
+    process = None
+    if process_started is not None:
+        with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.process_lookup"):
            started_at = parse_event_datetime(process_started.start_ts)
            if started_at is None:
                raise ValueError("ProcessStartedEvent.start_ts is required")
@@ -244,8 +303,9 @@ class ArchiveResultService(BaseService):
            )
            if process_started.pid:
                process_query = process_query.filter(pid=process_started.pid)
-            process = await process_query.order_by("-modified_at").afirst()
+            process = process_query.order_by("-modified_at").first()

+    with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.prepare_defaults"):
        start_ts = parse_event_datetime(event.start_ts)
        end_ts = parse_event_datetime(event.end_ts) or timezone.now()
        defaults = {
@@ -263,47 +323,82 @@ class ArchiveResultService(BaseService):
        if event.error:
            defaults["notes"] = event.error

-        key = (str(snapshot.id), event.plugin, event.hook_name)
-        lock = self._save_locks.setdefault(key, asyncio.Lock())
-        async with lock:
-            # Avoid update_or_create(): Django wraps it in transaction.atomic().
-            # On SQLite a lock retry inside that read-then-write transaction can
-            # keep the read transaction open and block the writer that would have
-            # released the lock. Use autocommit-sized writes instead.
-            result = await ArchiveResult.objects.filter(
-                snapshot=snapshot,
-                plugin=event.plugin,
-                hook_name=event.hook_name,
-            ).afirst()
-            if result is None:
-                try:
-                    result = await ArchiveResult.objects.acreate(
-                        snapshot=snapshot,
-                        plugin=event.plugin,
-                        hook_name=event.hook_name,
-                        **defaults,
-                    )
-                except IntegrityError:
-                    result = await ArchiveResult.objects.aget(
-                        snapshot=snapshot,
-                        plugin=event.plugin,
-                        hook_name=event.hook_name,
-                    )
+    with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_lookup"):
+        result = ArchiveResult.objects.filter(
+            snapshot=snapshot,
+            plugin=event.plugin,
+            hook_name=event.hook_name,
+        ).first()
+    if result is None:
+        try:
+            with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_create"):
+                result = ArchiveResult.objects.create(
+                    snapshot=snapshot,
+                    plugin=event.plugin,
+                    hook_name=event.hook_name,
+                    **defaults,
+                )
+        except IntegrityError:
+            with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_get_after_integrity"):
+                result = ArchiveResult.objects.get(
+                    snapshot=snapshot,
+                    plugin=event.plugin,
+                    hook_name=event.hook_name,
+                )

-            update_fields = []
-            for field, value in defaults.items():
-                if getattr(result, field) != value:
-                    setattr(result, field, value)
-                    update_fields.append(field)
-            if update_fields:
-                await result.asave(update_fields=[*update_fields, "modified_at"])
+    with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.diff_fields"):
+        update_fields = []
+        for field, value in defaults.items():
+            if result.__dict__[field] != value:
+                setattr(result, field, value)
+                update_fields.append(field)
+    if update_fields:
+        with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_update"):
+            result.save(update_fields=[*update_fields, "modified_at"])

-        if result.status in (ArchiveResult.StatusChoices.SUCCEEDED, ArchiveResult.StatusChoices.NORESULTS):
-            next_title = _extract_snapshot_title(str(snapshot.output_dir), event.plugin, result.output_str, snapshot_url=snapshot.url)
+    if result.status == ArchiveResult.StatusChoices.QUEUED:
+        # ArchiveResult has no retry_at column. If a shutdown/takeover projects
+        # a killed hook back to QUEUED, wake the parent Snapshot/Crawl so the
+        # next runner retries that exact hook instead of waiting on a stale
+        # active-state lease.
+        snapshot.update_and_requeue(retry_at=timezone.now())
+
+    if result.status in (ArchiveResult.StatusChoices.SUCCEEDED, ArchiveResult.StatusChoices.NORESULTS):
+        with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.title_update"):
+            title_output_str = result.output_str if result.status == ArchiveResult.StatusChoices.SUCCEEDED else ""
+            next_title = _extract_snapshot_title(str(plugin_dir.parent), event.plugin, title_output_str, snapshot_url=snapshot.url)
            if next_title and _should_update_snapshot_title(snapshot.title or "", next_title, snapshot_url=snapshot.url):
                snapshot.title = next_title
-                await snapshot.asave(update_fields=["title", "modified_at"])
+                snapshot.save(update_fields=["title", "modified_at"])

+
+class ArchiveResultService(BaseService):
+    LISTENS_TO = [ArchiveResultEvent, ProcessCompletedEvent]
+    EMITS = []
+
+    def __init__(self, bus):
+        self._completed_process_event_ids: set[str] = set()
+        self._save_locks: dict[tuple[str, str, str], asyncio.Lock] = {}
+        super().__init__(bus)
+        self.bus.on(ArchiveResultEvent, self.on_ArchiveResultEvent__save_to_db)
+        self.bus.on(ProcessCompletedEvent, self.on_ProcessCompletedEvent__save_to_db)
+
+    @_perf_trace("archivebox.ArchiveResultService.on_ArchiveResultEvent__save_to_db")
+    async def on_ArchiveResultEvent__save_to_db(self, event: ArchiveResultEvent) -> None:
+        with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.find_process_started"):
+            process_started = await self.bus.find(
+                ProcessStartedEvent,
+                past=True,
+                future=False,
+                where=lambda candidate: self.bus.event_is_child_of(event, candidate),
+            )
+
+        key = (str(event.snapshot_id), event.plugin, event.hook_name)
+        lock = self._save_locks.setdefault(key, asyncio.Lock())
+        async with lock:
+            await sync_to_async(_save_archiveresult_event_to_db, thread_sensitive=True)(event, process_started)
+
+    @_perf_trace("archivebox.ArchiveResultService.on_ProcessCompletedEvent__save_to_db")
    async def on_ProcessCompletedEvent__save_to_db(self, event: ProcessCompletedEvent) -> None:
        if event.event_id in self._completed_process_event_ids:
            return
@@ -311,16 +406,18 @@ class ArchiveResultService(BaseService):

        if not event.hook_name.startswith("on_Snapshot"):
            return
-        snapshot_event = await self.bus.find(
-            SnapshotEvent,
-            past=True,
-            future=False,
-            where=lambda candidate: self.bus.event_is_child_of(event, candidate),
-        )
+        with _perf_span("archivebox.ArchiveResultService.on_ProcessCompletedEvent.find_snapshot_event"):
+            snapshot_event = await self.bus.find(
+                SnapshotEvent,
+                past=True,
+                future=False,
+                where=lambda candidate: self.bus.event_is_child_of(event, candidate),
+            )
        if snapshot_event is None:
            return

-        records = _iter_archiveresult_records(event.stdout)
+        with _perf_span("archivebox.ArchiveResultService.on_ProcessCompletedEvent.parse_stdout_records"):
+            records = _iter_archiveresult_records(event.stdout)
        if records:
            if len(records) > 1:
                raise RuntimeError(
@@ -329,33 +426,45 @@ class ArchiveResultService(BaseService):
            for record in records:
                record_status = _normalize_status(record.get("status") or "")
                record_failed = record_status == "failed" or (not record_status and event.exit_code not in (0, PROCESS_EXIT_SKIPPED))
-                await event.emit(
-                    ArchiveResultEvent(
-                        snapshot_id=record.get("snapshot_id") or snapshot_event.snapshot_id,
-                        plugin=record.get("plugin") or event.plugin_name,
-                        hook_name=record.get("hook_name") or event.hook_name,
-                        status=record_status,
-                        output_str=record.get("output_str") or "",
-                        output_json=record.get("output_json") if isinstance(record.get("output_json"), dict) else None,
-                        output_files=event.output_files,
-                        start_ts=event.start_ts,
-                        end_ts=event.end_ts,
-                        error=record.get("error") or (event.stderr if record_failed else ""),
-                    ),
-                ).now()
+                with _perf_span("archivebox.ArchiveResultService.on_ProcessCompletedEvent.emit_archive_result_record"):
+                    await event.emit(
+                        ArchiveResultEvent(
+                            snapshot_id=record.get("snapshot_id") or snapshot_event.snapshot_id,
+                            plugin=record.get("plugin") or event.plugin_name,
+                            hook_name=record.get("hook_name") or event.hook_name,
+                            status=record_status,
+                            output_str=record.get("output_str") or "",
+                            output_json=record.get("output_json") if isinstance(record.get("output_json"), dict) else None,
+                            output_files=event.output_files,
+                            start_ts=event.start_ts,
+                            end_ts=event.end_ts,
+                            error=record.get("error") or (event.stderr if record_failed else ""),
+                        ),
+                    ).now()
            return

-        process_failed = event.exit_code not in (0, PROCESS_EXIT_SKIPPED)
-        await event.emit(
-            ArchiveResultEvent(
-                snapshot_id=snapshot_event.snapshot_id,
-                plugin=event.plugin_name,
-                hook_name=event.hook_name,
-                status="failed" if process_failed else ("succeeded" if _has_content_files(event.output_files) else "skipped"),
-                output_str=event.stderr if process_failed else "",
-                output_files=event.output_files,
-                start_ts=event.start_ts,
-                end_ts=event.end_ts,
-                error=event.stderr if process_failed else "",
-            ),
-        ).now()
+        # TODO: consider moving this fallback derivation into abx-dl itself.
+        # First try both patterns: if the whole abx-dl process crashes, restarting
+        # the snapshot may be enough, but don't guess before validating it.
+        process_interrupted = _is_signal_interrupted_exit(event.exit_code)
+        process_failed = event.exit_code not in (0, PROCESS_EXIT_SKIPPED) and not process_interrupted
+        with _perf_span("archivebox.ArchiveResultService.on_ProcessCompletedEvent.emit_archive_result_fallback"):
+            await event.emit(
+                ArchiveResultEvent(
+                    snapshot_id=snapshot_event.snapshot_id,
+                    plugin=event.plugin_name,
+                    hook_name=event.hook_name,
+                    status=(
+                        "queued"
+                        if process_interrupted
+                        else "failed"
+                        if process_failed
+                        else ("succeeded" if _has_content_files(event.output_files) else "noresult")
+                    ),
+                    output_str=event.stderr if process_failed else "",
+                    output_files=event.output_files,
+                    start_ts=event.start_ts,
+                    end_ts=event.end_ts,
+                    error=event.stderr if process_failed else "",
+                ),
+            ).now()
@@ -1,5 +1,6 @@
 from __future__ import annotations

+import asyncio
 import json
 from collections.abc import Mapping
 from pathlib import Path
@@ -186,38 +187,110 @@ class ArchiveBoxDBBinaryCacheBackend:
 class ArchiveBoxBinaryService(BaseService):
    """Preserve ArchiveBox's legacy Binary Process rows around abxpkg requests."""

-    LISTENS_TO = [BinaryRequestEvent]
+    LISTENS_TO = [BinaryRequestEvent, BinaryEvent]
    EMITS: list[type[BaseEvent]] = []

    def __init__(self, bus: EventBus):
        super().__init__(bus)
+        self.process_ids_by_request_id: dict[str, str] = {}
+        self._missing_finalize_tasks: set[asyncio.Task] = set()
        self.bus.on(BinaryRequestEvent, self.on_BinaryRequestEvent__project_process)
+        self.bus.on(BinaryRequestEvent, self.on_BinaryRequestEvent__schedule_missing_finalize)
+        self.bus.on(BinaryEvent, self.on_BinaryEvent__finalize_process)

    async def on_BinaryRequestEvent__project_process(self, request: BinaryRequestEvent) -> None:
-        from archivebox.machine.models import Binary, Machine, Process, _canonical_binary_name
-        from archivebox.services.process_service import current_network_interface_with_machine
+        from archivebox.machine.models import Machine, Process, _canonical_binary_name

        machine = await sync_to_async(Machine.current, thread_sensitive=True)()
        binary_name = _canonical_binary_name(request.name)
        if not binary_name:
            return
-        binary = await Binary.objects.filter(machine=machine, name=binary_name).order_by("-modified_at").afirst()
-        if binary is None:
-            return
+        binary = await self._get_or_create_binary(machine, binary_name, request)
+        started_at = timezone.now()
+        output_dir = self._process_output_dir(binary, request)
+        await sync_to_async(output_dir.mkdir, thread_sensitive=True)(parents=True, exist_ok=True)
+        process = await Process.objects.acreate(
+            machine=machine,
+            iface=None,
+            process_type=Process.TypeChoices.BINARY,
+            worker_type="",
+            pwd=str(output_dir),
+            cmd=self._process_cmd(request),
+            env={},
+            timeout=int(request.event_timeout or request.install_timeout or 600),
+            pid=None,
+            url=None,
+            started_at=started_at,
+            ended_at=None,
+            stdout="",
+            stderr="",
+            exit_code=None,
+            status=Process.StatusChoices.RUNNING,
+            retry_at=None,
+            binary=binary,
+        )
+        self.process_ids_by_request_id[request.event_id] = str(process.id)

-        binary_event = await self.bus.find(
-            BinaryEvent,
-            child_of=request,
+    async def on_BinaryEvent__finalize_process(self, event: BinaryEvent) -> None:
+        from archivebox.machine.models import Binary, Process, _canonical_binary_name
+
+        request = await self.bus.find(
+            BinaryRequestEvent,
            past=True,
            future=False,
-            name=request.name,
-            where=lambda candidate: bool(candidate.abspath),
+            where=lambda candidate: self.bus.event_is_child_of(event, candidate),
        )
-        iface = await sync_to_async(current_network_interface_with_machine, thread_sensitive=True)()
-        now = timezone.now()
-        success = isinstance(binary_event, BinaryEvent)
-        output_dir = self._process_output_dir(binary, request)
-        output_dir.mkdir(parents=True, exist_ok=True)
+        request = request if isinstance(request, BinaryRequestEvent) else None
+        process_id = self.process_ids_by_request_id.pop(request.event_id, "") if request is not None else ""
+        if not process_id:
+            return
+        process = await Process.objects.filter(id=process_id).select_related("binary").afirst()
+        if process is None:
+            return
+        binary_name = _canonical_binary_name(event.name)
+        binary = process.binary
+        if binary is not None and binary_name:
+            binary.abspath = event.abspath
+            if event.version:
+                binary.version = str(event.version)
+            if event.sha256:
+                binary.sha256 = str(event.sha256)
+            binary.binproviders = event.binproviders or binary.binproviders
+            binary.binprovider = event.binprovider or binary.binprovider
+            binary.status = Binary.StatusChoices.INSTALLED
+            binary.retry_at = None
+            await binary.asave(
+                update_fields=["abspath", "version", "sha256", "binproviders", "binprovider", "status", "retry_at", "modified_at"],
+            )
+        process.ended_at = timezone.now()
+        process.stdout = json.dumps(self._binary_event_json(event, binary)) + "\n"
+        process.stderr = ""
+        process.exit_code = 0
+        process.status = Process.StatusChoices.EXITED
+        await process.asave(update_fields=["ended_at", "stdout", "stderr", "exit_code", "status", "modified_at"])
+        if binary is not None:
+            await sync_to_async(self._write_binary_index, thread_sensitive=True)(binary, process, Path(process.pwd))
+
+    async def _get_or_create_binary(self, machine, binary_name: str, request: BinaryRequestEvent):
+        from archivebox.machine.models import Binary
+
+        binary_id = str(request.extra_context.get("binary_id") or "")
+        if binary_id:
+            binary = await Binary.objects.filter(id=binary_id).afirst()
+            if binary is not None:
+                return binary
+        binary = await Binary.objects.filter(machine=machine, name=binary_name).order_by("-modified_at").afirst()
+        if binary is not None:
+            return binary
+        return await Binary.objects.acreate(
+            machine=machine,
+            name=binary_name,
+            binproviders=_binproviders_to_str(request.binproviders),
+            overrides=_persisted_overrides_for_request(request),
+            status=Binary.StatusChoices.QUEUED,
+        )
+
+    def _process_cmd(self, request: BinaryRequestEvent) -> list[str]:
        cmd = [
            "abxpkg",
            "install",
@@ -226,29 +299,70 @@ class ArchiveBoxBinaryService(BaseService):
        ]
        if request.overrides:
            cmd.append(f"--overrides={json.dumps(request.overrides, sort_keys=True)}")
-        stdout = json.dumps(binary.to_json()) + "\n" if success else ""
-        stderr = "" if success else f"Binary request did not resolve: {request.name}"
-        process = await Process.objects.acreate(
-            machine=iface.machine,
-            iface=iface,
-            process_type=Process.TypeChoices.BINARY,
-            worker_type="",
-            pwd=str(output_dir),
-            cmd=cmd,
-            env={},
-            timeout=int(request.event_timeout or request.install_timeout or 600),
-            pid=None,
-            url=None,
-            started_at=now,
-            ended_at=now,
-            stdout=stdout,
-            stderr=stderr,
-            exit_code=0 if success else 1,
-            status=Process.StatusChoices.EXITED,
-            retry_at=None,
-            binary=binary,
+        return cmd
+
+    def _binary_event_json(self, event: BinaryEvent, binary) -> dict[str, Any]:
+        if binary is not None:
+            data = binary.to_json()
+        else:
+            data = {"type": "Binary", "name": event.name}
+        data.update(
+            {
+                "type": "Binary",
+                "name": event.name,
+                "binproviders": event.binproviders,
+                "binprovider": event.binprovider,
+                "abspath": event.abspath,
+                "version": str(event.version or ""),
+                "sha256": event.sha256 or "",
+                "status": "installed",
+            },
        )
-        self._write_binary_index(binary, process, output_dir)
+        return data
+
+    async def _finalize_missing_process(self, request: BinaryRequestEvent) -> None:
+        from archivebox.machine.models import Process
+
+        process_id = self.process_ids_by_request_id.pop(request.event_id, "")
+        if not process_id:
+            return
+        process = await Process.objects.filter(id=process_id).afirst()
+        if process is None or process.status == Process.StatusChoices.EXITED:
+            return
+        process.ended_at = timezone.now()
+        process.stderr = f"Binary request did not resolve: {request.name}"
+        process.exit_code = 1
+        process.status = Process.StatusChoices.EXITED
+        await process.asave(update_fields=["ended_at", "stderr", "exit_code", "status", "modified_at"])
+
+    async def _finalize_request_when_done(self, request: BinaryRequestEvent) -> None:
+        try:
+            await request.wait(timeout=request.event_timeout)
+        except TimeoutError:
+            await self._finalize_missing_process(request)
+            return
+        binary_event = await self.bus.find(
+            BinaryEvent,
+            child_of=request,
+            past=True,
+            future=False,
+            name=request.name,
+            where=lambda candidate: bool(candidate.abspath),
+        )
+        if not isinstance(binary_event, BinaryEvent):
+            await self._finalize_missing_process(request)
+
+    def _schedule_missing_finalize(self, request: BinaryRequestEvent) -> None:
+        task = asyncio.create_task(self._finalize_request_when_done(request))
+        self._missing_finalize_tasks.add(task)
+        task.add_done_callback(lambda done: self._missing_finalize_tasks.discard(done) or (None if done.cancelled() else done.exception()))
+
+    async def flush_missing_finalizers(self) -> None:
+        if self._missing_finalize_tasks:
+            await asyncio.gather(*tuple(self._missing_finalize_tasks), return_exceptions=False)
+
+    async def on_BinaryRequestEvent__schedule_missing_finalize(self, request: BinaryRequestEvent) -> None:
+        self._schedule_missing_finalize(request)

    def _process_output_dir(self, binary, request: BinaryRequestEvent) -> Path:
        raw_output_dir = str(request.extra_context.get("output_dir") or "").strip()
Author	SHA1	Message	Date
Nick Sweeting	0a3c89e73c	release: archivebox 0.9.34rc63	2026-06-06 23:27:54 -07:00
Nick Sweeting	36e0e3e2b6	release: archivebox 0.9.34rc62	2026-06-06 23:19:13 -07:00
Nick Sweeting	f9b13e3828	release: archivebox 0.9.34rc61	2026-06-06 23:09:44 -07:00
Nick Sweeting	a25adf22eb	Build ArchiveBox on compact abx-dl image	2026-06-06 20:41:21 -07:00
Nick Sweeting	0c984651a3	Avoid Docker bytecode cleanup	2026-06-06 17:28:42 -07:00
Nick Sweeting	a0aec802b7	Support URLs up to 8000 chars with TextField storage (#1817 ) ## Summary Changes the `Snapshot.url` field from a fixed-length `CharField(max_length=65535)` to a variable-length `TextField` while enforcing a practical 8000-character limit. This allows short URLs to use minimal storage space while still supporting very long URLs (e.g., data: URLs, deeply nested query strings) that are common in real-world scenarios. The TextField with a normal database index maintains fast exact, prefix, and substring lookups while reducing storage overhead for typical URLs. ## Related issues Improves URL handling to match practical web server/proxy limits (8000 chars is the de facto standard enforced by most infrastructure). ## Changes these areas - [x] Internal architecture - [x] Snapshot data layout on disk ## Details ### Changes Made 1. `archivebox/misc/util.py`: Reduced `MAX_URL_LENGTH` from 65535 to 8000 with detailed comments explaining the rationale 2. `archivebox/core/models.py`: Changed `Snapshot.url` field from `CharField(max_length=MAX_URL_LENGTH)` to `TextField(db_index=True)` with validation in the model's `save()` method 3. `archivebox/core/migrations/0049_alter_snapshot_url.py`: Django migration to alter the field type 4. `archivebox/tests/test_snapshot_url_length.py`: Comprehensive test suite covering: - URL length validation at the limit and over the limit - Full round-trip persistence through the database - Query operations (exact match, prefix search, substring search) - Uniqueness constraints per crawl - Cross-crawl URL reuse ### Why This Change - Storage efficiency: Variable-length TextField doesn't reserve 8000 bytes for every URL - Practical limit: 8000 chars matches the upper bound enforced by most web servers and proxies - Query performance: Normal database index on TextField maintains fast lookups for all query types - Real-world support: Handles legitimate long URLs (data: URIs, complex query strings) while rejecting pathological cases ## Test Plan Added comprehensive test suite (`test_snapshot_url_length.py`) that validates: - URLs at exactly the 8000-char limit are accepted and persisted correctly - URLs exceeding the limit are rejected with validation errors - Database queries (exact, prefix, substring) work correctly on long URLs - Uniqueness constraints per crawl are still enforced - Same URL can exist in different crawls All tests pass with the new implementation. https://claude.ai/code/session_01BLnGTL5GSoouD4ihaYp55n <!-- This is an auto-generated description by cubic. --> --- ## Summary by cubic Store `Snapshot.url` as an indexed `TextField` to support very long URLs efficiently while keeping exact, prefix, and substring searches fast. The supported URL length remains 65535 characters. - Refactors - Changed `Snapshot.url` from `CharField` to `TextField(db_index=True)`. - Kept `MAX_URL_LENGTH` at 65535 and validate in model/utilities; tests updated to the 65535 boundary, lookups, and per-crawl uniqueness. - Migration - Apply `0049_alter_snapshot_url`. - No data rewrite; existing rows remain. New/updated URLs must be ≤65535 chars. <sup>Written for commit `d6d479f50b`. Summary will update on new commits.</sup> <a href="https://cubic.dev/pr/ArchiveBox/ArchiveBox/pull/1817?utm_source=github" target="_blank" rel="noopener noreferrer" data-no-image-dialog="true"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cubic.dev/buttons/review-in-cubic-dark.svg"><source media="(prefers-color-scheme: light)" srcset="https://cubic.dev/buttons/review-in-cubic-light.svg"><img alt="Review in cubic" src="https://cubic.dev/buttons/review-in-cubic-dark.svg"></picture></a> <!-- End of auto-generated description by cubic. -->	2026-06-06 03:11:53 -04:00
Nick Sweeting	7cfebc7815	Simplify Docker image layering	2026-06-06 00:02:01 -07:00
Nick Sweeting	d6d479f50b	Merge branch 'dev' into claude/festive-hopper-0p7Bj	2026-06-06 02:38:33 -04:00
Nick Sweeting	f39c5d39f1	Speed up remove after flag test	2026-06-05 12:09:20 -07:00
Nick Sweeting	88dc690bd3	Fix queued hook recovery tests	2026-06-05 11:44:36 -07:00
Nick Sweeting	1987edb2ee	Use shared env opencode binary and process-linked crawl test	2026-06-05 10:54:02 -07:00
Nick Sweeting	1b19736b2f	Recover interrupted hook work by hook identity	2026-06-05 03:25:38 -07:00
Nick Sweeting	232de59e95	Fix ArchiveBox Docker runtime layering	2026-06-05 02:03:12 -07:00
Nick Sweeting	466238d7dd	Use published abx-dl image for Docker builds	2026-06-05 00:18:11 -07:00
Nick Sweeting	16cd3837e4	use archivebox installs in browser security tests	2026-06-04 23:38:56 -07:00
Nick Sweeting	a99321c030	fix disabled extractor test env scope	2026-06-04 23:10:43 -07:00
Nick Sweeting	4e4ee8cdb0	fix runner stdin and update maintenance lifecycle	2026-06-04 22:38:22 -07:00
Nick Sweeting	73587a1a4d	use archivebox plugin discovery for extraction queues	2026-06-04 21:57:33 -07:00
Nick Sweeting	7f8af6357d	fix index and binary runner lifecycle	2026-06-04 21:40:58 -07:00
Claude	e4df2d6bf2	Keep MAX_URL_LENGTH at 65535; only switch url storage to TextField The column change to a variable-length, still-indexed TextField is what matters for supporting long URLs efficiently — no reason to lower the supported limit. Adjust the long-URL tests to the real 65535 boundary.	2026-06-05 04:10:41 +00:00
Claude	b74861ef1e	Support URLs up to 8000 chars via variable-length indexed TextField Snapshot.url was a CharField(max_length=65535) which reserves a fixed-width column and is too long to index or constrain on real DB backends. Store it as a variable-length TextField instead, so short URLs don't waste space and very long URLs (up to MAX_URL_LENGTH=8000) are supported, while keeping a normal index on the field so exact, prefix, and substring (icontains) URL lookups all stay fast. - misc/util.py: MAX_URL_LENGTH 65535 -> 8000 (the practical web-server limit) - core/models.py: Snapshot.url CharField -> TextField(db_index=True) - migration 0049_alter_snapshot_url - tests covering 8000-char persistence, exact/prefix/substring lookups, over-length rejection, and per-crawl uniqueness for long URLs https://claude.ai/code/session_01BLnGTL5GSoouD4ihaYp55n	2026-06-04 22:35:27 +00:00
Nick Sweeting	bdff741c85	release: archivebox 0.9.34rc59	2026-06-04 10:05:16 -07:00
Nick Sweeting	ba895db80e	release: archivebox 0.9.34rc58	2026-06-04 09:20:18 -07:00
Nick Sweeting	364efdf409	fix release pypi retry loop	2026-06-04 09:17:34 -07:00
Nick Sweeting	9a1919c2f4	fix docker env syntax	2026-06-04 09:15:16 -07:00
Nick Sweeting	91f6106434	release: archivebox 0.9.34rc56	2026-06-04 09:14:36 -07:00
Nick Sweeting	cf4bd959c0	fix private snapshot replay auth	2026-06-04 09:12:56 -07:00
Nick Sweeting	8367578f9b	Update snapshot admin selection behavior	2026-06-04 08:00:25 -07:00
Nick Sweeting	5de536e06d	release: archivebox 0.9.34rc55	2026-06-04 01:26:32 -07:00
Nick Sweeting	caf0acdec8	release: archivebox 0.9.34rc54	2026-06-04 00:48:47 -07:00
Nick Sweeting	70b325f959	release: archivebox 0.9.34rc53	2026-06-04 00:19:42 -07:00
Nick Sweeting	083ee8cb7d	Remove abx-plugins Docker commit pin	2026-06-04 00:01:29 -07:00
Nick Sweeting	79b59945e5	release: v0.9.34rc52	2026-06-03 23:55:32 -07:00
Nick Sweeting	3c6d94d03a	release: v0.9.34rc51	2026-06-03 23:54:12 -07:00
Nick Sweeting	2e9686b901	release: v0.9.34rc51	2026-06-03 23:21:37 -07:00
Nick Sweeting	40d5dc06a4	release: v0.9.34rc50	2026-06-03 22:59:35 -07:00
Nick Sweeting	bb0bf81e02	release: v0.9.34rc49	2026-06-03 22:58:26 -07:00
Nick Sweeting	d26e27d139	release: archivebox 0.9.34rc49	2026-06-03 22:37:51 -07:00
Nick Sweeting	327e378a9e	release: archivebox 0.9.34rc48	2026-06-03 22:07:19 -07:00
Nick Sweeting	77ee827a58	release: archivebox 0.9.34rc47	2026-06-03 21:37:13 -07:00
Nick Sweeting	5b74eb5238	fix: publish pip builds from pypi environment	2026-06-03 20:01:42 -07:00
Nick Sweeting	8e1c7dccaa	fix: wait for released deps before docker build	2026-06-03 19:45:43 -07:00
Nick Sweeting	50ff26b524	release: archivebox 0.9.34rc46	2026-06-03 19:42:16 -07:00
Nick Sweeting	9e06c32c18	fix: install archivebox extras explicitly in ci	2026-06-03 19:34:08 -07:00
Nick Sweeting	bcd10ea139	fix: resolve local package deps in archivebox ci	2026-06-03 19:30:42 -07:00
Nick Sweeting	b8b70f3525	release: archivebox 0.9.34rc45	2026-06-03 19:13:11 -07:00
Nick Sweeting	74690eb52b	fix: wait for released deps before pip build	2026-06-03 19:02:35 -07:00
Nick Sweeting	afa958a6d5	release: archivebox 0.9.34rc44	2026-06-03 18:57:23 -07:00
Nick Sweeting	1cf0d18444	release: archivebox 0.9.34rc43	2026-06-03 18:24:54 -07:00
Nick Sweeting	7698155cb9	release: archivebox 0.9.34rc42	2026-06-03 17:54:41 -07:00
Nick Sweeting	1ab99027d8	release: archivebox 0.9.34rc41	2026-06-03 17:41:51 -07:00
Nick Sweeting	e0c3ca67e4	release: archivebox 0.9.34rc40	2026-06-03 17:32:17 -07:00
Nick Sweeting	c0fb8eb532	release: archivebox 0.9.34rc39	2026-06-03 17:19:48 -07:00
Nick Sweeting	6da1d3f5af	ci: give archivebox chrome install a real timeout	2026-06-03 00:02:26 -07:00
Nick Sweeting	83a2099851	fix: scope update search backfill runner	2026-06-02 21:54:25 -07:00
Nick Sweeting	3669133a05	fix: allow install to initialize collections	2026-06-02 21:25:20 -07:00
Nick Sweeting	4e9c8b1475	ci: install chrome through archivebox	2026-06-02 21:21:39 -07:00
Nick Sweeting	9f0544857c	test: require success in cli workflows	2026-06-02 21:19:21 -07:00
Nick Sweeting	acc830d30f	test: exercise extract cli with real outputs	2026-06-02 21:07:52 -07:00
Nick Sweeting	43606e17a0	test: let recursive crawl finish before assertions	2026-06-02 20:27:44 -07:00
Nick Sweeting	cd7e7cd132	test: make extract cli setup deterministic	2026-06-02 20:11:37 -07:00
Nick Sweeting	f4ec16c3c8	test: create extract snapshots via cli	2026-06-02 19:56:25 -07:00
Nick Sweeting	9b5aa6861d	test: stabilize api workflow and chrome title checks	2026-06-02 19:41:37 -07:00
Nick Sweeting	81de752456	test: provision chrome for title config check	2026-06-02 19:30:22 -07:00
Nick Sweeting	ad43842691	test: stabilize update api and title ci checks	2026-06-02 19:11:54 -07:00
Nick Sweeting	e4ec848da8	fix: keep background add crawls runnable	2026-06-02 18:53:59 -07:00
Nick Sweeting	39eac65ed0	Stabilize frozen config CLI test flows (cherry picked from commit `2bca869e32`)	2026-06-02 18:44:57 -07:00
Nick Sweeting	e9968b75f2	test: rely on archivewebpage plugin dependencies	2026-06-02 18:42:34 -07:00
Nick Sweeting	cdec5ea440	test: stabilize archivewebpage browser preview	2026-06-02 18:32:13 -07:00
Nick Sweeting	d8fec2e1cf	test: fix ui add runtime cli env call	2026-06-02 15:29:54 -07:00
Nick Sweeting	042339cdf7	test: use local site for recursive cli add	2026-06-02 15:17:51 -07:00
Nick Sweeting	2cd80904e1	test: install chrome deps for cli add coverage	2026-06-02 15:07:09 -07:00
Nick Sweeting	69619dbea4	test: tolerate sealed paused crawl resume	2026-06-02 14:53:57 -07:00
Nick Sweeting	f3af8a1a17	test: tolerate queued api archive results	2026-06-02 14:35:23 -07:00
Nick Sweeting	2dadc434b2	test: stabilize cli add status shards	2026-06-02 14:24:55 -07:00
Nick Sweeting	8aaa236a15	test: stabilize archivebox ci shards	2026-06-02 13:59:50 -07:00
Nick Sweeting	ce6dbdf1a0	test: keep cli update and extract shards focused	2026-06-02 13:45:05 -07:00
Nick Sweeting	1320d08627	ci: tolerate transient sqlite locks in api workflow	2026-06-02 13:30:19 -07:00
Nick Sweeting	c7cd4de3da	ci: stabilize archivebox parallel shards	2026-06-02 13:22:03 -07:00
Nick Sweeting	1fb2d106c1	ci: allow empty archivebox test modules	2026-06-02 12:50:27 -07:00
Nick Sweeting	7e9d765a94	release: archivebox 0.9.34rc38	2026-06-02 12:25:06 -07:00
Nick Sweeting	b46d142cc6	test cleanup	2026-06-02 12:13:47 -07:00
Nick Sweeting	96437e1ffd	Publish local ArchiveBox changes	2026-06-02 02:25:52 -07:00
Nick Sweeting	7dd738b5b7	release: archivebox 0.9.34rc37	2026-06-01 21:44:23 -07:00
Nick Sweeting	00cf5c9dd0	release: archivebox 0.9.34rc36	2026-06-01 19:51:28 -07:00
Nick Sweeting	bcfb8c12a5	release: archivebox 0.9.34rc35	2026-06-01 19:35:35 -07:00
Nick Sweeting	ac6e018672	release: archivebox 0.9.34rc34	2026-06-01 19:23:06 -07:00
Nick Sweeting	24e7552cc4	release: archivebox 0.9.34rc33	2026-06-01 18:33:47 -07:00
Nick Sweeting	e1c84e7d26	release: archivebox 0.9.34rc32	2026-06-01 18:10:44 -07:00
Nick Sweeting	42fd16b820	release: archivebox 0.9.34rc31	2026-06-01 17:52:19 -07:00
Nick Sweeting	62fb2c8659	release: archivebox 0.9.34rc30	2026-06-01 17:36:52 -07:00
Nick Sweeting	5c3161a5c1	release: archivebox 0.9.34rc29	2026-06-01 17:22:59 -07:00
Nick Sweeting	065fcfc0ba	Preserve queued index jobs during reindex	2026-06-01 15:29:38 -07:00
Nick Sweeting	051d16697e	Tighten state machine model typing	2026-06-01 15:14:41 -07:00
Nick Sweeting	40d9ab5ced	release: archivebox 0.9.34rc28	2026-06-01 15:08:25 -07:00
Nick Sweeting	c075d654d8	Consolidate runtime config handling	2026-06-01 15:03:40 -07:00
Nick Sweeting	453d998e7d	fix: schedule background admin crawls	2026-06-01 10:44:21 -07:00
Nick Sweeting	bc8d8bd3d7	fix: support replay assets without base url	2026-06-01 10:38:45 -07:00
Nick Sweeting	75441eb8d7	fix: restart supervised runner from add view	2026-06-01 10:37:44 -07:00
Nick Sweeting	323ab9a439	release: archivebox 0.9.34rc27	2026-06-01 10:27:18 -07:00
Nick Sweeting	4e38ab94d6	fix: pin docker validation binary paths	2026-06-01 10:22:04 -07:00
Nick Sweeting	ae43b27d94	release: archivebox 0.9.34rc26	2026-06-01 09:55:51 -07:00
Nick Sweeting	ddb861b1cf	fix: clean docker data dir before init	2026-06-01 09:54:13 -07:00
Nick Sweeting	c9bba6961d	release: archivebox 0.9.34rc25	2026-06-01 09:37:49 -07:00
Nick Sweeting	c789e980b5	fix: avoid optional docker binary validation	2026-06-01 09:36:12 -07:00
Nick Sweeting	f07bdba4be	release: archivebox 0.9.34rc24	2026-06-01 09:30:51 -07:00
Nick Sweeting	813723f966	fix: disable docker pip release age gate	2026-06-01 09:29:07 -07:00
Nick Sweeting	36047a4356	release: archivebox 0.9.34rc23	2026-06-01 09:22:25 -07:00
Nick Sweeting	a5bbdd425d	fix: skip optional captcha docker preinstall	2026-06-01 09:20:48 -07:00
Nick Sweeting	dd96e56846	release: archivebox 0.9.34rc22	2026-06-01 09:05:02 -07:00
Nick Sweeting	c77ef74879	fix: install docker validation binaries	2026-06-01 09:03:54 -07:00
Nick Sweeting	bb40308149	release: archivebox 0.9.34rc21	2026-06-01 08:39:07 -07:00
Nick Sweeting	492c897f0c	release: archivebox 0.9.34rc20	2026-06-01 08:30:12 -07:00
Nick Sweeting	7cbbe50f9b	fix: avoid aiohttp source builds in docker	2026-06-01 08:28:33 -07:00
Nick Sweeting	fdf57cb82a	release: archivebox 0.9.34rc19	2026-06-01 08:13:36 -07:00
Nick Sweeting	c46e826ec3	fix: pass abxpkg install timeout in docker builds	2026-06-01 08:11:49 -07:00
Nick Sweeting	26ada3366b	release: archivebox 0.9.34rc18	2026-06-01 07:50:57 -07:00
Nick Sweeting	28df80254e	release: archivebox 0.9.34rc17	2026-06-01 07:14:12 -07:00
Nick Sweeting	579ede090d	fix: pre-bake rss parser docker deps	2026-06-01 07:12:13 -07:00
Nick Sweeting	f72f8c9179	release: archivebox 0.9.34rc16	2026-06-01 07:00:52 -07:00
Nick Sweeting	cc2a594e96	release: archivebox 0.9.34rc15	2026-06-01 06:30:36 -07:00
Nick Sweeting	78e7604b39	fix: pre-bake gallery and ocr docker deps	2026-06-01 06:28:49 -07:00
Nick Sweeting	9bfb6fde1b	release: archivebox 0.9.34rc14	2026-06-01 06:00:03 -07:00
Nick Sweeting	ad13744e62	fix: project papersdl docker binary deps	2026-06-01 05:57:44 -07:00
Nick Sweeting	086abfa4d7	release: archivebox 0.9.34rc13	2026-06-01 05:51:59 -07:00
Nick Sweeting	6dbe37fa42	fix: project readability docker binary deps	2026-06-01 05:49:38 -07:00
Nick Sweeting	569fa554ed	release: archivebox 0.9.34rc12	2026-06-01 05:41:26 -07:00
Nick Sweeting	003f05bed0	fix: project mercury docker binary deps	2026-06-01 05:39:03 -07:00
Nick Sweeting	193e646e97	release: archivebox 0.9.34rc11	2026-06-01 05:30:40 -07:00
Nick Sweeting	053834843a	release: archivebox 0.9.34rc10	2026-06-01 05:13:08 -07:00
Nick Sweeting	a0a995810b	fix: project liteparse docker binary deps	2026-06-01 05:10:42 -07:00