Update SECURITY.md with version security details

Clarify security risks and protections for versions prior to v0.9.0.
release: v0.9.35rc44
2026-06-20 22:16:20 -04:00 · 2026-06-19 16:57:38 +00:00 · 2026-06-19 09:57:16 -07:00 · 2026-06-19 00:08:49 -04:00 · 2026-06-15 18:46:52 +00:00 · 2026-06-15 11:46:05 -07:00
308 changed files with 24709 additions and 13790 deletions
@@ -1,5 +0,0 @@
-{
-  "enabledPlugins": {
-    "pyright-lsp@claude-plugins-official": true
-  }
-}
@@ -1,51 +0,0 @@
-{
-  "permissions": {
-    "allow": [
-      "Read(**)",
-      "Glob(**)",
-      "Grep(**)",
-      "Bash(python -m archivebox:*)",
-      "Bash(ls:*)",
-      "Bash(xargs:*)",
-      "Bash(python -c:*)",
-      "Bash(printf:*)",
-      "Bash(pkill:*)",
-      "Bash(python3:*)",
-      "Bash(sqlite3:*)",
-      "WebFetch(domain:github.com)",
-      "Bash(uv add:*)",
-      "Bash(mkdir:*)",
-      "Bash(chmod:*)",
-      "Bash(python -m forum_dl:*)",
-      "Bash(archivebox manage migrate:*)",
-      "Bash(cat:*)",
-      "Bash(python archivebox/plugins/pip/on_Dependency__install_using_pip_provider.py:*)",
-      "Bash(forum-dl:*)",
-      "Bash(pip uninstall:*)",
-      "Bash(python:*)",
-      "Bash(source .venv/bin/activate)",
-      "Bash(mv:*)",
-      "Bash(echo:*)",
-      "Bash(grep:*)",
-      "WebFetch(domain:python-statemachine.readthedocs.io)",
-      "Bash(./bin/run_plugin_tests.sh:*)",
-      "Bash(done)",
-      "Bash(coverage erase:*)",
-      "Bash(gh api:*)"
-    ]
-  },
-  "hooks": {
-    "PreToolUse": [
-      {
-        "matcher": "Bash",
-        "hooks": [
-          {
-            "type": "command",
-            "command": "REPO_ROOT=$(git rev-parse --show-toplevel 2>/dev/null); if [ -n \"$REPO_ROOT\" ] && [ \"$PWD\" != \"$REPO_ROOT\" ]; then echo \"ERROR: Not in repo root ($REPO_ROOT). Current dir: $PWD\" >&2; exit 1; fi",
-            "statusMessage": "Checking working directory..."
-          }
-        ]
-      }
-    ]
-  }
-}
@@ -3,18 +3,33 @@
 /.heartbeat.json
 /package.json
 /package-lock.json
+__pycache__
+**/__pycache__
 *.pyc
-__pycache__/
+*.pyo
+*.py[cod]
+*$py.class
 .mypy_cache/
 .pytest_cache/
+.ruff_cache/
+.uv-cache/
+.cache/
+.cache*/
 .github/
 .pdm-build/
+.pdm-build/**
 .pdm-python
 .eggs/
+*.egg-info/
 .git/
+!.git/
+.git/*
 .vscode/
 !.git/HEAD
-!.git/refs/heads/*
+!.git/packed-refs
+!.git/refs/
+!.git/refs/heads/
+!.git/refs/heads/**

 venv/
 .venv/
@@ -31,6 +46,7 @@ chromeprofile/
 chrome_profile/
 lib/
 out/
+tests/out/
 users/
 archive/
 crawls/
@@ -38,6 +54,8 @@ snapshots/
 logs/
 archivebox-docker-smoke*/
 archivebox-compose-smoke*/
+docker-test/
+docker-test*/
 core
 *.core

@@ -50,6 +68,8 @@ dist/
 brew_dist/
 deb_dist/
 pip_dist/
+cache/
+caches/
 assets/
 docker/
 website/
@@ -69,7 +69,7 @@ body:
        0.8.6
        ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
        IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
-        EUID=502:20 UID=502:20 PUID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
+        EUID=502:20 UID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
        DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
        
        Binary Dependencies:
@@ -71,7 +71,7 @@ body:
        0.8.6
        ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
        IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
-        EUID=502:20 UID=502:20 PUID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
+        EUID=502:20 UID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
        DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
        
        Binary Dependencies:
@@ -2,18 +2,15 @@

 ---

-## Security Information
+## Security Documentation

-Please see this wiki page for important notices about ArchiveBox security, publishing your archives securely, and the dangers of executing archived JS:
-
-https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview
-
-Also see this section of the README about important caveats when running ArchiveBox:
-
-https://github.com/ArchiveBox/ArchiveBox?tab=readme-ov-file#caveats
-
-You can also read these pages for more information about ArchiveBox's internals, development environment, DB schema, and more:
+Please start by reading these links to understand our existing security model, known issues, API surfaces, and setup steps:

+- https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview
+- https://github.com/ArchiveBox/ArchiveBox/security/advisories/
+- https://github.com/ArchiveBox/ArchiveBox/tree/v0.7.4#security-risks-of-viewing-archived-js
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#security-concerns
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-up-Authentication
 - https://github.com/ArchiveBox/ArchiveBox#archive-layout
 - https://github.com/ArchiveBox/ArchiveBox#archivebox-development
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives
@@ -32,3 +29,43 @@ We use Github's built-in [Private Reporting](https://docs.github.com/en/code-sec
 3. Fill out the form to submit the details of the report and it will be securely sent to the maintainers

 You can also contact the maintainers via our public [Zulip Chat Server zulip.archivebox.io](https://zulip.archivebox.io) or [Twitter DMs @ArchiveBoxApp](https://twitter.com/ArchiveBoxApp).
+
+---
+
+## CVEs and Report Credits
+
+> [!IMPORTANT]
+> **Reports that only affect `dev` or beta versions will be converted to normal Github issues and will not be issued CVEs.**  
+> Reporters **will still receive credit in release notes** however.
+
+> [!TIP]
+> Reports that affect **stable published versions will be issued CVEs** and published w/ full credit.
+
+Please read our existing published [security advisories](https://github.com/ArchiveBox/ArchiveBox/security/advisories), [Security Overview Docs](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview), and [issues](https://github.com/ArchiveBox/ArchiveBox/issues) and avoid creating duplicates.
+
+---
+
+## Threat Model
+
+### All admin users are assumed to be trusted
+
+ArchiveBox does not support granular user permissions, there are only two types of users available in the Admin UI & REST API:
+
+- logged in admin superusers (which are assumed to have `root` on the ArchiveBox host machine already)
+- non-logged in anonymous guest users
+
+Because admin / REST API / CLI / Python API users are already assumed to have `root` on the host machine, there are deliberately no protections against admins inputting malicious config values, attempts to attack other admin users, or destructive/malicious actions. Permissions for non-logged in users are managed via the [`PERMISSIONS`](https://github.com/archiveBox/archiveBox/wiki/configuration#permissions) + [`PUBLIC_INDEX` + `PUBLIC_ADD_VIEW`](https://github.com/archiveBox/archiveBox/wiki/configuration#public_index--public_add_view) + other config options.
+
+### Sanitization of Config Values, Filesystem Paths, DB Fields, etc.
+
+Non-logged in users can only choose between public Persona config presets on the `/add/` page, but they should never be allowed to directly set config options like `*_ARGS`, `*_BINARY` or DB fields like `url`, `title`, `tag.name`, `notes` without sanitization/filtering on save and/or use.
+
+Non-logged in users should be tightly limited to prevent attacks like SQL/shell injection, path traversal attacks, XSS, cookie leaks between admin/non-admin domains, etc. Any reports related to non-logged-in users will be prioritized as this is a surface we care a lot about hardening.
+
+### XSS, CSRF, CORS, CSP
+
+`< v0.9.x` versions do not include any XSS/CSRF protections at all, as documented in the [readme](https://github.com/ArchiveBox/ArchiveBox/tree/v0.7.4#security-risks-of-viewing-archived-js), [wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#security-concerns), [issue #239](https://github.com/ArchiveBox/ArchiveBox/issues/239), and [existing advisory](https://github.com/ArchiveBox/ArchiveBox/security/advisories/GHSA-cr45-98w9-gwqx). Do not open advisories related to XSS or CSRF for those versions, as they are missing fundamental architectural changes added in `v0.9.0` needed to support per-snapshot replay isolation.
+
+[`SERVER_SECURITY_MODE`](https://github.com/archiveBox/archiveBox/wiki/configuration#server_security_mode) + CSRF/XSS defenses were only added in the new 0.9.0 `dev` work and are planned to land in `main` in mid-2026.
+
+Note archivebox is designed to be able to archive any URL the server can reach, including internal intranet URLs. The only URLs that are blocked are URLs hosted by the archivebox server itself.
@@ -1,214 +0,0 @@
-name: Build Debian package
-
-on:
-  workflow_dispatch:
-  workflow_call:
-  push:
-    branches: ['**']
-    paths:
-      - 'pkg/debian/**'
-      - 'bin/build_deb.sh'
-      - 'bin/release_deb.sh'
-      - '.github/workflows/debian.yml'
-      - 'pyproject.toml'
-  # release trigger is handled by release.yml to avoid double-runs
-
-permissions:
-  contents: write
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build:
-    runs-on: ubuntu-24.04
-    strategy:
-      matrix:
-        arch: [amd64, arm64]
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Get version
-        id: version
-        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
-
-      - name: Install nfpm
-        env:
-          NFPM_VERSION: "2.45.1"
-        run: |
-          curl -fsSL "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${NFPM_VERSION}_Linux_x86_64.tar.gz" \
-            | sudo tar -xz -C /usr/local/bin nfpm
-          nfpm --version
-
-      - name: Build .deb package
-        run: |
-          export VERSION="${{ steps.version.outputs.version }}"
-          export ARCH="${{ matrix.arch }}"
-          ./bin/build_deb.sh
-
-      - name: Verify .deb package contents
-        run: |
-          DEB_FILE="$(ls dist/archivebox*.deb | head -1)"
-          echo "=== Package info ==="
-          dpkg-deb --info "$DEB_FILE"
-          echo ""
-          echo "=== Package contents ==="
-          dpkg-deb --contents "$DEB_FILE"
-          echo ""
-          echo "=== Control fields ==="
-          dpkg-deb --field "$DEB_FILE"
-
-      - name: Upload .deb artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: archivebox-${{ steps.version.outputs.version }}-${{ matrix.arch }}.deb
-          path: dist/*.deb
-
-  test:
-    needs: build
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - arch: amd64
-            runner: ubuntu-24.04
-          - arch: arm64
-            runner: ubuntu-24.04-arm
-    runs-on: ${{ matrix.runner }}
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v4
-        with:
-          enable-cache: false
-
-      - name: Install build dependencies
-        uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0
-        with:
-          packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1
-          version: 1.0
-
-      - name: Build local wheel
-        run: |
-          uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
-          uv build --wheel --out-dir /tmp/wheels/
-
-      - name: Download .deb artifact
-        uses: actions/download-artifact@v4
-        with:
-          pattern: archivebox-*-${{ matrix.arch }}.deb
-          merge-multiple: true
-
-      - name: Install system dependencies
-        run: |
-          sudo apt-get install -y python3.13 python3.13-venv python3-pip git curl wget
-
-      - name: Pre-seed virtualenv with local wheel before dpkg install
-        run: |
-          # CI-only: pre-seed the venv with a local wheel so we test the
-          # unreleased code, not whatever is on PyPI. We explicitly use python3.13
-          # here (matching the system dep installed above) to ensure the venv is
-          # created with the correct Python version. On real installs, install.sh
-          # handles this by preferring python3.13 and failing if < 3.13.
-          # When postinstall.sh runs during dpkg -i, it finds the venv already
-          # populated and just upgrades deps.
-          sudo mkdir -p /opt/archivebox
-          sudo python3.13 -m venv /opt/archivebox/venv
-          sudo /opt/archivebox/venv/bin/python3 -m pip install --quiet --upgrade pip setuptools
-          sudo /opt/archivebox/venv/bin/pip install --quiet /tmp/wheels/archivebox-*.whl
-          echo "[√] Pre-seeded /opt/archivebox/venv with local wheel"
-
-      - name: Install .deb package
-        run: |
-          # dpkg install will run postinstall which creates the user,
-          # sets up systemd, and tries pip install (which finds it already installed)
-          sudo dpkg -i archivebox*.deb || sudo apt-get install -f -y
-
-      - name: Verify archivebox is installed
-        run: |
-          which archivebox
-          archivebox version
-
-      - name: Verify wrapper script works
-        run: |
-          /usr/bin/archivebox version
-          /usr/bin/archivebox --help | head -5
-
-      - name: Test archivebox init as archivebox user
-        run: |
-          # The postinstall should have created the user
-          id archivebox
-          sudo mkdir -p /tmp/archivebox-test
-          sudo chown archivebox:archivebox /tmp/archivebox-test
-          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox init'
-
-      - name: Test archivebox status
-        run: |
-          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox status'
-
-      - name: Test archivebox add
-        run: |
-          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox add "https://example.com"'
-
-      - name: Verify systemd service file exists
-        run: |
-          test -f /usr/lib/systemd/system/archivebox.service
-          cat /usr/lib/systemd/system/archivebox.service
-
-  # Upload .deb to GitHub Release only after tests pass
-  release:
-    if: github.event_name == 'release' || github.event_name == 'workflow_call'
-    needs: [build, test]
-    runs-on: ubuntu-24.04
-    permissions:
-      contents: write
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Determine release tag
-        id: tag
-        run: |
-          TAG="${{ github.event.release.tag_name }}"
-          if [ -z "$TAG" ]; then
-            TAG="v$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')"
-            echo "[i] No release tag in event context, using version from pyproject.toml: $TAG"
-          fi
-          echo "tag=$TAG" >> "$GITHUB_OUTPUT"
-
-      - name: Download all .deb artifacts
-        uses: actions/download-artifact@v4
-        with:
-          pattern: archivebox-*.deb
-          merge-multiple: true
-
-      - name: Upload .deb to GitHub Release
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          TAG="${{ steps.tag.outputs.tag }}"
-          # Verify the release exists before uploading
-          if ! gh release view "$TAG" >/dev/null 2>&1; then
-            echo "[!] No GitHub Release found for tag $TAG."
-            if [ -n "${{ github.event.release.tag_name }}" ]; then
-              echo "[X] This was triggered by a release event — the release should exist. Failing."
-              exit 1
-            fi
-            echo "[i] Skipping upload (workflow_dispatch without a release)."
-            echo "    Create a release first or trigger via the release event."
-            exit 0
-          fi
-          gh release upload "$TAG" *.deb --clobber
@@ -5,7 +5,7 @@ on:
  workflow_call:
  push:
    branches:
-      - dev
+      - "**"
    tags:
      - 'v*'
  # pull_request:
@@ -19,7 +19,7 @@ permissions:
  packages: write

 concurrency:
-  group: docker-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

 jobs:
@@ -45,6 +45,47 @@ jobs:
        #   submodules: true
        #   fetch-depth: 1

+      - name: Wait for released ArchiveBox deps on PyPI
+        run: |
+          python3 - <<'PY'
+          import json
+          import re
+          import sys
+          import time
+          import tomllib
+          import urllib.request
+
+          watched = {"abxbus", "abxpkg", "abx-plugins", "abx-dl"}
+          deps = tomllib.loads(open("pyproject.toml", "rb").read().decode())["project"]["dependencies"]
+          required = {}
+          for dep in deps:
+              for name in watched:
+                  match = re.match(rf"{re.escape(name)}\s*(==|>=)\s*([^,;\s]+)", dep)
+                  if match:
+                      required[name] = match.group(2)
+
+          deadline = time.monotonic() + 300
+          missing = required.copy()
+          while missing and time.monotonic() < deadline:
+              for name, version in list(missing.items()):
+                  try:
+                      with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/{version}/json", timeout=20):
+                          available = True
+                  except Exception:
+                      available = False
+                  if available:
+                      print(f"{name} {version} is available on PyPI")
+                      missing.pop(name)
+                  else:
+                      print(f"{name} {version} is not available on PyPI yet")
+              if missing:
+                  time.sleep(10)
+
+          if missing:
+              print(f"Missing PyPI releases after wait: {missing}", file=sys.stderr)
+              sys.exit(1)
+          PY
+
      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
@@ -58,6 +99,41 @@ jobs:
      - name: Available platforms
        run: echo ${{ steps.buildx.outputs.platforms }}

+      - name: Wait for published abx-dl image
+        id: abx_dl_image
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          ABX_DL_VERSION="$(python3 - <<'PY'
+          import re
+          import tomllib
+
+          with open("pyproject.toml", "rb") as f:
+              deps = tomllib.load(f)["project"]["dependencies"]
+
+          for dep in deps:
+              match = re.match(r"abx-dl\s*(?:==|>=)\s*([^,;\s]+)", dep)
+              if match:
+                  print(match.group(1))
+                  break
+          else:
+              raise SystemExit("Missing abx-dl dependency in pyproject.toml")
+          PY
+          )"
+          ABX_DL_IMAGE="${ABX_DL_IMAGE:-archivebox/abx-dl:${ABX_DL_VERSION}}"
+          deadline=$((SECONDS + 1800))
+          until docker buildx imagetools inspect "${ABX_DL_IMAGE}" >/tmp/abx-dl-image.json; do
+            if (( SECONDS >= deadline )); then
+              echo "Timed out waiting for published ${ABX_DL_IMAGE}" >&2
+              exit 1
+            fi
+            echo "${ABX_DL_IMAGE} is not published yet; waiting..."
+            sleep 30
+          done
+
+          echo "image=${ABX_DL_IMAGE}" >> "$GITHUB_OUTPUT"
+          docker buildx imagetools inspect "${ABX_DL_IMAGE}"
+
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        if: github.event_name != 'pull_request'
@@ -78,13 +154,10 @@ jobs:
        run: |
          set -Eeuo pipefail
          VERSION="$(python3 - <<'PY'
-          from pathlib import Path
-          import re
+          import tomllib

-          match = re.search(r'^version = "([^"]+)"$', Path("pyproject.toml").read_text(), re.MULTILINE)
-          if not match:
-              raise SystemExit("Failed to read version from pyproject.toml")
-          print(match.group(1))
+          with open("pyproject.toml", "rb") as f:
+              print(tomllib.load(f)["project"]["version"])
          PY
          )"

@@ -93,11 +166,12 @@ jobs:
            echo "org.opencontainers.image.version=${VERSION}"
            echo "org.opencontainers.image.revision=${GITHUB_SHA}"
            echo 'org.opencontainers.image.source=https://github.com/ArchiveBox/ArchiveBox'
+            echo "io.archivebox.abx-dl.image=${{ steps.abx_dl_image.outputs.image }}"
            echo 'EOF'
            echo "version=${VERSION}"
          } >> "$GITHUB_OUTPUT"

-          echo "[+] Building ${{ matrix.platform }} for ${VERSION}"
+          echo "[+] Building ${{ matrix.platform }} for ${VERSION} using ${{ steps.abx_dl_image.outputs.image }}"

      - name: Build and push digest
        id: docker_build
@@ -111,14 +185,47 @@ jobs:
            ${{ env.DOCKERHUB_IMAGE }}
            ${{ env.GHCR_IMAGE }}
          labels: ${{ steps.docker_meta.outputs.labels }}
+          build-args: |
+            ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
          cache-from: type=gha,scope=${{ matrix.cache_scope }}
          cache-to: type=gha,mode=max,scope=${{ matrix.cache_scope }}
+          pull: true
          platforms: ${{ matrix.platform }}
          outputs: type=image,push-by-digest=true,name-canonical=true,push=true

      - name: Image digest
        run: echo ${{ steps.docker_build.outputs.digest }}

+      - name: Validate pushed image version, commit, and size
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          IMAGE="${DOCKERHUB_IMAGE}@${{ steps.docker_build.outputs.digest }}"
+          SHORT_SHA="${GITHUB_SHA::7}"
+
+          DATA_DIR="$(mktemp -d)"
+          docker run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox init
+          docker run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox version | tee /tmp/archivebox-version.out
+          docker run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox status
+          docker run --rm --platform '${{ matrix.platform }}' --entrypoint cat "$IMAGE" /VERSION.txt > /tmp/archivebox-image-version.txt
+          grep -F "ArchiveBox v${{ steps.docker_meta.outputs.version }}" /tmp/archivebox-version.out
+          grep -F "COMMIT_HASH=${SHORT_SHA}" /tmp/archivebox-version.out
+          grep -Fx "COMMIT_HASH=${GITHUB_SHA}" /tmp/archivebox-image-version.txt
+
+          python3 - "$IMAGE" <<'PY'
+          import json
+          import subprocess
+          import sys
+
+          image = sys.argv[1]
+          limit = 780 * 1024 * 1024
+          manifest = json.loads(subprocess.check_output(["docker", "manifest", "inspect", image]))
+          total = manifest.get("config", {}).get("size", 0) + sum(layer.get("size", 0) for layer in manifest.get("layers", []))
+          print(f"{image} compressed_size={total / 1024 / 1024:.2f} MiB limit={limit / 1024 / 1024:.2f} MiB")
+          if total > limit:
+              raise SystemExit(f"{image} is over the compressed size limit")
+          PY
+
      - name: Export digest
        shell: bash
        run: |
@@ -177,36 +284,49 @@ jobs:
        run: |
          set -Eeuo pipefail
          VERSION="$(python3 - <<'PY'
-          from pathlib import Path
-          import re
+          import tomllib

-          match = re.search(r'^version = "([^"]+)"$', Path("pyproject.toml").read_text(), re.MULTILINE)
-          if not match:
-              raise SystemExit("Failed to read version from pyproject.toml")
-          print(match.group(1))
+          with open("pyproject.toml", "rb") as f:
+              print(tomllib.load(f)["project"]["version"])
          PY
          )"
-          SHORT_SHA="${GITHUB_SHA::8}"
+          BRANCH_TAG="$(printf '%s' "${GITHUB_REF_NAME}" | tr -c 'A-Za-z0-9_.-' '-' | sed -E 's/^-+//; s/-+$//; s/-+/-/g' | cut -c1-128)"
+          SHORT_SHA="${GITHUB_SHA::12}"
+          test -n "$BRANCH_TAG"
+          test -n "$SHORT_SHA"

          {
            echo 'dockerhub_tags<<EOF'
-            echo "${DOCKERHUB_IMAGE}:dev"
+            if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
+              echo "${DOCKERHUB_IMAGE}:latest"
+            fi
+            echo "${DOCKERHUB_IMAGE}:${BRANCH_TAG}"
            echo "${DOCKERHUB_IMAGE}:${VERSION}"
            echo "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
            echo 'EOF'
            echo 'ghcr_tags<<EOF'
-            echo "${GHCR_IMAGE}:dev"
+            if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
+              echo "${GHCR_IMAGE}:latest"
+            fi
+            echo "${GHCR_IMAGE}:${BRANCH_TAG}"
            echo "${GHCR_IMAGE}:${VERSION}"
            echo "${GHCR_IMAGE}:sha-${SHORT_SHA}"
            echo 'EOF'
            echo "version=${VERSION}"
+            echo "branch_tag=${BRANCH_TAG}"
            echo "short_sha=${SHORT_SHA}"
          } >> "$GITHUB_OUTPUT"

          echo "[+] Publishing Docker Hub tags:"
-          printf '%s\n' "${DOCKERHUB_IMAGE}:dev" "${DOCKERHUB_IMAGE}:${VERSION}" "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
+          if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
+            printf '%s\n' "${DOCKERHUB_IMAGE}:latest"
+          fi
+          printf '%s\n' "${DOCKERHUB_IMAGE}:${BRANCH_TAG}" "${DOCKERHUB_IMAGE}:${VERSION}" "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
          echo "[+] Publishing GHCR tags:"
-          printf '%s\n' "${GHCR_IMAGE}:dev" "${GHCR_IMAGE}:${VERSION}" "${GHCR_IMAGE}:sha-${SHORT_SHA}"
+          if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
+            printf '%s\n' "${GHCR_IMAGE}:latest"
+          fi
+          printf '%s\n' "${GHCR_IMAGE}:${BRANCH_TAG}" "${GHCR_IMAGE}:${VERSION}" "${GHCR_IMAGE}:sha-${SHORT_SHA}"

      - name: Create Docker Hub manifest
        shell: bash
@@ -250,8 +370,12 @@ jobs:
        shell: bash
        run: |
          set -Eeuo pipefail
-          docker buildx imagetools inspect "${DOCKERHUB_IMAGE}:sha-${{ steps.docker_meta.outputs.short_sha }}"
-          docker buildx imagetools inspect "${GHCR_IMAGE}:sha-${{ steps.docker_meta.outputs.short_sha }}"
+          while IFS= read -r tag; do
+            [[ -n "$tag" ]] && docker buildx imagetools inspect "$tag"
+          done <<< "${{ steps.docker_meta.outputs.dockerhub_tags }}"
+          while IFS= read -r tag; do
+            [[ -n "$tag" ]] && docker buildx imagetools inspect "$tag"
+          done <<< "${{ steps.docker_meta.outputs.ghcr_tags }}"

      - name: Update README
        uses: peter-evans/dockerhub-description@v4
@@ -1,233 +0,0 @@
-name: Build Homebrew formula
-
-on:
-  workflow_dispatch:
-  workflow_call:
-  push:
-    branches: ['**']
-    paths:
-      - 'brew_dist/**'
-      - 'bin/build_brew.sh'
-      - 'bin/release_brew.sh'
-      - '.github/workflows/homebrew.yml'
-      - 'pyproject.toml'
-  # release trigger is handled by release.yml to avoid double-runs
-
-permissions:
-  contents: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build-and-test:
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [macos-latest, ubuntu-24.04]
-    runs-on: ${{ matrix.os }}
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Get version
-        id: version
-        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
-
-      - name: Validate formula template syntax
-        run: ruby -c brew_dist/archivebox.rb
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v4
-        with:
-          enable-cache: false
-
-      - name: Install build dependencies (Linux)
-        if: runner.os == 'Linux'
-        uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0
-        with:
-          packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1
-          version: 1.0
-
-      - name: Build local sdist
-        run: |
-          uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
-          uv build --sdist --out-dir /tmp/sdist/
-
-      - name: Generate formula from local sdist
-        run: |
-          VERSION="${{ steps.version.outputs.version }}"
-          SDIST_PATH="$(ls /tmp/sdist/archivebox-*.tar.gz | head -1)"
-          SDIST_SHA256="$(shasum -a 256 "$SDIST_PATH" | awk '{print $1}')"
-
-          # Install archivebox + poet into a temp venv to generate resource stanzas
-          python3 -m venv /tmp/poet-venv
-          source /tmp/poet-venv/bin/activate
-          pip install --quiet "$SDIST_PATH" homebrew-pypi-poet
-          echo "[+] Generating resource stanzas with homebrew-pypi-poet..."
-          RESOURCES="$(poet archivebox)"
-          deactivate
-
-          # For CI: use file:// URL pointing to local sdist
-          # For release: this gets overridden with the PyPI URL
-          SDIST_URL="file://${SDIST_PATH}"
-
-          cat > /tmp/archivebox.rb << RUBY
-class Archivebox < Formula
-  include Language::Python::Virtualenv
-
-  desc "Self-hosted internet archiving solution"
-  homepage "https://github.com/ArchiveBox/ArchiveBox"
-  url "${SDIST_URL}"
-  sha256 "${SDIST_SHA256}"
-  license "MIT"
-
-  depends_on "python@3.13"
-
-${RESOURCES}
-
-  def install
-    virtualenv_install_with_resources
-  end
-
-  def post_install
-    data_dir = var/"archivebox"
-    data_dir.mkpath
-    ENV["DATA_DIR"] = data_dir.to_s
-    system bin/"archivebox", "init"
-  end
-
-  def caveats
-    <<~EOS
-      ArchiveBox data is stored in:
-        #{var}/archivebox
-
-      To start archiving, run:
-        cd #{var}/archivebox && archivebox add 'https://example.com'
-
-      To start the web UI:
-        cd #{var}/archivebox && archivebox server 0.0.0.0:8000
-    EOS
-  end
-
-  test do
-    assert_match version.to_s, shell_output("#{bin}/archivebox version")
-  end
-end
-RUBY
-
-          echo "[√] Generated formula:"
-          ruby -c /tmp/archivebox.rb
-          cat /tmp/archivebox.rb
-
-      - name: Install Homebrew (Linux only)
-        if: runner.os == 'Linux'
-        run: |
-          NONINTERACTIVE=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
-          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
-          echo "HOMEBREW_PREFIX=$HOMEBREW_PREFIX" >> "$GITHUB_ENV"
-          echo "HOMEBREW_CELLAR=$HOMEBREW_CELLAR" >> "$GITHUB_ENV"
-          echo "HOMEBREW_REPOSITORY=$HOMEBREW_REPOSITORY" >> "$GITHUB_ENV"
-          echo "$HOMEBREW_PREFIX/bin" >> "$GITHUB_PATH"
-          echo "$HOMEBREW_PREFIX/sbin" >> "$GITHUB_PATH"
-
-      - name: Install brew dependencies
-        run: brew install python@3.13
-
-      - name: Install archivebox via brew from local formula
-        run: |
-          brew install --build-from-source --verbose /tmp/archivebox.rb
-
-      - name: Verify archivebox version
-        run: archivebox version
-
-      - name: Test archivebox init
-        run: |
-          mkdir -p /tmp/archivebox-test && cd /tmp/archivebox-test
-          archivebox init
-
-      - name: Test archivebox status
-        run: |
-          cd /tmp/archivebox-test
-          archivebox status
-
-  # On release only: generate the real formula with PyPI URL and push to tap
-  release:
-    if: github.event_name == 'release' || github.event_name == 'workflow_call'
-    needs: build-and-test
-    runs-on: macos-latest
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Get version
-        id: version
-        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - name: Wait for PyPI package availability
-        run: |
-          VERSION="${{ steps.version.outputs.version }}"
-          echo "[+] Waiting for archivebox==${VERSION} to be available on PyPI..."
-          for i in $(seq 1 30); do
-            if pip index versions archivebox 2>/dev/null | grep -q "$VERSION"; then
-              echo "[√] archivebox==${VERSION} is available on PyPI"
-              break
-            fi
-            if [ "$i" -eq 30 ]; then
-              echo "[!] Timed out waiting for PyPI. Trying to install anyway..."
-              break
-            fi
-            echo "    Attempt $i/30 - not yet available, waiting 30s..."
-            sleep 30
-          done
-
-      - name: Generate release formula via build_brew.sh
-        run: ./bin/build_brew.sh
-
-      - name: Test formula install
-        run: |
-          brew install --build-from-source brew_dist/archivebox.rb
-          archivebox version
-
-      - name: Upload formula artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: archivebox.rb
-          path: brew_dist/archivebox.rb
-
-      - name: Push to homebrew-archivebox tap
-        env:
-          GH_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }}
-        run: |
-          VERSION="${{ steps.version.outputs.version }}"
-          git clone "https://x-access-token:${GH_TOKEN}@github.com/ArchiveBox/homebrew-archivebox.git" /tmp/tap
-
-          cp brew_dist/archivebox.rb /tmp/tap/Formula/archivebox.rb 2>/dev/null || \
-            cp brew_dist/archivebox.rb /tmp/tap/archivebox.rb
-
-          cd /tmp/tap
-          git config user.name "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-          git add -A
-          if git diff --cached --quiet; then
-            echo "[i] No changes to formula, skipping push."
-          else
-            git commit -m "Update archivebox to v${VERSION}"
-            git push origin HEAD
-            echo "[√] Formula pushed to homebrew-archivebox tap"
-          fi
@@ -39,12 +39,13 @@ jobs:
      - name: Install uv
        uses: astral-sh/setup-uv@v6
        with:
+          version: "0.10.6"
          enable-cache: false

      - name: Install dependencies with uv
        run: |
-          uv sync --extra sonic --extra debug --all-groups --no-sources --no-cache
-          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
+          uv venv
+          uv pip install --all-groups -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[sonic,debug]"

      - name: Run prek
        run: uv run --no-sync --no-sources prek run --all-files
@@ -22,12 +22,14 @@ jobs:
      id-token: write

    runs-on: ubuntu-24.04
+    environment: pypi
    steps:
      - uses: actions/checkout@v4

      - name: Install uv
        uses: astral-sh/setup-uv@v6
        with:
+          version: "0.10.6"
          enable-cache: false

      - name: Set up Python
@@ -42,8 +44,89 @@ jobs:
          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
          version: 1.0

+      - name: Wait for released ArchiveBox deps on PyPI
+        run: |
+          python - <<'PY'
+          import re
+          import sys
+          import time
+          import tomllib
+          import urllib.request
+
+          watched = {"abxbus", "abxpkg", "abx-plugins", "abx-dl"}
+          deps = tomllib.loads(open("pyproject.toml", "rb").read().decode())["project"]["dependencies"]
+          required = {}
+          for dep in deps:
+              for name in watched:
+                  match = re.match(rf"{re.escape(name)}\s*(==|>=)\s*([^,;\s]+)", dep)
+                  if match:
+                      required[name] = match.group(2)
+
+          deadline = time.monotonic() + 300
+          missing = required.copy()
+          while missing and time.monotonic() < deadline:
+              for name, version in list(missing.items()):
+                  try:
+                      with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/{version}/json", timeout=20):
+                          available = True
+                  except Exception:
+                      available = False
+                  if available:
+                      print(f"{name} {version} is available on PyPI")
+                      missing.pop(name)
+                  else:
+                      print(f"{name} {version} is not available on PyPI yet")
+              if missing:
+                  time.sleep(10)
+
+          if missing:
+              print(f"Missing PyPI releases after wait: {missing}", file=sys.stderr)
+              sys.exit(1)
+          PY
+
+      - name: Prepare released dependency sync project
+        run: |
+          python - <<'PY'
+          from pathlib import Path
+          import json
+          import re
+          import urllib.request
+
+          source = Path("pyproject.toml")
+          target_dir = Path(".ci-uv-project")
+          target_dir.mkdir(exist_ok=True)
+          text = source.read_text()
+          text = text.replace(
+              'environments = ["sys_platform == \'darwin\'", "sys_platform == \'linux\'"]',
+              'environments = ["sys_platform == \'linux\'"]',
+          )
+
+          # CI runs immediately after publishing internal abx packages. PyPI's
+          # simple index can lag the version JSON endpoint, so sync against the
+          # exact wheel URLs from the version endpoint while leaving package
+          # metadata in pyproject.toml unchanged for the actual build.
+          for package in ("abxpkg", "abx-plugins", "abx-dl"):
+              match = re.search(rf'"{re.escape(package)}>=(?P<version>[^"]+)"', text)
+              if not match:
+                  continue
+              version = match.group("version")
+              with urllib.request.urlopen(f"https://pypi.org/pypi/{package}/{version}/json", timeout=20) as response:
+                  data = json.load(response)
+              wheel_url = next(url["url"] for url in data["urls"] if url["filename"].endswith(".whl"))
+              text = re.sub(
+                  rf'"{re.escape(package)}>=[^"]+"',
+                  f'"{package} @ {wheel_url}"',
+                  text,
+                  count=1,
+              )
+
+          (target_dir / "pyproject.toml").write_text(text)
+          PY
+
      - name: UV install archivebox dev + run sub-dependencies
-        run: uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
+        env:
+          UV_PROJECT_ENVIRONMENT: ${{ github.workspace }}/.venv
+        run: uv sync --project .ci-uv-project --all-extras --no-install-project --no-install-workspace --no-sources --no-cache

      - name: UV build archivebox and archivebox/pkgs/* packages
        run: |
@@ -53,17 +136,22 @@ jobs:
        uses: pypa/gh-action-pypi-publish@release/v1
        # ignore when publish to PyPI fails due to duplicate tag
        continue-on-error: true
-        with:
-          password: ${{ secrets.PYPI_PAT_SECRET }}

      - name: UV install archivebox and archivebox/pkgs/* locally for tests
-        run: uv sync --all-extras --no-sources --no-cache
-
-      - name: UV run archivebox init + archivebox version
+        env:
+          UV_PROJECT_ENVIRONMENT: ${{ github.workspace }}/.venv
        run: |
-          mkdir -p data && cd data
-          uv run --no-sync --no-sources archivebox init \
-          && uv run --no-sync --no-sources archivebox version
-          # && uv run archivebox add 'https://example.com' \
-          # && uv run archivebox status \
-          # || (echo "UV Failed to run archivebox!" && exit 1)
+          uv sync --project .ci-uv-project --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
+          uv pip install --python .venv/bin/python --no-deps dist/archivebox-*.whl
+
+      - name: Verify built package full install
+        env:
+          UV_PROJECT_ENVIRONMENT: ${{ github.workspace }}/.venv
+        run: |
+          set -Eeuo pipefail
+          DATA_DIR="$(mktemp -d)"
+          cd "$DATA_DIR"
+          uv run --project "$GITHUB_WORKSPACE/.ci-uv-project" --no-sync --no-sources archivebox init
+          timeout 30m uv run --project "$GITHUB_WORKSPACE/.ci-uv-project" --no-sync --no-sources archivebox install
+          uv run --project "$GITHUB_WORKSPACE/.ci-uv-project" --no-sync --no-sources archivebox version
+          uv run --project "$GITHUB_WORKSPACE/.ci-uv-project" --no-sync --no-sources archivebox status
@@ -3,26 +3,30 @@ name: Release State
 on:
  push:
    branches:
-      - '**'
+      - dev
  workflow_dispatch:

 permissions:
+  actions: write
  contents: write
  id-token: write

 concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+  cancel-in-progress: false

 jobs:
  release-state:
+    if: github.event_name != 'push' || !startsWith(github.event.head_commit.message, 'release:')
    runs-on: ubuntu-latest
+    environment: pypi
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          submodules: true
          ref: ${{ github.ref_name }}
+          token: ${{ secrets.RELEASE_GH_TOKEN || github.token }}

      - uses: actions/setup-python@v5
        with:
@@ -30,6 +34,7 @@ jobs:

      - uses: astral-sh/setup-uv@v6
        with:
+          version: "0.10.6"
          enable-cache: false

      - uses: actions/setup-node@v4
@@ -43,7 +48,13 @@ jobs:

      - name: Run release script
        env:
-          DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
-          GH_TOKEN: ${{ github.token }}
-          PYPI_PAT_SECRET: ${{ secrets.PYPI_PAT_SECRET }}
+          DEFAULT_BRANCH: dev
+          GH_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
        run: ./bin/release.sh
+
+      - name: Dispatch image and tap updates
+        env:
+          GH_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
+        run: |
+          gh workflow run docker.yml --ref dev
+          gh workflow run update-homebrew-tap.yml --ref dev
@@ -2,8 +2,8 @@ name: Release

 # Orchestrates the full release pipeline:
 #   1. Build and publish pip package to PyPI
-#   2. Build .deb packages and Homebrew formula (in parallel, after pip)
-#   3. Build Docker images (in parallel with deb/brew)
+#   2. Build Homebrew formula after pip
+#   3. Build Docker images in parallel
 #
 # Individual workflows also run on push for CI (see their own triggers).
 # Manual fallback for the hosted release pipeline. The normal prerelease loop
@@ -28,12 +28,6 @@ jobs:
    uses: ./.github/workflows/pip.yml
    secrets: inherit

-  debian:
-    name: Build .deb packages
-    needs: pip
-    uses: ./.github/workflows/debian.yml
-    secrets: inherit
-
  homebrew:
    name: Update Homebrew formula
    needs: pip
@@ -49,8 +49,16 @@ jobs:

            # Extract a display name for the test
            name="main/$(basename $test_file .py | sed 's/^test_//')"
+            needs_chromium=false
+            if grep -Eiq '(chrom|archivewebpage|PLUGINS=.*title|--plugins=.*title|SAVE_TITLE.*[Tt]rue)' "$test_file"; then
+              needs_chromium=true
+            fi
+            needs_sonic=false
+            if grep -Eiq "shutil\\.which\\([\"']sonic|SEARCH_BACKEND_ENGINE=.*sonic|worker_sonic" "$test_file"; then
+              needs_sonic=true
+            fi

-            json_array+="{\"path\":\"$test_file\",\"name\":\"$name\"}"
+            json_array+="{\"path\":\"$test_file\",\"name\":\"$name\",\"needs_chromium\":$needs_chromium,\"needs_sonic\":$needs_sonic}"
          done
          json_array+="]"

@@ -79,8 +87,12 @@ jobs:
            if grep -Riq "chrom" "abx-plugins/abx_plugins/plugins/$plugin_name"; then
              needs_chromium=true
            fi
+            needs_sonic=false
+            if [ "$plugin_name" = "search_backend_sonic" ]; then
+              needs_sonic=true
+            fi

-            json_array+="{\"plugin\":\"$plugin_name\",\"name\":\"plugin/$plugin_name\",\"needs_chromium\":$needs_chromium}"
+            json_array+="{\"plugin\":\"$plugin_name\",\"name\":\"plugin/$plugin_name\",\"needs_chromium\":$needs_chromium,\"needs_sonic\":$needs_sonic}"
          done
          json_array+="]"

@@ -129,7 +141,7 @@ jobs:
      - name: Install uv
        uses: astral-sh/setup-uv@v6
        with:
-          version: "latest"
+          version: "0.10.6"

      - name: Set up Node JS
        uses: actions/setup-node@v4
@@ -151,13 +163,79 @@ jobs:

      - name: Install dependencies with uv
        run: |
-          uv sync --dev --all-extras --no-sources --no-cache
-          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
+          uv venv
+          uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
+
+      - name: Install Chrome through ArchiveBox
+        if: ${{ matrix.test.needs_chromium }}
+        env:
+          MIN_CHROMIUM_MAJOR: "149"
+          LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_INSTALL_TIMEOUT: "900"
+          ABX_CI_DATA_DIR: ${{ runner.temp }}/archivebox-ci-data
+        run: |
+          set -euo pipefail
+          mkdir -p "$LIB_DIR" "$ABX_CI_DATA_DIR"
+          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
+          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
+
+          uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox install chrome
+
+          candidate="$(uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="chromium", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
+          if [ ! -x "$candidate" ]; then
+            echo "ArchiveBox did not install an executable Chromium binary: ${candidate:-not found}" >&2
+            exit 1
+          fi
+          version="$("$candidate" --version || true)"
+          major="$(printf '%s\n' "$version" | sed -E 's/.* ([0-9]+)\..*/\1/' | head -1)"
+          case "$major" in
+            ''|*[!0-9]*) major=0 ;;
+          esac
+          if [ "$major" -lt "$MIN_CHROMIUM_MAJOR" ]; then
+            echo "ArchiveBox installed Chromium is too old: ${version:-unknown}" >&2
+            exit 1
+          fi
+          echo "Using ArchiveBox-installed Chromium: $candidate ($version)"
+          echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV"
+
+      - name: Install Sonic through ArchiveBox
+        if: ${{ matrix.test.needs_sonic }}
+        env:
+          LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_INSTALL_TIMEOUT: "900"
+          ABX_CI_DATA_DIR: ${{ runner.temp }}/archivebox-ci-data
+        run: |
+          set -euo pipefail
+          mkdir -p "$LIB_DIR" "$ABX_CI_DATA_DIR"
+          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
+          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
+
+          uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox install search_backend_sonic
+
+          candidate="$(uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="sonic", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
+          if [ ! -x "$candidate" ]; then
+            echo "ArchiveBox did not install an executable Sonic binary: ${candidate:-not found}" >&2
+            exit 1
+          fi
+          version="$("$candidate" --version || true)"
+          echo "Using ArchiveBox-installed Sonic: $candidate (${version:-version unknown})"
+          echo "SONIC_BINARY=$candidate" >> "$GITHUB_ENV"
+          echo "$(dirname "$candidate")" >> "$GITHUB_PATH"

      - name: Run test - ${{ matrix.test.name }}
        run: |
          mkdir -p tests/out
+          set +e
          uv run --no-sync --no-sources pytest -xvs "${{ matrix.test.path }}" --basetemp=tests/out --ignore=archivebox/pkgs
+          status=$?
+          set -e
+          if [ "$status" -eq 5 ]; then
+            echo "No tests collected from ${{ matrix.test.path }}; treating as an empty test module."
+            exit 0
+          fi
+          exit "$status"

  plugin-tests:
    name: ${{ matrix.plugin.name }}
@@ -199,7 +277,7 @@ jobs:
      - name: Install uv
        uses: astral-sh/setup-uv@v6
        with:
-          version: "latest"
+          version: "0.10.6"

      - name: Set up Node JS
        uses: actions/setup-node@v4
@@ -221,58 +299,28 @@ jobs:

      - name: Install dependencies with uv
        run: |
-          uv sync --dev --all-extras --no-sources --no-cache
-          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
+          uv venv
+          uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"

-      - name: Select Chromium for extension-capable CDP
+      - name: Install Chrome through ArchiveBox
        if: ${{ matrix.plugin.needs_chromium }}
        env:
          MIN_CHROMIUM_MAJOR: "149"
          LIB_DIR: ${{ runner.temp }}/abx-lib
          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_INSTALL_TIMEOUT: "900"
+          ABX_CI_DATA_DIR: ${{ runner.temp }}/archivebox-ci-data
        run: |
          set -euo pipefail
-          mkdir -p "$LIB_DIR"
+          mkdir -p "$LIB_DIR" "$ABX_CI_DATA_DIR"
          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"

-          candidate="/usr/bin/chromium"
-          version=""
-          major=0
-          if [ -x "$candidate" ]; then
-            version="$("$candidate" --version || true)"
-            major="$(printf '%s\n' "$version" | sed -E 's/.* ([0-9]+)\..*/\1/' | head -1)"
-            case "$major" in
-              ''|*[!0-9]*) major=0 ;;
-            esac
-          fi
+          uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox install chrome

-          if [ "$major" -ge "$MIN_CHROMIUM_MAJOR" ]; then
-            echo "Using runner Chromium: $candidate ($version)"
-            echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV"
-            exit 0
-          fi
-
-          echo "Runner Chromium is unavailable or too old: ${version:-not found}"
-          echo "Installing Playwright Chromium into $LIB_DIR..."
-          uv run --no-sync --no-sources abxpkg install \
-            --binproviders=npm \
-            --lib="$LIB_DIR" \
-            --postinstall-scripts=false \
-            --min-release-age=0 \
-            --no-cache \
-            --overrides='{"npm":{"install_args":["playwright@next"]}}' \
-            playwright
-          uv run --no-sync --no-sources abxpkg install \
-            --binproviders=playwright \
-            --lib="$LIB_DIR" \
-            --bin-dir="$LIB_DIR/env/bin" \
-            --min-version="$MIN_CHROMIUM_MAJOR" \
-            chromium
-
-          candidate="$LIB_DIR/env/bin/chromium"
+          candidate="$(uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="chromium", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
          if [ ! -x "$candidate" ]; then
-            echo "Playwright Chromium shim was not created at $candidate" >&2
+            echo "ArchiveBox did not install an executable Chromium binary: ${candidate:-not found}" >&2
            exit 1
          fi
          version="$("$candidate" --version || true)"
@@ -281,12 +329,37 @@ jobs:
            ''|*[!0-9]*) major=0 ;;
          esac
          if [ "$major" -lt "$MIN_CHROMIUM_MAJOR" ]; then
-            echo "Playwright Chromium is still too old: ${version:-unknown}" >&2
+            echo "ArchiveBox installed Chromium is too old: ${version:-unknown}" >&2
            exit 1
          fi
-          echo "Using Playwright Chromium: $candidate ($version)"
+          echo "Using ArchiveBox-installed Chromium: $candidate ($version)"
          echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV"

+      - name: Install Sonic through ArchiveBox
+        if: ${{ matrix.plugin.needs_sonic }}
+        env:
+          LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_INSTALL_TIMEOUT: "900"
+          ABX_CI_DATA_DIR: ${{ runner.temp }}/archivebox-ci-data
+        run: |
+          set -euo pipefail
+          mkdir -p "$LIB_DIR" "$ABX_CI_DATA_DIR"
+          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
+          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
+
+          uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox install search_backend_sonic
+
+          candidate="$(uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="sonic", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
+          if [ ! -x "$candidate" ]; then
+            echo "ArchiveBox did not install an executable Sonic binary: ${candidate:-not found}" >&2
+            exit 1
+          fi
+          version="$("$candidate" --version || true)"
+          echo "Using ArchiveBox-installed Sonic: $candidate (${version:-version unknown})"
+          echo "SONIC_BINARY=$candidate" >> "$GITHUB_ENV"
+          echo "$(dirname "$candidate")" >> "$GITHUB_PATH"
+
      - name: Run plugin tests - ${{ matrix.plugin.name }}
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
@@ -49,7 +49,7 @@ jobs:
      - name: Install uv
        uses: astral-sh/setup-uv@v4
        with:
-          version: "latest"
+          version: "0.10.6"

      - name: Set up Node JS
        uses: actions/setup-node@v4
@@ -72,8 +72,8 @@ jobs:

      - name: Install dependencies with uv
        run: |
-          uv sync --dev --all-extras --no-sources --no-cache
-          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
+          uv venv
+          uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"

      ### Run the tests
      - name: Directory listing for debugging
@@ -81,10 +81,15 @@ jobs:
          pwd
          ls

-      - name: Archivebox version
+      - name: ArchiveBox full install check
        run: |
-          mkdir -p tests/out/data
-          DATA_DIR="$PWD/tests/out/data" uv run --no-sync --no-sources archivebox version
+          set -Eeuo pipefail
+          DATA_DIR="$PWD/tests/out/data"
+          mkdir -p "$DATA_DIR"
+          uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox init
+          timeout 30m uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox install
+          uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox version
+          uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox status

      - name: Test built package with pytest
        # TODO: remove this exception for windows once we get tests passing on that platform
@@ -0,0 +1,37 @@
+name: Update downstream packages
+
+on:
+  push:
+    branches: [dev]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: update-downstream-packages-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  dispatch:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Dispatch downstream package updates
+        env:
+          DOWNSTREAM_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || secrets.HOMEBREW_TAP_TOKEN }}
+        run: |
+          if [ -z "$DOWNSTREAM_TOKEN" ]; then
+            echo "RELEASE_GH_TOKEN/HOMEBREW_TAP_TOKEN is not configured; downstream schedules will update packages."
+            exit 0
+          fi
+
+          for repo in homebrew-archivebox debian-archivebox; do
+            curl --fail-with-body \
+              -X POST \
+              -H "Accept: application/vnd.github+json" \
+              -H "Authorization: Bearer ${DOWNSTREAM_TOKEN}" \
+              -H "X-GitHub-Api-Version: 2022-11-28" \
+              "https://api.github.com/repos/ArchiveBox/${repo}/dispatches" \
+              -d '{"event_type":"archivebox-dev-updated","client_payload":{"ref":"dev"}}'
+          done
@@ -0,0 +1,87 @@
+# ArchiveBox Agent Guide
+
+ArchiveBox is the full self-hosted web archiving app. Keep this repo on the `dev` branch.
+
+## Shared Standards
+
+- Use `uv` and `uv run` for Python commands. Do not use system `python`, direct `.venv/bin/python`, or `pip` commands.
+- Prefer existing repo patterns, helper APIs, fixtures, scripts, and command surfaces.
+- Keep edits focused and minimal. Do not add wrappers, shims, aliases, or extra abstraction layers unless the current code path requires them.
+- Do not weaken assertions, skip tests, xfail tests, or accept flaky behavior.
+- No mocks, monkeypatches, fakes, simulated handlers, fake binaries, fake hooks, fake buses, or direct shortcuts around user-facing flows.
+- Tests and verification should use real CLI commands, REST/API calls, browser UI flows, real hooks, real installs, real subprocesses, real DB rows, real files, and existing fixtures.
+- Assertions must verify real correctness: exit codes, returned values, DB state, filesystem contents, field values, rendered output, and side effects.
+- Start behavior fixes with a red failing test when a test is requested or practical.
+- Trace root causes from observed behavior. Do not paper over failures with retries, wider timeouts, broad fallbacks, or looser assertions.
+- Read `README.md` for the full setup, CLI, Docker, API, and release surface.
+
+## Development Setup
+
+```bash
+uv sync --dev --all-extras
+mkdir -p data
+cd data
+uv run --project .. archivebox init --install
+```
+
+Run collection commands from inside an initialized data directory:
+
+```bash
+cd data
+uv run --project .. archivebox status
+uv run --project .. archivebox add 'https://example.com'
+uv run --project .. archivebox run
+```
+
+## User-Facing Setup
+
+Recommended CLI install:
+
+```bash
+uv tool install archivebox
+mkdir -p ~/archivebox/data
+cd ~/archivebox/data
+archivebox init --install
+archivebox add 'https://example.com'
+archivebox server 0.0.0.0:8000
+```
+
+Alternative install methods:
+
+- Docker Compose / Docker
+- Homebrew
+- Debian package
+- pip
+
+## Basic Usage
+
+```bash
+archivebox version
+archivebox help
+archivebox status
+archivebox install
+archivebox add 'https://example.com'
+archivebox add --extract=title,screenshot,pdf 'https://example.com'
+archivebox list --json --with-headers
+archivebox search 'example'
+archivebox update --filter-type=domain example.com
+archivebox remove --filter-type=exact 'https://example.com'
+archivebox run
+```
+
+## Verification
+
+Use targeted tests for focused work:
+
+<!--pytest.mark.skip(reason="pytest invocation")-->
+```bash
+uv run pytest archivebox/tests/test_cli_add.py -q
+uv run prek run --all-files
+```
+
+Use the full release/deploy loop only when requested:
+
+<!--pytest.mark.skip(reason="release/deploy script")-->
+```bash
+./bin/release_dev_stack.sh
+```
@@ -1,497 +0,0 @@
-# Claude Code Development Guide for ArchiveBox
-
-## Quick Start
-
-```bash
-# Set up dev environment (always use uv, never pip directly)
-uv sync --dev --all-extras
-
-# Run tests as non-root user (required - ArchiveBox always refuses to run as root)
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/ -v'
-```
-
-## Development Environment Setup
-
-### Prerequisites
- Python 3.11+ (3.13 recommended)
- uv package manager
- A non-root user for running tests (e.g., `testuser`)
-
-### Install Dependencies
-```bash
-uv sync --dev --all-extras  # Always use uv, never pip directly
-```
-
-### Activate Virtual Environment
-```bash
-source .venv/bin/activate
-```
-
-### Common Gotchas
-
-#### File Permissions
-New files created by root need permissions fixed for testuser:
-```bash
-chmod 644 archivebox/tests/test_*.py
-```
-
-#### DATA_DIR Environment Variable
-ArchiveBox commands must run inside a data directory. Tests use temp directories - the `run_archivebox()` helper sets `DATA_DIR` automatically.
-
-## Code Style Guidelines
-
-### Naming Conventions for Grep-ability
-Use consistent naming for everything to enable easy grep-ability and logical grouping:
-
-**Principle**: Fewest unique names. If you must create a new unique name, make it grep and group well.
-
-**Examples**:
-```python
-# Filesystem migration methods - all start with fs_
-def fs_migration_needed() -> bool: ...
-def fs_migrate() -> None: ...
-def _fs_migrate_from_0_7_0_to_0_8_0() -> None: ...
-def _fs_migrate_from_0_8_0_to_0_9_0() -> None: ...
-def _fs_next_version(current: str) -> str: ...
-
-# Logging methods - ALL must start with log_ or _log
-def log_migration_start(snapshot_id: str) -> None: ...
-def _log_error(message: str) -> None: ...
-def log_validation_result(ok: bool, msg: str) -> None: ...
-```
-
-**Rules**:
- Group related functions with common prefixes
- Use `_` prefix for internal/private helpers within the same family
- ALL logging-related methods MUST start with `log_` or `_log`
- Search for all migration functions: `grep -r "def.*fs_.*(" archivebox/`
- Search for all logging: `grep -r "def.*log_.*(" archivebox/`
-
-### Minimize Unique Names and Data Structures
-**Do not invent new data structures, variable names, or keys if possible.** Try to use existing field names and data structures exactly to keep the total unique data structures and names in the codebase to an absolute minimum.
-
-**Example - GOOD**:
-```python
-# Binary has overrides field
-binary = Binary(overrides={'TIMEOUT': '60s'})
-
-# Binary reuses the same field name and structure
-class Binary(models.Model):
-    overrides = models.JSONField(default=dict)  # Same name, same structure
-```
-
-**Example - BAD**:
-```python
-# Don't invent new names like custom_bin_cmds, binary_overrides, etc.
-class Binary(models.Model):
-    custom_bin_cmds = models.JSONField(default=dict)  # ❌ New unique name
-```
-
-**Principle**: If you're storing the same conceptual data (e.g., `overrides`), use the same field name across all models and keep the internal structure identical. This makes the codebase predictable and reduces cognitive load.
-
-## Testing
-
-### CRITICAL: Never Run as Root
-ArchiveBox has a root check that prevents running as root user. All ArchiveBox commands (including tests) must run as non-root user inside a data directory:
-
-```bash
-# Run all migration tests
-sudo -u testuser bash -c 'source /path/to/.venv/bin/activate && python -m pytest archivebox/tests/test_migrations_*.py -v'
-
-# Run specific test file
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_08_to_09.py -v'
-
-# Run single test
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_fresh.py::TestFreshInstall::test_init_creates_database -xvs'
-```
-
-### Test File Structure
-```
-archivebox/tests/
-├── test_migrations_helpers.py    # Schemas, seeding functions, verification helpers
-├── test_migrations_fresh.py      # Fresh install tests
-├── test_migrations_04_to_09.py   # 0.4.x → 0.9.x migration tests
-├── test_migrations_07_to_09.py   # 0.7.x → 0.9.x migration tests
-└── test_migrations_08_to_09.py   # 0.8.x → 0.9.x migration tests
-```
-
-### Test Writing Standards
-
-#### NO MOCKS - Real Tests Only
-Tests must exercise real code paths:
- Create real SQLite databases with version-specific schemas
- Seed with realistic test data
- Run actual `python -m archivebox` commands via subprocess
- Query SQLite directly to verify results
-
-**If something is hard to test**: Modify the implementation to make it easier to test, or fix the underlying issue. Never mock, skip, simulate, or exit early from a test because you can't get something working inside the test.
-
-#### NO SKIPS
-Never use `@skip`, `skipTest`, or `pytest.mark.skip`. Every test must run. If a test is difficult, fix the code or test environment - don't disable the test.
-
-#### Strict Assertions
- `init` command must return exit code 0 (not `[0, 1]`)
- Verify ALL data is preserved, not just "at least one"
- Use exact counts (`==`) not loose bounds (`>=`)
-
-### Example Test Pattern
-```python
-def test_migration_preserves_snapshots(self):
-    """Migration should preserve all snapshots."""
-    result = run_archivebox(self.work_dir, ['init'], timeout=45)
-    self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}")
-
-    ok, msg = verify_snapshot_count(self.db_path, expected_count)
-    self.assertTrue(ok, msg)
-```
-
-### Testing Gotchas
-
-#### Extractors Disabled for Speed
-Tests disable all extractors via environment variables for faster execution:
-```python
-env['SAVE_TITLE'] = 'False'
-env['SAVE_FAVICON'] = 'False'
-# ... etc
-```
-
-#### Timeout Settings
-Use appropriate timeouts for migration tests (45s for init, 60s default).
-
-### Plugin Testing & Code Coverage
-
-**Target: 80-90% coverage** for critical plugins (screenshot, chrome, singlefile, dom)
-
-```bash
-# Run plugin tests with coverage (both Python + JavaScript)
-bash bin/test_plugins.sh screenshot
-
-# View coverage reports
-bash bin/test_plugins.sh --coverage-report
-# Or individual reports:
-coverage report --show-missing --include='archivebox/plugins/*' --omit='*/tests/*'
-```
-
-#### Plugin Test Structure
-
-Tests are **completely isolated** from ArchiveBox - they replicate production directory structure in temp dirs:
-
-```python
-# Correct production paths:
-# Crawl:    DATA_DIR/users/{username}/crawls/YYYYMMDD/example.com/{crawl-id}/{plugin}/
-# Snapshot: DATA_DIR/users/{username}/snapshots/YYYYMMDD/example.com/{snapshot-uuid}/{plugin}/
-
-with tempfile.TemporaryDirectory() as tmpdir:
-    data_dir = Path(tmpdir)
-
-    # Crawl-level plugin (e.g., chrome launcher)
-    crawl_dir = data_dir / 'users' / 'testuser' / 'crawls' / '20240101' / 'example.com' / 'crawl-123'
-    chrome_dir = crawl_dir / 'chrome'
-    chrome_dir.mkdir(parents=True)
-
-    # Snapshot-level plugin (e.g., screenshot)
-    snapshot_dir = data_dir / 'users' / 'testuser' / 'snapshots' / '20240101' / 'example.com' / 'snap-456'
-    screenshot_dir = snapshot_dir / 'screenshot'
-    screenshot_dir.mkdir(parents=True)
-
-    # Run hook in its output directory
-    result = subprocess.run(
-        ['node', str(SCREENSHOT_HOOK), '--url=https://example.com'],
-        cwd=str(screenshot_dir),
-        env={**get_test_env(), 'EXTRA_CONTEXT': '{"snapshot_id":"snap-456"}'},
-        capture_output=True,
-        timeout=120
-    )
-```
-
-#### Coverage Improvement Loop
-
-To improve from ~20% to 80%+:
-
-1. **Run tests**: `bash bin/test_plugins.sh screenshot` → Shows: `19.1% (13/68 ranges)`
-2. **Identify gaps**: Check hook file for untested paths (session connection vs fallback, config branches, error cases)
-3. **Add tests**: Test both execution paths (connect to session + launch own browser), skip conditions, error cases, config variations
-4. **Verify**: Re-run tests → Should show: `85%+ (58+/68 ranges)`
-
-**Critical**: JavaScript hooks have TWO paths that both must be tested (connect to session ~50% + launch browser ~30% + shared ~20%). Testing only one path = max 50% coverage possible!
-
-## Database Migrations
-
-### Generate and Apply Migrations
-```bash
-# Generate migrations (run from archivebox subdirectory)
-cd archivebox
-./manage.py makemigrations
-
-# Apply migrations to test database
-cd data/
-archivebox init
-```
-
-### Schema Versions
- **0.4.x**: First Django version. Tags as comma-separated string, no ArchiveResult model
- **0.7.x**: Tag model with M2M, ArchiveResult model, AutoField PKs
- **0.8.x**: Crawl/Seed models, UUID PKs, status fields, depth/retry_at
- **0.9.x**: Seed model removed, seed_id FK removed from Crawl
-
-### Testing a Migration Path
-1. Create SQLite DB with source version schema (from `test_migrations_helpers.py`)
-2. Seed with realistic test data using `seed_0_X_data()`
-3. Run `archivebox init` to trigger migrations
-4. Verify data preservation with `verify_*` functions
-5. Test CLI commands work post-migration (`status`, `list`, `add`, etc.)
-
-### Squashed Migrations
-When testing 0.8.x (dev branch), you must record ALL replaced migrations:
-```python
-# The squashed migration replaces these - all must be recorded
-('core', '0023_alter_archiveresult_options_archiveresult_abid_and_more'),
-('core', '0024_auto_20240513_1143'),
-# ... all 52 migrations from 0023-0074 ...
-('core', '0023_new_schema'),  # Also record the squashed migration itself
-```
-
-### Migration Strategy
- Squashed migrations for clean installs
- Individual migrations recorded for upgrades from dev branch
- `replaces` attribute in squashed migrations lists what they replace
-
-### Migration Gotchas
-
-#### Circular FK References in Schemas
-SQLite handles circular references with `IF NOT EXISTS`. Order matters less than in other DBs.
-
-## Plugin System Architecture
-
-### Plugin Dependency Rules
-
-Like other plugins, chrome plugins **ARE NOT ALLOWED TO DEPEND ON ARCHIVEBOX OR DJANGO**.
-However, they are allowed to depend on two shared files ONLY:
- `archivebox/plugins/chrome/chrome_utils.js` ← source of truth API for all basic chrome ops
- `archivebox/plugins/chrome/tests/chrome_test_utils.py` ← use for your tests, do not implement launching/killing/pid files/cdp/etc. in python, just extend this file as needed.
-
-### Chrome-Dependent Plugins
-
-Many plugins depend on Chrome/Chromium via CDP (Chrome DevTools Protocol). When checking for script name references or debugging Chrome-related issues, check these plugins:
-
-**Main puppeteer-based chrome installer + launcher plugin**:
- `chrome` - Core Chrome integration (CDP, launch, navigation)
-
-**Metadata extraction using chrome/chrome_utils.js / CDP**:
- `dns` - DNS resolution info
- `ssl` - SSL certificate info
- `headers` - HTTP response headers
- `redirects` - Capture redirect chains
- `staticfile` - Direct file downloads (e.g. if the url itself is a .png, .exe, .zip, etc.)
- `responses` - Capture network responses
- `consolelog` - Capture console.log output
- `title` - Extract page title
- `accessibility` - Extract accessibility tree
- `seo` - Extract SEO metadata
-
-**Extensions installed using chrome/chrome_utils.js / controlled using CDP**:
- `ublock` - uBlock Origin ad blocking
- `istilldontcareaboutcookies` - Cookie banner dismissal
- `twocaptcha` - 2captcha CAPTCHA solver integration
-
-**Page-alteration plugins to prepare the content for archiving**:
- `modalcloser` - Modal dialog dismissal
- `infiniscroll` - Infinite scroll handler
-
-**Main Extractor Outputs**:
- `dom` - DOM snapshot extraction
- `pdf` - Generate PDF snapshots
- `screenshot` - Generate screenshots
- `singlefile` - SingleFile archival, can be single-file-cli that launches chrome, or singlefile extension running inside chrome
-
-**Crawl URL parsers** (post-process dom.html, singlefile.html, staticfile, responses, headers, etc. for URLs to re-emit as new queued Snapshots during recursive crawling):
- `parse_dom_outlinks` - Extract outlinks from DOM (special, uses CDP to directly query browser)
- `parse_html_urls` - Parse URLs from HTML (doesn't use chrome directly, just reads dom.html)
- `parse_jsonl_urls` - Parse URLs from JSONL (doesn't use chrome directly, just reads dom.html)
- `parse_netscape_urls` - Parse Netscape bookmark format (doesn't use chrome directly, just reads dom.html)
-
-### Finding Chrome-Dependent Plugins
-
-```bash
-# Find all files containing "chrom" (case-insensitive)
-grep -ri "chrom" archivebox/plugins/*/on_*.* --include="*.*" 2>/dev/null | cut -d: -f1 | sort -u
-
-# Or get just the plugin names
-grep -ri "chrom" archivebox/plugins/*/on_*.* --include="*.*" 2>/dev/null | cut -d/ -f3 | sort -u
-```
-
-**Note**: This list may not be complete. Always run the grep command above when checking for Chrome-related script references or debugging Chrome integration issues.
-
-## Architecture Notes
-
-### Crawl Model (0.9.x)
- Crawl groups multiple Snapshots from a single `add` command
- Each `add` creates one Crawl with one or more Snapshots
- Seed model was removed - crawls now store URLs directly
-
-## Code Coverage
-
-### Overview
-
-Coverage tracking is enabled for passive collection across all contexts:
- Unit tests (pytest)
- Integration tests
- Dev server (manual testing)
- CLI usage
-
-Coverage data accumulates in `.coverage` file and can be viewed/analyzed to find dead code.
-
-### Install Coverage Tools
-
-```bash
-uv sync --dev  # Installs pytest-cov and coverage
-```
-
-### Running with Coverage
-
-#### Unit Tests
-```bash
-# Run tests with coverage
-pytest --cov=archivebox --cov-report=term archivebox/tests/
-
-# Or run specific test file
-pytest --cov=archivebox --cov-report=term archivebox/tests/test_migrations_08_to_09.py
-```
-
-#### Dev Server with Coverage
-```bash
-# Start dev server with coverage tracking
-coverage run --parallel-mode -m archivebox server
-
-# Or CLI commands
-coverage run --parallel-mode -m archivebox init
-coverage run --parallel-mode -m archivebox add https://example.com
-```
-
-#### Manual Testing (Always-On)
-To enable coverage during ALL Python executions (passive tracking):
-
-```bash
-# Option 1: Use coverage run wrapper
-coverage run --parallel-mode -m archivebox [command]
-
-# Option 2: Set environment variable (tracks everything)
-export COVERAGE_PROCESS_START=pyproject.toml
-# Now all Python processes will track coverage
-archivebox server
-archivebox add https://example.com
-```
-
-### Viewing Coverage
-
-#### Text Report (Quick View)
-```bash
-# Combine all parallel coverage data
-coverage combine
-
-# View summary
-coverage report
-
-# View detailed report with missing lines
-coverage report --show-missing
-
-# View specific file
-coverage report --include="archivebox/core/models.py" --show-missing
-```
-
-#### JSON Report (LLM-Friendly)
-```bash
-# Generate JSON report
-coverage json
-
-# View the JSON
-cat coverage.json | jq '.files | keys'  # List all files
-
-# Find files with low coverage
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered < 50) | "\(.key): \(.value.summary.percent_covered)%"'
-
-# Find completely uncovered files (dead code candidates)
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered == 0) | .key'
-
-# Get missing lines for a specific file
-cat coverage.json | jq '.files["archivebox/core/models.py"].missing_lines'
-```
-
-#### HTML Report (Visual)
-```bash
-# Generate interactive HTML report
-coverage html
-
-# Open in browser
-open htmlcov/index.html
-```
-
-### Isolated Runs
-
-To measure coverage for specific scenarios:
-
-```bash
-# 1. Reset coverage data
-coverage erase
-
-# 2. Run your isolated test/scenario
-pytest --cov=archivebox archivebox/tests/test_migrations_fresh.py
-# OR
-coverage run --parallel-mode -m archivebox add https://example.com
-
-# 3. View results
-coverage combine
-coverage report --show-missing
-
-# 4. Optionally export for analysis
-coverage json
-```
-
-### Finding Dead Code
-
-```bash
-# 1. Run comprehensive tests + manual testing to build coverage
-pytest --cov=archivebox archivebox/tests/
-coverage run --parallel-mode -m archivebox server  # Use the app manually
-coverage combine
-
-# 2. Find files with 0% coverage (strong dead code candidates)
-coverage json
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered == 0) | .key'
-
-# 3. Find files with <10% coverage (likely dead code)
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered < 10) | "\(.key): \(.value.summary.percent_covered)%"' | sort -t: -k2 -n
-
-# 4. Generate detailed report for analysis
-coverage report --show-missing > coverage_report.txt
-```
-
-### Tips
-
- **Parallel mode** (`--parallel-mode`): Allows multiple processes to track coverage simultaneously without conflicts
- **Combine**: Always run `coverage combine` before viewing reports to merge parallel data
- **Reset**: Use `coverage erase` to start fresh for isolated measurements
- **Branch coverage**: Enabled by default - tracks if both branches of if/else are executed
- **Exclude patterns**: Config in `pyproject.toml` excludes tests, migrations, type stubs
-
-## Debugging Tips
-
-### Check Migration State
-```bash
-sqlite3 /path/to/index.sqlite3 "SELECT app, name FROM django_migrations WHERE app='core' ORDER BY id;"
-```
-
-### Check Table Schema
-```bash
-sqlite3 /path/to/index.sqlite3 "PRAGMA table_info(core_snapshot);"
-```
-
-### Verbose Test Output
-```bash
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_08_to_09.py -xvs 2>&1 | head -200'
-```
-
-### Kill Zombie Chrome Processes
-```bash
-./bin/kill_chrome.sh
-```
@@ -1,38 +1,185 @@
-# This is the Dockerfile for ArchiveBox, it bundles the following main dependencies:
-#     python3.13, uv, python3-ldap
-#     curl, wget, git, dig, ping, tree, nano
-#     node, npm
-#     ArchiveBox and plugin runtime dependencies installed by archivebox init --install
-# Usage:
-#     git clone https://github.com/ArchiveBox/ArchiveBox && cd ArchiveBox
-#     docker build . -t archivebox
-#     docker run -v "$PWD/data":/data archivebox init
-#     docker run -v "$PWD/data":/data archivebox add 'https://example.com'
-#     docker run -v "$PWD/data":/data -it archivebox manage createsuperuser
-#     docker run -v "$PWD/data":/data -p 8000:8000 archivebox server
-#     docker buildx build . --platform=linux/amd64,linux/arm64 --push -t archivebox/archivebox:dev -t archivebox/archivebox:sha-abc123
-# Read more here: https://github.com/ArchiveBox/ArchiveBox#archivebox-development
+# syntax=docker/dockerfile:1.7

+# Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image.
+# abx-dl owns Python, Node, Chromium, and downloader plugin runtimes.
+# ArchiveBox owns sonic, supervisor, Django, and the app runtime.
+# Build abx-dl first, then point this file at it:
+#   docker buildx build ../abx-dl -f ../abx-dl/Dockerfile \
+#       --build-context abxbus=../abxbus \
+#       --build-context abxpkg=../abxpkg \
+#       --build-context abx-plugins=../abx-plugins \
+#       -t archivebox/abx-dl:dev
+#   docker buildx build . -f Dockerfile \
+#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:latest \
+#       -t archivebox:multistage

-#########################################################################################
+ARG ABX_DL_IMAGE=archivebox/abx-dl:latest

-### Example: Using ArchiveBox in your own project's Dockerfile ########
-
-# FROM python:3.13-slim
-# WORKDIR /data
-# RUN pip install archivebox>=0.9.0   # use latest release here
-# RUN archivebox install
-# RUN useradd -ms /bin/bash archivebox && chown -R archivebox /data
-
-#########################################################################################
+FROM archivebox/sonic:1.4.9 AS sonic
+FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base

 ARG TARGETPLATFORM
 ARG TARGETOS
 ARG TARGETARCH
-ARG TARGETVARIANT=
+ARG TARGETVARIANT

-FROM archivebox/sonic:1.4.9 AS sonic
-FROM ubuntu:24.04
+ENV TZ=UTC \
+    LANGUAGE=en_US:en \
+    LC_ALL=C.UTF-8 \
+    LANG=C.UTF-8 \
+    DEBIAN_FRONTEND=noninteractive \
+    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
+    PYTHONIOENCODING=UTF-8 \
+    PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    PIP_NO_COMPILE=1 \
+    PIP_ONLY_BINARY=aiohttp \
+    npm_config_loglevel=error
+
+ENV PYTHON_VERSION=3.13 \
+    NODE_VERSION=24
+
+ENV ARCHIVEBOX_USER=archivebox \
+    DEFAULT_ARCHIVEBOX_UID=911 \
+    DEFAULT_ARCHIVEBOX_GID=911 \
+    IN_DOCKER=True
+
+ENV CODE_DIR=/app \
+    DATA_DIR=/data \
+    CONFIG_DIR=/opt/archivebox \
+    ABXPKG_LIB_DIR=/opt/archivebox/lib \
+    PLAYWRIGHT_BROWSERS_PATH=/opt/archivebox/lib/playwright/cache \
+    PERSONAS_DIR=/data/personas \
+    CHROME_HEADLESS=true \
+    CHROME_SANDBOX=false \
+    CHROME_ISOLATION=crawl \
+    CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
+
+ENV TMP_DIR=/tmp/archivebox \
+    PIP_VENV_PYTHON=/venv/bin/python3 \
+    GOOGLE_API_KEY=no \
+    GOOGLE_DEFAULT_CLIENT_ID=no \
+    GOOGLE_DEFAULT_CLIENT_SECRET=no
+
+ENV HOME=/home/archivebox \
+    XDG_CONFIG_HOME=/home/archivebox/.config \
+    XDG_CACHE_HOME=/opt/archivebox/lib/cache \
+    ABXPKG_INSTALL_TIMEOUT=600 \
+    ABXPKG_POSTINSTALL_SCRIPTS=True \
+    ABXPKG_MIN_RELEASE_AGE=0 \
+    TIMEOUT=600
+
+ENV UV_COMPILE_BYTECODE=false \
+    UV_PYTHON_PREFERENCE=managed \
+    UV_PYTHON_INSTALL_DIR=/opt/uv/python \
+    UV_LINK_MODE=copy \
+    UV_PROJECT_ENVIRONMENT=/venv \
+    VIRTUAL_ENV=/venv \
+    PATH="/venv/bin:/opt/node/bin:$PATH"
+
+SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
+WORKDIR "$CODE_DIR"
+
+RUN cp /VERSION.txt /ABX-DL-VERSION.txt \
+    && (echo "[i] Docker build for ArchiveBox multistage starting..." \
+    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
+    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
+    && uname -a \
+    && sed -n '1,7p' /etc/os-release \
+    && abxpkg load --binproviders=env node \
+    && abxpkg load --binproviders=env uv \
+    ) | tee -a /VERSION.txt
+
+FROM archivebox-runtime-base AS archivebox-builder
+
+WORKDIR "$CODE_DIR"
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
+    <<'EOF'
+echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..."
+echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache
+echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends
+echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests
+rm -f /etc/apt/apt.conf.d/docker-clean
+apt-get update -qq
+apt-get install -qq -y --no-install-recommends \
+    build-essential gcc libldap2-dev libsasl2-dev libssl-dev
+/usr/bin/uv venv --clear /venv --python "${PYTHON_VERSION}"
+/usr/bin/uv pip install setuptools pip wheel
+
+mkdir -p /tmp/archivebox-uv-project
+/venv/bin/python - <<'PY'
+from pathlib import Path
+import json
+import re
+import urllib.request
+
+source = Path("/app/pyproject.toml")
+target = Path("/tmp/archivebox-uv-project/pyproject.toml")
+text = source.read_text()
+text = text.replace(
+    'environments = ["sys_platform == \'darwin\'", "sys_platform == \'linux\'"]',
+    'environments = ["sys_platform == \'linux\'"]',
+)
+
+# Docker builds need the just-published internal abx wheels immediately, but
+# PyPI simple can lag the version JSON endpoints by tens of minutes. Generate a
+# Docker-only dependency view from the version JSON so the published package
+# metadata stays normal while image builds remain resumable after a release.
+for package in ("abxpkg", "abx-plugins", "abx-dl"):
+    match = re.search(rf'"{re.escape(package)}>=(?P<version>[^"]+)"', text)
+    if not match:
+        continue
+    version = match.group("version")
+    with urllib.request.urlopen(f"https://pypi.org/pypi/{package}/{version}/json", timeout=20) as response:
+        data = json.load(response)
+    wheel_url = next(url["url"] for url in data["urls"] if url["filename"].endswith(".whl"))
+    text = re.sub(
+        rf'"{re.escape(package)}>=[^"]+"',
+        f'"{package} @ {wheel_url}"',
+        text,
+        count=1,
+    )
+
+target.write_text(text)
+PY
+
+/usr/bin/uv sync \
+    --project /tmp/archivebox-uv-project \
+    --refresh \
+    --no-dev \
+    --inexact \
+    --no-install-project \
+    --no-install-workspace \
+    --no-sources
+(find /venv/lib/python3.*/site-packages -type f -name '*.so' -exec strip --strip-unneeded {} + 2>/dev/null || true)
+rm -f /venv/bin/uv /venv/bin/uvx
+apt-get purge -y build-essential gcc libldap2-dev libsasl2-dev libssl-dev
+apt-get autoremove -y
+rm -rf /var/lib/apt/lists/*
+EOF
+
+COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
+RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
+    && COMMIT_HASH="$( \
+        if [[ -f "$CODE_DIR/.git/HEAD" ]]; then \
+            HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"; \
+            if [[ "$HEAD_REF" =~ ^[0-9a-fA-F]{40}$ ]]; then \
+                echo "$HEAD_REF"; \
+            elif [[ "$HEAD_REF" == ref:\ * ]]; then \
+                REF_PATH="${HEAD_REF#ref: }"; \
+                cat "$CODE_DIR/.git/$REF_PATH" 2>/dev/null || awk -v ref="$REF_PATH" '$2 == ref {print $1}' "$CODE_DIR/.git/packed-refs" 2>/dev/null || true; \
+            fi; \
+        fi)" \
+    && if [[ "$COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt; fi \
+    && /usr/bin/uv pip install --no-deps "$CODE_DIR" \
+    && rm -f /venv/bin/uv /venv/bin/uvx \
+    && /usr/bin/uv pip show archivebox | tee -a /VERSION.txt
+
+FROM archivebox-runtime-base

 LABEL name="archivebox" \
    maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
@@ -43,369 +190,70 @@ LABEL name="archivebox" \
    org.opencontainers.image.vendor="ArchiveBox" \
    org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \
    org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \
-    com.docker.image.source.entrypoint="Dockerfile" \
-    # TODO: release ArchiveBox as a Docker Desktop extension (requires these labels):
-    # https://docs.docker.com/desktop/extensions-sdk/architecture/metadata/
-    com.docker.desktop.extension.api.version=">= 1.4.7" \
-    com.docker.desktop.extension.icon="https://archivebox.io/icon.png" \
-    com.docker.extension.publisher-url="https://archivebox.io" \
-    com.docker.extension.screenshots='[{"alt": "Screenshot of Admin UI", "url": "https://github.com/ArchiveBox/ArchiveBox/assets/511499/e8e0b6f8-8fdf-4b7f-8124-c10d8699bdb2"}]' \
-    com.docker.extension.detailed-description='See here for detailed documentation: https://wiki.archivebox.io' \
-    com.docker.extension.changelog='See here for release notes: https://github.com/ArchiveBox/ArchiveBox/releases' \
-    com.docker.extension.categories='database,utility-tools'
+    com.docker.image.source.entrypoint="Dockerfile"

-ARG TARGETPLATFORM
-ARG TARGETOS
-ARG TARGETARCH
-ARG TARGETVARIANT
-######### Environment Variables #################################
-
-# Global build-time and runtime environment constants + default pkg manager config
-ENV TZ=UTC \
-    LANGUAGE=en_US:en \
-    LC_ALL=C.UTF-8 \
-    LANG=C.UTF-8 \
-    DEBIAN_FRONTEND=noninteractive \
-    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
-    PYTHONIOENCODING=UTF-8 \
-    PYTHONUNBUFFERED=1 \
-    PIP_DISABLE_PIP_VERSION_CHECK=1 \
-    PIP_ONLY_BINARY=aiohttp \
-    npm_config_loglevel=error
-
-# Language Version config
-ENV PYTHON_VERSION=3.13 \
-    NODE_VERSION=22.22.3
-
-# Non-root User config
-ENV ARCHIVEBOX_USER="archivebox" \
-    DEFAULT_PUID=911 \
-    DEFAULT_PGID=911 \
-    IN_DOCKER=True
-
-# ArchiveBox Source Code + Lib + Data paths
-ENV CODE_DIR=/app \
-    DATA_DIR=/data \
-    LIB_DIR=/opt/archivebox/lib \
-    ABXPKG_LIB_DIR=/opt/archivebox/lib \
-    SONIC_BINARY=/opt/archivebox/lib/env/bin/sonic \
-    PLAYWRIGHT_BROWSERS_PATH=/browsers
-
-# Bash SHELL config
-# http://redsymbol.net/articles/unofficial-bash-strict-mode/
-SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] 
-
-######### System Environment ####################################
-
-# Detect ArchiveBox version number by reading pyproject.toml (also serves to invalidate the entire build cache whenever pyproject.toml changes)
-WORKDIR "$CODE_DIR"
-
-# Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up back-to-back Docker builds)
-RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
-    && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-intall-recommends \
-    && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-intall-suggests \
-    && rm -f /etc/apt/apt.conf.d/docker-clean
-
-# Print debug info about build and save it to disk, for human eyes only, not used by anything else
-RUN (echo "[i] Docker build for ArchiveBox starting..." \
-    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})" \
-    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
-    && echo \
-    && echo "PYTHON=${PYTHON_VERSION} NODE=${NODE_VERSION} PATH=${PATH}" \
-    && echo "CODE_DIR=${CODE_DIR} DATA_DIR=${DATA_DIR}" \
-    && echo \
-    && uname -a \
-    && sed -n '1,7p' /etc/os-release \
-    && which bash && bash --version | sed -n '1p' \
-    && which dpkg && dpkg --version | sed -n '1p' \
-    && echo -e '\n\n' && env && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
-# Create non-privileged user for archivebox and chrome
-RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
-    && groupadd --system $ARCHIVEBOX_USER \
-    && useradd --system --create-home --gid $ARCHIVEBOX_USER --groups audio,video $ARCHIVEBOX_USER \
-    && usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
-    && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
-    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u $ARCHIVEBOX_USER) PGID=$(id -g $ARCHIVEBOX_USER)\n\n" \
-    | tee -a /VERSION.txt
-    # DEFAULT_PUID and DEFAULT_PID are overridden by PUID and PGID in /bin/docker_entrypoint.sh at runtime
-    # https://docs.linuxserver.io/general/understanding-puid-and-pgid
-
-# Install system apt dependencies (adding backports to access more recent apt updates)
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    echo "[+] APT Installing base system dependencies for $TARGETPLATFORM..." \
-    && mkdir -p /etc/apt/keyrings \
-    && apt-get update -qq \
-    && apt-get install -qq -y \
-        # 1. packaging dependencies
-        apt-transport-https ca-certificates apt-utils gnupg2 curl wget \
-        # 2. docker and init system dependencies
-        zlib1g-dev dumb-init gosu cron unzip grep dnsutils git ripgrep python3.12-venv default-jre-headless \
-        # 3. frivolous CLI helpers to make debugging failed archiving easier
-        tree nano iputils-ping \
-        # nano iputils-ping dnsutils htop procps jq yq
-    && rm -rf /var/lib/apt/lists/*
-
-# Install sonic search backend
 COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
 COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
-RUN (which sonic && sonic --version) | tee -a /VERSION.txt

-######### Language Environments ####################################
+COPY --from=archivebox-builder /opt/uv/python /opt/uv/python
+COPY --from=archivebox-builder /venv /venv
+COPY --from=archivebox-builder /app /app
+COPY --from=archivebox-builder /VERSION.txt /VERSION.txt

-# Set up Python environment
-# NOT NEEDED because we're using a pre-built python image, keeping this here in case we switch back to custom-building our own:
-#RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-#    --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-# RUN echo "[+] APT Installing PYTHON $PYTHON_VERSION for $TARGETPLATFORM (skipped, provided by base image)..." \
-    # && apt-get update -qq \
-    # && apt-get install -qq -y --no-upgrade \
-    #     python${PYTHON_VERSION} python${PYTHON_VERSION}-minimal python3-pip python${PYTHON_VERSION}-venv pipx \
-    # && rm -rf /var/lib/apt/lists/* \
-    # tell PDM to allow using global system python site packages
-    # && rm /usr/lib/python3*/EXTERNALLY-MANAGED \
-    # && ln -s "$(which python${PYTHON_VERSION})" /usr/bin/python \
-    # create global virtual environment GLOBAL_VENV to use (better than using pip install --global)
-    # && python3 -m venv --system-site-packages --symlinks $GLOBAL_VENV \
-    # && python3 -m venv --system-site-packages $GLOBAL_VENV \
-    # && python3 -m venv $GLOBAL_VENV \
-    # install global dependencies / python build dependencies in GLOBAL_VENV
-    # && pip install --upgrade pip setuptools wheel \
-    # Save version info
-    # && ( \
-    #     which python3 && python3 --version | grep " $PYTHON_VERSION" \
-    #     && which pip && pip --version \
-    #     # && which pdm && pdm --version \
-    #     && echo -e '\n\n' \
-    # ) | tee -a /VERSION.txt
+RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_ARCHIVEBOX_UID}..." \
+    && printf 'export PATH="/venv/bin:/opt/node/bin:$PATH"\n' > /etc/profile.d/archivebox-path.sh \
+    && ln -sf /venv/bin/archivebox /usr/local/bin/archivebox \
+    && ln -sf /venv/bin/daphne /usr/local/bin/daphne \
+    && ln -sf /venv/bin/supervisord /usr/local/bin/supervisord \
+    && ln -sf /venv/bin/supervisorctl /usr/local/bin/supervisorctl \
+    && getent group "$ARCHIVEBOX_USER" >/dev/null || groupadd --system "$ARCHIVEBOX_USER" \
+    && id -u "$ARCHIVEBOX_USER" >/dev/null 2>&1 || useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \
+    && usermod --append --groups audio,video "$ARCHIVEBOX_USER" \
+    && [[ "$(id -u "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_UID" ]] || usermod -u "$DEFAULT_ARCHIVEBOX_UID" "$ARCHIVEBOX_USER" \
+    && [[ "$(id -g "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_GID" ]] || groupmod -g "$DEFAULT_ARCHIVEBOX_GID" "$ARCHIVEBOX_USER" \
+    && abxpkg load --binproviders=env sonic | tee -a /VERSION.txt \
+    && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$XDG_CACHE_HOME" "$PLAYWRIGHT_BROWSERS_PATH" \
+    && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "/home/$ARCHIVEBOX_USER" \
+    && chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" \
+    && chown -R "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "$ABXPKG_LIB_DIR" \
+    && openssl rand -hex 16 > /etc/machine-id \
+    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER ARCHIVEBOX_UID=$(id -u "$ARCHIVEBOX_USER") ARCHIVEBOX_GID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
+    && echo -e "TMP_DIR=$TMP_DIR\nABXPKG_LIB_DIR=$ABXPKG_LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt

-
-# Set up Node environment from the official platform tarball. This avoids
-# NodeSource apt dependencies pulling Ubuntu's python3-minimal postinst into
-# emulated Docker builds.
-RUN --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
-    case "$TARGETARCH" in \
-        amd64) NODE_DIST_ARCH="x64" ;; \
-        arm64) NODE_DIST_ARCH="arm64" ;; \
-        *) echo "Unsupported TARGETARCH=$TARGETARCH for Node binary install" >&2; exit 1 ;; \
-    esac \
-    && NODE_TARBALL="node-v${NODE_VERSION}-linux-${NODE_DIST_ARCH}.tar.gz" \
-    && echo "[+] Installing NODE $NODE_VERSION for linux/${NODE_DIST_ARCH}..." \
-    && curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TARBALL}" \
-    && curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/SHASUMS256.txt" \
-    && grep "  ${NODE_TARBALL}$" SHASUMS256.txt | sha256sum -c - \
-    && tar -xzf "$NODE_TARBALL" -C /usr/local --strip-components=1 --no-same-owner \
-    && rm "$NODE_TARBALL" SHASUMS256.txt \
-    # Save version info
-    && ( \
-        which node && node --version \
-        && which npm && npm --version \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
-
-# Set up uv and main app /venv
-RUN curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR=/bin sh
-ENV UV_COMPILE_BYTECODE=0 \
-    UV_PYTHON_PREFERENCE=managed \
-    UV_PYTHON_INSTALL_DIR=/opt/uv/python \
-    UV_LINK_MODE=copy \
-    UV_PROJECT_ENVIRONMENT=/venv
-WORKDIR "$CODE_DIR"
-# COPY --chown=root:root --chmod=755 pyproject.toml "$CODE_DIR/"
-RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[+] UV Creating /venv using python ${PYTHON_VERSION} for ${TARGETPLATFORM}..." \
-    && uv venv /venv --python ${PYTHON_VERSION}
-ENV VIRTUAL_ENV=/venv PATH="/venv/bin:$PATH"
-RUN uv pip install setuptools pip \
-    && ( \
-        which python3 && python3 --version \
-        && which uv && uv self version \
-        && uv python find --system && uv python find \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
-
-######### ArchiveBox & Extractor Dependencies ##################################
-
-# Install ArchiveBox C-compiled/apt-installed Python dependencies in app /venv (currently only used for python-ldap)
-WORKDIR "$CODE_DIR"
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    #--mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-    echo "[+] APT Installing + Compiling python3-ldap for PIP archivebox[ldap] on ${TARGETPLATFORM}..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends \
-        build-essential gcc \
-        python3-dev libssl-dev libldap2-dev libsasl2-dev python3-ldap \
-        python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \
-    && uv pip install \
-        "python-ldap>=3.4.3" \
-    && apt-get purge -y \
-        python3-dev build-essential gcc \
-    && apt-get autoremove -y \
-    && rm -rf /var/lib/apt/lists/*
-
-
-# Runtime config used by plugin hooks. Plugin binaries and npm packages are
-# installed into LIB_DIR below by archivebox init --install and resolved from
-# LIB_DIR by ArchiveBox/abxpkg, not by mutating the container PATH.
-ENV PERSONAS_DIR=/data/personas \
-    CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \
-    CHROME_HEADLESS=true \
-    CHROME_SANDBOX=false \
-    CHROME_ISOLATION=crawl \
-    CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
-
-######### Build Dependencies ####################################
-
-
-# Install ArchiveBox Python venv dependencies from pyproject.toml.
-RUN --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
-    --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[+] PIP Installing ArchiveBox dependencies from pyproject.toml..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends build-essential gcc python3-dev \
-    && uv --no-cache sync \
-        --refresh \
-        --no-dev \
-        --inexact \
-        --all-extras \
-        --no-install-project \
-        --no-install-workspace \
-        --no-sources \
-    && apt-get purge -y python3-dev build-essential gcc \
-    && apt-get autoremove -y \
-    && find /venv -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && rm -rf /var/lib/apt/lists/*
-    # installs the pip packages that archivebox depends on, defined in pyproject.toml dependencies
-
-# Setup ArchiveBox runtime config
-ENV TMP_DIR=/tmp/archivebox \
-    PIP_VENV_PYTHON=/usr/bin/python3.12 \
-    GOOGLE_API_KEY=no \
-    GOOGLE_DEFAULT_CLIENT_ID=no \
-    GOOGLE_DEFAULT_CLIENT_SECRET=no
-
-WORKDIR "$DATA_DIR"
-RUN openssl rand -hex 16 > /etc/machine-id \
-    && mkdir -p "$DATA_DIR" \
-    && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" \
-    && mkdir -p "$TMP_DIR" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$TMP_DIR" \
-    && mkdir -p "$LIB_DIR" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
-    && mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \
-    && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && echo -e "\nTMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
-
-# Pre-bake plugin-managed runtime dependencies using the same abx-dl installer
-# path users run later, before copying ArchiveBox source so source-only edits do
-# not invalidate the heavy browser/plugin dependency layer.
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/puppeteer,sharing=locked,id=puppeteer-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing plugin runtime dependencies into $LIB_DIR..." \
-    && export PERSONAS_DIR="$LIB_DIR/personas" \
-    && export CHROME_USER_DATA_DIR="$LIB_DIR/chrome_profile" \
-    && export ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 \
-    && mkdir -p "$LIB_DIR" \
-    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends build-essential tesseract-ocr tesseract-ocr-eng \
-    && abxpkg install --no-cache --binproviders=pip --bin-dir="$LIB_DIR/env/bin" gallery-dl \
-    && abxpkg install --no-cache --binproviders=pip --bin-dir="$LIB_DIR/env/bin" --overrides='{"pip":{"install_args":["--no-deps","forum-dl","chardet==5.2.0","pydantic==2.12.3","pydantic-core==2.41.4","typing-extensions>=4.14.1","annotated-types>=0.6.0","typing-inspection>=0.4.2","beautifulsoup4","soupsieve","lxml","requests","urllib3","certifi","idna","charset-normalizer","tenacity","python-dateutil","six","html2text","warcio"]}}' forum-dl \
-    && if [ "$TARGETARCH" = "arm64" ]; then \
-        abxpkg install --binproviders=npm --overrides='{"npm":{"install_args":["playwright@next"]}}' playwright; \
-        abxpkg install --no-cache --install-timeout=600 --binproviders=playwright --bin-dir="$LIB_DIR/env/bin" chromium; \
-    fi \
-    && ABXPKG_INSTALL_TIMEOUT=600 TIMEOUT=600 PUID=0 PGID=0 abx-dl plugins --install \
-        accessibility archivedotorg archivewebpage base chrome chrome_mhtml chrome_screencast \
-        claudechrome claudecode claudecodecleanup claudecodeextract consolelog defuddle dns dom \
-        favicon forumdl gallerydl git hashes headers htmltotext infiniscroll \
-        istilldontcareaboutcookies liteparse media mercury modalcloser opendataloader papersdl \
-        parse_dom_outlinks parse_html_urls parse_jsonl_urls parse_netscape_urls parse_rss_urls \
-        parse_txt_urls pdf readability redirects responses screenshot search_backend_ripgrep \
-        search_backend_sonic search_backend_sqlite seo singlefile ssl sslcerts staticfile title \
-        trafilatura ublock wget ytdlp \
-    && abxpkg install --no-cache --binproviders=chromewebstore --overrides='{"chromewebstore":{"install_args":["fpeoodllldobpkbkabpblcfaogecpndd","--name=archivewebpage"]}}' archivewebpage \
-    && test -f "$LIB_DIR/chromewebstore/extensions/fpeoodllldobpkbkabpblcfaogecpndd__archivewebpage/manifest.json" \
-    && mkdir -p "$LIB_DIR/env/bin" \
-    && ln -sf "$(command -v node)" "$LIB_DIR/env/bin/node" \
-    && ln -sf "$(command -v npm)" "$LIB_DIR/env/bin/npm" \
-    && ln -sf "$(command -v java)" "$LIB_DIR/env/bin/java" \
-    && ln -sf "$(command -v git)" "$LIB_DIR/env/bin/git" \
-    && ln -sf "$(command -v rg)" "$LIB_DIR/env/bin/rg" \
-    && ln -sf "$(command -v sonic)" "$LIB_DIR/env/bin/sonic" \
-    && find "$LIB_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find "$LIB_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && rm -rf "$LIB_DIR/personas" "$LIB_DIR/chrome_profile" /opt/archivebox/lib-layer \
-    && mkdir -p /opt/archivebox/lib-layer \
-    && cp -a "$LIB_DIR"/. /opt/archivebox/lib-layer/ \
-    && apt-get purge -y build-essential \
-    && apt-get autoremove -y \
-    && rm -rf /var/lib/apt/lists/* \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" /opt/archivebox/lib-layer
-
-RUN rm -rf "$LIB_DIR" \
-    && mv /opt/archivebox/lib-layer "$LIB_DIR" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR"
-
-# Install ArchiveBox Python package from the checked-out source.
-WORKDIR "$CODE_DIR"
-COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
-RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
-    && pip install \
-        --no-deps \
-        "$CODE_DIR" \
-    && ( \
-        pip show archivebox \
-        && which archivebox \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt \
-    && find /venv "$CODE_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete
-    # installs archivebox itself, and any other vendored packages in pkgs/*, defined in pyproject.toml workspaces
-
-# Initialize an empty image collection without rerunning dependency installs.
 WORKDIR "$DATA_DIR"
 RUN echo "[+] Initializing image collection..." \
    && find "$DATA_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} + \
-    && PUID=0 PGID=0 archivebox init \
-    && find "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
-    && (chown "$DEFAULT_PUID:$DEFAULT_PGID" \
+    && HOME="$TMP_DIR" archivebox init \
+    && (chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" \
        "$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \
        "$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \
        "$DATA_DIR"/archive "$DATA_DIR"/archive/users "$DATA_DIR"/personas \
        "$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \
-        2>/dev/null || true)
+        "$CONFIG_DIR" "$CONFIG_DIR"/config.env "$CONFIG_DIR"/derived.env \
+        "$TMP_DIR" "$ABXPKG_LIB_DIR" "$XDG_CACHE_HOME" "$PLAYWRIGHT_BROWSERS_PATH" \
+        2>/dev/null || true) \
+    && find "$TMP_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} +

-# Print version for nice docker finish summary
-RUN (echo -e "\n\n[√] Finished Docker build successfully. Saving build summary in: /VERSION.txt" \
-    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})\n" \
+RUN chmod +x "$CODE_DIR"/bin/*.sh \
+    && chmod g+w "$TMP_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"
+
+RUN --mount=type=cache,target=/tmp/abxpkg-cache,sharing=locked,mode=1777 \
+    for forbidden_bin in gcc g++ make; do ! abxpkg load --binproviders=env "$forbidden_bin" >/dev/null 2>&1 || (echo "Unexpected build tool in runtime: $forbidden_bin" >&2 && exit 1); done \
+    && stat -c "%U:%G %a %n" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$CONFIG_DIR" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$ABXPKG_LIB_DIR" \
+    && python3 -c 'from abx_dl.models import discover_plugins; [print(f"export {plugin.enabled_key}=True") for plugin in discover_plugins(runtime="archivebox").values() if plugin.enabled_key in plugin.config.properties]' > /tmp/archivebox-enable-plugins.env \
+    && sort /tmp/archivebox-enable-plugins.env | tee -a /VERSION.txt \
+    && source /tmp/archivebox-enable-plugins.env \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups env HOME=/tmp/abxpkg-cache ABXPKG_NO_CACHE=True ABXPKG_TMP_CACHE_DIR=/tmp/abxpkg-cache archivebox install \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups env HOME=/tmp/abxpkg-cache archivebox version 2>&1 | tee -a /VERSION.txt \
+    && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
+
+RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully." \
+    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
    && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
    ) | tee -a /VERSION.txt

-# Verify ArchiveBox is installed and write full version/dependency info.
-RUN chmod +x "$CODE_DIR"/bin/*.sh \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
-    && chmod g+w "$TMP_DIR" "$LIB_DIR" "$LIB_DIR"/bin "$PLAYWRIGHT_BROWSERS_PATH" \
-    && GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 gosu "$ARCHIVEBOX_USER" archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury papersdl parse_rss_urls readability search_backend_sonic opendataloader search_backend_ripgrep 2>&1 | tee -a /VERSION.txt \
-    && gosu "$ARCHIVEBOX_USER" archivebox version 2>&1 | tee -a /VERSION.txt \
-    && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
-
-####################################################
-
-# Expose ArchiveBox's main interfaces to the outside world
 WORKDIR "$DATA_DIR"
 VOLUME "$DATA_DIR"
 EXPOSE 8000
@@ -1,229 +0,0 @@
-# syntax=docker/dockerfile:1.7
-
-# Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image.
-# abx-dl owns Python, Node, Chromium, and downloader plugin runtimes.
-# ArchiveBox owns ripgrep, sonic, supervisor, Django, and the app runtime.
-# Build abx-dl first, then point this file at it:
-#   docker buildx build ../abx-dl -f ../abx-dl/Dockerfile \
-#       --build-context abxbus=../abxbus \
-#       --build-context abxpkg=../abxpkg \
-#       --build-context abx-plugins=../abx-plugins \
-#       -t archivebox/abx-dl:dev
-#   docker buildx build . -f Dockerfile.multistage \
-#       --build-context abx-dl=docker-image://archivebox/abx-dl:dev \
-#       --build-arg ABX_DL_IMAGE=abx-dl \
-#       -t archivebox:multistage
-
-ARG TARGETPLATFORM=linux/amd64
-ARG TARGETOS=linux
-ARG TARGETARCH=amd64
-ARG TARGETVARIANT=
-ARG ABX_DL_IMAGE=abx-dl
-
-FROM ${ABX_DL_IMAGE} AS abx-dl
-FROM archivebox/sonic:1.4.9 AS sonic
-FROM ubuntu:24.04 AS archivebox-runtime-base
-
-ARG TARGETPLATFORM=linux/amd64
-ARG TARGETOS=linux
-ARG TARGETARCH=amd64
-ARG TARGETVARIANT
-
-ENV TZ=UTC \
-    LANGUAGE=en_US:en \
-    LC_ALL=C.UTF-8 \
-    LANG=C.UTF-8 \
-    DEBIAN_FRONTEND=noninteractive \
-    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
-    PYTHONIOENCODING=UTF-8 \
-    PYTHONUNBUFFERED=1 \
-    PIP_DISABLE_PIP_VERSION_CHECK=1 \
-    PIP_ONLY_BINARY=aiohttp \
-    npm_config_loglevel=error
-
-ENV PYTHON_VERSION=3.13 \
-    NODE_VERSION=22.22.3
-
-ENV ARCHIVEBOX_USER=archivebox \
-    DEFAULT_PUID=911 \
-    DEFAULT_PGID=911 \
-    IN_DOCKER=True
-
-ENV CODE_DIR=/app \
-    DATA_DIR=/data \
-    LIB_DIR=/opt/archivebox/lib \
-    ABXPKG_LIB_DIR=/opt/archivebox/lib \
-    PLAYWRIGHT_BROWSERS_PATH=/opt/archivebox/lib/playwright/cache \
-    PERSONAS_DIR=/data/personas \
-    CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \
-    CHROME_HEADLESS=true \
-    CHROME_SANDBOX=false \
-    CHROME_ISOLATION=crawl \
-    CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
-
-ENV TMP_DIR=/tmp/archivebox \
-    PIP_VENV_PYTHON=/venv/bin/python3 \
-    GOOGLE_API_KEY=no \
-    GOOGLE_DEFAULT_CLIENT_ID=no \
-    GOOGLE_DEFAULT_CLIENT_SECRET=no
-
-ENV UV_COMPILE_BYTECODE=0 \
-    UV_PYTHON_PREFERENCE=managed \
-    UV_PYTHON_INSTALL_DIR=/opt/uv/python \
-    UV_LINK_MODE=copy \
-    UV_PROJECT_ENVIRONMENT=/venv \
-    VIRTUAL_ENV=/venv \
-    PATH="/venv/bin:/opt/node/bin:$PATH"
-
-SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
-WORKDIR "$CODE_DIR"
-
-RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
-    && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends \
-    && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests \
-    && rm -f /etc/apt/apt.conf.d/docker-clean
-
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    echo "[+] APT Installing ArchiveBox base runtime dependencies for $TARGETPLATFORM..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y \
-        apt-transport-https apt-utils ca-certificates curl wget gnupg2 \
-        dumb-init gosu unzip git grep ripgrep dnsutils iputils-ping procps tree nano \
-        cron openssl xz-utils zlib1g libldap2 libsasl2-2 libssl3 libsqlite3-0 \
-        libasound2t64 libatk-bridge2.0-0 libatk1.0-0 libcairo2 libcups2 \
-        libdbus-1-3 libdrm2 libgbm1 libglib2.0-0 libgtk-3-0 libnspr4 libnss3 \
-        libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxdamage1 \
-        libxext6 libxfixes3 libxkbcommon0 libxrandr2 libxshmfence1 \
-        fonts-liberation fonts-noto-color-emoji xdg-utils \
-        ffmpeg imagemagick tesseract-ocr tesseract-ocr-eng openjdk-21-jre-headless \
-    && rm -rf /var/lib/apt/lists/*
-
-# Runtime-owned layers copied from the abx-dl image.
-COPY --from=abx-dl /bin/uv /bin/uv
-COPY --from=abx-dl /opt/uv/python /opt/uv/python
-COPY --from=abx-dl /opt/node /opt/node
-COPY --from=abx-dl /venv /venv
-COPY --from=abx-dl /VERSION.txt /ABX-DL-VERSION.txt
-
-RUN (echo "[i] Docker build for ArchiveBox multistage starting..." \
-    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
-    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
-    && uname -a \
-    && sed -n '1,7p' /etc/os-release \
-    && which python3 && python3 --version \
-    && which node && node --version \
-    && which uv && uv self version \
-    ) | tee -a /VERSION.txt
-
-ENV PYTHONDONTWRITEBYTECODE=1
-
-FROM archivebox-runtime-base AS archivebox-builder
-
-WORKDIR "$CODE_DIR"
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
-    echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends \
-        build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
-    && uv sync \
-        --refresh \
-        --no-dev \
-        --inexact \
-        --all-extras \
-        --no-install-project \
-        --no-install-workspace \
-        --no-sources \
-    && apt-get purge -y build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
-    && apt-get autoremove -y \
-    && find /venv -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && rm -rf /var/lib/apt/lists/*
-
-COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
-RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
-    && uv pip install --no-deps "$CODE_DIR" \
-    && (uv pip show archivebox && which archivebox) | tee -a /VERSION.txt \
-    && find /venv "$CODE_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete
-
-FROM archivebox-runtime-base
-
-LABEL name="archivebox" \
-    maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
-    description="All-in-one self-hosted internet archiving solution" \
-    homepage="https://github.com/ArchiveBox/ArchiveBox" \
-    documentation="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker" \
-    org.opencontainers.image.title="ArchiveBox" \
-    org.opencontainers.image.vendor="ArchiveBox" \
-    org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \
-    org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \
-    com.docker.image.source.entrypoint="Dockerfile.multistage"
-
-COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
-COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
-
-COPY --from=archivebox-builder /venv /venv
-COPY --from=archivebox-builder /app /app
-COPY --from=archivebox-builder /VERSION.txt /VERSION.txt
-COPY --from=abx-dl --chown=911:911 /opt/archivebox/lib /opt/archivebox/lib
-
-RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
-    && groupadd --system "$ARCHIVEBOX_USER" \
-    && useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \
-    && usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
-    && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
-    && (which sonic && sonic --version) | tee -a /VERSION.txt \
-    && mkdir -p "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && openssl rand -hex 16 > /etc/machine-id \
-    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u "$ARCHIVEBOX_USER") PGID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
-    && echo -e "TMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
-
-WORKDIR "$DATA_DIR"
-RUN echo "[+] Initializing image collection..." \
-    && find "$DATA_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} + \
-    && PUID=0 PGID=0 archivebox init \
-    && find "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
-    && (chown "$DEFAULT_PUID:$DEFAULT_PGID" \
-        "$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \
-        "$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \
-        "$DATA_DIR"/archive "$DATA_DIR"/archive/users "$DATA_DIR"/personas \
-        "$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \
-        2>/dev/null || true)
-
-RUN chmod +x "$CODE_DIR"/bin/*.sh \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
-    && chmod g+w "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 gosu "$ARCHIVEBOX_USER" archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury papersdl parse_rss_urls readability search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \
-    && "$LIB_DIR/env/bin/chromium" --version | tee -a /VERSION.txt \
-    && "$LIB_DIR/pip/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \
-    && /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \
-    && /usr/local/bin/sonic --version | tee -a /VERSION.txt \
-    && /venv/bin/supervisord --version | tee -a /VERSION.txt \
-    && ! command -v gcc \
-    && ! command -v g++ \
-    && ! command -v make \
-    && gosu "$ARCHIVEBOX_USER" archivebox version 2>&1 | tee -a /VERSION.txt \
-    && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
-
-RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully." \
-    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
-    && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
-    ) | tee -a /VERSION.txt
-
-WORKDIR "$DATA_DIR"
-VOLUME "$DATA_DIR"
-EXPOSE 8000
-
-HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
-    CMD curl --fail --silent --show-error --max-time 5 --connect-timeout 2 'http://admin.archivebox.localhost:8000/health/' | grep -q 'OK'
-
-ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh"]
-CMD ["archivebox", "server", "--init", "0.0.0.0:8000"]
@@ -1,6 +1,6 @@
 MIT License

-Copyright (c) 2024 Nick Sweeting
+Copyright (c) 2026 Nick Sweeting

 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -29,7 +29,7 @@ Without active preservation effort, everything on the internet eventually disapp
 *ArchiveBox is an open source tool that lets organizations & individuals archive both public & private web content while retaining control over their data. It can be used to save copies of bookmarks, preserve evidence for legal cases, backup photos from FB/Insta/Flickr or media from YT/Soundcloud/etc., save research papers, and more...*
 <br/>

-> ➡️ Get ArchiveBox with `pip install archivebox` on [Linux](#quickstart)/[macOS](#quickstart), or via **[Docker](#quickstart)** ⭐️ on any OS.  
+> ➡️ Get ArchiveBox with `pip install archivebox>=0.9.5rc1` on [Linux](#quickstart)/[macOS](#quickstart), or via **[Docker](#quickstart)** ⭐️ on any OS.  

 *Once installed, you can interact with it through the: [Browser Extension](https://github.com/ArchiveBox/archivebox-browser-extension), [CLI](#usage), [self-hosted web interface](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive), [Python API](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage), or [filesystem](#static-archive-exporting).*

@@ -92,7 +92,7 @@ docker run -it -v $PWD:/data archivebox/archivebox init
 <br/>
 <br/>
 # Option C: Or install it with your preferred pkg manager (see Quickstart below for apt, brew, and more)
-pip install archivebox
+pip install 'archivebox>=0.9.5rc1'
 mkdir -p ~/archivebox/data && cd ~/archivebox/data
 archivebox init
 archivebox install
@@ -295,18 +295,18 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <summary><b><img src="https://user-images.githubusercontent.com/511499/117448075-49597580-af0c-11eb-91ba-f34fff10096b.png" alt="aptitude" height="28px" align="top"/> <code>apt</code></b> (Ubuntu/Debian/etc.)</summary>
 <br/>
 <ol>
-<li>Download and install the <code>.deb</code> package from the <a href="https://github.com/ArchiveBox/ArchiveBox/releases">latest release</a>.
-<pre lang="bash"><code style="white-space: pre-line"># download the .deb for your architecture (amd64 or arm64)
-ARCH="$(dpkg --print-architecture)"
-VERSION="$(curl -fsSL https://api.github.com/repos/ArchiveBox/ArchiveBox/releases/latest | python3 -c "import sys,json; print(json.load(sys.stdin)['tag_name'].lstrip('v'))")"
-curl -fsSL "https://github.com/ArchiveBox/ArchiveBox/releases/latest/download/archivebox_${VERSION}_${ARCH}.deb" -o /tmp/archivebox.deb
-sudo apt install /tmp/archivebox.deb
-archivebox version                         # make sure all dependencies are installed
+<li>Add the third-party ArchiveBox apt repo and install <code>archivebox</code>.
+<pre lang="bash"><code style="white-space: pre-line">echo 'deb [trusted=yes] https://archivebox.github.io/debian-archivebox dev main' | sudo tee /etc/apt/sources.list.d/archivebox.list
+sudo apt update
+sudo apt install archivebox
+archivebox version                         # make sure the package is installed
 </code></pre>
 </li>
 <li>Create a new empty directory and initialize your collection (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
-archivebox init --install
+archivebox init
+sudo archivebox install
+archivebox add 'https://example.com'
 </code></pre>
 <br/>
 </li>
@@ -476,6 +476,22 @@ For more discussion on managed and paid hosting options see here: <a href="https
 ArchiveBox commands can be run in a terminal [directly on your host](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage), or via [Docker](https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage-1)/[Docker Compose](https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage).  
 <sup>(depending on how you chose to install it above)</sup>

+<!--
+```bash
+set -euo pipefail
+__archivebox_docs_home="$(mktemp -d)"
+export HOME="$__archivebox_docs_home"
+mkdir -p ~/archivebox/data
+cd ~/archivebox/data
+archivebox init
+archivebox version
+archivebox help
+test -f index.sqlite3
+test -d archive
+rm -rf "$__archivebox_docs_home"
+```
+-->
+<!--pytest.mark.skip(reason='Mixed host/Docker alternatives include interactive Docker commands')-->
 ```bash
 mkdir -p ~/archivebox/data   # create a new data dir anywhere
 cd ~/archivebox/data         # IMPORTANT: cd into the directory
@@ -512,10 +528,12 @@ archivebox persona create --import=chrome personal
 # make sure you have pip-installed ArchiveBox and it's available in your $PATH first  
 <br/>
 # archivebox [subcommand] [--help]
-archivebox init --install      # safe to run init multiple times (also how you update versions)
+mkdir -p ~/archivebox/data && cd ~/archivebox/data
+archivebox init
+sudo archivebox install
+archivebox add 'https://example.com'
 archivebox version           # get archivebox version info + check dependencies
 archivebox help              # get list of archivebox subcommands that can be run
-archivebox add --depth=1 'https://news.ycombinator.com'
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage">Usage: CLI Usage</a> wiki. ➡️</i>
 </details>
@@ -529,10 +547,11 @@ archivebox add --depth=1 'https://news.ycombinator.com'
 # make sure you have `docker-compose.yml` from the Quickstart instructions first
 <br/>
 # docker compose run archivebox [subcommand] [--help]
-docker compose run archivebox init --install
+docker compose run archivebox init
+docker compose run archivebox install
 docker compose run archivebox version
 docker compose run archivebox help
-docker compose run archivebox add --depth=1 'https://news.ycombinator.com'
+docker compose run archivebox add 'https://example.com'
 # to start webserver: docker compose up
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage">Usage: Docker Compose CLI</a> wiki. ➡️</i>
@@ -547,10 +566,11 @@ docker compose run archivebox add --depth=1 'https://news.ycombinator.com'
 # make sure you create and cd into in a new empty directory first  
 <br/>
 # docker run -it -v $PWD:/data archivebox/archivebox [subcommand] [--help]
-docker run -v $PWD:/data -it archivebox/archivebox init --install
+docker run -v $PWD:/data -it archivebox/archivebox init
+docker run -v $PWD:/data -it archivebox/archivebox install
 docker run -v $PWD:/data -it archivebox/archivebox version
 docker run -v $PWD:/data -it archivebox/archivebox help
-docker run -v $PWD:/data -it archivebox/archivebox add --depth=1 'https://news.ycombinator.com'
+docker run -v $PWD:/data -it archivebox/archivebox add 'https://example.com'
 # to start webserver: docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage-1">Usage: Docker CLI</a> wiki. ➡️</i>
@@ -676,10 +696,36 @@ docker run -it -v $PWD:/data archivebox/archivebox add --depth=1 'https://exampl
 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/e1e5bd78-b0b6-45dc-914c-e1046fee4bc4" width="330px" align="right" style="float: right"/>


+<!--
+```bash
+set -euo pipefail
+__archivebox_docs_home="$(mktemp -d)"
+export HOME="$__archivebox_docs_home"
+mkdir -p ~/archivebox/data ~/Downloads
+cd ~/archivebox/data
+archivebox init
+cat > ~/Downloads/some_feed.xml <<'EOF'
+<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0">
+  <channel>
+    <title>ArchiveBox docs test feed</title>
+    <link>https://example.com/</link>
+    <description>ArchiveBox docs test feed</description>
+    <item>
+      <title>Feed item</title>
+      <link>https://example.com/from-feed</link>
+      <guid>https://example.com/from-feed</guid>
+    </item>
+  </channel>
+</rss>
+EOF
+```
+-->
+<!--pytest-codeblocks:cont-->
 ```bash
 # archivebox add --help
 archivebox add 'https://example.com/some/page'
-archivebox add --parser=generic_rss < ~/Downloads/some_feed.xml
+archivebox add --depth=1 --plugins=parse_rss_urls "file://$HOME/Downloads/some_feed.xml"
 archivebox add --depth=1 'https://news.ycombinator.com#2020-12-12'
 echo 'http://example.com' | archivebox add
 echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox add
@@ -689,6 +735,19 @@ echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox a
 # if using Docker Compose, add -T when piping stdin / stdout:
 # echo 'https://example.com' | docker compose run -T archivebox add
 ```
+<!--pytest-codeblocks:cont-->
+<!--
+```bash
+archivebox list --json > snapshots.json
+grep -q 'https://example.com/some/page' snapshots.json
+grep -q 'https://example.com/from-feed' snapshots.json
+grep -q 'https://news.ycombinator.com#2020-12-12' snapshots.json
+grep -q 'http://example.com' snapshots.json
+test -d archive
+test "$(find archive -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ')" -ge 4
+rm -rf "$__archivebox_docs_home"
+```
+-->

 See the [Usage: CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) page for documentation and examples.

@@ -1085,7 +1144,7 @@ Because ArchiveBox is designed to ingest a large volume of URLs with multiple co
 <li><strong>Don't store large collections on older filesystems like EXT3/FAT</strong> as they may not be able to handle more than 50k directory entries in the <code>data/archive/</code> folder.
 </li>
 <li><strong>Try to keep the <code>data/index.sqlite3</code> file on local drive (not a network mount)</strong> or SSD for maximum performance, however the <code>data/archive/</code> folder can be on a network mount or slower HDD.</li>
-<li>If using Docker or NFS/SMB/FUSE for the <code>data/archive/</code> folder, you may need to set <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#puid--pgid"><code>PUID</code> & <code>PGID</code></a> and <a href="https://github.com/ArchiveBox/ArchiveBox/issues/1304">disable <code>root_squash</code></a> on your fileshare server.
+<li>If using Docker or NFS/SMB/FUSE for the <code>data/archive/</code> folder, make sure the mounted data directory is writable by its intended owner and consider disabling <a href="https://github.com/ArchiveBox/ArchiveBox/issues/1304"><code>root_squash</code></a> on your fileshare server.
 </li>
 </ul>

@@ -1095,7 +1154,6 @@ Because ArchiveBox is designed to ingest a large volume of URLs with multiple co
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">Wiki: Usage (Disk Layout)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#output-folder">Wiki: Security Overview (Output-Folder)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#large-archives">Wiki: Usage (Large Archives)</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#puid--pgid">Wiki: Configuration (<code>PUID</code> & <code>GUID</code>)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#do-not-run-as-root">Wiki: Security Overview (Do Not Run as Root)</a></li>
 </ul>

@@ -1348,6 +1406,7 @@ For low hanging fruit / easy first tickets, see: <a href="https://github.com/Arc

 First make sure you have `uv` installed: https://docs.astral.sh/uv/getting-started/installation/

+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 git clone https://github.com/ArchiveBox/monorepo
 cd monorepo
@@ -1369,6 +1428,7 @@ Repos included in monorepo setup:

 #### 2. Option A: Install the Python, JS, and system dependencies directly on your machine

+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 # Install ArchiveBox runtime dependencies
 mkdir -p data && cd data
@@ -1384,6 +1444,7 @@ archivebox server 0.0.0.0:8000

 #### 2. Option B: Build the docker container and use that for development instead

+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 # Optional: develop via docker by mounting the code dir into the container
 # if you edit e.g. ./archivebox/core/models.py on the docker host, runserver
@@ -1413,6 +1474,7 @@ You can also run all these in Docker. For more examples see the GitHub Actions C

 <details><summary><i>Click to expand...</i></summary>

+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 # set up persistent DEBUG=True for all runs
 archivebox config --set DEBUG=True
@@ -1437,6 +1499,7 @@ https://stackoverflow.com/questions/1074212/how-can-i-see-the-raw-sql-queries-dj

 If you're looking for the latest `dev` Docker image, it's often available pre-built on Docker Hub, simply pull and use `archivebox/archivebox:dev`.

+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 docker pull archivebox/archivebox:dev
 docker run archivebox/archivebox:dev version
@@ -1444,9 +1507,10 @@ docker run archivebox/archivebox:dev version
 ```

 ##### Build Branch from Source
-  
+
 You can also build and run any branch yourself from source, for example to build & use `dev` locally:

+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 # docker-compose.yml:
 services:
@@ -1471,6 +1535,7 @@ archivebox install

 <details><summary><i>Click to expand...</i></summary>

+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 ./bin/lint.sh
 ./bin/test.sh
@@ -1484,6 +1549,7 @@ archivebox install

 <details><summary><i>Click to expand...</i></summary>

+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 # generate the database migrations after changes to models.py
 cd archivebox/
@@ -1548,6 +1614,7 @@ Copy a similar plugin as a template to modify, then open a new PR to add it in t
 <details><summary><i>Click to expand...</i></summary>

 (Normally CI takes care of this, but these scripts can be run to do it manually)
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 ./bin/build.sh

@@ -1564,6 +1631,7 @@ Copy a similar plugin as a template to modify, then open a new PR to add it in t
 <details><summary><i>Click to expand...</i></summary>

 (Normally CI takes care of this, but these scripts can be run to do it manually)
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 ./bin/release.sh

@@ -17,4 +17,4 @@ ASCII_LOGO_MINI = r"""
 """

 if __name__ == "__main__":
-    main(args=sys.argv[1:], stdin=sys.stdin)
+    main(args=sys.argv[1:])
@@ -51,6 +51,27 @@ def auth_using_token(token: str | None, request: HttpRequest | None = None) -> U
    return user


+def token_from_request(request: HttpRequest) -> str:
+    token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key") or ""
+    auth_header = request.headers.get("Authorization", "")
+    if not token and auth_header.lower().startswith("bearer "):
+        token = auth_header.split(None, 1)[1].strip()
+    return token
+
+
+def authenticated_user_from_request(request: HttpRequest) -> User | None:
+    user = request.user
+    if user.is_authenticated and user.is_active:
+        return user
+
+    token = token_from_request(request)
+    token_user = auth_using_token(token=token, request=request) if token else None
+    if token_user and token_user.is_active:
+        request.user = token_user
+        return token_user
+    return None
+
+
 def auth_using_password(username: str | None, password: str | None, request: HttpRequest | None = None) -> User | None:
    """Given a username and password, check if they are valid and return the corresponding user"""
    user: User | None = None
@@ -8,8 +8,11 @@ from enum import Enum
 from django.http import HttpRequest

 from ninja import Router, Schema
+from ninja.errors import HttpError
+from pydantic import Field

 from archivebox.misc.util import ansi_to_html
+from archivebox.core.models import SnapshotQuerySet


 # from .auth import API_AUTH_METHODS
@@ -21,6 +24,7 @@ router = Router(tags=["ArchiveBox CLI Sub-Commands"])
 # Schemas

 JSONType = list[Any] | dict[str, Any] | bool | int | str | None
+FILTER_PATTERNS_EXAMPLES = [["https://example.com"]]


 class CLICommandResponseSchema(Schema):
@@ -32,26 +36,11 @@ class CLICommandResponseSchema(Schema):
    stderr: str


-class FilterTypeChoices(str, Enum):
-    exact = "exact"
-    substring = "substring"
-    regex = "regex"
-    domain = "domain"
-    tag = "tag"
-    timestamp = "timestamp"
-
-
-class StatusChoices(str, Enum):
-    indexed = "indexed"
-    archived = "archived"
-    unarchived = "unarchived"
-    present = "present"
-    valid = "valid"
-    invalid = "invalid"
-    duplicate = "duplicate"
-    orphaned = "orphaned"
-    corrupted = "corrupted"
-    unrecognized = "unrecognized"
+FilterTypeChoices = Enum(
+    "FilterTypeChoices",
+    {filter_type: filter_type for filter_type in SnapshotQuerySet.FILTER_TYPE_CHOICES},
+    type=str,
+)


 class AddCommandSchema(Schema):
@@ -71,14 +60,27 @@ class AddCommandSchema(Schema):
    index_only: bool = False


-class UpdateCommandSchema(Schema):
-    resume: str | None = None
+class SnapshotFilterCommandSchema(Schema):
    after: float | None = 0
-    before: float | None = 999999999999999
+    before: float | None = None
    filter_type: str | None = FilterTypeChoices.substring
-    filter_patterns: list[str] | None = ["https://example.com"]
+    filter_patterns: list[str] | None = Field(default=None, examples=FILTER_PATTERNS_EXAMPLES)
+    status: str | None = None
+    url__icontains: str | None = None
+    url__istartswith: str | None = None
+    tag: str | None = None
+    crawl_id: str | None = None
+    limit: int | None = None
+    sort: str | None = None
+    search: str | None = None
+
+
+class UpdateCommandSchema(SnapshotFilterCommandSchema):
+    resume: str | None = None
    batch_size: int = 100
    continuous: bool = False
+    index_only: bool = False
+    migrate_only: bool = False


 class ScheduleCommandSchema(Schema):
@@ -97,37 +99,43 @@ class ScheduleCommandSchema(Schema):
    clear: bool = False


-class ListCommandSchema(Schema):
-    filter_patterns: list[str] | None = ["https://example.com"]
-    filter_type: str = FilterTypeChoices.substring
-    status: StatusChoices = StatusChoices.indexed
-    after: float | None = 0
-    before: float | None = 999999999999999
-    sort: str = "bookmarked_at"
+class ListCommandSchema(SnapshotFilterCommandSchema):
    as_json: bool = True
    as_html: bool = False
    as_csv: str | None = "timestamp,url"
    with_headers: bool = False


-class RemoveCommandSchema(Schema):
-    after: float | None = 0
-    before: float | None = 999999999999999
+class RemoveCommandSchema(SnapshotFilterCommandSchema):
    filter_type: str = FilterTypeChoices.exact
-    filter_patterns: list[str] | None = ["https://example.com"]
+    timeout: float = 60.0
+
+
+def snapshot_filter_kwargs(args: SnapshotFilterCommandSchema, *, default_filter_type: str) -> dict[str, Any]:
+    kwargs = args.dict()
+    kwargs["filter_patterns"] = kwargs.get("filter_patterns") or []
+    kwargs["filter_type"] = kwargs.get("filter_type") or default_filter_type
+    return kwargs


@router.post("/add", response=CLICommandResponseSchema, summary="archivebox add [args] [urls]")
 def cli_add(request: HttpRequest, args: AddCommandSchema):
    from archivebox.cli.archivebox_add import add
+    from archivebox.misc.util import validate_url

    config_overrides: dict[str, object] = {}
    if args.only_new is not None:
        config_overrides["ONLY_NEW"] = bool(args.only_new)
    if args.update or args.overwrite:
        config_overrides["ONLY_NEW"] = False
+    submitted_urls: str | list[str] = args.urls
+    if len(args.urls) == 1:
+        try:
+            validate_url(args.urls[0])
+        except ValueError:
+            submitted_urls = args.urls[0]
    crawl, snapshots = add(
-        urls=args.urls,
+        urls=submitted_urls,
        snapshot_ids=args.snapshot_ids,
        tag=args.tag,
        depth=args.depth,
@@ -165,24 +173,39 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):

@router.post("/update", response=CLICommandResponseSchema, summary="archivebox update [args] [filter_patterns]")
 def cli_update(request: HttpRequest, args: UpdateCommandSchema):
-    from archivebox.cli.archivebox_update import update
+    from archivebox.cli.archivebox_update import _build_filtered_snapshots_queryset, update
+    from archivebox.core.snapshot_status import normalize_snapshot_status

-    result = update(
-        filter_patterns=args.filter_patterns or [],
-        filter_type=args.filter_type or FilterTypeChoices.substring,
-        after=args.after,
-        before=args.before,
-        resume=args.resume,
-        batch_size=args.batch_size,
-        continuous=args.continuous,
-        stop_daemon_stack=False,
+    try:
+        status = normalize_snapshot_status(args.status)
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
+
+    update_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.substring)
+    update_kwargs["status"] = status
+    update_kwargs["stop_daemon_stack"] = False
+
+    is_filtered_update = any(
+        (update_kwargs.get(key) for key in (*SnapshotQuerySet.FILTER_ARG_KEYS, "resume") if key != "filter_type"),
    )
+    matched_snapshot_ids = []
+    if is_filtered_update:
+        matched_snapshot_ids = [
+            str(snapshot_id) for snapshot_id in _build_filtered_snapshots_queryset(**update_kwargs).values_list("id", flat=True)
+        ]
+
+    update(**update_kwargs)
    stdout = request.__dict__.get("stdout")
    stderr = request.__dict__.get("stderr")
    return {
        "success": True,
        "errors": [],
-        "result": result,
+        "result": {
+            "matched_count": len(matched_snapshot_ids),
+            "snapshot_ids": matched_snapshot_ids,
+        }
+        if is_filtered_update
+        else None,
        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
    }
@@ -225,29 +248,35 @@ def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):

@router.post("/search", response=CLICommandResponseSchema, summary="archivebox search [args] [filter_patterns]")
 def cli_search(request: HttpRequest, args: ListCommandSchema):
-    from archivebox.cli.archivebox_search import search
+    from archivebox.cli.archivebox_snapshot import build_snapshot_queryset

-    result = search(
-        filter_patterns=args.filter_patterns,
-        filter_type=args.filter_type,
-        status=args.status,
-        after=args.after,
-        before=args.before,
-        sort=args.sort,
-        csv=args.as_csv,
-        json=args.as_json,
-        html=args.as_html,
-        with_headers=args.with_headers,
-    )
+    search_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.substring)
+    as_json = search_kwargs.pop("as_json")
+    as_html = search_kwargs.pop("as_html")
+    as_csv = search_kwargs.pop("as_csv")
+    with_headers = search_kwargs.pop("with_headers")
+    try:
+        snapshots = build_snapshot_queryset(**search_kwargs).select_related("crawl", "crawl__created_by")
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err

    result_format = "txt"
-    if args.as_json:
+    if as_json:
        result_format = "json"
-        result = json.loads(result)
-    elif args.as_html:
+        result = [
+            json.loads(json.dumps(snapshot.to_dict(extended=True), default=str))
+            for snapshot in snapshots.prefetch_related("tags").iterator(chunk_size=500)
+        ]
+    elif as_html:
        result_format = "html"
-    elif args.as_csv:
+        result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500))
+    elif as_csv:
        result_format = "csv"
+        cols = [col.strip() for col in as_csv.split(",") if col.strip()]
+        rows = [snapshot.to_csv(cols=cols, separator=",") for snapshot in snapshots.prefetch_related("tags").iterator(chunk_size=500)]
+        result = "\n".join((",".join(cols), *rows) if with_headers else rows)
+    else:
+        result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500))

    stdout = request.__dict__.get("stdout")
    stderr = request.__dict__.get("stderr")
@@ -264,37 +293,26 @@ def cli_search(request: HttpRequest, args: ListCommandSchema):
@router.post("/remove", response=CLICommandResponseSchema, summary="archivebox remove [args] [filter_patterns]")
 def cli_remove(request: HttpRequest, args: RemoveCommandSchema):
    from archivebox.cli.archivebox_remove import remove
-    from archivebox.cli.archivebox_search import get_snapshots
    from archivebox.core.models import Snapshot

-    filter_patterns = args.filter_patterns or []
-    snapshots_to_remove = get_snapshots(
-        filter_patterns=filter_patterns,
-        filter_type=args.filter_type,
-        after=args.after,
-        before=args.before,
-    )
-    removed_snapshot_ids = [str(snapshot_id) for snapshot_id in snapshots_to_remove.values_list("id", flat=True)]
+    remove_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.exact)
+    timeout_arg = remove_kwargs.pop("timeout")
+    timeout = min(float(timeout_arg if timeout_arg is not None else 60.0), 60.0)
+    try:
+        snapshots_to_remove = Snapshot.objects.order_by("-created_at").search(**remove_kwargs)
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err

-    remove(
+    result = remove(
        yes=True,  # no way to interactively ask for confirmation via API, so we force yes
        snapshots=snapshots_to_remove,
-        before=args.before,
-        after=args.after,
-        filter_type=args.filter_type,
-        filter_patterns=filter_patterns,
+        timeout=timeout,
    )
-
-    result = {
-        "removed_count": len(removed_snapshot_ids),
-        "removed_snapshot_ids": removed_snapshot_ids,
-        "remaining_snapshots": Snapshot.objects.count(),
-    }
    stdout = request.__dict__.get("stdout")
    stderr = request.__dict__.get("stderr")
    return {
-        "success": True,
-        "errors": [],
+        "success": bool(result["success"]),
+        "errors": [str(result["error"])] if result["error"] else [],
        "result": result,
        "result_format": "json",
        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
@@ -29,7 +29,7 @@ from ninja.errors import HttpError

 from archivebox.core.models import Snapshot, ArchiveResult, Tag
 from archivebox.core.permissions import public_snapshots_queryset
-from archivebox.api.auth import auth_using_token
+from archivebox.api.auth import authenticated_user_from_request
 from archivebox.config.common import get_config
 from archivebox.core.routes_util import build_web_url
 from archivebox.misc.util import filter_queryset_by_uuid_substring, validate_url_length
@@ -49,9 +49,10 @@ from archivebox.core.tag_util import (
    rename_tag as rename_tag_record,
 )
 from archivebox.crawls.models import Crawl
-from archivebox.api.v1_crawls import CrawlSchema
+from archivebox.api.v1_crawls import CrawlSchema, get_crawl_by_ref
 from archivebox.search.config import get_search_mode, get_search_mode_backend
 from archivebox.search.query import apply_snapshot_search
+from archivebox.core.snapshot_status import filter_snapshots_by_status, normalize_snapshot_status


 router = Router(tags=["Core Models"])
@@ -79,7 +80,7 @@ class CustomPagination(PaginationBase):
    def paginate_queryset(self, queryset, pagination: Input, request: HttpRequest, **params):
        limit = min(pagination.limit, 500)
        offset = pagination.offset or (pagination.page * limit)
-        total = queryset.count()
+        total = queryset.values("pk").distinct().count() if queryset.query.distinct else queryset.count()
        total_pages = math.ceil(total / limit)
        current_page = math.ceil(offset / (limit + 1))
        items = queryset[offset : offset + limit]
@@ -212,7 +213,10 @@ class ArchiveResultFilterSchema(FilterSchema):
@paginate(CustomPagination)
 def get_archiveresults(request: HttpRequest, filters: Query[ArchiveResultFilterSchema]):
    """List all ArchiveResult entries matching these filters."""
-    return filters.filter(ArchiveResult.objects.all()).distinct()
+    queryset = filters.filter(ArchiveResult.objects.all())
+    if filters.search or filters.snapshot_tag:
+        return queryset.distinct()
+    return queryset


 def _uuid_ref_query(field_name: str, ref: str) -> Q:
@@ -854,6 +858,7 @@ class SnapshotFilterSchema(FilterSchema):
    modified_at__lt: Annotated[datetime | None, FilterLookup("modified_at__lt")] = None
    search: str | None = None
    search_mode: str | None = None
+    status: str | None = None
    url: Annotated[str | None, FilterLookup("url")] = None
    tag: Annotated[str | None, FilterLookup("tags__name")] = None
    title: Annotated[str | None, FilterLookup("title__icontains")] = None
@@ -867,13 +872,20 @@ class SnapshotFilterSchema(FilterSchema):
    def filter_search_mode(self, value: str | None) -> Q:
        return Q()

+    def filter_status(self, value: str | None) -> Q:
+        return Q()
+

@router.get("/snapshots", response=list[SnapshotSchema], url_name="get_snapshots")
@paginate(CustomPagination)
 def get_snapshots(request: HttpRequest, filters: Query[SnapshotFilterSchema], with_archiveresults: bool = False):
    """List all Snapshot entries matching these filters."""
    setattr(request, "with_archiveresults", with_archiveresults)
-    queryset = filters.filter(Snapshot.objects.all()).distinct()
+    try:
+        queryset = filter_snapshots_by_status(Snapshot.objects.all(), filters.status)
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
+    queryset = filters.filter(queryset).distinct()
    query = (filters.search or "").strip()
    if not query:
        return queryset
@@ -916,18 +928,16 @@ def get_snapshots_rss(
 def get_snapshot(request: HttpRequest, snapshot_id: str, with_archiveresults: bool = True):
    """Get a specific Snapshot by id."""
    setattr(request, "with_archiveresults", with_archiveresults)
-    queryset = Snapshot.objects.all()
-    try:
-        return queryset.get(_uuid_ref_query("id", snapshot_id) | Q(timestamp__startswith=snapshot_id))
-    except Snapshot.DoesNotExist:
-        return queryset.get(_uuid_ref_query("id", snapshot_id))
+    return _get_snapshot_by_ref(snapshot_id)


@router.post("/snapshots", response=SnapshotSchema, url_name="create_snapshot")
 def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
    tags = normalize_tag_list(data.tags)
-    if data.status is not None and data.status not in Snapshot.StatusChoices.values:
-        raise HttpError(400, f"Invalid status: {data.status}")
+    try:
+        status = normalize_snapshot_status(data.status)
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
    if not data.url.strip():
        raise HttpError(400, "URL is required")
    try:
@@ -938,7 +948,7 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
        raise HttpError(400, "depth must be between 0 and 4")

    if data.crawl_id:
-        crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), data.crawl_id).get()
+        crawl = get_crawl_by_ref(data.crawl_id)
        crawl_tags = normalize_tag_list(crawl.tags_str.split(","))
        tags = tags or crawl_tags
    else:
@@ -955,7 +965,7 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
        "depth": data.depth,
        "title": data.title,
        "timestamp": str(timezone.now().timestamp()),
-        "status": data.status or Snapshot.StatusChoices.QUEUED,
+        "status": status or Snapshot.StatusChoices.QUEUED,
        "retry_at": timezone.now(),
    }
    snapshot, _ = Snapshot.objects.get_or_create(
@@ -968,10 +978,8 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
    if data.title is not None and snapshot.title != data.title:
        snapshot.title = data.title
        update_fields.append("title")
-    if data.status is not None and snapshot.status != data.status:
-        if data.status not in Snapshot.StatusChoices.values:
-            raise HttpError(400, f"Invalid status: {data.status}")
-        snapshot.status = data.status
+    if status is not None and snapshot.status != status:
+        snapshot.status = status
        update_fields.append("status")
    if update_fields:
        update_fields.append("modified_at")
@@ -992,10 +1000,7 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
@router.patch("/snapshot/{snapshot_id}", response=SnapshotSchema, url_name="patch_snapshot")
 def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateSchema):
    """Update a snapshot (e.g., set status=sealed to cancel queued work)."""
-    try:
-        snapshot = Snapshot.objects.get(Q(id__startswith=snapshot_id) | Q(timestamp__startswith=snapshot_id))
-    except Snapshot.DoesNotExist:
-        snapshot = filter_queryset_by_uuid_substring(Snapshot.objects.all(), snapshot_id).get()
+    snapshot = _get_snapshot_by_ref(snapshot_id)

    payload = data.dict(exclude_unset=True)
    update_fields = ["modified_at"]
@@ -1018,9 +1023,10 @@ def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateS
        raise HttpError(400, f"Invalid action: {action}")

    if "status" in payload:
-        if payload["status"] not in Snapshot.StatusChoices.values:
-            raise HttpError(400, f"Invalid status: {payload['status']}")
-        snapshot.status = payload["status"]
+        try:
+            snapshot.status = normalize_snapshot_status(payload["status"])
+        except ValueError as err:
+            raise HttpError(400, str(err)) from err
        if snapshot.status == Snapshot.StatusChoices.SEALED and "retry_at" not in payload:
            snapshot.retry_at = None
        update_fields.append("status")
@@ -1289,16 +1295,7 @@ def _public_tag_listing_enabled() -> bool:


 def _request_has_tag_autocomplete_access(request: HttpRequest) -> bool:
-    user = request.user
-    if user.is_authenticated:
-        return True
-
-    token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key")
-    auth_header = request.headers.get("Authorization", "")
-    if not token and auth_header.lower().startswith("bearer "):
-        token = auth_header.split(None, 1)[1].strip()
-
-    if token and auth_using_token(token=token, request=request):
+    if authenticated_user_from_request(request):
        return True

    return _public_tag_listing_enabled()
@@ -1,5 +1,6 @@
 __package__ = "archivebox.api"

+import json
 from pathlib import Path
 from uuid import UUID
 from datetime import datetime
@@ -24,7 +25,7 @@ from archivebox.config.common import get_config
 from archivebox.crawls.models import Crawl
 from archivebox.misc.util import filter_queryset_by_uuid_substring

-from .auth import API_AUTH_METHODS, auth_using_token
+from .auth import API_AUTH_METHODS, authenticated_user_from_request

 router = Router(tags=["Crawl Models"], auth=API_AUTH_METHODS)

@@ -124,7 +125,7 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
    config = dict(data.config or {})
    config.setdefault("PERMISSIONS", str(get_config().PERMISSIONS))
    crawl = Crawl.objects.create(
-        urls="\n".join(urls),
+        urls="\n".join(json.dumps({"type": "CrawlSeed", "url": url, "depth": 0}, separators=(",", ":")) for url in urls),
        max_depth=data.max_depth,
        tags_str=",".join(tags),
        label=data.label,
@@ -137,12 +138,16 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
    return crawl


+def get_crawl_by_ref(crawl_id: str):
+    return filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
+
+
@router.get("/crawl/{crawl_id}", response=CrawlSchema, url_name="get_crawl")
 def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_snapshots: bool = False, with_archiveresults: bool = False):
    """Get a specific Crawl by id."""
    setattr(request, "with_snapshots", with_snapshots)
    setattr(request, "with_archiveresults", with_archiveresults)
-    crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
+    crawl = get_crawl_by_ref(crawl_id)

    if crawl and as_rss:
        query = request.GET.copy()
@@ -156,33 +161,18 @@ def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_sn
 def crawl_file(request: HttpRequest, crawl_id: str, path: str):
    # Try to resolve the crawl first; if it doesn't exist, return 404.
    try:
-        crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
+        crawl = get_crawl_by_ref(crawl_id)
    except Crawl.DoesNotExist:
        raise HttpError(404, "Crawl not found")

-    # Determine the effective viewer: session user takes precedence, otherwise
-    # fall back to an API token passed via ?api_key=, X-ArchiveBox-API-Key, or
-    # Authorization: Bearer ... (so that programmatic clients still work).
-    user = request.user
-    is_authenticated = bool(user.is_authenticated and user.is_active)
-    if not is_authenticated:
-        token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key")
-        auth_header = request.headers.get("Authorization", "")
-        if not token and auth_header.lower().startswith("bearer "):
-            token = auth_header.split(None, 1)[1].strip()
-        token_user = auth_using_token(token=token, request=request) if token else None
-        if token_user and token_user.is_active:
-            user = token_user
-            is_authenticated = True
-            # Re-bind so is_admin_user() / ownership checks below see the token user.
-            setattr(request, "user", token_user)
+    user = authenticated_user_from_request(request)

    # Gate access using the same model as SnapshotView/can_view_snapshot:
    # admins always pass; owners can see their own crawls; otherwise the crawl
    # must be PUBLIC or UNLISTED. Don't disclose existence of private crawls.
    if not is_admin_user(request):
        permissions = normalize_permissions(crawl.permissions)
-        is_owner = bool(is_authenticated and crawl.created_by_id == user.id)
+        is_owner = bool(user and crawl.created_by_id == user.id)
        if not is_owner and permissions not in {PERMISSIONS_PUBLIC, PERMISSIONS_UNLISTED}:
            raise HttpError(404, "Crawl not found")

@@ -217,7 +207,7 @@ def crawl_file_nested_2(request: HttpRequest, crawl_id: str, folder: str, subfol
@router.patch("/crawl/{crawl_id}", response=CrawlSchema, url_name="patch_crawl")
 def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
    """Update a crawl (e.g., set status=sealed to cancel queued work)."""
-    crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
+    crawl = get_crawl_by_ref(crawl_id)
    payload = data.dict(exclude_unset=True)
    update_fields = ["modified_at"]

@@ -259,7 +249,7 @@ def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):

@router.delete("/crawl/{crawl_id}", response=CrawlDeleteResponseSchema, url_name="delete_crawl")
 def delete_crawl(request: HttpRequest, crawl_id: str):
-    crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
+    crawl = get_crawl_by_ref(crawl_id)
    crawl_id_str = str(crawl.id)
    snapshot_count = crawl.snapshot_set.count()
    deleted_count, _ = crawl.delete()
@@ -8,8 +8,10 @@ from uuid import UUID
 from django.db.models import Q
 from django.http import HttpRequest
 from ninja import Router, Schema
+from ninja.pagination import paginate
 from pydantic import Field

+from archivebox.api.v1_core import CustomPagination
 from archivebox.personas.importers import validate_persona_name
 from archivebox.personas.models import Persona

@@ -113,6 +115,7 @@ def find_persona(extension_persona_id: str, name: str) -> Persona | None:


@router.get("/personas", response=list[PersonaSchema], url_name="get_personas")
+@paginate(CustomPagination)
 def get_personas(request: HttpRequest):
    """List personas available on this ArchiveBox server."""
    return Persona.objects.all().order_by("name")
@@ -115,6 +115,10 @@ class ModelWithHealthStats(models.Model):
    class Meta(TypedModelMeta):
        abstract = True

+    @property
+    def admin_change_url(self) -> str:
+        return f"/admin/{self._meta.app_label}/{self._meta.model_name}/{self.pk}/change/"
+
    @property
    def health(self) -> int:
        total = max(self.num_uses_failed + self.num_uses_succeeded, 1)
@@ -18,10 +18,11 @@ if "--debug" in sys.argv:

 # Universal `--init` flag: when passed to ANY subcommand (e.g. `archivebox server --init`,
 # `archivebox add --init`, `archivebox shell --init`), run a `quick` archivebox init before
-# the subcommand executes. Strip it from argv here so each subcommand's own click parser
-# never sees it. Ignored for `help` and `init` themselves.
-if "--init" in sys.argv:
-    sys.argv = [arg for arg in sys.argv if arg != "--init"]
+# the subcommand executes. `--quick-init` is the old server spelling. Strip either from
+# argv here so each subcommand's own click parser never sees it. Ignored for `help` and
+# `init` themselves.
+if "--init" in sys.argv or "--quick-init" in sys.argv:
+    sys.argv = [arg for arg in sys.argv if arg not in ("--init", "--quick-init")]
    os.environ["ARCHIVEBOX_WANTS_INIT"] = "1"


@@ -32,6 +33,7 @@ class ArchiveBoxGroup(click.Group):
        "help": "archivebox.cli.archivebox_help.main",
        "version": "archivebox.cli.archivebox_version.main",
        "mcp": "archivebox.cli.archivebox_mcp.main",
+        "oneshot": "archivebox.cli.archivebox_oneshot.main",
    }
    setup_commands = {
        "init": "archivebox.cli.archivebox_init.main",
@@ -66,10 +68,6 @@ class ArchiveBoxGroup(click.Group):
        # Introspection commands
        "pluginmap": "archivebox.cli.archivebox_pluginmap.main",
    }
-    legacy_model_commands = {
-        "crawl": "archivebox.cli.archivebox_crawl_compat.main",
-        "snapshot": "archivebox.cli.archivebox_snapshot_compat.main",
-    }
    all_subcommands = {
        **meta_commands,
        **setup_commands,
@@ -81,34 +79,29 @@ class ArchiveBoxGroup(click.Group):
        "import": "add",
        "archive": "add",
    }
-    legacy_model_subcommands = {
-        "crawl": {"create", "list", "update", "delete"},
-        "snapshot": {"create", "list", "update", "delete"},
-    }

    @classmethod
    def get_canonical_name(cls, cmd_name):
        return cls.renamed_commands.get(cmd_name, cmd_name)

    @classmethod
-    def _should_use_legacy_model_command(cls, cmd_name: str) -> bool:
-        if cmd_name not in cls.legacy_model_commands:
-            return False
+    def _needs_django_for_lazy_import(cls, cmd_name: str) -> bool:
+        wants_help = any(arg in ("-h", "--help", "--version") for arg in sys.argv[1:])
+        return not wants_help and (cmd_name in cls.archive_commands or cmd_name in cls.model_commands)

-        try:
-            arg_idx = sys.argv.index(cmd_name)
-        except ValueError:
-            return False
+    @classmethod
+    def _setup_django_for_lazy_import(cls, cmd_name: str) -> None:
+        if not cls._needs_django_for_lazy_import(cmd_name):
+            return

-        remaining_args = sys.argv[arg_idx + 1 :]
-        if not remaining_args:
-            return False
+        from django.apps import apps

-        first_arg = remaining_args[0]
-        if first_arg in ("-h", "--help"):
-            return False
+        if apps.ready:
+            return

-        return first_arg not in cls.legacy_model_subcommands[cmd_name]
+        from archivebox.config.django import setup_django
+
+        setup_django()

    def get_command(self, ctx, cmd_name):
        # handle renamed commands
@@ -120,11 +113,9 @@ class ArchiveBoxGroup(click.Group):
            cmd_name = new_name
            ctx.invoked_subcommand = cmd_name

-        if self._should_use_legacy_model_command(cmd_name):
-            return self._lazy_load(self.legacy_model_commands[cmd_name])
-
        # handle lazy loading of commands
        if cmd_name in self.all_subcommands:
+            self._setup_django_for_lazy_import(cmd_name)
            return self._lazy_load(cmd_name)

        # fall-back to using click's default command lookup
@@ -199,15 +190,12 @@ def cli(ctx, help=False):
                raise


-def main(args=None, prog_name=None, stdin=None):
+def main(args=None, prog_name=None):
    # show `docker run archivebox xyz` in help messages if running in docker
    IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes")
    IS_TTY = sys.stdin.isatty()
    prog_name = prog_name or (f"docker compose run{'' if IS_TTY else ' -T'} archivebox" if IN_DOCKER else "archivebox")

-    # stdin param allows passing input data from caller (used by __main__.py)
-    # currently not used by click-based CLI, but kept for backwards compatibility
-
    previous_unraisablehook = sys.unraisablehook

    def ignore_shutdown_unraisable(unraisable):
@@ -5,14 +5,15 @@ __package__ = "archivebox.cli"
 __command__ = "archivebox add"

 import sys
-import json
 import os
+import json
 from pathlib import Path

 from typing import Any, TYPE_CHECKING

 import rich_click as click

+from archivebox.config import CONSTANTS
 from archivebox.misc.util import enforce_types, docstring
 from archivebox.misc.util import parse_filesize_to_bytes

@@ -23,21 +24,28 @@ if TYPE_CHECKING:
    from archivebox.core.models import Snapshot


-def _collect_input_urls(args: tuple[str, ...]) -> list[str]:
+def _collect_input_urls(args: tuple[str, ...], *, parser: str = "auto") -> list[str]:
    from archivebox.misc.jsonl import read_args_or_stdin
+    from archivebox.misc.util import validate_url

    urls: list[str] = []
    for record in read_args_or_stdin(args):
        url = record.get("url")
        if isinstance(url, str) and url:
-            urls.append(url)
+            try:
+                urls.append(validate_url(url))
+            except ValueError as err:
+                raise click.BadParameter(str(err), param_hint="URL") from err

        urls_field = record.get("urls")
        if isinstance(urls_field, str):
            for line in urls_field.splitlines():
                line = line.strip()
                if line and not line.startswith("#"):
-                    urls.append(line)
+                    try:
+                        urls.append(validate_url(line))
+                    except ValueError as err:
+                        raise click.BadParameter(str(err), param_hint="URL") from err

    return urls

@@ -82,7 +90,6 @@ def add(
    crawl_max_size = parse_filesize_to_bytes(crawl_max_size)
    crawl_timeout = int(crawl_timeout or 0)
    snapshot_max_size = parse_filesize_to_bytes(snapshot_max_size)
-    from archivebox import CONSTANTS
    from archivebox.config.permissions import USER, HOSTNAME
    from archivebox.config.common import get_config

@@ -118,22 +125,12 @@ def add(
    created_by_id = created_by_id or get_or_create_system_user_pk()
    started_at = timezone.now()

-    if isinstance(urls, str):
-        url_list = [line.strip() for line in urls.splitlines() if line.strip()]
-    else:
-        url_list = [str(url).strip() for url in urls if str(url).strip()]
-    if snapshot_ids and len(snapshot_ids) != len(url_list):
-        raise ValueError("snapshot_ids length must match urls length")
-
-    # 1. Save the provided URLs to sources/2024-11-05__23-59-59__cli_add.txt
-    sources_file = CONSTANTS.SOURCES_DIR / f"{timezone.now().strftime('%Y-%m-%d__%H-%M-%S')}__cli_add.txt"
-    sources_file.parent.mkdir(parents=True, exist_ok=True)
-    if snapshot_ids:
-        sources_file.write_text(
-            "\n".join(json.dumps({"url": url, "id": snapshot_ids[index], "tags": tag, "depth": 0}) for index, url in enumerate(url_list)),
-        )
-    else:
-        sources_file.write_text("\n".join(url_list))
+    use_internal_input_root = isinstance(urls, str)
+    source_text = (
+        urls
+        if use_internal_input_root
+        else "\n".join(json.dumps({"type": "CrawlSeed", "url": str(url), "depth": 0}, separators=(",", ":")) for url in urls)
+    )

    # 2. Create a new Crawl with inline URLs
    cli_args = [*sys.argv]
@@ -143,11 +140,9 @@ def add(

    timestamp = timezone.now().strftime("%Y-%m-%d__%H-%M-%S")

-    # Read URLs directly into crawl
-    urls_content = sources_file.read_text()
    persona_name = (persona or "Default").strip() or "Default"
    plugins = plugins or ""
-    persona_obj, _ = Persona.objects.get_or_create(name=persona_name)
+    persona_obj = Persona.get_or_create_named(persona_name)
    persona_obj.ensure_dirs()
    effective_persona_config = get_config(persona=persona_obj)

@@ -174,9 +169,18 @@ def add(
    # stripped by Crawl.save() and rederived when hooks run.
    crawl_config.update(config_overrides)

+    # Crawl.urls is the user's submitted source, not a derived work queue for
+    # this add path. Keeping it byte-for-byte readable is what lets API/UI/CLI
+    # callers audit or resume imports without losing RSS/Netscape/JSON metadata
+    # that is not representable as one plain URL per line.
    crawl = Crawl.objects.create(
-        urls=urls_content,
-        max_depth=depth,
+        urls=source_text,
+        # Stdin/import text gets an extra hop because the synthetic
+        # archivebox://internal root lives at depth 0 and parser-discovered
+        # URLs land at depth 1; direct URL args become the depth=0 input
+        # snapshots themselves so --depth=N matches the deepest hop the user
+        # asked for.
+        max_depth=depth + 1 if use_internal_input_root else depth,
        tags_str=tag,
        persona_id=persona_obj.id,
        label=f"{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]",
@@ -210,12 +214,15 @@ def add(
        print(
            "[yellow]\\[*] URLs queued. The background runner will process them (run `archivebox server` or `archivebox run --daemon` if not already running).[/yellow]",
        )
+        from archivebox.services.runner import ensure_background_runner
+
+        ensure_background_runner(allow_under_pytest=True)
    else:
        # Foreground mode: run full crawl runner until all work is done
        print("[green]\\[*] Starting crawl runner to process crawl...[/green]")
        from archivebox.machine.models import Process
-        from archivebox.core.takeover_util import command_owns_runtime_stack, current_command, standby_until_runtime_stack_needed
-        from archivebox.workers.supervisord_util import run_runner_worker, stop_own_supervisord_process
+        from archivebox.core.takeover_util import command_owns_foreground_runner, current_command, standby_until_foreground_runner_needed
+        from archivebox.workers.supervisord_util import get_existing_supervisord_process, run_runner_worker, stop_own_supervisord_process

        command = current_command(Process.TypeChoices.ADD, data_dir=CONSTANTS.DATA_DIR, url=first_url)
        exit_code = 0
@@ -223,15 +230,27 @@ def add(
            try:
                with foreground_shutdown_signals(first_signal_message=None), foreground_parent_watchdog():
                    while True:
-                        standby_until_runtime_stack_needed(command, data_dir=CONSTANTS.DATA_DIR)
+                        standby_until_foreground_runner_needed(command, data_dir=CONSTANTS.DATA_DIR)
                        exit_code = run_runner_worker(
                            ["--crawl-id", str(crawl.id)],
                            name=f"worker_runner_add_{os.getpid()}",
                            interactive_interrupts=True,
                        )
+                        crawl.refresh_from_db(fields=["status", "retry_at"])
+                        if exit_code == 0 and crawl.status == crawl.StatusChoices.SEALED:
+                            break
+                        # A shared supervisord can be stopped by another
+                        # foreground owner (for example `archivebox server`
+                        # shutting down) and stop the one-shot add runner while
+                        # this add's crawl is still runnable. Keep the add
+                        # process alive so it can re-enter the normal
+                        # foreground-runner ownership loop and finish its crawl.
+                        supervisor_gone = exit_code == 1 and get_existing_supervisord_process(quiet=True) is None
+                        if crawl.status in crawl.RUNNABLE_STATES and (exit_code in (0, 130, 143) or supervisor_gone):
+                            continue
                        if exit_code == 0:
                            break
-                        if not command_owns_runtime_stack(command, data_dir=CONSTANTS.DATA_DIR):
+                        if not command_owns_foreground_runner(command, data_dir=CONSTANTS.DATA_DIR):
                            continue
                        raise SystemExit(exit_code)
            except KeyboardInterrupt:
@@ -312,6 +331,7 @@ def add(
@click.option("--url-denylist", "--domain-denylist", default="", help="Comma-separated URL/domain denylist for this crawl")
@click.option("--parser", default="auto", help="Parser for reading input URLs (auto, txt, html, rss, json, jsonl, netscape, ...)")
@click.option("--plugins", "-p", default="", help="Comma-separated list of plugins to run e.g. title,favicon,screenshot,singlefile,...")
+@click.option("--extract", default="", help="Comma-separated plugins or output types to extract e.g. title,pdf,text/html,image,...")
@click.option("--persona", default="Default", help="Authentication profile to use when archiving")
@click.option(
    "--only-new/--no-only-new",
@@ -333,7 +353,12 @@ def main(**kwargs):

    with foreground_shutdown_signals(), foreground_parent_watchdog():
        raw_urls = kwargs.pop("urls")
-        urls = _collect_input_urls(raw_urls)
+        if raw_urls:
+            urls = _collect_input_urls(raw_urls, parser=kwargs.get("parser", "auto"))
+        elif not sys.stdin.isatty():
+            urls = sys.stdin.read()
+        else:
+            urls = []
        if not urls:
            raise click.UsageError("No URLs provided. Pass URLs as arguments or via stdin.")
        if int(kwargs.get("max_urls") or 0) < 0:
@@ -356,10 +381,23 @@ def main(**kwargs):
        overwrite = kwargs.pop("overwrite", False)
        update = kwargs.pop("update", False)
        only_new = kwargs.pop("only_new", None)
+        extract = kwargs.pop("extract", "")
        if overwrite or update:
            only_new = False
        if only_new is not None:
            kwargs["config"] = {"ONLY_NEW": bool(only_new)}
+        if extract:
+            from abx_dl.models import discover_plugins, plugins_matching_output
+
+            all_plugins = discover_plugins()
+            tokens = [token.strip() for token in extract.split(",") if token.strip()]
+            plugin_names = {name.lower(): name for name in all_plugins}
+            selected = [plugin_names[token.lower()] for token in tokens if token.lower() in plugin_names]
+            selected += plugins_matching_output(all_plugins, tokens)
+            if not selected:
+                raise click.UsageError(f"No plugins found matching extract types: {extract}")
+            existing = [token.strip() for token in (kwargs.get("plugins") or "").split(",") if token.strip()]
+            kwargs["plugins"] = ",".join(dict.fromkeys([*existing, *selected]))

        add(urls=urls, **kwargs)

@@ -6,6 +6,7 @@ import sys
 import toml
 import rich_click as click
 from rich import print
+from pathlib import Path

 from archivebox.misc.util import docstring, enforce_types
 from archivebox.misc.toml_util import CustomTOMLEncoder
@@ -30,12 +31,17 @@ def config(
    from archivebox.misc.logging_util import printable_config
    from abx_plugins.plugins.base.utils import resolve_alias
    from archivebox.config.collection import write_config_file
+    from archivebox.config import CONSTANTS_CONFIG
    from archivebox.config.common import ArchiveBoxConfig, get_config, get_all_configs
    from archivebox.plugins.discovery import discover_plugin_configs

    check_data_folder()

    FLAT_CONFIG = get_config().as_dict()
+    runtime_derived_keys = ArchiveBoxConfig.runtime_derived_config_keys()
+    readonly_config = {key: val for key, val in CONSTANTS_CONFIG.items() if key.isupper() and isinstance(val, Path)}
+    writable_config = {key: val for key, val in FLAT_CONFIG.items() if key not in runtime_derived_keys and key not in readonly_config}
+    readable_config = {**writable_config, **readonly_config}
    CONFIGS = get_all_configs()
    plugin_schemas = {
        plugin_name: schema.get("properties", {}) for plugin_name, schema in discover_plugin_configs().items() if isinstance(schema, dict)
@@ -56,18 +62,23 @@ def config(
            config_options = [
                core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options
            ]
-            matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
+            matching_config = {key: readable_config[key] for key in config_options if key in readable_config}
            for config_section in CONFIGS.values():
                aliases = {str(field.alias): field_name for field_name, field in type(config_section).model_fields.items() if field.alias}

                for search_key in config_options:
                    # search all aliases in the section
                    for alias_key, key in aliases.items():
-                        if search_key.lower() in alias_key.lower():
+                        if key in readable_config and search_key.lower() in alias_key.lower():
                            matching_config[key] = dict(config_section)[key]

                    # search all keys and values in the section
                    for existing_key, value in dict(config_section).items():
+                        if existing_key in readable_config and (
+                            search_key.lower() in existing_key.lower() or search_key.lower() in str(value).lower()
+                        ):
+                            matching_config[existing_key] = value
+                    for existing_key, value in readonly_config.items():
                        if search_key.lower() in existing_key.lower() or search_key.lower() in str(value).lower():
                            matching_config[existing_key] = value

@@ -79,14 +90,14 @@ def config(
            config_options = [
                core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options
            ]
-            matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
-            failed_config = [key for key in config_options if key not in FLAT_CONFIG]
+            matching_config = {key: readable_config[key] for key in config_options if key in readable_config}
+            failed_config = [key for key in config_options if key not in readable_config]
            if failed_config:
                print("\n[red][X] These options failed to get[/red]")
                print("    {}".format("\n    ".join(config_options)))
                raise SystemExit(1)
        else:
-            matching_config = FLAT_CONFIG
+            matching_config = readable_config

        # Display core config sections
        for config_section in CONFIGS.values():
@@ -100,12 +111,18 @@ def config(
            print(_format_toml(kv_in_section))
            print("[grey53]################################################################[/grey53]")

+        readonly_keys = {key: val for key, val in readonly_config.items() if key in matching_config}
+        if readonly_keys:
+            print("[grey53]\\[CONSTANTS]                                        # (read-only)[/grey53]")
+            print(_format_toml(readonly_keys))
+            print("[grey53]################################################################[/grey53]")
+
        plugin_keys = {}

        # Collect all plugin config keys
        for schema in plugin_schemas.values():
            for key in schema.keys():
-                if key in matching_config:
+                if key in matching_config and key in writable_config:
                    plugin_keys[key] = matching_config[key]

        # Display all plugin config in single [PLUGINS] section
@@ -135,7 +152,7 @@ def config(
                    f"[yellow][i] Note: The config option {raw_key} has been renamed to {key}, please use the new name going forwards.[/yellow]",
                )

-            if key in FLAT_CONFIG:
+            if key in writable_config:
                new_config[key] = val.strip()
            else:
                failed_options.append(line)
@@ -65,6 +65,7 @@ def create_crawl(
        1: Failure
    """
    from archivebox.misc.jsonl import read_args_or_stdin, write_record, TYPE_CRAWL
+    from archivebox.misc.util import validate_url
    from archivebox.base_models.models import get_or_create_system_user_pk
    from archivebox.crawls.models import Crawl

@@ -98,7 +99,11 @@ def create_crawl(
        # Collect URLs
        url = record.get("url")
        if url:
-            url_list.append(url)
+            try:
+                url_list.append(validate_url(str(url)))
+            except ValueError as err:
+                rprint(f"[red]Invalid URL: {err}[/red]", file=sys.stderr)
+                return 1

        # Handle 'urls' field (newline-separated)
        urls_field = record.get("urls")
@@ -106,7 +111,11 @@ def create_crawl(
            for line in urls_field.split("\n"):
                line = line.strip()
                if line and not line.startswith("#"):
-                    url_list.append(line)
+                    try:
+                        url_list.append(validate_url(line))
+                    except ValueError as err:
+                        rprint(f"[red]Invalid URL: {err}[/red]", file=sys.stderr)
+                        return 1

    # Output pass-through records first
    if not is_tty:
@@ -1,27 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox crawl"
-
-import sys
-
-import rich_click as click
-
-from archivebox.cli.archivebox_add import add
-
-
-@click.command(context_settings={"ignore_unknown_options": True})
-@click.option("--depth", "-d", type=int, default=0, help="Max crawl depth (default: 0)")
-@click.option("--tag", "-t", default="", help="Comma-separated tags to add")
-@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
-@click.option("--wait/--no-wait", "wait", default=True, help="Accepted for backwards compatibility")
-@click.argument("urls", nargs=-1)
-def main(depth: int, tag: str, status: str, wait: bool, urls: tuple[str, ...]):
-    """Backwards-compatible `archivebox crawl URL...` entrypoint."""
-    del status, wait
-    add(list(urls), depth=depth, tag=tag, bg=True)
-    sys.exit(0)
-
-
-if __name__ == "__main__":
-    main()
@@ -48,10 +48,11 @@ def process_archiveresult_by_id(archiveresult_id: str) -> int:
    from rich import print as rprint
    from django.utils import timezone
    from archivebox.core.models import ArchiveResult
+    from archivebox.api.v1_core import _uuid_ref_query
    from archivebox.services.runner import run_crawl

    try:
-        archiveresult = ArchiveResult.objects.get(id=archiveresult_id)
+        archiveresult = ArchiveResult.objects.get(_uuid_ref_query("id", archiveresult_id))
    except ArchiveResult.DoesNotExist:
        rprint(f"[red]ArchiveResult {archiveresult_id} not found[/red]", file=sys.stderr)
        return 1
@@ -116,6 +117,7 @@ def run_plugins(
    wait: bool = True,
    emit_results: bool = True,
    show_progress: bool = True,
+    preserve_queued: bool = False,
 ) -> int:
    """
    Run plugins on Snapshots from input.
@@ -209,7 +211,7 @@ def run_plugins(
            if snapshot_id in existing_snapshot_ids
            for plugin_name in plugin_names
        )
-    plugins_by_name = discover_plugins()
+    plugins_by_name = discover_plugins(runtime="archivebox")
    requested_rows: set[tuple[str, str, str]] = set()
    for snapshot_id, plugin_name in requested_pairs:
        plugin = plugins_by_name.get(plugin_name)
@@ -219,6 +221,18 @@ def run_plugins(
        else:
            requested_rows.add((snapshot_id, plugin_name, ""))

+    queued_rows: set[tuple[str, str, str]] = set()
+    if preserve_queued and requested_rows:
+        queued_rows = {
+            (str(snapshot_id), plugin_name, hook_name)
+            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
+                snapshot_id__in=existing_snapshot_ids,
+                plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in requested_rows},
+                status=ArchiveResult.StatusChoices.QUEUED,
+            ).values_list("snapshot_id", "plugin", "hook_name")
+        }
+    rows_to_queue = requested_rows - queued_rows
+
    reset_fields = {
        "status": ArchiveResult.StatusChoices.QUEUED,
        "output_str": "",
@@ -230,23 +244,34 @@ def run_plugins(
        "end_ts": None,
        "modified_at": timezone.now(),
    }
-    if plugins_list:
-        ArchiveResult.objects.filter(snapshot_id__in=existing_snapshot_ids, plugin__in=plugins_list).update(**reset_fields)
-    elif requested_plugins_by_snapshot:
-        snapshot_ids_by_plugin: dict[str, set[str]] = defaultdict(set)
-        for snapshot_id, plugin_names in requested_plugins_by_snapshot.items():
-            if snapshot_id in existing_snapshot_ids:
-                for plugin_name in plugin_names:
-                    snapshot_ids_by_plugin[plugin_name].add(snapshot_id)
-        for plugin_name, plugin_snapshot_ids in snapshot_ids_by_plugin.items():
-            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name).update(**reset_fields)
-    existing_rows = set(
-        ArchiveResult.objects.filter(
-            snapshot_id__in=existing_snapshot_ids,
-            plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in requested_rows},
-        ).values_list("snapshot_id", "plugin", "hook_name"),
+    if rows_to_queue and plugins_list:
+        rows_to_reset_by_hook: dict[tuple[str, str], set[str]] = defaultdict(set)
+        for snapshot_id, plugin_name, hook_name in rows_to_queue:
+            rows_to_reset_by_hook[(plugin_name, hook_name)].add(snapshot_id)
+        for (plugin_name, hook_name), plugin_snapshot_ids in rows_to_reset_by_hook.items():
+            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name, hook_name=hook_name).update(
+                **reset_fields,
+            )
+    elif rows_to_queue and requested_plugins_by_snapshot:
+        snapshot_ids_by_hook: dict[tuple[str, str], set[str]] = defaultdict(set)
+        for snapshot_id, plugin_name, hook_name in rows_to_queue:
+            snapshot_ids_by_hook[(plugin_name, hook_name)].add(snapshot_id)
+        for (plugin_name, hook_name), plugin_snapshot_ids in snapshot_ids_by_hook.items():
+            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name, hook_name=hook_name).update(
+                **reset_fields,
+            )
+    existing_rows = (
+        {
+            (str(snapshot_id), plugin_name, hook_name)
+            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
+                snapshot_id__in=existing_snapshot_ids,
+                plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in rows_to_queue},
+            ).values_list("snapshot_id", "plugin", "hook_name")
+        }
+        if rows_to_queue
+        else set()
    )
-    missing_rows = requested_rows - {(str(snapshot_id), plugin_name, hook_name) for snapshot_id, plugin_name, hook_name in existing_rows}
+    missing_rows = rows_to_queue - existing_rows
    if missing_rows:
        ArchiveResult.objects.bulk_create(
            [
@@ -271,7 +296,23 @@ def run_plugins(
            # also keep status=paused here: `retry_at` only asks the orchestrator
            # to process the queued plugin rows, and run_due_snapshot restores
            # retry_at=MAX afterward instead of resuming the snapshot lifecycle.
-            for snapshot in Snapshot.objects.filter(id__in=existing_snapshot_ids).only("id", "status", "modified_at"):
+            affected_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in rows_to_queue}
+            if preserve_queued and queued_rows:
+                queued_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in queued_rows}
+                affected_snapshot_ids.update(
+                    str(snapshot_id)
+                    for snapshot_id in Snapshot.objects.filter(id__in=queued_snapshot_ids)
+                    .filter(retry_at__gt=queue_at)
+                    .values_list("id", flat=True)
+                )
+                affected_snapshot_ids.update(
+                    str(snapshot_id)
+                    for snapshot_id in Snapshot.objects.filter(id__in=queued_snapshot_ids, retry_at__isnull=True).values_list(
+                        "id",
+                        flat=True,
+                    )
+                )
+            for snapshot in Snapshot.objects.filter(id__in=affected_snapshot_ids).only("id", "status", "modified_at"):
                # Guard the read-time status so we never bump retry_at on a
                # row that's been re-queued / started by a concurrent runner.
                snapshot.safe_update(
@@ -378,16 +419,11 @@ def run_plugins(


 def is_archiveresult_id(value: str) -> bool:
-    """Check if value looks like an ArchiveResult UUID."""
-    import re
-
-    uuid_pattern = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.I)
-    if not uuid_pattern.match(value):
-        return False
-    # Verify it's actually an ArchiveResult (not a Snapshot or other object)
+    """Check if value resolves to an ArchiveResult ID."""
    from archivebox.core.models import ArchiveResult
+    from archivebox.api.v1_core import _uuid_ref_query

-    return ArchiveResult.objects.filter(id=value).exists()
+    return ArchiveResult.objects.filter(_uuid_ref_query("id", value)).exists()


@click.command()
@@ -13,6 +13,10 @@ from rich.panel import Panel


 def _command_doc(cmd: str, import_path: str) -> str:
+    def first_doc_line(docstring: str | None) -> str:
+        lines = (docstring or "").splitlines()
+        return lines[0] if lines else ""
+
    modname, _ = import_path.rsplit(".", 1)
    spec = importlib.util.find_spec(modname)
    if spec is None or spec.origin is None:
@@ -26,9 +30,9 @@ def _command_doc(cmd: str, import_path: str) -> str:
    for name in (cmd, "main"):
        for node in tree.body:
            if isinstance(node, ast.FunctionDef) and node.name == name:
-                return (ast.get_docstring(node) or "").splitlines()[0]
+                return first_doc_line(ast.get_docstring(node))

-    return (ast.get_docstring(tree) or "").splitlines()[0]
+    return first_doc_line(ast.get_docstring(tree))


 def help() -> None:
@@ -94,10 +94,18 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
    from archivebox.config.django import setup_django

    setup_django()
-    check_migrations(blocking=True, auto_apply=False)
+    previous_wants_init = os.environ.get("ARCHIVEBOX_WANTS_INIT")
+    os.environ["ARCHIVEBOX_WANTS_INIT"] = "1"
+    try:
+        check_migrations(blocking=True, auto_apply=False)

-    for migration_line in apply_migrations(CONSTANTS.DATA_DIR):
-        sys.stdout.write(f"    {migration_line}\n")
+        for migration_line in apply_migrations(CONSTANTS.DATA_DIR):
+            sys.stdout.write(f"    {migration_line}\n")
+    finally:
+        if previous_wants_init is None:
+            os.environ.pop("ARCHIVEBOX_WANTS_INIT", None)
+        else:
+            os.environ["ARCHIVEBOX_WANTS_INIT"] = previous_wants_init

    assert os.path.isfile(CONSTANTS.DATABASE_FILE) and os.access(CONSTANTS.DATABASE_FILE, os.R_OK)
    print()
@@ -145,8 +153,8 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non

    config = get_config()
    config.TMP_DIR.mkdir(parents=True, exist_ok=True)
-    config.LIB_DIR.mkdir(parents=True, exist_ok=True)
-    (config.LIB_DIR / "bin").mkdir(parents=True, exist_ok=True)
+    config.ABXPKG_LIB_DIR.mkdir(parents=True, exist_ok=True)
+    (config.ABXPKG_LIB_DIR / "bin").mkdir(parents=True, exist_ok=True)

    working_tmp_dir = get_or_create_working_tmp_dir(autofix=True, quiet=True)
    if working_tmp_dir:
@@ -155,7 +163,6 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
    working_lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True)
    if working_lib_dir:
        working_lib_dir.mkdir(parents=True, exist_ok=True)
-        (working_lib_dir / "bin").mkdir(parents=True, exist_ok=True)

    if install:
        from archivebox.cli.archivebox_install import install as install_method
@@ -179,7 +186,7 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
@click.command()
@click.option("--force", "-f", is_flag=True, help="Ignore unrecognized files in current directory and initialize anyway")
@click.option("--quick", "-q", is_flag=True, help="Run any updates or migrations without rechecking all snapshot dirs")
-@click.option("--install", "-s", is_flag=True, help="Automatically install dependencies and extras used for archiving")
+@click.option("--install", "--setup", "-s", is_flag=True, help="Automatically install dependencies and extras used for archiving")
@docstring(init.__doc__)
 def main(**kwargs) -> None:
    init(**kwargs)
@@ -3,6 +3,8 @@
 __package__ = "archivebox.cli"

 import os
+import shutil
+from pathlib import Path

 import rich_click as click
 from rich import print
@@ -10,6 +12,38 @@ from rich import print
 from archivebox.misc.util import docstring, enforce_types


+def ensure_data_dir_lib_symlink(data_dir: Path, abxpkg_lib_dir: Path) -> Path | None:
+    """Expose the shared abxpkg lib dir at DATA_DIR/lib after install."""
+    link_path = data_dir / "lib"
+    target_config_path = abxpkg_lib_dir.expanduser().absolute()
+    target_path = abxpkg_lib_dir.expanduser().resolve()
+    from archivebox.config.paths import get_machine_type
+
+    scoped_data_lib_path = (link_path / get_machine_type()).absolute()
+
+    if target_config_path == scoped_data_lib_path:
+        if link_path.is_symlink() or link_path.is_file():
+            link_path.unlink()
+        target_config_path.mkdir(parents=True, exist_ok=True)
+        return None
+
+    if link_path.resolve(strict=False) == target_path:
+        return None
+
+    target_path.mkdir(parents=True, exist_ok=True)
+
+    if link_path.is_symlink():
+        if link_path.resolve(strict=False) == target_path:
+            return link_path
+        link_path.unlink()
+    elif link_path.exists():
+        shutil.rmtree(link_path)
+
+    relative_target = os.path.relpath(target_path, start=link_path.parent)
+    link_path.symlink_to(relative_target, target_is_directory=True)
+    return link_path
+
+
@enforce_types
 def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bool = False) -> None:
    """Detect and install ArchiveBox dependencies by running the abx-dl install flow
@@ -67,6 +101,11 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo

    run_install(plugin_names=plugin_names or None)

+    from archivebox.config.common import get_config
+
+    config = get_config(include_machine=False)
+    ensure_data_dir_lib_symlink(CONSTANTS.DATA_DIR, config.ABXPKG_LIB_DIR)
+
    print()

    # Check for superuser
@@ -7,50 +7,15 @@ import sys

 import rich_click as click

-from archivebox.cli.archivebox_snapshot import list_snapshots
+from archivebox.cli.archivebox_snapshot import list_snapshots, snapshot_filter_options, snapshot_output_options


@click.command()
-@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
-@click.option("--url__icontains", help="Filter by URL contains")
-@click.option("--url__istartswith", help="Filter by URL starts with")
-@click.option("--tag", "-t", help="Filter by tag name")
-@click.option("--crawl-id", help="Filter by crawl ID")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title")
-@click.option("--with-headers", is_flag=True, help="Include column headers in structured output")
-@click.option("--search", help="Search mode to use for the query")
-@click.argument("query", nargs=-1)
-def main(
-    status: str | None,
-    url__icontains: str | None,
-    url__istartswith: str | None,
-    tag: str | None,
-    crawl_id: str | None,
-    limit: int | None,
-    sort: str | None,
-    csv: str | None,
-    with_headers: bool,
-    search: str | None,
-    query: tuple[str, ...],
-) -> None:
+@snapshot_output_options
+@snapshot_filter_options(default_filter_type="substring")
+def main(**kwargs) -> None:
    """List Snapshots."""
-    sys.exit(
-        list_snapshots(
-            status=status,
-            url__icontains=url__icontains,
-            url__istartswith=url__istartswith,
-            tag=tag,
-            crawl_id=crawl_id,
-            limit=limit,
-            sort=sort,
-            csv=csv,
-            with_headers=with_headers,
-            search=search,
-            query=" ".join(query),
-        ),
-    )
+    sys.exit(list_snapshots(**kwargs))


 if __name__ == "__main__":
@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+
+__package__ = "archivebox.cli"
+__command__ = "archivebox oneshot"
+
+import subprocess
+from pathlib import Path
+
+import rich_click as click
+
+from archivebox.config import CONSTANTS
+
+
+@click.command(add_help_option=False, context_settings=dict(ignore_unknown_options=True))
+@click.argument("args", nargs=-1)
+def main(args: tuple[str, ...] = ()) -> None:
+    """Download URLs using abx-dl"""
+    cwd = Path.cwd()
+    if any((path / CONSTANTS.SQL_INDEX_FILENAME).exists() for path in (cwd, *cwd.parents)):
+        raise click.ClickException(
+            "Refusing to run `archivebox oneshot` inside an ArchiveBox DATA_DIR. Use `archivebox add` here, or run oneshot from another directory.",
+        )
+    raise SystemExit(subprocess.run(["abx-dl", *args]).returncode)
+
+
+if __name__ == "__main__":
+    main()
@@ -23,6 +23,7 @@ from archivebox.misc.logging_util import (
    log_removal_finished,
    TimedProgress,
 )
+from archivebox.cli.archivebox_snapshot import snapshot_filter_options


@enforce_types
@@ -33,49 +34,56 @@ def remove(
    after: float | None = None,
    before: float | None = None,
    yes: bool = False,
+    delete: bool = False,
    out_dir: Path = CONSTANTS.DATA_DIR,
-) -> QuerySet:
+    timeout: float | None = None,
+    **kwargs,
+) -> dict[str, object]:
    """Remove the specified URLs from the archive"""

    setup_django()
    check_data_folder()
+    timeout = float(timeout) if timeout is not None else None

-    from archivebox.cli.archivebox_search import get_snapshots
+    from archivebox.core.models import Snapshot

+    filter_kwargs = {
+        **kwargs,
+        "filter_patterns": filter_patterns,
+        "filter_type": filter_type,
+        "after": after,
+        "before": before,
+    }
    pattern_list = list(filter_patterns)

    log_list_started(pattern_list or None, filter_type)
    timer = TimedProgress(360, prefix="      ")
    try:
-        snapshots = get_snapshots(
-            snapshots=snapshots,
-            filter_patterns=pattern_list or None,
-            filter_type=filter_type,
-            after=after,
-            before=before,
-        )
+        if snapshots is None:
+            snapshots = Snapshot.objects.order_by("-created_at").search(**filter_kwargs)
+        # Freeze the target set up-front so a concurrent daemon writing new
+        # snapshots can't extend the deletion under us, and so the cursor isn't
+        # held open across the per-row deletes below.
+        snapshot_pks = list(snapshots.values_list("pk", flat=True))
    finally:
        timer.end()

-    if not snapshots.exists():
+    if not snapshot_pks:
        log_removal_finished(0, 0)
        raise SystemExit(1)

-    log_list_finished(snapshots)
-    log_removal_started(snapshots, yes=yes)
+    if not yes:
+        log_list_finished(snapshots)
+        log_removal_started(snapshots, yes=False)

-    from archivebox.core.models import Snapshot
    from archivebox.search.query import flush_search_index

-    # Freeze the target set up-front so a concurrent daemon writing new
-    # snapshots can't extend the deletion under us, and so the cursor isn't
-    # held open across the per-row deletes below.
-    snapshot_pks = list(snapshots.values_list("pk", flat=True))
-    to_remove = len(snapshot_pks)
+    started_at = time.monotonic()
+    deadline = started_at + timeout if timeout is not None else None

    # Search-index flush touches a separate backend (FTS / sonic), not the
    # main index.sqlite3 writer lock, so it's safe to do once up front.
-    flush_search_index(snapshots=Snapshot.objects.filter(pk__in=snapshot_pks))
+    flush_search_index(snapshots=snapshots)

    # Delete one snapshot at a time. Each ``.delete()`` is its own short
    # Django-atomic block, so the writer lock is released between rows and
@@ -90,40 +98,60 @@ def remove(
    # own retry loop at this outer (non-atomic) level. Each attempt is a
    # fresh atomic; an exception cleanly rolls it back before we sleep.
    retry_interval = 1.0
-    retry_timeout = 60.0
-    for pk in snapshot_pks:
-        deadline = time.monotonic() + retry_timeout
+    deleted_snapshot_pks = []
+    timed_out = False
+    timeout_error = ""
+    for index, pk in enumerate(snapshot_pks):
+        if deadline is not None and time.monotonic() >= deadline:
+            timed_out = True
+            timeout_error = f"Remove timed out after {timeout:g}s with {len(snapshot_pks) - index} snapshots remaining."
+            break
        while True:
            try:
-                Snapshot.objects.filter(pk=pk).delete()
+                deleted_count, _ = Snapshot.objects.filter(pk=pk).delete()
+                if deleted_count:
+                    deleted_snapshot_pks.append(pk)
                break
            except OperationalError as err:
-                if "database is locked" not in str(err) or time.monotonic() >= deadline:
+                if "database is locked" not in str(err):
                    raise
-                time.sleep(retry_interval)
+                remaining_time = deadline - time.monotonic() if deadline is not None else None
+                if remaining_time is not None and remaining_time <= 0:
+                    timed_out = True
+                    timeout_error = f"Remove timed out after {timeout:g}s while waiting for the database lock."
+                    break
+                time.sleep(min(retry_interval, remaining_time) if remaining_time is not None else retry_interval)
+        if timed_out:
+            break

    all_snapshots = Snapshot.objects.all()
-    log_removal_finished(all_snapshots.count(), to_remove)
+    remaining_count = all_snapshots.count()
+    deleted_snapshot_id_set = set(deleted_snapshot_pks)
+    remaining_snapshot_pks = [snapshot_id for snapshot_id in snapshot_pks if snapshot_id not in deleted_snapshot_id_set]
+    log_removal_finished(remaining_count, len(deleted_snapshot_pks))

-    return all_snapshots
+    return {
+        "removed_count": len(deleted_snapshot_pks),
+        "removed_snapshot_ids": [str(snapshot_id) for snapshot_id in deleted_snapshot_pks],
+        "not_removed_count": len(remaining_snapshot_pks),
+        "not_removed_snapshot_ids": [str(snapshot_id) for snapshot_id in remaining_snapshot_pks],
+        "success": not timed_out,
+        "error": timeout_error,
+        "timeout": timeout,
+    }


@click.command()
@click.option("--yes", is_flag=True, help="Remove links instantly without prompting to confirm")
-@click.option("--before", type=float, help="Remove only URLs bookmarked before timestamp")
-@click.option("--after", type=float, help="Remove only URLs bookmarked after timestamp")
-@click.option(
-    "--filter-type",
-    "-f",
-    type=click.Choice(("exact", "substring", "domain", "regex", "tag")),
-    default="exact",
-    help="Type of pattern matching to use when filtering URLs",
-)
-@click.argument("filter_patterns", nargs=-1)
+@click.option("--delete", is_flag=True, help="Compatibility no-op; archived content is deleted automatically")
+@click.option("--timeout", type=float, default=None, help="Maximum seconds to spend deleting snapshots")
+@snapshot_filter_options(default_filter_type="exact")
@docstring(remove.__doc__)
 def main(**kwargs):
    """Remove the specified URLs from the archive"""
-    remove(**kwargs)
+    result = remove(**kwargs)
+    if not result["success"]:
+        raise SystemExit(124)


 if __name__ == "__main__":
@@ -243,11 +243,17 @@ def process_stdin_records() -> int:
                    crawl_id,
                    snapshot_ids=None if crawl_id in full_crawl_ids else sorted(snapshot_ids_by_crawl[crawl_id]),
                    selected_plugins=None if crawl_id in run_all_plugins_for_crawl else sorted(plugin_names_by_crawl[crawl_id]),
+                    selected_plugins_are_explicit=False,
                )
    return 0


-def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_only: bool = False) -> int:
+def run_runner(
+    daemon: bool = False,
+    crawl_id: str | None = None,
+    maintenance_only: bool = False,
+    maintenance_batch_size: int | None = None,
+) -> int:
    """
    Run the background runner loop.

@@ -259,16 +265,29 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on
    from archivebox.config import CONSTANTS
    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
    from archivebox.machine.models import Machine, Process
-    from archivebox.core.takeover_util import enter_single_runner_gate
-    from archivebox.services.runner import recover_orchestrator_state, run_pending_crawls
+    from archivebox.core.takeover_util import enter_single_runner_gate, standby_until_foreground_runner_needed
+    from archivebox.core.recovery_util import recover_orchestrator_state
+    from archivebox.services.runner import run_pending_crawls

    Machine.current()
    current = Process.current()
+    root_command = current.root
+    if daemon and root_command.process_type in (
+        Process.TypeChoices.SERVER,
+        Process.TypeChoices.ADD,
+        Process.TypeChoices.UPDATE,
+    ):
+        # Server-owned daemon runners are persistent supervisor workers, but
+        # foreground add/update commands are allowed to borrow runner/sonic
+        # leadership without taking down Daphne. Waiting here keeps the worker
+        # on the normal runner path while preventing a server restart loop from
+        # immediately stealing the single-runner gate back from the newer CLI.
+        standby_until_foreground_runner_needed(root_command, data_dir=CONSTANTS.DATA_DIR)
    if not enter_single_runner_gate(current, data_dir=CONSTANTS.DATA_DIR):
        current.mark_exited()
        return 0

-    recover_orchestrator_state(include_chrome=True)
+    recover_orchestrator_state(include_chrome=crawl_id is None, crawl_id=crawl_id)
    if crawl_id:
        from django.utils import timezone
        from archivebox.crawls.models import Crawl
@@ -305,9 +324,15 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on
                crawl_id=crawl_id,
                maintenance_only=maintenance_only,
                interactive_interrupts=interactive_interrupts,
+                **({"maintenance_batch_size": maintenance_batch_size} if maintenance_batch_size else {}),
            )
        return 0
-    except (KeyboardInterrupt, asyncio.CancelledError):
+    except KeyboardInterrupt:
+        return 0
+    except asyncio.CancelledError as e:
+        if daemon:
+            rprint(f"[red]Runner cancelled unexpectedly: {type(e).__name__}: {e}[/red]", file=sys.stderr)
+            return 1
        return 0
    except Exception as e:
        rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
@@ -324,7 +349,23 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on
@click.option("--snapshot-id", help="Run one snapshot through its crawl")
@click.option("--binary-id", help="Run one queued binary install directly on the bus")
@click.option("--maintenance-only", is_flag=True, help="Only process due maintenance ticks on sealed/paused snapshots")
-def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintenance_only: bool):
+@click.option(
+    "--maintenance-batch-size",
+    type=int,
+    default=None,
+    hidden=True,
+    help="Limit queued maintenance snapshots claimed per scheduler tick",
+)
+@click.option("--no-stdin", is_flag=True, hidden=True, help="Run the scheduler even when stdin is not a TTY")
+def main(
+    daemon: bool,
+    crawl_id: str,
+    snapshot_id: str,
+    binary_id: str,
+    maintenance_only: bool,
+    maintenance_batch_size: int | None,
+    no_stdin: bool,
+):
    """
    Process queued work.

@@ -347,7 +388,7 @@ def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintena
                ),
                foreground_parent_watchdog(enabled=False),
            ):
-                sys.exit(run_runner(daemon=True, maintenance_only=maintenance_only))
+                sys.exit(run_runner(daemon=True, maintenance_only=maintenance_only, maintenance_batch_size=maintenance_batch_size))
        except KeyboardInterrupt:
            sys.exit(0)

@@ -371,30 +412,49 @@ def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintena
                sys.exit(1)

        if crawl_id:
-            sys.exit(run_runner(daemon=False, crawl_id=crawl_id, maintenance_only=maintenance_only))
+            sys.exit(
+                run_runner(
+                    daemon=False,
+                    crawl_id=crawl_id,
+                    maintenance_only=maintenance_only,
+                    maintenance_batch_size=maintenance_batch_size,
+                ),
+            )

        if maintenance_only:
-            sys.exit(run_runner(daemon=daemon, maintenance_only=True))
+            sys.exit(run_runner(daemon=daemon, maintenance_only=True, maintenance_batch_size=maintenance_batch_size))

-        if not sys.stdin.isatty():
+        if not no_stdin and not sys.stdin.isatty():
            sys.exit(process_stdin_records())
        else:
-            sys.exit(run_runner(daemon=daemon, maintenance_only=maintenance_only))
+            sys.exit(run_runner(daemon=daemon, maintenance_only=maintenance_only, maintenance_batch_size=maintenance_batch_size))


 def run_snapshot_worker(snapshot_id: str) -> int:
+    from archivebox.config import CONSTANTS
+    from archivebox.core.takeover_util import enter_single_runner_gate
    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
+    from archivebox.machine.models import Process
    from archivebox.core.models import Snapshot
    from archivebox.services.runner import run_due_snapshot
    from django.utils import timezone

+    current = Process.current()
+    if not enter_single_runner_gate(current, data_dir=CONSTANTS.DATA_DIR):
+        current.mark_exited()
+        return 0
+
    snapshot = None
    try:
        with foreground_shutdown_signals(), foreground_parent_watchdog():
-            snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
-            if snapshot.retry_at is None:
-                snapshot.update_and_requeue(retry_at=timezone.now())
-            run_due_snapshot(snapshot, lock_seconds=60)
+            for _ in range(10):
+                snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
+                if snapshot.retry_at is None:
+                    snapshot.update_and_requeue(retry_at=timezone.now())
+                elif snapshot.retry_at > timezone.now():
+                    break
+                if not run_due_snapshot(snapshot, lock_seconds=60):
+                    break
        return 0
    except KeyboardInterrupt:
        try:
@@ -413,6 +473,10 @@ def run_snapshot_worker(snapshot_id: str) -> int:

        traceback.print_exc()
        return 1
+    finally:
+        current.refresh_from_db()
+        if current.status != Process.StatusChoices.EXITED:
+            current.mark_exited()


 if __name__ == "__main__":
@@ -3,255 +3,7 @@
 __package__ = "archivebox.cli"
 __command__ = "archivebox search"

-import sys
-from pathlib import Path
-from typing import TYPE_CHECKING
-from collections.abc import Callable
-
-import rich_click as click
-
-from django.db.models import Q, QuerySet
-
-from archivebox.config import CONSTANTS
-from archivebox.config.common import get_config
-from archivebox.misc.logging import stderr
-from archivebox.misc.util import enforce_types, docstring
-
-if TYPE_CHECKING:
-    from archivebox.core.models import Snapshot
-
-# Filter types for URL matching
-LINK_FILTERS: dict[str, Callable[[str], Q]] = {
-    "exact": lambda pattern: Q(url=pattern),
-    "substring": lambda pattern: Q(url__icontains=pattern),
-    "regex": lambda pattern: Q(url__iregex=pattern),
-    "domain": lambda pattern: (
-        Q(url__istartswith=f"http://{pattern}") | Q(url__istartswith=f"https://{pattern}") | Q(url__istartswith=f"ftp://{pattern}")
-    ),
-    "tag": lambda pattern: Q(tags__name=pattern),
-    "timestamp": lambda pattern: Q(timestamp=pattern),
-}
-
-STATUS_CHOICES = ["indexed", "archived", "unarchived"]
-
-
-def _apply_pattern_filters(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    filter_patterns: list[str],
-    filter_type: str,
-) -> QuerySet["Snapshot", "Snapshot"]:
-    filter_builder = LINK_FILTERS.get(filter_type)
-    if filter_builder is None:
-        stderr()
-        stderr(f"[X] Got invalid pattern for --filter-type={filter_type}", color="red")
-        raise SystemExit(2)
-
-    query = Q()
-    for pattern in filter_patterns:
-        query |= filter_builder(pattern)
-    return snapshots.filter(query)
-
-
-def _snapshots_to_json(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    *,
-    with_headers: bool,
-) -> str:
-    from datetime import datetime, timezone as tz
-
-    from archivebox.config import VERSION
-    from archivebox.misc.util import to_json
-
-    config = get_config()
-    main_index_header = (
-        {
-            "info": "This is an index of site data archived by ArchiveBox: The self-hosted web archive.",
-            "schema": "archivebox.index.json",
-            "copyright_info": config.FOOTER_INFO,
-            "meta": {
-                "project": "ArchiveBox",
-                "version": VERSION,
-                "git_sha": VERSION,
-                "website": "https://ArchiveBox.io",
-                "docs": "https://github.com/ArchiveBox/ArchiveBox/wiki",
-                "source": "https://github.com/ArchiveBox/ArchiveBox",
-                "issues": "https://github.com/ArchiveBox/ArchiveBox/issues",
-                "dependencies": {},
-            },
-        }
-        if with_headers
-        else {}
-    )
-
-    snapshot_dicts = [snapshot.to_dict(extended=True) for snapshot in snapshots.iterator(chunk_size=500)]
-    output: dict[str, object] | list[dict[str, object]]
-    if with_headers:
-        output = {
-            **main_index_header,
-            "num_links": len(snapshot_dicts),
-            "updated": datetime.now(tz.utc),
-            "last_run_cmd": sys.argv,
-            "links": snapshot_dicts,
-        }
-    else:
-        output = snapshot_dicts
-
-    return to_json(output, indent=4, sort_keys=True)
-
-
-def _snapshots_to_csv(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    *,
-    cols: list[str],
-    with_headers: bool,
-) -> str:
-    header = ",".join(cols) if with_headers else ""
-    rows = [snapshot.to_csv(cols=cols, separator=",") for snapshot in snapshots.iterator(chunk_size=500)]
-    return "\n".join((header, *rows))
-
-
-def _snapshots_to_html(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    *,
-    with_headers: bool,
-) -> str:
-    from datetime import datetime, timezone as tz
-
-    from django.template.loader import render_to_string
-
-    from archivebox.config import VERSION
-    from archivebox.config.version import get_COMMIT_HASH
-
-    config = get_config()
-    template = "static_index.html" if with_headers else "minimal_index.html"
-    snapshot_list = list(snapshots.iterator(chunk_size=500))
-
-    return render_to_string(
-        template,
-        {
-            "version": VERSION,
-            "git_sha": get_COMMIT_HASH() or VERSION,
-            "num_links": str(len(snapshot_list)),
-            "date_updated": datetime.now(tz.utc).strftime("%Y-%m-%d"),
-            "time_updated": datetime.now(tz.utc).strftime("%Y-%m-%d %H:%M"),
-            "links": snapshot_list,
-            "FOOTER_INFO": config.FOOTER_INFO,
-        },
-    )
-
-
-def get_snapshots(
-    snapshots: QuerySet["Snapshot", "Snapshot"] | None = None,
-    filter_patterns: list[str] | None = None,
-    filter_type: str = "substring",
-    after: float | None = None,
-    before: float | None = None,
-    out_dir: Path = CONSTANTS.DATA_DIR,
-) -> QuerySet["Snapshot", "Snapshot"]:
-    """Filter and return Snapshots matching the given criteria."""
-    from archivebox.core.models import Snapshot
-
-    if snapshots is not None:
-        result = snapshots
-    else:
-        result = Snapshot.objects.all()
-
-    if after is not None:
-        result = result.filter(timestamp__gte=after)
-    if before is not None:
-        result = result.filter(timestamp__lt=before)
-    if filter_patterns:
-        result = _apply_pattern_filters(result, filter_patterns, filter_type)
-
-    # Prefetch crawl relationship to avoid N+1 queries when accessing output_dir
-    result = result.select_related("crawl", "crawl__created_by")
-
-    if not result.exists():
-        stderr("[!] No Snapshots matched your filters:", filter_patterns, f"({filter_type})", color="lightyellow")
-
-    return result
-
-
-@enforce_types
-def search(
-    filter_patterns: list[str] | None = None,
-    filter_type: str = "substring",
-    status: str = "indexed",
-    before: float | None = None,
-    after: float | None = None,
-    sort: str | None = None,
-    json: bool = False,
-    html: bool = False,
-    csv: str | None = None,
-    with_headers: bool = False,
-):
-    """List, filter, and export information about archive entries"""
-
-    if with_headers and not (json or html or csv):
-        stderr("[X] --with-headers requires --json, --html or --csv\n", color="red")
-        raise SystemExit(2)
-
-    # Query DB directly - no filesystem scanning
-    snapshots = get_snapshots(
-        filter_patterns=list(filter_patterns) if filter_patterns else None,
-        filter_type=filter_type,
-        before=before,
-        after=after,
-    )
-
-    # Apply status filter
-    if status == "archived":
-        snapshots = snapshots.filter(downloaded_at__isnull=False)
-    elif status == "unarchived":
-        snapshots = snapshots.filter(downloaded_at__isnull=True)
-    # 'indexed' = all snapshots (no filter)
-
-    if sort:
-        snapshots = snapshots.order_by(sort)
-
-    # Export to requested format
-    if json:
-        output = _snapshots_to_json(snapshots, with_headers=with_headers)
-    elif html:
-        output = _snapshots_to_html(snapshots, with_headers=with_headers)
-    elif csv:
-        output = _snapshots_to_csv(snapshots, cols=csv.split(","), with_headers=with_headers)
-    else:
-        from archivebox.misc.logging_util import printable_folders
-
-        # Convert to dict for printable_folders
-        folders: dict[str, Snapshot | None] = {str(snapshot.output_dir): snapshot for snapshot in snapshots}
-        output = printable_folders(folders, with_headers)
-
-    # Structured exports must be written directly to stdout.
-    # rich.print() reflows long lines to console width, which corrupts JSON/CSV/HTML output.
-    sys.stdout.write(output)
-    if not output.endswith("\n"):
-        sys.stdout.write("\n")
-    return output
-
-
-@click.command()
-@click.option(
-    "--filter-type",
-    "-f",
-    type=click.Choice(["search", *LINK_FILTERS.keys()]),
-    default="substring",
-    help="Pattern matching type for filtering URLs",
-)
-@click.option("--status", "-s", type=click.Choice(STATUS_CHOICES), default="indexed", help="List snapshots with the given status")
-@click.option("--before", "-b", type=float, help="List snapshots bookmarked before the given UNIX timestamp")
-@click.option("--after", "-a", type=float, help="List snapshots bookmarked after the given UNIX timestamp")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--json", "-J", is_flag=True, help="Print output in JSON format")
-@click.option("--html", "-M", is_flag=True, help="Print output in HTML format (suitable for viewing statically without a server)")
-@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: created_at,url,title")
-@click.option("--with-headers", "-H", is_flag=True, help="Include extra CSV/HTML headers in the output")
-@click.help_option("--help", "-h")
-@click.argument("filter_patterns", nargs=-1)
-@docstring(search.__doc__)
-def main(**kwargs):
-    return search(**kwargs)
+from archivebox.cli.archivebox_list import main


 if __name__ == "__main__":
@@ -231,6 +231,7 @@ def server(
    debug: bool = False,
    daemonize: bool = False,
    nothreading: bool = False,
+    createsuperuser: bool = False,
 ) -> None:
    """Run the ArchiveBox HTTP server"""
    from archivebox.config.common import get_config
@@ -244,6 +245,12 @@ def server(

    from django.contrib.auth.models import User

+    if createsuperuser:
+        from archivebox.cli.archivebox_manage import manage
+
+        manage(args=["createsuperuser"])
+        print()
+
    if not User.objects.filter(is_superuser=True).exclude(username="system").exists():
        print()
        print(
@@ -313,6 +320,7 @@ def server(
    from archivebox.core.takeover_util import (
        command_owns_runtime_stack,
        current_command,
+        foreground_runner_owner,
        runtime_stack_owner,
        standby_until_runtime_stack_needed,
    )
@@ -354,7 +362,10 @@ def server(
        ):
            while True:
                standby_result = standby_until_runtime_stack_needed(command, data_dir=CONSTANTS.DATA_DIR)
-                older_owner = runtime_stack_owner(data_dir=CONSTANTS.DATA_DIR, exclude_id=command.id)
+                older_owner = runtime_stack_owner(data_dir=CONSTANTS.DATA_DIR, exclude_id=command.id) or foreground_runner_owner(
+                    data_dir=CONSTANTS.DATA_DIR,
+                    exclude_id=command.id,
+                )
                takeover_components = active_supervisord_runtime_components(config=config)
                if older_owner and takeover_components:
                    print(
@@ -400,6 +411,7 @@ def server(
@click.option("--debug", is_flag=True, help="Enable DEBUG=True mode with more verbose errors")
@click.option("--nothreading", is_flag=True, help="Force runserver to run in single-threaded mode")
@click.option("--daemonize", is_flag=True, help="Run the server in the background as a daemon")
+@click.option("--createsuperuser", is_flag=True, help="Run archivebox manage createsuperuser before starting the server")
@docstring(server.__doc__)
 def main(**kwargs):
    server(**kwargs)
@@ -12,9 +12,10 @@ from archivebox.misc.util import docstring
 def shell(args: Iterable[str] = ()) -> None:
    """Enter an interactive ArchiveBox Django shell"""

-    from django.core.management import call_command
+    from django.core.management import call_command, get_commands

-    call_command("shell_plus", *args)
+    shell_command = "shell_plus" if "shell_plus" in get_commands() else "shell"
+    call_command(shell_command, *args)


@click.command(add_help_option=False, context_settings=dict(ignore_unknown_options=True))
@@ -35,9 +35,10 @@ from collections.abc import Iterable

 import rich_click as click
 from rich import print as rprint
-from django.db.models import Case, IntegerField, QuerySet, When
+from django.db.models import QuerySet

-from archivebox.cli.cli_util import apply_filters
+SNAPSHOT_FILTER_TYPE_CHOICES = ("exact", "substring", "regex", "domain", "tag", "timestamp")
+SNAPSHOT_LIST_CHUNK_SIZE = 100


 # =============================================================================
@@ -66,7 +67,7 @@ def create_snapshots(
        TYPE_SNAPSHOT,
        TYPE_CRAWL,
    )
-    from archivebox.misc.util import validate_url_length
+    from archivebox.misc.util import validate_url
    from archivebox.base_models.models import get_or_create_system_user_pk
    from archivebox.core.models import Snapshot
    from archivebox.crawls.models import Crawl
@@ -111,7 +112,7 @@ def create_snapshots(
                # Create snapshots for each URL in the crawl
                for url in crawl.get_urls_list():
                    try:
-                        validate_url_length(url)
+                        validate_url(url)
                    except ValueError as err:
                        rprint(f"[red]Error creating snapshot: {err}[/red]", file=sys.stderr)
                        continue
@@ -134,7 +135,7 @@ def create_snapshots(
            elif record_type == TYPE_SNAPSHOT or record.get("url"):
                # Input is a Snapshot or plain URL
                if record.get("url"):
-                    validate_url_length(str(record["url"]))
+                    validate_url(str(record["url"]))
                if tag and not record.get("tags"):
                    record["tags"] = tag
                if status:
@@ -178,72 +179,63 @@ def create_snapshots(
 # =============================================================================


+def snapshot_filter_options(*, default_filter_type: str):
+    def decorate(func):
+        for decorator in reversed(
+            (
+                click.option("--status", "-s", help="Filter by status (queued, started, sealed)"),
+                click.option("--url__icontains", help="Filter by URL contains"),
+                click.option("--url__istartswith", help="Filter by URL starts with"),
+                click.option("--tag", "-t", help="Filter by tag name"),
+                click.option("--crawl-id", help="Filter by crawl ID"),
+                click.option("--limit", "-n", type=int, help="Limit number of results"),
+                click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at"),
+                click.option("--search", help="Search mode to use for positional query"),
+                click.option("--before", type=float, help="Only snapshots bookmarked before timestamp"),
+                click.option("--after", type=float, help="Only snapshots bookmarked after timestamp"),
+                click.option(
+                    "--filter-type",
+                    "-f",
+                    type=click.Choice(SNAPSHOT_FILTER_TYPE_CHOICES),
+                    default=default_filter_type,
+                    help="Type of pattern matching to use for positional filters",
+                ),
+                click.argument("filter_patterns", nargs=-1),
+            ),
+        ):
+            func = decorator(func)
+        return func
+
+    return decorate
+
+
+def snapshot_output_options(func):
+    for decorator in reversed(
+        (
+            click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title"),
+            click.option("--json", "as_json", is_flag=True, help="Print output as a JSON array"),
+            click.option("--html", "as_html", is_flag=True, help="Print output as HTML"),
+            click.option("--with-headers", is_flag=True, help="Include column headers in structured output"),
+        ),
+    ):
+        func = decorator(func)
+    return func
+
+
 def build_snapshot_queryset(
-    *,
-    status: str | None = None,
-    url__icontains: str | None = None,
-    url__istartswith: str | None = None,
-    tag: str | None = None,
-    crawl_id: str | None = None,
-    sort: str | None = None,
-    search: str | None = None,
-    query: str | None = None,
-    limit: int | None = None,
+    **kwargs,
 ) -> QuerySet:
    from archivebox.core.models import Snapshot
-    from archivebox.search.query import apply_snapshot_search

-    queryset = Snapshot.objects.order_by("-created_at")
-    queryset = apply_filters(
-        queryset,
-        {
-            "status": status,
-            "url__icontains": url__icontains,
-            "url__istartswith": url__istartswith,
-            "crawl_id": crawl_id,
-        },
-    )
-
-    if tag:
-        queryset = queryset.filter(tags__name__iexact=tag)
-
-    query = (query or "").strip()
-    if query:
-        try:
-            queryset = apply_snapshot_search(
-                queryset,
-                query,
-                search_mode=search,
-                ordering=("-created_at",) if not sort else None,
-                max_results=limit,
-                skip_backend_when_metadata_satisfies_limit=True,
-                include_metadata_for_forced_backend=True,
-            )
-        except Exception as err:
-            rprint(
-                f"[yellow]Search backend error, falling back to metadata search: {err}[/yellow]",
-                file=sys.stderr,
-            )
-            queryset = apply_snapshot_search(queryset, query, search_mode="meta")
-
-    if sort:
-        queryset = queryset.order_by(sort)
-
-    return queryset
+    return Snapshot.objects.order_by("-created_at").search(**kwargs)


 def list_snapshots(
-    status: str | None = None,
-    url__icontains: str | None = None,
-    url__istartswith: str | None = None,
-    tag: str | None = None,
-    crawl_id: str | None = None,
-    limit: int | None = None,
-    sort: str | None = None,
    csv: str | None = None,
+    as_json: bool = False,
+    as_html: bool = False,
    with_headers: bool = False,
-    search: str | None = None,
-    query: str | None = None,
+    **kwargs,
 ) -> int:
    """
    List Snapshots as JSONL with optional filters.
@@ -252,90 +244,71 @@ def list_snapshots(
        0: Success (even if no results)
    """
    from archivebox.misc.jsonl import write_record
-    from archivebox.core.models import Snapshot

-    if with_headers and not csv:
-        rprint("[red]--with-headers requires --csv[/red]", file=sys.stderr)
+    output_formats = sum(bool(output_format) for output_format in (csv, as_json, as_html))
+    if output_formats > 1:
+        rprint("[red]Choose only one output format: --csv, --json, or --html[/red]", file=sys.stderr)
+        return 2
+    if with_headers and not output_formats:
+        rprint("[red]--with-headers requires --csv, --json, or --html[/red]", file=sys.stderr)
        return 2

-    is_tty = sys.stdout.isatty() and not csv
+    is_tty = sys.stdout.isatty() and not output_formats

-    queryset = build_snapshot_queryset(
-        status=status,
-        url__icontains=url__icontains,
-        url__istartswith=url__istartswith,
-        tag=tag,
-        crawl_id=crawl_id,
-        sort=sort,
-        search=search,
-        query=query,
-        limit=limit,
-    )
-
-    if not is_tty:
-        if limit:
-            limited_ids = list(queryset.values_list("id", flat=True)[:limit])
-            preserved_order = Case(
-                *(When(id=snapshot_id, then=position) for position, snapshot_id in enumerate(limited_ids)),
-                output_field=IntegerField(),
-            )
-            queryset = Snapshot.objects.filter(id__in=limited_ids).order_by(preserved_order)
-        queryset = queryset.prefetch_related("tags")
-    elif limit:
-        queryset = queryset[:limit]
+    try:
+        queryset = build_snapshot_queryset(**kwargs)
+    except ValueError as err:
+        rprint(f"[red]{err}[/red]", file=sys.stderr)
+        return 2

    count = 0
+    if as_json:
+        queryset = queryset.prefetch_related("tags")
+        output = queryset.to_json(with_headers=with_headers)
+        sys.stdout.write(output)
+        if output and not output.endswith("\n"):
+            sys.stdout.write("\n")
+        rprint(f"[dim]Listed {queryset.count()} snapshots[/dim]", file=sys.stderr)
+        return 0
+
+    if as_html:
+        queryset = queryset.prefetch_related("tags")
+        output = queryset.to_html(with_headers=with_headers)
+        sys.stdout.write(output)
+        if output and not output.endswith("\n"):
+            sys.stdout.write("\n")
+        rprint(f"[dim]Listed {queryset.count()} snapshots[/dim]", file=sys.stderr)
+        return 0
+
    if csv:
        cols = [col.strip() for col in csv.split(",") if col.strip()]
        if not cols:
            rprint("[red]No CSV columns provided[/red]", file=sys.stderr)
            return 2
-        rows: list[str] = []
        if with_headers:
-            rows.append(",".join(cols))
-        simple_cols = {
-            "id",
-            "crawl_id",
-            "url",
-            "title",
-            "timestamp",
-            "depth",
-            "status",
-            "fs_version",
-            "bookmarked_at",
-            "created_at",
-            "modified_at",
-            "retry_at",
-            "downloaded_at",
-        }
-        from archivebox.misc.util import to_json
-
-        for snapshot in queryset.iterator(chunk_size=500):
-            if set(cols).issubset(simple_cols):
-                rows.append(
-                    ",".join(to_json(snapshot.serializable_value(col), indent=None) for col in cols),
-                )
-            else:
-                rows.append(snapshot.to_csv(cols=cols, separator=","))
+            sys.stdout.write(",".join(cols))
+            sys.stdout.write("\n")
+        for snapshot in queryset.prefetch_related("tags").iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE):
+            sys.stdout.write(snapshot.to_csv(cols=cols, separator=","))
+            sys.stdout.write("\n")
            count += 1
-        output = "\n".join(rows)
-        if output:
-            sys.stdout.write(output)
-            if not output.endswith("\n"):
-                sys.stdout.write("\n")
        rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
        return 0

-    for snapshot in queryset:
-        if is_tty:
-            status_color = {
-                "queued": "yellow",
-                "started": "blue",
-                "sealed": "green",
-            }.get(snapshot.status, "dim")
-            rprint(f"[{status_color}]{snapshot.status:8}[/{status_color}] [dim]{snapshot.id}[/dim] {snapshot.url[:60]}")
-        else:
+    if not is_tty:
+        for snapshot in queryset.prefetch_related("tags").iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE):
            write_record(snapshot.to_json())
+            count += 1
+        rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
+        return 0
+
+    for snapshot in queryset.iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE):
+        status_color = {
+            "queued": "yellow",
+            "started": "blue",
+            "sealed": "green",
+        }.get(snapshot.status, "dim")
+        rprint(f"[{status_color}]{snapshot.status:8}[/{status_color}] [dim]{snapshot.id}[/dim] {snapshot.url[:60]}")
        count += 1

    rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
@@ -494,46 +467,11 @@ def create_cmd(urls: tuple, tag: str, status: str, depth: int):


@main.command("list")
-@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
-@click.option("--url__icontains", help="Filter by URL contains")
-@click.option("--url__istartswith", help="Filter by URL starts with")
-@click.option("--tag", "-t", help="Filter by tag name")
-@click.option("--crawl-id", help="Filter by crawl ID")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title")
-@click.option("--with-headers", is_flag=True, help="Include column headers in structured output")
-@click.option("--search", help="Search mode to use for the query")
-@click.argument("query", nargs=-1)
-def list_cmd(
-    status: str | None,
-    url__icontains: str | None,
-    url__istartswith: str | None,
-    tag: str | None,
-    crawl_id: str | None,
-    limit: int | None,
-    sort: str | None,
-    csv: str | None,
-    with_headers: bool,
-    search: str | None,
-    query: tuple[str, ...],
-):
+@snapshot_output_options
+@snapshot_filter_options(default_filter_type="substring")
+def list_cmd(**kwargs):
    """List Snapshots as JSONL."""
-    sys.exit(
-        list_snapshots(
-            status=status,
-            url__icontains=url__icontains,
-            url__istartswith=url__istartswith,
-            tag=tag,
-            crawl_id=crawl_id,
-            limit=limit,
-            sort=sort,
-            csv=csv,
-            with_headers=with_headers,
-            search=search,
-            query=" ".join(query),
-        ),
-    )
+    sys.exit(list_snapshots(**kwargs))


@main.command("update")
@@ -1,24 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox snapshot"
-
-import sys
-
-import rich_click as click
-
-from archivebox.cli.archivebox_snapshot import create_snapshots
-
-
-@click.command(context_settings={"ignore_unknown_options": True})
-@click.option("--tag", "-t", default="", help="Comma-separated tags to add")
-@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
-@click.option("--depth", "-d", type=int, default=0, help="Crawl depth (default: 0)")
-@click.argument("urls", nargs=-1)
-def main(tag: str, status: str, depth: int, urls: tuple[str, ...]):
-    """Backwards-compatible `archivebox snapshot URL...` entrypoint."""
-    sys.exit(create_snapshots(urls, tag=tag, status=status, depth=depth))
-
-
-if __name__ == "__main__":
-    main()
@@ -105,7 +105,7 @@ def status(out_dir: Path = CONSTANTS.DATA_DIR) -> None:

    if num_indexed:
        print("    [violet]Hint:[/violet] You can list snapshots by status like so:")
-        print("        [green]archivebox list --status=<status>  (e.g. archived, queued, etc.)[/green]")
+        print("        [green]archivebox list --status=<status>  (e.g. sealed, queued, etc.)[/green]")

    if orphaned_dirs:
        print("    [violet]Hint:[/violet] To automatically import orphaned data directories into the main index, run:")
@@ -15,6 +15,7 @@ from pathlib import Path
 import rich_click as click

 from archivebox.misc.util import enforce_types, docstring
+from archivebox.cli.archivebox_snapshot import snapshot_filter_options

 if TYPE_CHECKING:
    from django.db.models import QuerySet
@@ -32,66 +33,36 @@ def _get_snapshot_crawl(snapshot: Snapshot) -> Crawl | None:


 def _get_search_indexing_plugins() -> list[str]:
-    from abx_dl.models import discover_plugins
+    from archivebox.config.common import get_config
+    from archivebox.plugins.hooks import discover_hooks
    from archivebox.plugins.discovery import get_search_backends

    available_backends = set(get_search_backends())
-    plugins = discover_plugins()
    return sorted(
        plugin_name
-        for plugin_name, plugin in plugins.items()
-        if plugin_name.startswith("search_backend_")
-        and plugin_name.removeprefix("search_backend_") in available_backends
-        and any("Snapshot" in hook.name and "index" in hook.name.lower() for hook in plugin.hooks)
+        for plugin_name in {
+            hook.parent.name
+            for hook in discover_hooks("Snapshot", config=get_config())
+            if hook.parent.name.startswith("search_backend_") and "index" in hook.name.lower()
+        }
+        if plugin_name.startswith("search_backend_") and plugin_name.removeprefix("search_backend_") in available_backends
    )


 def _build_filtered_snapshots_queryset(
-    *,
-    filter_patterns: Iterable[str],
-    filter_type: str,
-    status: str | None = None,
-    url__icontains: str | None = None,
-    url__istartswith: str | None = None,
-    tag: str | None = None,
-    crawl_id: str | None = None,
-    limit: int | None = None,
-    sort: str | None = None,
-    search: str | None = None,
-    before: float | None = None,
-    after: float | None = None,
-    resume: str | None = None,
+    **kwargs,
 ):
-    from datetime import datetime
+    from archivebox.core.models import Snapshot
    from archivebox.cli.archivebox_snapshot import build_snapshot_queryset

-    filter_patterns = tuple(filter_patterns)
-    snapshots = build_snapshot_queryset(
-        status=status,
-        url__icontains=url__icontains,
-        url__istartswith=url__istartswith,
-        tag=tag,
-        crawl_id=crawl_id,
-        sort=sort,
-        search=search,
-        query=" ".join(filter_patterns) if search else None,
-    )
-
-    if filter_patterns and not search:
-        snapshots = snapshots.filter_by_patterns(list(filter_patterns), filter_type)
-
-    if before:
-        snapshots = snapshots.filter(bookmarked_at__lt=datetime.fromtimestamp(before))
-    if after:
-        snapshots = snapshots.filter(bookmarked_at__gt=datetime.fromtimestamp(after))
-    if resume:
-        snapshots = snapshots.filter(timestamp__lte=resume)
-    if not sort:
-        snapshots = snapshots.order_by("-timestamp")
+    limit = kwargs.pop("limit", None)
+    snapshots = build_snapshot_queryset(**kwargs)
+    if kwargs.get("resume"):
+        snapshots = snapshots.filter(timestamp__lte=kwargs["resume"])
    snapshots = snapshots.select_related("crawl")
-    if limit:
-        limited_ids = list(snapshots.values_list("id", flat=True)[:limit])
-        snapshots = snapshots.model.objects.filter(id__in=limited_ids).select_related("crawl")
+    if limit is not None and limit > 0:
+        snapshot_ids = list(snapshots.values_list("id", flat=True)[:limit])
+        snapshots = Snapshot.objects.filter(id__in=snapshot_ids).select_related("crawl")

    return snapshots

@@ -105,9 +76,17 @@ def reindex_snapshots(
    wait_for_turn=None,
 ) -> dict[str, Any]:
    from archivebox.cli.archivebox_extract import run_plugins
+    from archivebox.core.models import ArchiveResult
+    from abx_dl.models import discover_plugins

-    stats: dict[str, Any] = {"processed": 0, "queued": 0, "reindexed": 0, "snapshot_ids": []}
+    stats: dict[str, Any] = {"processed": 0, "requested": 0, "queued": 0, "skipped_queued": 0, "reindexed": 0, "snapshot_ids": []}
    records: list[dict[str, str]] = []
+    plugins_by_name = discover_plugins(runtime="archivebox")
+    required_hooks_by_plugin = {
+        plugin_name: frozenset(hook.name for hook in plugins_by_name[plugin_name].filter_hooks("Snapshot"))
+        for plugin_name in search_plugins
+        if plugin_name in plugins_by_name
+    }

    total = snapshots.count()
    print(f"[*] Reindexing {total} snapshots with search plugins: {', '.join(search_plugins)}")
@@ -118,6 +97,31 @@ def reindex_snapshots(
        if wait_for_turn:
            wait_for_turn()
        batch_records = list(records)
+        snapshot_ids = {record["snapshot_id"] for record in batch_records}
+        plugin_names = {record["plugin"] for record in batch_records}
+        queued_rows = {
+            (str(snapshot_id), plugin_name, hook_name)
+            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
+                snapshot_id__in=snapshot_ids,
+                plugin__in=plugin_names,
+                status=ArchiveResult.StatusChoices.QUEUED,
+            ).values_list("snapshot_id", "plugin", "hook_name")
+        }
+        records_to_queue = []
+        for record in batch_records:
+            snapshot_id = record["snapshot_id"]
+            plugin_name = record["plugin"]
+            required_hooks = required_hooks_by_plugin.get(plugin_name, frozenset())
+            if required_hooks and all((snapshot_id, plugin_name, hook_name) in queued_rows for hook_name in required_hooks):
+                stats["skipped_queued"] += 1
+                continue
+            records_to_queue.append(record)
+        if not records_to_queue:
+            print(
+                f"    [{stats['processed']}/{total}] Already queued {len(batch_records)} index jobs",
+            )
+            records.clear()
+            return
        # `archivebox update --index-only` intentionally breaks the usual
        # "runner discovers work" rule by inserting synthetic queued
        # ArchiveResult rows for search backends. run_plugins() keeps this as
@@ -127,15 +131,17 @@ def reindex_snapshots(
        # finish.
        exit_code = run_plugins(
            args=(),
-            records=batch_records,
+            records=records_to_queue,
            wait=False,
            emit_results=False,
            show_progress=False,
+            preserve_queued=True,
        )
        if exit_code != 0:
            raise SystemExit(exit_code)
+        stats["queued"] += len(records_to_queue)
        print(
-            f"    [{stats['processed']}/{total}] Queued {len(batch_records)} index jobs for orchestrator",
+            f"    [{stats['processed']}/{total}] Queued {len(records_to_queue)} index jobs for orchestrator",
        )
        records.clear()

@@ -156,7 +162,7 @@ def reindex_snapshots(
                        "plugin": plugin_name,
                    },
                )
-                stats["queued"] += 1
+                stats["requested"] += 1
            if len(records) >= batch_size:
                run_batch()
        except KeyboardInterrupt as err:
@@ -219,27 +225,42 @@ def update(
    from archivebox.machine.models import Process
    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals, raise_if_shutdown_requested
    from archivebox.core.takeover_util import (
-        command_owns_runtime_stack,
+        command_owns_foreground_runner,
        current_command,
        ensure_daemon_stack,
-        standby_until_runtime_stack_needed,
+        foreground_runner_owner,
+        standby_until_foreground_runner_needed,
    )
-    from archivebox.workers.supervisord_util import run_runner_worker, stop_existing_supervisord_process, stop_own_supervisord_process
+    from archivebox.workers.supervisord_util import run_runner_worker, stop_own_supervisord_process

    command = current_command(Process.TypeChoices.UPDATE, data_dir=CONSTANTS.DATA_DIR)

+    def still_owns_foreground_runner() -> bool:
+        from django.db import connections
+
+        try:
+            return command_owns_foreground_runner(command, data_dir=CONSTANTS.DATA_DIR)
+        finally:
+            connections.close_all()
+
    def wait_for_turn() -> None:
        raise_if_shutdown_requested()
-        standby_until_runtime_stack_needed(command, data_dir=CONSTANTS.DATA_DIR)
+        standby_until_foreground_runner_needed(command, data_dir=CONSTANTS.DATA_DIR)
        raise_if_shutdown_requested()

-    def run_scoped_runner(*args: str) -> None:
+    def run_scoped_runner(*args: str, ensure_daemon_reason: str | None = None) -> None:
        while True:
            wait_for_turn()
-            exit_code = run_runner_worker(list(args), name=f"worker_runner_update_{os.getpid()}")
+            if ensure_daemon_reason:
+                ensure_daemon_stack(reason=ensure_daemon_reason)
+            exit_code = run_runner_worker(
+                list(args),
+                name=f"worker_runner_update_{os.getpid()}",
+                keep_running=still_owns_foreground_runner,
+            )
            if exit_code == 0:
                return
-            if not command_owns_runtime_stack(command, data_dir=CONSTANTS.DATA_DIR):
+            if not command_owns_foreground_runner(command, data_dir=CONSTANTS.DATA_DIR):
                continue
            raise SystemExit(exit_code)

@@ -263,8 +284,6 @@ def update(

    try:
        wait_for_turn()
-        if stop_daemon_stack:
-            stop_existing_supervisord_process()

        with foreground_shutdown_signals(), foreground_parent_watchdog():
            while True:
@@ -272,6 +291,9 @@ def update(
                do_index = index_only or not migrate_only
                do_run_until_idle = do_migrate or do_index
                ran_post_migrate_runner = False
+                full_update_empty = False
+                maintenance_work_queued = False
+                runner_work_queued = False

                if do_migrate:
                    if (
@@ -308,6 +330,8 @@ def update(
                        )
                        print_stats(stats)
                        touched_snapshot_ids.update(stats.get("snapshot_ids", []))
+                        maintenance_work_queued = stats.get("queued", 0) > 0
+                        runner_work_queued = runner_work_queued or maintenance_work_queued
                    else:
                        stats_combined = {"phase1": {}, "phase2": {}}

@@ -324,6 +348,17 @@ def update(
                            wait_for_turn=wait_for_turn,
                        )
                        print_combined_stats(stats_combined)
+                        full_update_empty = (
+                            stats_combined["phase1"].get("processed", 0) == 0 and stats_combined["phase2"].get("snapshots", 0) == 0
+                        )
+                        maintenance_work_queued = any(
+                            (
+                                stats_combined["phase1"].get("queued", 0),
+                                stats_combined["phase2"].get("queued", 0),
+                                stats_combined["phase2"].get("crawls_sealed", 0),
+                            ),
+                        )
+                        runner_work_queued = runner_work_queued or maintenance_work_queued

                    if do_run_until_idle:
                        # Filesystem migration is maintenance on existing
@@ -335,46 +370,108 @@ def update(
                        # rows first, runs the targeted plugins, and may leave
                        # the fs_version maintenance tick hidden behind that
                        # plugin work until another update pass.
-                        print("[*] Phase 3: Running filesystem maintenance until idle...")
-                        if is_filtered_update:
+                        if full_update_empty:
+                            print("[*] No snapshots or legacy archive directories found; skipping filesystem maintenance runner.")
+                        elif not maintenance_work_queued:
+                            print("[*] No filesystem maintenance work queued; skipping filesystem maintenance runner.")
+                        else:
+                            print("[*] Phase 3: Running filesystem maintenance until idle...")
+                        if full_update_empty:
+                            pass
+                        elif not maintenance_work_queued:
+                            pass
+                        elif is_filtered_update:
                            if not touched_snapshot_ids:
                                print("[*] No matching snapshots queued work for the runner.")
                            for snapshot_id in sorted(touched_snapshot_ids):
                                run_scoped_runner("--snapshot-id", snapshot_id)
                        else:
-                            run_scoped_runner("--maintenance-only")
+                            run_scoped_runner("--maintenance-only", "--maintenance-batch-size", str(batch_size))
                        ran_post_migrate_runner = True

                if do_index:
-                    ensure_daemon_stack(reason="search indexing")
-                    search_plugins = _get_search_indexing_plugins()
-                    if not search_plugins:
-                        print("[*] No search indexing plugins are available, nothing to backfill.")
+                    if full_update_empty:
+                        print("[*] No snapshots found; skipping search indexing backfill.")
                    else:
-                        snapshots = _build_filtered_snapshots_queryset(
-                            filter_patterns=filter_patterns,
-                            filter_type=filter_type,
-                            status=status,
-                            url__icontains=url__icontains,
-                            url__istartswith=url__istartswith,
-                            tag=tag,
-                            crawl_id=crawl_id,
-                            limit=limit,
-                            sort=sort,
-                            search=search,
-                            before=before,
-                            after=after,
-                            resume=resume,
-                        )
-                        stats = reindex_snapshots(
-                            snapshots,
-                            search_plugins=search_plugins,
-                            batch_size=batch_size,
-                            collect_ids=is_filtered_update,
-                            wait_for_turn=wait_for_turn,
-                        )
-                        print_index_stats(stats)
-                        touched_snapshot_ids.update(stats.get("snapshot_ids", []))
+                        ensure_daemon_stack(reason="search indexing")
+                        search_plugins = _get_search_indexing_plugins()
+                        if not search_plugins:
+                            print("[*] No search indexing plugins are available, nothing to backfill.")
+                        else:
+                            snapshots = _build_filtered_snapshots_queryset(
+                                filter_patterns=filter_patterns,
+                                filter_type=filter_type,
+                                status=status,
+                                url__icontains=url__icontains,
+                                url__istartswith=url__istartswith,
+                                tag=tag,
+                                crawl_id=crawl_id,
+                                limit=limit,
+                                sort=sort,
+                                search=search,
+                                before=before,
+                                after=after,
+                                resume=resume,
+                            )
+                            from django.db.models import Exists, OuterRef, Q
+                            from django.utils import timezone
+                            from archivebox.core.models import ArchiveResult, Snapshot
+
+                            scoped_snapshot_ids = snapshots.order_by().values("id") if is_filtered_update else None
+                            queued_index_results = ArchiveResult.objects.filter(
+                                status=ArchiveResult.StatusChoices.QUEUED,
+                                plugin__in=search_plugins,
+                            )
+                            if scoped_snapshot_ids is not None:
+                                queued_index_results = queued_index_results.filter(snapshot_id__in=scoped_snapshot_ids)
+
+                            if queued_index_results.exists():
+                                runner_work_queued = True
+                                now = timezone.now()
+                                queued_result_for_snapshot = queued_index_results.filter(snapshot_id=OuterRef("pk"))
+                                snapshots_to_wake = (
+                                    Snapshot.objects.filter(
+                                        status__in=(Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED),
+                                    )
+                                    .annotate(
+                                        has_queued_index_result=Exists(queued_result_for_snapshot),
+                                    )
+                                    .filter(
+                                        has_queued_index_result=True,
+                                    )
+                                    .filter(
+                                        Q(retry_at__isnull=True) | Q(retry_at__gt=now),
+                                    )
+                                )
+                                if scoped_snapshot_ids is not None:
+                                    snapshots_to_wake = snapshots_to_wake.filter(id__in=scoped_snapshot_ids)
+                                woken_count = snapshots_to_wake.update(
+                                    retry_at=now,
+                                    modified_at=now,
+                                )
+                                print(
+                                    "[*] Existing queued search index jobs found; "
+                                    f"skipping backfill scan and waking {woken_count} snapshot(s) for the runner.",
+                                )
+                            else:
+                                collect_index_ids = (
+                                    is_filtered_update
+                                    or foreground_runner_owner(
+                                        data_dir=CONSTANTS.DATA_DIR,
+                                        exclude_id=command.id,
+                                    )
+                                    is not None
+                                )
+                                stats = reindex_snapshots(
+                                    snapshots,
+                                    search_plugins=search_plugins,
+                                    batch_size=batch_size,
+                                    collect_ids=collect_index_ids,
+                                    wait_for_turn=wait_for_turn,
+                                )
+                                print_index_stats(stats)
+                                touched_snapshot_ids.update(stats.get("snapshot_ids", []))
+                                runner_work_queued = runner_work_queued or stats["queued"] > 0

                if do_run_until_idle and (do_index or not ran_post_migrate_runner):
                    # Search/index backfill intentionally queues targeted
@@ -384,14 +481,26 @@ def update(
                    # For a normal unfiltered `archivebox update`, keep the
                    # historical final pass broad enough to resume genuinely
                    # queued/interrupted crawl work after maintenance is done.
-                    print("[*] Phase 3: Running queued/interrupted crawl work until idle...")
-                    if is_filtered_update:
+                    if full_update_empty:
+                        print("[*] No snapshots found; skipping queued/interrupted crawl runner.")
+                    elif not runner_work_queued:
+                        print("[*] No queued/interrupted crawl work found; skipping queued/interrupted crawl runner.")
+                    else:
+                        print("[*] Phase 3: Running queued/interrupted crawl work until idle...")
+                    if full_update_empty:
+                        pass
+                    elif not runner_work_queued:
+                        pass
+                    elif touched_snapshot_ids and is_filtered_update:
                        if not touched_snapshot_ids:
                            print("[*] No matching snapshots queued work for the runner.")
                        for snapshot_id in sorted(touched_snapshot_ids):
                            run_scoped_runner("--snapshot-id", snapshot_id)
                    else:
-                        run_scoped_runner(*(["--maintenance-only"] if index_only or migrate_only else []))
+                        run_scoped_runner(
+                            *(["--maintenance-only", "--maintenance-batch-size", str(batch_size)] if index_only or migrate_only else []),
+                            ensure_daemon_reason="search indexing" if do_index else None,
+                        )

                if not continuous:
                    break
@@ -638,7 +747,7 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
        "updated_db": 0,
        "queued": 0,
        "sealed": 0,
-        "crawls_queued": 0,
+        "crawls_sealed": 0,
    }
    current_fs_version = Snapshot._fs_current_version()

@@ -646,6 +755,7 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
    if resume:
        queryset = queryset.filter(timestamp__lte=resume)
    total = queryset.count()
+    stats["snapshots"] = total
    print(f"[*] Processing {total} snapshots from database (most recent first)...")

    def update_in_batches(rows, *, label: str, **updates) -> int:
@@ -744,7 +854,10 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
    queue_stale_fs_batch()

    now = timezone.now()
-    stats["crawls_queued"] = (
+    # Crawls with no open child snapshots are already finished. Seal them here
+    # instead of waking the foreground runner; otherwise migration/update can
+    # accidentally re-enter full crawl execution for historical rows.
+    stats["crawls_sealed"] = (
        Crawl.objects.filter(
            status__in=Crawl.RUNNABLE_STATES,
        )
@@ -752,11 +865,12 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
            snapshot_set__status__in=Snapshot.OPEN_STATES,
        )
        .update(
-            retry_at=now,
+            status=Crawl.StatusChoices.SEALED,
+            retry_at=None,
            modified_at=now,
        )
    )
-    stats["updated_db"] += stats["crawls_queued"]
+    stats["updated_db"] += stats["crawls_sealed"]
    return stats


@@ -894,7 +1008,7 @@ Phase 2 (Process DB):
  Updated JSON:     {s2.get("updated_json", 0)}
  Updated DB rows:  {s2.get("updated_db", 0)}
  Sealed snapshots: {s2.get("sealed", 0)}
-  Queued crawls:    {s2.get("crawls_queued", 0)}
+  Sealed crawls:    {s2.get("crawls_sealed", 0)}
 """)


@@ -904,28 +1018,19 @@ def print_index_stats(stats: dict[str, Any]) -> None:
    print(f"""
 [green]Search Reindex Complete[/green]
  Scanned rows:      {stats["processed"]}
+  Requested jobs:    {stats.get("requested", stats["queued"])}
  Queued index jobs: {stats["queued"]}
+  Already queued:    {stats.get("skipped_queued", 0)}
 """)


@click.command()
@click.option("--resume", type=str, help="Resume from timestamp")
-@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
-@click.option("--url__icontains", help="Filter by URL contains")
-@click.option("--url__istartswith", help="Filter by URL starts with")
-@click.option("--tag", "-t", help="Filter by tag name")
-@click.option("--crawl-id", help="Filter by crawl ID")
-@click.option("--limit", "-n", type=int, help="Limit number of snapshots to update")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--search", help="Search mode to use for positional query")
-@click.option("--before", type=float, help="Only snapshots before timestamp")
-@click.option("--after", type=float, help="Only snapshots after timestamp")
-@click.option("--filter-type", type=click.Choice(["exact", "substring", "regex", "domain", "tag", "timestamp"]), default="exact")
@click.option("--batch-size", type=int, default=500, help="Commit every N records")
@click.option("--continuous", is_flag=True, help="Run continuously as background worker")
@click.option("--index-only", is_flag=True, help="Backfill available search indexes from existing archived content")
@click.option("--migrate-only", is_flag=True, help="Only migrate filesystem and update database/index state")
-@click.argument("filter_patterns", nargs=-1)
+@snapshot_filter_options(default_filter_type="exact")
@docstring(update.__doc__)
 def main(**kwargs):
    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
@@ -933,6 +1038,8 @@ def main(**kwargs):
    try:
        with foreground_shutdown_signals(), foreground_parent_watchdog():
            update(**kwargs)
+    except ValueError as err:
+        raise click.BadParameter(str(err), param_hint="--status") from err
    except KeyboardInterrupt:
        raise SystemExit(130) from None

@@ -29,8 +29,7 @@ def _format_binary_abspath(

    candidate_bases: tuple[tuple[Path, str], ...] = (
        (pwd, "./"),
-        (lib_dir, "LIB_DIR/"),
-        (Path(os.environ.get("LIB_DIR", "")), "LIB_DIR/") if os.environ.get("LIB_DIR") else (Path(), ""),
+        (lib_dir, "ABXPKG_LIB_DIR/"),
        (personas_dir, "PERSONAS_DIR/"),
        (home, "~/"),
    )
@@ -47,8 +46,8 @@ def _format_binary_abspath(
            relative_str = relative.as_posix()
            if prefix == "./":
                return "." if not relative_str else f"./{relative_str}"
-            if prefix == "LIB_DIR/":
-                return "LIB_DIR" if not relative_str else f"LIB_DIR/{relative_str}"
+            if prefix == "ABXPKG_LIB_DIR/":
+                return "ABXPKG_LIB_DIR" if not relative_str else f"ABXPKG_LIB_DIR/{relative_str}"
            if prefix == "PERSONAS_DIR/":
                return "PERSONAS_DIR" if not relative_str else f"PERSONAS_DIR/{relative_str}"
            return "~" if not relative_str else f"~/{relative_str}"
@@ -59,10 +58,10 @@ def _format_binary_abspath(
 def _render_binary_abspath(abspath: str):
    from rich.text import Text

-    if abspath.startswith("LIB_DIR/"):
-        return Text.assemble(("LIB_DIR", "bright_blue"), (abspath.removeprefix("LIB_DIR"), "green"))
-    if abspath == "LIB_DIR":
-        return Text("LIB_DIR", style="bright_blue")
+    if abspath.startswith("ABXPKG_LIB_DIR/"):
+        return Text.assemble(("ABXPKG_LIB_DIR", "bright_blue"), (abspath.removeprefix("ABXPKG_LIB_DIR"), "green"))
+    if abspath == "ABXPKG_LIB_DIR":
+        return Text("ABXPKG_LIB_DIR", style="bright_blue")
    if abspath.startswith("PERSONAS_DIR/"):
        return Text.assemble(("PERSONAS_DIR", "medium_purple"), (abspath.removeprefix("PERSONAS_DIR"), "green"))
    if abspath == "PERSONAS_DIR":
@@ -78,6 +77,62 @@ def _render_binary_abspath(abspath: str):
    return Text(abspath, style="green")


+def _build_binary_table(rows: list[dict[str, object]]):
+    from rich import box
+    from rich.table import Table
+
+    table = Table(title="Binary Dependencies", box=box.SIMPLE_HEAVY, expand=True)
+    table.add_column("Plugin", no_wrap=True, max_width=24)
+    table.add_column("State", no_wrap=True, width=8)
+    table.add_column("Status", justify="center", no_wrap=True, width=6)
+    table.add_column("Binary", no_wrap=True, max_width=28)
+    table.add_column("Version", no_wrap=True, width=16)
+    table.add_column("Provider", no_wrap=True, width=8)
+    table.add_column("Path", overflow="fold", ratio=1)
+    for row in rows:
+        table.add_row(
+            str(row["plugin"]),
+            str(row["state"]),
+            str(row["status"]),
+            str(row["binary"]),
+            str(row["version"]),
+            str(row["provider"]),
+            row["path"],
+            style=str(row.get("style") or ""),
+        )
+    return table
+
+
+def _print_binary_row(prnt, row: dict[str, object]) -> None:
+    prnt(
+        "",
+        row["status"],
+        str(row["plugin"]).ljust(24),
+        str(row["state"]).ljust(8),
+        str(row["binary"]).ljust(28),
+        str(row["version"]).ljust(16),
+        str(row["provider"]).ljust(8),
+        row["path"],
+        overflow="ignore",
+        crop=False,
+        style=str(row.get("style") or ""),
+    )
+
+
+def _binary_record_matches_runtime(installed, lib_dir: Path) -> bool:
+    if not installed or not installed.is_valid or not installed.version:
+        return False
+    try:
+        abspath = Path(installed.abspath).expanduser().resolve(strict=False)
+        if not abspath.exists():
+            return False
+        if installed.binprovider not in {"", "env", "apt", "brew"}:
+            abspath.relative_to(lib_dir)
+    except (OSError, ValueError):
+        return False
+    return True
+
+
@enforce_types
 def version(
    quiet: bool = False,
@@ -94,13 +149,14 @@ def version(

    from rich.panel import Panel
    from rich.console import Console
+    from rich.live import Live

    from archivebox.config import CONSTANTS
    from archivebox.config.version import get_COMMIT_HASH, get_BUILD_TIME
    from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, RUNNING_AS_UID, RUNNING_AS_GID, IN_DOCKER
    from archivebox.config.paths import get_data_locations, get_code_locations
    from archivebox.misc.logging_util import printable_folder_status
-    from archivebox.config.common import get_config
+    from archivebox.config.common import get_config, normalize_runtime_config

    console = Console()
    prnt = console.print
@@ -133,7 +189,7 @@ def version(
    try:
        DATA_DIR_STAT = CONSTANTS.DATA_DIR.stat()
        prnt(
-            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} PUID={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
+            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} ARCHIVEBOX_USER={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
            f"FS_UID={DATA_DIR_STAT.st_uid}:{DATA_DIR_STAT.st_gid}",
            f"FS_PERMS={config.OUTPUT_PERMISSIONS}",
            f"FS_ATOMIC={config.ENFORCE_ATOMIC_WRITES}",
@@ -141,7 +197,7 @@ def version(
        )
    except Exception:
        prnt(
-            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} PUID={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
+            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} ARCHIVEBOX_USER={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
        )

    prnt(
@@ -154,7 +210,8 @@ def version(
    )
    prnt()

-    if not (os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and os.access(CONSTANTS.CONFIG_FILE, os.R_OK)):
+    in_data_dir = os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and os.access(CONSTANTS.CONFIG_FILE, os.R_OK)
+    if not in_data_dir:
        PANEL_TEXT = "\n".join(
            (
                "",
@@ -175,92 +232,149 @@ def version(
            ),
        )
        prnt()
-        return []

    prnt("[pale_green1][i] Binary Dependencies:[/pale_green1]")
    failures = []

-    # Setup Django before importing models
+    from archivebox.plugins.discovery import get_enabled_plugins
+    from abx_dl.config import get_required_binary_requests
+    from abx_dl.dependencies import load_binary
+    from abx_dl.models import discover_plugins, filter_plugins
+
+    if isinstance(binaries, str):
+        requested_names = {name.strip() for name in binaries.split(",") if name.strip()}
+    else:
+        requested_names = {name for name in (binaries or ()) if name}
+
+    plugins = discover_plugins(runtime="archivebox")
+    enabled_plugins = filter_plugins(plugins, get_enabled_plugins(config=config), include_providers=True)
+    enabled_plugin_names = set(enabled_plugins)
+    runtime_config = normalize_runtime_config(config.for_crawl(), json_safe=False)
+    derived_config: dict[str, object] = {}
+    db_binaries = {}
+    db_available = False
+    if in_data_dir:
+        try:
+            from archivebox.config.django import setup_django
+
+            setup_django()
+
+            from archivebox.machine.models import Machine, Binary
+
+            machine = Machine.current()
+            derived_config = normalize_runtime_config(machine.config, json_safe=False)
+            for binary in Binary.objects.filter(machine=machine).order_by("name", "-modified_at"):
+                if _binary_record_matches_runtime(binary, config.ABXPKG_LIB_DIR):
+                    db_binaries.setdefault(binary.name, binary)
+            db_available = True
+
+        except Exception as e:
+            prnt()
+            prnt("", f"[yellow]Warning: Could not query collection binary records, falling back to abxpkg state: {e}[/yellow]")
+
+    rows: list[dict[str, object]] = []
+    any_rows = False
+    any_available = False
+    compact_paths = console.is_terminal
+    live_enabled = console.is_terminal
+    live_cm = Live(_build_binary_table(rows), console=console, refresh_per_second=8) if live_enabled else None
+    if not live_enabled:
+        prnt("", "Status", "Plugin".ljust(24), "State".ljust(8), "Binary".ljust(28), "Version".ljust(16), "Provider".ljust(8), "Path")
+
+    def emit_row(row: dict[str, object]) -> None:
+        rows.append(row)
+        if live_cm is not None:
+            live_cm.update(_build_binary_table(rows), refresh=True)
+        else:
+            _print_binary_row(prnt, row)
+
+    if live_cm is not None:
+        live_cm.start()
    try:
-        from archivebox.config.django import setup_django
-
-        setup_django()
-
-        from archivebox.machine.models import Machine, Binary
-
-        machine = Machine.current()
-
-        if isinstance(binaries, str):
-            requested_names = {name.strip() for name in binaries.split(",") if name.strip()}
-        else:
-            requested_names = {name for name in (binaries or ()) if name}
-
-        db_binaries: dict[str, Binary] = {}
-        for binary in Binary.objects.filter(machine=machine).order_by("name", "-modified_at"):
-            db_binaries.setdefault(binary.name, binary)
-
-        all_binary_names = sorted(requested_names or set(db_binaries.keys()))
-
-        if not all_binary_names:
-            prnt("", "[grey53]No binaries detected. Run [green]archivebox install[/green] to detect dependencies.[/grey53]")
-        else:
-            any_available = False
-            compact_paths = console.is_terminal
-            for name in all_binary_names:
-                if requested_names and name not in requested_names:
+        for plugin_name, plugin in plugins.items():
+            plugin_enabled = plugin_name in enabled_plugin_names
+            logical_records = get_required_binary_requests(
+                plugin,
+                plugin.config.required_binaries,
+                overrides=runtime_config,
+                derived_overrides=derived_config,
+                run_output_dir=CONSTANTS.DATA_DIR,
+            )
+            actual_records = get_required_binary_requests(
+                plugin,
+                plugin.config.required_binaries,
+                overrides=runtime_config,
+                derived_overrides=derived_config,
+                run_output_dir=CONSTANTS.DATA_DIR,
+                logical_names=False,
+            )
+            for logical_record, actual_record in zip(logical_records, actual_records, strict=False):
+                logical_name = str(logical_record["name"])
+                actual_name = str(actual_record["name"])
+                display_name = Path(actual_name).expanduser().name if ("/" in actual_name or actual_name.startswith("~")) else logical_name
+                if (
+                    requested_names
+                    and logical_name not in requested_names
+                    and actual_name not in requested_names
+                    and display_name not in requested_names
+                ):
                    continue

-                installed = db_binaries.get(name)
-                if installed and installed.is_valid:
-                    display_name = Path(name).expanduser().name if ("/" in name or name.startswith("~")) else name
+                any_rows = True
+                installed = db_binaries.get(logical_name) if db_available else None
+                if _binary_record_matches_runtime(installed, config.ABXPKG_LIB_DIR):
+                    abspath = installed.abspath
+                    version_str = (installed.version or "unknown")[:15]
+                    provider = (installed.binprovider or "env")[:8]
+                    valid = True
+                else:
+                    loaded = load_binary(actual_record)
+                    abspath = str(loaded.loaded_abspath or "")
+                    version_str = str(loaded.loaded_version or "unknown")[:15]
+                    provider = (loaded.loaded_binprovider.name if loaded.loaded_binprovider else "env")[:8]
+                    valid = loaded.is_valid
+
+                if valid:
                    display_path = (
                        _format_binary_abspath(
-                            installed.abspath,
+                            abspath,
                            pwd=Path.cwd(),
-                            lib_dir=config.LIB_DIR,
+                            lib_dir=config.ABXPKG_LIB_DIR,
                            personas_dir=CONSTANTS.PERSONAS_DIR,
                            home=Path.home(),
                        )
                        if compact_paths
-                        else installed.abspath
+                        else abspath
                    )
                    rendered_path = _render_binary_abspath(display_path) if compact_paths else display_path
-                    version_str = (installed.version or "unknown")[:15]
-                    provider = (installed.binprovider or "env")[:8]
-                    prnt(
-                        "",
-                        "[green]√[/green]",
-                        "",
-                        display_name.ljust(18),
-                        version_str.ljust(16),
-                        provider.ljust(8),
-                        rendered_path,
-                        overflow="ignore",
-                        crop=False,
-                    )
+                    status = "[green]√[/green]" if plugin_enabled else "[grey53]-[/grey53]"
                    any_available = True
-                    continue
+                else:
+                    rendered_path = "[grey53]not installed[/grey53]"
+                    status = "[red]X[/red]" if plugin_enabled else "[grey53]-[/grey53]"
+                    if plugin_enabled:
+                        failures.append(display_name)

-                status = (
-                    "[grey53]not recorded[/grey53]" if name in requested_names and installed is None else "[grey53]not installed[/grey53]"
+                emit_row(
+                    {
+                        "plugin": plugin_name,
+                        "state": "enabled" if plugin_enabled else "disabled",
+                        "status": status,
+                        "binary": display_name,
+                        "version": version_str if valid else "-",
+                        "provider": provider if valid else "-",
+                        "path": rendered_path,
+                        "style": "" if plugin_enabled else "dim",
+                    },
                )
-                prnt("", "[red]X[/red]", "", name.ljust(18), status, overflow="ignore", crop=False)
-                failures.append(name)
+    finally:
+        if live_cm is not None:
+            live_cm.stop()

-            if not any_available:
-                prnt("", "[grey53]No binaries detected. Run [green]archivebox install[/green] to detect dependencies.[/grey53]")
-
-        # Show hint if no binaries are installed yet
-        has_any_installed = Binary.objects.filter(machine=machine).exclude(abspath="").exists()
-        if not has_any_installed:
-            prnt()
-            prnt("", "[grey53]Run [green]archivebox install[/green] to detect and install dependencies.[/grey53]")
-
-    except Exception as e:
-        # Handle database errors gracefully (locked, missing, etc.)
-        prnt()
-        prnt("", f"[yellow]Warning: Could not query binaries from database: {e}[/yellow]")
-        prnt("", "[grey53]Run [green]archivebox init[/green] and [green]archivebox install[/green] to set up dependencies.[/grey53]")
+    if not any_rows:
+        prnt("", "[grey53]No required binaries declared for discovered plugins.[/grey53]")
+    elif not any_available:
+        prnt("", "[grey53]No binaries detected. Run [green]archivebox install[/green] to detect dependencies.[/grey53]")

    if not binaries:
        # Show code and data locations
@@ -176,13 +176,31 @@ def build_crawl_config_snapshot(
    plugin_owned_keys = set(_plugin_config_properties(PLUGIN_CONFIG_SCHEMAS)) - set(ArchiveBoxBaseConfig.model_fields)
    effective = get_config(persona=persona, base_config=base_config)
    frozen = effective.for_crawl_frozen(persona=persona)
+    for key in ("BIND_ADDR", "BASE_URL", "CSRF_TRUSTED_ORIGINS", "SERVER_SECURITY_MODE"):
+        value = getattr(effective, key, None)
+        if value is not None:
+            frozen[key] = value
+    if persona is not None:
+        persona_config = persona.get_derived_config()
+        for key in plugin_owned_keys - explicit_overrides:
+            if key in persona_config:
+                frozen.pop(key, None)
    if overrides:
        resolved = get_config(base_config=frozen, overrides=overrides, include_machine=False)
        resolved_payload = normalize_runtime_config(resolved)
        frozen = resolved.for_crawl_frozen(persona=persona)
+        for key in ("BIND_ADDR", "BASE_URL", "CSRF_TRUSTED_ORIGINS", "SERVER_SECURITY_MODE"):
+            value = getattr(resolved, key, None)
+            if value is not None:
+                frozen[key] = value
        for key in plugin_owned_keys & explicit_overrides:
            if ArchiveBoxConfig.scope_for_key(key) == _SCOPE_CRAWL_FROZEN and key in resolved_payload:
                frozen[key] = resolved_payload[key]
+        if persona is not None:
+            persona_config = persona.get_derived_config()
+            for key in plugin_owned_keys - explicit_overrides:
+                if key in persona_config:
+                    frozen.pop(key, None)
    return frozen


@@ -228,18 +246,14 @@ class StorageConfig(BaseConfigSet):
    _scope: str = PrivateAttr(default=_SCOPE_SERVER)

    # TMP_DIR must be a local, fast, readable/writable dir by archivebox user,
-    # must be a short path due to unix path length restrictions for socket files (<100 chars)
+    # must be a short path due to unix path length restrictions for socket files (<90 chars)
    # must be a local SSD/tmpfs for speed and because bind mounts/network mounts/FUSE dont support unix sockets
    TMP_DIR: Path = Field(default=CONSTANTS.DEFAULT_TMP_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})

-    # LIB_DIR must be a local, fast, readable/writable dir by archivebox user,
+    # ABXPKG_LIB_DIR must be a local, fast, readable/writable dir by archivebox user,
    # must be able to contain executable binaries (up to 5GB size)
    # should not be a remote/network/FUSE mount for speed reasons, otherwise extractors will be slow
-    LIB_DIR: Path = Field(default=CONSTANTS.DEFAULT_LIB_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
-
-    # LIB_BIN_DIR is an optional human-facing symlink convenience directory.
-    # Runtime lookup must use provider-specific paths under LIB_DIR instead.
-    LIB_BIN_DIR: Path = Field(default=CONSTANTS.DEFAULT_LIB_BIN_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
+    ABXPKG_LIB_DIR: Path = Field(default=CONSTANTS.DEFAULT_ABXPKG_LIB_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})

    OUTPUT_PERMISSIONS: str = Field(default="644")
    ENFORCE_ATOMIC_WRITES: bool = Field(default=True)
@@ -478,7 +492,7 @@ class SearchBackendConfig(BaseConfigSet):
    toml_section_header: str = "SEARCH_BACKEND_CONFIG"
    _scope: str = PrivateAttr(default=_SCOPE_SERVER)

-    SEARCH_BACKEND_ENGINE: str = Field(default="ripgrep")
+    SEARCH_BACKEND_ENGINE: str = Field(default="ripgrep", json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})


 def _plugin_user_config_value(value: Any) -> str:
@@ -522,6 +536,10 @@ def _archivebox_config_input_names() -> set[str]:
    for field in ArchiveBoxConfig.model_fields.values():
        if isinstance(field.alias, str):
            names.add(field.alias)
+    for prop in _plugin_config_properties(PLUGIN_CONFIG_SCHEMAS).values():
+        aliases = prop.get("x-aliases") if isinstance(prop, Mapping) else None
+        if isinstance(aliases, list):
+            names.update(str(alias) for alias in aliases if str(alias).strip())
    return names


@@ -598,18 +616,13 @@ class ArchiveBoxBaseConfig(
            if isinstance(prop_schema, Mapping) and prop_schema.get("x-scope"):
                scope = str(prop_schema["x-scope"])
            elif scope is None:
-                if str(plugin_name).startswith("search_backend_"):
-                    scope = _SCOPE_SERVER
-                else:
-                    scope = _SCOPE_CRAWL_FROZEN
+                scope = _SCOPE_CRAWL_FROZEN
        return scope

    @classmethod
    @lru_cache(maxsize=None)
    def scope_for_key(cls, key: str) -> str:
        for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items():
-            if str(plugin_name).startswith("search_backend_"):
-                continue
            properties = schema.get("properties") if isinstance(schema, dict) else None
            if isinstance(properties, dict) and key == f"{str(plugin_name).upper()}_ENABLED" and key in properties:
                return _SCOPE_CRAWL_EXECUTION
@@ -638,10 +651,11 @@ class ArchiveBoxBaseConfig(
        runtime_derived_keys = {
            "ABX_INSTALL_CACHE",
            "ACTIVE_PERSONA",
-            "CHROME_DOWNLOADS_DIR",
-            "CHROME_USER_DATA_DIR",
+            "CRAWL_DIR",
            "DEFAULT_PERSONA",
            "EXTRA_CONTEXT",
+            "PERSONAS_DIR",
+            "SNAP_DIR",
        }
        return frozenset(
            key for key, scope in cls._scope_by_key().items() if scope == _SCOPE_CRAWL_EXECUTION and key in runtime_derived_keys
@@ -654,7 +668,10 @@ class ArchiveBoxBaseConfig(

    def for_crawl(self) -> dict[str, Any]:
        """Config scoped to crawl execution, without runtime object overlays."""
-        return self._scoped_config(include_execution=True)
+        config = self._scoped_config(include_execution=True)
+        for key in type(self).runtime_derived_config_keys():
+            config.pop(key, None)
+        return config

    def for_crawl_frozen(self, *, persona: Any = None) -> dict[str, Any]:
        """Config safe to persist permanently on Crawl.config."""
@@ -680,10 +697,13 @@ class ArchiveBoxBaseConfig(
    ) -> dict[str, Any]:
        """Config payload safe to pass to crawl/snapshot hook execution."""
        config = self.for_crawl()
+        config["DATA_DIR"] = str(CONSTANTS.DATA_DIR)
        scope_by_key = type(self)._scope_by_key()
        model_fields = type(self).model_fields
-        for key in type(self).runtime_derived_config_keys():
-            config.pop(key, None)
+        # ArchiveBox-only selectors are consumed during model validation to
+        # derive hook-owned *_ENABLED flags. They are stripped below after
+        # crawl/snapshot overlays have had a chance to apply.
+        config.pop("SEARCH_BACKEND_ENGINE", None)
        if persona is not None:
            for key, value in persona.get_derived_config().items():
                if scope_by_key.get(key) == _SCOPE_CRAWL_EXECUTION:
@@ -704,6 +724,11 @@ class ArchiveBoxBaseConfig(
        if runtime_overrides:
            config.update(normalize_runtime_config(runtime_overrides, json_safe=False))

+        # Hooks should only see concrete plugin-local flags, never the
+        # ArchiveBox selectors used to derive them.
+        config.pop("PLUGINS", None)
+        config.pop("SEARCH_BACKEND_ENGINE", None)
+
        if extra_context:
            context: dict[str, Any] = {}
            if config.get("EXTRA_CONTEXT"):
@@ -714,23 +739,27 @@ class ArchiveBoxBaseConfig(
            context.update(dict(extra_context))
            config["EXTRA_CONTEXT"] = json.dumps(context, separators=(",", ":"), sort_keys=True)

-        _derive_plugin_enabled_config(config)
        return config

    @model_validator(mode="after")
    def resolve_runtime_paths(self):
-        lib_dir = self.LIB_DIR.expanduser()
+        lib_dir = self.ABXPKG_LIB_DIR.expanduser()
        if not lib_dir.is_absolute():
            lib_dir = CONSTANTS.DATA_DIR / lib_dir
-        self.LIB_DIR = lib_dir.resolve()
+        self.ABXPKG_LIB_DIR = lib_dir.resolve()

-        lib_bin_dir = self.LIB_BIN_DIR.expanduser()
-        if lib_bin_dir == CONSTANTS.DEFAULT_LIB_BIN_DIR and self.LIB_DIR != CONSTANTS.DEFAULT_LIB_DIR:
-            lib_bin_dir = self.LIB_DIR / "bin"
-        elif not lib_bin_dir.is_absolute():
-            lib_bin_dir = CONSTANTS.DATA_DIR / lib_bin_dir
-        self.LIB_BIN_DIR = lib_bin_dir.resolve()
+        return self

+    @model_validator(mode="after")
+    def derive_plugin_enabled_config(self):
+        plugin_names = _normalize_plugins_config_value(self.PLUGINS)
+        selected_plugins = _plugins_with_required_plugins(plugin_names) if plugin_names else set()
+        search_backend = self.SEARCH_BACKEND_ENGINE.strip().lower()
+        if search_backend:
+            selected_plugins.add(f"search_backend_{search_backend}")
+        for plugin_name, enabled_key in _plugin_enabled_config_keys().items():
+            if plugin_names or plugin_name in selected_plugins:
+                setattr(self, enabled_key, plugin_name in selected_plugins)
        return self


@@ -805,15 +834,6 @@ def _plugins_with_required_plugins(plugin_names: set[str]) -> set[str]:
    return selected


-def _derive_plugin_enabled_config(config: dict[str, Any]) -> None:
-    plugin_names = _normalize_plugins_config_value(config.get("PLUGINS"))
-    if not plugin_names:
-        return
-    selected_plugins = _plugins_with_required_plugins(plugin_names)
-    for plugin_name, enabled_key in _plugin_enabled_config_keys().items():
-        config[enabled_key] = plugin_name in selected_plugins
-
-
 def get_live_config_url(key: str) -> str:
    return f"{LIVE_CONFIG_BASE_URL}{quote(key)}/"

@@ -911,14 +931,11 @@ def find_config_type(key: str) -> str:

 def find_config_source(key: str, merged_config: Mapping[str, Any]) -> str:
    """Determine where a config value comes from."""
-    try:
-        from archivebox.machine.models import Machine
+    from archivebox.machine.models import Machine

-        machine = Machine.current()
-        if machine.config and key in machine.config:
-            return "Machine"
-    except Exception:
-        pass
+    machine = Machine.current()
+    if machine.config and key in machine.config:
+        return "Machine"

    if key in os.environ:
        return "Environment"
@@ -988,6 +1005,7 @@ def get_config(
        persona = crawl.resolve_persona()

    config_data: ConfigPayload = dict(defaults or {})
+    config_data["PERSONAS_DIR"] = str(CONSTANTS.PERSONAS_DIR)
    base_config_payload: ConfigPayload = {}
    if crawl_config_base:
        config_data.update(
@@ -1015,7 +1033,7 @@ def get_config(

        scope_overrides.update(
            normalize_runtime_config(
-                _sanitize_machine_config(machine.config, lib_dir=config_data.get("LIB_DIR")),
+                _sanitize_machine_config(machine.config, lib_dir=config_data.get("ABXPKG_LIB_DIR")),
                only_crawl_execution=crawl_config_base,
                exclude_runtime_derived=True,
                json_safe=False,
@@ -1023,14 +1041,20 @@ def get_config(
        )

    if persona is not None:
-        scope_overrides.update(
-            normalize_runtime_config(
-                persona.get_derived_config(),
-                only_crawl_execution=crawl_config_base,
-                exclude_runtime_derived=not crawl_config_base,
-                json_safe=False,
-            ),
+        persona_config = normalize_runtime_config(
+            persona.get_derived_config(),
+            exclude_runtime_derived=not crawl_config_base,
+            json_safe=False,
        )
+        if crawl_config_base:
+            scope_by_key = ArchiveBoxConfig._scope_by_key()
+            crawl_keys = set(dict(crawl.config or {}))
+            persona_config = {
+                key: value
+                for key, value in persona_config.items()
+                if scope_by_key.get(key) == _SCOPE_CRAWL_EXECUTION or key not in crawl_keys
+            }
+        scope_overrides.update(persona_config)

    if crawl is not None and crawl.config and not crawl_config_base:
        scope_overrides.update(normalize_runtime_config(crawl.config, exclude_crawl_execution=True, json_safe=False))
@@ -1089,8 +1113,6 @@ def get_config(
            config_data.update(normalize_runtime_config(dict(crawl.config or {}), exclude_crawl_execution=True, json_safe=False))
        config_data.update(archivebox_scope_overrides)

-    _derive_plugin_enabled_config(config_data)
-
    # Decode JSON-encoded complex values (dict/list fields) that came from
    # string-only sources before validation. ``IniConfigSettingsSource`` does
    # this for the ArchiveBox.conf path, but Machine.config (mirrored from the
@@ -1120,9 +1142,7 @@ def get_config(
            value = config[key]
            if is_sensitive_config_key(key) and value not in (None, ""):
                setattr(config, key, SENSITIVE_CONFIG_VALUE_REDACTED)
-    os.environ["LIB_DIR"] = str(config.LIB_DIR)
-    os.environ["LIB_BIN_DIR"] = str(config.LIB_BIN_DIR)
-    os.environ["ABXPKG_LIB_DIR"] = str(config.LIB_DIR)
+    os.environ["ABXPKG_LIB_DIR"] = str(config.ABXPKG_LIB_DIR)
    archiving_warning_key = (config.TIMEOUT, config.USE_COLOR)
    if archiving_warning_key not in _WARNED_ARCHIVING_CONFIGS:
        config.warn_if_invalid()
@@ -36,8 +36,8 @@ from .permissions import (
    IN_DOCKER,
    RUNNING_AS_UID,
    RUNNING_AS_GID,
-    DEFAULT_PUID,
-    DEFAULT_PGID,
+    DEFAULT_UID,
+    DEFAULT_GID,
    ARCHIVEBOX_USER,
    ARCHIVEBOX_GROUP,
 )
@@ -66,8 +66,8 @@ class ConstantsDict:
    ARCHIVEBOX_GROUP: int = ARCHIVEBOX_GROUP
    RUNNING_AS_UID: int = RUNNING_AS_UID
    RUNNING_AS_GID: int = RUNNING_AS_GID
-    DEFAULT_PUID: int = DEFAULT_PUID
-    DEFAULT_PGID: int = DEFAULT_PGID
+    DEFAULT_UID: int = DEFAULT_UID
+    DEFAULT_GID: int = DEFAULT_GID
    IS_INSIDE_VENV: bool = sys.prefix != sys.base_prefix

    # Source code dirs
@@ -113,12 +113,10 @@ class ConstantsDict:
    TMP_DIR_NAME: str = "tmp"
    DEFAULT_TMP_DIR: Path = DATA_DIR / TMP_DIR_NAME / MACHINE_ID  # ./data/tmp/abc3244323

-    LIB_DIR_NAME: str = "lib"
-    DEFAULT_LIB_DIR: Path = _env_path(
-        "LIB_DIR",
-        user_config_path("abx") / LIB_DIR_NAME,
+    DEFAULT_ABXPKG_LIB_DIR: Path = _env_path(
+        "ABXPKG_LIB_DIR",
+        user_config_path("abx") / "lib",
    )
-    DEFAULT_LIB_BIN_DIR: Path = _env_path("LIB_BIN_DIR", DEFAULT_LIB_DIR / "bin")

    RESERVED_ARCHIVE_DIR_NAMES: frozenset[str] = frozenset(
        (
@@ -241,7 +239,7 @@ class ConstantsDict:
            SOURCES_DIR_NAME,
            LOGS_DIR_NAME,
            CACHE_DIR_NAME,
-            LIB_DIR_NAME,
+            "lib",
            TMP_DIR_NAME,
            PERSONAS_DIR_NAME,
            CUSTOM_TEMPLATES_DIR_NAME,
@@ -41,6 +41,12 @@ def setup_django(check_db=False, in_memory_db=False) -> None:
        # TODO: figure out why CLI entrypoints with init_pending are running this twice sometimes
        return

+    # SQLite creates index.sqlite3 during django.setup()/migrate. Apply the
+    # ArchiveBox file-mode policy before any DB connection can create the file,
+    # otherwise a permissive parent umask can expose a just-created DB until a
+    # later chmod runs.
+    os.umask(0o777 - (int(CONFIG.OUTPUT_PERMISSIONS, base=8) | 0o111))
+
    # Third-party patches are only needed once Django/apps are about to load.
    # Keeping them out of archivebox.__init__ avoids paying Django/Daphne setup
    # cost for cheap CLI startup paths like `archivebox <cmd> --help`.
@@ -20,6 +20,8 @@ if TYPE_CHECKING:

 PACKAGE_DIR: Path = Path(__file__).resolve().parent.parent  # archivebox source code dir
 DATA_DIR: Path = Path(os.getcwd()).resolve()  # archivebox user data dir
+MAX_TMP_SOCKET_URL_LENGTH = 90
+SUPERVISORD_SOCKET_FILENAME = "supervisord.sock"


 def _env_path(key: str, default: Path) -> Path:
@@ -120,7 +122,7 @@ def dir_is_writable(dir_path: Path, uid: int | None = None, gid: int | None = No
    try:
        with SudoPermission(uid=uid, fallback=fallback):
            test_file.exists()
-            test_file.write_text(f"Checking if PUID={uid} PGID={gid} can write to dir")
+            test_file.write_text(f"Checking if uid={uid} gid={gid} can write to dir")
            test_file.unlink()
            return True
    except (OSError, PermissionError):
@@ -176,33 +178,45 @@ def create_and_chown_dir(dir_path: Path) -> None:


 def tmp_dir_socket_path_is_short_enough(dir_path: Path) -> bool:
-    socket_file = dir_path.absolute().resolve() / "supervisord.sock"
-    return len(f"file://{socket_file}") <= 96
+    socket_file = dir_path.absolute().resolve() / SUPERVISORD_SOCKET_FILENAME
+    return len(f"file://{socket_file}") < MAX_TMP_SOCKET_URL_LENGTH
+
+
+def tmp_dir_candidates(config: "ArchiveBoxConfig") -> list[Path]:
+    from archivebox.config.constants import CONSTANTS
+
+    collection_id = get_collection_id()
+    collection_id_short = collection_id[:4]
+    system_tmp_dir = Path(tempfile.gettempdir())
+    candidates = [
+        config.TMP_DIR,  # <user-specified>
+        CONSTANTS.DEFAULT_TMP_DIR,  # ./data/tmp/<machine_id>
+        Path("/var/run/archivebox") / collection_id,
+        Path("/tmp") / "archivebox" / collection_id,
+        Path("~/.tmp/archivebox").expanduser() / collection_id,
+        system_tmp_dir / "archivebox" / collection_id,
+        system_tmp_dir / "archivebox" / collection_id_short,
+        system_tmp_dir / "abx" / collection_id_short,
+    ]
+    seen = set()
+    unique_candidates = []
+    for path in candidates:
+        path_key = str(path.expanduser().absolute())
+        if path_key in seen:
+            continue
+        seen.add(path_key)
+        unique_candidates.append(path)
+    return unique_candidates


 def get_or_create_working_tmp_dir(autofix=True, quiet=True, config: "ArchiveBoxConfig | None" = None, **config_kwargs):
-    from archivebox.config.constants import CONSTANTS
    from archivebox.config.common import get_config
    from archivebox.misc.checks import check_tmp_dir

    config = config or get_config(**config_kwargs)
-    # try a few potential directories in order of preference
-    CANDIDATES = [
-        config.TMP_DIR,  # <user-specified>
-        CONSTANTS.DEFAULT_TMP_DIR,  # ./data/tmp/<machine_id>
-        Path("/var/run/archivebox") / get_collection_id(),  # /var/run/archivebox/abc5d8512
-        Path("/tmp") / "archivebox" / get_collection_id(),  # /tmp/archivebox/abc5d8512
-        Path("~/.tmp/archivebox").expanduser() / get_collection_id(),  # ~/.tmp/archivebox/abc5d8512
-        Path(tempfile.gettempdir())
-        / "archivebox"
-        / get_collection_id(),  # /var/folders/qy/6tpfrpx100j1t4l312nz683m0000gn/T/archivebox/abc5d8512
-        Path(tempfile.gettempdir())
-        / "archivebox"
-        / get_collection_id()[:4],  # /var/folders/qy/6tpfrpx100j1t4l312nz683m0000gn/T/archivebox/abc5d
-        Path(tempfile.gettempdir()) / "abx" / get_collection_id()[:4],  # /var/folders/qy/6tpfrpx100j1t4l312nz683m0000gn/T/abx/abc5
-    ]
+    candidates = tmp_dir_candidates(config)
    fallback_candidate = None
-    for candidate in CANDIDATES:
+    for candidate in candidates:
        try:
            create_and_chown_dir(candidate)
        except Exception:
@@ -231,7 +245,7 @@ def get_or_create_working_tmp_dir(autofix=True, quiet=True, config: "ArchiveBoxC
        return fallback_candidate

    if not quiet:
-        raise OSError(f"ArchiveBox is unable to find a writable TMP_DIR, tried {CANDIDATES}!")
+        raise OSError(f"ArchiveBox is unable to find a writable TMP_DIR, tried {candidates}!")


 def get_or_create_working_lib_dir(autofix=True, quiet=False, config: "ArchiveBoxConfig | None" = None, **config_kwargs):
@@ -240,8 +254,8 @@ def get_or_create_working_lib_dir(autofix=True, quiet=False, config: "ArchiveBox

    config = config or get_config(**config_kwargs)

-    # LIB_DIR is either the shared platformdirs default or an explicit env/config override.
-    CANDIDATES = [config.LIB_DIR]
+    # ABXPKG_LIB_DIR is either the shared platformdirs default or an explicit env/config override.
+    CANDIDATES = [config.ABXPKG_LIB_DIR]

    for candidate in CANDIDATES:
        try:
@@ -249,12 +263,12 @@ def get_or_create_working_lib_dir(autofix=True, quiet=False, config: "ArchiveBox
        except Exception:
            pass
        if check_lib_dir(candidate, throw=False, quiet=True, must_exist=True):
-            if autofix and config.LIB_DIR != candidate:
-                os.environ["LIB_DIR"] = str(candidate)
+            if autofix and config.ABXPKG_LIB_DIR != candidate:
+                os.environ["ABXPKG_LIB_DIR"] = str(candidate)
            return candidate

    if not quiet:
-        raise OSError(f"ArchiveBox is unable to find a writable LIB_DIR, tried {CANDIDATES}!")
+        raise OSError(f"ArchiveBox is unable to find a writable ABXPKG_LIB_DIR, tried {CANDIDATES}!")


 def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs):
@@ -329,7 +343,10 @@ def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
            "TMP_DIR": {
                "path": tmp_dir.resolve(),
                "enabled": True,
-                "is_valid": os.path.isdir(tmp_dir) and os.access(tmp_dir, os.R_OK) and os.access(tmp_dir, os.W_OK),  # read + write
+                "is_valid": os.path.isdir(tmp_dir)
+                and os.access(tmp_dir, os.R_OK)
+                and os.access(tmp_dir, os.W_OK)
+                and tmp_dir_socket_path_is_short_enough(tmp_dir),
            },
            # "CACHE_DIR": {
            #     "path": CACHE_DIR.resolve(),
@@ -347,11 +364,9 @@ def get_code_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs

    config = config or get_config(**config_kwargs)
    try:
-        lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True, config=config) or config.LIB_DIR
+        lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True, config=config) or config.ABXPKG_LIB_DIR
    except Exception:
-        lib_dir = config.LIB_DIR
-
-    lib_bin_dir = lib_dir / "bin"
+        lib_dir = config.ABXPKG_LIB_DIR

    return AttrDict(
        {
@@ -375,17 +390,10 @@ def get_code_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
                "enabled": os.path.isdir(CONSTANTS.USER_PLUGINS_DIR),
                "is_valid": os.path.isdir(CONSTANTS.USER_PLUGINS_DIR) and os.access(CONSTANTS.USER_PLUGINS_DIR, os.R_OK),  # read
            },
-            "LIB_DIR": {
+            "ABXPKG_LIB_DIR": {
                "path": lib_dir.resolve(),
                "enabled": True,
                "is_valid": os.path.isdir(lib_dir) and os.access(lib_dir, os.R_OK) and os.access(lib_dir, os.W_OK),  # read + write
            },
-            "LIB_BIN_DIR": {
-                "path": lib_bin_dir.resolve(),
-                "enabled": True,
-                "is_valid": os.path.isdir(lib_bin_dir)
-                and os.access(lib_bin_dir, os.R_OK)
-                and os.access(lib_bin_dir, os.W_OK),  # read + write
-            },
        },
    )
@@ -24,8 +24,8 @@ except PermissionError:
    DATA_DIR_UID = 0
    DATA_DIR_GID = 0

-DEFAULT_PUID = 911
-DEFAULT_PGID = 911
+DEFAULT_UID = 911
+DEFAULT_GID = 911
 RUNNING_AS_UID = os.getuid()
 RUNNING_AS_GID = os.getgid()
 EUID = os.geteuid()
@@ -42,23 +42,20 @@ IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE",

 FALLBACK_UID = RUNNING_AS_UID or SUDO_UID
 FALLBACK_GID = RUNNING_AS_GID or SUDO_GID
-if RUNNING_AS_UID == 0:
-    try:
-        # if we are running as root it's really hard to figure out what the correct archivebox user should be
-        # as a last resort instead of setting DATA_DIR ownership to 0:0 (which breaks it for non-root users)
-        # check if 911:911 archivebox user exists on host system, and use it instead of 0
-        if pwd.getpwuid(DEFAULT_PUID).pw_name == "archivebox":
-            FALLBACK_UID = DEFAULT_PUID
-            FALLBACK_GID = DEFAULT_PGID
-    except Exception:
-        pass
+try:
+    ARCHIVEBOX_ACCOUNT = pwd.getpwnam("archivebox")
+except KeyError:
+    ARCHIVEBOX_ACCOUNT = None

-
-os.environ.setdefault("PUID", str(DATA_DIR_UID or EUID or RUNNING_AS_UID or FALLBACK_UID))
-os.environ.setdefault("PGID", str(DATA_DIR_GID or EGID or RUNNING_AS_GID or FALLBACK_GID))
-
-ARCHIVEBOX_USER = int(os.environ["PUID"])
-ARCHIVEBOX_GROUP = int(os.environ["PGID"])
+if DATA_DIR_UID != 0:
+    ARCHIVEBOX_USER = DATA_DIR_UID
+    ARCHIVEBOX_GROUP = DATA_DIR_GID
+elif RUNNING_AS_UID == 0 and ARCHIVEBOX_ACCOUNT is not None:
+    ARCHIVEBOX_USER = ARCHIVEBOX_ACCOUNT.pw_uid
+    ARCHIVEBOX_GROUP = ARCHIVEBOX_ACCOUNT.pw_gid
+else:
+    ARCHIVEBOX_USER = EUID or RUNNING_AS_UID or FALLBACK_UID
+    ARCHIVEBOX_GROUP = EGID or RUNNING_AS_GID or FALLBACK_GID
 if not USER:
    try:
        # alternative method 1 to get username
@@ -94,13 +91,12 @@ except Exception:


 def drop_privileges():
-    """If running as root, drop privileges to the user that owns the data dir (or PUID)"""
+    """If running as root, drop privileges to the data dir owner or archivebox user."""

-    # always run archivebox as the user that owns the data dir, never as root
+    # Always run ArchiveBox as the user that owns the data dir, or as the
+    # archivebox service account when the data dir is root-owned.
    if os.getuid() == 0:
-        # drop permissions to the user that owns the data dir / provided PUID
        if os.geteuid() != ARCHIVEBOX_USER and ARCHIVEBOX_USER != 0 and ARCHIVEBOX_USER_EXISTS:
-            # drop our effective UID to the archivebox user's UID
            os.seteuid(ARCHIVEBOX_USER)

            # update environment variables so that subprocesses dont try to write to /root
@@ -6,6 +6,7 @@ import importlib.metadata
 from pathlib import Path
 from functools import cache
 from datetime import datetime
+import re

 #############################################################################################

@@ -41,11 +42,57 @@ def detect_installed_version(PACKAGE_DIR: Path = PACKAGE_DIR):

@cache
 def get_COMMIT_HASH() -> str | None:
+    for env_var in ("ARCHIVEBOX_COMMIT_HASH", "COMMIT_HASH"):
+        env_commit_hash = os.environ.get(env_var, "").strip()
+        if re.fullmatch(r"[0-9a-fA-F]{40}", env_commit_hash):
+            return env_commit_hash
+
+    if IN_DOCKER:
+        try:
+            version_text = Path("/VERSION.txt").read_text()
+            match = re.search(r"^COMMIT_HASH=([0-9a-fA-F]{40})$", version_text, re.MULTILINE)
+            if match:
+                return match.group(1)
+        except Exception:
+            pass
+
+    def _read_git_file(git_dir: Path, ref: str) -> str | None:
+        try:
+            return git_dir.joinpath(ref).read_text().strip()
+        except Exception:
+            pass
+
+        try:
+            packed_refs = git_dir.joinpath("packed-refs").read_text().splitlines()
+        except Exception:
+            return None
+
+        for line in packed_refs:
+            if line.startswith("#") or line.startswith("^") or not line.strip():
+                continue
+            commit_hash, packed_ref = line.split(" ", 1)
+            if packed_ref == ref:
+                return commit_hash.strip()
+
+        return None
+
    try:
        git_dir = PACKAGE_DIR.parent / ".git"
-        ref = (git_dir / "HEAD").read_text().strip().split(" ")[-1]
-        commit_hash = git_dir.joinpath(ref).read_text().strip()
-        return commit_hash
+        if git_dir.is_file():
+            gitdir_line = git_dir.read_text().strip()
+            gitdir_path = gitdir_line.removeprefix("gitdir:").strip()
+            git_dir = Path(gitdir_path)
+            if not git_dir.is_absolute():
+                git_dir = PACKAGE_DIR.parent / git_dir
+
+        head = (git_dir / "HEAD").read_text().strip()
+        if re.fullmatch(r"[0-9a-fA-F]{40}", head):
+            return head
+
+        ref = head.removeprefix("ref:").strip()
+        commit_hash = _read_git_file(git_dir, ref)
+        if commit_hash:
+            return commit_hash
    except Exception:
        pass

@@ -59,13 +106,10 @@ def get_COMMIT_HASH() -> str | None:

@cache
 def get_BUILD_TIME() -> str:
-    if IN_DOCKER:
-        try:
-            # if we're in the archivebox official docker image, /VERSION.txt will contain the build time
-            docker_build_end_time = Path("/VERSION.txt").read_text().rsplit("BUILD_END_TIME=")[-1].split("\n", 1)[0]
-            return docker_build_end_time
-        except Exception:
-            pass
+    for env_var in ("ARCHIVEBOX_BUILD_TIME", "BUILD_TIME"):
+        build_time = os.environ.get(env_var, "").strip()
+        if build_time:
+            return build_time

    src_last_modified_unix_timestamp = (PACKAGE_DIR / "README.md").stat().st_mtime
    return datetime.fromtimestamp(src_last_modified_unix_timestamp).strftime("%Y-%m-%d %H:%M:%S %s")
@@ -18,7 +18,6 @@ from archivebox.misc.util import parse_date

 from archivebox.machine.models import Binary

-LIVE_CONFIG_BASE_URL = "/admin/environment/config/"
 ENVIRONMENT_BINARIES_BASE_URL = "/admin/environment/binaries/"
 INSTALLED_BINARIES_BASE_URL = "/admin/machine/binary/"

@@ -32,10 +31,6 @@ def format_parsed_datetime(value: object) -> str:
    return parsed.strftime("%Y-%m-%d %H:%M:%S") if parsed else ""


-def get_live_config_url(key: str) -> str:
-    return f"{LIVE_CONFIG_BASE_URL}{quote(key)}/"
-
-
 def get_environment_binary_url(name: str) -> str:
    return f"{ENVIRONMENT_BINARIES_BASE_URL}{quote(name)}/"

@@ -75,6 +75,12 @@ def get_plugin_admin_url(plugin_name: str) -> str:
    return f"{LIVE_PLUGIN_BASE_URL}builtin.{quote(plugin_name)}/"


+def get_process_link_label(process) -> str:
+    if process.pid:
+        return str(process.pid)
+    return str(process.id)[-8:]
+
+
 def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
    """Render a nice inline list view of archive results with status, plugin, output, and actions."""

@@ -136,25 +142,31 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
        process = result.process_record
        process_display = "-"
        if process:
+            process_url = html.escape(reverse("admin:machine_process_change", args=[process.id]), quote=True)
+            process_label = html.escape(get_process_link_label(process), quote=True)
            process_display = f'''
-                <a href="{reverse("admin:machine_process_change", args=[process.id])}"
+                <a href="{process_url}"
                   style="color: #2563eb; text-decoration: none; font-family: ui-monospace, monospace; font-size: 12px;"
-                   title="View process">{process.pid or "-"}</a>
+                   title="View process">{process_label}</a>
            '''

        machine_display = "-"
        if process and process.machine_id:
+            machine_url = html.escape(reverse("admin:machine_machine_change", args=[process.machine_id]), quote=True)
+            machine_label = html.escape(str(process.machine.hostname or ""), quote=True)
            machine_display = f'''
-                <a href="{reverse("admin:machine_machine_change", args=[process.machine_id])}"
+                <a href="{machine_url}"
                   style="color: #2563eb; text-decoration: none; font-size: 12px;"
-                   title="View machine">{process.machine.hostname}</a>
+                   title="View machine">{machine_label}</a>
            '''

        # Truncate output for display
-        full_output = result.output_str_for_display() or "-"
-        output_display = full_output[:60]
-        if len(full_output) > 60:
-            output_display += "..."
+        full_output_raw = result.output_str_for_display() or "-"
+        output_display_raw = full_output_raw[:60]
+        if len(full_output_raw) > 60:
+            output_display_raw += "..."
+        full_output = html.escape(full_output_raw)
+        output_display = html.escape(output_display_raw)

        display_cmd = build_abx_dl_display_command(result)
        replay_cmd = build_abx_dl_replay_command(result, config=config)
@@ -168,9 +180,13 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
            output_link = build_snapshot_url(snapshot_id, embed_path, config=config)
        else:
            output_link = build_snapshot_url(snapshot_id, "", config=config)
+        output_link_attr = html.escape(output_link, quote=True)

        # Get version - try cmd_version field
-        version = result.cmd_version if result.cmd_version else "-"
+        version = html.escape(str(result.cmd_version if result.cmd_version else "-"), quote=True)
+        plugin_text = html.escape(str(result.plugin or ""), quote=True)
+        status_text = html.escape(str(status), quote=True)
+        pwd_text = html.escape(str(result.pwd or "-"), quote=True)

        # Unique ID for this row's expandable output
        row_id = f"output_{idx}_{str(result.id)[:8]}"
@@ -187,18 +203,18 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
                <td style="padding: 10px 12px; white-space: nowrap;">
                    <span style="display: inline-block; padding: 3px 10px; border-radius: 12px;
                                 font-size: 11px; font-weight: 600; text-transform: uppercase;
-                                 color: {color}; background: {bg};">{status}</span>
+                                 color: {color}; background: {bg};">{status_text}</span>
                </td>
-                <td style="padding: 10px 12px; white-space: nowrap; font-size: 20px;" title="{result.plugin}">
+                <td style="padding: 10px 12px; white-space: nowrap; font-size: 20px;" title="{plugin_text}">
                    {icon}
                </td>
                <td style="padding: 10px 12px; font-weight: 500; color: #334155;">
-                        <a href="{output_link}" target="_blank"
+                        <a href="{output_link_attr}" target="_blank"
                           style="color: #334155; text-decoration: none;"
                       title="View output fullscreen"
                       onmouseover="this.style.color='#2563eb'; this.style.textDecoration='underline';"
                       onmouseout="this.style.color='#334155'; this.style.textDecoration='none';">
-                        {result.plugin}
+                        {plugin_text}
                    </a>
                </td>
                <td style="padding: 10px 12px; max-width: 280px;">
@@ -225,7 +241,7 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
                </td>
                <td style="padding: 10px 8px; white-space: nowrap;">
                    <div style="display: flex; gap: 4px;">
-                        <a href="{output_link}" target="_blank"
+                        <a href="{output_link_attr}" target="_blank"
                           style="padding: 4px 8px; background: #f1f5f9; border-radius: 4px; color: #475569; text-decoration: none; font-size: 11px;"
                           title="View output">📄</a>
                        <a href="{reverse("admin:core_archiveresult_change", args=[result.id])}"
@@ -244,7 +260,7 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
                            <div style="font-size: 11px; color: #64748b; margin-bottom: 8px;">
                                <span style="margin-right: 16px;"><b>ID:</b> <code>{str(result.id)}</code></span>
                                <span style="margin-right: 16px;"><b>Version:</b> <code>{version}</code></span>
-                                <span style="margin-right: 16px;"><b>PWD:</b> <code>{result.pwd or "-"}</code></span>
+                                <span style="margin-right: 16px;"><b>PWD:</b> <code>{pwd_text}</code></span>
                            </div>
                            <div style="font-size: 11px; color: #64748b; margin-bottom: 8px;">
                                <b>Output:</b>
@@ -517,7 +533,6 @@ class ArchiveResultAdmin(BaseModelAdmin):
            super()
            .get_queryset(request)
            .defer(
-                "config",
                "notes",
                "output_json",
            )
@@ -660,7 +675,7 @@ class ArchiveResultAdmin(BaseModelAdmin):
        process = result.process_record
        if not process:
            return "-"
-        process_label = process.pid or "-"
+        process_label = get_process_link_label(process)
        return format_html(
            '<a href="{}"><code>{}</code></a>',
            reverse("admin:machine_process_change", args=[process.id]),
@@ -674,9 +689,8 @@ class ArchiveResultAdmin(BaseModelAdmin):
            return "-"
        machine = process.machine
        return format_html(
-            '<a href="{}"><code>{}</code> {}</a>',
+            '<a href="{}">{}</a>',
            reverse("admin:machine_machine_change", args=[machine.id]),
-            str(machine.id)[:8],
            machine.hostname,
        )

@@ -3,7 +3,15 @@ __package__ = "archivebox.core"
 from typing import TYPE_CHECKING, Any

 from django.contrib import admin
+from django.contrib.auth import REDIRECT_FIELD_NAME
+from django.contrib.auth.decorators import login_not_required
+from django.contrib.auth.views import LoginView
 from django.db import DatabaseError, connection
+from django.http import HttpResponseRedirect
+from django.urls import reverse
+from django.utils.decorators import method_decorator
+from django.utils.translation import gettext as _
+from django.views.decorators.cache import never_cache
 from admin_data_views.admin import (
    admin_data_index_view as adv_admin_data_index_view,
    get_admin_data_urls as adv_get_admin_data_urls,
@@ -12,6 +20,7 @@ from admin_data_views.admin import (

 from archivebox.config import VERSION
 from archivebox.config.version import get_COMMIT_HASH
+from archivebox.core.routes_util import is_allowed_archivebox_redirect_url

 if TYPE_CHECKING:
    from django.http import HttpRequest
@@ -21,6 +30,17 @@ if TYPE_CHECKING:
    from admin_data_views.typing import AppDict


+class ArchiveBoxLoginView(LoginView):
+    def get_redirect_url(self) -> str:
+        redirect_to = self.request.POST.get(
+            self.redirect_field_name,
+            self.request.GET.get(self.redirect_field_name),
+        )
+        if is_allowed_archivebox_redirect_url(redirect_to, request=self.request):
+            return redirect_to
+        return ""
+
+
 class ArchiveBoxAdmin(admin.AdminSite):
    site_header = "ArchiveBox"
    index_title = "Admin Views"
@@ -72,6 +92,34 @@ class ArchiveBoxAdmin(admin.AdminSite):
    def admin_data_index_view(self, request: "HttpRequest", **kwargs: Any) -> "TemplateResponse":
        return adv_admin_data_index_view(self, request, **kwargs)

+    @method_decorator(never_cache)
+    @login_not_required
+    def login(self, request: "HttpRequest", extra_context: dict[str, Any] | None = None) -> "TemplateResponse":
+        if request.method == "GET" and self.has_permission(request):
+            return HttpResponseRedirect(reverse("admin:index", current_app=self.name))
+
+        from django.contrib.admin.forms import AdminAuthenticationForm
+
+        context = {
+            **self.each_context(request),
+            "title": _("Log in"),
+            "subtitle": None,
+            "app_path": request.get_full_path(),
+            "username": request.user.get_username(),
+        }
+        if REDIRECT_FIELD_NAME not in request.GET and REDIRECT_FIELD_NAME not in request.POST:
+            context[REDIRECT_FIELD_NAME] = reverse("admin:index", current_app=self.name)
+        context.update(extra_context or {})
+
+        index_path = reverse("admin:index", current_app=self.name)
+        request.current_app = self.name
+        return ArchiveBoxLoginView.as_view(
+            extra_context=context,
+            authentication_form=self.login_form or AdminAuthenticationForm,
+            template_name=self.login_template or "admin/login.html",
+            next_page=index_path,
+        )(request)
+
    def index(self, request: "HttpRequest", extra_context: dict[str, Any] | None = None) -> "TemplateResponse":
        response = super().index(request, extra_context)
        if connection.vendor != "sqlite":
@@ -30,7 +30,9 @@ from archivebox.plugins.discovery import get_plugin_icon, get_plugin_name, get_p
 from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin

 from archivebox.core.models import Tag, Snapshot, ArchiveResult
+from archivebox.crawls.models import Crawl
 from archivebox.core.admin_archiveresults import render_archiveresults_list
+from archivebox.core.preview_util import EXTENSION_SCREENSHOT_PLUGIN
 from archivebox.progressmonitor.views import progress_endpoint
 from archivebox.core.permissions import (
    PERMISSIONS_CHOICES,
@@ -107,9 +109,7 @@ class SnapshotPermissionsListFilter(admin.SimpleListFilter):
    def queryset(self, request, queryset):
        value = self.value()
        if value:
-            return queryset.filter(
-                Q(permissions=value) | (Q(permissions__isnull=True) & Q(crawl__permissions=value)),
-            )
+            return queryset.filter(permissions=value)
        return queryset


@@ -229,8 +229,6 @@ class SnapshotResultHealthListFilter(admin.SimpleListFilter):
                        total_results=ArchiveResult.snapshot_count_expr(),
                        matching_results=ArchiveResult.snapshot_count_expr(status=status),
                    ).filter(matching_results__gt=F("total_results") / 2)
-                    queryset._archivebox_count_hint = "model_estimate"
-                    queryset.query._archivebox_count_hint = queryset._archivebox_count_hint
                    return queryset

                # Rare statuses are faster status-first: use the
@@ -277,13 +275,7 @@ class SnapshotChangeList(SearchResultsChangeList):
        super().get_results(request)
        if request.GET.get("_embedded") == "crawl":
            self.full_result_count = self.result_count
-        else:
-            self.full_result_count = self.model_admin.get_paginator(
-                request,
-                self.model._default_manager.all().order_by(),
-                self.list_per_page,
-            ).count
-        self.show_full_result_count = True
+            self.show_full_result_count = True
        self._attach_archiveresult_summaries()


@@ -320,10 +312,7 @@ class SnapshotAdminForm(forms.ModelForm):
        instance = super().save(commit=False)
        permissions = self.cleaned_data["permissions_config"]
        config = dict(instance.config or {})
-        if permissions == instance.crawl.permissions:
-            config.pop("PERMISSIONS", None)
-        else:
-            config["PERMISSIONS"] = permissions
+        config["PERMISSIONS"] = permissions
        instance.config = config

        # Handle tags_editor field
@@ -465,7 +454,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
        ),
    )

-    ordering = ["-timestamp"]
+    ordering = ["-created_at"]
    actions = [
        "add_tags",
        "remove_tags",
@@ -482,7 +471,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
    paginator = AcceleratedPaginator

    save_on_top = True
-    show_full_result_count = False
+    show_full_result_count = True

    def get_changelist(self, request, **kwargs):
        return SnapshotChangeList
@@ -578,12 +567,9 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
        if permissions not in dict(PERMISSIONS_CHOICES):
            return HttpResponseBadRequest("Invalid permissions value")

-        snapshot = get_object_or_404(Snapshot.objects.select_related("crawl"), pk=object_id)
+        snapshot = get_object_or_404(Snapshot, pk=object_id)
        config = dict(snapshot.config or {})
-        if permissions == snapshot.crawl.permissions:
-            config.pop("PERMISSIONS", None)
-        else:
-            config["PERMISSIONS"] = permissions
+        config["PERMISSIONS"] = permissions

        # Keep the quick-edit write to one targeted UPDATE so SQLite only holds
        # the write lock for the permission/config change itself. safe_update()
@@ -606,18 +592,10 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
        now = timezone.now()
        updated = 0
        batch = []
-        snapshots = (
-            queryset.select_related(None)
-            .select_related("crawl")
-            .only("id", "config", "crawl__id", "crawl__permissions")
-            .prefetch_related(None)
-        )
+        snapshots = queryset.select_related(None).only("id", "config").prefetch_related(None)
        for snapshot in snapshots.iterator(chunk_size=500):
            config = dict(snapshot.config or {})
-            if permissions == snapshot.crawl.permissions:
-                config.pop("PERMISSIONS", None)
-            else:
-                config["PERMISSIONS"] = permissions
+            config["PERMISSIONS"] = permissions
            snapshot.config = config
            snapshot.modified_at = now
            batch.append(snapshot)
@@ -665,13 +643,26 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
                        "plugin",
                        "status",
                        "output_size",
+                        "output_files",
+                    ),
+                ),
+            )
+        else:
+            prefetches.append(
+                Prefetch(
+                    "crawl",
+                    queryset=Crawl.objects.select_related("created_by").only(
+                        "id",
+                        "created_by_id",
+                        "created_by__id",
+                        "created_by__username",
                    ),
                ),
            )

-        qs = super().get_queryset(request).select_related("crawl__created_by")
+        qs = super().get_queryset(request)
        if is_change_view:
-            qs = qs.defer("notes")
+            qs = qs.select_related("crawl__created_by").defer("notes")
        else:
            qs = qs.only(
                "id",
@@ -685,13 +676,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
                "fs_version",
                "output_size",
                "permissions",
-                "crawl__id",
-                "crawl__permissions",
-                "crawl__persona_id",
-                "crawl__status",
-                "crawl__created_by_id",
-                "crawl__created_by__id",
-                "crawl__created_by__username",
            )
        qs = qs.prefetch_related(*prefetches)
        if needs_files_sort:
@@ -707,10 +691,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
    def permissions_badge(self, obj):
        permissions = obj.__dict__.get("snapshot_permissions")
        if permissions is None:
-            if obj.permissions:
-                permissions = obj.permissions
-            else:
-                permissions = obj.crawl.permissions
+            permissions = obj.permissions
        permissions = normalize_permissions(permissions)
        icon, label, fg, bg = SNAPSHOT_PERMISSION_META[permissions]
        menu_items = format_html_join(
@@ -959,30 +940,54 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
        config = request.archivebox_config
        results = self._get_prefetched_results(obj)
        if results is not None:
-            has_screenshot = any(r.plugin == "screenshot" for r in results)
-            has_favicon = any(r.plugin == "favicon" for r in results)
+            results = [r for r in results if r.plugin in ("screenshot", EXTENSION_SCREENSHOT_PLUGIN, "favicon")]
        else:
-            available_plugins = set(obj.archiveresult_set.filter(plugin__in=("screenshot", "favicon")).values_list("plugin", flat=True))
-            has_screenshot = "screenshot" in available_plugins
-            has_favicon = "favicon" in available_plugins
+            results = list(
+                obj.archiveresult_set.filter(plugin__in=("screenshot", EXTENSION_SCREENSHOT_PLUGIN, "favicon")).only(
+                    "snapshot_id",
+                    "plugin",
+                    "status",
+                    "output_files",
+                    "output_str",
+                ),
+            )

-        if not has_screenshot and not has_favicon:
+        def result_output_path(result, filename: str) -> str | None:
+            output_files = result.output_files or {}
+            file_info = output_files.get(filename)
+            if not isinstance(file_info, dict) or int(file_info.get("size") or 0) <= 0:
+                return None
+            if file_info.get("root_relative"):
+                return filename
+            return f"{result.plugin}/{filename}"
+
+        def result_urls(plugin: str, filenames: tuple[str, ...]) -> list[str]:
+            urls: list[str] = []
+            for result in results:
+                if result.plugin != plugin or result.status != ArchiveResult.StatusChoices.SUCCEEDED:
+                    continue
+                for filename in filenames:
+                    output_path = result_output_path(result, filename)
+                    if output_path:
+                        urls.append(build_snapshot_url(str(obj.id), output_path, request=request, config=config))
+            return urls
+
+        screenshot_urls = result_urls("screenshot", ("screenshot.png",))
+        extension_screenshot_urls = result_urls(EXTENSION_SCREENSHOT_PLUGIN, ("screenshot-1.png", "screenshot.png"))
+        favicon_urls = result_urls("favicon", ("favicon.ico",))
+
+        if not screenshot_urls and not extension_screenshot_urls and not favicon_urls:
            return None

-        if has_screenshot:
-            img_url = build_snapshot_url(str(obj.id), "screenshot/screenshot.png", request=request, config=config)
-            fallbacks = [
-                build_snapshot_url(str(obj.id), "screenshot.png", request=request, config=config),
-                build_snapshot_url(str(obj.id), "favicon/favicon.ico", request=request, config=config),
-                build_snapshot_url(str(obj.id), "favicon.ico", request=request, config=config),
-            ]
+        all_screenshot_urls = [*screenshot_urls, *extension_screenshot_urls]
+        if all_screenshot_urls:
+            img_url = all_screenshot_urls[0]
+            fallbacks = [*all_screenshot_urls[1:], *favicon_urls]
            img_alt = "Screenshot"
            preview_class = "screenshot"
        else:
-            img_url = build_snapshot_url(str(obj.id), "favicon/favicon.ico", request=request, config=config)
-            fallbacks = [
-                build_snapshot_url(str(obj.id), "favicon.ico", request=request, config=config),
-            ]
+            img_url = favicon_urls[0]
+            fallbacks = favicon_urls[1:]
            img_alt = "Favicon"
            preview_class = "favicon"

@@ -1000,6 +1005,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
            "preview_class": preview_class,
            "onerror_js": onerror_js,
            "fallback_list": fallback_list,
+            "favicon_url": favicon_urls[0] if favicon_urls else "",
+            "favicon_fallback_list": ",".join(favicon_urls[1:]),
        }

    @admin.display(description="", empty_value="")
@@ -1008,10 +1015,10 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
        if not preview:
            return ""

-        request = self.request
-        config = request.archivebox_config
-        favicon_url = build_snapshot_url(str(obj.id), "favicon/favicon.ico", request=request, config=config)
-        fallback_list = ",".join([build_snapshot_url(str(obj.id), "favicon.ico", request=request, config=config)])
+        favicon_url = preview.get("favicon_url") or ""
+        if not favicon_url:
+            return ""
+        fallback_list = preview.get("favicon_fallback_list") or ""
        onerror_js = (
            "this.dataset.fallbacks && this.dataset.fallbacks.length ? "
            "(this.src=this.dataset.fallbacks.split(',').shift(), "
@@ -117,7 +117,7 @@ class TagAdmin(BaseModelAdmin):
        sort = normalize_tag_sort((request.GET.get("sort") or "created_desc").strip())
        created_by = normalize_created_by_filter((request.GET.get("created_by") or "").strip())
        year = normalize_created_year_filter((request.GET.get("year") or "").strip())
-        has_snapshots = normalize_has_snapshots_filter((request.GET.get("has_snapshots") or "yes").strip())
+        has_snapshots = normalize_has_snapshots_filter((request.GET.get("has_snapshots") or "all").strip())
        context = {
            **self.admin_site.each_context(request),
            **(extra_context or {}),
@@ -6,11 +6,10 @@ from decimal import Decimal, InvalidOperation, ROUND_CEILING
 from django import forms

 from archivebox.misc.util import URL_REGEX, find_all_urls, parse_filesize_to_bytes
-from taggit.utils import edit_string_for_tags, parse_tags
 from archivebox.base_models.admin import KeyValueWidget
 from archivebox.crawls.schedule_util import validate_schedule
 from archivebox.config.common import get_config, parse_delete_after
-from archivebox.core.permissions import PERMISSIONS_CHOICES, PERMISSIONS_PUBLIC, filter_personas_by_permissions, is_admin_user
+from archivebox.core.permissions import PERMISSIONS_CHOICES, PERMISSIONS_PUBLIC, filter_personas_by_permissions
 from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget
 from archivebox.plugins.discovery import get_plugins
 from archivebox.plugins.forms import (
@@ -30,22 +29,84 @@ DEPTH_CHOICES = (
 )


+def _split_strip(value: str, delimiter: str) -> list[str]:
+    return [part.strip() for part in value.split(delimiter) if part.strip()]
+
+
+def parse_tag_string(value: str | None) -> list[str]:
+    """Parse the legacy tag-editing format used by the admin tag field."""
+    if not value:
+        return []
+
+    if "," not in value and '"' not in value:
+        return sorted(set(_split_strip(value, " ")))
+
+    tags: list[str] = []
+    buffer: list[str] = []
+    deferred_chunks: list[str] = []
+    saw_unquoted_comma = False
+    in_quote = False
+    chars = iter(value)
+
+    try:
+        while True:
+            char = next(chars)
+            if char == '"':
+                if buffer:
+                    deferred_chunks.append("".join(buffer))
+                    buffer = []
+                in_quote = True
+                char = next(chars)
+                while char != '"':
+                    buffer.append(char)
+                    char = next(chars)
+                tag = "".join(buffer).strip()
+                if tag:
+                    tags.append(tag)
+                buffer = []
+                in_quote = False
+            else:
+                if not saw_unquoted_comma and char == ",":
+                    saw_unquoted_comma = True
+                buffer.append(char)
+    except StopIteration:
+        if buffer:
+            if in_quote and "," in buffer:
+                saw_unquoted_comma = True
+            deferred_chunks.append("".join(buffer))
+
+    delimiter = "," if saw_unquoted_comma else " "
+    for chunk in deferred_chunks:
+        tags.extend(_split_strip(chunk, delimiter))
+
+    return sorted(set(tags))
+
+
+def edit_string_for_tag_names(tags) -> str:
+    names = []
+    for tag in tags:
+        name = tag.name
+        names.append(f'"{name}"' if "," in name or " " in name else name)
+    return ", ".join(sorted(names))
+
+
 class AddLinkForm(PluginConfigFormMixin, forms.Form):
    allow_crawl_execution_config_fields = False

    # Basic fields
    url = forms.CharField(
        label="URLs",
-        strip=True,
+        strip=False,
        widget=forms.Textarea(
            attrs={
                "data-url-regex": URL_REGEX.pattern,
                "placeholder": (
-                    "Enter URLs to archive, as one per line, CSV, JSON, or embedded in text "
-                    "(e.g. markdown, HTML, etc.). Examples:\n"
-                    "https://example.com\n"
-                    "https://news.ycombinator.com,https://news.google.com\n"
-                    "[ArchiveBox](https://github.com/ArchiveBox/ArchiveBox)"
+                    "\n",
+                    "Enter URL(s) to archive. Any format is ok: one per line, CSV, JSON, embedded in text, etc."
+                    "Examples:\n\n"
+                    "https://example.com\n\n"
+                    "https://news.ycombinator.com,https://news.google.com\n\n"
+                    "Or any text-based content [containing URLs](https://github.com/ArchiveBox/ArchiveBox)...",
                ),
            },
        ),
@@ -238,7 +299,10 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):

    def __init__(self, *args, **kwargs):
        self.request = kwargs.pop("request", None)
-        self.can_override_crawl_config = bool(self.request and is_admin_user(self.request))
+        self.can_override_crawl_config = False
+        if self.request is not None:
+            user = self.request.user
+            self.can_override_crawl_config = bool(user.is_authenticated and user.is_active and user.is_superuser)
        super().__init__(*args, **kwargs)

        default_persona = Persona.get_or_create_default()
@@ -314,12 +378,9 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):

    def clean_url(self):
        value = self.cleaned_data.get("url") or ""
-        valid_urls = []
-        for url in find_all_urls(value):
-            valid_urls.append(url)
-        if not valid_urls:
+        if not list(find_all_urls(value)):
            raise forms.ValidationError("Enter at least one valid URL.")
-        return "\n".join(valid_urls)
+        return value

    def clean_url_filters(self):
        from archivebox.crawls.models import Crawl
@@ -425,7 +486,7 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
 class TagWidget(forms.TextInput):
    def format_value(self, value):
        if value is not None and not isinstance(value, str):
-            value = edit_string_for_tags(value)
+            value = edit_string_for_tag_names(value)
        return super().format_value(value)


@@ -434,12 +495,7 @@ class TagField(forms.CharField):

    def clean(self, value):
        value = super().clean(value)
-        try:
-            return parse_tags(value)
-        except ValueError:
-            raise forms.ValidationError(
-                "Please provide a comma-separated list of tags.",
-            )
+        return parse_tag_string(value)

    def has_changed(self, initial, data):
        # Always return False if the field is disabled since self.bound_data
@@ -3,6 +3,7 @@ __package__ = "archivebox.core"
 import ipaddress
 import re
 from pathlib import Path
+from django.conf import settings
 from django.utils import timezone
 from django.contrib.auth.middleware import RemoteUserMiddleware
 from django.contrib.auth.models import AnonymousUser
@@ -24,7 +25,6 @@ from archivebox.core.routes_util import (
    get_base_host,
    get_listen_host,
    get_listen_subdomain,
-    get_public_host,
    get_web_host,
    host_matches,
    is_snapshot_subdomain,
@@ -42,7 +42,7 @@ def _admin_login_hint_cookie_domain(config) -> str | None:
    NOTE: this cookie carries only the single bit "user is logged in on
    admin somewhere"; it MUST NOT be confused with the session cookie,
    which stays admin-host-scoped (see core/settings.py
-    SESSION_COOKIE_DOMAIN comment — admin/public is a security boundary).
+    SESSION_COOKIE_DOMAIN comment — admin/web is a security boundary).

    Returns the hostname portion of ``get_base_host`` (which respects
    ``BASE_URL`` and falls back to the local-bind mapping). Strips the
@@ -85,6 +85,34 @@ def TimezoneMiddleware(get_response):
    return middleware


+def AdminCookieIsolationMiddleware(get_response):
+    def middleware(request):
+        response = get_response(request)
+
+        config = request.__dict__.get("archivebox_config")
+        if config is None:
+            config = get_config(resolve_plugins=False)
+            request.archivebox_config = config
+        if not config.USES_SUBDOMAIN_ROUTING:
+            return response
+
+        request_host = (request.get_host() or "").lower()
+        if host_matches(request_host, get_admin_host(config=config)):
+            return response
+
+        if host_matches(request_host, get_web_host(config=config)):
+            for cookie_name in tuple(response.cookies.keys()):
+                if cookie_name != ADMIN_LOGIN_HINT_COOKIE:
+                    response.cookies.pop(cookie_name, None)
+            return response
+
+        response.cookies.pop(settings.SESSION_COOKIE_NAME, None)
+        response.cookies.pop(settings.CSRF_COOKIE_NAME, None)
+        return response
+
+    return middleware
+
+
 def CacheControlMiddleware(get_response):
    snapshot_path_re = re.compile(r"^/[^/]+/\\d{8}/[^/]+/[0-9a-fA-F-]{8,36}/")
    static_cache_key = (get_COMMIT_HASH() or VERSION or "dev").strip()
@@ -176,7 +204,6 @@ def HostRoutingMiddleware(get_response):
        admin_host = get_admin_host(config=config)
        web_host = get_web_host(config=config)
        api_host = get_api_host(config=config)
-        public_host = get_public_host(config=config)
        listen_host = get_listen_host(config=config)
        subdomain = get_listen_subdomain(request_host, config=config)

@@ -185,25 +212,7 @@ def HostRoutingMiddleware(get_response):
        if request.path.startswith("/static/") or request.path in {"/favicon.ico", "/robots.txt"}:
            return get_response(request)

-        # In subdomain mode with no explicit BASE_URL we can't safely emit
-        # ``admin.``/``web.``/``snap-*.`` redirects: every URL builder uses the
-        # request's own Host (via the request-host fallback in get_base_url),
-        # so prepending ``admin.`` to whatever the client sent produces a
-        # redirect chain of ``admin.admin.admin.<host>``. Pass the request
-        # through; the misconfig banner on the rendered page tells the user
-        # to pin BASE_URL so the redirects can resume.
-        if config.USES_SUBDOMAIN_ROUTING and not config.BASE_URL:
-            return get_response(request)
-
-        if config.USES_SUBDOMAIN_ROUTING and not host_matches(request_host, admin_host):
-            # ``/add`` is admin-only unless ``PUBLIC_ADD_VIEW`` is on. Without
-            # this redirect, hitting it on public.* falls into AddView's
-            # auth check, bounces through ``/accounts/login/?next=/add/`` →
-            # ``/admin/login/?next=/add/``, and Django admin's LoginView
-            # silently drops ``next`` when the user already has an admin
-            # session — dumping the user on the admin homepage instead of
-            # the add form. Routing the request to admin.* directly lets
-            # AddView run on the host where the session lives.
+        if config.USES_SUBDOMAIN_ROUTING and config.BASE_URL and not host_matches(request_host, admin_host):
            add_should_redirect = not config.PUBLIC_ADD_VIEW and (request.path == "/add" or request.path.startswith("/add/"))
            if (
                request.path == "/admin"
@@ -217,6 +226,20 @@ def HostRoutingMiddleware(get_response):
                    target = f"{target}?{request.META['QUERY_STRING']}"
                return redirect(target)

+        if subdomain and is_snapshot_subdomain(subdomain):
+            view = SnapshotHostView.as_view()
+            return view(request, snapshot_id=subdomain, path=request.path.lstrip("/"))
+
+        # In subdomain mode with no explicit BASE_URL we can't safely emit
+        # ``admin.``/``web.``/``snap-*.`` redirects: every URL builder uses the
+        # request's own Host (via the request-host fallback in get_base_url),
+        # so prepending ``admin.`` to whatever the client sent produces a
+        # redirect chain of ``admin.admin.admin.<host>``. Pass the request
+        # through; the misconfig banner on the rendered page tells the user
+        # to pin BASE_URL so the redirects can resume.
+        if config.USES_SUBDOMAIN_ROUTING and not config.BASE_URL:
+            return get_response(request)
+
        if not config.USES_SUBDOMAIN_ROUTING:
            if host_matches(request_host, listen_host):
                return get_response(request)
@@ -282,16 +305,7 @@ def HostRoutingMiddleware(get_response):
            request._cached_user = request.user
            return get_response(request)

-        if host_matches(request_host, public_host):
-            if request.COOKIES.get(ADMIN_LOGIN_HINT_COOKIE) == "1" and (request.path == "/public" or request.path.startswith("/public/")):
-                target = build_admin_url("/admin/core/snapshot/", request=request)
-                return redirect(target)
-            return get_response(request)
-
        if subdomain:
-            if is_snapshot_subdomain(subdomain):
-                view = SnapshotHostView.as_view()
-                return view(request, snapshot_id=subdomain, path=request.path.lstrip("/"))
            view = OriginalDomainHostView.as_view()
            return view(request, domain=subdomain, path=request.path.lstrip("/"))

@@ -0,0 +1,17 @@
+# Generated by Django 6.0.6 on 2026-06-04 22:16
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0048_remove_archiveresult_config"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="snapshot",
+            name="url",
+            field=models.TextField(db_index=True),
+        ),
+    ]
@@ -0,0 +1,69 @@
+import django.db.models.fields.json
+import django.db.models.functions.comparison
+from django.db import migrations, models
+from django.db.models import Q
+
+
+VALID_PERMISSIONS = {"public", "unlisted", "private"}
+BATCH_SIZE = 1000
+
+
+def normalize_permissions(value, default="private"):
+    value = str(value or "").strip().lower()
+    return value if value in VALID_PERMISSIONS else default
+
+
+def hydrate_snapshot_permissions(apps, schema_editor):
+    Snapshot = apps.get_model("core", "Snapshot")
+    db_alias = schema_editor.connection.alias
+    missing_permissions = Q(permissions__isnull=True) | (Q(permissions__isnull=False) & ~Q(permissions__in=VALID_PERMISSIONS))
+    batch = []
+
+    snapshots = (
+        Snapshot.objects.using(db_alias)
+        .filter(missing_permissions)
+        .select_related("crawl")
+        .only("id", "config", "crawl__permissions")
+        .iterator(chunk_size=BATCH_SIZE)
+    )
+    for snapshot in snapshots:
+        config = dict(snapshot.config or {})
+        config["PERMISSIONS"] = normalize_permissions(snapshot.crawl.permissions)
+        snapshot.config = config
+        batch.append(snapshot)
+        if len(batch) >= BATCH_SIZE:
+            Snapshot.objects.using(db_alias).bulk_update(batch, ["config"], batch_size=BATCH_SIZE)
+            batch.clear()
+
+    if batch:
+        Snapshot.objects.using(db_alias).bulk_update(batch, ["config"], batch_size=BATCH_SIZE)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("crawls", "0016_hydrate_crawl_permissions"),
+        ("core", "0049_alter_snapshot_url"),
+    ]
+
+    operations = [
+        migrations.RunPython(hydrate_snapshot_permissions, migrations.RunPython.noop),
+        migrations.RemoveField(
+            model_name="snapshot",
+            name="permissions",
+        ),
+        migrations.AddField(
+            model_name="snapshot",
+            name="permissions",
+            field=models.GeneratedField(
+                db_index=True,
+                db_persist=True,
+                editable=False,
+                expression=django.db.models.functions.comparison.Coalesce(
+                    django.db.models.fields.json.KeyTextTransform("PERMISSIONS", "config"),
+                    models.Value("private"),
+                    output_field=models.CharField(max_length=16),
+                ),
+                output_field=models.CharField(max_length=16),
+            ),
+        ),
+    ]
@@ -1,7 +1,7 @@
 __package__ = "archivebox.core"

 from typing import TYPE_CHECKING, Optional, Any
-from collections.abc import Iterable, Sequence
+from collections.abc import Iterable, Mapping, Sequence
 import uuid
 from archivebox.uuid_compat import CompactUUIDField, uuid7
 from datetime import datetime, timedelta
@@ -14,7 +14,7 @@ from urllib.parse import urlparse
 from statemachine import State, registry

 from django.db import models, transaction
-from django.db.models import Case, Q, QuerySet, Sum, Value, When
+from django.db.models import Case, F, Q, QuerySet, Sum, Value, When
 from django.db.models.functions import Coalesce, Concat
 from django.db.models.fields.json import KT
 from django.utils.functional import cached_property
@@ -31,15 +31,15 @@ from archivebox.config import CONSTANTS
 from archivebox.config.common import get_config, rprint
 from archivebox.misc.system import atomic_write
 from archivebox.misc.util import (
-    MAX_URL_LENGTH,
    parse_date,
    domain as url_domain,
    to_json,
    ts_to_date_str,
    urlencode,
-    htmlencode,
+    htmldecode,
+    sanitize_html_text,
    urldecode,
-    validate_url_length,
+    validate_url,
 )
 from archivebox.plugins.discovery import (
    get_plugins,
@@ -95,6 +95,12 @@ class Tag(ModelWithUUID):
    def __str__(self):
        return self.name

+    def save(self, *args, **kwargs):
+        update_fields = kwargs.get("update_fields")
+        if update_fields is None or "name" in update_fields:
+            self.name = sanitize_html_text(self.name).strip()
+        super().save(*args, **kwargs)
+
    @property
    def slug(self) -> str:
        """ASCII-safe slugified form of the tag name (derived, not stored)."""
@@ -137,7 +143,7 @@ class Tag(ModelWithUUID):

        # Auto-attach to snapshot if in overrides
        if overrides and "snapshot" in overrides and tag:
-            overrides["snapshot"].tags.add(tag)
+            overrides["snapshot"].add_tag_ids([tag.pk])

        return tag

@@ -156,6 +162,32 @@ class SnapshotTag(models.Model):
 class SnapshotQuerySet(models.QuerySet):
    """Custom QuerySet for Snapshot model with export methods that persist through .filter() etc."""

+    def bulk_create(self, objs, *args, **kwargs):
+        objs = list(objs)
+        missing_crawl_ids = set()
+        from archivebox.core.permissions import PERMISSIONS_VALUES
+
+        for obj in objs:
+            if isinstance(obj, self.model):
+                config = dict(obj.config or {})
+                permission = str(config.get("PERMISSIONS") or "").strip().lower()
+                if permission not in PERMISSIONS_VALUES and obj.crawl_id:
+                    crawl = getattr(obj, "crawl", None)
+                    if not getattr(crawl, "permissions", None):
+                        missing_crawl_ids.add(str(obj.crawl_id))
+
+        crawl_permissions_by_id = {}
+        if missing_crawl_ids:
+            crawl_permissions_by_id = {
+                str(crawl_id): permissions
+                for crawl_id, permissions in Crawl.objects.filter(pk__in=missing_crawl_ids).values_list("pk", "permissions")
+            }
+
+        for obj in objs:
+            if isinstance(obj, self.model):
+                obj.ensure_permissions_config(crawl_permissions=crawl_permissions_by_id.get(str(obj.crawl_id)))
+        return super().bulk_create(objs, *args, **kwargs)
+
    def paged_iterator(self, chunk_size: int = 500):
        """
        Iterate snapshots using bounded keyset pages instead of one streaming cursor.
@@ -267,6 +299,22 @@ class SnapshotQuerySet(models.QuerySet):
        "tag": lambda pattern: models.Q(tags__name=pattern),
        "timestamp": lambda pattern: models.Q(timestamp=pattern),
    }
+    FILTER_TYPE_CHOICES = tuple(FILTER_TYPES)
+    FILTER_ARG_KEYS = (
+        "after",
+        "before",
+        "filter_type",
+        "filter_patterns",
+        "status",
+        "url__icontains",
+        "url__istartswith",
+        "tag",
+        "crawl_id",
+        "limit",
+        "sort",
+        "search",
+    )
+    SPECIAL_FILTER_ARG_KEYS = frozenset({"filter_patterns", "filter_type", "query", "search", "tag", "before", "after", "limit", "sort"})

    def filter_by_patterns(self, patterns: list[str], filter_type: str = "exact") -> "SnapshotQuerySet":
        """Filter snapshots by URL patterns using specified filter type"""
@@ -283,6 +331,62 @@ class SnapshotQuerySet(models.QuerySet):
                raise SystemExit(2)
        return self.filter(q_filter)

+    def search(self, **kwargs) -> "SnapshotQuerySet":
+        from datetime import timezone as dt_timezone
+
+        from archivebox.core.snapshot_status import filter_snapshots_by_status
+        from archivebox.search.query import apply_snapshot_search
+
+        queryset = self
+        filter_patterns = tuple(str(pattern) for pattern in kwargs.get("filter_patterns") or ())
+        filter_type = kwargs.get("filter_type") or "substring"
+        query = kwargs.get("query")
+        if isinstance(query, (list, tuple)):
+            query = " ".join(str(part) for part in query)
+        query = (query or (" ".join(filter_patterns) if kwargs.get("search") else "")).strip()
+
+        field_names = {field.name for field in self.model._meta.get_fields()}
+        field_names.update(field.attname for field in self.model._meta.fields)
+        field_filters = {
+            key: value
+            for key, value in kwargs.items()
+            if value is not None and key not in self.SPECIAL_FILTER_ARG_KEYS and key.split("__", 1)[0] in field_names
+        }
+        status = field_filters.pop("status", None)
+        queryset = filter_snapshots_by_status(queryset, status)
+        if field_filters:
+            queryset = queryset.filter(**field_filters)
+        if kwargs.get("tag"):
+            queryset = queryset.filter(tags__name__iexact=kwargs["tag"])
+        if kwargs.get("before") is not None:
+            queryset = queryset.filter(bookmarked_at__lt=datetime.fromtimestamp(float(kwargs["before"]), tz=dt_timezone.utc))
+        if kwargs.get("after") is not None:
+            queryset = queryset.filter(bookmarked_at__gt=datetime.fromtimestamp(float(kwargs["after"]), tz=dt_timezone.utc))
+
+        if query:
+            queryset = apply_snapshot_search(
+                queryset,
+                query,
+                search_mode=kwargs.get("search"),
+                ordering=("-created_at",) if not kwargs.get("sort") else None,
+                max_results=kwargs.get("limit"),
+                skip_backend_when_metadata_satisfies_limit=True,
+                include_metadata_for_forced_backend=True,
+            )
+        elif filter_patterns:
+            queryset = queryset.filter_by_patterns(list(filter_patterns), filter_type)
+
+        if kwargs.get("sort"):
+            queryset = queryset.order_by(kwargs["sort"])
+        elif not queryset.query.order_by:
+            queryset = queryset.order_by("-created_at")
+
+        limit = kwargs.get("limit")
+        if limit is not None and limit > 0:
+            queryset = queryset[:limit]
+
+        return queryset
+
    # =========================================================================
    # Export Methods
    # =========================================================================
@@ -392,11 +496,16 @@ class SnapshotManager(models.Manager.from_queryset(SnapshotQuerySet)):  # ty: ig


 class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHealthStats, ModelWithStateMachine):
+    INTERNAL_INPUT_URL = "archivebox://internal"
+
    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
    created_at = models.DateTimeField(default=timezone.now, db_index=True)
    modified_at = models.DateTimeField(auto_now=True)

-    url = models.CharField(max_length=MAX_URL_LENGTH, unique=False, db_index=True)  # URLs can appear in multiple crawls
+    # Stored as a variable-length TextField so short URLs don't reserve space and very long
+    # URLs (up to MAX_URL_LENGTH chars, enforced in save()) are supported, while keeping a
+    # normal index so exact, prefix, and substring lookups all stay fast.
+    url = models.TextField(db_index=True)  # URLs can appear in multiple crawls
    timestamp = models.CharField(max_length=32, unique=True, db_index=True, editable=False)
    bookmarked_at = models.DateTimeField(default=timezone.now, db_index=True)
    crawl: Crawl = models.ForeignKey(Crawl, on_delete=models.CASCADE, null=False, related_name="snapshot_set", db_index=True)  # type: ignore[assignment]
@@ -431,8 +540,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
    )
    config = models.JSONField(default=dict, null=False, blank=False, editable=True)
    permissions = models.GeneratedField(
-        expression=KT("config__PERMISSIONS"),
-        output_field=models.CharField(max_length=16, null=True),
+        expression=Coalesce(KT("config__PERMISSIONS"), Value("private"), output_field=models.CharField(max_length=16)),
+        output_field=models.CharField(max_length=16, null=False),
        db_persist=True,
        db_index=True,
        editable=False,
@@ -464,6 +573,20 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
    objects = SnapshotManager()
    archiveresult_set: models.Manager["ArchiveResult"]

+    if TYPE_CHECKING:
+
+        @property
+        def sm(self) -> "SnapshotMachine": ...
+
+    def add_tag_ids(self, tag_ids: Iterable[int | str]) -> None:
+        tag_ids = [tag_id for tag_id in dict.fromkeys(tag_ids) if tag_id]
+        if not tag_ids:
+            return
+        SnapshotTag.objects.bulk_create(
+            [SnapshotTag(snapshot_id=self.pk, tag_id=tag_id) for tag_id in tag_ids],
+            ignore_conflicts=True,
+        )
+
    class Meta(
        ModelWithDeleteAfter.Meta,
        ModelWithOutputDir.Meta,
@@ -677,7 +800,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
        )

    @classmethod
-    def is_archivebox_internal_url(cls, url: str) -> bool:
+    def is_archivebox_internal_url(cls, url: str, *, config: Mapping[str, Any] | Any | None = None) -> bool:
        parsed = urlparse((url or "").strip())
        if parsed.scheme not in ("http", "https") or not parsed.hostname:
            return False
@@ -685,24 +808,39 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
        from archivebox.core.routes_util import (
            get_admin_host,
            get_api_host,
+            get_base_host,
            get_listen_host,
-            get_public_host,
            get_web_host,
            split_host_port,
        )

-        config = get_config()
+        if config is None:
+            config = get_config()
+        elif isinstance(config, Mapping):
+            route_config = config
+
+            class RouteConfig:
+                BIND_ADDR = str(route_config.get("BIND_ADDR") or "")
+                BASE_URL = str(route_config.get("BASE_URL") or "")
+                CSRF_TRUSTED_ORIGINS = str(route_config.get("CSRF_TRUSTED_ORIGINS") or "")
+                SERVER_SECURITY_MODE = str(route_config.get("SERVER_SECURITY_MODE") or "")
+
+                @property
+                def USES_SUBDOMAIN_ROUTING(self) -> bool:
+                    return self.SERVER_SECURITY_MODE == "safe-subdomains-fullreplay"
+
+            config = RouteConfig()
        host = parsed.hostname.lower().strip(".")
        port = str(parsed.port) if parsed.port else None
-        protected_subdomains = {"admin", "web", "api", "public"}
+        protected_subdomains = {"admin", "web", "api"}
        protected_hosts: set[tuple[str, str | None]] = set()
        protected_roots: set[tuple[str, str | None]] = set()
        for host_value in (
            get_listen_host(config=config),
+            get_base_host(config=config),
            get_admin_host(config=config),
            get_web_host(config=config),
            get_api_host(config=config),
-            get_public_host(config=config),
        ):
            if not host_value:
                continue
@@ -755,17 +893,59 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW

        return Binary.objects.filter(process_set__archiveresult__snapshot_id=self.id).distinct()

+    def ensure_permissions_config(self, crawl_permissions: str | None = None) -> bool:
+        config = dict(self.config or {})
+        permission = str(config.get("PERMISSIONS") or "").strip().lower()
+        from archivebox.core.permissions import PERMISSIONS_PUBLIC, PERMISSIONS_VALUES, normalize_permissions
+
+        if permission not in PERMISSIONS_VALUES:
+            if self.crawl_id:
+                if not crawl_permissions:
+                    crawl_permissions = Crawl.objects.filter(pk=self.crawl_id).values_list("permissions", flat=True).first()
+            config["PERMISSIONS"] = normalize_permissions(
+                crawl_permissions,
+                default=PERMISSIONS_PUBLIC,
+            )
+            self.config = config
+            return True
+        elif config.get("PERMISSIONS") != permission:
+            config["PERMISSIONS"] = permission
+            self.config = config
+            return True
+        return False
+
+    def validate_url_for_archiving(self, *, config: Mapping[str, Any] | Any | None = None) -> None:
+        if self.is_internal_input_url():
+            return
+
+        try:
+            validate_url(self.url or "")
+        except ValueError as err:
+            raise ValidationError({"url": str(err)}) from err
+
+        if self.is_archivebox_internal_url(self.url, config=config):
+            raise ValidationError({"url": "ArchiveBox cannot archive its own admin, web, api, or snapshot URLs."})
+
+    def is_internal_input_url(self) -> bool:
+        return (self.url or "").strip() == self.INTERNAL_INPUT_URL and self.depth == 0 and bool(self.crawl_id)
+
    def save(self, *args, **kwargs):
        update_fields = kwargs.get("update_fields")
        validate_url_field = self._state.adding or update_fields is None or "url" in update_fields
-        if validate_url_field:
-            try:
-                validate_url_length(self.url or "")
-            except ValueError as err:
-                raise ValidationError({"url": str(err)}) from err
+        crawl_config_for_save = None
+        crawl_permissions_for_save = None
+        if self.crawl_id and validate_url_field:
+            crawl_row = Crawl.objects.filter(pk=self.crawl_id).values("config", "permissions").first()
+            if crawl_row:
+                crawl_config_for_save = crawl_row.get("config") or {}
+                crawl_permissions_for_save = crawl_row.get("permissions")

-            if self.is_archivebox_internal_url(self.url):
-                raise ValidationError({"url": "ArchiveBox cannot archive its own admin, web, api, or snapshot URLs."})
+        if self.ensure_permissions_config(crawl_permissions=crawl_permissions_for_save):
+            if update_fields is not None:
+                kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "config"]))
+
+        if validate_url_field:
+            self.validate_url_for_archiving(config=crawl_config_for_save if self.crawl_id else None)

        if not self.bookmarked_at:
            self.bookmarked_at = self.created_at or timezone.now()
@@ -774,14 +954,17 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW

        if self._state.adding or update_fields is None or "title" in update_fields:
            self.title = self._normalize_title_candidate(self.title, snapshot_url=self.url or "") or None
+        if self._state.adding or update_fields is None or "notes" in update_fields:
+            self.notes = sanitize_html_text(self.notes)

        # Migrate filesystem if needed (happens automatically on save)
-        if self.pk and self.fs_migration_needed:
+        existing_snapshot = self.pk and not self._state.adding
+        if existing_snapshot and self.fs_migration_needed:
            self.migrate_filesystem_to_current_version()
            update_fields = kwargs.get("update_fields")
            if update_fields is not None:
                kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "fs_version", "modified_at"]))
-        elif self.pk:
+        elif existing_snapshot:
            current_dir = self.get_storage_path_for_version(self._fs_current_version())
            source_dir = Path(self.output_dir)
            if source_dir.exists() and source_dir != current_dir and not source_dir.is_symlink():
@@ -794,8 +977,29 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
        def finish_snapshot_save():
            self.ensure_legacy_archive_symlink()
            self.ensure_crawl_symlink()
-            crawl = self.crawl
-            if not crawl.url_passes_filters(self.url, snapshot=self):
+            crawl = Crawl.objects.filter(pk=self.crawl_id).first()
+            if crawl is None:
+                return
+            crawl_tag_names = crawl.current_tag_names()
+            if crawl_tag_names:
+                # Snapshots can be created by parser hook side-effect records,
+                # direct ORM creates, or legacy crawl URL expansion. Crawl tags
+                # are user-facing metadata on the whole import, so attach them
+                # at the Snapshot.save() boundary instead of relying on every
+                # caller to remember to duplicate this fanout logic.
+                tags_by_name = {tag.name: tag for tag in Tag.objects.filter(name__in=crawl_tag_names)}
+                missing_tags = [Tag(name=name) for name in crawl_tag_names if name not in tags_by_name]
+                if missing_tags:
+                    Tag.objects.bulk_create(missing_tags, ignore_conflicts=True)
+                    tags_by_name = {tag.name: tag for tag in Tag.objects.filter(name__in=crawl_tag_names)}
+                self.add_tag_ids([tag.pk for name in crawl_tag_names if (tag := tags_by_name.get(name))])
+            # Snapshot.save() normally appends newly created URLs to Crawl.urls
+            # so legacy/direct crawls can keep their queue text in sync. For
+            # internal-input crawls that would corrupt the original submitted
+            # import text; parsed URLs are represented by child Snapshot rows.
+            if crawl.has_internal_input_root():
+                return
+            if not crawl.url_passes_filters(self.url, snapshot=self, use_effective_config=False):
                return
            # Best-effort skip if our URL is already recorded on the crawl;
            # the atomic UPDATE below is what actually prevents clobbering.
@@ -1504,7 +1708,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
            with transaction.atomic():
                for tag_name in new_tags:
                    tag, _ = Tag.objects.get_or_create(name=tag_name)
-                    self.tags.add(tag)
+                    self.add_tag_ids([tag.pk])

    def _merge_archive_results_from_index(self, index_data: dict, update_existing: bool = True):
        """Merge ArchiveResults one row per hook; retries update the existing row."""
@@ -1950,7 +2154,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW

            # Merge tags
            for tag in dup.tags.all():
-                keeper.tags.add(tag)
+                keeper.add_tag_ids([tag.pk])

            # Move ArchiveResults
            ArchiveResult.objects.filter(snapshot=dup).update(
@@ -1984,6 +2188,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW

    @admin.display(description="Tags")
    def tags_str(self, nocache=True) -> str | None:
+        if "_tags_str_cached" in self.__dict__:
+            return self.__dict__["_tags_str_cached"]
        calc_tags_str = lambda: ",".join(sorted(tag.name for tag in self.tags.all()))
        prefetched_cache = self.__dict__.get("_prefetched_objects_cache", {})
        if "tags" in prefetched_cache:
@@ -2121,7 +2327,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW

    @staticmethod
    def _normalize_title_candidate(candidate: str | None, *, snapshot_url: str) -> str:
-        title = " ".join(line.strip() for line in str(candidate or "").splitlines() if line.strip()).strip()
+        title = " ".join(line.strip() for line in sanitize_html_text(candidate).splitlines() if line.strip()).strip()
        if not title:
            return ""
        if title.lower() in {"pending...", "no title found", "unable to detect page title"}:
@@ -2396,7 +2602,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
    def save_tags(self, tags: Iterable[str] = ()) -> None:
        tags_id = [Tag.objects.get_or_create(name=tag)[0].pk for tag in tags if tag.strip()]
        self.tags.clear()
-        self.tags.add(*tags_id)
+        self.add_tag_ids(tags_id)

    def pending_archiveresults(self) -> QuerySet["ArchiveResult"]:
        return self.archiveresult_set.exclude(status__in=ArchiveResult.FINAL_OR_ACTIVE_STATES)
@@ -2577,15 +2783,18 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
                )
                rprint(f"[red]⚠️  Snapshot.from_json auto-created new crawl {crawl.id} for url={url}[/red]", file=sys.stderr)

-        # Parse tags (accept either a list ["tag1", "tag2"] or a comma-separated string "tag1,tag2")
+        # Parser hooks emit child Snapshot records through this generic
+        # dispatcher after crawling an internal import root. Those child rows
+        # must inherit crawl-level tags just like direct Crawl.urls snapshots;
+        # otherwise `archivebox add --tag ... < bookmarks.html` loses the tag
+        # unless the parser format also happened to provide its own tags.
        tags_raw = record.get("tags", "")
-        tag_list = []
+        tag_list = list(crawl.current_tag_names()) if crawl else []
        if isinstance(tags_raw, list):
-            tag_list = list(dict.fromkeys(tag.strip() for tag in tags_raw if tag.strip()))
+            tag_list.extend(tag.strip() for tag in tags_raw if tag.strip())
        elif tags_raw:
-            tag_list = list(
-                dict.fromkeys(tag.strip() for tag in re.split(config.TAG_SEPARATOR_PATTERN, tags_raw) if tag.strip()),
-            )
+            tag_list.extend(tag.strip() for tag in re.split(config.TAG_SEPARATOR_PATTERN, tags_raw) if tag.strip())
+        tag_list = list(dict.fromkeys(tag_list))

        # Check for existing snapshot with same URL in same crawl
        # (URLs can exist in multiple crawls, but should be unique within a crawl)
@@ -2658,7 +2867,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW

        return snapshot

-    def create_pending_archiveresults(self) -> list["ArchiveResult"]:
+    def create_pending_archiveresults(self, hooks: Iterable[tuple[str, str]] | None = None) -> list["ArchiveResult"]:
        """
        Create ArchiveResult records for all enabled hooks.

@@ -2669,18 +2878,23 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
        Creates one ArchiveResult per hook (not per plugin), with hook_name set.
        This enables step-based execution where all hooks in a step can run in parallel.
        """
-        from archivebox.plugins.hooks import discover_hooks
-        from archivebox.config.common import get_config
+        try:
+            self.validate_url_for_archiving()
+        except ValidationError as err:
+            rprint(f"[yellow][!] Skipping blocked snapshot URL: {(self.url or '')[:120]}... ({err})[/yellow]")
+            return []

-        # Get merged config with crawl-specific PLUGINS filter
-        config = get_config(crawl=self.crawl, snapshot=self)
-        hooks = discover_hooks("Snapshot", config=config)
+        if hooks is None:
+            from archivebox.plugins.hooks import discover_hooks
+            from archivebox.config.common import get_config
+
+            # Compatibility path for direct model callers. The runner passes its
+            # abx-dl hook inventory explicitly so queued rows match execution.
+            config = get_config(crawl=self.crawl, snapshot=self)
+            hooks = ((hook_path.parent.name, hook_path.stem) for hook_path in discover_hooks("Snapshot", config=config))
        archiveresults = []

-        for hook_path in hooks:
-            hook_name = hook_path.stem  # e.g., 'on_Snapshot__50_wget'
-            plugin = hook_path.parent.name  # e.g., 'wget'
-
+        for plugin, hook_name in hooks:
            # ArchiveResult output is one filesystem directory per plugin hook, so
            # retries must update this row in place instead of creating siblings.
            archiveresult, _created = ArchiveResult.objects.get_or_create(
@@ -3168,7 +3382,11 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
        """Convert to CSV string"""
        data = self.to_dict()
        cols = cols or ["timestamp", "is_archived", "url"]
-        return separator.join(to_json(data.get(col, ""), indent=None).ljust(ljust) for col in cols)
+        invalid_cols = [col for col in dict.fromkeys(cols) if col not in data]
+        if invalid_cols:
+            supported_cols = ", ".join(sorted(data))
+            raise ValueError(f"Invalid CSV field(s): {', '.join(invalid_cols)}\nSupported CSV fields: {supported_cols}")
+        return separator.join(to_json(data[col], indent=None).ljust(ljust) for col in cols)

    def write_json_details(self, out_dir: Path | str | None = None) -> None:
        """Write JSON index file for this snapshot to its output directory"""
@@ -3216,8 +3434,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
        context = {
            **self.to_dict(extended=True),
            "snapshot": self,
-            "title": htmlencode(self.resolved_title or (self.base_url if is_archived else TITLE_LOADING_MSG)),
-            "url_str": htmlencode(urldecode(self.base_url)),
+            "title": htmldecode(self.resolved_title or (self.base_url if is_archived else TITLE_LOADING_MSG)),
+            "url_str": htmldecode(urldecode(self.base_url)),
            "archive_url": urlencode(f"warc/{self.timestamp}" or (self.domain if is_archived else "")) or "about:blank",
            "extension": self.extension or "html",
            "tags": self.tags_str() or "untagged",
@@ -3798,21 +4016,49 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
            or "snapshot_id" in update_fields
        )
        old_snapshot_id = None
+        old_output_size = 0
        if refresh_snapshot_size and not is_new:
-            old_snapshot_id = type(self).objects.filter(pk=self.pk).values_list("snapshot_id", flat=True).first()
-
-        update_fields = kwargs.get("update_fields")
-        if self.delete_at is None:
-            self.set_delete_at_from_config()
-            if self.delete_at is not None and update_fields is not None:
-                kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "delete_at"]))
+            old_values = type(self).objects.filter(pk=self.pk).values("snapshot_id", "output_size").first()
+            if old_values:
+                old_snapshot_id = old_values["snapshot_id"]
+                old_output_size = int(old_values["output_size"] or 0)

+        # ArchiveResult rows are updated on every plugin event. Resolving
+        # DELETE_AFTER here is deceptively expensive because the effective
+        # value lives on Snapshot/Crawl config, so a save of an already-loaded
+        # result can still materialize parent objects and parse config. The
+        # orchestrator owns the repair pass for these rows instead: it fills
+        # missing delete_at values from fresh Snapshot/Crawl config when the
+        # queue is idle, outside the hook-result write hot path.
        # Skip ModelWithOutputDir.save() to avoid creating index.json in plugin directories
        # Call the Django Model.save() directly instead
        models.Model.save(self, *args, **kwargs)
        if refresh_snapshot_size:
-            snapshot_ids = {snapshot_id for snapshot_id in (old_snapshot_id, self.snapshot_id) if snapshot_id}
-            transaction.on_commit(lambda: type(self).refresh_snapshot_output_sizes(snapshot_ids))
+            current_snapshot_id = self.snapshot_id
+            snapshot_ids = {snapshot_id for snapshot_id in (old_snapshot_id, current_snapshot_id) if snapshot_id}
+            current_output_size = int(self.output_size or 0)
+            if len(snapshot_ids) > 1:
+                # Moving an ArchiveResult between Snapshots is rare and cannot
+                # be represented as a single delta on one parent row. Keep the
+                # conservative aggregate fallback for that shape.
+                transaction.on_commit(lambda: type(self).refresh_snapshot_output_sizes(snapshot_ids))
+            elif current_snapshot_id:
+                # Hook-result projection updates ArchiveResult rows at very
+                # high frequency during indexing. Re-aggregating every sibling
+                # row for the parent Snapshot on each save turns those short
+                # writes into a table-scan hot path. For the common case where
+                # the result stays attached to the same Snapshot, the persisted
+                # parent total is exactly the old total plus this row's size
+                # delta; F() keeps that update atomic with concurrent result
+                # saves for other plugins on the same Snapshot.
+                size_delta = current_output_size if is_new else current_output_size - old_output_size
+                if size_delta:
+                    transaction.on_commit(
+                        lambda: Snapshot.objects.filter(pk=current_snapshot_id).update(
+                            output_size=F("output_size") + size_delta,
+                            modified_at=timezone.now(),
+                        ),
+                    )
        if is_new or update_fields is None or "status" in update_fields or "snapshot" in update_fields or "snapshot_id" in update_fields:
            transaction.on_commit(type(self).clear_majority_status_cache)

@@ -1,6 +1,6 @@
 from __future__ import annotations

-from django.db.models import Q, QuerySet
+from django.db.models import QuerySet
 from django.http import HttpRequest

 PERMISSIONS_PUBLIC = "public"
@@ -32,10 +32,7 @@ def is_admin_user(request: HttpRequest) -> bool:


 def get_snapshot_permissions(snapshot) -> str:
-    permissions = snapshot.permissions
-    if not permissions:
-        permissions = snapshot.crawl.permissions
-    return normalize_permissions(permissions)
+    return normalize_permissions(snapshot.permissions)


 def can_view_snapshot(request: HttpRequest, snapshot) -> bool:
@@ -56,10 +53,7 @@ def filter_personas_by_permissions(queryset: QuerySet, allowed_permissions: set[
 def filter_snapshots_by_permissions(queryset: QuerySet, *, direct: bool = False, allowed_permissions: set[str] | None = None) -> QuerySet:
    allowed_permissions = allowed_permissions or ({PERMISSIONS_PUBLIC, PERMISSIONS_UNLISTED} if direct else {PERMISSIONS_PUBLIC})
    allowed = sorted(allowed_permissions)
-    return queryset.exclude(
-        (Q(permissions__isnull=False) & ~Q(permissions__in=allowed))
-        | (Q(permissions__isnull=True) & (Q(crawl__permissions__isnull=True) | ~Q(crawl__permissions__in=allowed))),
-    )
+    return queryset.filter(permissions__in=allowed)


 def public_snapshots_queryset(queryset: QuerySet) -> QuerySet:
@@ -0,0 +1,4 @@
+__package__ = "archivebox.core"
+
+
+EXTENSION_SCREENSHOT_PLUGIN = "chrome_extension_screenshot"
@@ -1,26 +1,39 @@
 from __future__ import annotations

+from pathlib import Path
+
 from django.utils import timezone
 from rich.console import Console


-def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int]:
+def _is_signal_interrupted_exit(exit_code: int | None) -> bool:
+    return exit_code is not None and (exit_code < 0 or exit_code >= 128)
+
+
+def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str | None = None) -> dict[str, int]:
    from archivebox.crawls.models import Crawl
    from archivebox.core.models import ArchiveResult, Snapshot
+    from archivebox.services.archive_result_service import _collect_output_metadata
    from archivebox.machine.models import Process
+    from django.core.exceptions import ValidationError
    from django.db.models import Exists, OuterRef, Q, Subquery, Value
    from django.db.models.functions import Coalesce

    now = timezone.now()
    recovery_console = Console(stderr=True, highlight=False, soft_wrap=True)
+    crawl_filter = {"id": crawl_id} if crawl_id else {}
+    snapshot_filter = {"crawl_id": crawl_id} if crawl_id else {}
+    result_filter = {"snapshot__crawl_id": crawl_id} if crawl_id else {}
    cleaned = {
-        "processes_stale_running": Process.cleanup_stale_running(),
-        "processes_orphaned_workers": Process.cleanup_orphaned_workers(),
-        "chrome_processes_orphaned": Process.cleanup_orphaned_chrome() if include_chrome else 0,
+        "processes_stale_running": 0 if crawl_id else Process.cleanup_stale_running(),
+        "processes_orphaned_workers": 0 if crawl_id else Process.cleanup_orphaned_workers(),
+        "chrome_processes_orphaned": Process.cleanup_orphaned_chrome() if include_chrome and not crawl_id else 0,
        "crawls_queued_without_retry_at": 0,
        "snapshots_queued_without_retry_at": 0,
        "archiveresults_backoff": 0,
+        "snapshots_queued_plugin_rows_waiting_on_stale_lease": 0,
        "archiveresults_started_without_running_process": 0,
+        "archiveresults_missing_for_orphaned_hook_processes": 0,
        "snapshots_started_without_running_results": 0,
        "crawls_started_with_due_snapshots": 0,
        "crawls_started_waiting_on_future_snapshots": 0,
@@ -48,14 +61,16 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
    cleaned["crawls_queued_without_retry_at"] = Crawl.objects.filter(
        status=Crawl.StatusChoices.QUEUED,
        retry_at__isnull=True,
+        **crawl_filter,
    ).update(retry_at=now, modified_at=now)
    cleaned["snapshots_queued_without_retry_at"] = Snapshot.objects.filter(
        status=Snapshot.StatusChoices.QUEUED,
        retry_at__isnull=True,
        crawl__status__in=Crawl.RUNNABLE_STATES,
+        **snapshot_filter,
    ).update(retry_at=now, modified_at=now)
-    backoff_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.BACKOFF)
-    orphaned_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.STARTED).exclude(
+    backoff_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.BACKOFF, **result_filter)
+    orphaned_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.STARTED, **result_filter).exclude(
        process__status=Process.StatusChoices.RUNNING,
    )
    # ArchiveResult has no retry_at scheduler. Wake only the parent Snapshots
@@ -73,6 +88,24 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
        retry_at__isnull=True,
    ).update(retry_at=now, modified_at=now)
    cleaned["archiveresults_backoff"] = backoff_results.update(status=ArchiveResult.StatusChoices.QUEUED, modified_at=now)
+    # Targeted plugin rows on final/paused Snapshots are scheduled through the
+    # parent Snapshot.retry_at. retry_at=NULL is the normal idle marker for a
+    # sealed Snapshot and must not be interpreted as queued work just because
+    # old/synthetic ArchiveResult rows exist. If takeover kills the runner
+    # after it leases the Snapshot but before queued ArchiveResult rows finish,
+    # the rows remain QUEUED while retry_at sits in the future. Recovery runs
+    # only after this runner has won the single-runner gate, so it can safely
+    # unlock those stale plugin leases for immediate processing instead of
+    # waiting out the previous owner's full lock timeout.
+    queued_plugin_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.QUEUED, **result_filter)
+    cleaned["snapshots_queued_plugin_rows_waiting_on_stale_lease"] = (
+        Snapshot.objects.filter(
+            id__in=queued_plugin_results.values("snapshot_id"),
+            status__in=[Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED],
+        )
+        .filter(retry_at__gt=now)
+        .update(retry_at=now, modified_at=now)
+    )
    # Impossible state repair: STARTED ArchiveResults without a live Process
    # have no owner left to emit completion. Requeue only the result row; the
    # snapshot/crawl schedulers will pick up normal retry processing.
@@ -81,8 +114,96 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
        process=None,
        modified_at=now,
    )
+    orphaned_hook_processes = Process.objects.filter(
+        process_type=Process.TypeChoices.HOOK,
+        archiveresult__isnull=True,
+    ).exclude(status=Process.StatusChoices.RUNNING)
+    crawl_snapshot_ids = (
+        {str(snapshot_id) for snapshot_id in Snapshot.objects.filter(crawl_id=crawl_id).values_list("id", flat=True)} if crawl_id else None
+    )
+    if crawl_snapshot_ids is not None:
+        # Targeted `archivebox run --crawl-id ...` is used by foreground add/update
+        # commands and by resume of one existing crawl. It must still repair bad
+        # hook state for that crawl, but it must not scan historical orphaned hook
+        # rows from unrelated crawls before the first snapshot can run.
+        if crawl_snapshot_ids:
+            snapshot_pwd_filter = Q()
+            for snapshot_id in crawl_snapshot_ids:
+                snapshot_pwd_filter |= Q(pwd__contains=snapshot_id)
+            orphaned_hook_processes = orphaned_hook_processes.filter(snapshot_pwd_filter)
+        else:
+            orphaned_hook_processes = orphaned_hook_processes.none()
+    for process in orphaned_hook_processes.only("id", "pwd", "cmd", "process_type", "status"):
+        hook_script_name = process.hook_script_name
+        if not hook_script_name or not process.pwd:
+            continue
+        plugin_dir = Path(process.pwd)
+        if crawl_snapshot_ids is not None and plugin_dir.parent.name not in crawl_snapshot_ids:
+            continue
+        try:
+            # Old or synthetic hook Process rows can point at arbitrary paths.
+            # Only paths whose parent directory is a valid Snapshot id can be
+            # reconstructed into ArchiveResult rows.
+            snapshot = Snapshot.objects.filter(id=plugin_dir.parent.name, **snapshot_filter).first()
+        except ValidationError:
+            continue
+        if snapshot is None:
+            continue
+        result, created = ArchiveResult.objects.get_or_create(
+            snapshot=snapshot,
+            plugin=plugin_dir.name,
+            hook_name=Path(hook_script_name).stem,
+            defaults={
+                "status": ArchiveResult.StatusChoices.QUEUED,
+            },
+        )
+        if result.status == ArchiveResult.StatusChoices.QUEUED:
+            requeue_snapshot = False
+            # A runner can die after the hook Process exits but before the
+            # ProcessCompletedEvent projector links/finalizes ArchiveResult.
+            # Reconstruct only that exact hook row from the durable Process row.
+            output_files, output_size, output_mimetypes = _collect_output_metadata(plugin_dir)
+            result.process = process
+            if _is_signal_interrupted_exit(process.exit_code):
+                # The owning runner died or was asked to stop while the hook was
+                # still active. Keep the work item queued so takeover retries the
+                # same hook; treating an unknown signal exit as success would
+                # silently skip unfinished side effects.
+                result.output_files = {}
+                result.output_size = 0
+                result.output_mimetypes = ""
+                result.output_str = ""
+                result.status = ArchiveResult.StatusChoices.QUEUED
+                requeue_snapshot = True
+            else:
+                result.output_files = output_files
+                result.output_size = output_size
+                result.output_mimetypes = output_mimetypes
+                result.output_str = process.stderr if process.exit_code not in (0, None) else ""
+                result.status = (
+                    ArchiveResult.StatusChoices.FAILED
+                    if process.exit_code not in (0, None)
+                    else (ArchiveResult.StatusChoices.SUCCEEDED if output_files else ArchiveResult.StatusChoices.NORESULTS)
+                )
+            result.save(
+                update_fields=[
+                    "process",
+                    "output_files",
+                    "output_size",
+                    "output_mimetypes",
+                    "output_str",
+                    "status",
+                    "modified_at",
+                ],
+            )
+            if requeue_snapshot:
+                Snapshot.objects.filter(id=snapshot.id).update(retry_at=now, modified_at=now)
+        if created:
+            cleaned["archiveresults_missing_for_orphaned_hook_processes"] += 1
+            Snapshot.objects.filter(id=snapshot.id).update(retry_at=now, modified_at=now)
    started_snapshots = Snapshot.objects.filter(status=Snapshot.StatusChoices.STARTED).filter(
        Q(retry_at__isnull=True) | Q(retry_at__gt=now),
+        **snapshot_filter,
    )

    # Broken lock repair: STARTED + retry_at=NULL or retry_at in the future
@@ -107,6 +228,7 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
    # are already final.
    recoverable_started_crawls = Crawl.objects.filter(status=Crawl.StatusChoices.STARTED).filter(
        Q(retry_at__isnull=True) | Q(retry_at__gt=now),
+        **crawl_filter,
    )

    due_started_crawls = recoverable_started_crawls.annotate(has_due_child=Exists(due_child_snapshots)).filter(has_due_child=True)
@@ -4,12 +4,14 @@ import re
 from typing import Any
 from urllib.parse import urlparse

+from django.utils.http import url_has_allowed_host_and_scheme
+
 from archivebox.config.common import get_config


 _SNAPSHOT_ID_RE = re.compile(r"^[0-9a-fA-F-]{8,36}$")
 _SNAPSHOT_SUBDOMAIN_RE = re.compile(r"^snap-(?P<suffix>[0-9a-fA-F]{12})$")
-_ROLE_SUBDOMAIN_LABELS = ("admin", "web", "api", "public")
+_ROLE_SUBDOMAIN_LABELS = ("admin", "web", "api")


 def split_host_port(host: str) -> tuple[str, str | None]:
@@ -102,7 +104,7 @@ def _with_port(host: str, port: str | None) -> str:


 def strip_role_subdomain(host: str) -> str:
-    """Strip leading ``admin.`` / ``web.`` / ``api.`` / ``public.`` / ``snap-*.``
+    """Strip leading ``admin.`` / ``web.`` / ``api.`` / ``snap-*.``
    labels from a host (preserving the port). Strips repeatedly so an
    already-compounded host like ``snap-X.snap-X.<base>`` reduces all the
    way down to ``<base>``.
@@ -172,13 +174,12 @@ def get_base_url(request=None, config: dict[str, Any] | None = None, **config_kw
            return f"{scheme}://{_with_port('archivebox.localhost', req_port)}"
        # C) Per-request fallback: when ``BASE_URL`` is unset and CSRF didn't
        # give us a single origin, trust the request's Host header — but first
-        # peel off any ``admin.`` / ``web.`` / ``api.`` / ``public.`` /
-        # ``snap-*.`` label. Otherwise the URL builders below prepend their own
-        # role label onto a host that already carries one, producing the
-        # ``snap-X.snap-X.snap-X.<base>`` compounding bug. Django has already
-        # admitted the host via ALLOWED_HOSTS; the misconfig banner surfaces
-        # the case where the resulting URL doesn't match what the operator
-        # probably intended.
+        # peel off any ``admin.`` / ``web.`` / ``api.`` / ``snap-*.`` label.
+        # Otherwise the URL builders below prepend their own role label onto a
+        # host that already carries one, producing the ``snap-X.snap-X.snap-X``
+        # compounding bug. Django has already admitted the host via
+        # ALLOWED_HOSTS; the misconfig banner surfaces the case where the
+        # resulting URL doesn't match what the operator probably intended.
        canonical_host = strip_role_subdomain(request.get_host())
        return f"{scheme}://{canonical_host}"

@@ -220,13 +221,6 @@ def get_api_host(config: dict[str, Any] | None = None, **config_kwargs: Any) ->
    return _build_base_host("api", config=config)


-def get_public_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return get_base_host(config=config)
-    return _build_base_host("public", config=config)
-
-
 def get_snapshot_subdomain(snapshot_id: str) -> str:
    normalized = re.sub(r"[^0-9a-fA-F]", "", snapshot_id or "")
    suffix = (normalized[-12:] if len(normalized) >= 12 else normalized).lower()
@@ -292,6 +286,54 @@ def host_matches(request_host: str, target_host: str) -> bool:
    return True


+def is_allowed_archivebox_redirect_url(url: str | None, request=None, config: dict[str, Any] | None = None) -> bool:
+    """Allow login redirects to this host or to the explicit ArchiveBox BASE_URL family.
+
+    Django already rejects arbitrary absolute ``next=`` URLs for the current host.
+    ArchiveBox also has legitimate cross-subdomain handoffs (admin/web/api/snap-*),
+    but only when BASE_URL is pinned. Without an explicit BASE_URL we do not widen
+    trust based on a request Host header that may be user supplied.
+    """
+    if not url:
+        return False
+
+    require_https = bool(request and request.is_secure())
+    request_host = request.get_host() if request else ""
+    if url_has_allowed_host_and_scheme(url=url, allowed_hosts={request_host}, require_https=require_https):
+        return True
+
+    parsed = urlparse(url)
+    if not parsed.netloc:
+        return False
+    if parsed.scheme not in {"http", "https"}:
+        return False
+    if require_https and parsed.scheme != "https":
+        return False
+    if parsed.username or parsed.password:
+        return False
+
+    config = config or get_config()
+    base_url = _normalize_base_url(config.BASE_URL)
+    if not base_url:
+        return False
+
+    base_host, base_port = split_host_port(urlparse(base_url).netloc)
+    if not base_host:
+        return False
+
+    redirect_host = (parsed.hostname or "").lower()
+    if redirect_host != base_host and not redirect_host.endswith(f".{base_host}"):
+        return False
+
+    try:
+        redirect_port = str(parsed.port) if parsed.port else None
+    except ValueError:
+        return False
+    if base_port:
+        return redirect_port == base_port
+    return redirect_port is None
+
+
 def _scheme_from_request(request=None, config: dict[str, Any] | None = None) -> str:
    config = config or get_config()
    override = _normalize_base_url(config.BASE_URL)
@@ -330,13 +372,6 @@ def get_api_base_url(request=None, config: dict[str, Any] | None = None, **confi
    return _build_base_url_for_host(_build_base_host("api", request=request, config=config), request=request, config=config)


-def get_public_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return get_base_url(request=request, config=config)
-    return _build_base_url_for_host(_build_base_host("public", request=request, config=config), request=request, config=config)
-
-
 def get_snapshot_base_url(snapshot_id: str, request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
    config = config or get_config(**config_kwargs)
    if not config.USES_SUBDOMAIN_ROUTING:
@@ -13,7 +13,7 @@ import archivebox

 from archivebox.config.constants import CONSTANTS
 from archivebox.config.common import get_config
-from archivebox.core.routes_util import normalize_base_url, get_admin_base_url, get_api_base_url
+from archivebox.core.routes_util import get_api_base_url, get_admin_base_url, get_base_url, normalize_base_url
 from .settings_logging import SETTINGS_LOGGING


@@ -72,15 +72,19 @@ INSTALLED_APPS = [
    "archivebox.crawls",  # handles Crawl and CrawlSchedule models and management (depends on core)
    "archivebox.progressmonitor",  # live progress endpoint and admin monitor template
    "archivebox.api",  # Django-Ninja-based Rest API interfaces, config, APIToken model, etc.
+    "abx_plugins.plugins.opencode",
    # 3rd-party apps from PyPI that need to be loaded last
    "admin_data_views",  # handles rendering some convenient automatic read-only views of data in Django admin
    "django_extensions",  # provides Django Debug Toolbar (and other non-debug helpers)
 ]

+DJANGO_OBJECT_ACTIONS_DEFAULT_HTTP_METHOD = "POST"
+

 MIDDLEWARE = [
    "archivebox.core.middleware.TimezoneMiddleware",
    "django.middleware.security.SecurityMiddleware",
+    "archivebox.core.middleware.AdminCookieIsolationMiddleware",
    "django.contrib.sessions.middleware.SessionMiddleware",
    "django.middleware.common.CommonMiddleware",
    "archivebox.api.middleware.ApiCorsMiddleware",
@@ -416,13 +420,28 @@ SECURE_BROWSER_XSS_FILTER = True
 SECURE_CONTENT_TYPE_NOSNIFF = True
 SECURE_REFERRER_POLICY = "strict-origin-when-cross-origin"

-CSRF_COOKIE_SECURE = False
-SESSION_COOKIE_SECURE = False
+# When BASE_URL is an https:// URL the deployment is HTTPS end-to-end, typically
+# behind a TLS-terminating proxy/tunnel (the bundled traefik/cloudflared profiles,
+# or your own caddy/traefik/nginx) where the proxy -> archivebox hop is plain HTTP, so
+# request.is_secure() / request.scheme would otherwise report http. Honour the
+# proxy's X-Forwarded-Proto so request-derived schemes are correct, and mark the
+# admin session + CSRF cookies Secure so auth cookies are never sent in cleartext.
+# Derived from the RESOLVED base URL's scheme — no separate flag to keep in sync.
+# get_base_url() also covers deployments that only set CSRF_TRUSTED_ORIGINS (the
+# implicit-BASE_URL fallback used on 0.7.x->0.9.x upgrades), so HTTPS hardening
+# isn't lost until BASE_URL is migrated. A plain-http base (e.g. local
+# http://archivebox.localhost:8000) keeps the defaults below.
+BASE_URL_IS_HTTPS = get_base_url(config=CONFIG).strip().lower().startswith("https://")
+if BASE_URL_IS_HTTPS:
+    SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
+
+CSRF_COOKIE_SECURE = BASE_URL_IS_HTTPS
+SESSION_COOKIE_SECURE = BASE_URL_IS_HTTPS
 SESSION_COOKIE_HTTPONLY = True
 SESSION_COOKIE_NAME = f"archivebox_sessionid_{CONSTANTS.COLLECTION_ID}"
 CSRF_COOKIE_NAME = f"archivebox_csrftoken_{CONSTANTS.COLLECTION_ID}"
 # Auth cookies are intentionally scoped to the exact host that set them so
-# the admin session is NOT readable from public.* / web.* / api.* — that
+# the admin session is NOT readable from web.* / api.* — that
 # split is a security boundary, not a UX choice. Subdomains that need to
 # render an "is the user logged in?" indicator must use the single-bit
 # `archivebox_admin_logged_in` hint cookie set by core/middleware.py
@@ -0,0 +1,25 @@
+__package__ = "archivebox.core"
+
+from django.db.models import QuerySet
+
+
+def snapshot_status_values() -> tuple[str, ...]:
+    from archivebox.core.models import Snapshot
+
+    return tuple(Snapshot.StatusChoices.values)
+
+
+def normalize_snapshot_status(status: str | None) -> str | None:
+    value = str(status or "").strip().lower()
+    if not value:
+        return None
+
+    valid_statuses = snapshot_status_values()
+    if value not in valid_statuses:
+        raise ValueError(f"Invalid snapshot status: {status}. Expected one of: {', '.join(valid_statuses)}")
+    return value
+
+
+def filter_snapshots_by_status(queryset: QuerySet, status: str | None) -> QuerySet:
+    value = normalize_snapshot_status(status)
+    return queryset.filter(status=value) if value else queryset
@@ -12,6 +12,7 @@ from django.http import HttpRequest
 from django.urls import reverse

 from archivebox.config.common import get_config
+from archivebox.misc.util import sanitize_html_text
 from archivebox.core.routes_util import build_snapshot_url, build_web_url
 from archivebox.core.models import Snapshot, SnapshotTag, Tag

@@ -33,7 +34,7 @@ TAG_HAS_SNAPSHOTS_CHOICES = (


 def normalize_tag_name(name: str) -> str:
-    return (name or "").strip()
+    return sanitize_html_text(name).strip()


 def normalize_tag_sort(sort: str = "created_desc") -> str:
@@ -114,9 +115,9 @@ def add_snapshot_counts(tags: list[Tag], snapshot_queryset: QuerySet[Snapshot] |
    queryset = SnapshotTag.objects.filter(tag_id__in=tag_ids)
    if snapshot_queryset is not None:
        queryset = queryset.filter(snapshot_id__in=snapshot_queryset.values("id"))
-    counts = {row["tag_id"]: row["num_snapshots"] for row in queryset.values("tag_id").annotate(num_snapshots=Count("snapshot_id"))}
+    counts = {int(row["tag_id"]): row["num_snapshots"] for row in queryset.values("tag_id").annotate(num_snapshots=Count("snapshot_id"))}
    for tag in tags:
-        tag.num_snapshots = counts.get(tag.pk, 0)
+        tag.num_snapshots = counts.get(int(tag.pk), 0)


 def get_tag_creator_choices() -> list[tuple[str, str]]:
@@ -18,10 +18,18 @@ def runtime_stack_owner_types():
    from archivebox.machine.models import Process

    return (
-        Process.TypeChoices.UPDATE,
        Process.TypeChoices.SERVER,
        Process.TypeChoices.ORCHESTRATOR,
+    )
+
+
+def foreground_runner_owner_types():
+    from archivebox.machine.models import Process
+
+    return (
+        Process.TypeChoices.SERVER,
        Process.TypeChoices.ADD,
+        Process.TypeChoices.UPDATE,
    )


@@ -70,15 +78,13 @@ def runtime_stack_owner(*, data_dir: str | Path, exclude_id=None):
    if exclude_id is not None:
        base_qs = base_qs.exclude(id=exclude_id)

-    top_level_types = (
-        Process.TypeChoices.UPDATE,
-        Process.TypeChoices.SERVER,
-        Process.TypeChoices.ADD,
-    )
    for qs in (
-        base_qs.filter(process_type__in=top_level_types),
+        # Only server parents own HTTP runtime leadership. Foreground add/update
+        # commands can own runner/sonic components, but server startup must never
+        # wait behind them before binding Daphne.
+        base_qs.filter(process_type=Process.TypeChoices.SERVER),
        # A foreground `archivebox run` process is allowed to own the runtime
-        # stack when no server/update/add parent is alive. A runner launched by
+        # stack when no server/add parent is alive. A runner launched by
        # supervisord is only a child worker; after its parent is killed it must
        # not keep stealing leadership from the next foreground command.
        base_qs.filter(process_type=Process.TypeChoices.ORCHESTRATOR).exclude(parent__process_type=Process.TypeChoices.SUPERVISORD),
@@ -95,6 +101,30 @@ def command_owns_runtime_stack(command, *, data_dir: str | Path) -> bool:
    return bool(owner and owner.id == command.id)


+def foreground_runner_owner(*, data_dir: str | Path, exclude_id=None):
+    from archivebox.machine.models import Machine, Process
+
+    machine = Machine.current()
+    qs = Process.objects.filter(
+        machine=machine,
+        status=Process.StatusChoices.RUNNING,
+        pwd=str(data_dir),
+        process_type__in=foreground_runner_owner_types(),
+    )
+    if exclude_id is not None:
+        qs = qs.exclude(id=exclude_id)
+    for proc in qs.order_by("-created_at", "-modified_at").iterator(chunk_size=50):
+        if proc.is_running:
+            return proc
+        proc.mark_exited(exit_code=proc.exit_code if proc.exit_code is not None else 0)
+    return None
+
+
+def command_owns_foreground_runner(command, *, data_dir: str | Path) -> bool:
+    owner = foreground_runner_owner(data_dir=data_dir)
+    return bool(owner and owner.id == command.id)
+
+
 def runtime_stack_component_label(*, owner=None, data_dir: str | Path) -> str:
    try:
        from archivebox.config.common import get_config
@@ -108,9 +138,9 @@ def runtime_stack_component_label(*, owner=None, data_dir: str | Path) -> str:
    if not names and owner is not None:
        from archivebox.machine.models import Process

-        if owner.process_type in {Process.TypeChoices.SERVER, Process.TypeChoices.ADD}:
+        if owner.process_type == Process.TypeChoices.SERVER:
            names = ["orchestrator", "server"]
-        elif owner.process_type in {Process.TypeChoices.UPDATE, Process.TypeChoices.ORCHESTRATOR}:
+        elif owner.process_type == Process.TypeChoices.ORCHESTRATOR:
            names = ["orchestrator"]

    return ", ".join(dict.fromkeys(names)) or "runtime stack"
@@ -292,3 +322,26 @@ def standby_until_runtime_stack_needed(command, *, data_dir: str | Path, interva
    command.modified_at = timezone.now()
    command.save(update_fields=["modified_at"])
    return {"resumed": announced, "previous_owner_pid": previous_owner_pid}
+
+
+def standby_until_foreground_runner_needed(command, *, data_dir: str | Path, interval: float = 2.0) -> dict[str, object]:
+    from archivebox.workers.supervisord_util import reap_foreground_supervisord_process
+
+    announced = False
+    previous_owner_pid = None
+    while not command_owns_foreground_runner(command, data_dir=data_dir):
+        reap_foreground_supervisord_process()
+        if not announced:
+            owner = foreground_runner_owner(data_dir=data_dir)
+            owner_pid = owner.pid if owner else "unknown"
+            previous_owner_pid = owner_pid
+            rprint(
+                f"[yellow][*] A newer archivebox process took over the orchestrator, sonic "
+                f"(pid={owner_pid}). Work will continue there, and will resume here if that process exits and work still remains.[/yellow]",
+                file=sys.stderr,
+            )
+            announced = True
+        time.sleep(interval)
+    command.modified_at = timezone.now()
+    command.save(update_fields=["modified_at"])
+    return {"resumed": announced, "previous_owner_pid": previous_owner_pid}
@@ -4,6 +4,9 @@ from django import template
 from django.contrib.admin.templatetags.base import InclusionAdminNode
 from django.utils.safestring import mark_safe
 from django.utils.html import escape
+from django.templatetags.static import static
+from django.utils import timezone
+from django.utils.text import Truncator

 from pathlib import Path

@@ -17,7 +20,6 @@ from archivebox.plugins.discovery import (
 from archivebox.core.routes_util import (
    canonical_base_host_for_request,
    get_admin_base_url,
-    get_public_base_url,
    get_web_base_url,
    get_snapshot_base_url,
    build_snapshot_url,
@@ -495,11 +497,6 @@ def web_base_url(context) -> str:
    return get_web_base_url(request=context.get("request"), config=context.get("CONFIG"))


-@register.simple_tag(takes_context=True)
-def public_base_url(context) -> str:
-    return get_public_base_url(request=context.get("request"), config=context.get("CONFIG"))
-
-
@register.simple_tag(takes_context=True)
 def snapshot_base_url(context, snapshot) -> str:
    snapshot_id = _snapshot_id(snapshot)
@@ -512,6 +509,202 @@ def snapshot_url(context, snapshot, path: str = "") -> str:
    return build_snapshot_url(str(snapshot_id), path, request=context.get("request"), config=context.get("CONFIG"))


+@register.simple_tag(takes_context=True)
+def snapshot_archiveresult_url(context, snapshot, plugin: str, filename: str) -> str:
+    snapshot_id = str(_snapshot_id(snapshot))
+    url_cache = snapshot.__dict__.setdefault("_snapshot_archiveresult_url_cache", {})
+    cache_key = (plugin, filename)
+    if cache_key in url_cache:
+        return url_cache[cache_key]
+
+    results = None
+    if "_admin_archiveresults" in snapshot.__dict__:
+        results = snapshot.__dict__["_admin_archiveresults"]
+    elif "archiveresult_set" in snapshot.__dict__.get("_prefetched_objects_cache", {}):
+        results = snapshot.archiveresult_set.all()
+    elif "_snapshot_archiveresult_url_results" in snapshot.__dict__:
+        results = snapshot.__dict__["_snapshot_archiveresult_url_results"]
+
+    if results is None:
+        from archivebox.core.models import ArchiveResult
+
+        results = list(
+            ArchiveResult.objects.filter(
+                snapshot_id=snapshot_id,
+                plugin__in=("screenshot", "chrome_extension_screenshot", "favicon"),
+                status=ArchiveResult.StatusChoices.SUCCEEDED,
+            ).only(
+                "plugin",
+                "status",
+                "output_files",
+            ),
+        )
+        snapshot.__dict__["_snapshot_archiveresult_url_results"] = results
+
+    for result in results:
+        if result.plugin != plugin or result.status != "succeeded":
+            continue
+        output_files = result.output_files or {}
+        file_info = output_files.get(filename)
+        if not isinstance(file_info, dict) or int(file_info.get("size") or 0) <= 0:
+            continue
+        output_path = filename if file_info.get("root_relative") else f"{plugin}/{filename}"
+        url_cache[cache_key] = build_snapshot_url(snapshot_id, output_path, request=context.get("request"), config=context.get("CONFIG"))
+        return url_cache[cache_key]
+
+    url_cache[cache_key] = ""
+    return ""
+
+
+@register.simple_tag(takes_context=True)
+def snapshot_index_row(context, link) -> str:
+    snapshot_id = str(_snapshot_id(link))
+    request = context.get("request")
+    config = context.get("CONFIG")
+    snapshot_base = get_snapshot_base_url(snapshot_id, request=request, config=config)
+
+    status = getattr(link, "status", None) or "unknown"
+    bookmarked_at = getattr(link, "bookmarked_at", None)
+    timestamp = getattr(link, "timestamp", "")
+    if bookmarked_at:
+        bookmarked_at = timezone.localtime(bookmarked_at)
+        date_text = bookmarked_at.strftime("%Y-%m-%d")
+        time_text = bookmarked_at.strftime("%H:%M")
+        sort_value = str(bookmarked_at.timestamp())
+        title_time = f"Bookmarked: {bookmarked_at} ({timestamp})"
+    else:
+        date_text = ""
+        time_text = ""
+        sort_value = ""
+        title_time = f"Bookmarked: ({timestamp})"
+
+    url = getattr(link, "url", "") or ""
+    title = getattr(link, "title", "") or ""
+    is_pending = status in {"queued", "started", "backoff"}
+    title_text = title or ("Loading..." if is_pending else url)
+    tags_str = link.tags_str() if callable(getattr(link, "tags_str", None)) else getattr(link, "tags_str", "")
+    tag_html = "".join(f'<span class="snapshot-tag">{escape(tag)}</span>' for tag in (tags_str or "").split(",") if tag)
+    if tag_html:
+        tag_cell = f'<span class="snapshot-tags">{tag_html}</span>'
+    else:
+        tag_cell = '<span class="empty-value">...</span>'
+
+    num_outputs = int(getattr(link, "num_outputs", 0) or 0)
+    icons = link.icons() if callable(getattr(link, "icons", None)) else getattr(link, "icons", "")
+    icons_cell = str(icons) if icons else '<span class="empty-value">...</span>'
+    archive_size = int(getattr(link, "archive_size", 0) or 0)
+    size_cell = file_size(archive_size) if archive_size else '<span class="empty-value">...</span>'
+    output_plural = "" if num_outputs == 1 else "s"
+
+    if is_pending:
+        preview_html = (
+            '<span class="snapshot-preview snapshot-preview-spinner" aria-label="Archiving in progress">'
+            f'<img src="{escape(static("spinner.gif"))}" alt="" decoding="async" loading="lazy">'
+            "</span>"
+        )
+    elif "_public_preview_paths" in link.__dict__:
+        preview_paths = list(getattr(link, "_public_preview_paths", []) or [])
+        if preview_paths:
+            preview_urls = [build_snapshot_url(snapshot_id, path, request=request, config=config) for path in preview_paths]
+            preview_html = (
+                f'<img src="{escape(preview_urls[0])}" '
+                f'data-fallbacks="{escape(",".join(preview_urls[1:]))}" '
+                'onerror="nextPublicSnapshotPreview(this)" class="snapshot-preview screenshot" alt="" decoding="async" loading="lazy">'
+            )
+        else:
+            preview_html = '<span class="snapshot-preview snapshot-preview-empty" aria-label="No preview available"></span>'
+    else:
+        preview_urls = [
+            url
+            for url in (
+                snapshot_archiveresult_url(context, link, "screenshot", "screenshot.png"),
+                snapshot_archiveresult_url(context, link, "chrome_extension_screenshot", "screenshot-1.png"),
+                snapshot_archiveresult_url(context, link, "chrome_extension_screenshot", "screenshot.png"),
+            )
+            if url
+        ]
+        if preview_urls:
+            preview_html = (
+                f'<img src="{escape(preview_urls[0])}" '
+                f'data-fallbacks="{escape(",".join(preview_urls[1:]))}" '
+                'onerror="nextPublicSnapshotPreview(this)" class="snapshot-preview screenshot" alt="" decoding="async" loading="lazy">'
+            )
+        else:
+            preview_html = '<span class="snapshot-preview snapshot-preview-empty" aria-label="No preview available"></span>'
+
+    if "_public_favicon_paths" in link.__dict__:
+        favicon_paths = list(getattr(link, "_public_favicon_paths", []) or [])
+        if favicon_paths:
+            favicon_urls = [build_snapshot_url(snapshot_id, path, request=request, config=config) for path in favicon_paths]
+            favicon_html = (
+                f'<img src="{escape(favicon_urls[0])}" '
+                f'data-fallbacks="{escape(",".join(favicon_urls[1:]))}" '
+                'onerror="nextPublicSnapshotPreview(this)" '
+                'class="link-favicon" alt="" decoding="async" loading="lazy">'
+            )
+        else:
+            favicon_html = '<span class="link-favicon link-favicon-empty" aria-hidden="true"></span>'
+    else:
+        favicon_url = snapshot_archiveresult_url(context, link, "favicon", "favicon.ico")
+        favicon_html = (
+            (
+                f'<img src="{escape(favicon_url)}" '
+                'data-fallbacks="" '
+                'onerror="nextPublicSnapshotPreview(this)" '
+                'class="link-favicon" alt="" decoding="async" loading="lazy">'
+            )
+            if favicon_url
+            else '<span class="link-favicon link-favicon-empty" aria-hidden="true"></span>'
+        )
+
+    html = f"""
+<tr class="snapshot-row status-{escape(status)}">
+    <td class="snapshot-time" title="{escape(title_time)}" data-sort="{escape(sort_value)}">
+        <a href="{escape(snapshot_base)}/index.html">
+            <span>{escape(date_text)}</span>
+            <small>{escape(time_text)}</small>
+        </a>
+    </td>
+    <td class="snapshot-preview-cell">
+        <a href="{escape(snapshot_base)}/index.html" title="Open archived snapshot">
+            {preview_html}
+        </a>
+    </td>
+    <td class="snapshot-title-cell" title="{escape(title or url)}">
+        <div class="snapshot-title-line">
+            <a href="{escape(snapshot_base)}/index.html" class="snapshot-favicon-link" title="Open archived snapshot">
+                {favicon_html}
+            </a>
+            <a href="{escape(snapshot_base)}/index.html" class="snapshot-title">
+                {escape(Truncator(title_text).chars(110))}
+            </a>
+        </div>
+        <a href="{escape(url)}" class="snapshot-url" title="{escape(url)}" target="_blank" rel="noopener noreferrer">
+            {escape(url)}
+        </a>
+    </td>
+    <td class="snapshot-tags-cell">
+        {tag_cell}
+    </td>
+    <td class="snapshot-status-cell">
+        <span class="snapshot-status">{escape(status)}</span>
+    </td>
+    <td class="snapshot-files-cell">
+        <span data-number-for="{escape(url)}" title="{num_outputs} successful outputs">
+            {icons_cell}
+        </span>
+    </td>
+    <td class="snapshot-size-cell">
+        <a href="{escape(snapshot_base)}/?files=1" title="View archived files">
+            {size_cell}
+        </a>
+        <small>{num_outputs} output{output_plural}</small>
+    </td>
+</tr>
+"""
+    return mark_safe(html)
+
+
@register.simple_tag(takes_context=True)
 def snapshot_preview_url(context, snapshot, path: str = "") -> str:
    snapshot_id = _snapshot_id(snapshot)
@@ -19,6 +19,7 @@ from archivebox.core.views import (
    SnapshotView,
    SnapshotPathView,
    SnapshotReplayView,
+    SnapshotReplayAuthView,
    OriginalDomainReplayView,
    PublicIndexView,
    AddView,
@@ -26,6 +27,8 @@ from archivebox.core.views import (
    HealthCheckView,
 )
 from archivebox.progressmonitor.views import live_progress_view
+from archivebox.search.views import public_snapshot_search_stream_view
+from abx_plugins.plugins.opencode.views import opencode_proxy_view


 # GLOBAL_CONTEXT doesn't work as-is, disabled for now: https://github.com/ArchiveBox/ArchiveBox/discussions/1306
@@ -41,6 +44,9 @@ urlpatterns = [
    path("robots.txt", static.serve, {"document_root": CONSTANTS.STATIC_DIR, "path": "robots.txt"}),
    path("favicon.ico", static.serve, {"document_root": CONSTANTS.STATIC_DIR, "path": "favicon.ico"}),
    path("docs/", RedirectView.as_view(url="https://github.com/ArchiveBox/ArchiveBox/wiki"), name="Docs"),
+    re_path(r"^admin/agent/?(?=$|opencode)", include("abx_plugins.plugins.opencode.urls")),
+    re_path(r"^(?P<path>assets/.*)$", opencode_proxy_view, name="opencode-assets"),
+    path("public/search-stream/", public_snapshot_search_stream_view, name="public-search-stream"),
    path("public/", PublicIndexView.as_view(), name="public-index"),
    path("public.html", RedirectView.as_view(url="/public/"), name="public-index-html"),
    path("archive/", RedirectView.as_view(url="/")),
@@ -65,6 +71,7 @@ urlpatterns = [
        name="snapshot-path-nodate",
    ),
    path("admin/core/snapshot/add/", RedirectView.as_view(url="/add/")),
+    path("admin/core/snapshot/replay-auth/", SnapshotReplayAuthView.as_view(), name="snapshot-replay-auth"),
    path("add/", AddView.as_view(), name="add"),
    # ``query_string=True`` preserves the ``?next=…`` param that Django's
    # auth/login mixins append, so e.g. ``UserPassesTestMixin`` redirecting
@@ -11,14 +11,19 @@ from urllib.parse import quote, urlparse

 from django.shortcuts import render, redirect
 from django.http import HttpRequest, HttpResponse, Http404, HttpResponseForbidden, QueryDict
-from django.utils.html import format_html
+from django.utils.html import format_html, format_html_join
 from django.utils.safestring import mark_safe
 from django.views import View
 from django.views.generic.list import ListView
 from django.views.generic import FormView
-from django.db.models import Q
+from django.db.models import Case, IntegerField, Q, Value, When
+from django.core.paginator import InvalidPage
 from django.contrib import messages
+from django.conf import settings
+from django.contrib.auth import HASH_SESSION_KEY, SESSION_KEY, get_user_model
 from django.contrib.auth.mixins import UserPassesTestMixin
+from django.contrib.sessions.models import Session
+from django.core import signing
 from django.views.decorators.csrf import csrf_exempt
 from django.utils.decorators import method_decorator

@@ -37,16 +42,20 @@ from archivebox.config.common import (
    get_config,
    get_all_configs,
    get_request_config,
+    _plugin_config_properties,
    redact_sensitive_config,
 )
+from archivebox.config.common import PLUGIN_CONFIG_SCHEMAS
 from archivebox.config.configset import BaseConfigSet
-from archivebox.misc.paginators import CountlessPaginator
+from archivebox.misc.paginators import AcceleratedPaginator
 from archivebox.misc.util import (
    base_url,
    filter_queryset_by_uuid_substring,
-    htmlencode,
+    htmldecode,
+    sanitize_html_text,
    ts_to_date_str,
    urldecode,
+    validate_url,
    without_fragment,
 )
 from archivebox.misc.serve_static import serve_static_with_byterange_support
@@ -57,11 +66,13 @@ from archivebox.search.config import (
    get_search_mode_base,
    get_search_mode_options,
 )
-from archivebox.search.query import apply_snapshot_search
+from archivebox.search.views import get_cached_public_search_state

-from archivebox.core.models import ArchiveResult, Snapshot
+from archivebox.core.models import ArchiveResult, Snapshot, SnapshotTag
 from archivebox.core.permissions import (
+    PERMISSIONS_PRIVATE,
    PERMISSIONS_PUBLIC,
+    PERMISSIONS_UNLISTED,
    can_view_snapshot,
    direct_snapshots_queryset,
    filter_personas_by_permissions,
@@ -82,7 +93,6 @@ from archivebox.core.routes_util import (
 from archivebox.core.forms import AddLinkForm
 from archivebox.plugins.forms import get_plugin_config_binary_urls
 from archivebox.crawls.models import Crawl
-from archivebox.workers.models import RETRY_AT_MAX
 from archivebox.plugins.discovery import discover_plugin_configs
 from archivebox.plugins.views import get_config_definition_link
 from archivebox.progressmonitor.views import live_progress_view, progress_endpoint
@@ -125,6 +135,125 @@ def _admin_login_redirect_or_forbidden(request: HttpRequest):
    return HttpResponseForbidden("ArchiveBox is running with the control plane disabled in this security mode.")


+REPLAY_AUTH_SALT = "archivebox.private-snapshot-replay"
+REPLAY_COOKIE_PREFIX = f"archivebox_replay_{CONSTANTS.COLLECTION_ID}_"
+REPLAY_GRANT_MAX_AGE = 60
+
+
+def _replay_cookie_name(snapshot: Snapshot) -> str:
+    return f"{REPLAY_COOKIE_PREFIX}{str(snapshot.id).replace('-', '')[-12:]}"
+
+
+def _clean_replay_next(path: str | None) -> str:
+    """Only allow same-snap relative replay paths; grants must never redirect off-host."""
+    path = f"/{(path or 'index.html').lstrip('/')}"
+    parsed = urlparse(path)
+    if parsed.scheme or parsed.netloc or path.startswith("//"):
+        return "/index.html"
+    return path
+
+
+def _replay_payload_is_valid(payload: dict, snapshot: Snapshot) -> bool:
+    """A replay cookie is not its own auth source; it must point at a live admin session.
+
+    Replayed pages can execute hostile JS, so admin cookies stay host-only on admin.*.
+    The snap host gets only this host-only HttpOnly cookie, and every request checks
+    that the original Django session still exists and still belongs to an active staff user.
+    Logout, session expiry, user deletion/deactivation, or password auth-hash rotation all
+    make the replay cookie inert without needing admin.* to delete a cookie on snap-*.
+    """
+    if payload.get("snapshot_id") != str(snapshot.id):
+        return False
+    try:
+        session = Session.objects.get(session_key=str(payload.get("session_key") or ""))
+        session_data = session.get_decoded()
+        user_id = str(session_data.get(SESSION_KEY) or "")
+        auth_hash = str(session_data.get(HASH_SESSION_KEY) or "")
+        user = get_user_model().objects.get(pk=user_id)
+    except Exception:
+        return False
+    return (
+        str(payload.get("user_id")) == user_id
+        and str(payload.get("auth_hash") or "") == auth_hash
+        and user.is_active
+        and user.is_staff
+        and auth_hash == user.get_session_auth_hash()
+    )
+
+
+def _has_replay_cookie(request: HttpRequest, snapshot: Snapshot) -> bool:
+    value = request.COOKIES.get(_replay_cookie_name(snapshot))
+    if not value:
+        return False
+    try:
+        payload = signing.loads(value, salt=REPLAY_AUTH_SALT, max_age=settings.SESSION_COOKIE_AGE)
+    except signing.BadSignature:
+        return False
+    return isinstance(payload, dict) and _replay_payload_is_valid(payload, snapshot)
+
+
+def _private_snapshot_auth_redirect(request: HttpRequest, snapshot: Snapshot, path: str = "", *, preserve_query: bool = True):
+    next_path = _clean_replay_next(path or "index.html")
+    if preserve_query and request.META.get("QUERY_STRING"):
+        next_path = f"{next_path}?{request.META['QUERY_STRING']}"
+    target = build_admin_url(
+        f"/admin/core/snapshot/replay-auth/?snapshot={snapshot.id}&next={quote(next_path, safe='')}",
+        request=request,
+    )
+    return redirect(target)
+
+
+def _replay_auth_response(request: HttpRequest, snapshot: Snapshot):
+    try:
+        payload = signing.loads(str(request.GET.get("grant") or ""), salt=REPLAY_AUTH_SALT, max_age=REPLAY_GRANT_MAX_AGE)
+    except signing.BadSignature:
+        return _private_snapshot_auth_redirect(request, snapshot, "index.html", preserve_query=False)
+
+    if not isinstance(payload, dict) or not _replay_payload_is_valid(payload, snapshot):
+        return _private_snapshot_auth_redirect(request, snapshot, "index.html", preserve_query=False)
+
+    cookie_value = signing.dumps(payload, salt=REPLAY_AUTH_SALT)
+    response = redirect(_clean_replay_next(request.GET.get("next")))
+    response.set_cookie(
+        _replay_cookie_name(snapshot),
+        cookie_value,
+        max_age=settings.SESSION_COOKIE_AGE,
+        secure=request.is_secure(),
+        httponly=True,
+        samesite="Lax",
+    )
+    return response
+
+
+class SnapshotReplayAuthView(View):
+    """Admin-only handoff that lets a snap host mint its own replay cookie.
+
+    admin.* cannot set a host-only cookie for snap-* (browsers forbid that), and
+    widening the real Django session cookie to *.archivebox.localhost would let XSS
+    in replayed pages hit the admin UI. Instead admin.* proves the user is logged in
+    with a short URL grant, then snap-* validates it and sets a snap-host-only cookie.
+    """
+
+    def get(self, request: HttpRequest):
+        if not is_admin_user(request):
+            return redirect(f"{build_admin_url('/admin/login/', request=request)}?next={quote(request.get_full_path(), safe='')}")
+
+        snapshot = _find_snapshot_by_ref(str(request.GET.get("snapshot") or ""))
+        if not snapshot:
+            raise Http404
+
+        payload = {
+            "snapshot_id": str(snapshot.id),
+            "user_id": str(request.user.pk),
+            "session_key": request.session.session_key,
+            "auth_hash": request.user.get_session_auth_hash(),
+        }
+        grant = signing.dumps(payload, salt=REPLAY_AUTH_SALT)
+        next_path = _clean_replay_next(request.GET.get("next"))
+        target = build_snapshot_url(str(snapshot.id), "_auth", request=request, config=get_request_config(request))
+        return redirect(f"{target}?grant={quote(grant, safe='')}&next={quote(next_path, safe='')}")
+
+
 class HomepageView(View):
    def get(self, request):
        request_config = get_request_config(request)
@@ -204,7 +333,7 @@ class SnapshotView(View):
        hidden_card_plugins = {"archivedotorg", "favicon", "title"}
        outputs = [
            out
-            for out in snapshot.discover_outputs(include_filesystem_fallback=False)
+            for out in snapshot.discover_outputs(include_filesystem_fallback=True)
            if (out.get("size") or 0) > 0 and out.get("name") not in hidden_card_plugins
        ]
        archiveresults = {out["name"]: out for out in outputs}
@@ -312,7 +441,7 @@ class SnapshotView(View):
            "progress_endpoint": progress_endpoint("snapshot", snapshot.id),
            "url": snapshot.url,
            "archive_path": snapshot.archive_path_from_db,
-            "title": htmlencode(snapshot.resolved_title or (snapshot.base_url if is_archived else TITLE_LOADING_MSG)),
+            "title": htmldecode(snapshot.resolved_title or (snapshot.base_url if is_archived else TITLE_LOADING_MSG)),
            "extension": snapshot.extension or "html",
            "tags": snapshot.tags_str() or "untagged",
            "size": printable_filesize(output_size) if output_size else "—",
@@ -362,7 +491,7 @@ class SnapshotView(View):
                try:
                    snapshot = Snapshot.objects.get(Q(timestamp=slug) | Q(id__startswith=slug))
                    if not can_view_snapshot(request, snapshot):
-                        return _admin_login_redirect_or_forbidden(request)
+                        return _private_snapshot_auth_redirect(request, snapshot, archivefile or "index.html")
                    canonical_base = snapshot.url_path
                    if canonical_base != snapshot.legacy_archive_path:
                        target_path = f"/{canonical_base}/{archivefile or 'index.html'}"
@@ -442,37 +571,43 @@ class SnapshotView(View):
                # Snapshot dir exists but file within does not e.g. 124235.324234/screenshot.png
                return HttpResponse(
                    format_html(
-                        (
-                            "<html><head>"
-                            "<title>Snapshot Not Found</title>"
-                            #'<script>'
-                            #'setTimeout(() => { window.location.reload(); }, 5000);'
-                            #'</script>'
-                            "</head><body>"
-                            "<center><br/><br/><br/>"
-                            f'Snapshot <a href="/{snapshot.archive_path}/index.html" target="_top"><b><code>[{snapshot.timestamp}]</code></b></a>: <a href="{snapshot.url}" target="_blank" rel="noreferrer">{snapshot.url}</a><br/>'
-                            f"was queued on {str(snapshot.bookmarked_at).split('.')[0]}, "
-                            f'but no files have been saved yet in:<br/><b><a href="/{snapshot.archive_path}/" target="_top"><code>{snapshot.timestamp}</code></a><code>/'
-                            "{}"
-                            f"</code></b><br/><br/>"
-                            "It's possible {} "
-                            f"during the last capture on {str(snapshot.bookmarked_at).split('.')[0]},<br/>or that the archiving process has not completed yet.<br/>"
-                            f"<pre><code># run this cmd to finish/retry archiving this Snapshot</code><br/>"
-                            f'<code style="user-select: all; color: #333">archivebox update -t timestamp {snapshot.timestamp}</code></pre><br/><br/>'
-                            '<div class="text-align: left; width: 100%; max-width: 400px">'
-                            "<i><b>Next steps:</i></b><br/>"
-                            f'- list all the <a href="/{snapshot.archive_path}/" target="_top">Snapshot files <code>.*</code></a><br/>'
-                            f'- view the <a href="/{snapshot.archive_path}/index.html" target="_top">Snapshot <code>./index.html</code></a><br/>'
-                            f'- go to the <a href="/admin/core/snapshot/{snapshot.pk}/change/" target="_top">Snapshot admin</a> to edit<br/>'
-                            f'- go to the <a href="/admin/core/snapshot/?id__exact={snapshot.id}" target="_top">Snapshot actions</a> to re-archive<br/>'
-                            '- or return to <a href="/" target="_top">the main index...</a></div>'
-                            "</center>"
-                            "</body></html>"
-                        ),
+                        """
+                        <html><head>
+                        <title>Snapshot Not Found</title>
+                        </head><body>
+                        <center><br/><br/><br/>
+                        Snapshot <a href="/{}/index.html" target="_top"><b><code>[{}]</code></b></a>: <a href="{}" target="_blank" rel="noreferrer">{}</a><br/>
+                        was queued on {}, but no files have been saved yet in:<br/><b><a href="/{}/" target="_top"><code>{}</code></a><code>/{}</code></b><br/><br/>
+                        It's possible {} during the last capture on {},<br/>or that the archiving process has not completed yet.<br/>
+                        <pre><code># run this cmd to finish/retry archiving this Snapshot</code><br/>
+                        <code style="user-select: all; color: #333">archivebox update -t timestamp {}</code></pre><br/><br/>
+                        <div class="text-align: left; width: 100%; max-width: 400px">
+                        <i><b>Next steps:</i></b><br/>
+                        - list all the <a href="/{}/" target="_top">Snapshot files <code>.*</code></a><br/>
+                        - view the <a href="/{}/index.html" target="_top">Snapshot <code>./index.html</code></a><br/>
+                        - go to the <a href="/admin/core/snapshot/{}/change/" target="_top">Snapshot admin</a> to edit<br/>
+                        - go to the <a href="/admin/core/snapshot/?id__exact={}" target="_top">Snapshot actions</a> to re-archive<br/>
+                        - or return to <a href="/" target="_top">the main index...</a></div>
+                        </center>
+                        </body></html>
+                        """,
+                        snapshot.archive_path,
+                        snapshot.timestamp,
+                        snapshot.url,
+                        snapshot.url,
+                        str(snapshot.bookmarked_at).split(".")[0],
+                        snapshot.archive_path,
+                        snapshot.timestamp,
                        archivefile if str(archivefile) != "None" else "",
                        f"the {archivefile} resource could not be fetched"
                        if str(archivefile) != "None"
                        else "the original site was not available",
+                        str(snapshot.bookmarked_at).split(".")[0],
+                        snapshot.timestamp,
+                        snapshot.archive_path,
+                        snapshot.archive_path,
+                        snapshot.pk,
+                        snapshot.id,
                    ),
                    content_type="text/html",
                    status=404,
@@ -571,7 +706,7 @@ class SnapshotPathView(View):
        if snapshot_id:
            snapshot = _find_snapshot_by_ref(snapshot_id)
            if snapshot and not can_view_snapshot(request, snapshot):
-                return _admin_login_redirect_or_forbidden(request)
+                return _private_snapshot_auth_redirect(request, snapshot, path or "index.html")
        else:
            # fuzzy lookup by date + domain/url (most recent)
            username_lookup = "system" if username == "web" else username
@@ -685,6 +820,34 @@ def _safe_archive_relpath(path: str) -> str | None:
    return cleaned


+def _resolve_archiveresult_relpath(snapshot: Snapshot, rel_path: str) -> tuple[str, bool]:
+    """Resolve plugin-relative output paths through ArchiveResult.output_files."""
+    parts = Path(rel_path).parts
+    if len(parts) < 2:
+        return rel_path, False
+
+    plugin = parts[0]
+    plugin_relpath = posixpath.join(*parts[1:])
+    result = (
+        ArchiveResult.objects.filter(snapshot=snapshot, plugin=plugin, status=ArchiveResult.StatusChoices.SUCCEEDED)
+        .only("output_files")
+        .first()
+    )
+    if not result or not result.output_files:
+        return rel_path, False
+
+    output_files = result.output_files or {}
+    for candidate in (plugin_relpath, rel_path):
+        file_info = output_files.get(candidate)
+        if not isinstance(file_info, dict):
+            continue
+        if file_info.get("root_relative"):
+            return candidate, True
+        return rel_path, True
+
+    return rel_path, True
+
+
 def _coerce_sort_timestamp(value: str | float | None) -> float:
    if value is None:
        return 0.0
@@ -730,7 +893,7 @@ def _replay_path_visible(request: HttpRequest, path: Path) -> bool:
    if not snapshot_id:
        return False
    snapshot = Snapshot.objects.filter(id=snapshot_id).select_related("crawl", "crawl__created_by").first()
-    if not snapshot or not can_view_snapshot(request, snapshot):
+    if not snapshot or (not can_view_snapshot(request, snapshot) and not _has_replay_cookie(request, snapshot)):
        return False
    request.archivebox_config = get_request_config(request, resolve_plugins=False)
    return True
@@ -878,6 +1041,8 @@ def _serve_snapshot_replay(request: HttpRequest, snapshot: Snapshot, path: str =
    if rel_path is None:
        raise Http404

+    rel_path, _is_known_plugin_output = _resolve_archiveresult_relpath(snapshot, rel_path)
+
    try:
        return serve_static_with_byterange_support(
            request,
@@ -953,8 +1118,10 @@ class SnapshotHostView(View):

        if not snapshot:
            raise Http404
-        if not can_view_snapshot(request, snapshot):
-            return _admin_login_redirect_or_forbidden(request)
+        if path == "_auth":
+            return _replay_auth_response(request, snapshot)
+        if not can_view_snapshot(request, snapshot) and not _has_replay_cookie(request, snapshot):
+            return _private_snapshot_auth_redirect(request, snapshot, path)

        canonical_host = get_snapshot_host(str(snapshot.id), config=request_config)
        if not host_matches(request.get_host(), canonical_host):
@@ -973,8 +1140,10 @@ class SnapshotReplayView(View):
        snapshot = _find_snapshot_by_ref(snapshot_id)
        if not snapshot:
            raise Http404
-        if not can_view_snapshot(request, snapshot):
-            return _admin_login_redirect_or_forbidden(request)
+        if path == "_auth":
+            return _replay_auth_response(request, snapshot)
+        if not can_view_snapshot(request, snapshot) and not _has_replay_cookie(request, snapshot):
+            return _private_snapshot_auth_redirect(request, snapshot, path)

        return _serve_snapshot_replay(request, snapshot, path)

@@ -997,7 +1166,8 @@ class PublicIndexView(ListView):
    template_name = "public_index.html"
    model = Snapshot
    ordering = ["-bookmarked_at", "-created_at"]
-    paginator_class = CountlessPaginator
+    paginator_class = AcceleratedPaginator
+    public_page_scan_chunk_size = 50

    def get_paginate_by(self, queryset):
        runtime_config = self.__dict__.get("runtime_config")
@@ -1005,12 +1175,66 @@ class PublicIndexView(ListView):
            self.runtime_config = runtime_config = get_request_config(self.request, resolve_plugins=False)
        return runtime_config.SNAPSHOTS_PER_PAGE

+    def _base_public_snapshot_fields(self) -> tuple[str, ...]:
+        return (
+            "id",
+            "created_at",
+            "modified_at",
+            "url",
+            "timestamp",
+            "bookmarked_at",
+            "title",
+            "downloaded_at",
+            "status",
+            "output_size",
+            "permissions",
+        )
+
+    def _ordered_public_page_from_order_index(self, *, page_number: int, page_size: int) -> list[Snapshot] | None:
+        target_count = page_number * page_size
+        public_snapshots: list[Snapshot] = []
+        scanned = 0
+        chunk_size = max(self.public_page_scan_chunk_size, page_size)
+        ordered_snapshots = Snapshot.objects.order_by(*self.ordering).only(*self._base_public_snapshot_fields())
+
+        while len(public_snapshots) < target_count:
+            chunk = list(ordered_snapshots[scanned : scanned + chunk_size])
+            if not chunk:
+                break
+            scanned += len(chunk)
+            public_snapshots.extend(snapshot for snapshot in chunk if snapshot.permissions == PERMISSIONS_PUBLIC)
+
+        start = (page_number - 1) * page_size
+        return public_snapshots[start:target_count]
+
+    def paginate_queryset(self, queryset, page_size):
+        if self.request.GET.get("q", default="").strip():
+            return super().paginate_queryset(queryset, page_size)
+
+        public_count = self.get_exact_public_snapshot_count()
+        paginator = self.get_paginator(range(public_count), page_size)
+        page_kwarg = self.kwargs.get(self.page_kwarg)
+        page_query = self.request.GET.get(self.page_kwarg)
+        page_number = page_kwarg or page_query or 1
+
+        try:
+            page = paginator.page(page_number)
+        except InvalidPage as err:
+            raise Http404(f"Invalid page ({page_number}): {err}") from err
+
+        object_list = self._ordered_public_page_from_order_index(page_number=page.number, page_size=page_size)
+        page.object_list = object_list
+        return paginator, page, object_list, page.has_other_pages()
+
    def get_context_data(self, **kwargs):
        runtime_config = self.__dict__.get("runtime_config")
        if runtime_config is None:
            self.runtime_config = runtime_config = get_request_config(self.request, resolve_plugins=False)
        search_mode = get_search_mode(self.request.GET.get("search_mode"), config=runtime_config)
        search_mode_backend = get_search_mode_backend(search_mode, config=runtime_config)
+        query = self.request.GET.get("q", default="").strip()
+        public_search_state = self.__dict__.get("public_search_state")
+        public_search_pending = bool(query and (public_search_state is None or not public_search_state.get("done")))
        context = {
            **super().get_context_data(**kwargs),
            "VERSION": VERSION,
@@ -1020,15 +1244,20 @@ class PublicIndexView(ListView):
            "WEB_BASE_URL": build_web_url(request=self.request, config=runtime_config),
            "search_mode": search_mode,
            "search_mode_options": get_search_mode_options(config=runtime_config),
+            "public_search_stream_pending": public_search_pending,
        }
        context["show_search_index_hint"] = bool(
-            self.request.GET.get("q")
+            query
+            and not public_search_pending
            and get_search_mode_base(search_mode, config=runtime_config) == "deep"
            and search_mode_backend
            and context["paginator"].count == 0,
        )
        snapshots = list(context.get("object_list") or ())
        icons_by_snapshot: dict[str, set[str]] = {str(snapshot.id): set() for snapshot in snapshots}
+        tag_names_by_snapshot: dict[str, list[str]] = {str(snapshot.id): [] for snapshot in snapshots}
+        preview_paths_by_snapshot: dict[str, list[tuple[int, int, str]]] = {str(snapshot.id): [] for snapshot in snapshots}
+        favicon_paths_by_snapshot: dict[str, list[str]] = {str(snapshot.id): [] for snapshot in snapshots}
        progress_by_snapshot: dict[str, dict[str, int]] = {
            str(snapshot.id): {
                "total": 0,
@@ -1041,20 +1270,59 @@ class PublicIndexView(ListView):
            for snapshot in snapshots
        }
        if icons_by_snapshot:
-            for snapshot_id, plugin, status in (
+            for snapshot_id, tag_name in (
+                SnapshotTag.objects.filter(snapshot_id__in=icons_by_snapshot.keys())
+                .order_by("tag__name")
+                .values_list("snapshot_id", "tag__name")
+                .iterator(chunk_size=1000)
+            ):
+                tag_names_by_snapshot[str(snapshot_id)].append(tag_name)
+
+            preview_plugin_order = {
+                "screenshot": 0,
+                "chrome_extension_screenshot": 1,
+            }
+            preview_candidates = {
+                "screenshot": ("screenshot.png",),
+                "chrome_extension_screenshot": ("screenshot-1.png", "screenshot.png"),
+            }
+
+            def result_output_path(result: ArchiveResult, filename: str) -> str | None:
+                output_files = result.output_files or {}
+                file_info = output_files.get(filename)
+                if not isinstance(file_info, dict) or int(file_info.get("size") or 0) <= 0:
+                    return None
+                if file_info.get("root_relative"):
+                    return filename
+                return f"{result.plugin}/{filename}"
+
+            for result in (
                ArchiveResult.objects.filter(
                    snapshot_id__in=icons_by_snapshot.keys(),
                )
                .exclude(plugin="")
-                .values_list("snapshot_id", "plugin", "status")
+                .only("snapshot_id", "plugin", "status", "output_files", "output_str")
                .iterator(chunk_size=1000)
            ):
+                snapshot_id = result.snapshot_id
+                plugin = result.plugin
+                status = result.status
                snapshot_key = str(snapshot_id)
                progress = progress_by_snapshot[snapshot_key]
                progress["total"] += 1
                if status == ArchiveResult.StatusChoices.SUCCEEDED:
                    icons_by_snapshot[snapshot_key].add(plugin)
                    progress["succeeded"] += 1
+                    if plugin in preview_candidates:
+                        plugin_rank = preview_plugin_order[plugin]
+                        for filename_rank, filename in enumerate(preview_candidates[plugin]):
+                            output_path = result_output_path(result, filename)
+                            if output_path:
+                                preview_paths_by_snapshot[snapshot_key].append((plugin_rank, filename_rank, output_path))
+                    elif plugin == "favicon":
+                        output_path = result_output_path(result, "favicon.ico")
+                        if output_path:
+                            favicon_paths_by_snapshot[snapshot_key].append(output_path)
                elif status == ArchiveResult.StatusChoices.FAILED:
                    progress["failed"] += 1
                elif status == ArchiveResult.StatusChoices.STARTED:
@@ -1068,41 +1336,41 @@ class PublicIndexView(ListView):
            snapshot._icons_compact = True
            snapshot._icons_archive_results = icons_by_snapshot.get(str(snapshot.id), set())
            snapshot._icons_progress_stats = progress_by_snapshot.get(str(snapshot.id), {})
+            snapshot.num_outputs_cached = snapshot._icons_progress_stats.get("succeeded", 0)
+            snapshot._tags_str_cached = ",".join(tag_names_by_snapshot.get(str(snapshot.id), []))
+            snapshot._public_preview_paths = [
+                output_path for _plugin_rank, _filename_rank, output_path in sorted(preview_paths_by_snapshot.get(str(snapshot.id), []))
+            ]
+            snapshot._public_favicon_paths = favicon_paths_by_snapshot.get(str(snapshot.id), [])
            snapshot._is_archived_cached = bool(snapshot.downloaded_at or snapshot.status == Snapshot.StatusChoices.SEALED)
        context["object_list"] = snapshots
        return context

+    def get_exact_public_snapshot_count(self) -> int:
+        hidden_count = Snapshot.objects.filter(permissions=PERMISSIONS_PRIVATE).count()
+        hidden_count += Snapshot.objects.filter(permissions=PERMISSIONS_UNLISTED).count()
+        return Snapshot.objects.count() - hidden_count
+
    def get_queryset(self, **kwargs):
-        qs = (
-            public_snapshots_queryset(super().get_queryset(**kwargs))
-            .select_related("crawl__created_by")
-            .annotate(
-                num_outputs_cached=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.SUCCEEDED),
-            )
-            .prefetch_related(
-                "tags",
-            )
-        )
+        qs = public_snapshots_queryset(super().get_queryset(**kwargs)).only(*self._base_public_snapshot_fields())
        query = self.request.GET.get("q", default="").strip()

        if not query:
            return qs

-        runtime_config = self.__dict__.get("runtime_config")
-        search_mode = get_search_mode(self.request.GET.get("search_mode"), config=runtime_config)
-        try:
-            return apply_snapshot_search(
-                qs,
-                query,
-                search_mode=search_mode,
-                config=runtime_config,
-                ordering=self.ordering,
-            )
-        except Exception as err:
-            print(f"[!] Error while using search backend: {err.__class__.__name__} {err}")
-            if get_search_mode_backend(search_mode, config=runtime_config):
+        cached_state = get_cached_public_search_state(self.request)
+        self.public_search_state = cached_state
+        if cached_state is not None:
+            cached_ids = cached_state.get("ids") or []
+            if not cached_ids:
                return qs.none()
-            return apply_snapshot_search(qs, query, search_mode="meta", config=runtime_config)
+            search_rank = Case(
+                *(When(pk=snapshot_id, then=Value(index)) for index, snapshot_id in enumerate(cached_ids)),
+                output_field=IntegerField(),
+            )
+            return qs.filter(pk__in=cached_ids).annotate(search_rank=search_rank).order_by("search_rank", *self.ordering)
+
+        return qs.none()

    def get(self, *args, **kwargs):
        if self.request.user.is_authenticated:
@@ -1137,7 +1405,8 @@ class AddView(UserPassesTestMixin, FormView):
        return get_request_config(self.request).PUBLIC_ADD_VIEW or self.request.user.is_authenticated

    def _can_override_crawl_config(self) -> bool:
-        return is_admin_user(self.request)
+        user = self.request.user
+        return bool(user.is_authenticated and user.is_active and user.is_superuser)

    def _get_custom_config_overrides(self, form: AddLinkForm) -> dict:
        custom_config = form.cleaned_data.get("config") or {}
@@ -1212,8 +1481,22 @@ class AddView(UserPassesTestMixin, FormView):
        }

    def _create_crawl_from_form(self, form, *, created_by_id=None) -> Crawl:
-        urls = form.cleaned_data["url"]
-        print(f"[+] Adding URL: {urls}")
+        from archivebox.cli.archivebox_add import add
+
+        urls_input = form.cleaned_data["url"]
+        urls = urls_input
+        submitted_lines = [line.strip() for line in urls_input.splitlines() if line.strip()]
+        if len(submitted_lines) == 1:
+            try:
+                # A lone URL pasted into /add/ is the same user-facing input as
+                # `archivebox add https://...`: queue that URL directly so a
+                # narrow plugin selection like `wget` can archive it without
+                # also needing parser plugins. Multi-line or formatted text
+                # remains verbatim import content for the internal parser root.
+                urls = [validate_url(submitted_lines[0])]
+            except ValueError:
+                pass
+        print(f"[+] Adding URL: {urls_input}")

        # Extract all form fields
        tag = form.cleaned_data["tag"]
@@ -1244,8 +1527,6 @@ class AddView(UserPassesTestMixin, FormView):
        if persona:
            persona.ensure_dirs()

-        from archivebox.config.permissions import HOSTNAME
-
        if created_by_id is None:
            if self.request.user.is_authenticated:
                created_by_id = self.request.user.pk
@@ -1254,60 +1535,43 @@ class AddView(UserPassesTestMixin, FormView):

                created_by_id = get_or_create_system_user_pk()

-        created_by_name = self.request.user.username if self.request.user.is_authenticated else "web"
-
-        # 1. save the provided urls to sources/2024-11-05__23-59-59__web_ui_add_by_user_<user_pk>.txt
-        sources_file = CONSTANTS.SOURCES_DIR / f"{timezone.now().strftime('%Y-%m-%d__%H-%M-%S')}__web_ui_add_by_user_{created_by_id}.txt"
-        sources_file.parent.mkdir(parents=True, exist_ok=True)
-        sources_file.write_text(urls if isinstance(urls, str) else "\n".join(urls))
-
-        # 2. create a new Crawl with the URLs from the file
-        timestamp = timezone.now().strftime("%Y-%m-%d__%H-%M-%S")
-        urls_content = sources_file.read_text()
-        # Store explicit crawl-scoped overrides; Crawl.save() freezes them
-        # over the resolved persona/user/machine defaults at creation time.
        config = {}
-        if plugins:
-            config["PLUGINS"] = plugins
        effective_config = get_config(persona=persona) if persona else get_config()
-        if crawl_max_concurrent_snapshots != int(effective_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS):
-            config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] = crawl_max_concurrent_snapshots
        if delete_after != str(effective_config.DELETE_AFTER):
            config["DELETE_AFTER"] = delete_after
-        config["PERMISSIONS"] = permissions
-        if max_urls:
-            config["CRAWL_MAX_URLS"] = max_urls
-        if crawl_max_size:
-            config["CRAWL_MAX_SIZE"] = crawl_max_size
-        if crawl_timeout:
-            config["CRAWL_TIMEOUT"] = crawl_timeout
        if timeout is not None and int(timeout) != int(effective_config.TIMEOUT):
            config["TIMEOUT"] = int(timeout)
-        if snapshot_max_size:
-            config["SNAPSHOT_MAX_SIZE"] = snapshot_max_size
+        if permissions:
+            config["PERMISSIONS"] = permissions

-        # Merge custom config overrides
        config.update(plugin_config)
        config.update(custom_config)
        if bool(url_filters.get("only_new")) != bool(effective_config.ONLY_NEW):
            config["ONLY_NEW"] = bool(url_filters.get("only_new"))
-        if url_filters.get("allowlist"):
-            config["URL_ALLOWLIST"] = url_filters["allowlist"]
-        if url_filters.get("denylist"):
-            config["URL_DENYLIST"] = url_filters["denylist"]
-
-        crawl = Crawl.create_scheduler_row(
-            urls=urls_content,
-            max_depth=depth,
-            tags_str=tag,
-            notes=notes,
-            label=f"{created_by_name}@{HOSTNAME}{self.request.path} {timestamp}",
+        crawl, _snapshots = add(
+            urls=urls,
+            depth=depth,
+            max_urls=max_urls,
+            crawl_max_size=crawl_max_size,
+            crawl_timeout=crawl_timeout,
+            snapshot_max_size=snapshot_max_size,
+            crawl_max_concurrent_snapshots=crawl_max_concurrent_snapshots,
+            tag=tag,
+            url_allowlist=url_filters.get("allowlist") or "",
+            url_denylist=url_filters.get("denylist") or "",
+            plugins=plugins,
+            persona=persona.name if persona else "Default",
+            bg=True,
            created_by_id=created_by_id,
            config=config,
-            persona_id=persona.id if persona else None,
-            status=Crawl.StatusChoices.PAUSED if start_paused else Crawl.StatusChoices.QUEUED,
-            retry_at=RETRY_AT_MAX if start_paused else timezone.now(),
        )
+        if notes:
+            crawl.safe_update({"notes": sanitize_html_text(notes)}, refresh=False)
+        if permissions and crawl.config.get("PERMISSIONS") != permissions:
+            next_config = {**crawl.config, "PERMISSIONS": permissions}
+            crawl.safe_update({"config": next_config}, refresh=True)
+        if start_paused:
+            crawl.pause()

        # 3. create a CrawlSchedule if schedule is provided
        if schedule:
@@ -1325,11 +1589,6 @@ class AddView(UserPassesTestMixin, FormView):
            crawl.schedule = crawl_schedule
            crawl.safe_update({"schedule": crawl_schedule}, refresh=False)

-        if not start_paused:
-            from archivebox.services.runner import ensure_background_runner
-
-            ensure_background_runner()
-
        return crawl

    def form_valid(self, form):
@@ -1339,15 +1598,17 @@ class AddView(UserPassesTestMixin, FormView):
        schedule = form.cleaned_data.get("schedule", "").strip()
        rough_url_count = len([url for url in urls.splitlines() if url.strip()])

-        # Build success message with schedule link if created
        schedule_msg = ""
        if schedule and crawl.schedule_id:
-            schedule_msg = f" and <a href='{crawl.schedule.admin_change_url}'>scheduled to repeat {schedule}</a>"
+            schedule_msg = format_html(" and <a href='{}'>scheduled to repeat {}</a>", crawl.schedule.admin_change_url, schedule)

        messages.success(
            self.request,
-            mark_safe(
-                f"Created crawl with {rough_url_count} starting URL(s){schedule_msg}. Snapshots will be created and archived in the background. <a href='{crawl.admin_change_url}'>View Crawl →</a>",
+            format_html(
+                "Created crawl with {} starting URL(s){}. Snapshots will be created and archived in the background. <a href='{}'>View Crawl →</a>",
+                rough_url_count,
+                schedule_msg,
+                crawl.admin_change_url,
            ),
        )

@@ -1485,7 +1746,7 @@ def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:

            # Use merged config value (includes machine overrides)
            actual_value = merged_config.get(key, dict(section)[key])
-            rows["Value"].append(mark_safe(f"<code>{actual_value}</code>"))
+            rows["Value"].append(format_html("<code>{}</code>", actual_value))

            # Show where the value comes from
            source = find_config_source(key, merged_config)
@@ -1493,8 +1754,10 @@ def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
            rows["Source"].append(format_html('<code style="color: {}">{}</code>', source_colors.get(source, "gray"), source))

            rows["Default"].append(
-                mark_safe(
-                    f'<a href="https://github.com/search?q=repo%3AArchiveBox%2FArchiveBox+path%3Aconfig+{key}&type=code"><code style="text-decoration: underline">{find_config_default(key) or "See here..."}</code></a>',
+                format_html(
+                    '<a href="https://github.com/search?q=repo%3AArchiveBox%2FArchiveBox+path%3Aconfig+{}&type=code"><code style="text-decoration: underline">{}</code></a>',
+                    key,
+                    find_config_default(key) or "See here...",
                ),
            )
            # rows['Documentation'].append(mark_safe(f'Wiki: <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#{key.lower()}">{key}</a>'))
@@ -1508,8 +1771,10 @@ def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
        rows["Value"].append(format_html("<code>{}</code>", redact_sensitive_config(CONSTANTS_CONFIG).get(key)))
        rows["Source"].append(mark_safe('<code style="color: gray">Constant</code>'))
        rows["Default"].append(
-            mark_safe(
-                f'<a href="https://github.com/search?q=repo%3AArchiveBox%2FArchiveBox+path%3Aconfig+{key}&type=code"><code style="text-decoration: underline">{find_config_default(key) or "See here..."}</code></a>',
+            format_html(
+                '<a href="https://github.com/search?q=repo%3AArchiveBox%2FArchiveBox+path%3Aconfig+{}&type=code"><code style="text-decoration: underline">{}</code></a>',
+                key,
+                find_config_default(key) or "See here...",
            ),
        )
        # rows['Documentation'].append(mark_safe(f'Wiki: <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#{key.lower()}">{key}</a>'))
@@ -1535,15 +1800,10 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont
    sources_info = []

    # Machine config
-    machine = None
-    machine_admin_url = None
-    try:
-        machine = Machine.current()
-        machine_admin_url = f"/admin/machine/machine/{machine.id}/change/"
-        if machine.config and key in machine.config:
-            sources_info.append(("Machine", redact_sensitive_config(machine.config).get(key), "purple"))
-    except Exception:
-        pass
+    machine = Machine.current()
+    machine_admin_url = machine.admin_change_url
+    if machine.config and key in machine.config:
+        sources_info.append(("Machine", redact_sensitive_config(machine.config).get(key), "purple"))

    # Environment variable
    if key in os.environ:
@@ -1557,36 +1817,67 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont

    # Default value
    default_val = find_config_default(key)
-    if default_val:
+    if key in _plugin_config_properties(PLUGIN_CONFIG_SCHEMAS):
+        sources_info.append(("Plugin Default", default_val, "gray"))
+    elif default_val:
        sources_info.append(("Default", default_val, "gray"))

    # Final computed value
    config_source = find_config_source(key, merged_config)
    final_value = merged_config.get(key, CONFIGS.get(key, None))
-    if config_source == "Environment":
-        final_value = get_config(include_machine=False, redact_sensitive=True).model_dump(mode="json").get(key, CONFIGS.get(key, None))
    is_redacted = final_value == SENSITIVE_CONFIG_VALUE_REDACTED

    # Build sources display
-    sources_html = "<br/>".join([f'<b style="color: {color}">{source}:</b> <code>{value}</code>' for source, value, color in sources_info])
+    sources_html = format_html_join(
+        mark_safe("<br/>"),
+        '<b style="color: {}">{}:</b> <code>{}</code>',
+        ((color, source, value) for source, value, color in sources_info),
+    )

    # aliases = USER_CONFIG.get(key, {}).get("aliases", [])
    aliases = []

    if key in CONSTANTS_CONFIG:
-        section_header = mark_safe(
-            f'[CONSTANTS]   &nbsp; <b><code style="color: lightgray">{key}</code></b> &nbsp; <small>(read-only, hardcoded by ArchiveBox)</small>',
+        section_header = format_html(
+            '[CONSTANTS]   &nbsp; <b><code style="color: lightgray">{}</code></b> &nbsp; <small>(read-only, hardcoded by ArchiveBox)</small>',
+            key,
        )
    elif key in merged_config:
-        section_header = mark_safe(
-            f'data / ArchiveBox.conf &nbsp; [{find_config_section(key)}]  &nbsp; <b><code style="color: lightgray">{key}</code></b>',
+        section_header = format_html(
+            'data / ArchiveBox.conf &nbsp; [{}]  &nbsp; <b><code style="color: lightgray">{}</code></b>',
+            find_config_section(key),
+            key,
        )
    else:
-        section_header = mark_safe(
-            f'[DYNAMIC CONFIG]   &nbsp; <b><code style="color: lightgray">{key}</code></b> &nbsp; <small>(read-only, calculated at runtime)</small>',
+        section_header = format_html(
+            '[DYNAMIC CONFIG]   &nbsp; <b><code style="color: lightgray">{}</code></b> &nbsp; <small>(read-only, calculated at runtime)</small>',
+            key,
        )

    definition_url, definition_label = get_config_definition_link(key)
+    redacted_message = (
+        mark_safe(
+            '<b style="color: red">Value is redacted for your security. (Passwords, secrets, API tokens, etc. cannot be viewed in the Web UI)</b><br/><br/>',
+        )
+        if is_redacted
+        else ""
+    )
+    default_command_value = val.strip("'") if (val := find_config_default(key)) else str(final_value).strip("'")
+    machine_config_link = (
+        format_html('<br/><a href="{}">→ Edit <code>{}</code> in Machine.config for this server</a>', machine_admin_url, key)
+        if machine_admin_url
+        else ""
+    )
+    machine_config_tip = (
+        format_html(
+            '<br/><b>Tip:</b> To override <code>{}</code> on this machine, <a href="{}">edit the Machine.config field</a> and add:<br/><code>{}</code>',
+            key,
+            machine_admin_url,
+            f'{{"{key}": "your_value_here"}}',
+        )
+        if machine_admin_url and key not in CONSTANTS_CONFIG
+        else ""
+    )

    section_data = cast(
        SectionData,
@@ -1600,49 +1891,65 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont
                "Currently read from": config_source,
            },
            "help_texts": {
-                "Key": mark_safe(f"""
-                <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#{key.lower()}">Documentation</a>  &nbsp;
-                <span style="display: {"inline" if aliases else "none"}">
-                    Aliases: {", ".join(aliases)}
+                "Key": format_html(
+                    """
+                <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#{}">Documentation</a>  &nbsp;
+                <span style="display: {}">
+                    Aliases: {}
                </span>
-            """),
-                "Type": mark_safe(f'''
-                <a href="{definition_url}" target="_blank" rel="noopener noreferrer">
-                    See full definition in <code>{definition_label}</code>...
+            """,
+                    key.lower(),
+                    "inline" if aliases else "none",
+                    ", ".join(aliases),
+                ),
+                "Type": format_html(
+                    """
+                <a href="{}" target="_blank" rel="noopener noreferrer">
+                    See full definition in <code>{}</code>...
                </a>
-            '''),
-                "Value": mark_safe(f'''
-                {
-                    '<b style="color: red">Value is redacted for your security. (Passwords, secrets, API tokens, etc. cannot be viewed in the Web UI)</b><br/><br/>'
-                    if is_redacted
-                    else ""
-                }
+            """,
+                    definition_url,
+                    definition_label,
+                ),
+                "Value": format_html(
+                    """
+                {}
                <br/><hr/><br/>
                <b>Configuration Sources (highest priority first):</b><br/><br/>
-                {sources_html}
+                {}
                <br/><br/>
-                <p style="display: {"block" if key in merged_config and key not in CONSTANTS_CONFIG else "none"}">
+                <p style="display: {}">
                    <i>To change this value, edit <code>data/ArchiveBox.conf</code> or run:</i>
                    <br/><br/>
-                    <code>archivebox config --set {key}="{
-                    val.strip("'") if (val := find_config_default(key)) else str(final_value).strip("'")
-                }"</code>
+                    <code>archivebox config --set {}="{}"</code>
                </p>
-            '''),
-                "Currently read from": mark_safe(f"""
-                The value shown in the "Value" field comes from the <b>{config_source}</b> source.
+            """,
+                    redacted_message,
+                    sources_html,
+                    "block" if key in merged_config and key not in CONSTANTS_CONFIG else "none",
+                    key,
+                    default_command_value,
+                ),
+                "Currently read from": format_html(
+                    """
+                The value shown in the "Value" field comes from the <b>{}</b> source.
                <br/><br/>
                Priority order (highest to lowest):
                <ol>
-                    <li><b style="color: blue">Environment</b> - Environment variables</li>
                    <li><b style="color: purple">Machine</b> - Machine-specific overrides
-                        {f'<br/><a href="{machine_admin_url}">→ Edit <code>{key}</code> in Machine.config for this server</a>' if machine_admin_url else ""}
+                        {}
                    </li>
-                    <li><b style="color: green">Config File</b> - data/ArchiveBox.conf</li>
+                    <li><b style="color: blue">Environment</b> - process defaults from environment variables</li>
+                    <li><b style="color: green">File</b> - data/ArchiveBox.conf</li>
+                    <li><b style="color: gray">Plugin Default</b> - Default value from plugin config.json</li>
                    <li><b style="color: gray">Default</b> - Default value from code</li>
                </ol>
-                {f'<br/><b>Tip:</b> To override <code>{key}</code> on this machine, <a href="{machine_admin_url}">edit the Machine.config field</a> and add:<br/><code>{{"\\"{key}\\": "your_value_here"}}</code>' if machine_admin_url and key not in CONSTANTS_CONFIG else ""}
-            """),
+                {}
+            """,
+                    config_source,
+                    machine_config_link,
+                    machine_config_tip,
+                ),
            },
        },
    )
@@ -40,6 +40,16 @@ class TagEditorWidget(forms.Widget):
            normalized = f"t_{normalized}"
        return normalized

+    def _json_for_inline_script(self, value):
+        """Serialize JSON so it cannot close the surrounding inline <script> tag."""
+        return json.dumps(value).translate(
+            {
+                ord(">"): "\\u003E",
+                ord("<"): "\\u003C",
+                ord("&"): "\\u0026",
+            },
+        )
+
    def _tag_style(self, value):
        """Compute a stable pastel color style for a tag value."""
        tag = (value or "").strip().lower()
@@ -106,6 +116,8 @@ class TagEditorWidget(forms.Widget):
                </span>
            '''

+        tags_json = self._json_for_inline_script(tags)
+
        # Build the widget HTML
        html = f'''
        <div id="{widget_id}_container" class="tag-editor-container" onclick="focusTagInput_{widget_id}(event)">
@@ -128,7 +140,7 @@ class TagEditorWidget(forms.Widget):

        <script>
        (function() {{
-            var currentTags_{widget_id} = {json.dumps(tags)};
+            var currentTags_{widget_id} = {tags_json};
            var autocompleteTimeout_{widget_id} = null;

            window.focusTagInput_{widget_id} = function(event) {{
@@ -8,13 +8,14 @@ from django import forms
 from django.core.paginator import Paginator
 from django.http import JsonResponse, HttpRequest, HttpResponseBadRequest, HttpResponseNotAllowed
 from django.shortcuts import get_object_or_404, redirect
+from django.template.response import TemplateResponse
 from django.template.loader import render_to_string
 from django.urls import path, reverse
 from django.utils.html import escape, format_html, format_html_join
 from django.utils import timezone
 from django.utils.safestring import mark_safe
 from django.contrib import admin, messages
-from django.db.models import Case, CharField, Count, Q, Value, When
+from django.db.models import Count, F, Q


 from django_object_actions import action
@@ -79,16 +80,7 @@ def render_snapshots_list(snapshots_qs, request=None, crawl=None, page_size=50,
        failed_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.FAILED),
        started_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.STARTED),
        skipped_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.SKIPPED),
-        snapshot_permissions=Case(
-            When(permissions=PERMISSIONS_PUBLIC, then=Value(PERMISSIONS_PUBLIC)),
-            When(permissions=PERMISSIONS_UNLISTED, then=Value(PERMISSIONS_UNLISTED)),
-            When(permissions=PERMISSIONS_PRIVATE, then=Value(PERMISSIONS_PRIVATE)),
-            When(crawl__permissions=PERMISSIONS_PUBLIC, then=Value(PERMISSIONS_PUBLIC)),
-            When(crawl__permissions=PERMISSIONS_UNLISTED, then=Value(PERMISSIONS_UNLISTED)),
-            When(crawl__permissions=PERMISSIONS_PRIVATE, then=Value(PERMISSIONS_PRIVATE)),
-            default=Value(PERMISSIONS_PRIVATE),
-            output_field=CharField(),
-        ),
+        snapshot_permissions=F("permissions"),
    )

    page_number = request.GET.get(page_param, 1) if request is not None else 1
@@ -737,6 +729,8 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
        self.crawl_admin_base_config = request.archivebox_config
        self.stop_reason_cache = {}
        response = super().changelist_view(request, extra_context)
+        if not isinstance(response, TemplateResponse):
+            return response
        cl = response.context_data.get("cl")
        if cl is not None and not self.should_annotate_snapshot_counts(request):
            self.hydrate_visible_snapshot_counts(cl.result_list)
@@ -925,11 +919,14 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
        now = timezone.now()
        updated = 0
        batch = []
-        for crawl in queryset.only("id", "config").iterator(chunk_size=500):
+        crawls_to_update = []
+        for crawl in queryset.only("id", "config", "permissions").iterator(chunk_size=500):
+            old_permissions = crawl.permissions
            config = dict(crawl.config or {})
            config["PERMISSIONS"] = permissions
            crawl.config = config
            crawl.modified_at = now
+            crawls_to_update.append((crawl, old_permissions))
            batch.append(crawl)
            if len(batch) >= 500:
                Crawl.objects.bulk_update(batch, ["config", "modified_at"], batch_size=500)
@@ -938,9 +935,11 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
        if batch:
            Crawl.objects.bulk_update(batch, ["config", "modified_at"], batch_size=500)
            updated += len(batch)
+        for crawl, old_permissions in crawls_to_update:
+            crawl.update_child_snapshot_permissions(old_permissions, permissions)
        return updated

-    @action(label="Recrawl", description="Create a new crawl with the same settings")
+    @action(label="Recrawl", description="Create a new crawl with the same settings", methods=("POST",))
    def recrawl(self, request, obj):
        """Duplicate this crawl as a new crawl with the same URLs and settings."""

@@ -2,7 +2,6 @@ __package__ = "archivebox.crawls"

 from typing import TYPE_CHECKING, Any
 from collections.abc import Iterable, Mapping
-from io import StringIO
 import uuid
 import json
 import re
@@ -35,7 +34,7 @@ from archivebox.base_models.models import (
 )
 from archivebox.workers.models import RETRY_AT_MAX, ModelWithStateMachine, BaseStateMachine
 from archivebox.crawls.schedule_util import next_run_for_schedule, validate_schedule
-from archivebox.misc.util import validate_url, validate_url_length
+from archivebox.misc.util import parse_date, sanitize_html_text, validate_url, validate_url_length

 if TYPE_CHECKING:
    from archivebox.core.models import Snapshot
@@ -70,9 +69,14 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes):
        return str(reverse_lazy("api-1:get_any", args=[self.id]))

    def save(self, *args, **kwargs):
+        update_fields = kwargs.get("update_fields")
+        if update_fields is None or "label" in update_fields:
+            self.label = sanitize_html_text(self.label).strip()
+        if update_fields is None or "notes" in update_fields:
+            self.notes = sanitize_html_text(self.notes)
        self.schedule = (self.schedule or "").strip()
        validate_schedule(self.schedule)
-        self.label = self.label or (self.template.label if self.template else "")
+        self.label = self.label or (sanitize_html_text(self.template.label).strip() if self.template else "")
        super().save(*args, **kwargs)
        if self.template:
            self.template.safe_update(
@@ -170,10 +174,14 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
    INACTIVE_STATES = (StatusChoices.PAUSED, StatusChoices.SEALED)

    schedule_id: uuid.UUID | None
-    sm: "CrawlMachine"

    snapshot_set: models.Manager["Snapshot"]

+    if TYPE_CHECKING:
+
+        @property
+        def sm(self) -> "CrawlMachine": ...
+
    class Meta(
        ModelWithDeleteAfter.Meta,
        ModelWithOutputDir.Meta,
@@ -276,6 +284,12 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith

    def save(self, *args, **kwargs):
        update_fields = kwargs.get("update_fields")
+        if update_fields is None or "label" in update_fields:
+            self.label = sanitize_html_text(self.label).strip()
+        if update_fields is None or "notes" in update_fields:
+            self.notes = sanitize_html_text(self.notes)
+        if update_fields is None or "tags_str" in update_fields:
+            self.tags_str = ",".join(self.parse_tag_names(self.tags_str or ""))
        sync_tags = update_fields is None or "tags_str" in update_fields
        old_crawl = type(self).objects.filter(pk=self.pk).first() if self.pk else None
        previous_tag_names = set()
@@ -307,6 +321,9 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
                kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "config"]))

        super().save(*args, **kwargs)
+        old_permissions = getattr(old_crawl, "permissions", None)
+        if old_crawl is not None and old_permissions != self.permissions:
+            transaction.on_commit(lambda: self.update_child_snapshot_permissions(old_permissions, self.permissions))
        if sync_tags:
            next_tag_names = set(self.parse_tag_names(self.tags_str or ""))
            added_tag_names = next_tag_names - previous_tag_names
@@ -336,6 +353,29 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
        #         },
        #     )

+    def update_child_snapshot_permissions(self, old_permissions: str | None, new_permissions: str | None) -> int:
+        from archivebox.core.models import Snapshot
+
+        normalized_new_permissions = normalize_permissions(new_permissions)
+        now = timezone.now()
+        batch = []
+        updated = 0
+        queryset = self.snapshot_set.filter(Q(permissions=old_permissions) | Q(permissions__isnull=True)).only("id", "config")
+        for snapshot in queryset.iterator(chunk_size=500):
+            config = dict(snapshot.config or {})
+            config["PERMISSIONS"] = normalized_new_permissions
+            snapshot.config = config
+            snapshot.modified_at = now
+            batch.append(snapshot)
+            if len(batch) >= 500:
+                Snapshot.objects.bulk_update(batch, ["config", "modified_at"], batch_size=500)
+                updated += len(batch)
+                batch.clear()
+        if batch:
+            Snapshot.objects.bulk_update(batch, ["config", "modified_at"], batch_size=500)
+            updated += len(batch)
+        return updated
+
    @property
    def api_url(self) -> str:
        return str(reverse_lazy("api-1:get_crawl", args=[self.id]))
@@ -346,7 +386,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
        tag_names: list[str] = []
        seen: set[str] = set()
        for raw_tag in raw_tags:
-            tag_name = str(raw_tag or "").strip()
+            tag_name = sanitize_html_text(raw_tag).strip()
            if not tag_name:
                continue
            lowered = tag_name.lower()
@@ -473,12 +513,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
        from archivebox.core.models import Snapshot

        date_str = self.created_at.strftime("%Y%m%d")
-        first_url = ""
-        for raw_line in StringIO(self.urls or ""):
-            candidate = raw_line.strip()
-            if candidate and not candidate.startswith("#"):
-                first_url = candidate
-                break
+        first_url = next((url for url in self.get_urls_list() if url), "")
        domain = Snapshot.extract_domain_from_url(first_url) if first_url else "unknown"

        output_dir = CONSTANTS.USERS_DIR / self.created_by.username / CONSTANTS.CRAWLS_DIR_NAME / date_str / domain / str(self.id)
@@ -489,7 +524,19 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
        """Get list of URLs from urls field, filtering out comments and empty lines."""
        if not self.urls:
            return []
-        return [url.strip() for url in self.urls.split("\n") if url.strip() and not url.strip().startswith("#")]
+        return [url for _raw_line, url in self._iter_url_lines() if url]
+
+    def has_internal_input_root(self) -> bool:
+        """Return True when Crawl.urls is preserved source text, not the work queue.
+
+        The runner creates a synthetic root snapshot for raw import text so
+        parser hooks use the same Snapshot lifecycle as every other extractor.
+        In that mode the submitted text must remain in Crawl.urls forever;
+        parsed URLs live as child Snapshot rows and should not be appended back.
+        """
+        from archivebox.core.models import Snapshot
+
+        return self.snapshot_set.filter(url=Snapshot.INTERNAL_INPUT_URL, depth=0).exists()

    @staticmethod
    def normalize_domain(value: str) -> str:
@@ -558,22 +605,29 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
        except re.error:
            return False

+    def get_current_config(self, *, refresh: bool = False) -> dict[str, Any]:
+        if refresh and self.pk:
+            config = type(self).objects.filter(pk=self.pk).values_list("config", flat=True).first()
+            if config is not None:
+                self.config = config
+        return dict(self.config or {})
+
    def get_url_allowlist(self, *, use_effective_config: bool = False, snapshot=None) -> list[str]:
        if use_effective_config:
-            from archivebox.config.common import get_config
-
-            config = get_config(crawl=self, snapshot=snapshot)
+            config = self.get_current_config(refresh=True)
        else:
-            config = self.config or {}
+            config = self.get_current_config()
+        if snapshot is not None and snapshot.config:
+            config.update(snapshot.config)
        return self.split_filter_patterns(config.get("URL_ALLOWLIST", ""))

    def get_url_denylist(self, *, use_effective_config: bool = False, snapshot=None) -> list[str]:
        if use_effective_config:
-            from archivebox.config.common import get_config
-
-            config = get_config(crawl=self, snapshot=snapshot)
+            config = self.get_current_config(refresh=True)
        else:
-            config = self.config or {}
+            config = self.get_current_config()
+        if snapshot is not None and snapshot.config:
+            config.update(snapshot.config)
        return self.split_filter_patterns(config.get("URL_DENYLIST", ""))

    def url_passes_filters(self, url: str, *, snapshot=None, use_effective_config: bool = True) -> bool:
@@ -652,7 +706,14 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
                continue
            try:
                entry = json.loads(stripped)
-                entries.append((raw_line.rstrip(), str(entry.get("url", "") or "").strip()))
+                # Crawl.urls accepts plain lines and JSONL URL records. Other
+                # valid JSON values, e.g. a quoted string from hostile input,
+                # are not records and must stay inert text instead of raising
+                # during later Snapshot.save() bookkeeping.
+                if isinstance(entry, dict):
+                    entries.append((raw_line.rstrip(), str(entry.get("url", "") or "").strip()))
+                else:
+                    entries.append((raw_line.rstrip(), stripped))
            except json.JSONDecodeError:
                entries.append((raw_line.rstrip(), stripped))
        return entries
@@ -674,9 +735,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
        return len(urls)

    def remaining_url_capacity(self) -> int | None:
-        from archivebox.config.common import get_config
-
-        max_urls = int(get_config(crawl=self).CRAWL_MAX_URLS or 0)
+        max_urls = int(self._config_value(self.get_current_config(refresh=True), "CRAWL_MAX_URLS", 0) or 0)
        if max_urls <= 0:
            return None
        return max(max_urls - self.count_urls_for_limit(), 0)
@@ -686,9 +745,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
        return remaining is None or remaining > 0

    def remaining_snapshot_capacity(self) -> int | None:
-        from archivebox.config.common import get_config
-
-        max_urls = int(get_config(crawl=self).CRAWL_MAX_URLS or 0)
+        max_urls = int(self._config_value(self.get_current_config(refresh=True), "CRAWL_MAX_URLS", 0) or 0)
        if max_urls <= 0:
            return None
        return max(max_urls - self.snapshot_set.count(), 0)
@@ -918,21 +975,27 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
            List of newly created Snapshot objects
        """
        from archivebox.core.models import Snapshot, Tag
-        from archivebox.config.common import get_config
        from archivebox.misc.util import fix_url_from_markdown, sanitize_extracted_url

        if self.status == self.StatusChoices.SEALED:
            return []
+        # Internal-input crawls preserve the submitted text verbatim in
+        # Crawl.urls. The root snapshot's parser hooks are the only supported
+        # path for turning that text into child snapshots, otherwise a later
+        # runner pass could reinterpret plain URL-looking lines as direct
+        # depth-0 work and bypass format-specific metadata parsing.
+        if self.has_internal_input_root():
+            return []

        created_snapshots = []
        crawl_tag_names = self.current_tag_names()
        tags_by_name: dict[str, Tag] = {}
-        config = get_config(crawl=self)
-        only_new_urls = bool(config.ONLY_NEW)

        for line in self.urls.splitlines():
            if not line.strip():
                continue
+            config = self.get_current_config(refresh=True)
+            only_new_urls = bool(self._config_value(config, "ONLY_NEW", True))

            # Parse JSONL or plain URL
            try:
@@ -958,10 +1021,10 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
            except ValueError as err:
                print(f"[yellow][!] Skipping invalid snapshot URL: {url[:120]}... ({err})[/yellow]")
                continue
-            if Snapshot.is_archivebox_internal_url(url):
+            if Snapshot.is_archivebox_internal_url(url, config=config):
                print(f"[yellow][!] Skipping internal ArchiveBox snapshot URL: {url}[/yellow]")
                continue
-            if not self.url_passes_filters(url):
+            if not self.url_passes_filters(url, use_effective_config=False):
                continue
            if only_new_urls and Snapshot.objects.filter(url=url).exists():
                continue
@@ -1031,7 +1094,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
                        # relation without clearing any non-crawl snapshot tags.
                        Tag.objects.bulk_create(missing_tags, ignore_conflicts=True)
                        tags_by_name.update({tag.name: tag for tag in Tag.objects.filter(name__in=missing_names)})
-                snapshot.tags.add(*[tag.pk for tag_name in tag_names if (tag := tags_by_name.get(tag_name))])
+                snapshot.add_tag_ids([tag.pk for tag_name in tag_names if (tag := tags_by_name.get(tag_name))])

            # Symlink creation touches the filesystem and can be slow on remote disks.
            # Defer it until after any active DB transaction commits so SQLite does
@@ -1069,7 +1132,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
    ) -> list["Snapshot"]:
        """Create child snapshots from discovered URL records after filtering and deduping once."""
        from archivebox.core.models import Snapshot, SnapshotTag, Tag
-        from archivebox.config.common import get_config
        from archivebox.misc.util import fix_url_from_markdown, sanitize_extracted_url

        if self.status == self.StatusChoices.SEALED:
@@ -1079,30 +1141,71 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
            return []

        crawl_tag_names = self.current_tag_names()
-        config = get_config(crawl=self, snapshot=parent_snapshot)
+        config = self.get_current_config(refresh=True)
+        if parent_snapshot is not None and parent_snapshot.config:
+            config.update(parent_snapshot.config)
        allowlist = self.split_filter_patterns(config.get("URL_ALLOWLIST", ""))
        denylist = self.split_filter_patterns(config.get("URL_DENYLIST", ""))

+        def metadata_score(record: Mapping[str, Any]) -> int:
+            # Multiple parsers can discover the same URL from one import root.
+            # Keep the record with the richest user-facing metadata so generic
+            # text/HTML extraction does not erase RSS/Netscape/JSON fields.
+            return sum(bool(record.get(field)) for field in ("title", "bookmarked_at", "timestamp", "tags"))
+
        deduped_records: dict[str, Mapping[str, Any]] = {}
        for record in records:
            url = sanitize_extracted_url(fix_url_from_markdown(str(record.get("url") or "").strip()))
-            if not url or url in deduped_records:
+            if not url:
                continue
            try:
                validate_url(url)
            except ValueError as err:
                print(f"[yellow][!] Skipping invalid discovered snapshot URL: {url[:120]}... ({err})[/yellow]")
                continue
-            if Snapshot.is_archivebox_internal_url(url):
+            if Snapshot.is_archivebox_internal_url(url, config=config):
                print(f"[yellow][!] Skipping internal ArchiveBox discovered snapshot URL: {url}[/yellow]")
                continue
            if self.url_passes_compiled_filters(url, allowlist=allowlist, denylist=denylist):
-                deduped_records[url] = record
+                existing_record = deduped_records.get(url)
+                if existing_record is None or metadata_score(record) > metadata_score(existing_record):
+                    deduped_records[url] = record

        if not deduped_records:
            return []

-        existing_scope = Snapshot.objects if bool(config.ONLY_NEW) else self.snapshot_set
+        existing_in_crawl = {
+            snapshot.url: snapshot for snapshot in self.snapshot_set.prefetch_related("tags").filter(url__in=deduped_records.keys())
+        }
+        for url, snapshot in existing_in_crawl.items():
+            record = deduped_records[url]
+            update_fields = []
+            title = Snapshot._normalize_title_candidate(str(record.get("title") or "").strip()[:512], snapshot_url=url)
+            if title and (not snapshot.title or len(title) > len(snapshot.title or "")):
+                snapshot.title = title
+                update_fields.append("title")
+            bookmarked_at = None
+            try:
+                bookmarked_at = parse_date(record.get("bookmarked_at") or record.get("timestamp"))
+            except (TypeError, ValueError, OSError):
+                pass
+            if bookmarked_at and snapshot.bookmarked_at != bookmarked_at:
+                snapshot.bookmarked_at = bookmarked_at
+                update_fields.append("bookmarked_at")
+            if update_fields:
+                snapshot.save(update_fields=[*update_fields, "modified_at"])
+            tag_names = {
+                *crawl_tag_names,
+                *self.parse_tag_names(
+                    str(record.get("tags") or ""),
+                    pattern=self._config_value(config, "TAG_SEPARATOR_PATTERN", r"[,]"),
+                ),
+            }
+            if tag_names:
+                tag_ids = [Tag.objects.get_or_create(name=tag_name)[0].pk for tag_name in tag_names]
+                snapshot.add_tag_ids(tag_ids)
+
+        existing_scope = Snapshot.objects if bool(self._config_value(config, "ONLY_NEW", True)) else self.snapshot_set
        existing_urls = set(existing_scope.filter(url__in=deduped_records.keys()).values_list("url", flat=True))
        urls = [url for url in deduped_records.keys() if url not in existing_urls]
        remaining = self.remaining_snapshot_capacity()
@@ -1112,27 +1215,34 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
            return []

        now = timezone.now()
-        snapshots = [
-            Snapshot(
-                url=url,
-                timestamp=str((now + timedelta(microseconds=index)).timestamp()),
-                title=Snapshot._normalize_title_candidate(
-                    str(deduped_records[url].get("title") or "").strip()[:512],
-                    snapshot_url=url,
-                )
-                or None,
-                crawl=self,
-                parent_snapshot=parent_snapshot,
-                depth=depth,
-                status=Snapshot.StatusChoices.QUEUED,
-                retry_at=now,
-                bookmarked_at=now,
-                created_at=now,
+        snapshots = []
+        for index, url in enumerate(urls):
+            record = deduped_records[url]
+            bookmarked_at = now
+            try:
+                bookmarked_at = parse_date(record.get("bookmarked_at") or record.get("timestamp")) or now
+            except (TypeError, ValueError, OSError):
+                pass
+            snapshots.append(
+                Snapshot(
+                    url=url,
+                    timestamp=str((now + timedelta(microseconds=index)).timestamp()),
+                    title=Snapshot._normalize_title_candidate(
+                        str(record.get("title") or "").strip()[:512],
+                        snapshot_url=url,
+                    )
+                    or None,
+                    crawl=self,
+                    parent_snapshot=parent_snapshot,
+                    depth=depth,
+                    status=Snapshot.StatusChoices.QUEUED,
+                    retry_at=now,
+                    bookmarked_at=bookmarked_at,
+                    created_at=now,
+                ),
            )
-            for index, url in enumerate(urls)
-        ]
        for snapshot in snapshots:
-            snapshot.set_delete_at_from_config(config.DELETE_AFTER)
+            snapshot.set_delete_at_from_config(self._config_value(config, "DELETE_AFTER", "0"))

        created_snapshots = []
        for snapshot in snapshots:
@@ -1151,7 +1261,11 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith

        crawl_urls = {url for _raw_line, url in self._iter_url_lines() if url}
        new_url_lines = [snapshot.url for snapshot in created_snapshots if snapshot.url not in crawl_urls]
-        if new_url_lines:
+        # For internal-input crawls, Crawl.urls is the immutable source text.
+        # Child snapshots are the parsed/indexed representation, so appending
+        # discovered URLs here would both duplicate state and destroy the exact
+        # import artifact users submitted through CLI/API/UI.
+        if new_url_lines and not self.has_internal_input_root():
            self.urls = (self.urls.rstrip() + "\n" + "\n".join(new_url_lines)).lstrip("\n")
            self.save(update_fields=["urls", "modified_at"])

@@ -1159,7 +1273,10 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
        for snapshot in created_snapshots:
            tag_names = {
                *crawl_tag_names,
-                *self.parse_tag_names(str(deduped_records[snapshot.url].get("tags") or ""), pattern=config.TAG_SEPARATOR_PATTERN),
+                *self.parse_tag_names(
+                    str(deduped_records[snapshot.url].get("tags") or ""),
+                    pattern=self._config_value(config, "TAG_SEPARATOR_PATTERN", r"[,]"),
+                ),
            }
            if tag_names:
                tag_names_by_url[snapshot.url] = tag_names
@@ -1282,7 +1399,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
            The root Snapshot for this crawl, or None for system crawls that don't create snapshots
        """
        import time
-        from archivebox.plugins.hooks import run_hook, discover_hooks, process_hook_records, is_finite_background_hook
+        from archivebox.plugins.hooks import run_hook, discover_hooks, process_hook_records
        from archivebox.config.common import get_config
        from archivebox.machine.models import Binary, Machine

@@ -1340,19 +1457,15 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
                print(f"[yellow]⏱️  Hook {hook.name} took {hook_elapsed:.2f}s[/yellow]")

            if process.status == process.StatusChoices.RUNNING:
-                if not is_finite_background_hook(hook.name):
-                    return set()
-                try:
-                    process.wait(timeout=process.timeout)
-                except Exception:
+                if process.poll() is None:
                    return set()

            from archivebox.plugins.hooks import extract_records_from_process

            records = []
-            # Finite background hooks can exit before their completed Process
-            # metadata is visible. Give successful hooks a brief chance to
-            # flush JSONL stdout into the Process row before downstream hooks.
+            # A hook can exit before its completed Process metadata is visible.
+            # Give successful hooks a brief chance to flush JSONL stdout into
+            # the Process row before downstream hooks.
            for delay in (0.0, 0.05, 0.1, 0.25, 0.5):
                if delay:
                    time.sleep(delay)
@@ -434,6 +434,9 @@ class ProcessAdmin(BaseModelAdmin):
        "crawl_link",
        "cmd_display",
        "env_display",
+        "stdout_display",
+        "stderr_display",
+        "archiveresult_output_display",
        "timeout",
        "pid",
        "exit_code",
@@ -475,7 +478,7 @@ class ProcessAdmin(BaseModelAdmin):
        (
            "Output",
            {
-                "fields": ("stdout", "stderr"),
+                "fields": ("stdout_display", "stderr_display", "archiveresult_output_display"),
                "classes": ("card", "wide", "collapse"),
            },
        ),
@@ -548,6 +551,7 @@ class ProcessAdmin(BaseModelAdmin):
        label="Kill",
        description="Kill this process if it is still running",
        attrs={"class": "deletelink"},
+        methods=("POST",),
    )
    def kill_process(self, request, obj):
        self._terminate_processes(request, [obj])
@@ -739,6 +743,28 @@ class ProcessAdmin(BaseModelAdmin):
            return "-"
        return _render_copy_block(env_text, multiline=True)

+    @admin.display(description="Stdout")
+    def stdout_display(self, process):
+        if not process.stdout:
+            return "-"
+        return _render_copy_block(process.stdout, multiline=True)
+
+    @admin.display(description="Stderr")
+    def stderr_display(self, process):
+        if not process.stderr:
+            return "-"
+        return _render_copy_block(process.stderr, multiline=True)
+
+    @admin.display(description="ArchiveResult Output")
+    def archiveresult_output_display(self, process):
+        try:
+            output = process.archiveresult.output_str
+        except Process.archiveresult.RelatedObjectDoesNotExist:
+            return "-"
+        if not output:
+            return "-"
+        return _render_copy_block(output, multiline=True)
+

 def register_admin(admin_site):
    admin_site.register(Machine, MachineAdmin)
@@ -0,0 +1,130 @@
+import json
+from pathlib import Path
+
+from django.db import migrations
+from django.utils import timezone
+
+
+def _cmd_array(value):
+    if isinstance(value, list):
+        return value
+    if isinstance(value, str):
+        try:
+            parsed = json.loads(value)
+        except json.JSONDecodeError:
+            return [value] if value else []
+        return parsed if isinstance(parsed, list) else []
+    return []
+
+
+def _ensure_placeholder_iface(NetworkInterface, machine_id, hostname):
+    iface = NetworkInterface.objects.filter(machine_id=machine_id).order_by("-modified_at", "-created_at").first()
+    if iface is not None:
+        return iface
+
+    now = timezone.now()
+    return NetworkInterface.objects.create(
+        machine_id=machine_id,
+        created_at=now,
+        modified_at=now,
+        mac_address="00:00:00:00:00:00",
+        ip_public="0.0.0.0",
+        ip_local="0.0.0.0",
+        dns_server="0.0.0.0",
+        hostname=(hostname or "unknown")[:63],
+        iface="unknown",
+        isp="",
+        city="",
+        region="",
+        country="",
+    )
+
+
+def _get_or_create_binary(Binary, machine_id, reference):
+    reference = str(reference or "").strip()
+    if not reference:
+        return None
+
+    name = Path(reference).name or reference
+    qs = Binary.objects.filter(machine_id=machine_id)
+    binary = qs.filter(abspath=reference).order_by("-modified_at", "-created_at").first()
+    if binary is None:
+        binary = qs.filter(name=name).order_by("-modified_at", "-created_at").first()
+    if binary is not None:
+        return binary
+
+    now = timezone.now()
+    return Binary.objects.create(
+        machine_id=machine_id,
+        created_at=now,
+        modified_at=now,
+        name=name[:63],
+        binproviders="env",
+        overrides={},
+        binprovider="env",
+        abspath=reference[:255],
+        version="",
+        sha256="",
+        status="installed",
+        retry_at=None,
+    )
+
+
+def repair_process_binary_iface_links(apps, schema_editor):
+    Binary = apps.get_model("machine", "Binary")
+    Machine = apps.get_model("machine", "Machine")
+    NetworkInterface = apps.get_model("machine", "NetworkInterface")
+    Process = apps.get_model("machine", "Process")
+
+    machines = {machine.id: machine for machine in Machine.objects.only("id", "hostname").iterator(chunk_size=100)}
+    iface_by_machine = {}
+    binary_by_key = {}
+
+    qs = Process.objects.filter(machine_id__isnull=False).filter(binary_id__isnull=True) | Process.objects.filter(
+        machine_id__isnull=False,
+        iface_id__isnull=True,
+    )
+    for process in qs.distinct().only("id", "machine_id", "binary_id", "iface_id", "cmd").iterator(chunk_size=500):
+        update_fields = []
+        machine = machines.get(process.machine_id)
+
+        if process.iface_id is None:
+            iface = iface_by_machine.get(process.machine_id)
+            if iface is None:
+                iface = _ensure_placeholder_iface(
+                    NetworkInterface,
+                    process.machine_id,
+                    machine.hostname if machine is not None else "",
+                )
+                iface_by_machine[process.machine_id] = iface
+            process.iface_id = iface.id
+            update_fields.append("iface_id")
+
+        if process.binary_id is None:
+            cmd = _cmd_array(process.cmd)
+            reference = str(cmd[0]).strip() if cmd else ""
+            if reference:
+                key = (process.machine_id, reference)
+                binary = binary_by_key.get(key)
+                if binary is None:
+                    binary = _get_or_create_binary(Binary, process.machine_id, reference)
+                    binary_by_key[key] = binary
+                if binary is not None:
+                    process.binary_id = binary.id
+                    update_fields.append("binary_id")
+
+        if update_fields:
+            process.modified_at = timezone.now()
+            process.save(update_fields=[*update_fields, "modified_at"])
+
+
+class Migration(migrations.Migration):
+    atomic = False
+
+    dependencies = [
+        ("machine", "0019_single_active_runner_constraint"),
+    ]
+
+    operations = [
+        migrations.RunPython(repair_process_binary_iface_links, migrations.RunPython.noop),
+    ]
@@ -3,6 +3,7 @@ from __future__ import annotations
 __package__ = "archivebox.machine"

 import os
+import signal
 import sys
 import uuid
 import socket
@@ -55,6 +56,13 @@ PROCESS_TIMEOUT_GRACE = timedelta(seconds=30)  # Extra margin before force-clean
 START_TIME_TOLERANCE = 5.0  # Seconds tolerance for start time matching


+def _default_exit_code_for_unowned_process(process_type: str) -> int:
+    # Hooks are externally visible work items. If their owning runner disappeared
+    # before recording the real exit code, retrying is safer than converting an
+    # unknown interrupted extraction into a durable success/no-result row.
+    return 128 + signal.SIGTERM if process_type == Process.TypeChoices.HOOK else 0
+
+
 def _find_existing_binary_for_reference(machine: Machine, reference: str) -> Binary | None:
    reference = str(reference or "").strip()
    if not reference:
@@ -132,7 +140,7 @@ def _sanitize_machine_config(config: dict[str, Any] | None, *, lib_dir: str | Pa
    """Validate ``Machine.config`` in place.

    Drops stale ``*_BINARY`` overrides whose path no longer exists or whose
-    path falls outside of ``LIB_DIR`` (so a binary uninstall or a lib_dir
+    path falls outside of ``ABXPKG_LIB_DIR`` (so a binary uninstall or a lib_dir
    move clears the override automatically). Non-``_BINARY`` keys
    (``BASE_URL``, ``SERVER_SECURITY_MODE``, plugin tunables, etc.) are
    pass-through — they're arbitrary config overrides and not ours to filter.
@@ -279,7 +287,7 @@ class Machine(ModelWithHealthStats):
    def _sanitize_config(cls, machine: Machine) -> Machine:
        from archivebox.config.common import get_config

-        sanitized = _sanitize_machine_config(machine.config, lib_dir=get_config(include_machine=False).LIB_DIR)
+        sanitized = _sanitize_machine_config(machine.config, lib_dir=get_config(include_machine=False).ABXPKG_LIB_DIR)
        current = machine.config or {}
        if sanitized != current:
            machine.config = sanitized
@@ -412,14 +420,25 @@ class NetworkInterface(ModelWithHealthStats):
    def current(cls, refresh: bool = False) -> NetworkInterface:
        global _CURRENT_INTERFACE
        machine = Machine.current(refresh=refresh)
-        if _CURRENT_INTERFACE:
-            if (
-                not refresh
-                and _CURRENT_INTERFACE.machine_id == machine.id
-                and timezone.now() < _CURRENT_INTERFACE.modified_at + timedelta(seconds=NETWORK_INTERFACE_RECHECK_INTERVAL)
-            ):
+        if _CURRENT_INTERFACE and _CURRENT_INTERFACE.machine_id == machine.id:
+            if not refresh:
+                # Callers that pass refresh=False are asking for attribution to
+                # the currently known interface, not for public-IP/ISP probing.
+                # Maintenance paths create many short-lived services per run;
+                # expiring this in-memory object by age forced every crawl to
+                # hit external network APIs even though Process rows only need
+                # a stable existing FK. Active downloading paths opt into live
+                # detection with refresh=True.
                return _CURRENT_INTERFACE
-            _CURRENT_INTERFACE = None
+            if timezone.now() < _CURRENT_INTERFACE.modified_at + timedelta(seconds=NETWORK_INTERFACE_RECHECK_INTERVAL):
+                return _CURRENT_INTERFACE
+        _CURRENT_INTERFACE = None
+
+        if not refresh:
+            _CURRENT_INTERFACE = cls.objects.filter(machine=machine).order_by("-modified_at", "-created_at").first()
+            if _CURRENT_INTERFACE is not None:
+                return _CURRENT_INTERFACE
+
        net_info = get_host_network()
        lookup = dict(
            machine=machine,
@@ -548,9 +567,15 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):

    state_machine_name: str | None = "archivebox.machine.models.BinaryMachine"
    active_state: str = StatusChoices.QUEUED
+    warn_on_save_outside_runner = False

    objects = BinaryManager()  # pyright: ignore[reportIncompatibleVariableOverride]

+    if TYPE_CHECKING:
+
+        @property
+        def sm(self) -> BinaryMachine: ...
+
    class Meta(ModelWithHealthStats.Meta, ModelWithStateMachine.Meta):
        app_label = "machine"
        verbose_name = "Binary"
@@ -633,22 +658,6 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
        binary_overrides = record.get("overrides", {})
        normalized_overrides = binary_overrides if isinstance(binary_overrides, dict) else {}

-        # abx-plugins currently emits a GitHub install URL for readability-extractor,
-        # but the package is published on npm. Prefer the registry package to avoid
-        # long git-based installs in CI while still using canonical install_args.
-        if (
-            name == "readability-extractor"
-            and isinstance(normalized_overrides.get("npm"), dict)
-            and normalized_overrides["npm"].get("install_args") == ["https://github.com/ArchiveBox/readability-extractor"]
-        ):
-            normalized_overrides = {
-                **normalized_overrides,
-                "npm": {
-                    **normalized_overrides["npm"],
-                    "install_args": ["readability-extractor"],
-                },
-            }
-
        # Case 1: Already installed (from on_Crawl hooks) - has abspath AND binproviders
        # This happens when on_Crawl hooks detect already-installed binaries
        abspath = record.get("abspath")
@@ -672,7 +681,7 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
            )
            from archivebox.config.common import get_config

-            binary.symlink_to_lib_bin_after_commit(get_config().LIB_BIN_DIR)
+            binary.symlink_to_lib_bin_after_commit(get_config().ABXPKG_LIB_DIR / "bin")
            return binary

        # Case 2: From binaries.json - create queued binary (needs installation)
@@ -705,7 +714,7 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
            )
            from archivebox.config.common import get_config

-            binary.symlink_to_lib_bin_after_commit(get_config().LIB_BIN_DIR)
+            binary.symlink_to_lib_bin_after_commit(get_config().ABXPKG_LIB_DIR / "bin")
            return binary

        return None
@@ -741,15 +750,15 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):

    def symlink_to_lib_bin(self, lib_bin_dir: str | Path) -> Path | None:
        """
-        Symlink this binary into LIB_BIN_DIR for human-facing convenience.
+        Symlink this binary into a derived lib/bin directory for human-facing convenience.

        After a binary is installed by any binprovider (pip, npm, brew, apt, etc),
        we can optionally expose a flat convenience directory for shell users.
-        ArchiveBox/abx-dl runtime lookup must use the provider-specific LIB_DIR
+        ArchiveBox/abx-dl runtime lookup must use the provider-specific ABXPKG_LIB_DIR
        paths, not this indirection.

        Args:
-            lib_bin_dir: Path to LIB_BIN_DIR (e.g., /data/lib/arm64-darwin/bin)
+            lib_bin_dir: Path to the derived convenience bin dir (e.g., /data/lib/bin)

        Returns:
            Path to the created symlink, or None if symlinking failed
@@ -773,14 +782,14 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
        except StopIteration:
            app_index = -1

-        # Create LIB_BIN_DIR if it doesn't exist
+        # Create the derived convenience bin dir if it doesn't exist.
        try:
            lib_bin_dir.mkdir(parents=True, exist_ok=True)
        except (OSError, PermissionError) as e:
-            print(f"Failed to create LIB_BIN_DIR {lib_bin_dir}: {e}", file=sys.stderr)
+            print(f"Failed to create lib/bin convenience dir {lib_bin_dir}: {e}", file=sys.stderr)
            return None

-        # Expose the canonical Binary.name in LIB_BIN_DIR. Some providers point
+        # Expose the canonical Binary.name in the convenience bin dir. Some providers point
        # abspath at implementation files like cli.js or manifest.json; those
        # are valid targets, but they are not user-facing binary names.
        binary_name = _canonical_binary_name(self.name) or binary_abspath.name
@@ -1535,9 +1544,11 @@ class Process(ModelWithDeleteAfter, models.Model):
        avoids any race window where the row briefly exists as ADD before a
        higher-level demotion.
        """
-        argv_str = " ".join(sys.argv).lower()
+        argv = [str(arg) for arg in sys.argv]
+        argv_str = " ".join(argv).lower()
+        executable = Path(argv[0]).name.lower() if argv else ""

-        if "supervisord" in argv_str:
+        if executable == "supervisord" or any(arg.startswith("--configuration=") for arg in argv[1:]):
            return cls.TypeChoices.SUPERVISORD
        elif "runner_watch" in argv_str:
            return cls.TypeChoices.WORKER
@@ -1607,7 +1618,9 @@ class Process(ModelWithDeleteAfter, models.Model):
                    is_stale = True  # Process no longer exists

            if is_stale:
-                proc.mark_exited(exit_code=proc.exit_code if proc.exit_code is not None else 0)
+                proc.mark_exited(
+                    exit_code=proc.exit_code if proc.exit_code is not None else _default_exit_code_for_unowned_process(proc.process_type),
+                )
                cleaned += 1

        return cleaned
@@ -2137,8 +2150,7 @@ class Process(ModelWithDeleteAfter, models.Model):
                # TODO: Uncomment to cleanup (keeping for debugging for now)
                # self.stderr_file.unlink(missing_ok=True)

-            # Try to get exit code from proc or default to unknown
-            self.exit_code = self.exit_code if self.exit_code is not None else 0
+            self.exit_code = self.exit_code if self.exit_code is not None else _default_exit_code_for_unowned_process(self.process_type)
            if self.exit_code == -1:
                self.exit_code = 137
            self.ended_at = timezone.now()
@@ -2502,7 +2514,9 @@ class Process(ModelWithDeleteAfter, models.Model):
        # orphaned Process backlog cannot be materialized in memory at once.
        for proc in running_children.iterator(chunk_size=100):
            if not proc.is_running:
-                proc.mark_exited(exit_code=proc.exit_code if proc.exit_code is not None else 0)
+                proc.mark_exited(
+                    exit_code=proc.exit_code if proc.exit_code is not None else _default_exit_code_for_unowned_process(proc.process_type),
+                )
                cleaned += 1
                continue

@@ -2526,7 +2540,9 @@ class Process(ModelWithDeleteAfter, models.Model):
            ):
                continue

-            proc.mark_exited(exit_code=proc.exit_code if proc.exit_code is not None else 0)
+            proc.mark_exited(
+                exit_code=proc.exit_code if proc.exit_code is not None else _default_exit_code_for_unowned_process(proc.process_type),
+            )
            cleaned += 1

        if cleaned:
@@ -220,14 +220,14 @@ def check_not_inside_source_dir():
 def check_data_dir_permissions(config=None, **config_kwargs):
    from archivebox import DATA_DIR
    from archivebox.misc.logging import STDERR
-    from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, DEFAULT_PUID, DEFAULT_PGID, IS_ROOT, USER
+    from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, DEFAULT_UID, DEFAULT_GID, IS_ROOT, USER
    from archivebox.config.paths import get_or_create_working_tmp_dir, get_or_create_working_lib_dir

    data_dir_stat = Path(DATA_DIR).stat()
    data_dir_uid, data_dir_gid = data_dir_stat.st_uid, data_dir_stat.st_gid
    data_owned_by_root = data_dir_uid == 0

-    # data_owned_by_default_user = data_dir_uid == DEFAULT_PUID or data_dir_gid == DEFAULT_PGID
+    # data_owned_by_default_user = data_dir_uid == DEFAULT_UID or data_dir_gid == DEFAULT_GID
    data_owner_doesnt_match = (data_dir_uid != ARCHIVEBOX_USER and data_dir_gid != ARCHIVEBOX_GROUP) if not IS_ROOT else False
    data_not_writable = not (os.path.isdir(DATA_DIR) and os.access(DATA_DIR, os.W_OK))
    if data_not_writable:
@@ -245,9 +245,9 @@ def check_data_dir_permissions(config=None, **config_kwargs):

    if data_not_writable:
        STDERR.print(
-            f"[violet]Hint:[/violet] Change the current ownership [red]{data_dir_uid}[/red]:{data_dir_gid} (PUID:PGID) to the user & group that will run ArchiveBox, e.g.:",
+            f"[violet]Hint:[/violet] Change the current ownership [red]{data_dir_uid}[/red]:{data_dir_gid} to the user & group that will run ArchiveBox, e.g.:",
        )
-        STDERR.print(f"    [grey53]sudo[/grey53] chown -R [blue]{DEFAULT_PUID}:{DEFAULT_PGID}[/blue] {DATA_DIR.resolve()}")
+        STDERR.print(f"    [grey53]sudo[/grey53] chown -R [blue]{DEFAULT_UID}:{DEFAULT_GID}[/blue] {DATA_DIR.resolve()}")
        STDERR.print("    Avoid recursive chown on very large archives unless you know the full tree needs repair.")
        STDERR.print()
        STDERR.print("[blue]More info:[/blue]")
@@ -273,9 +273,9 @@ def check_data_dir_permissions(config=None, **config_kwargs):
        tmp_dir = config.TMP_DIR

    try:
-        lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True, config=config) or config.LIB_DIR
+        lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True, config=config) or config.ABXPKG_LIB_DIR
    except Exception:
-        lib_dir = config.LIB_DIR
+        lib_dir = config.ABXPKG_LIB_DIR

    # Check /tmp dir permissions
    check_tmp_dir(tmp_dir, throw=False, must_exist=True, config=config)
@@ -289,7 +289,14 @@ def check_data_dir_permissions(config=None, **config_kwargs):


 def check_tmp_dir(tmp_dir=None, throw=False, quiet=False, must_exist=True, config=None, **config_kwargs):
-    from archivebox.config.paths import assert_dir_can_contain_unix_sockets, dir_is_writable, get_or_create_working_tmp_dir
+    from archivebox.config.paths import (
+        MAX_TMP_SOCKET_URL_LENGTH,
+        SUPERVISORD_SOCKET_FILENAME,
+        assert_dir_can_contain_unix_sockets,
+        dir_is_writable,
+        get_or_create_working_tmp_dir,
+        tmp_dir_socket_path_is_short_enough,
+    )
    from archivebox.misc.logging import STDERR
    from archivebox.misc.logging_util import pretty_path
    from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
@@ -297,20 +304,22 @@ def check_tmp_dir(tmp_dir=None, throw=False, quiet=False, must_exist=True, confi

    config = config or get_config(**config_kwargs)
    tmp_dir = tmp_dir or config.TMP_DIR
-    socket_file = tmp_dir.absolute().resolve() / "supervisord.sock"
+    socket_file = tmp_dir.absolute().resolve() / SUPERVISORD_SOCKET_FILENAME

    if not must_exist and not os.path.isdir(tmp_dir):
        # just check that its viable based on its length (because dir may not exist yet, we cant check if its writable)
-        return len(f"file://{socket_file}") <= 96
+        return tmp_dir_socket_path_is_short_enough(tmp_dir)

    tmp_is_valid = False
    try:
        tmp_is_valid = dir_is_writable(tmp_dir)
        if not config.ALLOW_NO_UNIX_SOCKETS:
            tmp_is_valid = tmp_is_valid and assert_dir_can_contain_unix_sockets(tmp_dir)
-        assert tmp_is_valid, f"ArchiveBox user PUID={ARCHIVEBOX_USER} PGID={ARCHIVEBOX_GROUP} is unable to write to TMP_DIR={tmp_dir}"
-        assert len(f"file://{socket_file}") <= 96, (
-            f"ArchiveBox TMP_DIR={tmp_dir} is too long, dir containing unix socket files must be <90 chars."
+        assert tmp_is_valid, f"ArchiveBox user {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP} is unable to write to TMP_DIR={tmp_dir}"
+        socket_url_len = len(f"file://{socket_file}")
+        assert tmp_dir_socket_path_is_short_enough(tmp_dir), (
+            f"ArchiveBox TMP_DIR={tmp_dir} is too long, file://{socket_file} is {socket_url_len} chars "
+            f"and must be <{MAX_TMP_SOCKET_URL_LENGTH} chars."
        )
        return True
    except Exception as e:
@@ -324,7 +333,7 @@ def check_tmp_dir(tmp_dir=None, throw=False, quiet=False, must_exist=True, confi
                    "",
                    "[blue]Info:[/blue] [grey53]The TMP_DIR is used for the supervisord unix socket file and other temporary files.",
                    "  - It [red]must[/red] be on a local drive (not inside a docker volume, remote network drive, or FUSE mount).",
-                    f"  - It [red]must[/red] be readable and writable by the ArchiveBox user (PUID={ARCHIVEBOX_USER}, PGID={ARCHIVEBOX_GROUP}).",
+                    f"  - It [red]must[/red] be readable and writable by the ArchiveBox user ({ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}).",
                    "  - It [red]must[/red] be a *short* path (less than 90 characters) due to UNIX path length restrictions for sockets.",
                    "  - It [yellow]should[/yellow] be able to hold at least 200MB of data (in-progress downloads can be large).[/grey53]",
                    "",
@@ -356,7 +365,7 @@ def check_lib_dir(lib_dir: Path | None = None, throw=False, quiet=False, must_ex
    from archivebox.config.common import get_config

    config = config or get_config(**config_kwargs)
-    lib_dir = lib_dir or config.LIB_DIR
+    lib_dir = lib_dir or config.ABXPKG_LIB_DIR

    if not must_exist and not os.path.isdir(lib_dir):
        return True
@@ -364,7 +373,7 @@ def check_lib_dir(lib_dir: Path | None = None, throw=False, quiet=False, must_ex
    lib_is_valid = False
    try:
        lib_is_valid = dir_is_writable(lib_dir)
-        assert lib_is_valid, f"ArchiveBox user PUID={ARCHIVEBOX_USER} PGID={ARCHIVEBOX_GROUP} is unable to write to LIB_DIR={lib_dir}"
+        assert lib_is_valid, f"ArchiveBox user {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP} is unable to write to ABXPKG_LIB_DIR={lib_dir}"
        return True
    except Exception as e:
        if not quiet:
@@ -372,16 +381,16 @@ def check_lib_dir(lib_dir: Path | None = None, throw=False, quiet=False, must_ex
            ERROR_TEXT = "\n".join(
                (
                    "",
-                    f"[red]:cross_mark: ArchiveBox is unable to use LIB_DIR={pretty_path(lib_dir)}[/red]",
+                    f"[red]:cross_mark: ArchiveBox is unable to use ABXPKG_LIB_DIR={pretty_path(lib_dir)}[/red]",
                    f"   [yellow]{e}[/yellow]",
                    "",
-                    "[blue]Info:[/blue] [grey53]The LIB_DIR is used to store ArchiveBox auto-installed plugin library and binary dependencies.",
-                    f"  - It [red]must[/red] be readable and writable by the ArchiveBox user (PUID={ARCHIVEBOX_USER}, PGID={ARCHIVEBOX_GROUP}).",
+                    "[blue]Info:[/blue] [grey53]The ABXPKG_LIB_DIR is used to store ArchiveBox auto-installed plugin library and binary dependencies.",
+                    f"  - It [red]must[/red] be readable and writable by the ArchiveBox user ({ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}).",
                    "  - It [yellow]should[/yellow] be on a local (ideally fast) drive like an SSD or HDD (not on a network drive or external HDD).",
                    "  - It [yellow]should[/yellow] be able to hold at least 1GB of data (some dependencies like Chrome can be large).[/grey53]",
                    "",
-                    "[violet]Hint:[/violet] Fix it by setting LIB_DIR to a path that meets these requirements, e.g.:",
-                    f"      [green]archivebox config --set LIB_DIR={get_or_create_working_lib_dir(autofix=False, quiet=True) or config.LIB_DIR}[/green]",
+                    "[violet]Hint:[/violet] Fix it by setting ABXPKG_LIB_DIR to a path that meets these requirements, e.g.:",
+                    f"      [green]archivebox config --set ABXPKG_LIB_DIR={get_or_create_working_lib_dir(autofix=False, quiet=True) or config.ABXPKG_LIB_DIR}[/green]",
                    "",
                ),
            )
@@ -390,11 +399,13 @@ def check_lib_dir(lib_dir: Path | None = None, throw=False, quiet=False, must_ex
                    ERROR_TEXT,
                    expand=False,
                    border_style="red",
-                    title="[red]:cross_mark: Error with configured LIB_DIR[/red]",
+                    title="[red]:cross_mark: Error with configured ABXPKG_LIB_DIR[/red]",
                    subtitle="[yellow]Dependencies may not auto-install properly until fixed.[/yellow]",
                ),
            )
            STDERR.print()
        if throw:
-            raise OSError(f"LIB_DIR={lib_dir} is invalid, ArchiveBox is unable to use it and dependencies will fail to install.") from e
+            raise OSError(
+                f"ABXPKG_LIB_DIR={lib_dir} is invalid, ArchiveBox is unable to use it and dependencies will fail to install.",
+            ) from e
    return False
@@ -18,6 +18,9 @@ Typed JSONL Format:
 Plain URLs (also supported):
    https://example.com
    https://foo.com
+
+Local filesystem paths and file:// URLs are intentionally not accepted here.
+Pipe file contents through stdin instead of passing a path/URI as a crawl URL.
 """

 __package__ = "archivebox.misc"
@@ -27,7 +30,6 @@ __package__ = "archivebox.misc"

 import sys
 import json
-import select
 from typing import Any, TextIO
 from collections.abc import Iterable, Iterator
 from pathlib import Path
@@ -81,7 +83,7 @@ def parse_line(line: str) -> dict[str, Any] | None:
            pass

    # Treat as plain URL if it looks like one
-    if line.startswith("http://") or line.startswith("https://") or line.startswith("file://"):
+    if line.startswith("http://") or line.startswith("https://"):
        return {"type": TYPE_SNAPSHOT, "url": line}

    # Could be a snapshot ID (UUID with dashes or compact 32-char hex)
@@ -112,14 +114,6 @@ def read_stdin(stream: TextIO | None = None) -> Iterator[dict[str, Any]]:
    if active_stream.isatty():
        return

-    try:
-        ready, _, _ = select.select([active_stream], [], [], 0)
-    except (OSError, ValueError):
-        ready = [active_stream]
-
-    if not ready:
-        return
-
    for line in active_stream:
        record = parse_line(line)
        if record:
@@ -144,17 +138,13 @@ def read_args_or_stdin(args: Iterable[str], stream: TextIO | None = None) -> Ite
    Read from CLI arguments if provided, otherwise from stdin.

    Handles both URLs and JSONL from either source.
+    Does not expand local file path arguments; pipe file contents via stdin.
    """
    if args:
        for arg in args:
-            # Check if it's a file path
-            path = Path(arg)
-            if path.exists() and path.is_file():
-                yield from read_file(path)
-            else:
-                record = parse_line(arg)
-                if record:
-                    yield record
+            record = parse_line(arg)
+            if record:
+                yield record
    else:
        yield from read_stdin(stream)

@@ -1,63 +1,12 @@
 __package__ = "archivebox.misc"

 from django.core.paginator import Paginator
-from django.core.paginator import Page
-from django.core.paginator import EmptyPage
-from django.db import connection
 from django.db.models import QuerySet
 from django.utils.functional import cached_property


-class CountlessPage(Page):
-    def __init__(self, object_list, number, paginator, has_next_page=False):
-        self.has_next_page = has_next_page
-        super().__init__(object_list, number, paginator)
-
-    def has_next(self):
-        return self.has_next_page
-
-
-class CountlessPaginator(Paginator):
-    has_exact_count = False
-
-    def __init__(self, *args, **kwargs):
-        self._count_hint = 0
-        super().__init__(*args, **kwargs)
-
-    @cached_property
-    def count(self):
-        return self._count_hint
-
-    @cached_property
-    def num_pages(self):
-        return self.count
-
-    def validate_number(self, number):
-        try:
-            number = int(number)
-        except (TypeError, ValueError):
-            raise EmptyPage("That page number is not an integer")
-        if number < 1:
-            raise EmptyPage("That page number is less than 1")
-        return number
-
-    def page(self, number):
-        number = self.validate_number(number)
-        bottom = (number - 1) * self.per_page
-        page_objects = list(self.object_list[bottom : bottom + self.per_page + 1])
-        has_next_page = len(page_objects) > self.per_page
-        if has_next_page:
-            page_objects = page_objects[: self.per_page]
-        self._count_hint = bottom + len(page_objects) + (1 if has_next_page else 0)
-        return CountlessPage(page_objects, number, self, has_next_page=has_next_page)
-
-
 class AcceleratedPaginator(Paginator):
-    """
-    Accelerated paginator ignores DISTINCT when counting total number of rows.
-    Speeds up SELECT Count(*) on Admin views by >20x.
-    https://hakibenita.com/optimizing-the-django-admin-paginator
-    """
+    """Paginator that accepts exact count hints for already-optimized query paths."""

    @cached_property
    def count(self):
@@ -69,39 +18,8 @@ class AcceleratedPaginator(Paginator):
        if count_hint is None:
            count_hint = query.__dict__.get("_archivebox_count_hint")
        if count_hint is not None:
-            model = self.object_list.model
-            if count_hint == "model_estimate":
-                return self._model_count_estimate(model)
            if callable(count_hint):
                return count_hint()
            return count_hint

-        if query.distinct or query.where.children:
-            # fallback to normal count method on filtered queryset
-            return super().count
-
-        # otherwise count total rows in a separate fast query
-        return self._model_count_estimate(self.object_list.model)
-
-    def _model_count_estimate(self, model):
-        if connection.vendor == "sqlite":
-            table_name = model._meta.db_table
-            with connection.cursor() as cursor:
-                try:
-                    cursor.execute("SELECT stat FROM sqlite_stat1 WHERE tbl = %s", [table_name])
-                    stats = [int(str(row[0]).split()[0]) for row in cursor.fetchall() if row and row[0]]
-                except Exception:
-                    stats = []
-            if stats:
-                return max(stats)
-
-        return model.objects.count()
-
-        # Alternative approach for PostgreSQL: fallback count takes > 200ms
-        # from django.db import connection, transaction, OperationalError
-        # with transaction.atomic(), connection.cursor() as cursor:
-        #     cursor.execute('SET LOCAL statement_timeout TO 200;')
-        #     try:
-        #         return super().count
-        #     except OperationalError:
-        #         return 9999999999999
+        return super().count
@@ -711,6 +711,10 @@ def _apply_archive_replay_headers(
    return response


+def _is_asgi_request(request) -> bool:
+    return isinstance(request, ASGIRequest) or "scope" in request.__dict__
+
+
 def serve_static_with_byterange_support(request, path, document_root=None, show_indexes=False, is_archive_replay: bool = False):
    """
    Overrides Django's built-in django.views.static.serve function to support byte range requests.
@@ -718,7 +722,9 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
    https://github.com/satchamo/django/commit/2ce75c5c4bee2a858c0214d136bfcd351fcde11d
    """
    assert document_root
-    config = request.archivebox_config
+    config = request.__dict__.get("archivebox_config")
+    if config is None:
+        config = get_config(resolve_plugins=False)
    fullpath, path = _resolve_archive_path(document_root, path)
    if os.access(fullpath, os.R_OK) and fullpath.is_dir():
        if request.GET.get("download") == "zip" and show_indexes:
@@ -726,7 +732,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
                fullpath,
                path,
                is_archive_replay=is_archive_replay,
-                use_async_stream=isinstance(request, ASGIRequest),
+                use_async_stream=_is_asgi_request(request),
                config=config,
            )
        if show_indexes:
@@ -908,7 +914,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
            response.headers["Content-Security-Policy"] = (
                "default-src 'self' data: blob:; "
                "script-src 'unsafe-inline'; "
-                "style-src 'unsafe-inline'; "
+                "style-src 'unsafe-inline' data: blob:; "
                "connect-src 'self'; "
                "frame-src 'self' data: blob:; "
                "object-src 'none'; "
@@ -977,7 +983,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
    # setup response object
    ranged_file = RangedFileReader(open(fullpath, "rb"))
    response = StreamingHttpResponse(
-        _stream_ranged_file_async(ranged_file) if isinstance(request, ASGIRequest) else ranged_file,
+        _stream_ranged_file_async(ranged_file) if _is_asgi_request(request) else ranged_file,
        content_type=content_type,
    )
    response.headers["Last-Modified"] = http_date(statobj.st_mtime)
@@ -66,6 +66,22 @@ htmlencode = lambda s: s and escape(s, quote=True)
 htmldecode = lambda s: s and unescape(s)


+def sanitize_html_text(value: Any) -> str:
+    """Strip all HTML from user-editable text before storing it."""
+    if value is None:
+        return ""
+    import bleach
+
+    return bleach.clean(
+        str(value),
+        tags=[],
+        attributes={},
+        protocols=[],
+        strip=True,
+        strip_comments=True,
+    )
+
+
 def ts_to_date_str(ts: Any) -> str | None:
    parsed = parse_date(ts)
    return None if parsed is None else parsed.strftime("%Y-%m-%d %H:%M")
@@ -86,6 +102,11 @@ URL_REGEX = re.compile(
    re.IGNORECASE | re.UNICODE,
 )

+# Maximum supported URL length. Very long URLs are rare but must be supported correctly
+# (e.g. data: URLs, deeply nested query strings). The Snapshot.url column is stored as a
+# variable-length TextField (so short URLs don't reserve space and very long URLs still
+# fit) while keeping a normal index on the field so exact, prefix, and substring lookups
+# all keep working.
 MAX_URL_LENGTH = 65535

 QUOTE_DELIMITERS = (
@@ -62,7 +62,7 @@ class PersonaAdminForm(PluginConfigFormMixin, forms.ModelForm):
        label="Absolute path or CDP URL",
        widget=forms.TextInput(
            attrs={
-                "placeholder": "/Users/alice/Library/Application Support/Google/Chrome  or  ws://127.0.0.1:9222/devtools/browser/...",
+                "placeholder": "/Users/alice/Library/Application Support/Google/Chrome  or  http://127.0.0.1:9222  or  ws://127.0.0.1:9222/devtools/browser/...",
                "style": "width: 100%; font-family: monospace;",
            },
        ),
@@ -47,6 +47,12 @@ VOLATILE_PROFILE_COPY_PATTERNS = (
    "ShaderCache",
    "Service Worker",
    "GCM Store",
+    "chrome-extension_*",
+    "DNR Extension Rules",
+    "Extension Rules",
+    "Extension Scripts",
+    "Extension State",
+    "Local Extension Settings",
    "*.log",
    "Crashpad",
    "BrowserMetrics",
@@ -378,7 +384,6 @@ def discover_persona_template_profiles(personas_dir: Path | None = None) -> list
        candidate_roots.extend(
            [
                CONSTANTS.PERSONAS_DIR.expanduser(),
-                Path.home() / ".config" / "abx" / "personas",
            ],
        )

@@ -612,7 +617,7 @@ def export_browser_state(
    if not state_script.exists():
        return False, None, f"Browser state export script not found at {state_script}"

-    node_modules_dir = get_config().LIB_DIR / "npm" / "node_modules"
+    node_modules_dir = get_config().ABXPKG_LIB_DIR / "pnpm" / "packages" / "chrome" / "node_modules"
    chrome_plugin_dir = Path(get_plugins_dir()).resolve()

    env = os.environ.copy()
@@ -11,14 +11,12 @@ Each persona has its own:
 __package__ = "archivebox.personas"

 import shutil
-import subprocess
-import sys
 from contextlib import contextmanager
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
 from collections.abc import Mapping

-from django.db import models
+from django.db import IntegrityError, models
 from django.db.models.fields.json import KT
 from django.conf import settings
 from django.utils import timezone
@@ -62,11 +60,7 @@ VOLATILE_PROFILE_FILE_NAMES = {

 def derive_persona_config(*, name: str, config: Mapping[str, Any] | None, persona_dir: Path) -> dict[str, Any]:
    derived = dict(config or {})
-
-    if "CHROME_USER_DATA_DIR" not in derived:
-        derived["CHROME_USER_DATA_DIR"] = str(persona_dir / "chrome_profile")
-    if "CHROME_DOWNLOADS_DIR" not in derived:
-        derived["CHROME_DOWNLOADS_DIR"] = str(persona_dir / "chrome_downloads")
+    derived["PERSONAS_DIR"] = str(persona_dir.parent)

    cookies_path = persona_dir / "cookies.txt"
    if "COOKIES_FILE" not in derived and cookies_path.exists():
@@ -165,8 +159,7 @@ class Persona(ModelWithConfig):

        Returns dict with:
        - All values from self.config JSONField
-        - CHROME_USER_DATA_DIR (derived from persona path)
-        - CHROME_DOWNLOADS_DIR (derived from persona path)
+        - PERSONAS_DIR (derived from DATA_DIR/personas)
        - COOKIES_FILE (derived from persona path, if file exists)
        - AUTH_STORAGE_FILE (derived from persona path, if file exists)
        - ACTIVE_PERSONA (set to this persona's name)
@@ -228,6 +221,18 @@ class Persona(ModelWithConfig):
                if fcntl is not None:
                    fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)

+    @classmethod
+    def get_or_create_named(cls, name: str) -> "Persona":
+        persona_name = (name or "Default").strip() or "Default"
+        persona = cls.objects.filter(name=persona_name).first()
+        if persona is not None:
+            return persona
+
+        try:
+            return cls.objects.create(name=persona_name)
+        except IntegrityError:
+            return cls.objects.get(name=persona_name)
+
    def runtime_root_for_crawl(self, crawl) -> Path:
        return Path(crawl.output_dir) / ".persona" / self.name

@@ -237,28 +242,26 @@ class Persona(ModelWithConfig):
    def runtime_downloads_dir_for_crawl(self, crawl) -> Path:
        return self.runtime_root_for_crawl(crawl) / "chrome_downloads"

+    def runtime_root_for_snapshot(self, snapshot) -> Path:
+        return Path(snapshot.output_dir) / ".persona" / self.name
+
+    def runtime_profile_dir_for_snapshot(self, snapshot) -> Path:
+        return self.runtime_root_for_snapshot(snapshot) / "chrome_profile"
+
+    def runtime_downloads_dir_for_snapshot(self, snapshot) -> Path:
+        return self.runtime_root_for_snapshot(snapshot) / "chrome_downloads"
+
    def copy_chrome_profile(self, source_dir: Path, destination_dir: Path) -> None:
+        from archivebox.personas.importers import VOLATILE_PROFILE_COPY_PATTERNS
+
        destination_dir.parent.mkdir(parents=True, exist_ok=True)
        shutil.rmtree(destination_dir, ignore_errors=True)
-        destination_dir.mkdir(parents=True, exist_ok=True)
-
-        copy_cmd: list[str] | None = None
-        source_contents = f"{source_dir}/."
-
-        if sys.platform == "darwin":
-            copy_cmd = ["cp", "-cR", source_contents, str(destination_dir)]
-        elif sys.platform.startswith("linux"):
-            copy_cmd = ["cp", "-a", source_contents, str(destination_dir)]
-
-        if copy_cmd:
-            result = subprocess.run(copy_cmd, capture_output=True, text=True)
-            if result.returncode == 0:
-                return
-
-            shutil.rmtree(destination_dir, ignore_errors=True)
-            destination_dir.mkdir(parents=True, exist_ok=True)
-
-        shutil.copytree(source_dir, destination_dir, symlinks=True, dirs_exist_ok=True)
+        shutil.copytree(
+            source_dir,
+            destination_dir,
+            symlinks=True,
+            ignore=shutil.ignore_patterns(*VOLATILE_PROFILE_COPY_PATTERNS),
+        )

    def prepare_runtime_for_crawl(self, crawl, chrome_binary: str = "") -> dict[str, str]:
        self.ensure_dirs()
@@ -285,8 +288,43 @@ class Persona(ModelWithConfig):
                (runtime_root / "chrome_binary.txt").write_text(chrome_binary)

        return {
-            "CHROME_USER_DATA_DIR": str(runtime_profile_dir),
-            "CHROME_DOWNLOADS_DIR": str(runtime_downloads_dir),
+            # Hooks derive CHROME_USER_DATA_DIR/CHROME_DOWNLOADS_DIR from
+            # PERSONAS_DIR + ACTIVE_PERSONA. Point PERSONAS_DIR at the
+            # per-crawl runtime root here so CHROME_ISOLATION=crawl never
+            # leaks or reuses the template profile while keeping Chrome path
+            # derivation centralized in the Chrome plugin helpers.
+            "PERSONAS_DIR": str(runtime_root.parent),
+            "ACTIVE_PERSONA": self.name,
+        }
+
+    def prepare_runtime_for_snapshot(self, snapshot, chrome_binary: str = "") -> dict[str, str]:
+        crawl_runtime_profile_dir = self.runtime_profile_dir_for_crawl(snapshot.crawl)
+        template_dir = crawl_runtime_profile_dir if crawl_runtime_profile_dir.exists() else Path(self.CHROME_USER_DATA_DIR)
+        runtime_root = self.runtime_root_for_snapshot(snapshot)
+        runtime_profile_dir = self.runtime_profile_dir_for_snapshot(snapshot)
+        runtime_downloads_dir = self.runtime_downloads_dir_for_snapshot(snapshot)
+
+        if runtime_root.exists():
+            shutil.rmtree(runtime_root, ignore_errors=True)
+        if template_dir.exists() and any(template_dir.iterdir()):
+            self.copy_chrome_profile(template_dir, runtime_profile_dir)
+        else:
+            runtime_profile_dir.mkdir(parents=True, exist_ok=True)
+
+        runtime_downloads_dir.mkdir(parents=True, exist_ok=True)
+        self.cleanup_chrome_profile(runtime_profile_dir)
+
+        (runtime_root / "persona_name.txt").write_text(self.name)
+        (runtime_root / "template_dir.txt").write_text(str(template_dir))
+        if chrome_binary:
+            (runtime_root / "chrome_binary.txt").write_text(chrome_binary)
+
+        return {
+            # See prepare_runtime_for_crawl(): snapshot isolation changes the
+            # persona root, not individual CHROME_* config keys, so standalone
+            # Chrome hooks and ArchiveBox-driven hooks resolve paths the same way.
+            "PERSONAS_DIR": str(runtime_root.parent),
+            "ACTIVE_PERSONA": self.name,
        }

    def cleanup_runtime_for_crawl(self, crawl) -> None:
@@ -295,8 +333,7 @@ class Persona(ModelWithConfig):
    @classmethod
    def get_or_create_default(cls) -> "Persona":
        """Get or create the Default persona."""
-        persona, _ = cls.objects.get_or_create(name="Default")
-        return persona
+        return cls.get_or_create_named("Default")

    @classmethod
    def cleanup_chrome_all(cls) -> int:
@@ -195,12 +195,15 @@ def get_search_backends() -> dict[str, Any]:
    )


+@lru_cache(maxsize=1)
 def discover_plugin_configs() -> dict[str, dict[str, Any]]:
    """
    Discover all plugin config.json schemas.

    Each plugin can define a config.json file with JSONSchema defining
-    its configuration options.
+    its configuration options. This is intentionally cached because these
+    schemas are plugin package metadata, not live user config; runtime values
+    still come from env/db config at each callsite.
    """
    configs = {}

@@ -9,6 +9,7 @@ from typing import Any
 from django import forms
 from django.utils.html import format_html

+from archivebox.config import CONSTANTS_CONFIG
 from archivebox.config.common import ArchiveBoxConfig, get_config
 from archivebox.plugins.discovery import discover_plugin_configs, get_plugin_icon, get_plugins

@@ -129,6 +130,7 @@ HIDDEN_PLUGIN_CONFIG_UI_PLUGINS = {
    "env",
    "media",
    "npm",
+    "opencode",
    "pip",
    "puppeteer",
    "search_backend_ripgrep",
@@ -317,6 +319,7 @@ class PluginConfigFormMixin:
                for config_key, prop_schema in properties.items()
                if (
                    isinstance(prop_schema, dict)
+                    and str(config_key) not in CONSTANTS_CONFIG
                    and (self.allow_crawl_execution_config_fields or ArchiveBoxConfig.scope_for_key(str(config_key)) == "crawl_frozen")
                )
            ]
@@ -431,6 +434,8 @@ class PluginConfigFormMixin:
                input_name = _plugin_config_input_name(plugin_name, config_key)
                if input_name not in self.data:
                    continue
+                if str(config_key) in CONSTANTS_CONFIG:
+                    continue
                if not self.allow_crawl_execution_config_fields and ArchiveBoxConfig.scope_for_key(str(config_key)) != "crawl_frozen":
                    continue

@@ -32,7 +32,7 @@ Execution order:
    - After all foreground hooks complete, background hooks receive SIGTERM and must finalize

 Hook naming convention:
-    on_{EventFamily}__{run_order}_{description}[.finite.bg|.daemon.bg].{ext}
+    on_{EventFamily}__{run_order}_{description}[.bg].{ext}

 API:
    discover_hooks(event)     -> List[Path]     Find hook scripts for a hook-backed event family
@@ -104,11 +104,6 @@ def is_background_hook(hook_name: str) -> bool:
    return ".bg." in hook_name or "__background" in hook_name


-def is_finite_background_hook(hook_name: str) -> bool:
-    """Check if a background hook is finite-lived and should be awaited."""
-    return ".finite.bg." in hook_name
-
-
 def normalize_hook_event_name(event_name: str) -> str | None:
    """
    Normalize a hook event family or event class name to its on_* prefix.
@@ -284,14 +279,13 @@ def run_hook(
            records = process.get_records()  # Get parsed JSONL output
    """
    from archivebox.machine.models import Process, Machine, NetworkInterface
-    from archivebox.config.common import get_config, normalize_runtime_config
-    import sys
+    from archivebox.config.common import ArchiveBoxConfig, _archivebox_config_input_names, get_config, normalize_runtime_config

    config_scope = {key.removeprefix("config_"): kwargs.pop(key) for key in list(kwargs) if key.startswith("config_")}
    config_overrides = _config_to_overrides(config)
    resolved_config = get_config(overrides=config_overrides, **config_scope)
    hook_config = normalize_runtime_config(
-        config_overrides if config is not None else resolved_config.for_crawl(),
+        resolved_config.for_crawl_runtime(),
        json_safe=False,
    )

@@ -333,12 +327,15 @@ def run_hook(
        )
        return process

-    # Determine the interpreter based on file extension
+    # Python hooks carry their runtime contract in the shebang
+    # (usually `abxpkg run --script python3`), so execute them directly.
+    # For shell/JS hooks we still dispatch through the conventional
+    # interpreter because those hooks do not need per-script Python env setup.
    ext = script.suffix.lower()
    if ext == ".sh":
        cmd = ["bash", str(script)]
    elif ext == ".py":
-        cmd = [sys.executable, str(script)]
+        cmd = [str(script)]
    elif ext == ".js":
        cmd = ["node", str(script)]
    else:
@@ -368,6 +365,9 @@ def run_hook(

    # Set up environment with base paths
    env = os.environ.copy()
+    archivebox_config_input_names = _archivebox_config_input_names()
+    for key in archivebox_config_input_names:
+        env.pop(key, None)
    env["DATA_DIR"] = str(CONSTANTS.DATA_DIR)
    env["LIBRARY_VERSION"] = VERSION
    env.setdefault("MACHINE_ID", os.environ.get("MACHINE_ID", CONSTANTS.MACHINE_ID))
@@ -378,19 +378,18 @@ def run_hook(
    if crawl_dir:
        env["CRAWL_DIR"] = str(crawl_dir)

-    # Export runtime library roots; abx-dl/abxpkg own executable lookup env.
-    lib_dir = resolved_config.LIB_DIR
+    # Export the runtime library root; abx-dl/abxpkg own executable lookup env.
+    lib_dir = hook_config.get("ABXPKG_LIB_DIR")
    if lib_dir:
-        env["LIB_DIR"] = str(lib_dir)
        env["ABXPKG_LIB_DIR"] = str(lib_dir)

    # Set Node.js module resolution paths.
    # NODE_PATH may be a path list, but NODE_MODULES_DIR is a single canonical directory.
-    node_modules_dir = resolved_config.get("NODE_MODULES_DIR")
+    node_modules_dir = hook_config.get("NODE_MODULES_DIR")
    if not node_modules_dir and lib_dir:
-        node_modules_dir = Path(lib_dir) / "npm" / "node_modules"
+        node_modules_dir = Path(lib_dir) / "pnpm" / "packages" / "chrome" / "node_modules"

-    node_path_parts = [part for part in str(resolved_config.get("NODE_PATH") or "").split(os.pathsep) if part]
+    node_path_parts = [part for part in str(hook_config.get("NODE_PATH") or "").split(os.pathsep) if part]
    if node_modules_dir:
        node_modules_dir = Path(node_modules_dir)
        node_modules_dir.mkdir(parents=True, exist_ok=True)
@@ -403,12 +402,10 @@ def run_hook(
        env["NODE_PATH"] = os.pathsep.join(node_path_parts)

    # Export all config values to environment (already merged by get_config())
-    # Skip keys we've already handled specially above (PATH, LIB_DIR, NODE_PATH, etc.)
+    # Skip keys we've already handled specially above (PATH, ABXPKG_LIB_DIR, NODE_PATH, etc.)
    SKIP_KEYS = {
        "PATH",
-        "LIB_DIR",
        "ABXPKG_LIB_DIR",
-        "LIB_BIN_DIR",
        "NODE_PATH",
        "NODE_MODULES_DIR",
        "NODE_MODULE_DIR",
@@ -417,9 +414,12 @@ def run_hook(
        "SNAP_DIR",
        "CRAWL_DIR",
    }
+    canonical_config_keys = set(ArchiveBoxConfig.model_fields)
    for key, value in hook_config.items():
        if key in SKIP_KEYS:
            continue  # Already handled specially above, don't overwrite
+        if key in archivebox_config_input_names and key not in canonical_config_keys:
+            continue
        if value is None:
            continue
        elif isinstance(value, bool):
@@ -432,8 +432,8 @@ def run_hook(
    # Create output directory if needed
    output_dir.mkdir(parents=True, exist_ok=True)

-    # Detect if this is a background hook (long-running daemon)
-    # Background hooks use the .daemon.bg. or .finite.bg. filename convention.
+    # Detect if this is a background hook.
+    # Background hooks use the .bg. filename marker.
    # Old convention: __background in stem (for backwards compatibility)
    is_background = ".bg." in script.name or "__background" in script.stem

@@ -14,13 +14,13 @@ from django.utils.safestring import mark_safe
 from admin_data_views.typing import ItemContext, SectionData, TableContext
 from admin_data_views.utils import ItemLink, render_with_item_view, render_with_table_view

+from archivebox.config.common import get_live_config_url
 from archivebox.config.views import get_environment_binary_url, is_superuser
 from archivebox.plugins.discovery import BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR, discover_plugin_configs, iter_plugin_dirs


 ABX_PLUGINS_DOCS_BASE_URL = "https://archivebox.github.io/abx-plugins/"
 ABX_PLUGINS_GITHUB_BASE_URL = "https://github.com/ArchiveBox/abx-plugins/tree/main/abx_plugins/plugins/"
-LIVE_CONFIG_BASE_URL = "/admin/environment/config/"
 LIVE_PLUGIN_BASE_URL = "/admin/environment/plugins/"


@@ -73,18 +73,10 @@ def get_plugin_hook_source_url(plugin_name: str, hook_name: str) -> str:
    return f"{ABX_PLUGINS_GITHUB_BASE_URL}{quote(plugin_name)}/{quote(hook_name)}"


-def get_live_config_url(key: str) -> str:
-    return f"{LIVE_CONFIG_BASE_URL}{quote(key)}/"
-
-
 def get_machine_admin_url() -> str | None:
-    try:
-        from archivebox.machine.models import Machine
+    from archivebox.machine.models import Machine

-        machine = Machine.current()
-        return machine.admin_change_url or f"/admin/machine/machine/{machine.id}/change/"
-    except Exception:
-        return None
+    return Machine.current().admin_change_url


 def render_code_tag_list(values: list[str]) -> str:
@@ -35,6 +35,9 @@
        top: 0;
        z-index: 10;
    }
+    #progress-monitor.collapsed .header-bar {
+        cursor: pointer;
+    }
    #progress-monitor .header-left {
        display: flex;
        align-items: center;
@@ -71,7 +74,12 @@
        animation: idle-pulse 5s infinite;
    }
    #progress-monitor .status-dot.stopped {
-        background: #6e7681;
+        background: #d29922;
+        box-shadow: 0 0 4px #d29922;
+    }
+    #progress-monitor .status-dot.error {
+        background: #f85149;
+        box-shadow: 0 0 8px #f85149;
    }
    #progress-monitor .status-dot.flash {
        animation: flash 0.3s ease-out;
@@ -967,19 +975,19 @@
            <div class="stats">
                <div class="stat">
                    <span class="stat-label">Crawls</span>
-                    <span class="stat-value compact info"><span id="crawls-active">0</span> active · <span id="crawls-queued">0</span> queued</span>
+                    <span class="stat-value compact info"><span id="crawls-active-segment"><span id="crawls-active">0</span> active · </span><span id="crawls-queued">0</span> queued</span>
                </div>
                <div class="stat">
                    <span class="stat-label">Snapshots</span>
-                    <span class="stat-value compact info"><span id="snapshots-active">0</span> active · <span id="snapshots-queued">0</span> queued</span>
+                    <span class="stat-value compact info"><span id="snapshots-active-segment"><span id="snapshots-active">0</span> active · </span><span id="snapshots-queued">0</span> queued</span>
                </div>
                <div class="stat">
                    <span class="stat-label">Downloads</span>
-                    <span class="stat-value compact warning"><span id="downloads-active">0</span> active · <span id="downloads-queued">0</span> queued</span>
+                    <span class="stat-value compact warning"><span id="downloads-active-segment"><span id="downloads-active">0</span> active · </span><span id="downloads-queued">0</span> queued</span>
                </div>
                <div class="stat">
                    <span class="stat-label">Indexing</span>
-                    <span class="stat-value compact success"><span id="indexing-active">0</span> active · <span id="indexing-queued">0</span> queued</span>
+                    <span class="stat-value compact success"><span id="indexing-active-segment"><span id="indexing-active">0</span> active · </span><span id="indexing-queued">0</span> queued</span>
                </div>
            </div>
        </div>
@@ -1682,17 +1690,24 @@
        `;
    }

+    function setOrchestratorState(state, label) {
+        const dot = document.getElementById('orchestrator-dot');
+        dot.classList.remove('stopped', 'idle', 'running', 'error');
+        dot.classList.add(state);
+        document.getElementById('orchestrator-text').textContent = label;
+        return dot;
+    }
+
    function updateProgress(data) {
+        idleMessage.style.color = '';
        function setCount(id, value) {
            const el = document.getElementById(id);
            if (el) el.textContent = Number(value || 0).toLocaleString();
        }
-        function setOrchestratorState(state, label) {
-            const dot = document.getElementById('orchestrator-dot');
-            dot.classList.remove('stopped', 'idle', 'running');
-            dot.classList.add(state);
-            document.getElementById('orchestrator-text').textContent = label;
-            return dot;
+        function setActiveCount(id, value) {
+            setCount(id, value);
+            const segment = document.getElementById(`${id}-segment`);
+            if (segment) segment.style.display = Number(value || 0) > 0 ? '' : 'none';
        }

        // Calculate if there's activity
@@ -1715,17 +1730,15 @@
        // Update orchestrator status - show "Running" only when there are active workers.
        const pidEl = document.getElementById('orchestrator-pid');
        const hasWorkers = data.total_workers > 0;
-        const hasBlockedCrawl = (data.active_crawls || []).some(c => c.worker_state === 'crashed');
        let dot = null;

        if (hasWorkers) {
            dot = setOrchestratorState('running', 'Running');
-        } else if (hasActivity && hasBlockedCrawl && !data.orchestrator_running) {
+        } else if (!data.orchestrator_running) {
            dot = setOrchestratorState('stopped', 'Runner stopped');
        } else if (hasActivity) {
-            dot = setOrchestratorState('idle', data.orchestrator_running ? 'Idle' : 'Waiting');
+            dot = setOrchestratorState('idle', 'Idle');
        } else {
-            // No activity - show as idle (whether orchestrator process exists or not)
            dot = setOrchestratorState('idle', 'Idle');
        }

@@ -1742,15 +1755,17 @@
        setTimeout(() => dot.classList.remove('flash'), 300);

        [
-            ['crawls-active', data.crawls_active],
            ['crawls-queued', data.crawls_queued],
-            ['snapshots-active', data.snapshots_active],
            ['snapshots-queued', data.snapshots_queued],
-            ['downloads-active', data.downloads_active],
            ['downloads-queued', data.downloads_queued],
-            ['indexing-active', data.indexing_active],
            ['indexing-queued', data.indexing_queued],
        ].forEach(([id, value]) => setCount(id, value));
+        [
+            ['crawls-active', data.crawls_active],
+            ['snapshots-active', data.snapshots_active],
+            ['downloads-active', data.downloads_active],
+            ['indexing-active', data.indexing_active],
+        ].forEach(([id, value]) => setActiveCount(id, value));
        updateScreencastPanel(data);

        // Render crawl tree
@@ -1784,18 +1799,24 @@

    function fetchProgress() {
        fetch(progressEndpoint, { credentials: 'same-origin' })
-            .then(response => response.json())
+            .then(response => {
+                if (!response.ok) throw new Error(`HTTP ${response.status}`);
+                return response.json();
+            })
            .then(data => {
                if (data.error) {
                    console.error('Progress API error:', data.error, data.traceback);
+                    setOrchestratorState('error', 'Backend not responding');
                    idleMessage.textContent = 'API Error: ' + data.error;
                    idleMessage.style.color = '#f85149';
+                    return;
                }
                monitor.classList.toggle('is-guest', data.is_admin === false);
                updateProgress(data);
            })
            .catch(error => {
                console.error('Progress fetch error:', error);
+                setOrchestratorState('error', 'Backend not responding');
                idleMessage.textContent = 'Fetch Error: ' + error.message;
                idleMessage.style.color = '#f85149';
            });
@@ -1866,9 +1887,14 @@
    }

    // Collapse toggle
-    collapseBtn.addEventListener('click', function() {
+    collapseBtn.addEventListener('click', function(event) {
+        event.stopPropagation();
        setCollapsedState(!isCollapsed);
    });
+    monitor.querySelector('.header-bar').addEventListener('click', function(event) {
+        if (event.target.closest('button, a, input, select, textarea, [role="button"]')) return;
+        if (isCollapsed) setCollapsedState(false);
+    });

    crawlTree.addEventListener('click', function(event) {
        const actionBtn = event.target.closest('.crawl-action-btn');
@@ -4,6 +4,7 @@ from functools import lru_cache
 from pathlib import Path
 from typing import Literal

+from django.conf import settings
 from django.db.models import CharField, Count, Q, Sum
 from django.db.models.functions import Cast
 from django.http import HttpResponse, JsonResponse
@@ -327,8 +328,20 @@ def live_progress_view(request):
            .order_by("-modified_at")[:max_active_crawls],
        )
        paused_crawls = list(
-            crawl_scope.filter(status=Crawl.StatusChoices.PAUSED, created_at__gte=paused_crawl_cutoff)
+            crawl_scope.filter(
+                Q(status=Crawl.StatusChoices.PAUSED, created_at__gte=paused_crawl_cutoff)
+                | Q(
+                    status=Crawl.StatusChoices.PAUSED,
+                    snapshot_set__status__in=Snapshot.RUNNABLE_STATES,
+                    snapshot_set__retry_at__lte=now,
+                )
+                | Q(
+                    status=Crawl.StatusChoices.PAUSED,
+                    snapshot_set__archiveresult__status=ArchiveResult.StatusChoices.QUEUED,
+                ),
+            )
            .values(*active_crawl_fields)
+            .distinct()
            .order_by("-modified_at")[:max_active_crawls],
        )
        queued_crawls = list(
@@ -923,27 +936,26 @@ def live_progress_view(request):
        except ImportError:
            return JsonResponse(payload)
    except Exception as e:
-        import traceback
+        error_payload = {
+            "error": str(e),
+            "orchestrator_running": False,
+            "total_workers": 0,
+            "crawls_active": 0,
+            "crawls_queued": 0,
+            "crawls_recent": 0,
+            "snapshots_active": 0,
+            "snapshots_queued": 0,
+            "archiveresults_active": 0,
+            "archiveresults_queued": 0,
+            "downloads_active": 0,
+            "downloads_queued": 0,
+            "indexing_active": 0,
+            "indexing_queued": 0,
+            "active_crawls": [],
+            "server_time": timezone.now().isoformat(),
+        }
+        if settings.DEBUG:
+            import traceback

-        return JsonResponse(
-            {
-                "error": str(e),
-                "traceback": traceback.format_exc(),
-                "orchestrator_running": False,
-                "total_workers": 0,
-                "crawls_active": 0,
-                "crawls_queued": 0,
-                "crawls_recent": 0,
-                "snapshots_active": 0,
-                "snapshots_queued": 0,
-                "archiveresults_active": 0,
-                "archiveresults_queued": 0,
-                "downloads_active": 0,
-                "downloads_queued": 0,
-                "indexing_active": 0,
-                "indexing_queued": 0,
-                "active_crawls": [],
-                "server_time": timezone.now().isoformat(),
-            },
-            status=500,
-        )
+            error_payload["traceback"] = traceback.format_exc()
+        return JsonResponse(error_payload, status=500)
@@ -2,6 +2,7 @@ __package__ = "archivebox.search"

 from django.contrib import admin
 from django.contrib.admin.views.main import ChangeList
+from django.db.models import Case, IntegerField, Value, When

 from archivebox.search.config import (
    get_default_search_mode,
@@ -71,7 +72,13 @@ class SearchResultsAdminMixin(admin.ModelAdmin):
        if queryset.model._meta.label_lower == "core.snapshot" and request.GET.get("_embedded") != "crawl":
            cached_ids = get_cached_admin_search_ids(request)
            if cached_ids is not None:
-                return queryset.filter(pk__in=cached_ids) if cached_ids else queryset.none(), False
+                if not cached_ids:
+                    return queryset.none(), False
+                search_rank = Case(
+                    *(When(pk=snapshot_id, then=Value(index)) for index, snapshot_id in enumerate(cached_ids)),
+                    output_field=IntegerField(),
+                )
+                return queryset.filter(pk__in=cached_ids).annotate(search_rank=search_rank).order_by("search_rank"), False
            return queryset.none(), False

        if get_search_mode_base(search_mode, config=request.archivebox_config) == "meta":
@@ -62,7 +62,7 @@ def get_search_mode_options(config: dict[str, Any] | None = None, **config_kwarg
    ]
    options = [
        {"value": "meta", "label": "meta"},
-        {"value": "contents", "label": "contents"},
+        {"value": "contents", "label": "deep"},
    ]
    if backend_names:
        options.extend(
@@ -4,20 +4,23 @@ import asyncio
 import hashlib
 import json
 import threading
+import time
 from copy import copy
 from queue import Full, Queue
 from urllib.parse import urlsplit
 from uuid import UUID

 from django.core.cache import cache
+from django.db import connections
 from django.db.models import Q
-from django.http import QueryDict, StreamingHttpResponse
+from django.http import HttpResponseForbidden, QueryDict, StreamingHttpResponse

 from archivebox.search.config import get_search_mode, get_search_mode_base
 from archivebox.search.query import crawl_config_values_search_wave, iter_query_search_ids


 SEARCH_RESULT_CACHE_TTL = 60
+URL_PREFIX_SEARCH_LIMIT = 500


 def get_admin_search_cache_key(request, url: str | None = None) -> str:
@@ -34,6 +37,17 @@ def get_admin_search_cache_key(request, url: str | None = None) -> str:
    return f"abx:admin-search:{hashlib.sha256(payload.encode()).hexdigest()}"


+def get_public_search_cache_key(request, url: str | None = None) -> str:
+    """Build the cache key for one public search URL."""
+    payload = json.dumps(
+        {
+            "url": url or request.get_full_path(),
+        },
+        sort_keys=True,
+    )
+    return f"abx:public-search:{hashlib.sha256(payload.encode()).hexdigest()}"
+
+
 def get_cached_admin_search_ids(request) -> list[str] | None:
    """Return streamed admin search IDs from Django cache."""
    cached = cache.get(get_admin_search_cache_key(request))
@@ -42,7 +56,100 @@ def get_cached_admin_search_ids(request) -> list[str] | None:
    return None


-def iter_admin_meta_search_ids(query, queryset):
+def get_cached_public_search_ids(request) -> list[str] | None:
+    """Return streamed public search IDs from Django cache."""
+    cached = get_cached_public_search_state(request)
+    if isinstance(cached, dict):
+        return cached.get("ids") or []
+    return None
+
+
+def get_cached_public_search_state(request) -> dict | None:
+    """Return streamed public search state from Django cache."""
+    cached = cache.get(get_public_search_cache_key(request))
+    return cached if isinstance(cached, dict) else None
+
+
+def iter_url_search_prefixes(query: str):
+    """Yield URL prefixes that can use indexed startswith scans for common search input."""
+    query = query.strip().lower()
+    if not query or any(char.isspace() for char in query):
+        return
+
+    prefixes = []
+
+    def add(prefix: str):
+        if prefix and prefix not in prefixes:
+            prefixes.append(prefix)
+
+    add(query)
+    if "://" in query:
+        parsed = urlsplit(query)
+        if parsed.scheme and parsed.netloc:
+            host = parsed.netloc
+            path = parsed.path or ""
+            if parsed.query:
+                path = f"{path}?{parsed.query}"
+            if host.startswith("www."):
+                add(f"{parsed.scheme}://{host[4:]}{path}")
+            else:
+                add(f"{parsed.scheme}://www.{host}{path}")
+    else:
+        trimmed = query.lstrip("/")
+        for scheme in ("https://", "http://"):
+            add(f"{scheme}{trimmed}")
+            if trimmed.startswith("www."):
+                add(f"{scheme}{trimmed[4:]}")
+            else:
+                add(f"{scheme}www.{trimmed}")
+
+    yield from prefixes
+
+
+def url_prefix_upper_bound(prefix: str) -> str:
+    """Return the exclusive upper bound for an indexed URL prefix range."""
+    if not prefix:
+        return prefix
+    return f"{prefix[:-1]}{chr(ord(prefix[-1]) + 1)}"
+
+
+def iter_url_prefix_search_ids(prefix: str, queryset):
+    """Yield IDs for one URL prefix using the URL index, then apply caller filters."""
+    if not prefix:
+        return
+
+    model = queryset.model
+    db_alias = queryset.db
+    connection = connections[db_alias]
+    table = connection.ops.quote_name(model._meta.db_table)
+    pk_column = connection.ops.quote_name(model._meta.pk.column)
+    url_column = connection.ops.quote_name(model._meta.get_field("url").column)
+    upper_bound = url_prefix_upper_bound(prefix)
+    raw_ids = []
+
+    with connection.cursor() as cursor:
+        cursor.execute(
+            f"""
+            SELECT {pk_column}
+            FROM {table}
+            WHERE {url_column} >= %s AND {url_column} < %s
+            ORDER BY {url_column}
+            LIMIT %s
+            """,
+            [prefix, upper_bound, URL_PREFIX_SEARCH_LIMIT],
+        )
+        raw_ids = [str(row[0]).replace("-", "") for row in cursor.fetchall()]
+
+    if not raw_ids:
+        return
+
+    valid_ids = {str(pk).replace("-", "") for pk in queryset.filter(pk__in=raw_ids).values_list("pk", flat=True)}
+    for snapshot_id in raw_ids:
+        if snapshot_id in valid_ids:
+            yield snapshot_id
+
+
+def iter_meta_search_ids(query, queryset):
    """Yield metadata search matches from a filtered Snapshot queryset."""
    seen = set()
    try:
@@ -54,18 +161,20 @@ def iter_admin_meta_search_ids(query, queryset):
            seen.add(pk)
            yield pk

+    for prefix in iter_url_search_prefixes(query):
+        for pk in iter_url_prefix_search_ids(prefix, queryset):
+            if pk in seen:
+                continue
+            seen.add(pk)
+            yield pk
+
    waves = [
-        Q(timestamp__startswith=query) | Q(url__istartswith=query) | Q(title__istartswith=query),
+        Q(timestamp__startswith=query) | Q(title__istartswith=query),
        Q(url__icontains=query),
        Q(title__icontains=query),
        Q(tags__name__icontains=query),
-        Q(notes__icontains=query) | Q(crawl__notes__icontains=query) | Q(crawl__label__icontains=query),
-        Q(crawl__created_by__username=query),
+        Q(notes__icontains=query),
    ]
-    config_wave = crawl_config_values_search_wave(query)
-    if config_wave is not None:
-        waves.append(config_wave)
-
    for wave in waves:
        for pk in queryset.filter(wave).values_list("pk", flat=True).distinct().iterator(chunk_size=500):
            if pk in seen:
@@ -73,38 +182,157 @@ def iter_admin_meta_search_ids(query, queryset):
            seen.add(pk)
            yield pk

+    crawl_metadata_wave = Q(crawl__notes__icontains=query) | Q(crawl__label__icontains=query) | Q(crawl__created_by__username=query)
+    if not seen:
+        for pk in queryset.filter(crawl_metadata_wave).values_list("pk", flat=True).distinct().iterator(chunk_size=500):
+            seen.add(pk)
+            yield pk

-def iter_admin_backend_search_ids(iterator, queryset):
-    """Yield backend search IDs that still match the filtered queryset."""
+    config_wave = crawl_config_values_search_wave(query)
+    if config_wave is not None and not seen:
+        for pk in queryset.filter(config_wave).values_list("pk", flat=True).distinct().iterator(chunk_size=500):
+            seen.add(pk)
+            yield pk
+
+
+def normalize_search_result_id(snapshot_id) -> str | None:
+    """Return a compact Snapshot ID string from a search provider result."""
+    snapshot_id = str(snapshot_id).strip().lower().replace("-", "")
+    if len(snapshot_id) != 32:
+        return None
+    return snapshot_id
+
+
+def iter_filtered_search_result_ids(iterator, queryset, *, flush_max_delay=0.05):
+    """Yield provider IDs that still match the filtered queryset.
+
+    This is the single intersection/dedupe path used for metadata and every
+    search backend. It flushes by elapsed time so sparse providers stream rows
+    as soon as IDs are found instead of waiting for a fixed batch size.
+    """
    batch = []
    seen = set()
+    queued = set()
+    last_flush_at = 0.0

    def flush_batch():
-        valid = {str(pk) for pk in queryset.filter(pk__in=batch).values_list("pk", flat=True)}
-        for snapshot_id in batch:
+        nonlocal batch, queued, last_flush_at
+        if not batch:
+            return
+        batch_ids = batch
+        batch = []
+        queued = set()
+        last_flush_at = time.monotonic()
+        valid = {str(pk).replace("-", "") for pk in queryset.filter(pk__in=batch_ids).values_list("pk", flat=True)}
+        for snapshot_id in batch_ids:
            if snapshot_id in valid and snapshot_id not in seen:
                seen.add(snapshot_id)
                yield snapshot_id

    for snapshot_id in iterator:
-        snapshot_id = str(snapshot_id).strip().lower().replace("-", "")
-        if len(snapshot_id) != 32:
+        snapshot_id = normalize_search_result_id(snapshot_id)
+        if not snapshot_id or snapshot_id in seen or snapshot_id in queued:
            continue
        batch.append(snapshot_id)
-        if len(batch) >= 200:
+        queued.add(snapshot_id)
+        if not seen or time.monotonic() - last_flush_at >= flush_max_delay:
            yield from flush_batch()
-            batch = []
    if batch:
        yield from flush_batch()


+def iter_search_result_ids(query, base_queryset, *, search_mode, config):
+    """Yield filtered Snapshot IDs from the selected search provider."""
+    search_mode_base = get_search_mode_base(search_mode, config=config)
+    provider = (
+        iter_meta_search_ids(query, base_queryset)
+        if search_mode_base == "meta"
+        else iter_query_search_ids(query, search_mode=search_mode, config=config)
+    )
+    yield from iter_filtered_search_result_ids(provider, base_queryset)
+
+
+def snapshot_search_stream_response(query, base_queryset, *, search_mode, config, cache_key, thread_name):
+    """Stream Snapshot search progress and cache matching IDs for a list view."""
+    if not query:
+        return StreamingHttpResponse((), content_type="text/plain")
+
+    async def snapshot_ids():
+        seen = set()
+        ids = []
+        last_sent = 0
+        last_sent_at = time.monotonic()
+        stream_max_delay = 0.05
+        stream_padding = " " * 4096
+        cache.set(cache_key, {"ids": [], "done": False}, SEARCH_RESULT_CACHE_TTL)
+        queue = Queue(maxsize=8)
+        stop_event = threading.Event()
+
+        def emit(item):
+            while not stop_event.is_set():
+                try:
+                    queue.put(item, timeout=0.1)
+                    return
+                except Full:
+                    continue
+
+        def publish_count(done=False):
+            nonlocal last_sent, last_sent_at
+            cache.set(cache_key, {"ids": list(ids), "done": done}, SEARCH_RESULT_CACHE_TTL)
+            last_sent = len(ids)
+            last_sent_at = time.monotonic()
+            emit(f"{last_sent}{stream_padding}\n")
+
+        def run_search():
+            iterator = None
+            try:
+                iterator = iter_search_result_ids(query, base_queryset, search_mode=search_mode, config=config)
+                for snapshot_id in iterator:
+                    if stop_event.is_set():
+                        break
+                    snapshot_id = normalize_search_result_id(snapshot_id)
+                    if not snapshot_id or snapshot_id in seen:
+                        continue
+                    seen.add(snapshot_id)
+                    ids.append(snapshot_id)
+                    if len(ids) == 1 or time.monotonic() - last_sent_at >= stream_max_delay:
+                        publish_count()
+                if not stop_event.is_set() and len(ids) != last_sent:
+                    publish_count(done=True)
+            except BaseException as err:
+                emit(err)
+            finally:
+                if iterator is not None:
+                    try:
+                        iterator.close()
+                    except AttributeError:
+                        pass
+                cache.set(cache_key, {"ids": list(ids), "done": True}, SEARCH_RESULT_CACHE_TTL)
+                emit(None)
+
+        threading.Thread(target=run_search, name=thread_name, daemon=True).start()
+        yield f"0{stream_padding}\n"
+        try:
+            while True:
+                item = await asyncio.to_thread(queue.get)
+                if item is None:
+                    break
+                if isinstance(item, BaseException):
+                    raise item
+                yield item
+        finally:
+            stop_event.set()
+
+    response = StreamingHttpResponse(snapshot_ids(), content_type="text/plain")
+    response["X-Accel-Buffering"] = "no"
+    return response
+
+
 def admin_snapshot_search_stream_view(model_admin, request):
    """Stream admin Snapshot search progress and cache matching IDs."""
    query = (request.GET.get("q") or "").strip()
    config = request.archivebox_config
    search_mode = get_search_mode(request.GET.get("search_mode"), config=config)
-    if not query:
-        return StreamingHttpResponse((), content_type="text/plain")

    search_url = request.GET.get("search_url") or request.get_full_path()
    target_url = urlsplit(search_url)
@@ -127,77 +355,36 @@ def admin_snapshot_search_stream_view(model_admin, request):
    finally:
        model_admin.request = current_request

-    async def snapshot_ids():
-        seen = set()
-        ids = []
-        last_sent = 0
-        stream_batch_size = 100
-        stream_padding = " " * 4096
-        cache_key = get_admin_search_cache_key(request, search_url)
-        cache.set(cache_key, {"ids": ids, "done": False}, SEARCH_RESULT_CACHE_TTL)
-        yield f"0{stream_padding}\n"
-        queue = Queue(maxsize=8)
-        stop_event = threading.Event()
+    return snapshot_search_stream_response(
+        query,
+        base_queryset,
+        search_mode=search_mode,
+        config=config,
+        cache_key=get_admin_search_cache_key(request, search_url),
+        thread_name="admin-snapshot-search-stream",
+    )

-        def emit(item):
-            while not stop_event.is_set():
-                try:
-                    queue.put(item, timeout=0.1)
-                    return
-                except Full:
-                    continue

-        def run_search():
-            nonlocal last_sent
-            iterator = None
-            try:
-                search_mode_base = get_search_mode_base(search_mode, config=config)
-                iterator = (
-                    iter_admin_meta_search_ids(query, base_queryset)
-                    if search_mode_base == "meta"
-                    else iter_admin_backend_search_ids(
-                        iter_query_search_ids(query, search_mode=search_mode, config=config),
-                        base_queryset,
-                    )
-                )
-                for snapshot_id in iterator:
-                    if stop_event.is_set():
-                        break
-                    snapshot_id = str(snapshot_id).strip().lower().replace("-", "")
-                    if len(snapshot_id) != 32 or snapshot_id in seen:
-                        continue
-                    seen.add(snapshot_id)
-                    ids.append(snapshot_id)
-                    if len(ids) - last_sent >= stream_batch_size:
-                        cache.set(cache_key, {"ids": ids, "done": False}, SEARCH_RESULT_CACHE_TTL)
-                        last_sent = len(ids)
-                        emit(f"{last_sent}{stream_padding}\n")
-                if not stop_event.is_set() and len(ids) != last_sent:
-                    cache.set(cache_key, {"ids": ids, "done": False}, SEARCH_RESULT_CACHE_TTL)
-                    emit(f"{len(ids)}{stream_padding}\n")
-            except BaseException as err:
-                emit(err)
-            finally:
-                if iterator is not None:
-                    try:
-                        iterator.close()
-                    except AttributeError:
-                        pass
-                cache.set(cache_key, {"ids": ids, "done": True}, SEARCH_RESULT_CACHE_TTL)
-                emit(None)
+def public_snapshot_search_stream_view(request):
+    """Stream public Snapshot search progress and cache matching IDs."""
+    from archivebox.config.common import get_request_config
+    from archivebox.core.models import Snapshot
+    from archivebox.core.permissions import public_snapshots_queryset

-        threading.Thread(target=run_search, name="admin-snapshot-search-stream", daemon=True).start()
-        try:
-            while True:
-                item = await asyncio.to_thread(queue.get)
-                if item is None:
-                    break
-                if isinstance(item, BaseException):
-                    raise item
-                yield item
-        finally:
-            stop_event.set()
+    config = getattr(request, "archivebox_config", None) or get_request_config(request, resolve_plugins=False)
+    if not request.user.is_authenticated and not config.PUBLIC_INDEX:
+        return HttpResponseForbidden("Public index is disabled")

-    response = StreamingHttpResponse(snapshot_ids(), content_type="text/plain")
-    response["X-Accel-Buffering"] = "no"
-    return response
+    query = (request.GET.get("q") or "").strip()
+    search_mode = get_search_mode(request.GET.get("search_mode"), config=config)
+    search_url = request.GET.get("search_url") or request.get_full_path()
+    base_queryset = public_snapshots_queryset(Snapshot.objects.all())
+
+    return snapshot_search_stream_response(
+        query,
+        base_queryset,
+        search_mode=search_mode,
+        config=config,
+        cache_key=get_public_search_cache_key(request, search_url),
+        thread_name="public-snapshot-search-stream",
+    )