Bump release version to 0.9.35rc482

Bump for abx-dl 1.12.300
Bump release version to 0.9.35rc481
2026-09-18 08:22:03 +00:00 · 2026-09-18 08:21:44 +00:00 · 2026-09-18 08:02:08 +00:00 · 2026-09-18 08:01:47 +00:00 · 2026-09-18 07:48:43 +00:00 · 2026-09-18 07:48:16 +00:00
233 changed files with 12490 additions and 12237 deletions
@@ -75,6 +75,15 @@ docker/
 website/
 typings/

+# The runtime COPY needs ArchiveBox's package and bin/ entrypoint, not the
+# development UI screenshots or test trees. PNGs are already compressed, so
+# including them creates a large image layer that later cleanup cannot remove.
+publicsite/
+old/
+test/
+tests/
+archivebox/tests/
+
 tmp/
 .tmp/
 data/
@@ -191,7 +191,6 @@ body:
        In the meantime please consider:  
          
          - 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
-          - 👨‍✈️ [Hiring us for corporate deployments](https://docs.monadical.com/s/archivebox-consulting-services) with professional support, custom feature development, and help with CAPTCHAs/rate-limits
          - 🔍 [Searching the Documentation](https://docs.archivebox.io/) for answers to common questions
          - 📚 Reading the [Troubleshooting Guide](https://github.com/ArchiveBox/ArchiveBox/wiki)
          - ✨ Testing out a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases) (issues are often already fixed in our latest `BETA` releases)
@@ -109,9 +109,6 @@ body:
          required: false
        - label: "I've had a lot of difficulty getting ArchiveBox set up"
          required: false
-        - label: "I would pay $10/mo for a hosted version of ArchiveBox if it had this feature"
-          required: false
-
  - type: markdown
    attributes:
      value: |
@@ -123,6 +120,5 @@ body:
        In the meantime please consider:  
          
          - 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
-          - 📊 [Hiring us for corporate deployments](https://docs.monadical.com/s/archivebox-consulting-services) with professional support, custom feature development, and help with CAPTCHAs/rate-limits
          - 🔍 [Searching the Documentation](https://docs.archivebox.io/) for answers to common questions
          - ✨ Testing out a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases) (issues are often already fixed in our latest `BETA` releases)
@@ -47,6 +47,5 @@ body:
        In the meantime please consider:  
          
          - 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
-          - 👨‍✈️ [Hiring us for corporate deployments](https://docs.monadical.com/s/archivebox-consulting-services) with professional support, custom feature development, and help with CAPTCHAs/rate-limits
          - 🔍 [Checking out the new ReadTheDocs Documentation](https://docs.archivebox.io/)
          - ✨ Helping us test a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases)
@@ -6,6 +6,3 @@ contact_links:
  - name: 💬 Chat with the dev team & community on Zulip
    url: https://zulip.archivebox.io
    about: "Join us on our Zulip forum to chat with the developers and other users (it's similar to Discord but self-hosted)."
-  - name: 💁‍♂️ Hire us for professional support with fast response times
-    url: https://docs.monadical.com/s/archivebox-consulting-services
-    about: "We provide hosting, development, and support, including on-prem/cloud w/ SSO & storage, CAPTCHA-solving, proxies, etc."
@@ -52,7 +52,7 @@ jobs:
    name: Tested Docker images
    uses: ./.github/workflows/docker.yml
    with:
-      push_digests: ${{ github.event_name == 'push' }}
+      push_digests: false
    secrets: inherit

  required:
@@ -1,63 +1,50 @@
 name: Deploy Publicsite to GitHub Pages
-
 on:
  push:
-    branches:
-      - dev
-    paths:
-      - publicsite/**
-      - archivebox/**
-      - bin/collect_ui_screenshots.*
-      - bin/generate_ui_screenshot_gallery.py
-      - bin/take_screenshot.js
-      - docs/Screenshots.md
-      - pyproject.toml
-      - uv.lock
-      - .github/workflows/deploy-publicsite.yml
+    branches: [dev]
+  pull_request:
  workflow_dispatch:
-
+  workflow_run:
+    workflows: ['Capture publicsite screenshots']
+    types: [completed]
+    branches: [dev]
 permissions:
  contents: read
-  pages: write
-  id-token: write
-
+  actions: read
 concurrency:
-  group: ${{ github.event_name == 'workflow_dispatch' && format('github-pages-publicsite-manual-{0}', github.run_id) || 'github-pages-publicsite' }}
-  cancel-in-progress: ${{ github.event_name != 'workflow_dispatch' }}
-
+  # Failed captures must not displace a pending push deployment.
+  group: fast-pages-${{ github.event_name == 'workflow_run' && github.event.workflow_run.conclusion != 'success' && github.run_id || github.ref }}
+  cancel-in-progress: false
 jobs:
+  build:
+    if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success'
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event_name == 'pull_request' && github.ref || 'dev' }}
+      - uses: astral-sh/setup-uv@v7
+      - name: Restore latest complete screenshots
+        run: uv run --no-project python bin/restore_site_screenshots.py archivebox publicsite/screenshots
+        env:
+          GH_TOKEN: ${{ github.token }}
+      - name: Render current gallery layout without recapturing images
+        run: uv run --no-project python bin/generate_ui_screenshot_gallery.py refresh publicsite/screenshots/index.html
+      - uses: actions/upload-pages-artifact@v3
+        with:
+          path: publicsite
  deploy:
+    if: github.event_name != 'pull_request'
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      pages: write
+      id-token: write
    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}
-    runs-on: ubuntu-24.04
    steps:
-      - name: Checkout
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 1
-
-      - name: Setup Python and uv
-        uses: astral-sh/setup-uv@v6
-        with:
-          enable-cache: false
-
-      - name: Install ArchiveBox and screenshot dependencies
-        run: uv sync --no-cache --frozen
-
-      - name: Build UI screenshot gallery
-        run: bin/collect_ui_screenshots.sh
-        env:
-          BASE_URL: http://archivebox.localhost:8000
-
-      - name: Setup Pages
-        uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5
-
-      - name: Upload artifact
-        uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3
-        with:
-          path: ./publicsite
-
-      - name: Deploy to GitHub Pages
+      - uses: actions/configure-pages@v5
+      - uses: actions/deploy-pages@v4
        id: deployment
-        uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4
@@ -8,6 +8,11 @@ on:
        required: false
        default: false
        type: boolean
+      full_tests:
+        description: Run storage, CLI, and size validation in addition to release smoke checks
+        required: false
+        default: true
+        type: boolean

 env:
  DOCKERHUB_IMAGE: archivebox/archivebox
@@ -28,12 +33,10 @@ jobs:
        include:
          - platform: linux/amd64
            runner: ubuntu-24.04
-            cache_scope: docker-amd64
            artifact_name: digest-linux-amd64
            local_tag: archivebox-ci:amd64
          - platform: linux/arm64
            runner: ubuntu-24.04-arm
-            cache_scope: docker-arm64
            artifact_name: digest-linux-arm64
            local_tag: archivebox-ci:arm64
    steps:
@@ -58,26 +61,27 @@ jobs:
        shell: bash
        run: |
          set -Eeuo pipefail
-          ABXPKG_VERSION="$(uv run --no-cache --no-project python - <<'PY'
+          ABXPKG_SPEC="$(uv run --no-cache --no-project python - <<'PY'
          import tomllib
          from pathlib import Path

          packages = tomllib.loads(Path("uv.lock").read_text())["package"]
-          matches = [package["version"] for package in packages if package["name"] == "abxpkg"]
+          matches = [package for package in packages if package["name"] == "abxpkg"]
          if len(matches) != 1:
              raise SystemExit(f"Expected one locked abxpkg package, found {len(matches)}")
-          print(matches[0])
+          wheel = matches[0]["wheels"][0]
+          print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))
          PY
          )"
-          test -n "$ABXPKG_VERSION"
+          test -n "$ABXPKG_SPEC"
          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
          export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
-          echo "ABXPKG_VERSION=$ABXPKG_VERSION" >> "$GITHUB_ENV"
+          echo "ABXPKG_SPEC=$ABXPKG_SPEC" >> "$GITHUB_ENV"
          echo "$ABXPKG_LIB_DIR/env/bin" >> "$GITHUB_PATH"

          docker_env="$(
-            uv run --no-cache --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
+            uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env \
              --install \
              --json \
              --lib="$ABXPKG_LIB_DIR" \
@@ -202,32 +206,48 @@ jobs:
            ARCHIVEBOX_COMMIT_HASH=${{ github.sha }}
          pull: true
          platforms: ${{ matrix.platform }}
+          # Keep the released image as the one cache authority. A second GHA
+          # cache can hold gzip variants of identical layers, and BuildKit may
+          # prefer those over :dev's zstd blobs. That counter-intuitively makes
+          # a cache hit larger than the released image. :dev advances only
+          # after both architectures pass the release checks.
+          cache-from: type=registry,ref=${{ env.DOCKERHUB_IMAGE }}:dev
+          # Inline metadata reuses the released compressed blobs directly;
+          # force-compression would recompress the whole ~740 MiB image. The
+          # compressed-size gate below remains the regression guard.
+          cache-to: |
+            ${{ inputs.push_digests && 'type=inline' || '' }}
          outputs: |
-            type=docker,name=${{ matrix.local_tag }}
-            type=oci,dest=${{ runner.temp }}/archivebox-image.tar
-            type=image,"name=${{ env.DOCKERHUB_IMAGE }},${{ env.GHCR_IMAGE }}",push-by-digest=true,name-canonical=true,push=true
+            ${{ inputs.full_tests && format('type=oci,dest={0}/archivebox-image.tar,compression=zstd,compression-level=3', runner.temp) || '' }}
+            ${{ inputs.push_digests && format('type=image,"name={0},{1}",push-by-digest=true,name-canonical=true,push=true,compression=zstd,compression-level=3', env.DOCKERHUB_IMAGE, env.GHCR_IMAGE) || '' }}

      - name: Build pull request image
+        id: docker_build_pr
        if: ${{ !inputs.push_digests }}
        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
        with:
          context: ./
          file: ./Dockerfile
          builder: ${{ steps.buildx.outputs.name }}
-          load: true
-          tags: ${{ matrix.local_tag }}
          labels: ${{ steps.docker_meta.outputs.labels }}
          build-args: |
            ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
            ARCHIVEBOX_COMMIT_HASH=${{ github.sha }}
          pull: true
          platforms: ${{ matrix.platform }}
+          # Test the same layer lineage users receive. Reusing :dev avoids a
+          # second cache whose compression can drift from the released image.
+          cache-from: type=registry,ref=${{ env.DOCKERHUB_IMAGE }}:dev
+          outputs: |
+            type=docker,name=${{ matrix.local_tag }}
+            type=oci,dest=${{ runner.temp }}/archivebox-image.tar

      - name: Image digest
        if: inputs.push_digests
        run: echo ${{ steps.docker_build.outputs.digest }}

      - name: Validate exact built image version and commit
+        if: ${{ !inputs.push_digests }}
        shell: bash
        run: |
          set -Eeuo pipefail
@@ -261,13 +281,32 @@ jobs:
          "$DOCKER_BINARY" run --rm --user 911:911 --platform '${{ matrix.platform }}' --entrypoint /venv/bin/python3 "$IMAGE" -c \
            'import json, pathlib, stat; cache_file = "/opt/archivebox/lib/env/derived.env"; raw = open(cache_file).read().strip(); records = json.loads(raw[len("ABXPKG_DERIVED_CACHE=\x27"):-1]); projections = [projection for record in records.values() for projection in record.get("request_exec_projections", {}).values()]; plans = [projection["validation"] for projection in projections]; fingerprints = [fingerprint for plan in plans for fingerprint in plan["fingerprint"]]; actual = lambda fingerprint: (lambda path, value: {"path": str(path.resolve(strict=False)), "size": value.st_size, "mtime_ns": value.st_mtime_ns, "mode": stat.S_IMODE(value.st_mode), "euid": value.st_uid})(pathlib.Path(fingerprint["path"]), pathlib.Path(fingerprint["path"]).stat()); mismatches = [(fingerprint, actual(fingerprint)) for fingerprint in fingerprints if fingerprint != actual(fingerprint)]; assert projections and {plan.get("euid") for plan in plans} == {911}; assert all(not record.get("script_exec_plans") for record in records.values()); assert "ABXPKG_TMP_CACHE_DIR" not in json.dumps(projections); assert not mismatches, mismatches[:3]'

+      - name: Validate compressed candidate size
+        if: inputs.push_digests
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          DIGEST='${{ steps.docker_build.outputs.digest }}'
+          [[ "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]
+          MANIFEST="$("$DOCKER_BINARY" buildx imagetools inspect --raw "${DOCKERHUB_IMAGE}@${DIGEST}")"
+          # Provenance wraps the real platform image and its attestation in an
+          # OCI index. Measure the image manifest, not the index/attestation.
+          IMAGE_DIGEST="$(MANIFEST="$MANIFEST" PLATFORM='${{ matrix.platform }}' "$UV_BINARY" run --no-cache --no-project python -c 'import json, os; manifest = json.loads(os.environ["MANIFEST"]); platform_os, platform_arch = os.environ["PLATFORM"].split("/")[:2]; print(next((item["digest"] for item in manifest.get("manifests", []) if item.get("platform", {}).get("os") == platform_os and item.get("platform", {}).get("architecture") == platform_arch), ""))')"
+          if [[ -n "$IMAGE_DIGEST" ]]; then
+            MANIFEST="$("$DOCKER_BINARY" buildx imagetools inspect --raw "${DOCKERHUB_IMAGE}@${IMAGE_DIGEST}")"
+          fi
+          COMPRESSED_SIZE="$(MANIFEST="$MANIFEST" "$UV_BINARY" run --no-cache --no-project python -c 'import json, os; manifest = json.loads(os.environ["MANIFEST"]); print(manifest["config"]["size"] + sum(layer["size"] for layer in manifest["layers"]))')"
+          LIMIT=$((750 * 1024 * 1024))
+          echo "${DIGEST} compressed_size=$((COMPRESSED_SIZE / 1024 / 1024)) MiB limit=750 MiB"
+          (( COMPRESSED_SIZE < LIMIT ))
+
      - name: Resolve real Docker CLI test dependencies through abxpkg
-        if: matrix.platform == 'linux/amd64'
+        if: inputs.full_tests && matrix.platform == 'linux/amd64'
        shell: bash
        run: |
          set -Eeuo pipefail
          mount_env="$(
-            uv run --no-cache --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
+            uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env \
              --install \
              --json \
              --lib="$ABXPKG_LIB_DIR" \
@@ -284,7 +323,7 @@ jobs:
          done

      - name: Validate Docker UID and GID behavior on real NFS and SMB mounts
-        if: matrix.platform == 'linux/amd64'
+        if: inputs.full_tests && matrix.platform == 'linux/amd64'
        shell: bash
        run: |
          set -Eeuo pipefail
@@ -318,7 +357,7 @@ jobs:
          "$SUDO_BINARY" "$MKDIR_BINARY" -p /etc/exports.d
          "$SUDO_BINARY" "$CHMOD_BINARY" 0777 "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
          "$SUDO_BINARY" "$BASH_BINARY" -c \
-            "printf '%s *(rw,sync,no_subtree_check,no_root_squash,insecure)\\n' '$NFS_SOURCE' > /etc/exports.d/archivebox-ci.exports"
+            "printf '%s *(rw,sync,no_subtree_check,root_squash,insecure)\\n' '$NFS_SOURCE' > /etc/exports.d/archivebox-ci.exports"
          "$SUDO_BINARY" "$BASH_BINARY" -c \
            "printf '\\n[archivebox-ci]\\npath = %s\\nbrowseable = yes\\nread only = no\\nguest ok = yes\\nguest only = yes\\nforce user = nobody\\ncreate mask = 0777\\ndirectory mask = 0777\\n' '$SMB_SOURCE' >> /etc/samba/smb.conf"

@@ -340,7 +379,7 @@ jobs:
            "$BASH_BINARY" bin/validate_docker_uid_gid.sh --local-only

      - name: Validate Docker CLI add/list and compose behavior
-        if: matrix.platform == 'linux/amd64'
+        if: inputs.full_tests && matrix.platform == 'linux/amd64'
        shell: bash
        env:
          ARCHIVEBOX_IMAGE: ${{ matrix.local_tag }}
@@ -362,29 +401,29 @@ jobs:
          "$DOCKER_BINARY" compose run --rm archivebox init
          "$DOCKER_BINARY" compose up -d --wait
          "$DOCKER_BINARY" network connect archivebox_default www-nginx
-          index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:8000')"
+          index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:8000/public/')"
          admin_js="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:8000/static/admin/js/jquery.init.js')"
          [[ "$index_html" == *ArchiveBox* ]]
          [[ "$admin_js" == *window.django* ]]

          "$DOCKER_BINARY" compose run --rm archivebox add --plugins=parse_txt_urls 'http://www-nginx/#test_docker'
-          index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:8000')"
+          index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:8000/public/')"
          [[ "$index_html" == *'http://www-nginx/#test_docker'* ]]
          "$DOCKER_BINARY" compose down

      - name: Validate compressed image size
-        if: inputs.push_digests
+        if: inputs.full_tests && !inputs.push_digests
        env:
-          BUILD_METADATA: ${{ steps.docker_build.outputs.metadata }}
+          BUILD_METADATA: ${{ steps.docker_build_pr.outputs.metadata }}
        shell: bash
        run: |
          set -Eeuo pipefail
          LOCAL_IMAGE_ID="$("$DOCKER_BINARY" image inspect '${{ matrix.local_tag }}' --format '{{.Id}}')"
          "$UV_BINARY" run --no-cache --no-project python - \
            '${{ runner.temp }}/archivebox-image.tar' \
-            '${{ steps.docker_build.outputs.digest }}' \
+            '${{ steps.docker_build_pr.outputs.digest }}' \
            "$LOCAL_IMAGE_ID" \
-            "$((910 * 1024 * 1024))" <<'PY'
+            "$((750 * 1024 * 1024))" <<'PY'
          from hashlib import sha256
          import json
          import os
@@ -67,6 +67,7 @@ jobs:
    timeout-minutes: 15
    strategy:
      fail-fast: false
+      max-parallel: 4
      matrix: ${{ fromJSON(needs.discover.outputs.matrix) }}
    steps:
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
@@ -98,9 +99,10 @@ jobs:
          if [[ '${{ matrix.environment }}' != macos ]]; then
            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docs_binaries")
          fi
-          if [[ '${{ matrix.environment }}' == ubuntu ]]; then
+          if [[ '${{ matrix.name }}' == ubuntu-AGENTS || '${{ matrix.name }}' == ubuntu-skills-archivebox-SKILL || '${{ matrix.name }}' == ubuntu-docs-Configuration || '${{ matrix.name }}' == ubuntu-docs-Setting-up-Authentication ]]; then
            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:required_binaries")
-          elif [[ '${{ matrix.environment }}' == docker ]]; then
+          fi
+          if [[ '${{ matrix.environment }}' == docker ]]; then
            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries")
          elif [[ '${{ matrix.environment }}' == macos ]]; then
            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-macos-brew.json:required_binaries")
@@ -1,96 +0,0 @@
-name: Duplicate Issue Detection
-
-on:
-  issues:
-    types: [opened]
-
-jobs:
-  check-duplicates:
-    runs-on: ubuntu-24.04
-    permissions:
-      contents: read
-      issues: write
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 1
-
-      - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: "3.13"
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-
-      - name: Resolve OpenCode and workflow tools through abxpkg
-        env:
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-        run: |
-          set -Eeuo pipefail
-          uv sync --no-cache --locked --no-sources --no-install-project
-          OPENCODE_CONFIG="$(uv run --no-cache --no-sync --no-sources python -c 'from abx_plugins import get_plugins_dir; print(get_plugins_dir() / "opencode" / "config.json")')"
-          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
-          export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
-          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
-          echo "$ABXPKG_LIB_DIR/env/bin" >> "$GITHUB_PATH"
-          opencode_env="$(
-            uv run --no-cache --no-sync --no-sources abxpkg env \
-              --install \
-              --json \
-              --lib="$ABXPKG_LIB_DIR" \
-              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \
-              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:github_binaries" \
-              --deps-from="$OPENCODE_CONFIG:required_binaries"
-          )"
-          jq_binary="$ABXPKG_LIB_DIR/env/bin/jq"
-          "$jq_binary" -r 'to_entries[] | "\(.key)=\(.value)"' \
-            <<< "$opencode_env" >> "$GITHUB_ENV"
-
-          opencode_binary="$ABXPKG_LIB_DIR/env/bin/opencode"
-          echo "OPENCODE_BINARY=$opencode_binary" >> "$GITHUB_ENV"
-          test -L "$opencode_binary"
-          test -x "$opencode_binary"
-
-      - name: Check for duplicate issues
-        env:
-          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          OPENCODE_PERMISSION: |
-            {
-              "bash": {
-                "gh issue*": "allow",
-                "gh pr*": "allow",
-                "*": "deny"
-              },
-              "webfetch": "allow"
-            }
-        run: |
-          "$OPENCODE_BINARY" run -m anthropic/claude-haiku-4-5 "A new issue has been created:
-
-          Issue number:
-          ${{ github.event.issue.number }}
-
-          Lookup this issue and search through existing issues and PRs (excluding #${{ github.event.issue.number }}) in this repository to find any potential duplicates of this new issue.
-          Consider:
-          1. Similar titles or descriptions
-          2. Same error messages or symptoms
-          3. Related functionality or components
-          4. Similar feature requests
-
-          If you find any potential duplicates, please comment on the new issue with:
-          - A brief explanation of why it might be a duplicate
-          - Links to the potentially duplicate issues or PRs
-          - A suggestion to check those issues first
-
-          Use this format for the comment:
-          'This issue might be a duplicate of existing issues. Please check:
-          - #[issue_number]: [brief description of similarity]
-
-          Feel free to ignore if none of these address your specific case.'
-
-          If no clear duplicates are found, do not comment."
@@ -2,6 +2,12 @@ name: Build Pip package

 on:
  workflow_call:
+    inputs:
+      full_tests:
+        description: Run the full package compatibility matrix instead of one release smoke check
+        required: false
+        default: true
+        type: boolean

 env:
  UV_VERSION: "0.11.3"
@@ -58,6 +64,21 @@ jobs:
          Path("dist/SHA256SUMS").write_text("\n".join(checksum_lines) + "\n")
          PY

+      - name: Release wheel import and CLI smoke
+        if: ${{ !inputs.full_tests }}
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          shopt -s nullglob
+          wheels=(dist/archivebox-*.whl)
+          [[ "${#wheels[@]}" -eq 1 ]]
+          uv sync --locked --no-cache --no-sources --no-dev --no-install-project
+          uv pip install --no-cache --no-deps --python .venv/bin/python "${wheels[0]}"
+          unset PYTHONPATH
+          cd "$(mktemp -d)"
+          VIRTUAL_ENV="$GITHUB_WORKSPACE/.venv" uv run --no-cache --active --no-project python -c 'import archivebox'
+          VIRTUAL_ENV="$GITHUB_WORKSPACE/.venv" uv run --no-cache --active --no-project archivebox version
+
      - name: Upload every distribution
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
@@ -68,6 +89,7 @@ jobs:

  install-smoke:
    name: install ${{ matrix.artifact }} / Python ${{ matrix.python }} / ${{ matrix.os }}
+    if: inputs.full_tests
    needs: build
    runs-on: ${{ matrix.os }}
    strategy:
@@ -0,0 +1,51 @@
+name: Release candidate
+
+on:
+  push:
+    branches: [dev]
+
+permissions:
+  actions: read
+  contents: write
+  packages: write
+
+concurrency:
+  group: release-candidate-dev
+  cancel-in-progress: true
+
+jobs:
+  prepare:
+    runs-on: ubuntu-24.04
+    outputs:
+      action: ${{ steps.version.outputs.action }}
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          fetch-depth: 0
+          submodules: true
+          token: ${{ secrets.RELEASE_GH_TOKEN }}
+
+      - id: version
+        uses: ArchiveBox/monorepo/.github/actions/prepare-release-version@main
+        with:
+          package: archivebox
+          branch: dev
+          tag-prefix: v
+          version-scheme: rc
+
+  python-artifacts:
+    needs: prepare
+    if: needs.prepare.outputs.action == 'ready' || needs.prepare.outputs.action == 'reserved'
+    uses: ./.github/workflows/pip.yml
+    with:
+      full_tests: false
+    secrets: inherit
+
+  docker-digests:
+    needs: prepare
+    if: needs.prepare.outputs.action == 'ready' || needs.prepare.outputs.action == 'reserved'
+    uses: ./.github/workflows/docker.yml
+    with:
+      push_digests: true
+      full_tests: false
+    secrets: inherit
@@ -2,7 +2,7 @@ name: Release

 on:
  workflow_run:
-    workflows: [CI]
+    workflows: [CI, Release candidate]
    types: [completed]
    branches: [dev, main]

@@ -20,9 +20,35 @@ env:
  CI_RUN_ID: ${{ github.event.workflow_run.id }}

 jobs:
+  candidate:
+    if: >-
+      github.event.workflow_run.conclusion == 'success' &&
+      github.event.workflow_run.event == 'push' &&
+      ((github.event.workflow_run.head_branch == 'dev' && github.event.workflow_run.name == 'Release candidate') ||
+       (github.event.workflow_run.head_branch == 'main' && github.event.workflow_run.name == 'CI'))
+    runs-on: ubuntu-24.04
+    outputs:
+      ready: ${{ steps.owner.outputs.ready }}
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          fetch-depth: 0
+          ref: ${{ env.RELEASE_SHA }}
+      - id: owner
+        shell: bash
+        run: |
+          if [[ "$RELEASE_BRANCH" == main ]]; then
+            echo 'ready=true' >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          VERSION="$(sed -nE 's/^version = "([^"]+)".*/\1/p' pyproject.toml | head -n 1)"
+          TARGET="$(git ls-remote origin "refs/tags/release-candidate/$VERSION" | awk '{print $1}')"
+          [[ "$TARGET" == "$RELEASE_SHA" ]] && echo 'ready=true' >> "$GITHUB_OUTPUT" || echo 'ready=false' >> "$GITHUB_OUTPUT"
+
  python-release:
    name: Publish tested Python artifacts
-    if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'push'
+    needs: candidate
+    if: needs.candidate.outputs.ready == 'true'
    runs-on: ubuntu-24.04
    environment: pypi
    permissions:
@@ -57,9 +83,9 @@ jobs:
        shell: bash
        run: |
          set -Eeuo pipefail
-          ABXPKG_VERSION="$(uv run --no-cache --no-project python -c 'import tomllib; print(next(package["version"] for package in tomllib.load(open("uv.lock", "rb"))["package"] if package["name"] == "abxpkg"))')"
+          ABXPKG_SPEC="$(uv run --no-cache --no-project python -c 'import tomllib; package = next(item for item in tomllib.load(open("uv.lock", "rb"))["package"] if item["name"] == "abxpkg"); wheel = package["wheels"][0]; print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))')"
          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
-          uv run --no-cache --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env --install \
+          uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env --install \
            --lib="$ABXPKG_LIB_DIR" \
            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" >/dev/null
          for name in uv gh git jq curl; do
@@ -133,34 +159,11 @@ jobs:
            echo "ready=false" >> "$GITHUB_OUTPUT"
          fi

-      - name: Verify published PyPI package installs and runs
-        if: steps.release-state.outputs.ready == 'true'
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          VERSION='${{ steps.version.outputs.version }}'
-          TOOL_ROOT="$(mktemp -d)"
-          export UV_TOOL_DIR="$TOOL_ROOT/tools"
-          export UV_TOOL_BIN_DIR="$TOOL_ROOT/bin"
-          for attempt in {1..40}; do
-            if "$UV_BINARY" tool install --no-cache --force "archivebox==$VERSION"; then
-              break
-            fi
-            if [[ "$attempt" == 40 ]]; then
-              echo "archivebox==$VERSION did not become installable from PyPI within 10 minutes" >&2
-              exit 1
-            fi
-            sleep 15
-          done
-          INSTALLED_VERSION="$("$UV_TOOL_BIN_DIR/archivebox" --version)"
-          test "$INSTALLED_VERSION" = "$VERSION"
-
  docker-release:
    name: Publish tested multiarch image
    if: >-
      github.event.workflow_run.conclusion == 'success' &&
-      github.event.workflow_run.event == 'push' &&
-      needs.python-release.outputs.release_ready == 'true'
+      github.event.workflow_run.event == 'push'
    needs: python-release
    runs-on: ubuntu-24.04
    permissions:
@@ -189,9 +192,9 @@ jobs:
        shell: bash
        run: |
          set -Eeuo pipefail
-          ABXPKG_VERSION="$(uv run --no-cache --no-project python -c 'import tomllib; print(next(package["version"] for package in tomllib.load(open("uv.lock", "rb"))["package"] if package["name"] == "abxpkg"))')"
+          ABXPKG_SPEC="$(uv run --no-cache --no-project python -c 'import tomllib; package = next(item for item in tomllib.load(open("uv.lock", "rb"))["package"] if item["name"] == "abxpkg"); wheel = package["wheels"][0]; print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))')"
          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
-          uv run --no-cache --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env --install \
+          uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env --install \
            --lib="$ABXPKG_LIB_DIR" \
            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" \
            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries" >/dev/null
@@ -296,7 +299,8 @@ jobs:
        run: |
          set -Eeuo pipefail
          VERSION='${{ needs.python-release.outputs.version }}'
-          for image in "${DOCKERHUB_IMAGE}:${VERSION}" "${GHCR_IMAGE}:${VERSION}"; do
+          SHORT_SHA="${RELEASE_SHA::12}"
+          for image in "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}" "${GHCR_IMAGE}:sha-${SHORT_SHA}"; do
            "$DOCKER_BINARY" buildx imagetools inspect "$image" >/dev/null
            INSTALLED_VERSION="$("$DOCKER_BINARY" run --rm "$image" --version)"
            test "$INSTALLED_VERSION" = "$VERSION"
@@ -311,14 +315,17 @@ jobs:

  cascade:
    name: Cascade verified release
+    if: needs.python-release.outputs.release_ready == 'true'
    needs:
      - python-release
      - docker-release
    permissions:
+      actions: read
      contents: read
    uses: ArchiveBox/monorepo/.github/workflows/cascade-release.yml@main
    with:
      upstream_package: archivebox
      upstream_version: ${{ needs.python-release.outputs.version }}
+      upstream_artifact_name: python-distributions
    secrets:
      release_token: ${{ secrets.RELEASE_GH_TOKEN }}
@@ -0,0 +1,70 @@
+name: Capture publicsite screenshots
+
+on:
+  push:
+    branches:
+      - dev
+    paths:
+      - publicsite/**
+      - archivebox/**
+      - bin/collect_ui_screenshots.*
+      - bin/generate_ui_screenshot_gallery.py
+      - bin/take_screenshot.js
+      - docs/Screenshots.md
+      - pyproject.toml
+      - uv.lock
+      - .github/workflows/screenshots.yml
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.event_name == 'workflow_dispatch' && format('publicsite-captures-manual-{0}', github.run_id) || 'publicsite-captures' }}
+  cancel-in-progress: false
+
+jobs:
+  deploy:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          fetch-depth: 1
+
+      - name: Setup Python and uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: false
+
+      - name: Install ArchiveBox and screenshot dependencies
+        run: uv sync --no-cache --frozen
+
+      - name: Build UI screenshot gallery
+        run: bin/collect_ui_screenshots.sh
+        env:
+          BASE_URL: http://archivebox.localhost:8000
+
+      - name: Verify gallery was generated from this exact revision
+        run: |
+          uv run --no-cache --no-sync python - <<'PY'
+          import json
+          import os
+          import tomllib
+          from pathlib import Path
+
+          provenance = json.loads(Path("publicsite/screenshots/build.json").read_text())
+          version = tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"]
+          assert provenance["revision"] == os.environ["GITHUB_SHA"], provenance
+          assert provenance["version"] == version, provenance
+          assert provenance["capture_count"] >= 3 * 35, provenance
+          assert len(provenance["files"]) == provenance["capture_count"], provenance
+          PY
+
+      - name: Save validated screenshots
+        uses: actions/upload-artifact@v4
+        with:
+          name: site-screenshots
+          path: publicsite/screenshots
+          if-no-files-found: error
+          retention-days: 90
@@ -38,12 +38,12 @@ jobs:
          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
        run: |
          set -Eeuo pipefail
-          ABXPKG_VERSION="$(uv run --no-cache --no-project python -c 'import tomllib; print(next(package["version"] for package in tomllib.load(open("uv.lock", "rb"))["package"] if package["name"] == "abxpkg"))')"
-          test -n "$ABXPKG_VERSION"
+          ABXPKG_SPEC="$(uv run --no-cache --no-project python -c 'import tomllib; package = next(item for item in tomllib.load(open("uv.lock", "rb"))["package"] if item["name"] == "abxpkg"); wheel = package["wheels"][0]; print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))')"
+          test -n "$ABXPKG_SPEC"
          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
          export PYTHON_BINARY="$pythonLocation/bin/python"
          test -x "$PYTHON_BINARY"
-          uv run --no-cache --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
+          uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env \
            --install \
            --lib="$ABXPKG_LIB_DIR" \
            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries"
@@ -106,8 +106,8 @@ jobs:
          TEST_PATHS: ${{ matrix.test.paths_arg }}
        run: |
          set -Eeuo pipefail
-          ABXPKG_VERSION="$(uv run --no-cache --no-project python -c 'import tomllib; print(next(package["version"] for package in tomllib.load(open("uv.lock", "rb"))["package"] if package["name"] == "abxpkg"))')"
-          test -n "$ABXPKG_VERSION"
+          ABXPKG_SPEC="$(uv run --no-cache --no-project python -c 'import tomllib; package = next(item for item in tomllib.load(open("uv.lock", "rb"))["package"] if item["name"] == "abxpkg"); wheel = package["wheels"][0]; print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))')"
+          test -n "$ABXPKG_SPEC"
          dependency_sources=()
          if [[ -n "$TEST_EXTRA" ]]; then
            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:required_binaries")
@@ -115,7 +115,7 @@ jobs:
          if [[ "$TEST_PATHS" == *test_postgres_backend* ]]; then
            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:postgres_binaries")
          fi
-          uv run --no-cache --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
+          uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env \
            --install \
            --no-cache \
            --json \
@@ -15,6 +15,13 @@ ArchiveBox is the full self-hosted web archiving app. Keep this repo on the `dev
 - Trace root causes from observed behavior. Do not paper over failures with retries, wider timeouts, broad fallbacks, or looser assertions.
 - Read `README.md` for the full setup, CLI, Docker, API, and release surface.

+## Concurrency Contract
+
+- A collection has one orchestrator at a time. Local PID/process checks may warn about obvious same-machine duplicates, but must not claim to enforce ownership across machines or shared filesystems.
+- SQLite remains supported with concurrent short writes from CLI and server processes. Keep the SQLite database on a local filesystem, never NFS/SMB, and use short autocommit/CAS updates instead of long transactions or database locks.
+- Network calls, hook execution, filesystem migrations, and other long work belong in the orchestrator. Never hold a database transaction or lock across that work.
+- PostgreSQL and shared data directories are the path to future multi-machine scheduling. Coordinate that work through database-backed per-crawl/per-snapshot claims; do not introduce file leases or timer-based orchestrator election.
+
 ## Development Setup

 ```bash
@@ -10,10 +10,10 @@
 #       --build-context abx-plugins=../abx-plugins \
 #       -t archivebox/abx-dl:dev
 #   docker buildx build . -f Dockerfile \
-#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:1.12.158 \
+#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:1.12.300 \
 #       -t archivebox:multistage

-ARG ABX_DL_IMAGE=archivebox/abx-dl:1.12.158
+ARG ABX_DL_IMAGE=archivebox/abx-dl:1.12.300

 FROM archivebox/sonic:1.4.9 AS sonic
 FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base
@@ -96,15 +96,15 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
    --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
    <<'EOF'
 echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..."
-echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache
-echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends
-echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests
+export UV_NO_CACHE=true
 rm -f /etc/apt/apt.conf.d/docker-clean
 apt-get update -qq
-apt-get install -qq -y --no-install-recommends \
-    build-essential gcc libldap2-dev libsasl2-dev libssl-dev
-/usr/bin/uv venv --no-cache --clear /venv --python "${PYTHON_VERSION}"
-/usr/bin/uv pip install --no-cache setuptools pip wheel
+apt-get install -qq -y --no-install-recommends x11-utils
+# Extend abx-dl's existing venv instead of clearing it. Clearing and later
+# copying a complete replacement over the base duplicates every venv byte in
+# the final overlay history even when most packages are unchanged.
+# Do not rewrite inherited native libraries merely to strip them again.
+/usr/bin/find /venv/lib/python3.*/site-packages -type f -name '*.so' -print > /tmp/archivebox-inherited-libraries

 mkdir -p /tmp/archivebox-uv-project
 /venv/bin/python - <<'PY'
@@ -153,24 +153,37 @@ PY
    --inexact \
    --no-install-project \
    --no-install-workspace \
+    --no-install-package abxbus \
+    --no-install-package abxpkg \
+    --no-install-package abx-plugins \
+    --no-install-package abx-dl \
    --no-sources
-ABXPKG_NO_CACHE=True abxpkg env --install --binproviders=env,apt --lib="$ABXPKG_LIB_DIR" --overrides='{"apt":{"install_args":["binutils"]}}' strip >/dev/null
+builder_abxpkg_lib_dir=/tmp/archivebox-builder-abxpkg
+ABXPKG_NO_CACHE=True abxpkg env --install --binproviders=env,apt --lib="$builder_abxpkg_lib_dir" --overrides='{"apt":{"install_args":["binutils"]}}' strip >/dev/null
 /usr/bin/find /venv/lib/python3.*/site-packages -type f -name '*.so' -print0 > /tmp/archivebox-native-libraries
 while IFS= read -r -d '' native_library; do
+    if grep -Fxq "$native_library" /tmp/archivebox-inherited-libraries; then
+        continue
+    fi
    magic=''
    if IFS= read -r -N 4 magic < "$native_library" && [[ "$magic" == $'\x7fELF' ]]; then
-        "$ABXPKG_LIB_DIR/env/bin/strip" --strip-unneeded "$native_library" || exit $?
+        "$builder_abxpkg_lib_dir/env/bin/strip" --strip-unneeded "$native_library" || exit $?
    fi
 done < /tmp/archivebox-native-libraries
-rm -f /tmp/archivebox-native-libraries
+rm -f /tmp/archivebox-native-libraries /tmp/archivebox-inherited-libraries
 rm -f /venv/bin/uv /venv/bin/uvx
-abxpkg run --binproviders=env --lib="$ABXPKG_LIB_DIR" apt-get purge -y binutils build-essential gcc libldap2-dev libsasl2-dev libssl-dev
-abxpkg run --binproviders=env --lib="$ABXPKG_LIB_DIR" apt-get autoremove -y
-/usr/bin/find "$ABXPKG_LIB_DIR/env/bin" -maxdepth 1 -type l -name strip -delete
-rm -rf /var/lib/apt/lists/*
+abxpkg run --binproviders=env --lib="$builder_abxpkg_lib_dir" apt-get purge -y binutils
+abxpkg run --binproviders=env --lib="$builder_abxpkg_lib_dir" apt-get autoremove -y
+rm -rf "$builder_abxpkg_lib_dir"
+rm -rf /venv/lib/python3.*/site-packages/pip* \
+    /venv/lib/python3.*/site-packages/wheel* \
+    /venv/bin/pip /venv/bin/pip3 /venv/bin/pip3.* /venv/bin/wheel
+rm -rf /var/lib/apt/lists/* /tmp/archivebox-uv-project
 EOF

 COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
+# Compile only ArchiveBox's installed package. Touching all of /venv here would
+# copy every inherited abx-dl file into a new layer merely to change its mtime.
 RUN echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
    && COMMIT_HASH="$( \
        if [[ "$ARCHIVEBOX_COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then \
@@ -187,10 +200,17 @@ RUN echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
    && if [[ "$COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt; fi \
    && /usr/bin/uv pip install --no-cache --no-deps "$CODE_DIR" \
    && rm -f /venv/bin/uv /venv/bin/uvx \
-    && find /venv -exec touch -h -d "@$(date +%s)" {} + \
+    && cd / \
+    && ARCHIVEBOX_PY_DIR="$(/venv/bin/python -c 'import pathlib, archivebox; print(pathlib.Path(archivebox.__file__).parent)')" \
+    && /venv/bin/python -m compileall --invalidation-mode checked-hash -q "$ARCHIVEBOX_PY_DIR" \
+    && test -f "$(/venv/bin/python -c 'import archivebox; print(archivebox.__cached__)')" \
    && /usr/bin/uv pip show archivebox | tee -a /VERSION.txt

-FROM archivebox-runtime-base
+# The builder installs and purges stripping tools in one layer, so its final
+# filesystem is already runtime-clean. Preserve that ancestry: starting again
+# from archivebox-runtime-base and COPYing /venv would bake a second full venv
+# over the inherited one instead of recording only ArchiveBox's package delta.
+FROM archivebox-builder

 LABEL name="archivebox" \
    maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
@@ -206,11 +226,11 @@ LABEL name="archivebox" \
 COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
 COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg

-COPY --from=archivebox-builder /venv /venv
-COPY --from=archivebox-builder /app /app
-COPY --from=archivebox-builder /VERSION.txt /VERSION.txt
-
-RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_ARCHIVEBOX_UID}..." \
+# The builder invokes abxpkg as root for temporary ELF stripping, which can
+# rewrite derived state ownership. Restore UID 911 before resolving binaries
+# as the runtime user; the old copy-based final stage hid this dependency.
+RUN --mount=type=cache,target=/opt/archivebox/lib/cache,sharing=locked,mode=1777,id=archivebox-runtime-config-$TARGETARCH \
+    echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_ARCHIVEBOX_UID}..." \
    && printf 'export PATH="/venv/bin:/opt/node/bin:$PATH"\n' > /etc/profile.d/archivebox-path.sh \
    && ln -sf /venv/bin/archivebox /usr/local/bin/archivebox \
    && ln -sf /venv/bin/daphne /usr/local/bin/daphne \
@@ -221,11 +241,11 @@ RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_ARCHIVEBOX_UID}..."
    && usermod --append --groups audio,video "$ARCHIVEBOX_USER" \
    && [[ "$(id -u "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_UID" ]] || usermod -u "$DEFAULT_ARCHIVEBOX_UID" "$ARCHIVEBOX_USER" \
    && [[ "$(id -g "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_GID" ]] || groupmod -g "$DEFAULT_ARCHIVEBOX_GID" "$ARCHIVEBOX_USER" \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abxpkg load --binproviders=env sonic | tee -a /VERSION.txt \
    && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$XDG_CACHE_HOME" "$PLAYWRIGHT_BROWSERS_PATH" \
    && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "/home/$ARCHIVEBOX_USER" \
    && chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" \
-    && chown -R "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "$ABXPKG_LIB_DIR" \
+    && find "$ABXPKG_LIB_DIR" \( ! -user "$DEFAULT_ARCHIVEBOX_UID" -o ! -group "$DEFAULT_ARCHIVEBOX_GID" \) -exec chown -h "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" {} + \
+    && UV_NO_CACHE=true setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abxpkg load --binproviders=env sonic | tee -a /VERSION.txt \
    && openssl rand -hex 16 > /etc/machine-id \
    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER ARCHIVEBOX_UID=$(id -u "$ARCHIVEBOX_USER") ARCHIVEBOX_GID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
    && echo -e "TMP_DIR=$TMP_DIR\nABXPKG_LIB_DIR=$ABXPKG_LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
@@ -23,7 +23,7 @@ Without active preservation effort, everything on the internet eventually disapp
 *ArchiveBox is an open source tool that lets organizations & individuals archive both public & private web content while retaining control over their data. It can be used to save copies of bookmarks, preserve evidence for legal cases, backup photos from FB/Insta/Flickr or media from YT/Soundcloud/etc., save research papers, and more...*
 <br/>

-> ➡️ Get ArchiveBox with `uv tool install --python 3.13 --prerelease explicit --upgrade 'archivebox>=0.9.0rc0,<0.10'` on [Linux](#quickstart)/[macOS](#quickstart), or via **[Docker](#quickstart)** ⭐️ on Linux/macOS.
+> ➡️ Get ArchiveBox with `pip install archivebox>=0.9.0rc0'` on [Linux](#quickstart)/[macOS](#quickstart), or via **[Docker](#quickstart)** ⭐️ on Linux/macOS.

 *Once installed, you can interact with it through the: [Browser Extension](https://github.com/ArchiveBox/archivebox-browser-extension), [CLI](#usage), [self-hosted web interface](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive), [Python API](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage), or [filesystem](#static-archive-exporting).*

@@ -73,17 +73,15 @@ mkdir -p ~/archivebox/data && cd ~/archivebox
 curl -fsSL 'https://docker-compose.archivebox.io' > docker-compose.yml
 docker compose pull
 docker compose up -d --wait                                                # initializes new collections automatically
-docker compose exec archivebox archivebox manage createsuperuser          # create the first Web UI user
+# open http://admin.archivebox.localhost:8000 to finish setup
 # docker compose run --rm archivebox add 'https://example.com'
 # docker compose run --rm archivebox help
 <br/>
 <br/>
 # Option B: Or use it as a plain Docker container:
 mkdir -p ~/archivebox/data && cd ~/archivebox/data
-docker run --rm -it -v "$PWD:/data" archivebox/archivebox:dev init
-docker run --rm -it -v "$PWD:/data" archivebox/archivebox:dev install
-docker run --rm -it -v "$PWD:/data" archivebox/archivebox:dev manage createsuperuser
 docker run -d --name archivebox -v "$PWD:/data" -p 8000:8000 archivebox/archivebox:dev
+# open http://admin.archivebox.localhost:8000 to finish setup
 # docker run -it -v $PWD:/data archivebox/archivebox:dev add 'https://example.com'
 # docker run -it -v $PWD:/data archivebox/archivebox:dev help
 <br/>
@@ -98,7 +96,7 @@ archivebox install
 # archivebox server 0.0.0.0:8000
 <br/>
 <br/>
-# Option D: Or use the optional auto setup script to install it
+# Option D: Or use the uv install shortcut for Option C
 curl -fsSL 'https://get.archivebox.io' | bash
 </code></pre>
 <br/>
@@ -138,27 +136,6 @@ curl -fsSL 'https://get.archivebox.io' | bash

 <br/>

-## 🤝 Professional Integration
-
-ArchiveBox is free for everyone to self-host, but we also provide support, security review, and custom integrations to help NGOs, governments, and other organizations [run ArchiveBox professionally](https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102):
-
- **Journalists:**
-  `crawling during research`, `preserving cited pages`, `fact-checking & review`  
- **Lawyers:**
-  `collecting & preserving evidence`, `detecting changes`, `tagging & review`  
- **Researchers:**
-  `analyzing social media trends`, `getting LLM training data`, `crawling pipelines`
- **Individuals:**
-  `saving bookmarks`, `preserving portfolio content`, `legacy / memoirs archival`
- **Governments:**
-  `snapshotting public service sites`, `recordkeeping compliance`
-
-> ***[Contact us](https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102)** if your org wants help using ArchiveBox professionally.*  
-> We offer: setup & support, CAPTCHA/ratelimit unblocking, SSO, audit logging/chain-of-custody, and more  
-> *ArchiveBox is a 🏛️ 501(c)(3) [nonprofit FSP](https://hackclub.com/hcb/) and all our work supports open-source development.* 
-
-<br/>
-
 <div align="center" style="text-align: center">
 <br/>
 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/0db52ea7-4a2c-441d-b47f-5553a5d8fe96" width="49%" alt="grass"/><img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/0db52ea7-4a2c-441d-b47f-5553a5d8fe96" width="49%" alt="grass"/>
@@ -174,6 +151,17 @@ ArchiveBox is free for everyone to self-host, but we also provide support, secur

 #### ✳️&nbsp; Easy Setup

+<details>
+<summary><b><img src="https://archivebox.github.io/ios-archivebox/docs/icons/apple.svg" alt="Apple" height="28px" align="top"/> <code>ArchiveBox Server.app</code></b> (macOS)</summary>
+<br/>
+<ol>
+<li><a href="https://github.com/ArchiveBox/ios-archivebox/releases">Download <code>ArchiveBox Server.app</code></a>, move it to Applications, and open it.</li>
+<li>Set up an admin account and generate an API Key.</li>
+<li>Use the Web UI or <a href="https://archivebox.github.io/ios-archivebox/"><code>ArchiveBox.app</code></a>, and optionally connect REST API clients, the <a href="https://archivebox.github.io/ios-archivebox/">mobile app</a>, the <a href="https://archivebox.github.io/archivebox-browser-extension/">browser extension</a>, and other integrations to your server using your API Key.</li>
+</ol>
+<br/>
+</details>
+
 <details>
 <summary><b><img src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" height="28px" align="top"/> <code>docker-compose</code></b>  (macOS/Linux) &nbsp; <b>👈&nbsp; recommended</b> &nbsp; <i>(click to expand)</i></summary>
 <br/>
@@ -186,11 +174,10 @@ ArchiveBox is free for everyone to self-host, but we also provide support, secur
 curl -fsSL 'https://docker-compose.archivebox.io' > docker-compose.yml
 docker compose pull
 </code></pre></li>
-<li>Start the server, which initializes a new collection automatically, then create the first admin user.
+<li>Start the server, which initializes a new collection automatically.
 <pre lang="bash"><code style="white-space: pre-line">docker compose up -d --wait
-docker compose exec archivebox archivebox manage createsuperuser
 </code></pre></li>
-<li>Next steps: Log in to the Admin UI at <a href="http://admin.archivebox.localhost:8000">http://admin.archivebox.localhost:8000</a>.
+<li>Open <code>/admin/</code> on the hostname or IP used to reach ArchiveBox (local example: <a href="http://admin.archivebox.localhost:8000/admin/">http://admin.archivebox.localhost:8000/admin/</a>) to create the first admin. If <code>BASE_URL</code> is not configured yet, continue through the web setup wizard.
 <pre lang="bash"><code style="white-space: pre-line">
 # run CLI commands inside the server container started above
 docker compose exec archivebox archivebox add 'https://example.com'
@@ -209,17 +196,16 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <br/>
 <ol>
 <li>Install <a href="https://docs.docker.com/get-docker/">Docker</a> on your system (if not already installed).</li>
-<li>Create a new empty directory and initialize your collection (can be anywhere).
+<li>Create a new empty directory and start the server, which initializes the collection automatically (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
-docker run --rm -v $PWD:/data -it archivebox/archivebox:dev init
-docker run --rm -v $PWD:/data -it archivebox/archivebox:dev install
+docker run -d --name archivebox -v $PWD:/data -p 8000:8000 archivebox/archivebox:dev
 </code></pre>
 </li>
-<li>Optional: Start the server then log in to the Admin UI at <a href="http://admin.archivebox.localhost:8000">http://admin.archivebox.localhost:8000</a>.
-<pre lang="bash"><code style="white-space: pre-line">docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox:dev
+<li>Open <code>/admin/</code> on the hostname or IP used to reach ArchiveBox (local example: <a href="http://admin.archivebox.localhost:8000/admin/">http://admin.archivebox.localhost:8000/admin/</a>) to create the first admin. If <code>BASE_URL</code> is not configured yet, continue through the web setup wizard.
+<pre lang="bash"><code style="white-space: pre-line">
 # completely optional, CLI can always be used without running a server
-# docker run -v $PWD:/data -it archivebox/archivebox:dev [subcommand] [--help]
-docker run -v $PWD:/data -it archivebox/archivebox:dev help
+# docker exec archivebox archivebox [subcommand] [--help]
+docker exec archivebox archivebox help
 </code></pre>
 <i>For more info, see <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-a-docker--docker-compose-setup-%EF%B8%8F">Install: Docker Compose</a> in the Wiki. ➡️</i>
 </li>
@@ -230,18 +216,17 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 </details>

 <details>
-<summary><b><img src="https://user-images.githubusercontent.com/511499/117456282-08665e80-af16-11eb-91a1-8102eff54091.png" alt="curl sh automatic setup script" height="28px" align="top"/> <code>bash</code> auto-setup script</b>  (macOS/Linux)</summary>
+<summary><b><img src="https://user-images.githubusercontent.com/511499/117456282-08665e80-af16-11eb-91a1-8102eff54091.png" alt="curl sh uv install shortcut" height="28px" align="top"/> <code>bash</code> uv install shortcut</b>  (macOS/Linux/BSD)</summary>
 <br/>
 <ol>
-<li>Install <a href="https://docs.docker.com/get-docker/">Docker</a> on your system (optional, highly recommended but not required).</li>
-<li>Run the automatic setup script.
+<li>Run the shortcut for the <code>uv</code> install method. It installs <code>uv</code> first when needed.
 <pre lang="bash"><code style="white-space: pre-line">curl -fsSL 'https://get.archivebox.io' | bash</code></pre>
 <i>For more info, see <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-b-automatic-setup-script">Install: Bare Metal</a> in the Wiki. ➡️</i>
 </li>
 </ol>

 See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.<br/>
-See <a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/bin/setup.sh"><code>setup.sh</code></a> for the source code of the auto-install script.<br/>
+See <a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/bin/setup.sh"><code>setup.sh</code></a> for the source code of the <code>uv</code> install shortcut.<br/>
 See <a href="https://docs.sweeting.me/s/against-curl-sh">"Against curl | sh as an install method"</a> blog post for my thoughts on the shortcomings of this install method.
 <br/><br/>
 </details>
@@ -271,9 +256,8 @@ archivebox init     # initialize a new collection
 archivebox install  # install all the runtime dependencies (e.g. chrome, single-file, yt-dlp, etc.)
 </code></pre>
 </li>
-<li>Create an admin account, then optionally start the server and log in to the Admin UI at <a href="http://admin.archivebox.localhost:8000">http://admin.archivebox.localhost:8000</a>.
-<pre lang="bash"><code style="white-space: pre-line">archivebox manage createsuperuser
-archivebox server 0.0.0.0:8000
+<li>Optionally start the server, then open <code>/admin/</code> on the hostname or IP used to reach ArchiveBox (local example: <a href="http://admin.archivebox.localhost:8000/admin/">http://admin.archivebox.localhost:8000/admin/</a>) to create the first admin. If <code>BASE_URL</code> is not configured yet, continue through the same web setup wizard used by Docker installs.
+<pre lang="bash"><code style="white-space: pre-line">archivebox server 0.0.0.0:8000
 # completely optional, CLI can always be used without running a server
 # archivebox [subcommand] [--help]
 archivebox help
@@ -303,14 +287,13 @@ sudo apt install archivebox
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data
 cd ~/archivebox/data
 archivebox init
-archivebox install
+sudo archivebox install
 archivebox add 'https://example.com'
 </code></pre>
 <br/>
 </li>
-<li>Create an admin account, then optionally start the server and log in to the Admin UI at <a href="http://admin.archivebox.localhost:8000">http://admin.archivebox.localhost:8000</a>.
-<pre lang="bash"><code style="white-space: pre-line">archivebox manage createsuperuser
-archivebox server 0.0.0.0:8000
+<li>Optionally start the server, then open <code>/admin/</code> on the hostname or IP used to reach ArchiveBox (local example: <a href="http://admin.archivebox.localhost:8000/admin/">http://admin.archivebox.localhost:8000/admin/</a>) to create the first admin. If <code>BASE_URL</code> is not configured yet, continue through the same web setup wizard used by Docker installs.
+<pre lang="bash"><code style="white-space: pre-line">archivebox server 0.0.0.0:8000
 # completely optional, CLI can always be used without running a server
 # archivebox [subcommand] [--help]
 archivebox help
@@ -318,7 +301,7 @@ archivebox help
 </li>
 </ol>
 See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.<br/>
-<sub>The apt package is a thin dev-channel wrapper around the normal Python install flow. See the <a href="https://github.com/ArchiveBox/debian-archivebox"><code>debian-archivebox</code></a> repo for details, and run <code>sudo archivebox install</code> only if you want it to install missing system packages via apt.</sub>
+<sub>The apt package is a thin dev-channel wrapper around the normal Python install flow. <code>sudo archivebox install</code> uses apt for missing system dependencies while preserving ownership of the user-owned collection. See the <a href="https://github.com/ArchiveBox/debian-archivebox"><code>debian-archivebox</code></a> repo for details.</sub>
 <br/><br/>
 </details>

@@ -342,9 +325,8 @@ archivebox init
 archivebox install
 </code></pre>
 </li>
-<li>Create an admin account, then optionally start the server and log in to the Admin UI at <a href="http://admin.archivebox.localhost:8000">http://admin.archivebox.localhost:8000</a>.
-<pre lang="bash"><code style="white-space: pre-line">archivebox manage createsuperuser
-archivebox server 0.0.0.0:8000
+<li>Optionally start the server, then open <code>/admin/</code> on the hostname or IP used to reach ArchiveBox (local example: <a href="http://admin.archivebox.localhost:8000/admin/">http://admin.archivebox.localhost:8000/admin/</a>) to create the first admin. If <code>BASE_URL</code> is not configured yet, continue through the same web setup wizard used by Docker installs.
+<pre lang="bash"><code style="white-space: pre-line">archivebox server 0.0.0.0:8000
 # completely optional, CLI can always be used without running a server
 # archivebox [subcommand] [--help]
 archivebox help
@@ -365,7 +347,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C

 <ul>
 <li>Arch: <a href="https://aur.archlinux.org/packages/archivebox/"><code>yay -S archivebox</code></a> (contributed by <a href="https://github.com/imlonghao"><code>@imlonghao</code></a>, maintained by <a href="https://github.com/jasongodev"><code>@jasongodev</code></a>)</li>
-<li>FreeBSD: <a href="https://github.com/ArchiveBox/ArchiveBox#%EF%B8%8F-easy-setup"><code>curl -fsSL 'https://get.archivebox.io' | bash</code></a> (uses <code>pkg</code> + <code>uv</code> under-the-hood)</li>
+<li>FreeBSD: <a href="https://github.com/ArchiveBox/ArchiveBox#%EF%B8%8F-easy-setup"><code>curl -fsSL 'https://get.archivebox.io' | bash</code></a> (uses <code>uv</code>)</li>
 <li>Nix: <a href="https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/misc/archivebox/default.nix"><code>nix-env --install archivebox</code></a> (contributed by <a href="https://github.com/siraben"><code>@siraben</code></a>)</li>
 <li>Guix: <a href="https://packages.guix.gnu.org/packages/archivebox/"><code>guix install archivebox</code></a> (contributed by <a href="https://github.com/rakino"><code>@rakino</code></a>)</li>
 <li>More: <a href="https://github.com/ArchiveBox/ArchiveBox/issues/new"><i>contribute another distribution...!</i></a></li>
@@ -422,18 +404,9 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for usage examples using the CLI, W
 </details>

 <details>
-<summary><img src="https://user-images.githubusercontent.com/511499/117448723-1663b180-af0d-11eb-837f-d43959227810.png" alt="paid" height="27px" align="top"/> Paid hosting solutions (cloud VPS)</summary>
+<summary><img src="https://user-images.githubusercontent.com/511499/117448723-1663b180-af0d-11eb-837f-d43959227810.png" alt="paid" height="27px" align="top"/> Third-party paid hosting options (cloud VPS)</summary>
 <br/>
 <ul>
-<li><a href="https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102">
- <img src="https://img.shields.io/badge/Custom_Development-ArchiveBox.io-%231a1a1a.svg?style=flat" height="22px"/>
-</a> (<a href="https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102">get hosting, support, and feature customization directly from us</a>)</li>
-<li><a href="https://monadical.com">
- <img src="https://img.shields.io/badge/General_Dev_Consulting-Monadical.com-%231a1a1a.svg?style=flat" height="22px"/>
-</a> (<a href="https://monadical.com/contact-us.html">generalist consultancy that has ArchiveBox experience</a>)</li>
-<br/>
-Other providers of paid ArchiveBox hosting (not officially endorsed):<br/>
-<br/><br/>
 <li><a href="https://elest.io/open-source/archivebox"><img src="https://img.shields.io/badge/Managed_Hosting-Elest.io-%23193f7e.svg?style=flat" height="22px"/></a></li>
 <li><a href="https://www.stellarhosted.com/archivebox/"><img src="https://img.shields.io/badge/Semi_Managed_Hosting-StellarHosted.com-%23193f7e.svg?style=flat" height="22px"/></a> (USD $29-250/mo, <a href="https://www.stellarhosted.com/archivebox/#pricing">pricing</a>)</li>
 <li><a href="https://www.pikapods.com/pods?run=archivebox"><img src="https://img.shields.io/badge/Semi_Managed_Hosting-PikaPods.com-%2343a047.svg?style=flat" height="22px"/></a> (from USD $2.6/mo)</li>
@@ -455,7 +428,7 @@ Other providers of paid ArchiveBox hosting (not officially endorsed):<br/>
 <sub><i>Referral links marked 🎗 provide $5-10 of free credit for new users and help pay for our <a href="https://demo.archivebox.io">demo server</a> hosting costs.</i></sub>
 </ul>

-For more discussion on managed and paid hosting options see here: <a href="https://github.com/ArchiveBox/ArchiveBox/issues/531">Issue #531</a>.
+For more discussion on third-party hosting options see here: <a href="https://github.com/ArchiveBox/ArchiveBox/issues/531">Issue #531</a>.

 </details>

@@ -464,7 +437,7 @@ For more discussion on managed and paid hosting options see here: <a href="https
 #### ➡️&nbsp; Next Steps

 - Import URLs from some of the supported [Input Formats](#input-formats) or view the supported [Output Formats](#output-formats)...
- (Optional) Create a persona and import browser cookies to archive logged-in sites: `archivebox persona create --import=chrome personal`
+- (Optional) [Import browser cookies and settings into a persona](docs/Chromium-Install.md#import-an-existing-browser-profile).
 - Tweak your UI or archiving behavior [Configuration](#configuration), read about some of the [Caveats](#caveats), or [Troubleshoot](https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting)
 - Read about the [Dependencies](#dependencies) used for archiving, the [Upgrading Process](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives), or the [Archive Layout](#archive-layout) on disk...
 - Or check out our full [Documentation](#documentation) or [Community Wiki](#internet-archiving-ecosystem)...
@@ -492,11 +465,6 @@ docker compose run --rm archivebox help
 # equivalent: docker run -it -v $PWD:/data archivebox/archivebox:dev [subcommand] [--help]
 docker run -it -v $PWD:/data archivebox/archivebox:dev help

-# optional: import your browser cookies into a persona for logged-in archiving
-archivebox persona create --import=chrome personal
-# supported: chrome/chromium/brave/edge (Chromium-based only)
-# use --profile to target a specific profile (e.g. Default, Profile 1)
-# re-running import merges/dedupes cookies.txt (by domain/path/name) but replaces chrome_user_data
 ```

 #### ArchiveBox Subcommands
@@ -532,12 +500,10 @@ archivebox help              # get list of archivebox subcommands that can be ru
 <pre lang="bash"><code style="white-space: pre-line">
 # make sure you have `docker-compose.yml` from the Quickstart instructions first
 <br/>
-# docker compose run --rm archivebox [subcommand] [--help]
-docker compose run --rm archivebox init
-docker compose run --rm archivebox install
-docker compose run --rm archivebox version
-docker compose run --rm archivebox help
-docker compose run --rm archivebox add 'https://example.com'
+# docker compose exec archivebox archivebox [subcommand] [--help]
+docker compose exec archivebox archivebox version
+docker compose exec archivebox archivebox help
+docker compose exec archivebox archivebox add 'https://example.com'
 # to start webserver: docker compose up
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage">Usage: Docker Compose CLI</a> wiki. ➡️</i>
@@ -549,15 +515,12 @@ docker compose run --rm archivebox add 'https://example.com'
 <summary><img src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" height="22px" align="top"/> <b>CLI Usage Examples: Docker</b></summary>
 <br/>
 <pre lang="bash"><code style="white-space: pre-line">
-# make sure you create and cd into in a new empty directory first  
+# make sure the `archivebox` server container from the Quickstart is running first
 <br/>
-# docker run -it -v $PWD:/data archivebox/archivebox:dev [subcommand] [--help]
-docker run -v $PWD:/data -it archivebox/archivebox:dev init
-docker run -v $PWD:/data -it archivebox/archivebox:dev install
-docker run -v $PWD:/data -it archivebox/archivebox:dev version
-docker run -v $PWD:/data -it archivebox/archivebox:dev help
-docker run -v $PWD:/data -it archivebox/archivebox:dev add 'https://example.com'
-# to start webserver: docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox:dev
+# docker exec archivebox archivebox [subcommand] [--help]
+docker exec archivebox archivebox version
+docker exec archivebox archivebox help
+docker exec archivebox archivebox add 'https://example.com'
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage-1">Usage: Docker CLI</a> wiki. ➡️</i>
 </details>
@@ -581,19 +544,17 @@ find ./archive/users -path '*/snapshots/*/*/*/index.html'  # inspect snapshot da
 <summary><b>🖥&nbsp; Web UI & API Usage</b></summary>
 <pre lang="bash"><code style="white-space: pre-line">
 # Start the server on bare metal (uv/apt/brew):
-archivebox manage createsuperuser              # create a new admin user via CLI
 archivebox server 0.0.0.0:8000                 # start the server
 <br/>
 # Or with Docker Compose:
-docker compose run --rm archivebox manage createsuperuser
 docker compose up                              # start the server
 <br/>
 # Or with a Docker container:
-docker run -v $PWD:/data -it archivebox/archivebox:dev manage createsuperuser
 docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox:dev
 </code></pre>

-<sup>Open <a href="http://web.archivebox.localhost:8000"><code>http://web.archivebox.localhost:8000</code></a> for the public UI and <a href="http://admin.archivebox.localhost:8000"><code>http://admin.archivebox.localhost:8000</code></a> for the admin UI ➡️</sup><br/>
+<sup>Open <a href="http://admin.archivebox.localhost:8000/admin/"><code>http://admin.archivebox.localhost:8000/admin/</code></a> to create the first admin and finish web setup. Use <a href="http://web.archivebox.localhost:8000"><code>http://web.archivebox.localhost:8000</code></a> for the public UI. ➡️</sup><br/>
+<sup>Advanced: <code>archivebox manage createsuperuser</code> remains available for creating accounts from the CLI.</sup><br/>
 <sup>Set <code>BASE_URL</code> to change the public base domain. The default <code>auto</code> mode uses <code>web.</code> and <code>admin.</code> subdomains on <code>*.localhost</code>, but one host for ordinary DNS names. <code>BIND_ADDR</code> only controls the local listen address.</sup>
 <br/><br/>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#ui-usage">Usage: Web UI</a> wiki. ➡️</i>
@@ -611,6 +572,20 @@ docker compose run --rm archivebox config --set PERMISSIONS=public
 </code></pre>
 </details>

+<br/>
+
+<details>
+<summary><b><img src="https://archivebox.github.io/ios-archivebox/docs/icons/apple.svg" alt="Apple" height="22px" align="top"/> <img src="https://cdn.jsdelivr.net/gh/devicons/devicon@v2.17.0/icons/windows11/windows11-original.svg" alt="Windows" height="22px" align="top"/> 🐧&nbsp; Desktop / Mobile / Extension Usage</b></summary>
+<br/>
+
+- 📱 [<code>ArchiveBox</code> iOS Mobile App](https://archivebox.github.io/ios-archivebox/)
+- 🖥️ [<code>ArchiveBox.app</code> macOS Desktop App](https://archivebox.github.io/ios-archivebox/)
+- 🧩 [ArchiveBox Browser Extension](https://archivebox.github.io/archivebox-browser-extension/)
+- 💻 [Linux/Windows Desktop App (Requires Docker Desktop)](https://github.com/ArchiveBox/electron-archivebox)
+
+<br/>
+</details>
+
 <br/>
 <br/>

@@ -664,7 +639,10 @@ docker run -it -v $PWD:/data archivebox/archivebox:dev add --depth=1 'https://ex


 - <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/ff20d251-5347-4b85-ae9b-83037d0ac01e" height="28px"/> <b>From the official <a href="https://github.com/ArchiveBox/archivebox-extension">ArchiveBox Browser Extension</a></b>  
-  <i>Provides realtime archiving of browsing history or selected pages from Chrome/Chromium/Firefox browsers.</i>
+  <i>Provides realtime archiving of browsing history or selected pages from Chrome/Chromium/Firefox/Safari browsers.</i>
+
+- <img src="https://archivebox.github.io/ios-archivebox/App/AppIcon.icon/Assets/ArchiveBox.png" alt="ArchiveBox app icon" height="28px"/> <b>From the official <a href="https://archivebox.github.io/ios-archivebox/">ArchiveBox macOS Desktop App or iOS Mobile App</a></b><br/>
+  <i>Save directly from the Share menu or Siri/Shortcuts to your server.</i>

 - <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/64078483-21d7-4eb1-aa6e-9ad55afe45b8" height="22px"/> From manual imports of URLs from RSS, JSON, CSV, TXT, SQL, HTML, Markdown, etc. files  
  <i>ArchiveBox supports injesting URLs in [any text-based format](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Import-a-list-of-URLs-from-a-text-file).</i>
@@ -719,7 +697,7 @@ It uses all available methods out-of-the-box, but you can disable extractors and
 <summary><i>Expand to see the full list of ways it saves each page...</i></summary>


-<code>data/archive/{Snapshot.id}/</code><br/>
+<code>data/archive/users/{username}/snapshots/{YYYYMMDD}/{domain}/{Snapshot.id}/</code><br/>
 <ul>
 <li><strong>Index:</strong> <code>index.html</code> &amp; <code>index.json</code> HTML and JSON index files containing metadata and details</li>
 <li><strong>Title</strong>, <strong>Favicon</strong>, <strong>Headers</strong> Response headers, site favicon, and parsed site title</li>
@@ -886,7 +864,7 @@ The on-disk layout is optimized to be easy to browse by hand and durable long-te
                                ...
 </code></pre>

-Each snapshot subfolder includes static metadata and plain extractor output files. ArchiveBox also maintains a backwards-compatible <code>data/archive/TIMESTAMP</code> symlink for each snapshot.
+Each snapshot subfolder includes static metadata and plain extractor output files. Current releases do not create top-level <code>data/archive/TIMESTAMP</code> projections; legacy timestamp directories are migrated into the user-scoped tree by <code>archivebox update --migrate-only</code>.

 <h4>Learn More</h4>
 <ul>
@@ -977,7 +955,7 @@ archivebox manage createsuperuser
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive">Wiki: Publishing Your Archive</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview">Wiki: Security Overview</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile">Wiki: Chromium Install (Setting Up a User Profile)</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Personas">Wiki: Personas (browser profiles and cookies)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#import-an-existing-browser-profile">Wiki: Chrome profiles and cookies</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#default_persona">Wiki: Configuration (<code>DEFAULT_PERSONA</code>)</a></li>
 </ul>

@@ -1021,7 +999,7 @@ archivebox config --set SERVER_SECURITY_MODE=safe-onedomain-nojsreplay

 ### Working Around Sites that Block Archiving

-For various reasons, many large sites (Reddit, Twitter, Cloudflare, etc.) actively block archiving or bots in general. There are a number of approaches to work around this, and we also provide <a href="https://docs.monadical.com/s/archivebox-consulting-services">consulting services</a> to help here.
+For various reasons, many large sites (Reddit, Twitter, Cloudflare, etc.) actively block archiving or bots in general. There are a number of approaches to work around this.

 <br/>
 <details>
@@ -1031,7 +1009,7 @@ For various reasons, many large sites (Reddit, Twitter, Cloudflare, etc.) active

 <ul>
 <li>Set <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#curl_user_agent"><code>CHROME_USER_AGENT</code>, <code>WGET_USER_AGENT</code>, <code>CURL_USER_AGENT</code></a> to impersonate a real browser (by default, ArchiveBox reveals that it's a bot when using the default user agent settings)</li>
-<li>Set up a logged-in browser session for archiving by <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Personas">importing a dedicated browser profile into a persona</a></li>
+<li>Set up a logged-in browser session for archiving by <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#import-an-existing-browser-profile">importing a dedicated browser profile into a persona</a></li>
 <li>Rewrite your URLs before archiving to swap in alternative frontends that are more bot-friendly e.g.<br>
 <code>reddit.com/some/url</code> -&gt; <code>teddit.net/some/url</code>: <a href="https://github.com/mendel5/alternative-front-ends">https://github.com/mendel5/alternative-front-ends</a></li>
 </ul>
@@ -1086,8 +1064,8 @@ Because ArchiveBox is designed to ingest a large volume of URLs with multiple co
 </li>
 <li><strong>Don't store large collections on older filesystems like EXT3/FAT</strong> as they may not be able to handle more than 50k directory entries in the <code>data/archive/</code> folder.
 </li>
-<li><strong>Try to keep the <code>data/index.sqlite3</code> file on local drive (not a network mount)</strong> or SSD for maximum performance, however the <code>data/archive/</code> folder can be on a network mount or slower HDD.</li>
-<li>If using Docker or NFS/SMB/FUSE for the <code>data/archive/</code> folder, make sure the mounted data directory is writable by its intended owner and consider disabling <a href="https://github.com/ArchiveBox/ArchiveBox/issues/1304"><code>root_squash</code></a> on your fileshare server.
+<li><strong>Keep the <code>data/index.sqlite3</code> file on a reliable local drive (not a network mount)</strong> or SSD when using SQLite; the <code>data/archive/</code> folder can be on a network mount or slower HDD.</li>
+<li>If using Docker or NFS/SMB/FUSE for the <code>data/archive/</code> folder, configure the server-side UID/GID mapping or ACL so ArchiveBox's non-root user can create and remove files. <code>root_squash</code> is compatible when the selected non-root identity has write access; do not rely on container root to override server-side permissions.
 </li>
 </ul>

@@ -1261,13 +1239,6 @@ ArchiveBox is neither the highest fidelity nor the simplest tool available for s

 <br/>

-**Need help building a custom archiving solution?**
-
-> ✨ **[Hire the team that built Archivebox](https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102) to solve archiving for your org.** ([@ArchiveBoxApp](https://twitter.com/ArchiveBoxApp))
-
-<br/>
-
-
 <div align="center" style="text-align: center">
 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/897f7a88-1265-4aab-b80c-b1640afaad1f" width="100%" alt="documentation graphic">
 </div>
@@ -1389,14 +1360,11 @@ archivebox server 0.0.0.0:8000
 # inside the container will reload and pick up your changes
 ./bin/build_docker.sh dev

-docker run -it -v $PWD/data:/data archivebox/archivebox:dev init
-docker run -it -v $PWD/data:/data archivebox/archivebox:dev install
-
 # Run the development server w/ autoreloading (but no bg workers)
-docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server --debug --reload 0.0.0.0:8000
+docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server --init --debug --reload 0.0.0.0:8000

 # Run the production server (with bg workers but no autoreloading)
-docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server
+docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server --init

 # (remove the --reload flag and add the --nothreading flag when profiling with the django debug toolbar)
 # When using --reload, make sure any files you create can be read by the user in the Docker container, eg with 'chmod a+rX'.
@@ -1457,7 +1425,7 @@ services:

 # or with plain Docker:
 docker build -t archivebox:dev https://github.com/ArchiveBox/ArchiveBox.git#dev
-docker run -it -v $PWD:/data archivebox:dev init
+docker run -it -v $PWD:/data -p 8000:8000 archivebox:dev

 # or with uv:
 uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
@@ -1570,8 +1538,6 @@ Copy a similar plugin as a template to modify, then open a new PR to add it in t

 <br/>
 <div align="center" style="text-align: center">
-<b><a href="https://docs.sweeting.me/s/archivebox-consulting-services">🏛️ Contact us for professional support 💬</a></b><br/>
-<br/><br/>
 <a href="https://hcb.hackclub.com/donations/start/archivebox"><img src="https://img.shields.io/badge/Donate-Directly-%13DE5D26.svg"/></a> &nbsp;
 <a href="https://github.com/sponsors/pirate"><img src="https://img.shields.io/badge/Github_Sponsors-%23B7CDFE.svg"/></a> &nbsp;
 <a href="https://www.patreon.com/theSquashSH"><img src="https://img.shields.io/badge/Patreon-%23DD5D76.svg"/></a> &nbsp;
@@ -4,7 +4,7 @@ from django.http import HttpRequest

 from ninja import Router, Schema

-from archivebox.api.auth import auth_using_token, auth_using_password, get_or_create_api_token
+from archivebox.api.auth import auth_using_token, auth_using_password, get_or_create_api_token, HeaderTokenAuth, BearerTokenAuth


 router = Router(tags=["Authentication"], auth=None)
@@ -63,3 +63,44 @@ def check_api_token(request: HttpRequest, token_data: TokenAuthSchema):
        return {"success": True, "user_id": str(user.pk)}

    return {"success": False, "user_id": None}
+
+
+@router.post(
+    "/browser_session",
+    auth=[HeaderTokenAuth(), BearerTokenAuth()],
+    summary="Exchange an administrator API key for a normal browser session",
+)
+def browser_session(request: HttpRequest):
+    from django.conf import settings
+    from django.contrib.auth import login
+    from django.http import JsonResponse
+    from django.utils import timezone
+    from ninja.errors import HttpError
+    from archivebox.core.routes_util import get_admin_base_url
+
+    user = request.auth
+    if not user.is_active or not user.is_superuser:
+        raise HttpError(403, "An active administrator is required")
+    login(request, user, backend="django.contrib.auth.backends.ModelBackend")
+    # A browser session must not outlive the key used to establish it.
+    token = request._api_token
+    lifetime = settings.SESSION_COOKIE_AGE
+    if token.expires:
+        lifetime = min(lifetime, max(0, int((token.expires - timezone.now()).total_seconds())))
+    request.session.set_expiry(lifetime)
+    request.session.save()
+    response = JsonResponse(
+        {
+            "admin_url": get_admin_base_url(request=request).rstrip("/") + "/admin/",
+            "cookie": {
+                "name": settings.SESSION_COOKIE_NAME,
+                "value": request.session.session_key,
+                "expires": request.session.get_expiry_date().timestamp(),
+                "secure": settings.SESSION_COOKIE_SECURE,
+            },
+        },
+    )
+    # The native client installs a host-only HttpOnly cookie on the admin origin;
+    # API and admin hosts can differ, so a Set-Cookie on the API host is insufficient.
+    response["Cache-Control"] = "no-store"
+    return response
@@ -133,6 +133,15 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
            validate_url(args.urls[0])
        except ValueError:
            submitted_urls = args.urls[0]
+    else:
+        submitted_urls = []
+        for url in args.urls:
+            try:
+                submitted_urls.append(validate_url(url))
+            except ValueError:
+                continue
+        if not submitted_urls:
+            raise HttpError(400, "No valid URLs were submitted")
    crawl, snapshots = add(
        urls=submitted_urls,
        snapshot_ids=args.snapshot_ids,
@@ -156,7 +165,7 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
        "crawl_id": str(crawl.id),
        "num_snapshots": len(snapshot_ids),
        "snapshot_ids": snapshot_ids,
-        "queued_urls": args.urls,
+        "queued_urls": args.urls if isinstance(submitted_urls, str) else submitted_urls,
    }
    stdout = request.__dict__.get("stdout")
    stderr = request.__dict__.get("stderr")
@@ -11,7 +11,7 @@ from uuid import UUID
 from typing import Union, Any, Annotated
 from datetime import datetime, time

-from django.db import transaction
+from django.db import IntegrityError
 from django.db.models import Model, Q
 from django.http import HttpRequest, HttpResponse
 from django.http.multipartparser import MultiPartParser, MultiPartParserError
@@ -59,7 +59,7 @@ from archivebox.core.snapshot_status import filter_snapshots_by_status, normaliz

 router = Router(tags=["Core Models"])

-ARCHIVERESULT_UPLOAD_HOOK_NAME = "on_Snapshot__archivebox_browser_extension_upload"
+ARCHIVERESULT_UPLOAD_HOOK_NAME = Snapshot.BROWSER_EXTENSION_UPLOAD_HOOK_NAME
 ARCHIVERESULT_UPLOAD_PLUGIN_RE = re.compile(r"^[A-Za-z0-9_.-]{1,32}$")


@@ -342,21 +342,10 @@ def _parse_archiveresult_upload_int(value: str, field_name: str, *, default: int


 def _summarize_archiveresult_output_files(output_files: dict[str, dict[str, Any]]) -> tuple[int, str]:
-    mime_sizes: dict[str, int] = defaultdict(int)
-    total_size = 0
-    for metadata in output_files.values():
-        if not isinstance(metadata, dict):
-            continue
-        try:
-            size = max(int(metadata.get("size") or 0), 0)
-        except (TypeError, ValueError):
-            size = 0
-        mime_type = str(metadata.get("mimetype") or "").strip()
-        total_size += size
-        if mime_type and size:
-            mime_sizes[mime_type] += size
-    output_mimetypes = ",".join(mime for mime, _size in sorted(mime_sizes.items(), key=lambda item: item[1], reverse=True))
-    return total_size, output_mimetypes
+    from abx_dl.output_files import OutputManifest
+
+    manifest = OutputManifest.from_value(output_files)
+    return manifest.total_size, ",".join(manifest.mimetypes)


 def _get_snapshot_by_ref(snapshot_id: str):
@@ -389,13 +378,6 @@ def _queue_archiveresult_snapshot_maintenance(snapshot: Snapshot) -> None:
    snapshot.safe_update(updates, refresh=False)


-def _merge_archiveresult_output_file_maps(results: list[ArchiveResult]) -> dict[str, dict[str, Any]]:
-    output_files: dict[str, dict[str, Any]] = {}
-    for result in results:
-        output_files.update(result.output_file_map())
-    return output_files
-
-
 def _write_archiveresult_files(
    request: HttpRequest,
    snapshot: Snapshot,
@@ -535,67 +517,53 @@ def create_archiveresult(
    normalized_status = ArchiveResult.normalize_status(status)
    parsed_output_json = _parse_archiveresult_output_json(output_json)
    hook = hook_name or ARCHIVERESULT_UPLOAD_HOOK_NAME
-    matching_results = list(
-        ArchiveResult.objects.filter(
-            snapshot=snapshot,
-            plugin=plugin_name,
-            hook_name=hook,
-        ).order_by("created_at", "id"),
-    )
-    existing_result = matching_results[0] if matching_results else None
-    existing_output_files = _merge_archiveresult_output_file_maps(matching_results)
-    output_files = _write_archiveresult_files(
+    result_lookup = {
+        "snapshot": snapshot,
+        "plugin": plugin_name,
+        "hook_name": hook,
+    }
+    uploaded_output_files = _write_archiveresult_files(
        request,
        snapshot,
        plugin_name,
-        existing_output_files=existing_output_files,
        allow_empty=True,
    )
-    now = timezone.now()
-
-    with transaction.atomic():
-        matching_results = list(
-            ArchiveResult.objects.filter(
-                snapshot=snapshot,
-                plugin=plugin_name,
-                hook_name=hook,
-            ).order_by("created_at", "id"),
-        )
-        if matching_results:
-            existing_result = matching_results[0]
-            output_files = {
-                **_merge_archiveresult_output_file_maps(matching_results),
-                **output_files,
-            }
-            duplicate_ids = [result.id for result in matching_results[1:]]
-            if duplicate_ids:
-                ArchiveResult.objects.filter(id__in=duplicate_ids).delete()
-            result = existing_result
-        else:
-            existing_result = None
-            result = ArchiveResult(
-                snapshot=snapshot,
-                plugin=plugin_name,
-                hook_name=hook,
-            )
-
-        if (
-            existing_result
-            and normalized_status == ArchiveResult.StatusChoices.STARTED
-            and existing_result.status != ArchiveResult.StatusChoices.STARTED
-        ):
-            normalized_status = existing_result.status
+    result = ArchiveResult.objects.filter(**result_lookup).first()
+    for _attempt in range(3):
+        output_files = {
+            **(result.output_file_map() if result else {}),
+            **uploaded_output_files,
+        }
        output_size, output_mimetypes = _summarize_archiveresult_output_files(output_files)
        output_file_paths = list(output_files.keys())
-        result.status = normalized_status
-        result.output_str = output_str or (output_file_paths[0] if output_file_paths else "")
-        result.output_json = parsed_output_json
-        result.output_files = output_files
-        result.output_size = output_size
-        result.output_mimetypes = output_mimetypes
-        result.start_ts = result.start_ts or now
-        result.end_ts = now
-        result.save()
+        result_status = normalized_status
+        if result and result_status == ArchiveResult.StatusChoices.STARTED and result.status != ArchiveResult.StatusChoices.STARTED:
+            result_status = result.status
+        now = timezone.now()
+        values = {
+            "status": result_status,
+            "output_str": output_str or (output_file_paths[0] if output_file_paths else ""),
+            "output_json": parsed_output_json,
+            "output_files": output_files,
+            "output_size": output_size,
+            "output_mimetypes": output_mimetypes,
+            "start_ts": result.start_ts or now if result else now,
+            "end_ts": now,
+        }
+        if result:
+            if result.safe_update(values):
+                break
+            continue
+        result, created = ArchiveResult.get_or_create_by_hook(
+            snapshot,
+            plugin_name,
+            hook,
+            defaults=values,
+        )
+        if created:
+            break
+    else:
+        raise HttpError(409, "ArchiveResult changed while upload metadata was being updated")

    if result.status != ArchiveResult.StatusChoices.STARTED:
        _queue_archiveresult_snapshot_maintenance(snapshot)
@@ -609,44 +577,46 @@ def patch_archiveresult(
 ):
    """Append or replace files on an existing ArchiveResult."""
    result = ArchiveResult.objects.select_related("snapshot__crawl__created_by").get(_uuid_ref_query("id", archiveresult_id))
-    output_files = _write_archiveresult_files(
+    uploaded_output_files = _write_archiveresult_files(
        request,
        result.snapshot,
        result.plugin,
-        existing_output_files=result.output_file_map(),
    )
-    latest_result = ArchiveResult.objects.only("output_files", "status").get(pk=result.pk)
-    output_files = {
-        **latest_result.output_file_map(),
-        **output_files,
-    }
-    output_size, output_mimetypes = _summarize_archiveresult_output_files(output_files)
-
-    update_fields = ["output_files", "output_size", "output_mimetypes", "end_ts", "modified_at"]
-    result.output_files = output_files
-    result.output_size = output_size
-    result.output_mimetypes = output_mimetypes
-    result.end_ts = timezone.now()
    output_str = _get_archiveresult_upload_form_value(request, "output_str")
    status = _get_archiveresult_upload_form_value(request, "status")
    output_json = _get_archiveresult_upload_form_value(request, "output_json")
-    if output_str:
-        result.output_str = output_str
-        update_fields.append("output_str")
-    if status:
-        normalized_status = ArchiveResult.normalize_status(status)
-        if normalized_status == ArchiveResult.StatusChoices.STARTED and latest_result.status != ArchiveResult.StatusChoices.STARTED:
-            normalized_status = latest_result.status
-        result.status = normalized_status
-        update_fields.append("status")
-    elif latest_result.status == ArchiveResult.StatusChoices.QUEUED and ArchiveResult.output_files_upload_complete(output_files):
-        result.status = ArchiveResult.StatusChoices.SUCCEEDED
-        update_fields.append("status")
-    if output_json:
-        result.output_json = _parse_archiveresult_output_json(output_json)
-        update_fields.append("output_json")
+    parsed_output_json = _parse_archiveresult_output_json(output_json) if output_json else None
+
+    if not ArchiveResult.output_files_upload_complete(uploaded_output_files):
+        result.output_files = {**result.output_file_map(), **uploaded_output_files}
+        result.output_size, result.output_mimetypes = _summarize_archiveresult_output_files(result.output_files)
+        return result
+
+    for _attempt in range(3):
+        output_files = {**result.output_file_map(), **uploaded_output_files}
+        output_size, output_mimetypes = _summarize_archiveresult_output_files(output_files)
+        values: dict[str, Any] = {
+            "output_files": output_files,
+            "output_size": output_size,
+            "output_mimetypes": output_mimetypes,
+            "end_ts": timezone.now(),
+        }
+        if output_str:
+            values["output_str"] = output_str
+        if status:
+            normalized_status = ArchiveResult.normalize_status(status)
+            if normalized_status == ArchiveResult.StatusChoices.STARTED and result.status != ArchiveResult.StatusChoices.STARTED:
+                normalized_status = result.status
+            values["status"] = normalized_status
+        elif result.status == ArchiveResult.StatusChoices.QUEUED:
+            values["status"] = ArchiveResult.StatusChoices.SUCCEEDED
+        if output_json:
+            values["output_json"] = parsed_output_json
+        if result.safe_update(values):
+            break
+    else:
+        raise HttpError(409, "ArchiveResult changed while upload metadata was being updated")

-    result.save(update_fields=update_fields)
    if result.status != ArchiveResult.StatusChoices.STARTED:
        _queue_archiveresult_snapshot_maintenance(result.snapshot)

@@ -963,38 +933,47 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
            created_by=request.user if isinstance(request.user, User) else None,
        )

-    with crawl_lifecycle_lock(str(crawl.id)):
-        snapshot_defaults = {
-            "depth": data.depth,
-            "title": data.title,
-            "timestamp": str(timezone.now().timestamp()),
-            "status": status or Snapshot.StatusChoices.QUEUED,
-            "retry_at": timezone.now(),
-        }
-        snapshot, _ = Snapshot.objects.get_or_create(
-            url=data.url,
-            crawl=crawl,
-            defaults=snapshot_defaults,
+    # Browser uploads must not wait behind the runner's crawl-wide lifecycle
+    # lock. The unique insert recovery and CAS update below are the request-side
+    # coordination boundary for this idempotent metadata sync.
+    snapshot = Snapshot.objects.filter(url=data.url, crawl=crawl).first()
+    if snapshot is None:
+        try:
+            snapshot = Snapshot.objects.create(
+                url=data.url,
+                crawl=crawl,
+                depth=data.depth,
+                title=data.title,
+                timestamp=str(timezone.now().timestamp()),
+                status=status or Snapshot.StatusChoices.QUEUED,
+                retry_at=timezone.now(),
+            )
+        except IntegrityError:
+            snapshot = Snapshot.objects.filter(url=data.url, crawl=crawl).first()
+            if snapshot is None:
+                raise
+
+    for _attempt in range(3):
+        updates: dict[str, Any] = {}
+        if data.title is not None and snapshot.title != data.title:
+            updates["title"] = data.title
+        if status is not None and snapshot.status != status:
+            updates["status"] = status
+        if not updates or snapshot.safe_update(updates, extra_filter={"modified_at": snapshot.modified_at}):
+            break
+    else:
+        raise HttpError(409, "Snapshot changed while metadata was being updated")
+
+    if tags:
+        snapshot.save_tags(
+            tags,
+            created_by=request.user if isinstance(request.user, User) else None,
        )

-        update_fields: list[str] = []
-        if data.title is not None and snapshot.title != data.title:
-            snapshot.title = data.title
-            update_fields.append("title")
-        if status is not None and snapshot.status != status:
-            snapshot.status = status
-            update_fields.append("status")
-        if update_fields:
-            update_fields.append("modified_at")
-            snapshot.save(update_fields=update_fields)
-
-        if tags:
-            snapshot.save_tags(tags)
-
-        try:
-            snapshot.ensure_crawl_symlink()
-        except Exception:
-            pass
+    try:
+        snapshot.ensure_crawl_symlink()
+    except Exception:
+        pass

    setattr(request, "with_archiveresults", False)
    return snapshot
@@ -1042,7 +1021,10 @@ def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateS
            update_fields.append("retry_at")

        if tags is not None:
-            snapshot.save_tags(normalize_tag_list(tags))
+            snapshot.save_tags(
+                normalize_tag_list(tags),
+                created_by=request.user if isinstance(request.user, User) else None,
+            )

        if payload.get("status") == Snapshot.StatusChoices.SEALED:
            snapshot.cancel()
@@ -1429,8 +1411,7 @@ def tags_add_to_snapshot(request: HttpRequest, data: TagSnapshotRequestSchema):
    else:
        raise HttpError(400, "Either tag_name or tag_id is required")

-    # Add the tag to the snapshot
-    snapshot.tags.add(tag.pk)
+    snapshot.add_tag_ids([tag.pk])

    return {
        "success": True,
@@ -1458,8 +1439,7 @@ def tags_remove_from_snapshot(request: HttpRequest, data: TagSnapshotRequestSche
    else:
        raise HttpError(400, "Either tag_name or tag_id is required")

-    # Remove the tag from the snapshot
-    snapshot.tags.remove(tag.pk)
+    snapshot.remove_tag_ids([tag.pk])

    return {
        "success": True,
@@ -1,6 +1,5 @@
 __package__ = "archivebox.api"

-import json
 from pathlib import Path
 from uuid import UUID
 from datetime import datetime
@@ -23,7 +22,7 @@ from archivebox.core.permissions import (
 )
 from archivebox.config.common import get_config
 from archivebox.crawls.models import Crawl
-from archivebox.misc.util import filter_queryset_by_uuid_substring
+from archivebox.misc.util import filter_queryset_by_uuid_substring, validate_url

 from .auth import API_AUTH_METHODS, authenticated_user_from_request

@@ -113,7 +112,10 @@ def get_crawls(request: HttpRequest):

@router.post("/crawls", response=CrawlSchema, url_name="create_crawl")
 def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
-    urls = [url.strip() for url in data.urls if url and url.strip()]
+    try:
+        urls = [validate_url(url) for url in data.urls if url and url.strip()]
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
    if not urls:
        raise HttpError(400, "At least one URL is required")
    if data.max_depth not in (0, 1, 2, 3, 4):
@@ -121,9 +123,10 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema):

    tags = normalize_tag_list(data.tags, data.tags_str)
    config = dict(data.config or {})
+    config.setdefault("PARSER", "url_list")
    config.setdefault("PERMISSIONS", str(get_config().PERMISSIONS))
    crawl = Crawl.objects.create(
-        urls="\n".join(json.dumps({"type": "CrawlSeed", "url": url, "depth": 0}, separators=(",", ":")) for url in urls),
+        urls="\n".join(urls),
        max_depth=data.max_depth,
        tags_str=",".join(tags),
        label=data.label,
@@ -284,16 +284,20 @@ class ModelWithOutputDir(ModelWithUUID):
            elif path.is_dir():
                shutil.rmtree(path, ignore_errors=True)

+    def schedule_delete_cleanup(self, *, using: str | None = None) -> None:
+        """Capture output paths before DB deletion and remove them after commit."""
+        paths = self.validate_output_paths_for_delete(self.output_paths_for_delete())
+        transaction.on_commit(lambda: self.delete_output_paths(paths), using=using)
+
    @classmethod
    def register_delete_signal(cls) -> None:
        if cls._delete_signal_registered:
            return

-        def schedule_output_dir_cleanup(sender, instance, **kwargs):
+        def schedule_output_dir_cleanup(sender, instance, using, **kwargs):
            if not isinstance(instance, ModelWithOutputDir):
                return
-            paths = instance.validate_output_paths_for_delete(instance.output_paths_for_delete())
-            transaction.on_commit(lambda paths=paths: instance.delete_output_paths(paths))
+            instance.schedule_delete_cleanup(using=using)

        pre_delete.connect(
            schedule_output_dir_cleanup,
@@ -186,6 +186,10 @@ def cli(ctx, help=False):


 def main(args=None, prog_name=None):
+    from archivebox.config.constants import CONSTANTS
+
+    os.environ.setdefault("ABX_PLUGINS_DIR", str(CONSTANTS.USER_PLUGINS_DIR))
+
    # show `docker run archivebox xyz` in help messages if running in docker
    IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes")
    IS_TTY = sys.stdin.isatty()
@@ -6,7 +6,6 @@ __command__ = "archivebox add"

 import sys
 import os
-import json
 from pathlib import Path

 from typing import Any, TYPE_CHECKING
@@ -71,17 +70,7 @@ def add(
    created_by_id: int | None = None,
    config: dict[str, Any] | None = None,
 ) -> tuple[Crawl, QuerySet[Snapshot]]:
-    """Add a new URL or list of URLs to your archive.
-
-    The flow is:
-    1. Save URLs to sources file
-    2. Create Crawl with URLs and max_depth
-    3. Crawl runner creates Snapshots from Crawl URLs (depth=0)
-    4. Crawl runner runs parser extractors on root snapshots
-    5. Parser extractors output to urls.jsonl
-    6. URLs are added to Crawl.urls and child Snapshots are created
-    7. Repeat until max_depth is reached
-    """
+    """Add a URL list or imported URL document to a new Crawl."""

    from rich import print

@@ -125,12 +114,12 @@ def add(
    created_by_id = created_by_id or get_or_create_system_user_pk()
    started_at = timezone.now()

-    use_internal_input_root = isinstance(urls, str)
-    source_text = (
-        urls
-        if use_internal_input_root
-        else "\n".join(json.dumps({"type": "CrawlSeed", "url": str(url), "depth": 0}, separators=(",", ":")) for url in urls)
-    )
+    if isinstance(urls, str):
+        source_text = urls
+    else:
+        from archivebox.misc.util import validate_url
+
+        source_text = "\n".join(validate_url(str(url)) for url in urls)

    # 2. Create a new Crawl with inline URLs
    # Foreground add must claim runner ownership before publishing runnable
@@ -187,12 +176,7 @@ def add(
    try:
        crawl = Crawl.objects.create(
            urls=source_text,
-            # Stdin/import text gets an extra hop because the synthetic
-            # archivebox://internal root lives at depth 0 and parser-discovered
-            # URLs land at depth 1; direct URL args become the depth=0 input
-            # snapshots themselves so --depth=N matches the deepest hop the user
-            # asked for.
-            max_depth=depth + 1 if use_internal_input_root else depth,
+            max_depth=depth,
            tags_str=tag,
            persona_id=persona_obj.id,
            label=f"{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]",
@@ -213,9 +197,8 @@ def add(
    print(f"[green]\\[+] Created Crawl {crawl.id} with max_depth={depth}[/green]")
    print(f"    [dim]First URL: {first_url}[/dim]")

-    # 3. The CrawlMachine will create Snapshots from all URLs when started
-    #    Parser extractors run on snapshots and discover more URLs
-    #    Discovered URLs become child Snapshots (depth+1)
+    # The runner parses Crawl.urls after claiming the Crawl, then persists the
+    # resulting depth-0 Snapshot facts. Recursive discoveries start at depth 1.

    if index_only:
        print("[yellow]\\[*] Index-only mode - URLs queued, runner not started[/yellow]")
@@ -244,11 +227,22 @@ def add(

        assert command is not None
        exit_code = 0
+
+        def crawl_is_complete() -> bool:
+            crawl.refresh_from_db(fields=["status"])
+            return crawl.status not in crawl.RUNNABLE_STATES
+
        try:
            try:
                with foreground_shutdown_signals(first_signal_message=None), foreground_parent_watchdog():
                    while True:
-                        standby_until_foreground_runner_needed(command, data_dir=CONSTANTS.DATA_DIR)
+                        standby = standby_until_foreground_runner_needed(
+                            command,
+                            data_dir=CONSTANTS.DATA_DIR,
+                            work_is_complete=crawl_is_complete,
+                        )
+                        if standby["work_completed"]:
+                            break
                        exit_code = run_runner_worker(
                            ["--crawl-id", str(crawl.id)],
                            name=f"worker_runner_add_{os.getpid()}",
@@ -394,10 +388,6 @@ def main(**kwargs):
            raise click.BadParameter(str(err), param_hint="--snapshot-max-size") from err
        if kwargs.get("crawl_max_concurrent_snapshots") is not None and int(kwargs["crawl_max_concurrent_snapshots"]) < 1:
            raise click.BadParameter("crawl_max_concurrent_snapshots must be at least 1.", param_hint="--crawl-max-concurrent-snapshots")
-        if not kwargs.get("index_only"):
-            from archivebox.workers.supervisord_util import require_crawl_memory
-
-            require_crawl_memory()

        # Translate --only-new/--no-only-new into a crawl config override.
        # add() takes config overrides as a dict; no per-flag kwargs.
@@ -410,13 +400,13 @@ def main(**kwargs):
        if only_new is not None:
            kwargs["config"] = {"ONLY_NEW": bool(only_new)}
        if extract:
-            from abx_dl.models import discover_plugins, plugins_matching_output
+            from archivebox.plugins.discovery import get_plugin_catalog

-            all_plugins = discover_plugins()
+            catalog = get_plugin_catalog()
            tokens = [token.strip() for token in extract.split(",") if token.strip()]
-            plugin_names = {name.lower(): name for name in all_plugins}
+            plugin_names = {name.lower(): name for name in catalog}
            selected = [plugin_names[token.lower()] for token in tokens if token.lower() in plugin_names]
-            selected += plugins_matching_output(all_plugins, tokens)
+            selected += catalog.matching_output(tokens)
            if not selected:
                raise click.UsageError(f"No plugins found matching extract types: {extract}")
            existing = [token.strip() for token in (kwargs.get("plugins") or "").split(",") if token.strip()]
@@ -6,13 +6,13 @@ archivebox archiveresult <action> [args...] [--filters]
 Manage ArchiveResult records (plugin extraction results).

 Actions:
-    create  - Create ArchiveResults for Snapshots (queue extractions)
+    create  - Emit plugin extraction request records for Snapshots
    list    - List ArchiveResults as JSONL (with optional filters)
    update  - Update ArchiveResults from stdin JSONL
    delete  - Delete ArchiveResults from stdin JSONL

 Examples:
-    # Create ArchiveResults for snapshots (queue for extraction)
+    # Emit extraction requests; `archivebox run` schedules their parent snapshots
    archivebox snapshot list --status=queued | archivebox archiveresult create
    archivebox archiveresult create --plugin=screenshot --snapshot-id=<uuid>

@@ -20,9 +20,6 @@ Examples:
    archivebox archiveresult list --status=failed
    archivebox archiveresult list --plugin=screenshot --status=succeeded

-    # Update (reset failed extractions to queued)
-    archivebox archiveresult list --status=failed | archivebox archiveresult update --status=queued
-
    # Delete
    archivebox archiveresult list --plugin=singlefile | archivebox archiveresult delete --yes

@@ -144,21 +141,20 @@ def create_archiveresults(

    created_count = 0
    for snapshot in snapshots:
-        if plugin:
+        config = get_config(crawl=snapshot.crawl, snapshot=snapshot)
+        hooks = [
+            hook
+            for hook in discover_hooks("Snapshot", filter_disabled=not plugin, config=config)
+            if not plugin or hook.parent.name == plugin
+        ]
+        for hook_path in hooks:
+            hook_name = hook_path.stem
+            plugin_name = hook_path.parent.name
            if not is_tty:
-                write_record(build_archiveresult_request(snapshot.id, plugin, status=status))
+                write_record(build_archiveresult_request(snapshot.id, plugin_name, hook_name=hook_name, status=status))
            created_count += 1
-        else:
-            config = get_config(crawl=snapshot.crawl, snapshot=snapshot)
-            hooks = discover_hooks("Snapshot", config=config)
-            for hook_path in hooks:
-                hook_name = hook_path.stem
-                plugin_name = hook_path.parent.name
-                if not is_tty:
-                    write_record(build_archiveresult_request(snapshot.id, plugin_name, hook_name=hook_name, status=status))
-                created_count += 1

-    rprint(f"[green]Created {created_count} archive result request records[/green]", file=sys.stderr)
+    rprint(f"[green]Created {created_count} extraction request records[/green]", file=sys.stderr)
    return 0


@@ -342,7 +338,7 @@ def main():
@click.option("--plugin", "-p", help="Plugin name (e.g., screenshot, singlefile)")
@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
 def create_cmd(snapshot_id: str | None, plugin: str | None, status: str):
-    """Create ArchiveResults for Snapshots from stdin JSONL."""
+    """Emit Snapshot plugin extraction requests as JSONL."""
    sys.exit(create_archiveresults(snapshot_id=snapshot_id, plugin=plugin, status=status))


@@ -1,95 +1,135 @@
 #!/usr/bin/env python3

-"""
-archivebox extract [snapshot_ids...] [--plugins=NAMES]
-
-Run plugins on Snapshots. Accepts snapshot IDs as arguments, from stdin, or via JSONL.
-
-Input formats:
-    - Snapshot UUIDs (one per line)
-    - JSONL: {"type": "Snapshot", "id": "...", "url": "..."}
-    - JSONL: {"type": "ArchiveResult", "snapshot_id": "...", "plugin": "..."}
-
-Output (JSONL):
-    {"type": "ArchiveResult", "id": "...", "snapshot_id": "...", "plugin": "...", "status": "..."}
-
-Examples:
-    # Extract specific snapshot
-    archivebox extract 01234567-89ab-cdef-0123-456789abcdef
-
-    # Pipe from snapshot command
-    archivebox snapshot https://example.com | archivebox extract
-
-    # Run specific plugins only
-    archivebox extract --plugins=screenshot,singlefile 01234567-89ab-cdef-0123-456789abcdef
-
-    # Chain commands
-    archivebox crawl https://example.com | archivebox snapshot | archivebox extract
-"""
+"""Run abx-dl snapshot hooks for existing ArchiveBox snapshots."""

 __package__ = "archivebox.cli"
 __command__ = "archivebox extract"

 import sys
 from collections import defaultdict
-from itertools import product
+from contextlib import redirect_stdout

 import rich_click as click


-def process_archiveresult_by_id(archiveresult_id: str) -> int:
-    """
-    Re-run extraction for a single ArchiveResult by ID.
+def _resolve_requests(records: list[dict], plugins: str) -> tuple[dict[str, set[str]], set[str]]:
+    """Resolve CLI records to snapshot-level execution requests.

-    ArchiveResults are projected status rows, not queued work items. Re-running
-    a single result means resetting that row and queueing its parent snapshot
-    through the shared crawl runner with the corresponding plugin selected.
+    ArchiveResult input is accepted as a convenient reference to its parent
+    Snapshot and plugin. It is never reset or converted into queued work.
    """
-    from rich import print as rprint
-    from archivebox.core.models import ArchiveResult
    from archivebox.api.v1_core import _uuid_ref_query
+    from archivebox.core.models import ArchiveResult, Snapshot
+    from archivebox.misc.jsonl import TYPE_ARCHIVERESULT
+
+    explicit_plugins = {name.strip() for name in plugins.split(",") if name.strip()}
+    requested: dict[str, set[str]] = defaultdict(set)
+    missing: set[str] = set()
+
+    for record in records:
+        record_type = record.get("type")
+        record_id = str(record.get("id") or "")
+
+        if record_type == TYPE_ARCHIVERESULT and record.get("snapshot_id"):
+            requested[str(record["snapshot_id"])].update(explicit_plugins or {str(record.get("plugin") or "")})
+            requested[str(record["snapshot_id"])].discard("")
+            continue
+
+        if record_type == TYPE_ARCHIVERESULT:
+            result = ArchiveResult.objects.filter(_uuid_ref_query("id", record_id)).only("snapshot_id", "plugin").first()
+            if result is not None:
+                requested[str(result.snapshot_id)].update(explicit_plugins or {result.plugin})
+                continue
+
+        snapshot_id = str(record.get("snapshot_id") or record_id or "")
+        snapshot = Snapshot.objects.filter(id=snapshot_id).only("id").first() if snapshot_id else None
+        if snapshot is None and record.get("url"):
+            snapshot = Snapshot.objects.filter(url=record["url"]).order_by("-created_at").only("id").first()
+        # Bare UUID CLI arguments are parsed as Snapshot records because the
+        # input layer cannot know which model owns them. Preserve the public
+        # convenience of passing an ArchiveResult ID by trying that reference
+        # only after the Snapshot lookup misses.
+        if snapshot is None and record_id and not record.get("url"):
+            result = ArchiveResult.objects.filter(_uuid_ref_query("id", record_id)).only("snapshot_id", "plugin").first()
+            if result is not None:
+                requested[str(result.snapshot_id)].update(explicit_plugins or {result.plugin})
+                continue
+        if snapshot is None:
+            missing.add(snapshot_id or str(record.get("url") or ""))
+            continue
+        requested[str(snapshot.id)].update(explicit_plugins)
+
+    return requested, missing
+
+
+def _run_snapshot_requests(requested: dict[str, set[str]], *, wait: bool, show_progress: bool) -> int:
+    from django.utils import timezone
+    from rich import print as rprint
+
+    from archivebox.core.models import Snapshot
    from archivebox.services.runner import run_crawl

-    try:
-        archiveresult = ArchiveResult.objects.get(_uuid_ref_query("id", archiveresult_id))
-    except ArchiveResult.DoesNotExist:
-        rprint(f"[red]ArchiveResult {archiveresult_id} not found[/red]", file=sys.stderr)
+    snapshots = {str(snapshot.id): snapshot for snapshot in Snapshot.objects.filter(id__in=requested).select_related("crawl")}
+    if not snapshots:
+        rprint("[red]No snapshots to process[/red]", file=sys.stderr)
        return 1

-    rprint(f"[blue]Extracting {archiveresult.plugin} for {archiveresult.snapshot.url}[/blue]", file=sys.stderr)
+    from archivebox.config.common import get_config
+    from archivebox.plugins.discovery import get_enabled_plugins

-    try:
-        archiveresult.reset_for_retry()
-        snapshot = archiveresult.snapshot
-        snapshot.queue_for_extraction()
-        crawl = snapshot.crawl
-        if not crawl.claim_processing_lock(lock_seconds=10):
-            rprint(
-                f"[yellow]Crawl {crawl.id} is already owned by another runner[/yellow]",
-                file=sys.stderr,
-            )
-            return 1
+    for snapshot_id, plugin_names in requested.items():
+        snapshot = snapshots.get(snapshot_id)
+        if snapshot is not None and not plugin_names:
+            plugin_names.update(get_enabled_plugins(config=get_config(crawl=snapshot.crawl, snapshot=snapshot)))

-        run_crawl(str(snapshot.crawl_id), snapshot_ids=[str(snapshot.id)], selected_plugins=[archiveresult.plugin])
-        archiveresult.refresh_from_db()
+    if wait and any("search_backend_sonic" in plugin_names for plugin_names in requested.values()):
+        from archivebox.core.takeover_util import ensure_daemon_stack

-        if archiveresult.status == ArchiveResult.StatusChoices.SUCCEEDED:
-            print(f"[green]Extraction succeeded: {archiveresult.output_str}[/green]")
-            return 0
-        elif archiveresult.status == ArchiveResult.StatusChoices.NORESULTS:
-            print(f"[dim]Extraction completed with no results: {archiveresult.output_str}[/dim]")
-            return 0
-        elif archiveresult.status == ArchiveResult.StatusChoices.FAILED:
-            print(f"[red]Extraction failed: {archiveresult.output_str}[/red]", file=sys.stderr)
-            return 1
-        else:
-            # Still in progress or backoff - not a failure
-            print(f"[yellow]Extraction status: {archiveresult.status}[/yellow]")
-            return 0
+        with redirect_stdout(sys.stderr):
+            ensure_daemon_stack(reason="Sonic snapshot indexing")

-    except Exception as e:
-        print(f"[red]Extraction error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
-        return 1
+    # Explicit extraction resumes open/paused snapshots at the snapshot level.
+    # Sealed snapshots stay sealed during targeted maintenance backfills.
+    if wait:
+        for snapshot in snapshots.values():
+            if snapshot.status != Snapshot.StatusChoices.SEALED:
+                snapshot.update_and_requeue(
+                    status=Snapshot.StatusChoices.QUEUED,
+                    retry_at=timezone.now(),
+                )
+
+    if not wait:
+        for snapshot_id, plugin_names in requested.items():
+            snapshot = snapshots.get(snapshot_id)
+            if snapshot is None:
+                continue
+            if plugin_names:
+                snapshot.schedule_plugin_run(plugin_names, when=timezone.now())
+            elif snapshot.status == Snapshot.StatusChoices.SEALED:
+                snapshot.update_and_requeue(retry_at=timezone.now())
+            else:
+                snapshot.update_and_requeue(
+                    status=Snapshot.StatusChoices.QUEUED,
+                    retry_at=timezone.now(),
+                )
+        if show_progress:
+            rprint(f"[blue]Queued {len(snapshots)} snapshots for extraction[/blue]", file=sys.stderr)
+        return 0
+
+    grouped: dict[tuple[str, tuple[str, ...]], list[str]] = defaultdict(list)
+    for snapshot_id, plugin_names in requested.items():
+        snapshot = snapshots.get(snapshot_id)
+        if snapshot is not None:
+            grouped[(str(snapshot.crawl_id), tuple(sorted(plugin_names)))].append(snapshot_id)
+
+    for (crawl_id, plugin_names), snapshot_ids in grouped.items():
+        run_crawl(
+            crawl_id,
+            snapshot_ids=sorted(snapshot_ids),
+            selected_plugins=list(plugin_names) or None,
+            show_progress=show_progress,
+        )
+    return 0


 def run_plugins(
@@ -99,366 +139,64 @@ def run_plugins(
    wait: bool = True,
    emit_results: bool = True,
    show_progress: bool = True,
-    preserve_queued: bool = False,
 ) -> int:
-    """
-    Run plugins on Snapshots from input.
-
-    Reads Snapshot IDs or JSONL from args/stdin, runs plugins, outputs JSONL.
-
-    Exit codes:
-        0: Success
-        1: Failure
-    """
+    """Execute selected plugins through the snapshot-level runner."""
    from rich import print as rprint
-    from django.utils import timezone

-    from archivebox.misc.jsonl import (
-        read_args_or_stdin,
-        write_record,
-        TYPE_SNAPSHOT,
-        TYPE_ARCHIVERESULT,
-    )
    from archivebox.core.models import Snapshot
-    from archivebox.core.models import ArchiveResult
-    from archivebox.services.runner import run_crawl
-    from abx_dl.models import discover_plugins
+    from archivebox.misc.jsonl import read_args_or_stdin, write_record

-    is_tty = sys.stdout.isatty()
-
-    # Parse comma-separated plugins list once (reused in creation and filtering)
-    plugins_list = [p.strip() for p in plugins.split(",") if p.strip()] if plugins else []
-
-    # Parse stdin/args exactly once per CLI invocation.
-    # `main()` may already have consumed stdin to distinguish Snapshot input from
-    # ArchiveResult IDs; if so, it must pass the parsed records through here
-    # instead of asking this helper to reread an already-drained pipe.
    if records is None:
        records = list(read_args_or_stdin(args))
-
    if not records:
        rprint("[yellow]No snapshots provided. Pass snapshot IDs as arguments or via stdin.[/yellow]", file=sys.stderr)
        return 1

-    # Gather snapshot IDs and optional plugin constraints to process
-    snapshot_ids = set()
-    requested_plugins_by_snapshot: dict[str, set[str]] = defaultdict(set)
-    for record in records:
-        record_type = record.get("type")
-
-        if record_type == TYPE_SNAPSHOT:
-            snapshot_id = record.get("id")
-            if snapshot_id:
-                snapshot_ids.add(str(snapshot_id))
-            elif record.get("url"):
-                # Look up by URL (get most recent if multiple exist)
-                snap = Snapshot.objects.filter(url=record["url"]).order_by("-created_at").first()
-                if snap:
-                    snapshot_ids.add(str(snap.id))
-                else:
-                    rprint(f"[yellow]Snapshot not found for URL: {record['url']}[/yellow]", file=sys.stderr)
-
-        elif record_type == TYPE_ARCHIVERESULT:
-            snapshot_id = record.get("snapshot_id")
-            if snapshot_id:
-                snapshot_ids.add(str(snapshot_id))
-                plugin_name = record.get("plugin")
-                if plugin_name and not plugins_list:
-                    requested_plugins_by_snapshot[str(snapshot_id)].add(str(plugin_name))
-
-        elif "id" in record:
-            # Assume it's a snapshot ID
-            snapshot_ids.add(str(record["id"]))
-
-    if not snapshot_ids:
-        rprint("[red]No valid snapshot IDs found in input[/red]", file=sys.stderr)
+    requested, missing = _resolve_requests(records, plugins)
+    for value in sorted(missing):
+        rprint(f"[yellow]Snapshot or ArchiveResult not found: {value}[/yellow]", file=sys.stderr)
+    if not requested:
        return 1

-    existing_snapshots = list(Snapshot.objects.filter(id__in=snapshot_ids).values_list("id", "crawl_id"))
-    existing_snapshot_ids = {str(snapshot_id) for snapshot_id, _crawl_id in existing_snapshots}
-    existing_crawl_ids = {str(crawl_id) for _snapshot_id, crawl_id in existing_snapshots}
-    missing_snapshot_ids = sorted(str(snapshot_id) for snapshot_id in snapshot_ids - existing_snapshot_ids)
-    for snapshot_id in missing_snapshot_ids:
-        rprint(f"[yellow]Snapshot {snapshot_id} not found[/yellow]", file=sys.stderr)
-
-    # Queue only the target plugin rows. Bulk updates keep large reindex runs
-    # from doing one SELECT+UPDATE per snapshot/plugin before hooks even start.
-    requested_pairs: set[tuple[str, str]] = set()
-    if plugins_list:
-        requested_pairs.update((snapshot_id, plugin_name) for snapshot_id, plugin_name in product(existing_snapshot_ids, plugins_list))
-    else:
-        requested_pairs.update(
-            (snapshot_id, plugin_name)
-            for snapshot_id, plugin_names in requested_plugins_by_snapshot.items()
-            if snapshot_id in existing_snapshot_ids
-            for plugin_name in plugin_names
-        )
-    plugins_by_name = discover_plugins(runtime="archivebox")
-    requested_rows: set[tuple[str, str, str]] = set()
-    for snapshot_id, plugin_name in requested_pairs:
-        plugin = plugins_by_name.get(plugin_name)
-        hooks = plugin.filter_hooks("Snapshot") if plugin is not None else []
-        if hooks:
-            requested_rows.update((snapshot_id, plugin_name, hook.name) for hook in hooks)
-        else:
-            requested_rows.add((snapshot_id, plugin_name, ""))
-
-    queued_rows: set[tuple[str, str, str]] = set()
-    if preserve_queued and requested_rows:
-        queued_rows = {
-            (str(snapshot_id), plugin_name, hook_name)
-            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
-                snapshot_id__in=existing_snapshot_ids,
-                plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in requested_rows},
-                status=ArchiveResult.StatusChoices.QUEUED,
-            ).values_list("snapshot_id", "plugin", "hook_name")
-        }
-    rows_to_queue = requested_rows - queued_rows
-
-    reset_fields = {
-        "status": ArchiveResult.StatusChoices.QUEUED,
-        "output_str": "",
-        "output_json": None,
-        "output_files": {},
-        "output_size": 0,
-        "output_mimetypes": "",
-        "start_ts": None,
-        "end_ts": None,
-        "modified_at": timezone.now(),
-    }
-    if rows_to_queue and plugins_list:
-        rows_to_reset_by_hook: dict[tuple[str, str], set[str]] = defaultdict(set)
-        for snapshot_id, plugin_name, hook_name in rows_to_queue:
-            rows_to_reset_by_hook[(plugin_name, hook_name)].add(snapshot_id)
-        for (plugin_name, hook_name), plugin_snapshot_ids in rows_to_reset_by_hook.items():
-            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name, hook_name=hook_name).update(
-                **reset_fields,
-            )
-    elif rows_to_queue and requested_plugins_by_snapshot:
-        snapshot_ids_by_hook: dict[tuple[str, str], set[str]] = defaultdict(set)
-        for snapshot_id, plugin_name, hook_name in rows_to_queue:
-            snapshot_ids_by_hook[(plugin_name, hook_name)].add(snapshot_id)
-        for (plugin_name, hook_name), plugin_snapshot_ids in snapshot_ids_by_hook.items():
-            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name, hook_name=hook_name).update(
-                **reset_fields,
-            )
-    existing_rows = (
-        {
-            (str(snapshot_id), plugin_name, hook_name)
-            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
-                snapshot_id__in=existing_snapshot_ids,
-                plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in rows_to_queue},
-            ).values_list("snapshot_id", "plugin", "hook_name")
-        }
-        if rows_to_queue
-        else set()
-    )
-    missing_rows = rows_to_queue - existing_rows
-    if missing_rows:
-        ArchiveResult.objects.bulk_create(
-            [
-                ArchiveResult(
-                    snapshot_id=snapshot_id,
-                    plugin=plugin_name,
-                    hook_name=hook_name,
-                    status=ArchiveResult.StatusChoices.QUEUED,
-                )
-                for snapshot_id, plugin_name, hook_name in sorted(missing_rows)
-            ],
-            batch_size=500,
-        )
-
-    processed_count = len(existing_snapshot_ids)
-    queue_at = timezone.now()
-    if existing_snapshot_ids:
-        if requested_rows:
-            # Search indexing on a sealed Snapshot is the only targeted hook
-            # allowed to bypass the normal lifecycle. Every other requested
-            # plugin requeues its Snapshot through the unified state machine.
-            affected_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in rows_to_queue}
-            if preserve_queued and queued_rows:
-                queued_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in queued_rows}
-                affected_snapshot_ids.update(
-                    str(snapshot_id)
-                    for snapshot_id in Snapshot.objects.filter(id__in=queued_snapshot_ids)
-                    .filter(retry_at__gt=queue_at)
-                    .values_list("id", flat=True)
-                )
-                affected_snapshot_ids.update(
-                    str(snapshot_id)
-                    for snapshot_id in Snapshot.objects.filter(id__in=queued_snapshot_ids, retry_at__isnull=True).values_list(
-                        "id",
-                        flat=True,
-                    )
-                )
-            requested_plugins_by_id: dict[str, set[str]] = defaultdict(set)
-            for snapshot_id, plugin_name, _hook_name in requested_rows:
-                requested_plugins_by_id[snapshot_id].add(plugin_name)
-            for snapshot in Snapshot.objects.filter(id__in=affected_snapshot_ids).only("id", "status", "modified_at"):
-                # Guard the read-time status so we never bump retry_at on a
-                # row that's been re-queued / started by a concurrent runner.
-                plugin_names = requested_plugins_by_id.get(str(snapshot.id), set())
-                sealed_search_backfill = (
-                    snapshot.status == Snapshot.StatusChoices.SEALED
-                    and plugin_names
-                    and all(plugin_name.startswith("search_backend_") for plugin_name in plugin_names)
-                )
-                if sealed_search_backfill:
-                    snapshot.safe_update(
-                        {"retry_at": queue_at, "modified_at": queue_at},
-                        refresh=False,
-                        extra_filter={"status": snapshot.status},
-                    )
-                else:
-                    snapshot.update_and_requeue(
-                        status=Snapshot.StatusChoices.QUEUED,
-                        retry_at=queue_at,
-                        current_step=0,
-                    )
-        else:
-            # No plugin rows were requested, so this is a full snapshot retry.
-            for snapshot in Snapshot.objects.filter(id__in=existing_snapshot_ids).only("id", "status", "retry_at", "modified_at"):
-                snapshot.safe_update(
-                    {
-                        "status": Snapshot.StatusChoices.QUEUED,
-                        "retry_at": queue_at,
-                        "current_step": 0,
-                        "modified_at": queue_at,
-                    },
-                    refresh=False,
-                    extra_filter={"status": snapshot.status},
-                )
-    if existing_crawl_ids and not requested_rows:
-        from archivebox.crawls.models import Crawl
-
-        for crawl in Crawl.objects.filter(id__in=existing_crawl_ids).only("id", "status", "retry_at", "modified_at"):
-            update_fields = {
-                "retry_at": queue_at,
-                "modified_at": queue_at,
-            }
-            if crawl.status != Crawl.StatusChoices.STARTED:
-                update_fields["status"] = Crawl.StatusChoices.QUEUED
-            crawl.safe_update(
-                update_fields,
-                refresh=False,
-                extra_filter={"status": crawl.status},
-            )
-
-    if processed_count == 0:
-        rprint("[red]No snapshots to process[/red]", file=sys.stderr)
-        return 1
-
-    if show_progress:
-        rprint(f"[blue]Queued {processed_count} snapshots for extraction[/blue]", file=sys.stderr)
-
-    # Run orchestrator if --wait (default)
-    if wait:
-        if show_progress:
-            rprint("[blue]Running plugins...[/blue]", file=sys.stderr)
-        snapshot_ids_by_crawl: dict[str, set[str]] = defaultdict(set)
-        for snapshot_id, crawl_id in existing_snapshots:
-            snapshot_ids_by_crawl[str(crawl_id)].add(str(snapshot_id))
-
-        for crawl_id, crawl_snapshot_ids in snapshot_ids_by_crawl.items():
-            from archivebox.crawls.models import Crawl
-
-            crawl = Crawl.objects.get(id=crawl_id)
-            if not crawl.claim_processing_lock(lock_seconds=10):
-                rprint(
-                    f"[yellow]Crawl {crawl_id} is already owned by another runner[/yellow]",
-                    file=sys.stderr,
-                )
-                return 1
-            selected_plugins = (
-                plugins_list
-                or sorted(
-                    {plugin for snapshot_id in crawl_snapshot_ids for plugin in requested_plugins_by_snapshot.get(str(snapshot_id), set())},
-                )
-                or None
-            )
-            run_crawl(
-                crawl_id,
-                snapshot_ids=sorted(crawl_snapshot_ids),
-                selected_plugins=selected_plugins,
-                show_progress=show_progress,
-            )
-
-    if not emit_results:
-        return 0
-
-    # Output results as JSONL (when piped) or human-readable (when TTY)
-    for snapshot_id in snapshot_ids:
-        try:
-            snapshot = Snapshot.objects.get(id=snapshot_id)
-            results = snapshot.archiveresult_set.all()
-            if plugins_list:
-                results = results.filter(plugin__in=plugins_list)
-
-            for result in results:
-                if is_tty:
-                    status_color = {
-                        "succeeded": "green",
-                        "failed": "red",
-                        "skipped": "yellow",
-                    }.get(result.status, "dim")
-                    rprint(
-                        f"  [{status_color}]{result.status}[/{status_color}] {result.plugin} → {result.output_str or ''}",
-                        file=sys.stderr,
-                    )
-                else:
-                    write_record(result.to_json())
-        except Snapshot.DoesNotExist:
-            continue
+    exit_code = _run_snapshot_requests(requested, wait=wait, show_progress=show_progress)
+    if exit_code or not emit_results:
+        return exit_code

+    is_tty = sys.stdout.isatty()
+    for snapshot in Snapshot.objects.filter(id__in=requested):
+        results = snapshot.archiveresult_set.all()
+        requested_plugins = requested[str(snapshot.id)]
+        if requested_plugins:
+            results = results.filter(plugin__in=requested_plugins)
+        for result in results:
+            if is_tty:
+                color = {"succeeded": "green", "failed": "red", "skipped": "yellow"}.get(result.status, "dim")
+                rprint(f"  [{color}]{result.status}[/{color}] {result.plugin} → {result.output_str or ''}", file=sys.stderr)
+            else:
+                write_record(result.to_json())
    return 0


-def is_archiveresult_id(value: str) -> bool:
-    """Check if value resolves to an ArchiveResult ID."""
-    from archivebox.core.models import ArchiveResult
-    from archivebox.api.v1_core import _uuid_ref_query
-
-    return ArchiveResult.objects.filter(_uuid_ref_query("id", value)).exists()
+def process_archiveresult_by_id(archiveresult_id: str) -> int:
+    """Re-run the parent Snapshot plugin referenced by an ArchiveResult."""
+    return run_plugins((), records=[{"id": archiveresult_id}], wait=True)


@click.command()
-@click.option("--plugins", "--plugin", "-p", default="", help="Comma-separated list of plugins to run (e.g., screenshot,singlefile)")
+@click.option("--plugins", "--plugin", "-p", default="", help="Comma-separated list of plugins to run")
@click.option("--wait/--no-wait", default=True, help="Wait for plugins to complete (default: wait)")
@click.argument("args", nargs=-1)
 def main(plugins: str, wait: bool, args: tuple):
-    """Run plugins on Snapshots, or process existing ArchiveResults by ID"""
+    """Run plugins on Snapshots; ArchiveResult IDs select their parent plugin."""
    from archivebox.misc.jsonl import read_args_or_stdin

-    # Read all input
    records = list(read_args_or_stdin(args))
-
    if not records:
        from rich import print as rprint

        rprint("[yellow]No Snapshot IDs or ArchiveResult IDs provided. Pass as arguments or via stdin.[/yellow]", file=sys.stderr)
        sys.exit(1)
-
-    # Check if input looks like existing ArchiveResult IDs to process
-    all_are_archiveresult_ids = all(is_archiveresult_id(r.get("id") or r.get("url", "")) for r in records)
-
-    if all_are_archiveresult_ids:
-        # Process existing ArchiveResults by ID
-        from rich import print as rprint
-
-        exit_code = 0
-        for record in records:
-            archiveresult_id = record.get("id") or record.get("url")
-            if not isinstance(archiveresult_id, str):
-                rprint(f"[red]Invalid ArchiveResult input: {record}[/red]", file=sys.stderr)
-                exit_code = 1
-                continue
-            result = process_archiveresult_by_id(archiveresult_id)
-            if result != 0:
-                exit_code = result
-        sys.exit(exit_code)
-    else:
-        # Default behavior: run plugins on Snapshots from input
-        sys.exit(run_plugins(args, records=records, plugins=plugins, wait=wait))
+    sys.exit(run_plugins(args, records=records, plugins=plugins, wait=wait))


 if __name__ == "__main__":
@@ -5,6 +5,8 @@ __command__ = "archivebox help"
 import ast
 import importlib.util
 import os
+import sys
+from datetime import UTC, datetime
 from pathlib import Path

 import click
@@ -40,11 +42,14 @@ def help() -> None:

    from archivebox.cli import ArchiveBoxGroup
    from archivebox.config import CONSTANTS
-    from archivebox.config.common import get_config
    from archivebox.config.permissions import IN_DOCKER
-    from archivebox.misc.logging_util import log_cli_command

-    log_cli_command("help", [], None, ".")
+    version_msg = (
+        f"[dark_magenta]\\[{datetime.now(UTC):%Y-%m-%d %H:%M:%S}][/dark_magenta] "
+        f"[dark_red]ArchiveBox[/dark_red] [dark_goldenrod]v{CONSTANTS.VERSION}[/dark_goldenrod]: "
+        "[green4]archivebox [green3]help"
+    )
+    print(Panel(version_msg), file=sys.stderr)

    COMMANDS_HELP_TEXT = (
        "\n    ".join(f"[green]{cmd.ljust(20)}[/green] {_command_doc(cmd, path)}" for cmd, path in ArchiveBoxGroup.meta_commands.items())
@@ -89,7 +94,6 @@ def help() -> None:
    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration]https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration[/link]
 """)

-    get_config()
    if os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and CONSTANTS.ARCHIVE_DIR.is_dir():
        pretty_out_dir = str(CONSTANTS.DATA_DIR).replace(str(Path("~").expanduser()), "~")
        EXAMPLE_USAGE = f"""
@@ -176,10 +176,13 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
        install_method()

    if Snapshot.objects.count() < 25:  # hide the hints for experienced users
+        from archivebox.core.routes_util import build_admin_url
+
+        admin_url = build_admin_url("/admin/", config=config)
        print()
-        print("    [violet]Hint:[/violet] To view your archive index, run:")
+        print("    [violet]Hint:[/violet] To finish web setup, run:")
        print(
-            "        archivebox server  # then visit [deep_sky_blue4][link=http://127.0.0.1:8000]http://127.0.0.1:8000[/link][/deep_sky_blue4]",
+            f"        archivebox server  # then visit [deep_sky_blue4][link={admin_url}]{admin_url}[/link][/deep_sky_blue4]",
        )
        print()
        print("    To add new links, you can run:")
@@ -17,12 +17,12 @@ def _resolve_install_targets(
    requested_names: tuple[str, ...],
 ) -> tuple[list[str], list[str]]:
    """Resolve plugin names and declared binary aliases, leaving unknown tokens raw."""
-    from abx_dl.models import discover_plugins, filter_plugins
+    from archivebox.plugins.discovery import get_plugin_catalog

-    plugins = discover_plugins(runtime="archivebox")
-    plugin_names_by_lower = {plugin_name.lower(): plugin_name for plugin_name in plugins}
+    catalog = get_plugin_catalog()
+    plugin_names_by_lower = {plugin_name.lower(): plugin_name for plugin_name in catalog}
    plugin_names_by_binary_alias: dict[str, set[str]] = {}
-    for plugin_name, plugin in plugins.items():
+    for plugin_name, plugin in catalog.items():
        for required_binary in plugin.config.required_binaries:
            aliases = {required_binary.name.lower()}
            template_match = re.fullmatch(r"\{([A-Za-z_][A-Za-z0-9_]*)\}", required_binary.name)
@@ -47,14 +47,14 @@ def _resolve_install_targets(
        else:
            raw_binary_names.append(name)

-    selected_plugins = filter_plugins(plugins, requested_plugins, include_providers=True) if requested_plugins else {}
+    selected_plugins = catalog.select(requested_plugins) if requested_plugins else catalog.select([])
    selected_plugin_names = {name.lower() for name in selected_plugins}
    raw_binary_names = [name for name in raw_binary_names if name.lower() not in selected_plugin_names]
    return sorted(selected_plugins), sorted(set(raw_binary_names))


 def _install_raw_binary_names(binary_names: list[str], binproviders: str) -> None:
-    """Install user-requested standalone binaries through the Binary state machine."""
+    """Install user-requested standalone binaries through the Binary lifecycle."""
    from django.utils import timezone

    from archivebox.machine.models import Binary, Machine, _canonical_binary_name
@@ -179,7 +179,7 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo
        run_install(plugin_names=install_plugin_names or None)

    if raw_binary_names:
-        print(f"[+] Running direct binary installer via ArchiveBox binary state machine: {', '.join(raw_binary_names)}")
+        print(f"[+] Running direct binary installer via ArchiveBox binary lifecycle: {', '.join(raw_binary_names)}")
        print()
        _install_raw_binary_names(raw_binary_names, binproviders)

@@ -196,8 +196,10 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo
    User = get_user_model()

    if not User.objects.filter(is_superuser=True).exclude(username="system").exists():
-        stderr("\n[+] Don't forget to create a new admin user for the Web UI...", color="green")
-        stderr("    archivebox manage createsuperuser")
+        from archivebox.core.routes_util import build_admin_url
+
+        stderr("\n[+] Open the Admin UI to create the first admin and finish web setup:", color="green")
+        stderr(f"    {build_admin_url('/admin/', config=get_config(resolve_plugins=False))}")

    print()

@@ -27,11 +27,10 @@ Examples:
 __package__ = "archivebox.cli"
 __command__ = "archivebox persona"

-import os
 import sys
 import shutil
-import platform
 from pathlib import Path
+from dataclasses import replace
 from collections.abc import Iterable

 import rich_click as click
@@ -41,116 +40,6 @@ from archivebox.cli.cli_util import apply_filters
 from archivebox.personas import importers as persona_importers


-# =============================================================================
-# Browser Profile Locations
-# =============================================================================
-
-
-def get_chrome_user_data_dir() -> Path | None:
-    """Get the default Chrome user data directory for the current platform."""
-    system = platform.system()
-    home = Path.home()
-
-    if system == "Darwin":  # macOS
-        candidates = [
-            home / "Library" / "Application Support" / "Google" / "Chrome",
-            home / "Library" / "Application Support" / "Chromium",
-        ]
-    elif system == "Linux":
-        candidates = [
-            home / ".config" / "google-chrome",
-            home / ".config" / "chromium",
-            home / ".config" / "chrome",
-            home / "snap" / "chromium" / "common" / "chromium",
-        ]
-    elif system == "Windows":
-        local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
-        candidates = [
-            local_app_data / "Google" / "Chrome" / "User Data",
-            local_app_data / "Chromium" / "User Data",
-        ]
-    else:
-        candidates = []
-
-    for candidate in candidates:
-        if candidate.exists() and (candidate / "Default").exists():
-            return candidate
-
-    return None
-
-
-def get_brave_user_data_dir() -> Path | None:
-    """Get the default Brave user data directory for the current platform."""
-    system = platform.system()
-    home = Path.home()
-
-    if system == "Darwin":
-        candidates = [
-            home / "Library" / "Application Support" / "BraveSoftware" / "Brave-Browser",
-        ]
-    elif system == "Linux":
-        candidates = [
-            home / ".config" / "BraveSoftware" / "Brave-Browser",
-        ]
-    elif system == "Windows":
-        local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
-        candidates = [
-            local_app_data / "BraveSoftware" / "Brave-Browser" / "User Data",
-        ]
-    else:
-        candidates = []
-
-    for candidate in candidates:
-        if candidate.exists() and (candidate / "Default").exists():
-            return candidate
-
-    return None
-
-
-def get_edge_user_data_dir() -> Path | None:
-    """Get the default Edge user data directory for the current platform."""
-    system = platform.system()
-    home = Path.home()
-
-    if system == "Darwin":
-        candidates = [
-            home / "Library" / "Application Support" / "Microsoft Edge",
-        ]
-    elif system == "Linux":
-        candidates = [
-            home / ".config" / "microsoft-edge",
-            home / ".config" / "microsoft-edge-beta",
-            home / ".config" / "microsoft-edge-dev",
-        ]
-    elif system == "Windows":
-        local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
-        candidates = [
-            local_app_data / "Microsoft" / "Edge" / "User Data",
-        ]
-    else:
-        candidates = []
-
-    for candidate in candidates:
-        if candidate.exists() and (candidate / "Default").exists():
-            return candidate
-
-    return None
-
-
-BROWSER_PROFILE_FINDERS = {
-    "chrome": get_chrome_user_data_dir,
-    "chromium": get_chrome_user_data_dir,  # Same locations
-    "brave": get_brave_user_data_dir,
-    "edge": get_edge_user_data_dir,
-}
-
-CHROMIUM_BROWSERS = {"chrome", "chromium", "brave", "edge"}
-
-
-# =============================================================================
-# Cookie Extraction via CDP
-# =============================================================================
-
 # =============================================================================
 # Validation Helpers
 # =============================================================================
@@ -218,6 +107,8 @@ def create_personas(
    names: Iterable[str],
    import_from: str | None = None,
    profile: str | None = None,
+    source: str | None = None,
+    browser_binary: str | None = None,
 ) -> int:
    """
    Create Personas from names.
@@ -239,25 +130,26 @@ def create_personas(
        rprint("[yellow]No persona names provided. Pass names as arguments.[/yellow]", file=sys.stderr)
        return 1

-    # Validate import source if specified
-    source_profile_dir = None
+    import_source = None
+    if source and not import_from:
+        rprint("[red]--source requires --import (the source browser name).[/red]", file=sys.stderr)
+        return 1
    if import_from:
-        import_from = import_from.lower()
-        if import_from not in BROWSER_PROFILE_FINDERS:
-            rprint(f"[red]Unknown browser: {import_from}[/red]", file=sys.stderr)
-            rprint(f"[dim]Supported browsers: {', '.join(BROWSER_PROFILE_FINDERS.keys())}[/dim]", file=sys.stderr)
+        try:
+            if source:
+                import_source = persona_importers.resolve_custom_import_source(source, profile_dir=profile)
+                if import_source.browser != "persona":
+                    import_source = replace(import_source, browser=import_from.lower(), browser_binary=browser_binary)
+            elif import_from.startswith(("http://", "https://", "ws://", "wss://")) or Path(import_from).expanduser().is_absolute():
+                import_source = persona_importers.resolve_custom_import_source(import_from)
+            else:
+                import_source = persona_importers.resolve_browser_import_source(import_from, profile_dir=profile)
+            if browser_binary:
+                import_source = replace(import_source, browser_binary=browser_binary)
+        except ValueError as err:
+            rprint(f"[red]{err}[/red]", file=sys.stderr)
            return 1

-        source_profile_dir = BROWSER_PROFILE_FINDERS[import_from]()
-        if not source_profile_dir:
-            rprint(f"[red]Could not find {import_from} profile directory[/red]", file=sys.stderr)
-            return 1
-
-        rprint(f"[dim]Found {import_from} profile: {source_profile_dir}[/dim]", file=sys.stderr)
-
-        if profile is None and (source_profile_dir / "Default").exists():
-            profile = "Default"
-
    created_count = 0
    for name in name_list:
        name = name.strip()
@@ -275,16 +167,16 @@ def create_personas(
        if created:
            persona.ensure_dirs()
            created_count += 1
-            rprint(f"[green]Created persona: {name}[/green]", file=sys.stderr)
+
        else:
            rprint(f"[dim]Persona already exists: {name}[/dim]", file=sys.stderr)

        cookies_file = Path(persona.path) / "cookies.txt"

        # Import browser profile if requested
-        if import_from in CHROMIUM_BROWSERS and source_profile_dir is not None:
+        if import_source is not None:
            try:
-                import_source = persona_importers.resolve_browser_import_source(import_from, profile_dir=profile)
+                rprint(f"[dim]Importing {import_source.display_label}; copying settings and exporting cookies...[/dim]", file=sys.stderr)
                import_result = persona_importers.import_persona_from_source(
                    persona,
                    import_source,
@@ -292,20 +184,25 @@ def create_personas(
                    import_cookies=True,
                    capture_storage=False,
                )
-            except Exception as e:
+            except (Exception, KeyboardInterrupt) as e:
+                if created:
+                    shutil.rmtree(persona.path, ignore_errors=True)
+                    persona.delete()
                rprint(f"[red]Failed to import browser profile: {e}[/red]", file=sys.stderr)
                return 1

            if import_result.profile_copied:
                rprint("[green]Copied browser profile to persona[/green]", file=sys.stderr)
            if import_result.cookies_imported:
-                rprint(f"[green]Extracted cookies to {cookies_file}[/green]", file=sys.stderr)
+                rprint(f"[green]Extracted {import_result.cookie_count} cookies to {cookies_file}[/green]", file=sys.stderr)
            elif not import_result.profile_copied:
                rprint("[yellow]Could not import cookies automatically.[/yellow]", file=sys.stderr)

            for warning in import_result.warnings:
                rprint(f"[yellow]{warning}[/yellow]", file=sys.stderr)

+        if created:
+            rprint(f"[green]Created persona: {name}[/green]", file=sys.stderr)
        if not is_tty:
            write_record(
                {
@@ -544,11 +441,17 @@ def main():

@main.command("create")
@click.argument("names", nargs=-1)
-@click.option("--import", "import_from", help="Import profile from browser (chrome, chromium, brave, edge)")
+@click.option("--import", "import_from", help="Import from chrome, chromium, brave, edge, a persona directory, or a live CDP URL")
+@click.option("--source", help="Source browser user-data directory or exact profile path (including Docker mounts)")
+@click.option(
+    "--browser-binary",
+    type=click.Path(exists=True, dir_okay=False),
+    help="Source Chromium browser executable; required for other Chromium-based browsers",
+)
@click.option("--profile", help="Profile directory name under the user data dir (e.g. Default, Profile 1)")
-def create_cmd(names: tuple, import_from: str | None, profile: str | None):
+def create_cmd(names: tuple, import_from: str | None, profile: str | None, source: str | None, browser_binary: str | None):
    """Create Personas, optionally importing from a browser profile."""
-    sys.exit(create_personas(names, import_from=import_from, profile=profile))
+    sys.exit(create_personas(names, import_from=import_from, profile=profile, source=source, browser_binary=browser_binary))


@main.command("list")
@@ -560,6 +463,84 @@ def list_cmd(name: str | None, name__icontains: str | None, limit: int | None):
    sys.exit(list_personas(name=name, name__icontains=name__icontains, limit=limit))


+@main.command("open")
+@click.argument("name", default="Default")
+def open_cmd(name: str):
+    """Open a persona's browser, including its imported logins."""
+    import json
+    import os
+    import subprocess
+
+    import psutil
+
+    from abx_plugins import get_plugins_dir
+    from archivebox.config.common import get_config
+    from archivebox.personas.models import Persona
+
+    valid, reason = validate_persona_name(name)
+    if not valid:
+        raise click.ClickException(reason)
+    persona = Persona.get_or_create_named(name)
+    persona.ensure_dirs()
+    config = get_config(persona=persona)
+    if os.environ.get("DISPLAY"):
+        from archivebox.config import CONSTANTS
+
+        for process in psutil.process_iter():
+            try:
+                args = process.cmdline()
+                profile = next((arg.split("=", 1)[1] for arg in args if arg.startswith("--user-data-dir=")), None)
+                if (
+                    not profile
+                    or any(arg.startswith(("--type=", "--headless")) for arg in args)
+                    or not Path(profile).resolve().is_relative_to(CONSTANTS.DATA_DIR.resolve())
+                    or process.environ().get("DISPLAY") != os.environ["DISPLAY"]
+                ):
+                    continue
+                rprint("An ArchiveBox browser is already open on this display; waiting for it to close.")
+                process.wait()
+            except (psutil.NoSuchProcess, psutil.AccessDenied):
+                continue
+    plugins_dir = Path(get_plugins_dir()).resolve()
+    env = os.environ.copy()
+    dependencies = subprocess.run(
+        [
+            str(Path(sys.executable).with_name("abxpkg")),
+            "env",
+            "--install",
+            "--json",
+            f"--lib={config.ABXPKG_LIB_DIR}",
+            f"--deps-from={plugins_dir / 'chrome' / 'config.json'}:required_binaries",
+        ],
+        capture_output=True,
+        text=True,
+        env=env,
+    )
+    if dependencies.returncode:
+        raise click.ClickException(dependencies.stderr.strip())
+    env.update({key: str(value) for key, value in json.loads(dependencies.stdout).items()})
+    env["ARCHIVEBOX_ABX_PLUGINS_DIR"] = str(plugins_dir)
+    env["ACTIVE_PERSONA"] = persona.name
+    env["PERSONAS_DIR"] = str(persona.path.parent)
+    env["NODE_MODULES_DIR"] = str(config.ABXPKG_LIB_DIR / "pnpm" / "packages" / "chrome" / "node_modules")
+    payload = config.model_dump(mode="json")
+    payload.update(
+        ACTIVE_PERSONA=persona.name,
+        CHROME_HEADLESS=False,
+        CHROME_USER_DATA_DIR=str(persona.CHROME_USER_DATA_DIR),
+        AUTH_STORAGE_FILE=str(persona.AUTH_STORAGE_FILE or ""),
+        COOKIES_FILE=str(persona.COOKIES_FILE or ""),
+    )
+    script = Path(__file__).parent.parent / "personas" / "open_browser.js"
+    result = subprocess.run(
+        [env.get("NODE_BINARY") or "node", str(script)],
+        input=json.dumps(payload),
+        text=True,
+        env=env,
+    )
+    sys.exit(result.returncode)
+
+
@main.command("update")
@click.option("--name", "-n", help="Set new name")
 def update_cmd(name: str | None):
@@ -18,7 +18,7 @@ EVENT_FLOW_DIAGRAM = """
 │         └─ BinaryRequestEvent                                               │
 │              └─ abxpkg BinaryService builtin providers                      │
 │                   └─ BinaryEvent                                            │
-│                        └─ BinaryCacheService / project cache backend        │
+│                        └─ ArchiveBox DB history projection                  │
 │                                                                             │
 │  CrawlEvent                                                                 │
 │    └─ CrawlSetupEvent                                                       │
@@ -48,7 +48,7 @@ def pluginmap(
    Show the current abx-dl event phases and their associated plugin hooks.

    This command reflects the new bus-driven runtime, not the legacy ArchiveBox
-    state-machine executor. Event names are normalized to hook prefixes by
+    event runtime. Event names are normalized to hook prefixes by
    stripping a trailing `Event`, then ArchiveBox checks whether any matching
    `on_{EventFamily}__*` scripts actually exist.
    """
@@ -54,6 +54,11 @@ RUNNER_DAEMON_ENV = "ARCHIVEBOX_RUNNER_DAEMON"


 def _exit_daemon_runner_on_signal(sig: signal.Signals) -> None:
+    # A supervised `archivebox run --daemon` is intentionally a disposable
+    # child. If it receives SIGINT/SIGTERM directly, exit with the conventional
+    # signal status so supervisord treats it as an unexpected worker death and
+    # restarts only the runner. The parent `archivebox server` owns supervisord
+    # shutdown and must not be pulled down by a killed daemon worker.
    os._exit(128 + int(sig))


@@ -67,8 +72,8 @@ def process_stdin_records() -> int:

    Outputs JSONL of all processed records (for chaining).

-    Handles any record type: Crawl, Snapshot, ArchiveResult.
-    Auto-cascades: Crawl → Snapshots → ArchiveResults.
+    Handles Crawl and Snapshot work records. ArchiveResult records are accepted
+    as references to their parent Snapshot and plugin.

    Returns exit code (0 = success, 1 = error).
    """
@@ -85,6 +90,7 @@ def process_stdin_records() -> int:
    )
    from archivebox.base_models.models import get_or_create_system_user_pk
    from archivebox.core.models import Snapshot, ArchiveResult
+    from archivebox.api.v1_core import _uuid_ref_query
    from archivebox.crawls.models import Crawl
    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
    from archivebox.machine.models import Binary
@@ -151,33 +157,15 @@ def process_stdin_records() -> int:
                    queued_count += 1

            elif record_type == TYPE_ARCHIVERESULT:
-                if record_id:
-                    # Existing archiveresult - re-queue
-                    try:
-                        archiveresult = ArchiveResult.objects.get(id=record_id)
-                    except ArchiveResult.DoesNotExist:
-                        archiveresult = None
-                else:
-                    archiveresult = None
-
-                snapshot_id = record.get("snapshot_id")
-                plugin_name = record.get("plugin")
-                snapshot = None
-                if archiveresult:
-                    if archiveresult.status in [
-                        ArchiveResult.StatusChoices.FAILED,
-                        ArchiveResult.StatusChoices.SKIPPED,
-                        ArchiveResult.StatusChoices.NORESULTS,
-                        ArchiveResult.StatusChoices.BACKOFF,
-                    ]:
-                        archiveresult.reset_for_retry()
-                    snapshot = archiveresult.snapshot
-                    plugin_name = plugin_name or archiveresult.plugin
-                elif snapshot_id:
-                    try:
-                        snapshot = Snapshot.objects.get(id=snapshot_id)
-                    except Snapshot.DoesNotExist:
-                        snapshot = None
+                snapshot_id = str(record.get("snapshot_id") or "")
+                plugin_name = str(record.get("plugin") or "")
+                archiveresult = None
+                if not snapshot_id and record_id:
+                    archiveresult = ArchiveResult.objects.filter(_uuid_ref_query("id", str(record_id))).select_related("snapshot").first()
+                    if archiveresult:
+                        snapshot_id = str(archiveresult.snapshot_id)
+                        plugin_name = plugin_name or archiveresult.plugin
+                snapshot = Snapshot.objects.filter(id=snapshot_id).first() if snapshot_id else None

                if snapshot:
                    snapshot.queue_for_extraction()
@@ -185,7 +173,7 @@ def process_stdin_records() -> int:
                    snapshot_ids_by_crawl[crawl_id].add(str(snapshot.id))
                    if plugin_name:
                        plugin_names_by_crawl[crawl_id].add(str(plugin_name))
-                    output_records.append(record if not archiveresult else archiveresult.to_json())
+                    output_records.append(archiveresult.to_json() if archiveresult else record)
                    queued_count += 1

            elif record_type in {TYPE_BINARYREQUEST, TYPE_BINARY}:
@@ -243,7 +231,6 @@ def process_stdin_records() -> int:
                    crawl_id,
                    snapshot_ids=None if crawl_id in full_crawl_ids else sorted(snapshot_ids_by_crawl[crawl_id]),
                    selected_plugins=None if crawl_id in run_all_plugins_for_crawl else sorted(plugin_names_by_crawl[crawl_id]),
-                    selected_plugins_are_explicit=False,
                )
    return 0

@@ -252,7 +239,6 @@ def run_runner(
    daemon: bool = False,
    crawl_id: str | None = None,
    maintenance_only: bool = False,
-    maintenance_batch_size: int | None = None,
 ) -> int:
    """
    Run the background runner loop.
@@ -305,6 +291,7 @@ def run_runner(
    interactive_interrupts = current.root.process_type == Process.TypeChoices.ADD
    if daemon:
        os.environ[RUNNER_DAEMON_ENV] = "1"
+
    try:
        with (
            foreground_shutdown_signals(
@@ -318,7 +305,6 @@ def run_runner(
                crawl_id=crawl_id,
                maintenance_only=maintenance_only,
                interactive_interrupts=interactive_interrupts,
-                **({"maintenance_batch_size": maintenance_batch_size} if maintenance_batch_size else {}),
            )
        return 0
    except KeyboardInterrupt:
@@ -343,13 +329,6 @@ def run_runner(
@click.option("--snapshot-id", help="Run one snapshot through its crawl")
@click.option("--binary-id", help="Run one queued binary install directly on the bus")
@click.option("--maintenance-only", is_flag=True, help="Only process sealed Snapshot maintenance and search-index backfills")
-@click.option(
-    "--maintenance-batch-size",
-    type=int,
-    default=None,
-    hidden=True,
-    help="Limit queued maintenance snapshots claimed per scheduler tick",
-)
@click.option("--no-stdin", is_flag=True, hidden=True, help="Run the scheduler even when stdin is not a TTY")
 def main(
    daemon: bool,
@@ -357,7 +336,6 @@ def main(
    snapshot_id: str,
    binary_id: str,
    maintenance_only: bool,
-    maintenance_batch_size: int | None,
    no_stdin: bool,
 ):
    """
@@ -382,7 +360,7 @@ def main(
                ),
                foreground_parent_watchdog(enabled=False),
            ):
-                sys.exit(run_runner(daemon=True, maintenance_only=maintenance_only, maintenance_batch_size=maintenance_batch_size))
+                sys.exit(run_runner(daemon=True, maintenance_only=maintenance_only))
        except KeyboardInterrupt:
            sys.exit(0)

@@ -411,17 +389,16 @@ def main(
                    daemon=False,
                    crawl_id=crawl_id,
                    maintenance_only=maintenance_only,
-                    maintenance_batch_size=maintenance_batch_size,
                ),
            )

        if maintenance_only:
-            sys.exit(run_runner(daemon=daemon, maintenance_only=True, maintenance_batch_size=maintenance_batch_size))
+            sys.exit(run_runner(daemon=daemon, maintenance_only=True))

        if not no_stdin and not sys.stdin.isatty():
            sys.exit(process_stdin_records())
        else:
-            sys.exit(run_runner(daemon=daemon, maintenance_only=maintenance_only, maintenance_batch_size=maintenance_batch_size))
+            sys.exit(run_runner(daemon=daemon, maintenance_only=maintenance_only))


 def run_snapshot_worker(snapshot_id: str) -> int:
@@ -37,6 +37,7 @@ def schedule(
        "created_schedule_ids": [],
        "disabled_count": 0,
        "run_all_enqueued": 0,
+        "run_all_maintained": 0,
        "active_schedule_ids": [],
    }

@@ -57,12 +58,12 @@ def schedule(

        created_by_id = get_or_create_system_user_pk()
        is_update_schedule = not import_path
-        template_urls = import_path or "archivebox://update"
+        template_urls = import_path or ""
        template_label = (f"Scheduled import: {template_urls}" if import_path else "Scheduled ArchiveBox update")[:64]
        template_notes = (
            f"Created by archivebox schedule for {template_urls}"
            if import_path
-            else "Created by archivebox schedule to queue recurring archivebox://update maintenance crawls."
+            else "Created by archivebox schedule to run recurring ArchiveBox maintenance."
        )

        template = Crawl.objects.create(
@@ -76,7 +77,6 @@ def schedule(
            retry_at=None,
            config={
                "DEPTH": 0 if is_update_schedule else depth,
-                "SCHEDULE_KIND": "update" if is_update_schedule else "crawl",
                # Caller-supplied overrides (e.g. {"ONLY_NEW": False}) win over the
                # template defaults. Anything left unset falls through to the
                # standard config stack at crawl-resolution time.
@@ -86,6 +86,7 @@ def schedule(
        crawl_schedule = CrawlSchedule.objects.create(
            template=template,
            schedule=schedule_str,
+            config={**template.config, "SCHEDULE_KIND": "update" if is_update_schedule else "crawl"},
            is_enabled=True,
            label=template_label,
            notes=template_notes,
@@ -119,12 +120,19 @@ def schedule(

    if run_all:
        enqueued = 0
+        maintained = 0
        now = timezone.now()
        for scheduled_crawl in schedules:
-            scheduled_crawl.enqueue(queued_at=now)
-            enqueued += 1
+            if scheduled_crawl.dispatch(queued_at=now) is None:
+                maintained += 1
+            else:
+                enqueued += 1
        result["run_all_enqueued"] = enqueued
+        result["run_all_maintained"] = maintained
        print(f"[green]\\[*] Enqueued {enqueued} scheduled crawl(s) immediately.[/green]")
+        if maintained:
+            run_pending_crawls(maintenance_only=True)
+            print(f"[green]\\[*] Ran {maintained} scheduled maintenance update(s).[/green]")
        if enqueued:
            print(
                "[yellow]\\[*] Start `archivebox server`, `archivebox run --daemon`, or `archivebox schedule --foreground` to process the queued crawls.[/yellow]",
@@ -171,7 +179,7 @@ def schedule(
@click.option("--clear", is_flag=True, help="Disable all currently enabled schedules")
@click.option("--show", is_flag=True, help="Print all currently enabled schedules")
@click.option("--foreground", "-f", is_flag=True, help="Run the global crawl runner in the foreground (no crontab required)")
-@click.option("--run-all", is_flag=True, help="Enqueue all enabled schedules immediately and process them once")
+@click.option("--run-all", is_flag=True, help="Dispatch all enabled schedules immediately")
@click.argument("import_path", required=False)
@docstring(schedule.__doc__)
 def main(**kwargs):
@@ -11,6 +11,8 @@ from collections.abc import Iterable

 import rich_click as click
 from rich import print
+from rich.style import Style
+from rich.text import Text

 from archivebox.config import CONSTANTS
 from archivebox.misc.util import docstring, enforce_types
@@ -251,14 +253,6 @@ def server(
        manage(args=["createsuperuser"])
        print()

-    if not User.objects.filter(is_superuser=True).exclude(username="system").exists():
-        print()
-        print(
-            "[violet]Hint:[/violet] To create an [bold]admin username & password[/bold] for the [deep_sky_blue3][underline][link=http://{host}:{port}/admin]Admin UI[/link][/underline][/deep_sky_blue3], run:",
-        )
-        print("      [green]archivebox manage createsuperuser[/green]")
-        print()
-
    # First non-empty positional arg is the bind spec; otherwise inherit from
    # config (which defaults to "127.0.0.1:8000"). _parse_and_validate_bind_spec
    # hard-errors on hostnames so the rest of the server can assume a numeric
@@ -266,6 +260,17 @@ def server(
    bind_spec = next((arg for arg in runserver_args if arg), "")
    host, port = _parse_and_validate_bind_spec(bind_spec)

+    if not User.objects.filter(is_superuser=True).exclude(username="system").exists():
+        from archivebox.core.routes_util import build_admin_url
+
+        print()
+        print("[violet]Hint:[/violet] Open the Admin UI to create the first admin and finish web setup:")
+        runtime_config = config.model_copy(update={"BIND_ADDR": f"{host}:{port}"})
+        print(f"      [green]{build_admin_url('/admin/', config=runtime_config)}[/green]")
+        if not config.BASE_URL and host not in ("127.0.0.1", "localhost"):
+            print("      (When running remotely, replace localhost with this server's IP address or hostname.)")
+        print()
+
    if daemonize and os.environ.get("ARCHIVEBOX_SERVER_DAEMON_CHILD") != "1":
        from archivebox.workers.supervisord_util import resolve_env_binary

@@ -339,12 +344,15 @@ def server(
        print("[green][+] Starting ArchiveBox webserver in DEBUG mode...[/green]")
    else:
        print("[green][+] Starting ArchiveBox webserver...[/green]")
-    print(
-        f"    [blink][green]>[/green][/blink] Starting ArchiveBox webserver on [dim]BIND_ADDR[/dim] [deep_sky_blue4][link=http://{host}:{port}]http://{host}:{port}[/link][/deep_sky_blue4]",
-    )
-    print(
-        f"    [green]>[/green] Log in to ArchiveBox Admin UI on [dim]BASE_URL [/dim] [deep_sky_blue3][link={admin_url}]{admin_url}[/link][/deep_sky_blue3]",
+    bind_url = f"http://{host}:{port}"
+    bind_message = Text.from_markup(
+        "    [blink][green]>[/green][/blink] Starting ArchiveBox webserver on [dim]BIND_ADDR[/dim] ",
    )
+    bind_message.append(bind_url, style=Style(color="deep_sky_blue4", link=bind_url))
+    print(bind_message)
+    admin_message = Text.from_markup("    [green]>[/green] Log in to ArchiveBox Admin UI on [dim]BASE_URL [/dim] ")
+    admin_message.append(admin_url, style=Style(color="deep_sky_blue3", link=admin_url))
+    print(admin_message)
    print("    > Writing ArchiveBox error log to ./logs/errors.log")
    print()

@@ -352,16 +360,13 @@ def server(
    # security-mode + base-url warnings see the effective values.
    runtime_config = get_config()
    _print_server_startup_warnings(runtime_config, host, port)
-    bind_url = f"http://{host}:{port}"
    command = current_command(Process.TypeChoices.SERVER, data_dir=CONSTANTS.DATA_DIR, url=bind_url)

    def still_owns_runtime_stack() -> bool:
-        from django.db import connections
-
-        try:
-            return command_owns_runtime_stack(command, data_dir=CONSTANTS.DATA_DIR)
-        finally:
-            connections.close_all()
+        # This is polled at the log-tail cadence. Keep the autocommit
+        # connection open so SQLite can reuse its page cache between polls;
+        # reconnecting here repeatedly rereads the process table while idle.
+        return command_owns_runtime_stack(command, data_dir=CONSTANTS.DATA_DIR)

    shutdown_state = None
    try:
@@ -409,8 +414,13 @@ def server(
    except KeyboardInterrupt:
        pass
    finally:
-        if not shutdown_state or not shutdown_state.signal_name:
-            command.mark_exited()
+        from django.db import connections
+
+        try:
+            if not shutdown_state or not shutdown_state.signal_name:
+                command.mark_exited()
+        finally:
+            connections.close_all()
    print("\n[i][green][🟩] ArchiveBox server shut down gracefully.[/green][/i]")


@@ -131,9 +131,11 @@ def status(out_dir: Path = CONSTANTS.DATA_DIR) -> None:
        print(f"    Last changes: {str(last_downloaded.downloaded_at)[:16]}")

    if not users:
+        from archivebox.core.routes_util import build_admin_url
+
        print()
-        print("    [violet]Hint:[/violet] You can create an admin user by running:")
-        print("        [green]archivebox manage createsuperuser[/green]")
+        print("    [violet]Hint:[/violet] Open the Admin UI to create the first admin and finish web setup:")
+        print(f"        [green]{build_admin_url('/admin/', config=config)}[/green]")

    print()
    recent_snapshots = snapshots_qs.order_by(
@@ -77,107 +77,71 @@ def reindex_snapshots(
 ) -> dict[str, Any]:
    from archivebox.cli.archivebox_extract import run_plugins
    from archivebox.core.models import ArchiveResult, Snapshot
-    from abx_dl.models import discover_plugins
+    from django.db.models import Exists, OuterRef

    # Search backfill is the one maintenance hook allowed to execute without
    # reopening a Snapshot. Restrict that exception to already-sealed rows;
-    # every open lifecycle state remains owned by the normal state machine.
+    # every open lifecycle state remains owned by the normal runner lifecycle.
    snapshots = snapshots.filter(status=Snapshot.StatusChoices.SEALED)

-    stats: dict[str, Any] = {"processed": 0, "requested": 0, "queued": 0, "skipped_queued": 0, "reindexed": 0, "snapshot_ids": []}
-    records: list[dict[str, str]] = []
-    plugins_by_name = discover_plugins(runtime="archivebox")
-    required_hooks_by_plugin = {
-        plugin_name: frozenset(hook.name for hook in plugins_by_name[plugin_name].filter_hooks("Snapshot"))
-        for plugin_name in search_plugins
-        if plugin_name in plugins_by_name
-    }
+    stats: dict[str, Any] = {"processed": 0, "requested": 0, "queued": 0, "reindexed": 0, "snapshot_ids": []}
+    print(f"[*] Reindexing missing search indexes with: {', '.join(search_plugins)}")

-    total = snapshots.count()
-    print(f"[*] Reindexing {total} snapshots with search plugins: {', '.join(search_plugins)}")
-
-    def run_batch() -> None:
-        if not records:
-            return
-        if wait_for_turn:
-            wait_for_turn()
-        batch_records = list(records)
-        snapshot_ids = {record["snapshot_id"] for record in batch_records}
-        plugin_names = {record["plugin"] for record in batch_records}
-        queued_rows = {
-            (str(snapshot_id), plugin_name, hook_name)
-            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
-                snapshot_id__in=snapshot_ids,
-                plugin__in=plugin_names,
-                status=ArchiveResult.StatusChoices.QUEUED,
-            ).values_list("snapshot_id", "plugin", "hook_name")
-        }
-        records_to_queue = []
-        for record in batch_records:
-            snapshot_id = record["snapshot_id"]
-            plugin_name = record["plugin"]
-            required_hooks = required_hooks_by_plugin.get(plugin_name, frozenset())
-            if required_hooks and all((snapshot_id, plugin_name, hook_name) in queued_rows for hook_name in required_hooks):
-                stats["skipped_queued"] += 1
-                continue
-            records_to_queue.append(record)
-        if not records_to_queue:
-            print(
-                f"    [{stats['processed']}/{total}] Already queued {len(batch_records)} index jobs",
-            )
-            records.clear()
-            return
-        # `archivebox update --index-only` intentionally breaks the usual
-        # "runner discovers work" rule by inserting synthetic queued
-        # ArchiveResult rows for search backends. run_plugins() keeps this as
-        # statement-sized UPDATE/bulk_create work, then bumps Snapshot.retry_at
-        # so the orchestrator owns actual hook execution. Paused snapshots stay
-        # PAUSED; run_due_snapshot restores retry_at=MAX after targeted rows
-        # finish.
-        exit_code = run_plugins(
-            args=(),
-            records=records_to_queue,
-            wait=False,
-            emit_results=False,
-            show_progress=False,
-            preserve_queued=True,
+    completed_statuses = [ArchiveResult.StatusChoices.SUCCEEDED, ArchiveResult.StatusChoices.NORESULTS]
+    for plugin_name in search_plugins:
+        completed_result = ArchiveResult.objects.filter(
+            snapshot_id=OuterRef("pk"),
+            plugin=plugin_name,
+            status__in=completed_statuses,
        )
-        if exit_code != 0:
-            raise SystemExit(exit_code)
-        stats["queued"] += len(records_to_queue)
-        print(
-            f"    [{stats['processed']}/{total}] Queued {len(records_to_queue)} index jobs for orchestrator",
-        )
-        records.clear()
-
-    for snapshot in snapshots.select_related("crawl").paged_iterator(chunk_size=batch_size):
-        try:
-            stats["processed"] += 1
-
-            if _get_snapshot_crawl(snapshot) is None:
-                continue
-
+        candidates = snapshots.annotate(has_completed_index=Exists(completed_result)).filter(has_completed_index=False).order_by("id")
+        after_id = None
+        while True:
+            if wait_for_turn:
+                wait_for_turn()
+            page = candidates.filter(id__gt=after_id) if after_id is not None else candidates
+            batch = list(page.select_related(None).only("id", "timestamp")[:batch_size])
+            if not batch:
+                break
+            after_id = batch[-1].id
+            records = [{"type": "Snapshot", "id": str(snapshot.id)} for snapshot in batch]
+            stats["processed"] += len(batch)
+            stats["requested"] += len(batch)
            if collect_ids:
-                stats["snapshot_ids"].append(str(snapshot.id))
-            for plugin_name in search_plugins:
-                records.append(
-                    {
-                        "type": "ArchiveResult",
-                        "snapshot_id": str(snapshot.id),
-                        "plugin": plugin_name,
-                    },
-                )
-                stats["requested"] += 1
-            if len(records) >= batch_size:
-                run_batch()
-        except KeyboardInterrupt as err:
-            err.archivebox_resume = snapshot.timestamp
-            raise
-
-    run_batch()
+                stats["snapshot_ids"].extend(str(snapshot.id) for snapshot in batch)
+            exit_code = run_plugins(
+                args=(),
+                records=records,
+                plugins=plugin_name,
+                wait=True,
+                emit_results=False,
+                show_progress=False,
+            )
+            if exit_code != 0:
+                raise SystemExit(exit_code)
+            stats["reindexed"] += len(batch)
+            print(f"    [{plugin_name}] indexed {stats['reindexed']} missing snapshots")
    return stats


+def run_scheduled_maintenance(*, batch_size: int = 500) -> dict[str, Any]:
+    """Queue stale filesystem rows and backfill missing search facts."""
+    from archivebox.core.models import Snapshot
+
+    filesystem_stats = process_all_db_snapshots(batch_size=batch_size)
+    search_plugins = _get_search_indexing_plugins()
+    search_stats = (
+        reindex_snapshots(
+            Snapshot.objects.filter(fs_version=Snapshot._fs_current_version()),
+            search_plugins=search_plugins,
+            batch_size=batch_size,
+        )
+        if search_plugins
+        else {"processed": 0, "requested": 0, "queued": 0, "reindexed": 0, "snapshot_ids": []}
+    )
+    return {"filesystem": filesystem_stats, "search": search_stats}
+
+
@enforce_types
 def update(
    filter_patterns: Iterable[str] = (),
@@ -203,9 +167,9 @@ def update(
    Update snapshots: migrate old dirs, reconcile DB, and re-queue for archiving.

    Three-phase operation (without filters):
-    - Phase 1: Drain old archive/ dirs by moving to new fs location (0.8.x → 0.9.x)
-    - Phase 2: O(n) scan over entire DB from most recent to least recent
-    - No orphan scans needed (trust 1:1 mapping between DB and filesystem after phase 1)
+    - Phase 1: Drain legacy archive/ directories into the current layout
+    - Phase 2: Select only stale fs_version rows through the indexed column
+    - Phase 3: Run queued snapshot-level filesystem maintenance until idle

    With filters: Only phase 2 (DB query), no filesystem operations.
    Without filters: All phases (full update).
@@ -232,7 +196,6 @@ def update(
    from archivebox.core.takeover_util import (
        command_owns_foreground_runner,
        current_command,
-        foreground_runner_owner,
        standby_until_foreground_runner_needed,
    )
    from archivebox.workers.supervisord_util import run_runner_worker, stop_own_supervisord_process
@@ -291,11 +254,6 @@ def update(
            while True:
                do_migrate = migrate_only or not index_only
                do_index = index_only or not migrate_only
-                do_run_until_idle = do_migrate or do_index
-                ran_post_migrate_runner = False
-                full_update_empty = False
-                maintenance_work_queued = False
-                runner_work_queued = False

                if do_migrate:
                    if (
@@ -327,13 +285,11 @@ def update(
                            after=after,
                            resume=resume,
                            batch_size=batch_size,
-                            queue_for_archiving=do_run_until_idle,
+                            queue_for_archiving=True,
                            wait_for_turn=wait_for_turn,
                        )
                        print_stats(stats)
                        touched_snapshot_ids.update(stats.get("snapshot_ids", []))
-                        maintenance_work_queued = stats.get("queued", 0) > 0
-                        runner_work_queued = runner_work_queued or maintenance_work_queued
                    else:
                        stats_combined = {"phase1": {}, "phase2": {}}

@@ -343,162 +299,54 @@ def update(
                            batch_size=batch_size,
                        )

-                        print("[*] Phase 2: Processing all database snapshots (most recent first)...")
+                        print("[*] Phase 2: Selecting database snapshots with stale filesystem versions...")
                        stats_combined["phase2"] = process_all_db_snapshots(
                            batch_size=batch_size,
                            resume=resume,
                            wait_for_turn=wait_for_turn,
                        )
                        print_combined_stats(stats_combined)
-                        full_update_empty = (
-                            stats_combined["phase1"].get("processed", 0) == 0 and stats_combined["phase2"].get("snapshots", 0) == 0
-                        )
-                        maintenance_work_queued = any(
-                            (
-                                stats_combined["phase1"].get("queued", 0),
-                                stats_combined["phase2"].get("queued", 0),
-                                stats_combined["phase2"].get("crawls_sealed", 0),
-                            ),
-                        )
-                        runner_work_queued = runner_work_queued or maintenance_work_queued
-
-                    if do_run_until_idle:
-                        # Filesystem migration is maintenance on existing
-                        # Snapshot rows: Snapshot.save() moves archive/<ts> to
-                        # the current output_dir and preserves the lifecycle
-                        # status. Drain those retry_at ticks before queuing
-                        # search backfill below. Otherwise the sealed search
-                        # runner branch correctly sees queued ArchiveResult
-                        # rows first, runs the targeted plugins, and may leave
-                        # the fs_version maintenance tick hidden behind that
-                        # plugin work until another update pass.
-                        if full_update_empty:
-                            print("[*] No snapshots or legacy archive directories found; skipping filesystem maintenance runner.")
-                        elif not maintenance_work_queued:
-                            print("[*] No filesystem maintenance work queued; skipping filesystem maintenance runner.")
-                        else:
-                            print("[*] Phase 3: Running filesystem maintenance until idle...")
-                        if full_update_empty:
-                            pass
-                        elif not maintenance_work_queued:
-                            pass
-                        elif is_filtered_update:
-                            if not touched_snapshot_ids:
-                                print("[*] No matching snapshots queued work for the runner.")
-                            for snapshot_id in sorted(touched_snapshot_ids):
-                                run_scoped_runner("--snapshot-id", snapshot_id)
-                        else:
-                            run_scoped_runner("--maintenance-only", "--maintenance-batch-size", str(batch_size))
-                        ran_post_migrate_runner = True
-
-                if do_index:
-                    if full_update_empty:
-                        print("[*] No snapshots found; skipping search indexing backfill.")
-                    else:
-                        search_plugins = _get_search_indexing_plugins()
-                        if not search_plugins:
-                            print("[*] No search indexing plugins are available, nothing to backfill.")
-                        else:
-                            snapshots = _build_filtered_snapshots_queryset(
-                                filter_patterns=filter_patterns,
-                                filter_type=filter_type,
-                                status=status,
-                                url__icontains=url__icontains,
-                                url__istartswith=url__istartswith,
-                                tag=tag,
-                                crawl_id=crawl_id,
-                                limit=limit,
-                                sort=sort,
-                                search=search,
-                                before=before,
-                                after=after,
-                                resume=resume,
-                            )
-                            from django.db.models import Exists, OuterRef, Q
-                            from django.utils import timezone
-                            from archivebox.core.models import ArchiveResult, Snapshot
-
-                            scoped_snapshot_ids = snapshots.order_by().values("id") if is_filtered_update else None
-                            queued_index_results = ArchiveResult.objects.filter(
-                                status=ArchiveResult.StatusChoices.QUEUED,
-                                plugin__in=search_plugins,
-                            )
-                            if scoped_snapshot_ids is not None:
-                                queued_index_results = queued_index_results.filter(snapshot_id__in=scoped_snapshot_ids)
-
-                            if queued_index_results.exists():
-                                runner_work_queued = True
-                                now = timezone.now()
-                                queued_result_for_snapshot = queued_index_results.filter(snapshot_id=OuterRef("pk"))
-                                snapshots_to_wake = (
-                                    Snapshot.objects.filter(status=Snapshot.StatusChoices.SEALED)
-                                    .annotate(
-                                        has_queued_index_result=Exists(queued_result_for_snapshot),
-                                    )
-                                    .filter(
-                                        has_queued_index_result=True,
-                                    )
-                                    .filter(
-                                        Q(retry_at__isnull=True) | Q(retry_at__gt=now),
-                                    )
-                                )
-                                if scoped_snapshot_ids is not None:
-                                    snapshots_to_wake = snapshots_to_wake.filter(id__in=scoped_snapshot_ids)
-                                woken_count = snapshots_to_wake.update(
-                                    retry_at=now,
-                                    modified_at=now,
-                                )
-                                print(
-                                    "[*] Existing queued search index jobs found; "
-                                    f"skipping backfill scan and waking {woken_count} snapshot(s) for the runner.",
-                                )
-                            else:
-                                collect_index_ids = (
-                                    is_filtered_update
-                                    or foreground_runner_owner(
-                                        data_dir=CONSTANTS.DATA_DIR,
-                                        exclude_id=command.id,
-                                    )
-                                    is not None
-                                )
-                                stats = reindex_snapshots(
-                                    snapshots,
-                                    search_plugins=search_plugins,
-                                    batch_size=batch_size,
-                                    collect_ids=collect_index_ids,
-                                    wait_for_turn=wait_for_turn,
-                                )
-                                print_index_stats(stats)
-                                touched_snapshot_ids.update(stats.get("snapshot_ids", []))
-                                runner_work_queued = runner_work_queued or stats["queued"] > 0
-
-                if do_run_until_idle and (do_index or not ran_post_migrate_runner):
-                    # Search/index backfill intentionally queues targeted
-                    # ArchiveResult rows without reopening sealed snapshots.
-                    # This second runner pass drains those plugin
-                    # rows after filesystem maintenance has had its own turn.
-                    # For a normal unfiltered `archivebox update`, keep the
-                    # historical final pass broad enough to resume genuinely
-                    # queued/interrupted crawl work after maintenance is done.
-                    if full_update_empty:
-                        print("[*] No snapshots found; skipping queued/interrupted crawl runner.")
-                    elif not runner_work_queued:
-                        print("[*] No queued/interrupted crawl work found; skipping queued/interrupted crawl runner.")
-                    else:
-                        print("[*] Phase 3: Running queued/interrupted crawl work until idle...")
-                    if full_update_empty:
-                        pass
-                    elif not runner_work_queued:
-                        pass
-                    elif touched_snapshot_ids and is_filtered_update:
-                        if not touched_snapshot_ids:
-                            print("[*] No matching snapshots queued work for the runner.")
+                    # The due selectors are indexed and cheap when empty, so
+                    # always drain them instead of preceding the runner with
+                    # whole-table counts merely to decide whether to call it.
+                    print("[*] Phase 3: Running filesystem maintenance until idle...")
+                    if is_filtered_update:
                        for snapshot_id in sorted(touched_snapshot_ids):
                            run_scoped_runner("--snapshot-id", snapshot_id)
+                    elif migrate_only:
+                        run_scoped_runner("--maintenance-only")
                    else:
-                        run_scoped_runner(
-                            *(["--maintenance-only", "--maintenance-batch-size", str(batch_size)] if index_only or migrate_only else []),
+                        run_scoped_runner()
+
+                if do_index:
+                    search_plugins = _get_search_indexing_plugins()
+                    if not search_plugins:
+                        print("[*] No search indexing plugins are available, nothing to backfill.")
+                    else:
+                        snapshots = _build_filtered_snapshots_queryset(
+                            filter_patterns=filter_patterns,
+                            filter_type=filter_type,
+                            status=status,
+                            url__icontains=url__icontains,
+                            url__istartswith=url__istartswith,
+                            tag=tag,
+                            crawl_id=crawl_id,
+                            limit=limit,
+                            sort=sort,
+                            search=search,
+                            before=before,
+                            after=after,
+                            resume=resume,
                        )
+                        stats = reindex_snapshots(
+                            snapshots,
+                            search_plugins=search_plugins,
+                            batch_size=batch_size,
+                            collect_ids=is_filtered_update,
+                            wait_for_turn=wait_for_turn,
+                        )
+                        print_index_stats(stats)
+                        touched_snapshot_ids.update(stats.get("snapshot_ids", []))

                if not continuous:
                    break
@@ -561,21 +409,20 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500
    """
    Drain old archive/ directories (0.8.x → 0.9.x migration).

-    Only processes real directories (skips symlinks - those are already migrated).
+    Removes obsolete timestamp symlinks and processes real legacy directories.
    For each old dir found in archive/:
      1. Load or create DB snapshot
      2. Trigger fs migration on save() to move to data/archive/users/{user}/...
-      3. Leave symlink in archive/ pointing to new location
+      3. Remove the old timestamp path after the verified migration commits

-    After this drains, archive/ should only contain symlinks and we can trust
-    1:1 mapping between DB and filesystem.
+    After this drains, current snapshot data exists only under archive/users/.
    """
    from archivebox.core.models import Snapshot
    from archivebox.config import CONSTANTS
    from archivebox.crawls.models import Crawl
    from django.utils import timezone

-    stats = {"processed": 0, "migrated": 0, "queued": 0, "skipped": 0, "invalid": 0}
+    stats = {"processed": 0, "migrated": 0, "queued": 0, "skipped": 0, "invalid": 0, "removed_symlinks": 0}
    crawl_url_lines: dict[str, list[str]] = {}
    crawl_url_sets: dict[str, set[str]] = {}
    dirty_crawl_ids: set[str] = set()
@@ -606,8 +453,23 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500
            if changed:
                Crawl.objects.filter(pk=crawl.pk).update(urls="\n".join(lines), modified_at=timezone.now())

-    # Scan for real directories only (skip symlinks - they're already migrated)
+    # Compatibility timestamp projections are harmful in portable exports and
+    # duplicate the obsolete 0.7.x-looking namespace without containing data.
    all_entries = list(os.scandir(archive_dir))
+    for entry in all_entries:
+        entry_path = Path(entry.path)
+        if entry.is_symlink() and Snapshot.is_legacy_archive_dir(entry_path):
+            try:
+                target_path = entry_path.resolve(strict=True)
+                points_into_current_layout = target_path.is_relative_to(CONSTANTS.USERS_DIR.resolve(strict=True))
+            except OSError:
+                points_into_current_layout = False
+
+            if points_into_current_layout:
+                entry_path.unlink(missing_ok=True)
+                stats["removed_symlinks"] += 1
+
+    # Scan real legacy directories only; these still contain data to migrate.
    entries = [
        (e.stat().st_mtime, e.path)
        for e in all_entries
@@ -723,18 +585,8 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500


 def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, wait_for_turn=None) -> dict[str, int]:
-    """
-    O(n) scan over entire DB from most recent to least recent.
-
-    For each snapshot:
-      1. Reconcile index.json with DB (merge titles, tags, archive results)
-      2. Mark migrated snapshots sealed unless explicitly re-queued elsewhere
-
-    No orphan detection needed - we trust 1:1 mapping between DB and filesystem
-    after Phase 1 has drained all old archive/ directories.
-    """
+    """Queue only snapshots whose indexed filesystem version is stale."""
    from archivebox.core.models import Snapshot
-    from archivebox.crawls.models import Crawl
    from django.db.models import Q
    from django.utils import timezone

@@ -747,128 +599,35 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
        "sealed": 0,
        "crawls_sealed": 0,
    }
-    current_fs_version = Snapshot._fs_current_version()
-
-    queryset = Snapshot.objects.all()
+    queryset = Snapshot.objects.filter(fs_version__in=Snapshot._FS_VERSION_MIGRATION_PATHS)
    if resume:
        queryset = queryset.filter(timestamp__lte=resume)
-    total = queryset.count()
-    stats["snapshots"] = total
-    print(f"[*] Processing {total} snapshots from database (most recent first)...")
-
-    def update_in_batches(rows, *, label: str, **updates) -> int:
-        updated = 0
-        checked = 0
-        while True:
-            if wait_for_turn:
-                wait_for_turn()
-            batch = list(rows.only("id", "modified_at").order_by("-timestamp")[:batch_size])
-            if not batch:
-                if updated:
-                    print(f"    [{label}] complete: {updated} rows updated")
-                return updated
-            checked += len(batch)
-            print(f"    [{label}] updating next {len(batch)} rows (seen {checked})...")
-            for snapshot in batch:
-                # This maintenance scan intentionally bypasses save(); it is
-                # only normalizing scheduler fields, and Snapshot.save() may
-                # do filesystem migration work that belongs in the runner.
-                # Guard each single-row UPDATE with modified_at so stale scan
-                # pages cannot overwrite newer runner/admin writes.
-                updated += int(
-                    snapshot.safe_update(
-                        updates,
-                        refresh=False,
-                        extra_filter={"modified_at": snapshot.modified_at},
-                    ),
-                )
-            print(f"    [{label}] updated {updated} rows so far")
-
-    now = timezone.now()
-    updated_rows = update_in_batches(
-        queryset.exclude(
-            status__in=[
-                Snapshot.StatusChoices.QUEUED,
-                Snapshot.StatusChoices.STARTED,
-                Snapshot.StatusChoices.PAUSED,
-                Snapshot.StatusChoices.SEALED,
-            ],
-        ),
-        label="snapshot status normalization",
-        status=Snapshot.StatusChoices.SEALED,
-        retry_at=None,
-        modified_at=now,
-    )
-    stats["sealed"] += updated_rows
-    stats["updated_db"] += updated_rows
-
-    fs_version_rows = queryset.exclude(fs_version=current_fs_version).filter(Q(retry_at__isnull=True) | Q(retry_at__gt=now))
-    stale_batch = []
-
-    def queue_stale_fs_batch() -> None:
-        if not stale_batch:
-            return
+    initial_now = timezone.now()
+    rows_to_wake = queryset.filter(Q(retry_at__isnull=True) | Q(retry_at__gt=initial_now))
+    after_id = None
+    while True:
        if wait_for_turn:
            wait_for_turn()
+        page = rows_to_wake.filter(id__gt=after_id) if after_id is not None else rows_to_wake
+        batch = list(page.only("id", "fs_version", "modified_at").order_by("id")[:batch_size])
+        if not batch:
+            break
+        after_id = batch[-1].id
        now = timezone.now()
-        # Do not bump fs_version here. The orchestrator calls Snapshot.save(),
-        # which performs the idempotent filesystem migration and commits the new
-        # fs_version in the same serialized worker path as normal crawls.
        updated = 0
-        for snapshot in stale_batch:
-            # Each row gets its own short autocommit UPDATE because this scan
-            # can touch millions of snapshots while a server is also alive.
-            # The modified_at predicate is the CAS guard: if the runner or
-            # admin changed the snapshot after paged_iterator read it, skip it
-            # and let the newer state decide whether migration is still due.
+        for snapshot in batch:
            updated += int(
                snapshot.safe_update(
-                    {
-                        "retry_at": now,
-                        "modified_at": now,
-                    },
+                    {"retry_at": now, "modified_at": now},
                    refresh=False,
                    extra_filter={"fs_version": snapshot.fs_version},
                ),
            )
-        stats["processed"] += len(stale_batch)
+        stats["processed"] += len(batch)
        stats["updated_db"] += updated
        stats["queued"] += updated
-        print(f"    [{stats['processed']}/{total}] Queued {updated} filesystem migrations for orchestrator...")
-        stale_batch.clear()
-
-    for snapshot in (
-        fs_version_rows.only("id", "crawl_id", "timestamp", "fs_version", "modified_at")
-        .order_by("-timestamp")
-        .paged_iterator(chunk_size=batch_size)
-    ):
-        try:
-            stale_batch.append(snapshot)
-            if len(stale_batch) >= batch_size:
-                queue_stale_fs_batch()
-        except KeyboardInterrupt as err:
-            err.archivebox_resume = snapshot.timestamp
-            raise
-    queue_stale_fs_batch()
-
-    now = timezone.now()
-    # Crawls with no open child snapshots are already finished. Seal them here
-    # instead of waking the foreground runner; otherwise migration/update can
-    # accidentally re-enter full crawl execution for historical rows.
-    stats["crawls_sealed"] = (
-        Crawl.objects.filter(
-            status__in=Crawl.RUNNABLE_STATES,
-        )
-        .exclude(
-            snapshot_set__status__in=Snapshot.OPEN_STATES,
-        )
-        .update(
-            status=Crawl.StatusChoices.SEALED,
-            retry_at=None,
-            modified_at=now,
-        )
-    )
-    stats["updated_db"] += stats["crawls_sealed"]
+        print(f"    Queued {stats['queued']} stale filesystem snapshots so far...")
+    stats["snapshots"] = stats["processed"]
    return stats


@@ -928,8 +687,6 @@ def process_filtered_snapshots(
            stats["snapshot_ids"].append(str(snapshot.id))
            update_values = {}
            updated = 0
-            if not isinstance(snapshot.current_step, int):
-                update_values["current_step"] = 0
            if queue_for_archiving:
                update_values.update(
                    {
@@ -1015,10 +772,9 @@ def print_index_stats(stats: dict[str, Any]) -> None:

    print(f"""
 [green]Search Reindex Complete[/green]
-  Scanned rows:      {stats["processed"]}
-  Requested jobs:    {stats.get("requested", stats["queued"])}
-  Queued index jobs: {stats["queued"]}
-  Already queued:    {stats.get("skipped_queued", 0)}
+  Missing index runs: {stats["processed"]}
+  Requested runs:    {stats.get("requested", 0)}
+  Indexed snapshots: {stats.get("reindexed", 0)}
 """)


@@ -162,7 +162,8 @@ def version(
    )

    try:
-        OUTPUT_IS_REMOTE_FS = get_data_locations().DATA_DIR.is_mount or get_data_locations().ARCHIVE_DIR.is_mount
+        data_locations = get_data_locations(config=config)
+        OUTPUT_IS_REMOTE_FS = data_locations.DATA_DIR.is_mount or data_locations.ARCHIVE_DIR.is_mount
    except Exception:
        OUTPUT_IS_REMOTE_FS = False

@@ -249,7 +250,7 @@ def version(
        prnt()
        prnt("[deep_sky_blue3][i] Code locations:[/deep_sky_blue3]")
        try:
-            for name, path in get_code_locations().items():
+            for name, path in get_code_locations(config=config).items():
                if isinstance(name, str) and isinstance(path, dict):
                    prnt(printable_folder_status(name, path), overflow="ignore", crop=False)
        except Exception as e:
@@ -264,15 +265,14 @@ def version(
    seen_failures: set[str] = set()
    seen_rows: set[tuple[str, str, str, str]] = set()

-    from archivebox.plugins.discovery import get_enabled_plugins
+    from archivebox.plugins.discovery import get_enabled_plugins, get_plugin_catalog
    from abx_dl.config import get_required_binary_requests
    from abx_dl.dependencies import resolve_binary_requests
-    from abx_dl.models import discover_plugins, filter_plugins
    from abx_dl.orchestrator import create_bus
    from abxpkg.binary_service import BinaryEvent, BinaryService

-    plugins = discover_plugins(runtime="archivebox")
-    enabled_plugins = filter_plugins(plugins, get_enabled_plugins(config=config), include_providers=True)
+    plugins = get_plugin_catalog()
+    enabled_plugins = plugins.select(get_enabled_plugins(config=config))
    enabled_plugin_names = set(enabled_plugins)
    runtime_config = normalize_runtime_config(config.for_crawl(), json_safe=False)
    derived_config: dict[str, object] = {}
@@ -501,7 +501,7 @@ def version(
        prnt()
        prnt("[deep_sky_blue3][i] Code locations:[/deep_sky_blue3]")
        try:
-            for name, path in get_code_locations().items():
+            for name, path in get_code_locations(config=config).items():
                if isinstance(name, str) and isinstance(path, dict):
                    prnt(printable_folder_status(name, path), overflow="ignore", crop=False)
        except Exception as e:
@@ -511,7 +511,7 @@ def version(
        if os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) or os.access(CONSTANTS.CONFIG_FILE, os.R_OK):
            prnt("[bright_yellow][i] Data locations:[/bright_yellow]")
            try:
-                for name, path in get_data_locations().items():
+                for name, path in get_data_locations(config=config).items():
                    if isinstance(name, str) and isinstance(path, dict):
                        prnt(printable_folder_status(name, path), overflow="ignore", crop=False)
            except Exception as e:
@@ -7,7 +7,7 @@ from collections.abc import Mapping
 from typing import Any

 from archivebox.config.constants import CONSTANTS
-from archivebox.config.configset import CaseConfigParser
+from archivebox.config.configset import CaseConfigParser, decode_config_inputs
 from archivebox.misc.logging import AttrDict


@@ -17,7 +17,7 @@ CONFIG_FILE_HEADER = (
    "# You can add options here manually in INI format, or automatically by running:\n"
    "#    archivebox config --set KEY=VALUE\n"
    "#\n"
-    "# This file is kept in sync 1:1 with Machine.config in the index DB —\n"
+    "# This file is kept in sync 1:1 with the current Machine.config in the DB —\n"
    "# editing either side propagates to the other. ``archivebox init`` reads\n"
    "# this file on startup; the admin Machine.config editor writes both.\n"
    "#\n"
@@ -160,7 +160,7 @@ def _coerce_from_str_dict(file_config: dict[str, str]) -> dict[str, Any]:
    so they round-trip through INI's string-only storage. When reading the
    file back into ``Machine.config`` (a JSONField that holds native types)
    those strings have to be decoded — otherwise downstream consumers like
-    ``_emit_machine_config`` → ``MachineEvent`` → abx-dl see a JSON string
+    ``MachineEvent`` → abx-dl see a JSON string
    where they expect a dict and raise ``TypeError``.
    Declared fields go through pydantic-settings' own ``field_is_complex`` /
    ``prepare_field_value`` so they're decoded per annotation. Undeclared
@@ -169,31 +169,8 @@ def _coerce_from_str_dict(file_config: dict[str, str]) -> dict[str, Any]:
    shape ``_coerce_to_str_dict`` writes them as.
    """
    from archivebox.config.common import ArchiveBoxConfig
-    from archivebox.config.configset import IniConfigSettingsSource

-    decoder = IniConfigSettingsSource(ArchiveBoxConfig)
-    decoded: dict[str, Any] = dict(file_config)
-    declared_fields = set(ArchiveBoxConfig.model_fields)
-    for field_name, field in ArchiveBoxConfig.model_fields.items():
-        if field_name not in decoded:
-            continue
-        raw = decoded[field_name]
-        if not isinstance(raw, str) or not raw:
-            continue
-        if decoder.field_is_complex(field):
-            decoded[field_name] = decoder.prepare_field_value(field_name, field, raw, True)
-    for key, raw in list(decoded.items()):
-        if key in declared_fields:
-            continue
-        if not isinstance(raw, str) or not raw:
-            continue
-        first = raw[:1]
-        if first not in ("{", "["):
-            continue
-        try:
-            decoded[key] = json.loads(raw)
-        except (TypeError, ValueError):
-            continue
+    decoded = decode_config_inputs(ArchiveBoxConfig, file_config, decode_unknown_json=True)
    if not str(decoded.get("BASE_URL") or "").strip():
        from archivebox.config.common import base_url_from_legacy_server_config

@@ -16,13 +16,13 @@ from pathlib import Path
 from typing import Any, ClassVar, cast
 from urllib.parse import quote, urlparse

-from abx_plugins.plugins.base.utils import BASE_CONFIG_PATH, build_config_model, resolve_plugin_configs
+from abx_plugins.plugins.base.utils import build_config_model
 from django.db import DatabaseError
-from pydantic import BaseModel, Field, PrivateAttr, create_model, field_validator, model_validator
-from pydantic_settings import SettingsConfigDict
+from pydantic import BaseModel, Field, PrivateAttr, TypeAdapter, create_model, field_validator, model_validator
+from pydantic_settings import BaseSettings, SettingsConfigDict
 from rich.console import Console

-from archivebox.config.configset import COMPUTED_CONFIG_KEYS, BaseConfigSet, IniConfigSettingsSource
+from archivebox.config.configset import COMPUTED_CONFIG_KEYS, BaseConfigSet, decode_config_inputs

 from .constants import CONSTANTS
 from .ldap import LDAPConfig
@@ -406,7 +406,11 @@ class DatabaseConfig(BaseConfigSet):
    DATABASE_USER: str = Field(default="archivebox", alias="ARCHIVEBOX_DATABASE_USER")
    DATABASE_PASSWORD: str = Field(default="", alias="ARCHIVEBOX_DATABASE_PASSWORD")
    SQLITE_JOURNAL_MODE: str = Field(
-        default="WAL",
+        # Docker collections commonly live on a host bind mount. WAL's -shm
+        # locking is only safe when every SQLite process is on the same host;
+        # Docker Desktop/OrbStack place the container and host in different
+        # locking domains and a host-side reader can corrupt the live DB.
+        default="DELETE" if IN_DOCKER else "WAL",
        alias="ARCHIVEBOX_SQLITE_JOURNAL_MODE",
        pattern=r"(?i)^(DELETE|TRUNCATE|PERSIST|MEMORY|WAL|OFF)$",
    )
@@ -419,6 +423,15 @@ class DatabaseConfig(BaseConfigSet):
    SQLITE_LOCK_RETRY_TIMEOUT: float = Field(default=60.0, alias="ARCHIVEBOX_SQLITE_LOCK_RETRY_TIMEOUT", ge=0)
    SQLITE_LOCK_RETRY_INTERVAL: float = Field(default=5.0, alias="ARCHIVEBOX_SQLITE_LOCK_RETRY_INTERVAL", gt=0)

+    @model_validator(mode="after")
+    def reject_docker_sqlite_wal(self):
+        if IN_DOCKER and self.DATABASE_ENGINE.lower() == "sqlite" and self.SQLITE_JOURNAL_MODE.upper() == "WAL":
+            raise ValueError(
+                "SQLITE_JOURNAL_MODE=WAL is unsafe for Docker collections because host bind mounts cross SQLite "
+                "locking domains; use DELETE (the Docker default) or PostgreSQL",
+            )
+        return self
+

 class ArchivingConfig(BaseConfigSet):
    toml_section_header: str = "ARCHIVING_CONFIG"
@@ -428,6 +441,7 @@ class ArchivingConfig(BaseConfigSet):
        default="",
        description="Comma-separated plugin selection for this run. Empty means use enabled plugin defaults.",
    )
+    PARSER: str = Field(default="auto", description="Parser used to read the submitted Crawl input document.")

    ONLY_NEW: bool = Field(default=True)
    INDEX_ONLY: bool = Field(default=False)
@@ -447,7 +461,8 @@ class ArchivingConfig(BaseConfigSet):
    USER_AGENT: str = Field(
        default=f"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)",
    )
-    COOKIES_FILE: Path | None = Field(default=None)
+    COOKIES_FILE: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
+    AUTH_STORAGE_FILE: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})

    URL_DENYLIST: str = Field(default=r"\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$", alias="URL_BLACKLIST")
    URL_ALLOWLIST: str | None = Field(default=None, alias="URL_WHITELIST")
@@ -588,13 +603,9 @@ def _explicit_plugin_enabled_keys(config: Mapping[str, object]) -> set[str]:


 def _discover_plugin_config_schemas() -> PluginSchemaDocuments:
-    from archivebox.plugins.discovery import discover_plugin_configs
+    from archivebox.plugins.discovery import get_plugin_config_resolver

-    schemas: PluginSchemaDocuments = {}
-    if BASE_CONFIG_PATH.exists():
-        schemas["base"] = json.loads(BASE_CONFIG_PATH.read_text())
-    schemas.update(discover_plugin_configs())
-    return schemas
+    return get_plugin_config_resolver().schemas


 def _plugin_config_properties(plugin_schemas: PluginSchemaDocuments) -> dict[str, dict[str, Any]]:
@@ -843,13 +854,11 @@ class ArchiveBoxBaseConfig(
            disabled_plugins = [plugin_name for plugin_name, enabled_key in enabled_config_keys.items() if not getattr(self, enabled_key)]
        selected_plugin_roots = plugin_names
        if selected_plugin_roots:
-            from abx_dl.models import discover_plugins, filter_plugins
+            from archivebox.plugins.discovery import get_plugin_catalog

            selected_plugins = set(
-                filter_plugins(
-                    discover_plugins(runtime="archivebox"),
+                get_plugin_catalog().select(
                    sorted(selected_plugin_roots),
-                    include_providers=True,
                    disabled_names=disabled_plugins,
                ),
            )
@@ -885,6 +894,12 @@ PLUGIN_CONFIG_SCHEMAS = _discover_plugin_config_schemas()
 ArchiveBoxConfig = _build_archivebox_config_model(PLUGIN_CONFIG_SCHEMAS)


+class ArchiveBoxSourceSettings(ArchiveBoxConfig, BaseSettings):
+    """Environment and INI source loader for the pure runtime config model."""
+
+    model_config = SettingsConfigDict(**{**ArchiveBoxConfig.model_config, "extra": "ignore"})
+
+
 def _normalize_plugins_config_value(value: Any) -> set[str]:
    if value is None:
        return set()
@@ -1041,7 +1056,9 @@ def get_request_config(request: Any, *, resolve_plugins: bool = False) -> Archiv
    request_config = request_state.get("archivebox_config")
    request_config_resolves_plugins = bool(request_state.get("_archivebox_config_resolves_plugins", False))
    if request_config is None or (resolve_plugins and not request_config_resolves_plugins):
-        request_config = get_config(resolve_plugins=resolve_plugins)
+        from django.conf import settings
+
+        request_config = get_config(base_config=settings.CONFIG, resolve_plugins=resolve_plugins)
        request._archivebox_config_resolves_plugins = resolve_plugins
    if request_config.SERVER_SECURITY_MODE == "auto":
        request_host = (urlparse(f"//{request.get_host()}").hostname or "").lower().rstrip(".")
@@ -1096,9 +1113,11 @@ def get_config(
    config_data: ConfigPayload = dict(defaults or {})
    config_data["PERSONAS_DIR"] = str(CONSTANTS.PERSONAS_DIR)
    base_config_payload: ConfigPayload = {}
-    base_config_model = ArchiveBoxConfig()
+    file_config: dict[str, Any] | None = None
+    base_config_model = ArchiveBoxSourceSettings() if base_config is None else None

    if crawl_config_base:
+        assert base_config_model is not None
        config_data.update(
            normalize_runtime_config(base_config_model.model_dump(mode="json"), exclude_runtime_derived=True, json_safe=False),
        )
@@ -1110,10 +1129,12 @@ def get_config(
            base_config_payload.update(dict(base_config))
        config_data.update(normalize_runtime_config(base_config_payload, exclude_runtime_derived=True, json_safe=False))
    else:
+        assert base_config_model is not None
        config_data.update(
            normalize_runtime_config(base_config_model.model_dump(mode="json"), exclude_runtime_derived=True, json_safe=False),
        )
-        legacy_config = {**BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE), **os.environ}
+        file_config = BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE)
+        legacy_config = {**file_config, **os.environ}
        legacy_permissions = permissions_from_legacy_public_flags(legacy_config)
        if legacy_permissions:
            config_data["PERMISSIONS"] = legacy_permissions
@@ -1170,7 +1191,6 @@ def get_config(

    explicit_plugin_enabled_keys: set[str] = set()
    if resolve_plugins:
-        plugin_schemas = {plugin_name: schema for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items() if isinstance(schema, dict)}
        plugin_global_config = {key: str(value) if isinstance(value, Path) else value for key, value in config_data.items()}
        crawl_selected_plugins = crawl_config_base and bool(_normalize_plugins_config_value(dict(crawl.config or {}).get("PLUGINS")))
        # A frozen crawl-level PLUGINS selector is the exact extractor set for
@@ -1187,14 +1207,16 @@ def get_config(
            },
        )
        if not crawl_config_base:
-            file_config = BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE)
+            if file_config is None:
+                file_config = BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE)
            explicit_plugin_enabled_keys.update(_explicit_plugin_enabled_keys(file_config))
            plugin_user_config = {
                **_plugin_user_config(_plugin_input_config(file_config)),
                **plugin_user_config,
            }
-        plugin_sections = resolve_plugin_configs(
-            plugin_schemas,
+        from archivebox.plugins.discovery import get_plugin_config_resolver
+
+        plugin_sections = get_plugin_config_resolver().resolve(
            global_config=plugin_global_config,
            user_config=plugin_user_config,
            environ={},
@@ -1223,28 +1245,7 @@ def get_config(
            for enabled_key in _plugin_enabled_config_keys().values():
                config_data.pop(enabled_key, None)

-    # Decode JSON-encoded complex values (dict/list fields) that came from
-    # string-only sources before validation. ``IniConfigSettingsSource`` does
-    # this for the ArchiveBox.conf path, but Machine.config (mirrored from the
-    # INI via ``_coerce_to_str_dict``) and plugin/env scope overrides bypass
-    # pydantic-settings sources entirely — they feed JSON strings directly
-    # into ``model_validate``, which rejects ``"{...}"`` for a ``dict[str, str]``
-    # field. Run pydantic-settings' own complex-value decoder here so every
-    # source converges on the same shape before validation.
-    _complex_decoder = IniConfigSettingsSource(ArchiveBoxConfig)
-    for _field_name, _field in ArchiveBoxConfig.model_fields.items():
-        if _field_name not in config_data:
-            continue
-        _raw = config_data[_field_name]
-        if not isinstance(_raw, str) or not _raw:
-            continue
-        if _complex_decoder.field_is_complex(_field):
-            config_data[_field_name] = _complex_decoder.prepare_field_value(
-                _field_name,
-                _field,
-                _raw,
-                True,
-            )
+    config_data = decode_config_inputs(ArchiveBoxConfig, config_data)

    config = ArchiveBoxConfig.model_validate(config_data)
    if config.SERVER_SECURITY_MODE == "auto":
@@ -1253,7 +1254,7 @@ def get_config(
            config = config.model_copy(update={"SERVER_SECURITY_MODE": "safe-subdomains-fullreplay"})
    for key in explicit_plugin_enabled_keys:
        if key in config_data:
-            setattr(config, key, config_data[key])
+            setattr(config, key, TypeAdapter(bool).validate_python(config_data[key]))
    if config.PLUGINS:
        config._derive_plugin_enabled_config(respect_current_enabled=True)
    if redact_sensitive:
@@ -1261,7 +1262,6 @@ def get_config(
            value = config[key]
            if is_sensitive_config_key(key) and value not in (None, ""):
                setattr(config, key, SENSITIVE_CONFIG_VALUE_REDACTED)
-    os.environ["ABXPKG_LIB_DIR"] = str(config.ABXPKG_LIB_DIR)
    archiving_warning_key = (config.TIMEOUT, config.USE_COLOR)
    if archiving_warning_key not in _WARNED_ARCHIVING_CONFIGS:
        config.warn_if_invalid()
@@ -1271,13 +1271,14 @@ def get_config(

 def get_all_configs() -> dict[str, BaseConfigSet]:
    """Get all config section objects as a dictionary."""
+    resolved = ArchiveBoxSourceSettings().model_dump(mode="json")
    return {
-        "SHELL_CONFIG": ShellConfig(),
-        "STORAGE_CONFIG": StorageConfig(),
-        "GENERAL_CONFIG": GeneralConfig(),
-        "SERVER_CONFIG": ServerConfig(),
-        "DATABASE_CONFIG": DatabaseConfig(),
-        "ARCHIVING_CONFIG": ArchivingConfig(),
-        "SEARCH_BACKEND_CONFIG": SearchBackendConfig(),
-        "LDAP_CONFIG": LDAPConfig(),
+        "SHELL_CONFIG": ShellConfig.model_validate(resolved),
+        "STORAGE_CONFIG": StorageConfig.model_validate(resolved),
+        "GENERAL_CONFIG": GeneralConfig.model_validate(resolved),
+        "SERVER_CONFIG": ServerConfig.model_validate(resolved),
+        "DATABASE_CONFIG": DatabaseConfig.model_validate(resolved),
+        "ARCHIVING_CONFIG": ArchivingConfig.model_validate(resolved),
+        "SEARCH_BACKEND_CONFIG": SearchBackendConfig.model_validate(resolved),
+        "LDAP_CONFIG": LDAPConfig.model_validate(resolved),
    }
@@ -2,10 +2,13 @@

 __package__ = "archivebox.config"

+import json
+from collections.abc import Mapping
+from configparser import ConfigParser
 from pathlib import Path
 from typing import Any, ClassVar
-from configparser import ConfigParser

+from pydantic import BaseModel
 from pydantic_settings import BaseSettings, PydanticBaseSettingsSource, SettingsConfigDict

 COMPUTED_CONFIG_KEYS = (
@@ -30,31 +33,17 @@ class CaseConfigParser(ConfigParser):
        return optionstr


-# ``IniConfigSettingsSource.get_field_value`` is called once per pydantic field
-# (hundreds of fields across the GlobalConfig + plugin configs). Without
-# caching, every field re-opens and re-parses ``ArchiveBox.conf`` from disk —
-# that's the dominant cost of ``get_config()`` (each call was ~130ms; profile
-# showed 632 ``parser.read(config_path)`` invocations per call). The cache is
-# keyed on (path, mtime) so external edits to the file still get picked up.
-_INI_CACHE: dict[tuple[str, float], dict[str, Any]] = {}
-
-
-def _read_ini_config_cached(config_path_str: str) -> dict[str, Any]:
-    config_path = Path(config_path_str)
+def read_ini_config(config_path: str | Path) -> dict[str, Any]:
+    """Read and flatten an ArchiveBox INI config file."""
+    config_path = Path(config_path)
    try:
-        mtime = config_path.stat().st_mtime
-    except (FileNotFoundError, PermissionError):
+        if not config_path.exists():
+            return {}
+        parser = CaseConfigParser(interpolation=None)
+        parser.read(config_path)
+    except OSError:
        return {}
-    cache_key = (str(config_path), mtime)
-    cached = _INI_CACHE.get(cache_key)
-    if cached is not None:
-        return cached
-    parser = CaseConfigParser()
-    parser.read(config_path)
-    flat = {key.upper(): value for section in parser.sections() for key, value in parser.items(section)}
-    _INI_CACHE.clear()
-    _INI_CACHE[cache_key] = flat
-    return flat
+    return {key.upper(): value for section in parser.sections() for key, value in parser.items(section)}


 class IniConfigSettingsSource(PydanticBaseSettingsSource):
@@ -75,18 +64,7 @@ class IniConfigSettingsSource(PydanticBaseSettingsSource):
        return field_value, field_name, value_is_complex

    def __call__(self) -> dict[str, Any]:
-        # Use the per-field path (``get_field_value`` + ``prepare_field_value``)
-        # so complex types get JSON-decoded. The previous flat-dict return
-        # skipped pydantic-settings' complex-value handling entirely.
-        result: dict[str, Any] = {}
-        for field_name, field in self.settings_cls.model_fields.items():
-            value, key, value_is_complex = self.get_field_value(field, field_name)
-            if value is None:
-                continue
-            prepared = self.prepare_field_value(field_name, field, value, value_is_complex)
-            if prepared is not None:
-                result[key] = prepared
-        return result
+        return decode_config_inputs(self.settings_cls, self._load_config_file())

    def _load_config_file(self) -> dict[str, Any]:
        try:
@@ -96,14 +74,42 @@ class IniConfigSettingsSource(PydanticBaseSettingsSource):
        except ImportError:
            return {}

-        return _read_ini_config_cached(str(config_path))
+        return read_ini_config(config_path)


-class BaseConfigSet(BaseSettings):
+def decode_config_inputs(
+    settings_cls: type[BaseSettings],
+    config: Mapping[str, Any],
+    *,
+    decode_unknown_json: bool = False,
+) -> dict[str, Any]:
+    """Decode string-only config values once at the boundary to typed config."""
+    decoder = IniConfigSettingsSource(settings_cls)
+    decoded: dict[str, Any] = dict(config)
+    for source_key, raw in list(decoded.items()):
+        if not isinstance(raw, str) or not raw:
+            continue
+        field_name = source_key if source_key in settings_cls.model_fields else source_key.upper()
+        field = settings_cls.model_fields.get(field_name)
+        if field is None:
+            if decode_unknown_json and raw[:1] in ("{", "["):
+                try:
+                    decoded[source_key] = json.loads(raw)
+                except (TypeError, ValueError):
+                    pass
+            continue
+        if decoder.field_is_complex(field):
+            decoded[field_name] = decoder.prepare_field_value(field_name, field, raw, True)
+            if source_key != field_name:
+                decoded.pop(source_key, None)
+    return decoded
+
+
+class BaseConfigSet(BaseModel):
    """
-    Base class for config sections.
+    Pure typed runtime model for config sections.

-    Automatically loads values from (highest to lowest priority):
+    Source-loading subclasses combine this model with ``BaseSettings`` and load:
    1. Environment variables
    2. ArchiveBox.conf file (INI format, flattened)
    3. Default values
@@ -146,14 +152,7 @@ class BaseConfigSet(BaseSettings):
    @classmethod
    def load_from_file(cls, config_path: Path) -> dict[str, str]:
        """Load config values from INI file."""
-        if not config_path.exists():
-            return {}
-
-        parser = CaseConfigParser()
-        parser.read(config_path)
-
-        # Flatten all sections into single namespace
-        return {key.upper(): value for section in parser.sections() for key, value in parser.items(section)}
+        return read_ini_config(config_path)

    def __getitem__(self, key: str) -> Any:
        if key in type(self).model_fields:
@@ -15,6 +15,7 @@ from .common import get_config
 from .constants import CONSTANTS

 CONFIG = get_config()
+os.environ.setdefault("ABXPKG_LIB_DIR", str(CONFIG.ABXPKG_LIB_DIR))

 if not CONFIG.USE_COLOR:
    os.environ["NO_COLOR"] = "1"
@@ -222,7 +222,7 @@ def get_or_create_working_tmp_dir(autofix=True, quiet=True, config: "ArchiveBoxC
            create_and_chown_dir(candidate)
        except Exception:
            pass
-        if check_tmp_dir(candidate, throw=False, quiet=True, must_exist=True):
+        if check_tmp_dir(candidate, throw=False, quiet=True, must_exist=True, config=config):
            if autofix and config.TMP_DIR != candidate:
                os.environ["TMP_DIR"] = str(candidate)
            return candidate
@@ -263,7 +263,7 @@ def get_or_create_working_lib_dir(autofix=True, quiet=False, config: "ArchiveBox
            create_and_chown_dir(candidate)
        except Exception:
            pass
-        if check_lib_dir(candidate, throw=False, quiet=True, must_exist=True):
+        if check_lib_dir(candidate, throw=False, quiet=True, must_exist=True, config=config):
            if autofix and config.ABXPKG_LIB_DIR != candidate:
                os.environ["ABXPKG_LIB_DIR"] = str(candidate)
            return candidate
@@ -272,10 +272,10 @@ def get_or_create_working_lib_dir(autofix=True, quiet=False, config: "ArchiveBox
        raise OSError(f"ArchiveBox is unable to find a writable ABXPKG_LIB_DIR, tried {CANDIDATES}!")


-def _sql_index_location() -> dict:
+def _sql_index_location(config: "ArchiveBoxConfig") -> dict:
    from archivebox.misc.db import database_display_location, database_exists, is_postgres

-    if is_postgres():
+    if is_postgres(config):
        return {
            "path": database_display_location(),
            "enabled": True,
@@ -316,7 +316,7 @@ def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
                and os.access(CONSTANTS.CONFIG_FILE, os.R_OK)
                and os.access(CONSTANTS.CONFIG_FILE, os.W_OK),
            },
-            "SQL_INDEX": _sql_index_location(),
+            "SQL_INDEX": _sql_index_location(config),
            "ARCHIVE_DIR": {
                "path": CONSTANTS.ARCHIVE_DIR.resolve(),
                "enabled": True,
@@ -10,9 +10,12 @@ from pathlib import Path
 from urllib.parse import quote

 from django.contrib import admin
-from django.core.exceptions import ValidationError
-from django.db.models import Min, Prefetch, Q, TextField
+from django.contrib.admin.actions import delete_selected
+from django.contrib.admin.helpers import ACTION_CHECKBOX_NAME
+from django.core.exceptions import PermissionDenied, SuspiciousOperation, ValidationError
+from django.db.models import Count, Min, Prefetch, Q, Subquery, TextField, Window
 from django.db.models.functions import Cast
+from django.shortcuts import redirect
 from django.urls import resolve, reverse
 from django.utils import timezone
 from django.utils.html import format_html
@@ -24,6 +27,7 @@ from archivebox.core.models import ArchiveResult, Snapshot
 from archivebox.core.routes_util import build_snapshot_url
 from archivebox.core.widgets import InlineTagEditorWidget
 from archivebox.machine.env_util import env_to_shell_exports
+from archivebox.misc.logging_util import printable_filesize
 from archivebox.misc.paginators import AcceleratedPaginator
 from archivebox.plugins.discovery import get_plugin_icon
 from archivebox.plugins.views import LIVE_PLUGIN_BASE_URL
@@ -79,24 +83,22 @@ def get_process_link_label(process) -> str:
    return str(process.id)[-8:]


-def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
+def render_archiveresults_list(archiveresults_qs, limit=50, config=None, can_delete=False):
    """Render a nice inline list view of archive results with status, plugin, output, and actions."""

-    result_ids = list(archiveresults_qs.order_by("plugin").values_list("pk", flat=True)[:limit])
-    if not result_ids:
-        return mark_safe('<div style="color: #64748b; font-style: italic; padding: 16px 0;">No Archive Results yet...</div>')
-
-    results_by_id = {
-        result.pk: result
-        for result in ArchiveResult.objects.filter(pk__in=result_ids).select_related(
+    results = list(
+        ArchiveResult.objects.filter(pk__in=Subquery(archiveresults_qs.order_by().values("pk")))
+        .order_by("plugin")
+        .annotate(_inline_total_count=Window(expression=Count("pk")))
+        .select_related(
            "snapshot",
            "snapshot__crawl",
            "snapshot__crawl__created_by",
            "process",
            "process__machine",
        )
-    }
-    results = [results_by_id[result_id] for result_id in result_ids if result_id in results_by_id]
+        .all()[:limit],
+    )

    if not results:
        return mark_safe('<div style="color: #64748b; font-style: italic; padding: 16px 0;">No Archive Results yet...</div>')
@@ -114,6 +116,7 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
    }

    rows = []
+    delete_url = html.escape(reverse("admin:core_archiveresult_changelist"), quote=True)
    for idx, result in enumerate(results):
        status = result.status or "queued"
        color, bg = status_colors.get(status, ("#6b7280", "#f3f4f6"))
@@ -128,6 +131,8 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
                output_file_count = 0
        else:
            output_file_count = 0
+        output_size = int(result.output_size or 0)
+        output_size_display = html.escape(printable_filesize(output_size))

        # Get plugin icon
        icon = get_plugin_icon(result.plugin)
@@ -192,9 +197,15 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):

        # Unique ID for this row's expandable output
        row_id = f"output_{idx}_{str(result.id)[:8]}"
+        delete_button = ""
+        if can_delete:
+            delete_button = f'''
+                <button type="button" data-archive-result-ids="{result.id}" data-delete-url="{delete_url}"
+                        title="Delete this output">×</button>
+            '''

        rows.append(f'''
-            <tr style="border-bottom: 1px solid #f1f5f9; transition: background 0.15s;" onmouseover="this.style.background='#f8fafc'" onmouseout="this.style.background='transparent'">
+            <tr data-output-size="{output_size}" style="border-bottom: 1px solid #f1f5f9; transition: background 0.15s;" onmouseover="this.style.background='#f8fafc'" onmouseout="this.style.background='transparent'">
                <td style="padding: 10px 12px; white-space: nowrap;">
                    <a href="{reverse("admin:core_archiveresult_change", args=[result.id])}"
                       style="color: #2563eb; text-decoration: none; font-family: ui-monospace, monospace; font-size: 11px;"
@@ -230,6 +241,9 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
                <td class="archive-results-files" style="padding: 10px 12px; color: #64748b; font-size: 12px; text-align: right;">
                    {output_file_count}
                </td>
+                <td class="archive-results-size" style="padding: 10px 12px; color: #64748b; font-size: 12px; text-align: right; white-space: nowrap;">
+                    {output_size_display}
+                </td>
                <td class="archive-results-completed" style="padding: 10px 12px; color: #64748b; font-size: 12px;">
                    {end_time}
                </td>
@@ -242,19 +256,18 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
                <td style="padding: 10px 12px; white-space: nowrap; font-family: ui-monospace, monospace; font-size: 11px; color: #64748b;">
                    {version}
                </td>
-                <td style="padding: 10px 8px; white-space: nowrap;">
-                    <div style="display: flex; gap: 4px;">
+                <td class="archive-results-actions-cell">
+                    <div class="archive-results-actions">
                        <a href="{output_link_attr}" target="_blank"
-                           style="padding: 4px 8px; background: #f1f5f9; border-radius: 4px; color: #475569; text-decoration: none; font-size: 11px;"
                           title="View output">📄</a>
                        <a href="{reverse("admin:core_archiveresult_change", args=[result.id])}"
-                           style="padding: 4px 8px; background: #f1f5f9; border-radius: 4px; color: #475569; text-decoration: none; font-size: 11px;"
                           title="Edit">✏️</a>
+                        {delete_button}
                    </div>
                </td>
            </tr>
            <tr style="border-bottom: 1px solid #e2e8f0;">
-                <td colspan="11" style="padding: 0 12px 10px 12px;">
+                <td colspan="12" style="padding: 0 12px 10px 12px;">
                    <details id="{row_id}" style="margin: 0;">
                        <summary style="cursor: pointer; font-size: 11px; color: #94a3b8; user-select: none;">
                            Details &amp; Output
@@ -287,12 +300,12 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
            </tr>
        ''')

-    total_count = archiveresults_qs.count()
+    total_count = results[0]._inline_total_count
    footer = ""
    if total_count > limit:
        footer = f"""
-            <tr>
-                <td colspan="11" style="padding: 12px; text-align: center; color: #64748b; font-size: 13px; background: #f8fafc;">
+            <tr data-output-footer>
+                <td colspan="12" style="padding: 12px; text-align: center; color: #64748b; font-size: 13px; background: #f8fafc;">
                    Showing {limit} of {total_count} results &nbsp;
                    <a href="/admin/core/archiveresult/?snapshot__id__exact={results[0].snapshot_id if results else ""}"
                       style="color: #2563eb;">View all →</a>
@@ -311,11 +324,14 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
                        <th class="archive-results-plugin" style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Plugin</th>
                        <th class="archive-results-output" style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Output</th>
                        <th class="archive-results-files" style="padding: 10px 12px; text-align: right; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Files</th>
+                        <th class="archive-results-size" style="padding: 10px 12px; text-align: right; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">
+                            <button type="button" data-output-size-sort style="all: unset; cursor: pointer;">Size ↕</button>
+                        </th>
                        <th class="archive-results-completed" style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Completed</th>
                        <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Process</th>
                        <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Machine</th>
                        <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Version</th>
-                        <th style="padding: 10px 8px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Actions</th>
+                        <th class="archive-results-actions-cell" style="text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Actions</th>
                    </tr>
                </thead>
                <tbody>
@@ -328,7 +344,7 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):


 class ArchiveResultInline(admin.TabularInline):
-    name = "Archive Results Log"
+    name = "Archive Results"
    model = ArchiveResult
    parent_model = Snapshot
    extra = 0
@@ -530,10 +546,44 @@ class ArchiveResultAdmin(BaseModelAdmin):

    def changelist_view(self, request, extra_context=None):
        self.request = request
+        selected = request.GET.getlist(ACTION_CHECKBOX_NAME)
+        if request.method == "GET" and request.GET.get("action") == "delete_selected" and selected:
+            if not request.user.is_superuser:
+                raise PermissionDenied
+            if len(selected) > 100:
+                raise SuspiciousOperation("Too many ArchiveResults selected for deletion")
+            try:
+                queryset = self.get_queryset(request).filter(pk__in=selected)
+                if not queryset.exists():
+                    snapshot = Snapshot.objects.only("id").filter(pk=request.GET.get("snapshot")).first()
+                    return redirect(build_snapshot_url(str(snapshot.id), "index.html", request=request) if snapshot else request.path)
+            except (ValidationError, ValueError):
+                return redirect(request.path)
+            return delete_selected(self, request, queryset)
+        handoff_snapshot = (
+            request.GET.get("snapshot") if request.method == "POST" and request.GET.get("action") == "delete_selected" else None
+        )
+        if handoff_snapshot:
+            request.GET = request.GET.copy()
+            request.GET.clear()
+            request.META["QUERY_STRING"] = ""
+            try:
+                handoff_snapshot = Snapshot.objects.only("id").filter(pk=handoff_snapshot).first()
+            except (ValidationError, ValueError):
+                handoff_snapshot = None
        saved_list_per_page = self.list_per_page
        self.list_per_page = request.archivebox_config.SNAPSHOTS_PER_PAGE
        try:
-            return super().changelist_view(request, extra_context)
+            response = super().changelist_view(request, extra_context)
+            if (
+                handoff_snapshot
+                and response.status_code in (301, 302)
+                and not ArchiveResult.objects.filter(
+                    pk__in=request.POST.getlist(ACTION_CHECKBOX_NAME),
+                ).exists()
+            ):
+                return redirect(build_snapshot_url(str(handoff_snapshot.id), "index.html", request=request))
+            return response
        finally:
            self.list_per_page = saved_list_per_page

@@ -554,10 +604,6 @@ class ArchiveResultAdmin(BaseModelAdmin):
        qs = (
            super()
            .get_queryset(request)
-            .defer(
-                "notes",
-                "output_json",
-            )
            .prefetch_related(
                Prefetch(
                    "snapshot",
@@ -566,6 +612,10 @@ class ArchiveResultAdmin(BaseModelAdmin):
                "process__machine",
            )
        )
+        if request.resolver_match.url_name == "core_archiveresult_change":
+            qs = qs.defer("notes")
+        else:
+            qs = qs.defer("notes", "output_json")
        if "tags_inline" in ordering_fields:
            qs = qs.annotate(snapshot_first_tag=Min("snapshot__tags__name"))
        return qs
@@ -12,11 +12,13 @@ from admin_data_views.admin import (
    get_app_list as adv_get_app_list,
 )
 from django.contrib import admin
-from django.contrib.auth import REDIRECT_FIELD_NAME
+from django.contrib.auth import REDIRECT_FIELD_NAME, get_user_model, login as auth_login
 from django.contrib.auth.decorators import login_not_required
+from django.contrib.auth.forms import UserCreationForm
 from django.contrib.auth.views import LoginView
-from django.db import DatabaseError, connection
+from django.db import DatabaseError, connection, transaction
 from django.http import HttpResponseRedirect
+from django.template.response import TemplateResponse
 from django.urls import reverse
 from django.utils.decorators import method_decorator
 from django.utils.translation import gettext as _
@@ -103,12 +105,15 @@ class ArchiveBoxAdmin(admin.AdminSite):

        from django.contrib.admin.forms import AdminAuthenticationForm

+        User = get_user_model()
+        first_admin_setup = not User.objects.filter(is_superuser=True).exclude(username="system").exists()
        context = {
            **self.each_context(request),
-            "title": _("Log in"),
+            "title": _("Set up ArchiveBox") if first_admin_setup else _("Log in"),
            "subtitle": None,
            "app_path": request.get_full_path(),
            "username": request.user.get_username(),
+            "first_admin_setup": first_admin_setup,
        }
        if REDIRECT_FIELD_NAME not in request.GET and REDIRECT_FIELD_NAME not in request.POST:
            context[REDIRECT_FIELD_NAME] = reverse("admin:index", current_app=self.name)
@@ -116,6 +121,30 @@ class ArchiveBoxAdmin(admin.AdminSite):

        index_path = reverse("admin:index", current_app=self.name)
        request.current_app = self.name
+        if first_admin_setup:
+            form = UserCreationForm(request.POST or None)
+            if request.method == "POST" and form.is_valid():
+                with transaction.atomic():
+                    # Fresh collections contain the system superuser, which also
+                    # provides a row lock while the first real admin is created.
+                    list(User.objects.select_for_update().filter(is_superuser=True).values_list("pk", flat=True))
+                    if User.objects.filter(is_superuser=True).exclude(username="system").exists():
+                        form.add_error(None, _("An admin account was already created. Log in instead."))
+                    else:
+                        user = form.save(commit=False)
+                        user.is_staff = True
+                        user.is_superuser = True
+                        user.save()
+                        auth_login(request, user, backend="django.contrib.auth.backends.ModelBackend")
+                        return HttpResponseRedirect(index_path)
+
+            context["form"] = form
+            return TemplateResponse(
+                request,
+                self.login_template or "admin/login.html",
+                context,
+            )
+
        return ArchiveBoxLoginView.as_view(
            extra_context=context,
            authentication_form=self.login_form or AdminAuthenticationForm,
@@ -225,11 +225,7 @@ class SnapshotResultHealthListFilter(admin.SimpleListFilter):

                # Rare statuses are faster status-first: use the
                # (status, snapshot_id) index to find candidate snapshots.
-                snapshot_ids = ArchiveResult.cached_snapshot_ids_with_majority_status(status)
-                queryset = queryset.filter(pk__in=snapshot_ids)
-                queryset._archivebox_count_hint = len(snapshot_ids)
-                queryset.query._archivebox_count_hint = queryset._archivebox_count_hint
-                return queryset
+                return queryset.filter(pk__in=ArchiveResult.snapshot_ids_with_majority_status(status))
        return queryset


@@ -247,7 +243,21 @@ class SnapshotChangeList(SearchResultsChangeList):
        if not snapshot_ids:
            return

-        results_by_snapshot = {snapshot_id: [] for snapshot_id in snapshot_ids}
+        status_counts_by_snapshot = {
+            row["snapshot_id"]: row
+            for row in ArchiveResult.objects.filter(snapshot_id__in=snapshot_ids)
+            .values("snapshot_id")
+            .annotate(
+                total=Count("pk"),
+                succeeded=Count("pk", filter=Q(status=ArchiveResult.StatusChoices.SUCCEEDED)),
+                failed=Count("pk", filter=Q(status=ArchiveResult.StatusChoices.FAILED)),
+                running=Count("pk", filter=Q(status=ArchiveResult.StatusChoices.STARTED)),
+                skipped=Count("pk", filter=Q(status=ArchiveResult.StatusChoices.SKIPPED)),
+                noresults=Count("pk", filter=Q(status=ArchiveResult.StatusChoices.NORESULTS)),
+            )
+        }
+
+        output_results_by_snapshot = {snapshot_id: [] for snapshot_id in snapshot_ids}
        seen_plugins = {snapshot_id: set() for snapshot_id in snapshot_ids}
        rows = (
            ArchiveResult.objects.filter(snapshot_id__in=snapshot_ids, status=ArchiveResult.StatusChoices.SUCCEEDED, output_size__gt=0)
@@ -258,12 +268,32 @@ class SnapshotChangeList(SearchResultsChangeList):
            if plugin in seen_plugins[snapshot_id]:
                continue
            seen_plugins[snapshot_id].add(plugin)
-            results_by_snapshot[snapshot_id].append(
+            output_results_by_snapshot[snapshot_id].append(
                SimpleNamespace(plugin=plugin, status=status, output_size=output_size, output_files=output_files),
            )

        for obj in self.result_list:
-            obj.__dict__["_admin_archiveresults"] = results_by_snapshot[obj.pk]
+            counts = status_counts_by_snapshot.get(obj.pk, {})
+            total = int(counts.get("total") or 0)
+            succeeded = int(counts.get("succeeded") or 0)
+            failed = int(counts.get("failed") or 0)
+            running = int(counts.get("running") or 0)
+            skipped = int(counts.get("skipped") or 0)
+            noresults = int(counts.get("noresults") or 0)
+            completed = succeeded + failed + skipped + noresults
+            obj.__dict__["_admin_progress_stats"] = {
+                "total": total,
+                "succeeded": succeeded,
+                "failed": failed,
+                "running": running,
+                "pending": max(total - completed - running, 0),
+                "skipped": skipped,
+                "noresults": noresults,
+                "percent": int((completed / total * 100) if total else 0),
+                "output_size": obj.output_size or 0,
+                "is_sealed": obj.status not in (obj.StatusChoices.QUEUED, obj.StatusChoices.STARTED, obj.StatusChoices.PAUSED),
+            }
+            obj.__dict__["_admin_output_results"] = output_results_by_snapshot[obj.pk]

    def get_results(self, request):
        super().get_results(request)
@@ -316,19 +346,8 @@ class SnapshotAdminForm(forms.ModelForm):

            # Parse and save tags from tags_editor
            tags_str = self.cleaned_data.get("tags_editor", "")
-            if tags_str:
-                tag_names = [name.strip() for name in tags_str.split(",") if name.strip()]
-                tags = []
-                for name in tag_names:
-                    tag, _ = Tag.objects.get_or_create(
-                        name__iexact=name,
-                        defaults={"name": name},
-                    )
-                    tag = Tag.objects.filter(name__iexact=name).first() or tag
-                    tags.append(tag)
-                instance.tags.set(tags)
-            else:
-                instance.tags.clear()
+            tag_names = [name.strip() for name in tags_str.split(",") if name.strip()]
+            instance.save_tags(tag_names)

        return instance

@@ -337,6 +356,16 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
    form = SnapshotAdminForm
    raw_id_fields = ("crawl", "parent_snapshot")
    list_select_related = ()
+
+    def save_related(self, request, form, formsets, change):
+        super().save_related(request, form, formsets, change)
+        tags_str = form.cleaned_data.get("tags_editor", "")
+        tag_names = [name.strip() for name in tags_str.split(",") if name.strip()]
+        form.instance.save_tags(
+            tag_names,
+            created_by=request.user if request.user.is_authenticated else None,
+        )
+
    list_display = (
        "permissions_badge",
        "created_at",
@@ -475,19 +504,18 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
            return []
        return super().get_ordering(request)

-    def change_view(self, request, object_id, form_url="", extra_context=None):
+    def render_change_form(self, request, context, add=False, change=False, form_url="", obj=None):
        self.request = request
-        extra_context = extra_context or {}
-        extra_context["CONFIG"] = request.archivebox_config
-        snapshot = self.get_object(request, object_id)
-        if snapshot and snapshot.status in {
+        context["CONFIG"] = request.archivebox_config
+        if obj and obj.status in {
            Snapshot.StatusChoices.QUEUED,
            Snapshot.StatusChoices.STARTED,
            Snapshot.StatusChoices.PAUSED,
        }:
-            extra_context["progress_auto_expand"] = True
-            extra_context["progress_endpoint"] = progress_endpoint("snapshot", snapshot.id)
-        return super().change_view(request, object_id, form_url, extra_context | GLOBAL_CONTEXT)
+            context["progress_auto_expand"] = True
+            context["progress_endpoint"] = progress_endpoint("snapshot", obj.id)
+        context.update(GLOBAL_CONTEXT)
+        return super().render_change_form(request, context, add=add, change=change, form_url=form_url, obj=obj)

    def changelist_view(self, request, extra_context=None):
        self.request = request
@@ -844,7 +872,12 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
    @admin.display(description="Archive Results")
    def archiveresults_list(self, obj):
        request = self.request
-        return render_archiveresults_list(obj.archiveresult_set.all(), limit=8, config=request.archivebox_config)
+        return render_archiveresults_list(
+            obj.archiveresult_set.all(),
+            limit=8,
+            config=request.archivebox_config,
+            can_delete=request.user.is_superuser,
+        )

    @admin.display(
        description="Title",
@@ -1277,6 +1310,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
        return stats

    def _get_prefetched_results(self, obj):
+        if "_admin_output_results" in obj.__dict__:
+            return obj.__dict__["_admin_output_results"]
        if "_admin_archiveresults" in obj.__dict__:
            return obj.__dict__["_admin_archiveresults"]
        if "archiveresult_set" in obj.__dict__.get("_prefetched_objects_cache", {}):
@@ -96,7 +96,7 @@ class CustomUserAdmin(UserAdmin):
            + f'<br/><a href="/admin/core/snapshot/?created_by__id__exact={obj.pk}">{total_count} total records...<a>',
        )

-    @admin.display(description="Archive Result Logs")
+    @admin.display(description="Archive Results")
    def archiveresult_set(self, obj):
        total_count = obj.archiveresult_set.count()
        return mark_safe(
@@ -10,7 +10,6 @@ class CoreConfig(AppConfig):

    def ready(self):
        """Register the archivebox.core.admin_site as the main django admin site"""
-        import sys
        from django.utils.autoreload import DJANGO_AUTORELOAD_ENV

        from archivebox.core.admin_site import register_admin_site
@@ -28,11 +27,6 @@ class CoreConfig(AppConfig):

        pre_save.connect(truncate_overlong_charfields, dispatch_uid="archivebox_truncate_overlong_charfields")

-        # Import models to register state machines with the registry
-        # Skip during makemigrations to avoid premature state machine access
-        if "makemigrations" not in sys.argv:
-            from archivebox.core import models  # noqa: F401
-
        def _should_prepare_runtime() -> bool:
            if os.environ.get("ARCHIVEBOX_RUNSERVER") == "1":
                if os.environ.get("ARCHIVEBOX_AUTORELOAD") == "1":
@@ -13,64 +13,12 @@ from archivebox.config.django import setup_django

 setup_django(check_db=True)

-
-def _patch_thread_sensitive_context_shutdown() -> None:
-    """Stop ``ThreadSensitiveContext.__aexit__`` from blocking the daphne loop.
-
-    Django 6.0's ASGIHandler wraps every request in ``async with
-    ThreadSensitiveContext():`` (django/core/handlers/asgi.py:169). On exit
-    asgiref calls ``executor.shutdown()`` with the default ``wait=True``
-    (asgiref/sync.py:148), which is a *synchronous* ``Thread.join()`` inside
-    an async function — so it blocks the daphne event loop until the
-    executor's worker thread exits.
-
-    That's normally fine because the request handler has already awaited
-    every ``sync_to_async`` it submitted, so the worker is idle and dies as
-    soon as the shutdown sentinel reaches it. The blocking turns into a
-    problem when a client disconnects mid-request:
-
-    * ``SyncToAsync.__call__`` shields the executor work with
-      ``await asyncio.shield(exec_coro)`` (asgiref/sync.py:506) so that the
-      sync DB call doesn't get torn down halfway through.
-    * On cancellation it calls ``exec_coro.cancel()`` (line 522) which only
-      flips the asyncio ``Future`` to cancelled — the underlying thread
-      keeps running the SQL query.
-    * Control unwinds to ``__aexit__`` while the orphaned thread is still
-      mid-query. ``shutdown(wait=True)`` then blocks the event loop until
-      that orphan finishes.
-
-    Under heavy SQLite contention (the load-test scenario that surfaced
-    this on cabbage) those orphan threads can take 30 seconds each waiting
-    for write locks, and the daphne loop is single-threaded — so every
-    such orphan stalls every other in-flight request, healthchecks time
-    out, and the container goes ``unhealthy``.
-
-    Switching to ``shutdown(wait=False)`` queues the sentinel and returns
-    immediately; the worker thread still exits cleanly once its current
-    task finishes, and asgiref's ``WeakKeyDictionary`` releases the
-    executor as soon as the request's context is GC'd. No per-request
-    teardown guarantee is lost — there was no caller relying on it.
-    """
-    from asgiref import sync as _asgiref_sync
-
-    original_aexit = _asgiref_sync.ThreadSensitiveContext.__aexit__
-
-    async def __aexit__(self, exc, value, tb):  # type: ignore[no-redef]
-        if not self.token:
-            return
-        executor = _asgiref_sync.SyncToAsync.context_to_thread_executor.pop(self, None)
-        if executor is not None:
-            executor.shutdown(wait=False)
-        _asgiref_sync.SyncToAsync.thread_sensitive_context.reset(self.token)
-
-    # Idempotent: only patch once even if asgi.py is reloaded.
-    if getattr(original_aexit, "_archivebox_patched", False):
-        return
-    __aexit__._archivebox_patched = True  # type: ignore[attr-defined]
-    _asgiref_sync.ThreadSensitiveContext.__aexit__ = __aexit__
+django_application = get_asgi_application()


-_patch_thread_sensitive_context_shutdown()
+async def application(scope, receive, send):
+    if scope["type"] == "websocket":
+        from archivebox.opencode.views import websocket_view

-# Standard Django ASGI application (no websockets/channels needed)
-application = get_asgi_application()
+        return await websocket_view(scope, receive, send)
+    return await django_application(scope, receive, send)
@@ -12,9 +12,9 @@ from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget
 from archivebox.crawls.schedule_util import validate_schedule
 from archivebox.misc.util import URL_REGEX, find_all_urls, parse_filesize_to_bytes
 from archivebox.personas.models import Persona
-from archivebox.plugins.discovery import get_plugins
+from archivebox.plugins.discovery import get_plugin_catalog
 from archivebox.plugins.forms import (
-    PLUGIN_GROUP_DEFINITIONS,
+    PLUGIN_GROUPS,
    TIMEOUT_INPUT_PATTERN,
    PluginConfigFormMixin,
    get_choice_field,
@@ -333,10 +333,9 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
        if self.can_override_crawl_config:
            self.build_plugin_groups(get_config(persona=selected_persona) if selected_persona else get_config())
        else:
-            all_plugins = get_plugins()
-            for field_name, *_rest, plugin_names in PLUGIN_GROUP_DEFINITIONS:
-                get_choice_field(self, field_name).choices = [(p, p) for p in all_plugins if p in plugin_names]
-            get_choice_field(self, "other_plugins").choices = [(p, p) for p in all_plugins]
+            grouped_plugins = get_plugin_catalog().groups()
+            for category, field_name, _title in PLUGIN_GROUPS:
+                get_choice_field(self, field_name).choices = [(plugin.name, plugin.name) for plugin in grouped_plugins.get(category, [])]
            self.plugin_groups = []

    def clean(self):
@@ -15,7 +15,7 @@ from django.utils import timezone
 from django.utils.http import http_date

 from archivebox.config import VERSION
-from archivebox.config.common import get_config
+from archivebox.config.common import get_config, get_request_config
 from archivebox.config.version import get_COMMIT_HASH
 from archivebox.core.routes_util import (
    build_admin_url,
@@ -39,8 +39,8 @@ ADMIN_LOGIN_HINT_COOKIE = "archivebox_admin_logged_in"
 def _admin_login_hint_cookie_domain(config) -> str | None:
    """Resolve the parent domain to scope the cross-subdomain login hint.

-    NOTE: this cookie carries only the single bit "user is logged in on
-    admin somewhere"; it MUST NOT be confused with the session cookie,
+    NOTE: this cookie carries only the single bit "a superuser is logged in
+    on admin somewhere"; it MUST NOT be confused with the session cookie,
    which stays admin-host-scoped (see core/settings.py
    SESSION_COOKIE_DOMAIN comment — admin/web is a security boundary).

@@ -88,6 +88,11 @@ def AdminCookieIsolationMiddleware(get_response):
    def middleware(request):
        response = get_response(request)

+        if request.path == "/admin" or request.path.startswith("/admin/"):
+            if not getattr(response, "xframe_options_exempt", False):
+                response.headers["X-Frame-Options"] = "DENY"
+                response.headers["Content-Security-Policy"] = "frame-ancestors 'none'"
+
        config = request.__dict__.get("archivebox_config")
        if config is None or config.SERVER_SECURITY_MODE == "auto":
            from archivebox.config.common import get_request_config
@@ -178,6 +183,11 @@ def ServerSecurityModeMiddleware(get_response):
                api_host, _api_port = split_host_port(get_api_host(config=base_config))
                web_host, _web_port = split_host_port(get_web_host(config=base_config))
                control_hosts = {host for host in (base_host, admin_host, api_host, web_host) if host}
+                credentialed_api_request = request.path == "/api/v1/auth/check_api_token" or bool(
+                    request.META.get("HTTP_X_ARCHIVEBOX_API_KEY") or request.META.get("HTTP_AUTHORIZATION"),
+                )
+                if not base_config.BASE_URL and credentialed_api_request:
+                    control_hosts.add(request_host)
                first_run_setup_request = not base_config.BASE_URL and (
                    request.path == "/admin/login/"
                    or (
@@ -192,6 +202,21 @@ def ServerSecurityModeMiddleware(get_response):

            config = get_request_config(request, resolve_plugins=False)

+        if config.USES_SUBDOMAIN_ROUTING and config.BASE_URL and request.method.upper() not in allowed_methods:
+            request_host, _request_port = split_host_port((request.get_host() or "").lower())
+            control_hosts = {
+                split_host_port(host)[0]
+                for host in (
+                    get_base_host(config=config),
+                    get_admin_host(config=config),
+                    get_api_host(config=config),
+                    get_web_host(config=config),
+                )
+                if host
+            }
+            if request_host not in control_hosts:
+                return HttpResponseForbidden("ArchiveBox is running with the control plane disabled on this host.")
+
        if config.CONTROL_PLANE_ENABLED:
            return get_response(request)

@@ -225,14 +250,13 @@ def HostRoutingMiddleware(get_response):

        request_host = (request.get_host() or "").lower()
        config = request.__dict__.get("archivebox_config")
-        if config is None:
-            config = get_config(resolve_plugins=False)
-            request.archivebox_config = config
-        admin_host = get_admin_host(config=config)
-        web_host = get_web_host(config=config)
-        api_host = get_api_host(config=config)
+        if config is None or config.SERVER_SECURITY_MODE == "auto":
+            config = get_request_config(request, resolve_plugins=False)
+        admin_host = get_admin_host(config=config, request=request)
+        web_host = get_web_host(config=config, request=request)
+        api_host = get_api_host(config=config, request=request)
        listen_host = get_listen_host(config=config)
-        subdomain = get_listen_subdomain(request_host, config=config)
+        subdomain = get_listen_subdomain(request_host, config=config, request=request)

        # Framework-owned assets must bypass snapshot/original-domain replay routing.
        # Otherwise pages on snapshot subdomains can receive HTML for JS/CSS requests.
@@ -315,7 +339,12 @@ def HostRoutingMiddleware(get_response):
                return redirect(target)
            response = get_response(request)
            hint_cookie_domain = _admin_login_hint_cookie_domain(config)
-            if request.user.is_authenticated and not request.path.startswith("/admin/logout"):
+            if (
+                request.user.is_authenticated
+                and request.user.is_active
+                and request.user.is_superuser
+                and not request.path.startswith("/admin/logout")
+            ):
                response.set_cookie(
                    ADMIN_LOGIN_HINT_COOKIE,
                    "1",
@@ -386,7 +415,7 @@ class ReverseProxyAuthMiddleware(RemoteUserMiddleware):
    def process_request(self, request):
        config = request.__dict__.get("archivebox_config")
        if config is None:
-            config = get_config(resolve_plugins=False)
+            config = get_config(base_config=settings.CONFIG, resolve_plugins=False)
            request.archivebox_config = config
        self.header = "HTTP_{normalized}".format(normalized=config.REVERSE_PROXY_USER_HEADER.replace("-", "_").upper())
        if config.REVERSE_PROXY_WHITELIST == "":
@@ -0,0 +1,16 @@
+# Generated by Django 6.1 on 2026-08-30 18:20
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0051_postgres_url_pattern_ops_index"),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name="archiveresult",
+            options={"verbose_name": "Archive Result", "verbose_name_plural": "Archive Results"},
+        ),
+    ]
@@ -0,0 +1,26 @@
+# Generated by Django 6.1 on 2026-09-02
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0053_alter_archiveresult_options"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="snapshot",
+            name="current_step",
+        ),
+        migrations.AlterField(
+            model_name="snapshot",
+            name="fs_version",
+            field=models.CharField(
+                db_index=True,
+                default="0.9.0",
+                help_text='Filesystem version of this snapshot (e.g., "0.7.0", "0.8.0", "0.9.4").',
+                max_length=10,
+            ),
+        ),
+    ]
@@ -10,10 +10,15 @@ def _is_signal_interrupted_exit(exit_code: int | None) -> bool:
    return exit_code is not None and (exit_code < 0 or exit_code >= 128)


+def _canonical_hook_name(hook_name: str) -> str:
+    hook_name = Path(hook_name).name
+    return Path(hook_name).stem if Path(hook_name).suffix in {".py", ".js", ".sh"} else hook_name
+
+
 def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str | None = None) -> dict[str, int]:
    from archivebox.crawls.models import Crawl
    from archivebox.core.models import ArchiveResult, Snapshot
-    from archivebox.services.archive_result_service import _collect_output_metadata
+    from abx_dl.output_files import OutputManifest
    from archivebox.machine.models import Process
    from django.core.exceptions import ValidationError
    from django.db.models import Exists, OuterRef, Q, Subquery, Value
@@ -30,9 +35,8 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
        "chrome_processes_orphaned": Process.cleanup_orphaned_chrome() if include_chrome and not crawl_id else 0,
        "crawls_queued_without_retry_at": 0,
        "snapshots_queued_without_retry_at": 0,
-        "archiveresults_backoff": 0,
-        "snapshots_queued_plugin_rows_waiting_on_stale_lease": 0,
-        "archiveresults_started_without_running_process": 0,
+        "snapshots_sealed_with_extension_uploads_only": 0,
+        "archiveresults_interrupted_without_running_process": 0,
        "archiveresults_missing_for_orphaned_hook_processes": 0,
        "snapshots_started_without_running_results": 0,
        "crawls_started_with_due_snapshots": 0,
@@ -40,10 +44,10 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
        "crawls_started_without_active_snapshots": 0,
    }

-    running_archiveresults = ArchiveResult.objects.filter(
-        snapshot_id=OuterRef("pk"),
-        status=ArchiveResult.StatusChoices.STARTED,
-        process__status=Process.StatusChoices.RUNNING,
+    running_hook_processes = Process.objects.filter(
+        archiveresult__snapshot_id=OuterRef("pk"),
+        process_type=Process.TypeChoices.HOOK,
+        status=Process.StatusChoices.RUNNING,
    )
    active_child_snapshots = Snapshot.objects.filter(
        crawl_id=OuterRef("pk"),
@@ -69,49 +73,52 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
        crawl__status__in=Crawl.RUNNABLE_STATES,
        **snapshot_filter,
    ).update(retry_at=now, modified_at=now)
-    backoff_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.BACKOFF, **result_filter)
+
+    extension_upload_results = ArchiveResult.objects.filter(
+        snapshot_id=OuterRef("pk"),
+        hook_name=Snapshot.BROWSER_EXTENSION_UPLOAD_HOOK_NAME,
+    )
+    server_results = ArchiveResult.objects.filter(snapshot_id=OuterRef("pk")).exclude(
+        hook_name=Snapshot.BROWSER_EXTENSION_UPLOAD_HOOK_NAME,
+    )
+    extension_only_snapshots = (
+        Snapshot.objects.filter(
+            status=Snapshot.StatusChoices.SEALED,
+            crawl__status__in=[Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED, Crawl.StatusChoices.SEALED],
+            **snapshot_filter,
+        )
+        .annotate(
+            has_extension_upload=Exists(extension_upload_results),
+            has_server_result=Exists(server_results),
+        )
+        .filter(has_extension_upload=True, has_server_result=False)
+    )
+    # Older browser-extension uploads could win a race with runner startup:
+    # their successful external rows made the fresh Snapshot look finished
+    # before its configured server-hook workset was materialized. Reopen only
+    # that exact, recognizable state so the normal runner adds the missing
+    # server rows to the same Snapshot and preserves the uploaded outputs.
+    Crawl.objects.filter(
+        id__in=Subquery(extension_only_snapshots.values("crawl_id")),
+        status=Crawl.StatusChoices.SEALED,
+    ).update(status=Crawl.StatusChoices.STARTED, retry_at=now, modified_at=now)
+    cleaned["snapshots_sealed_with_extension_uploads_only"] = extension_only_snapshots.update(
+        status=Snapshot.StatusChoices.QUEUED,
+        retry_at=now,
+        modified_at=now,
+    )
    orphaned_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.STARTED, **result_filter).exclude(
        process__status=Process.StatusChoices.RUNNING,
    )
-    # ArchiveResult has no retry_at scheduler. Wake only the parent Snapshots
-    # for result rows we are about to repair, then requeue those exact rows.
-    # Using subqueries keeps million-row recovery in SQLite instead of building
-    # Python ID lists or scanning all sealed snapshots.
-    Snapshot.objects.filter(
-        id__in=backoff_results.values("snapshot_id"),
-        status__in=[Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED],
-        retry_at__isnull=True,
-    ).update(retry_at=now, modified_at=now)
+    # ArchiveResult rows are projections, never work items. Close interrupted
+    # projections as failed and wake their parent Snapshot so abx-dl can replay
+    # the snapshot-level sequence. Indexed subqueries keep this bounded.
    Snapshot.objects.filter(
        id__in=orphaned_results.values("snapshot_id"),
-        status__in=[Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED],
-        retry_at__isnull=True,
+        status=Snapshot.StatusChoices.STARTED,
    ).update(retry_at=now, modified_at=now)
-    cleaned["archiveresults_backoff"] = backoff_results.update(status=ArchiveResult.StatusChoices.QUEUED, modified_at=now)
-    # Targeted plugin rows on final/paused Snapshots are scheduled through the
-    # parent Snapshot.retry_at. retry_at=NULL is the normal idle marker for a
-    # sealed Snapshot and must not be interpreted as queued work just because
-    # old/synthetic ArchiveResult rows exist. If takeover kills the runner
-    # after it leases the Snapshot but before queued ArchiveResult rows finish,
-    # the rows remain QUEUED while retry_at sits in the future. Recovery runs
-    # only after this runner has won the single-runner gate, so it can safely
-    # unlock those stale plugin leases for immediate processing instead of
-    # waiting out the previous owner's full lock timeout.
-    queued_plugin_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.QUEUED, **result_filter)
-    cleaned["snapshots_queued_plugin_rows_waiting_on_stale_lease"] = (
-        Snapshot.objects.filter(
-            id__in=queued_plugin_results.values("snapshot_id"),
-            status__in=[Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED],
-        )
-        .filter(retry_at__gt=now)
-        .update(retry_at=now, modified_at=now)
-    )
-    # Impossible state repair: STARTED ArchiveResults without a live Process
-    # have no owner left to emit completion. Requeue only the result row; the
-    # snapshot/crawl schedulers will pick up normal retry processing.
-    cleaned["archiveresults_started_without_running_process"] = orphaned_results.update(
-        status=ArchiveResult.StatusChoices.QUEUED,
-        process=None,
+    cleaned["archiveresults_interrupted_without_running_process"] = orphaned_results.update(
+        status=ArchiveResult.StatusChoices.FAILED,
        modified_at=now,
    )
    orphaned_hook_processes = Process.objects.filter(
@@ -133,11 +140,23 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
            orphaned_hook_processes = orphaned_hook_processes.filter(snapshot_pwd_filter)
        else:
            orphaned_hook_processes = orphaned_hook_processes.none()
-    for process in orphaned_hook_processes.only("id", "pwd", "cmd", "process_type", "status"):
+    for process in orphaned_hook_processes.only(
+        "id",
+        "pwd",
+        "cmd",
+        "process_type",
+        "status",
+        "exit_code",
+        "stdout",
+        "stderr",
+        "started_at",
+        "ended_at",
+    ).order_by("-started_at", "-id"):
        hook_script_name = process.hook_script_name
        if not hook_script_name or not process.pwd:
            continue
        plugin_dir = Path(process.pwd)
+        hook_name = _canonical_hook_name(hook_script_name)
        if crawl_snapshot_ids is not None and plugin_dir.parent.name not in crawl_snapshot_ids:
            continue
        try:
@@ -149,49 +168,75 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
            continue
        if snapshot is None:
            continue
-        result, created = ArchiveResult.objects.get_or_create(
-            snapshot=snapshot,
-            plugin=plugin_dir.name,
-            hook_name=Path(hook_script_name).stem,
+        result, created = ArchiveResult.get_or_create_by_hook(
+            snapshot,
+            plugin_dir.name,
+            hook_name,
            defaults={
-                "status": ArchiveResult.StatusChoices.QUEUED,
+                "status": ArchiveResult.StatusChoices.FAILED,
            },
        )
-        if result.status == ArchiveResult.StatusChoices.QUEUED:
+        process_is_newer = bool(process.started_at and (result.start_ts is None or process.started_at >= result.start_ts))
+        if created or process_is_newer:
            requeue_snapshot = False
            # A runner can die after the hook Process exits but before the
            # ProcessCompletedEvent projector links/finalizes ArchiveResult.
            # Reconstruct only that exact hook row from the durable Process row.
-            output_files, output_size, output_mimetypes = _collect_output_metadata(plugin_dir)
+            manifest = OutputManifest.scan(plugin_dir, containment_root=snapshot.output_dir)
+            output_files = manifest.as_mapping()
+            output_size = manifest.total_size
+            output_mimetypes = ",".join(manifest.mimetypes)
+            emitted_records = [
+                record
+                for record in Process.parse_records_from_text(process.stdout or "")
+                if record.get("type") == "ArchiveResult"
+                and (record.get("plugin") or plugin_dir.name) == plugin_dir.name
+                and _canonical_hook_name(str(record.get("hook_name") or hook_name)) == hook_name
+            ]
+            emitted_result = emitted_records[-1] if emitted_records else {}
            result.process = process
+            result.start_ts = process.started_at
+            result.end_ts = process.ended_at
            if _is_signal_interrupted_exit(process.exit_code):
-                # The owning runner died or was asked to stop while the hook was
-                # still active. Keep the work item queued so takeover retries the
-                # same hook; treating an unknown signal exit as success would
-                # silently skip unfinished side effects.
+                # The Process is a durable fact; record the interruption as a
+                # failure and wake the parent Snapshot for replay.
                result.output_files = {}
                result.output_size = 0
                result.output_mimetypes = ""
                result.output_str = ""
-                result.status = ArchiveResult.StatusChoices.QUEUED
+                result.output_json = None
+                result.status = ArchiveResult.StatusChoices.FAILED
                requeue_snapshot = True
            else:
                result.output_files = output_files
                result.output_size = output_size
                result.output_mimetypes = output_mimetypes
-                result.output_str = process.stderr if process.exit_code not in (0, None) else ""
+                result.output_str = (
+                    emitted_result.get("output_str")
+                    or emitted_result.get("output")
+                    or (process.stderr if process.exit_code not in (0, None) else "")
+                )
+                result.output_json = emitted_result.get("output_json") if isinstance(emitted_result.get("output_json"), dict) else None
+                emitted_status = emitted_result.get("status")
                result.status = (
-                    ArchiveResult.StatusChoices.FAILED
-                    if process.exit_code not in (0, None)
-                    else (ArchiveResult.StatusChoices.SUCCEEDED if output_files else ArchiveResult.StatusChoices.NORESULTS)
+                    emitted_status
+                    if emitted_status in ArchiveResult.StatusChoices.values
+                    else (
+                        ArchiveResult.StatusChoices.FAILED
+                        if process.exit_code not in (0, None)
+                        else (ArchiveResult.StatusChoices.SUCCEEDED if output_files else ArchiveResult.StatusChoices.NORESULTS)
+                    )
                )
            result.save(
                update_fields=[
                    "process",
+                    "start_ts",
+                    "end_ts",
                    "output_files",
                    "output_size",
                    "output_mimetypes",
                    "output_str",
+                    "output_json",
                    "status",
                    "modified_at",
                ],
@@ -200,7 +245,6 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
                Snapshot.objects.filter(id=snapshot.id).update(retry_at=now, modified_at=now)
        if created:
            cleaned["archiveresults_missing_for_orphaned_hook_processes"] += 1
-            Snapshot.objects.filter(id=snapshot.id).update(retry_at=now, modified_at=now)
    started_snapshots = Snapshot.objects.filter(status=Snapshot.StatusChoices.STARTED).filter(
        Q(retry_at__isnull=True) | Q(retry_at__gt=now),
        **snapshot_filter,
@@ -214,8 +258,8 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
    # We only unlock scheduling; normal Snapshot runner code owns the next
    # transition and side effects.
    cleaned["snapshots_started_without_running_results"] = (
-        started_snapshots.annotate(has_running_results=Exists(running_archiveresults))
-        .filter(has_running_results=False)
+        started_snapshots.annotate(has_running_process=Exists(running_hook_processes))
+        .filter(has_running_process=False)
        .update(
            retry_at=now,
            modified_at=now,
@@ -224,7 +268,7 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |

    # Broken lock repair: STARTED + retry_at=NULL is an orphaned ownership
    # lease. Recovery only unlocks scheduling; the runner owns any subsequent
-    # state-machine transition, including sealing rows whose children/results
+    # lifecycle transition, including sealing rows whose children/results
    # are already final.
    recoverable_started_crawls = Crawl.objects.filter(status=Crawl.StatusChoices.STARTED).filter(
        Q(retry_at__isnull=True) | Q(retry_at__gt=now),
@@ -268,13 +312,13 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
            "Starting {count} Snapshot(s) that were queued but never started "
            "(ArchiveBox may have been interrupted before it was able to archive those URLs)."
        ),
-        "archiveresults_backoff": (
-            "Retrying {count} extractor result(s) that were waiting to retry "
-            "(ArchiveBox may have been interrupted before it was able to try them again; affected outputs will be retried)."
+        "snapshots_sealed_with_extension_uploads_only": (
+            "Finishing {count} browser-extension Snapshot(s) that received uploaded files before server extractors started "
+            "(uploaded and server-created results will remain together on the same Snapshot)."
        ),
-        "archiveresults_started_without_running_process": (
-            "Retrying {count} extractor result(s) that were interrupted before finishing "
-            "(ArchiveBox may have been interrupted before it was able to save their final status; partial files will be overwritten with fresh results upon retry)."
+        "archiveresults_interrupted_without_running_process": (
+            "Closing {count} interrupted extractor result projection(s) "
+            "(their parent Snapshots are resumed through the normal snapshot-level runner)."
        ),
        "snapshots_started_without_running_results": (
            "Resuming {count} Snapshot(s) that were interrupted before finishing "
@@ -199,25 +199,25 @@ def _build_base_host(subdomain: str | None, request=None, config: dict[str, Any]
    return _with_port(full_host, port)


-def get_admin_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
+def get_admin_host(config: dict[str, Any] | None = None, request=None, **config_kwargs: Any) -> str:
+    config = config or (get_request_config(request) if request is not None else get_config(**config_kwargs))
    if not config.USES_SUBDOMAIN_ROUTING:
-        return get_base_host(config=config)
-    return _build_base_host("admin", config=config)
+        return get_base_host(request=request, config=config)
+    return _build_base_host("admin", request=request, config=config)


-def get_web_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
+def get_web_host(config: dict[str, Any] | None = None, request=None, **config_kwargs: Any) -> str:
+    config = config or (get_request_config(request) if request is not None else get_config(**config_kwargs))
    if not config.USES_SUBDOMAIN_ROUTING:
-        return get_base_host(config=config)
-    return _build_base_host("web", config=config)
+        return get_base_host(request=request, config=config)
+    return _build_base_host("web", request=request, config=config)


-def get_api_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
+def get_api_host(config: dict[str, Any] | None = None, request=None, **config_kwargs: Any) -> str:
+    config = config or (get_request_config(request) if request is not None else get_config(**config_kwargs))
    if not config.USES_SUBDOMAIN_ROUTING:
-        return get_base_host(config=config)
-    return _build_base_host("api", config=config)
+        return get_base_host(request=request, config=config)
+    return _build_base_host("api", request=request, config=config)


 def get_snapshot_subdomain(snapshot_id: str) -> str:
@@ -226,18 +226,18 @@ def get_snapshot_subdomain(snapshot_id: str) -> str:
    return f"snap-{suffix}"


-def get_snapshot_host(snapshot_id: str, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
+def get_snapshot_host(snapshot_id: str, config: dict[str, Any] | None = None, request=None, **config_kwargs: Any) -> str:
+    config = config or (get_request_config(request) if request is not None else get_config(**config_kwargs))
    if not config.USES_SUBDOMAIN_ROUTING:
-        return get_base_host(config=config)
-    return _build_base_host(get_snapshot_subdomain(snapshot_id), config=config)
+        return get_base_host(request=request, config=config)
+    return _build_base_host(get_snapshot_subdomain(snapshot_id), request=request, config=config)


-def get_original_host(domain: str, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
+def get_original_host(domain: str, config: dict[str, Any] | None = None, request=None, **config_kwargs: Any) -> str:
+    config = config or (get_request_config(request) if request is not None else get_config(**config_kwargs))
    if not config.USES_SUBDOMAIN_ROUTING:
-        return get_base_host(config=config)
-    return _build_base_host(domain, config=config)
+        return get_base_host(request=request, config=config)
+    return _build_base_host(domain, request=request, config=config)


 def is_snapshot_subdomain(subdomain: str) -> bool:
@@ -255,12 +255,12 @@ def get_snapshot_lookup_key(snapshot_ref: str) -> str:
    return value


-def get_listen_subdomain(request_host: str, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
+def get_listen_subdomain(request_host: str, config: dict[str, Any] | None = None, request=None, **config_kwargs: Any) -> str:
+    config = config or (get_request_config(request) if request is not None else get_config(**config_kwargs))
    if not config.USES_SUBDOMAIN_ROUTING:
        return ""
    req_host, req_port = split_host_port(request_host)
-    base_host, base_port = split_host_port(get_base_host(config=config))
+    base_host, base_port = split_host_port(get_base_host(request=request, config=config))
    if not base_host:
        return ""
    if base_port and req_port and base_port != req_port:
@@ -11,7 +11,7 @@ from pathlib import Path
 from django.conf.locale.en import formats as en_formats  # type: ignore

 import archivebox
-from archivebox.config.common import get_config
+from archivebox.config.django import CONFIG
 from archivebox.config.constants import CONSTANTS
 from archivebox.core.routes_util import get_admin_base_url, get_api_base_url, get_base_url, normalize_base_url

@@ -25,7 +25,6 @@ IS_MIGRATING = "makemigrations" in sys.argv[:3] or "migrate" in sys.argv[:3]
 IS_TESTING = "test" in sys.argv[:3]
 IS_SHELL = "shell" in sys.argv[:3] or "shell_plus" in sys.argv[:3]
 IS_GETTING_VERSION_OR_HELP = "version" in sys.argv or "help" in sys.argv or "--version" in sys.argv or "--help" in sys.argv
-CONFIG = get_config()
 PACKAGE_DIR = CONSTANTS.PACKAGE_DIR
 logger = logging.getLogger(__name__)

@@ -77,7 +76,6 @@ INSTALLED_APPS = [
    "archivebox.crawls",  # handles Crawl and CrawlSchedule models and management (depends on core)
    "archivebox.progressmonitor",  # live progress endpoint and admin monitor template
    "archivebox.api",  # Django-Ninja-based Rest API interfaces, config, APIToken model, etc.
-    "archivebox.opencode",
    # 3rd-party apps from PyPI that need to be loaded last
    "admin_data_views",  # handles rendering some convenient automatic read-only views of data in Django admin
    "django_extensions",  # provides Django Debug Toolbar (and other non-debug helpers)
@@ -250,7 +248,7 @@ SQLITE_CONNECTION_OPTIONS = {
    },
 }

-if is_postgres():
+if is_postgres(CONFIG):
    DATABASES = {
        "default": {
            "ENGINE": "django.db.backends.postgresql",
@@ -152,15 +152,8 @@ def get_or_create_tag(name: str, created_by: User | None = None) -> tuple[Tag, b
    if not normalized_name:
        raise ValueError("Tag name is required")

-    existing = Tag.objects.filter(name__iexact=normalized_name).first()
-    if existing:
-        return existing, False
-
-    tag = Tag.objects.create(
-        name=normalized_name,
-        created_by=created_by,
-    )
-    return tag, True
+    defaults = {"created_by": created_by} if created_by is not None else None
+    return Tag.get_or_create_by_name(normalized_name, defaults=defaults)


 def rename_tag(tag: Tag, name: str) -> Tag:
@@ -1,7 +1,22 @@
+"""Coordinate local foreground processes without pretending to provide a distributed lock.
+
+ArchiveBox wants one orchestrator per collection, but Process rows can only prove
+liveness for PIDs visible on the current machine/PID namespace. We therefore
+enforce one active runner per ``(Machine, DATA_DIR)`` locally, retire stale rows
+from sequential containers on that machine, and only warn about rows owned by a
+different machine. Foreign-machine rows must never block progress or be killed:
+multi-machine coordination belongs in the Crawl/Snapshot CAS claim layer, not in
+process takeover.
+
+These helpers only hand local supervisord/runner ownership between CLI parents.
+They must not hold database transactions or filesystem locks while work runs.
+"""
+
 from __future__ import annotations

 import time
 import sys
+from collections.abc import Callable
 from pathlib import Path

 from django.db import IntegrityError
@@ -14,25 +29,6 @@ RUNNER_WAITING_WORKER_TYPE = "runner_waiting"
 RUNNER_GATE_WORKER_TYPES = (RUNNER_ACTIVE_WORKER_TYPE, RUNNER_WAITING_WORKER_TYPE, "")


-def runtime_stack_owner_types():
-    from archivebox.machine.models import Process
-
-    return (
-        Process.TypeChoices.SERVER,
-        Process.TypeChoices.ORCHESTRATOR,
-    )
-
-
-def foreground_runner_owner_types():
-    from archivebox.machine.models import Process
-
-    return (
-        Process.TypeChoices.SERVER,
-        Process.TypeChoices.ADD,
-        Process.TypeChoices.UPDATE,
-    )
-
-
 def current_command(process_type: str, *, data_dir: str | Path, url: str | None = None):
    from archivebox.machine.models import Process

@@ -41,31 +37,8 @@ def current_command(process_type: str, *, data_dir: str | Path, url: str | None
    return proc


-def live_processes(*, process_type: str, data_dir: str | Path, url: str | None = None):
-    from archivebox.machine.models import Machine, Process
-
-    qs = Process.objects.filter(
-        machine=Machine.current(),
-        process_type=process_type,
-        status=Process.StatusChoices.RUNNING,
-        pwd=str(data_dir),
-    )
-    if url is not None:
-        qs = qs.filter(url=url)
-    return [proc for proc in qs.order_by("-created_at", "-modified_at").iterator(chunk_size=50) if proc.is_running]
-
-
-def newest_live_process(*, process_type: str, data_dir: str | Path, url: str | None = None):
-    processes = live_processes(process_type=process_type, data_dir=data_dir, url=url)
-    return processes[0] if processes else None
-
-
-def command_is_newest(command, *, process_type: str, data_dir: str | Path, url: str | None = None) -> bool:
-    leader = newest_live_process(process_type=process_type, data_dir=data_dir, url=url)
-    return bool(leader and leader.id == command.id)
-
-
 def runtime_stack_owner(*, data_dir: str | Path, exclude_id=None):
+    """Return the live local parent allowed to own the server runtime stack."""
    from archivebox.machine.models import Machine, Process

    machine = Machine.current()
@@ -73,7 +46,7 @@ def runtime_stack_owner(*, data_dir: str | Path, exclude_id=None):
        machine=machine,
        status=Process.StatusChoices.RUNNING,
        pwd=str(data_dir),
-        process_type__in=runtime_stack_owner_types(),
+        process_type__in=(Process.TypeChoices.SERVER, Process.TypeChoices.ORCHESTRATOR),
    )
    if exclude_id is not None:
        base_qs = base_qs.exclude(id=exclude_id)
@@ -102,6 +75,7 @@ def command_owns_runtime_stack(command, *, data_dir: str | Path) -> bool:


 def foreground_runner_owner(*, data_dir: str | Path, exclude_id=None):
+    """Return the newest live local parent allowed to borrow runner/sonic."""
    from archivebox.machine.models import Machine, Process

    machine = Machine.current()
@@ -109,7 +83,7 @@ def foreground_runner_owner(*, data_dir: str | Path, exclude_id=None):
        machine=machine,
        status=Process.StatusChoices.RUNNING,
        pwd=str(data_dir),
-        process_type__in=foreground_runner_owner_types(),
+        process_type__in=(Process.TypeChoices.SERVER, Process.TypeChoices.ADD, Process.TypeChoices.UPDATE),
    )
    if exclude_id is not None:
        qs = qs.exclude(id=exclude_id)
@@ -125,26 +99,6 @@ def command_owns_foreground_runner(command, *, data_dir: str | Path) -> bool:
    return bool(owner and owner.id == command.id)


-def runtime_stack_component_label(*, owner=None, data_dir: str | Path) -> str:
-    try:
-        from archivebox.workers.supervisord_util import active_supervisord_runtime_components
-
-        components = active_supervisord_runtime_components()
-    except Exception:
-        components = []
-
-    names = list(components)
-    if not names and owner is not None:
-        from archivebox.machine.models import Process
-
-        if owner.process_type == Process.TypeChoices.SERVER:
-            names = ["orchestrator", "server"]
-        elif owner.process_type == Process.TypeChoices.ORCHESTRATOR:
-            names = ["orchestrator"]
-
-    return ", ".join(dict.fromkeys(names)) or "runtime stack"
-
-
 def ensure_daemon_stack(*, reason: str = ""):
    from archivebox.config.common import get_config
    from archivebox.workers.supervisord_util import (
@@ -172,64 +126,86 @@ def ensure_daemon_stack(*, reason: str = ""):

    supervisor = get_existing_supervisord_process() or get_or_create_supervisord_process(daemonize=False)
    worker = get_worker(supervisor, sonic_worker["name"])
-    if isinstance(worker, dict) and worker.get("statename") in ("STARTING", "RUNNING"):
-        return worker
+    if not (isinstance(worker, dict) and worker.get("statename") in ("STARTING", "RUNNING")):
+        if reason:
+            rprint(f"[yellow][*] Starting daemon stack for {reason}...[/yellow]")
+        worker = start_worker(supervisor, sonic_worker)

-    if reason:
-        rprint(f"[yellow][*] Starting daemon stack for {reason}...[/yellow]")
-    return start_worker(supervisor, sonic_worker)
+    # Supervisord marks this worker RUNNING with startsecs=0, before Sonic has
+    # opened its search socket. A query launched at that point fails even though
+    # the daemon is about to become ready.
+    deadline = time.monotonic() + 30.0
+    while time.monotonic() < deadline:
+        if is_port_listening(sonic_event.host, sonic_event.port):
+            if time.monotonic() < deadline:
+                return worker
+            break
+        worker = get_worker(supervisor, sonic_worker["name"])
+        if not isinstance(worker, dict) or worker.get("statename") not in ("STARTING", "RUNNING"):
+            raise RuntimeError(f"Sonic search backend worker failed to start: {worker}")
+        time.sleep(min(0.1, max(0.0, deadline - time.monotonic())))
+    raise RuntimeError(f"Sonic search backend is not listening at {sonic_event.url}")


-def healthy_orchestrator(*, data_dir: str | Path):
-    from archivebox.machine.models import Machine, Process
-    from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker
+def live_runner_processes(*, data_dir: str | Path):
+    """Return locally verifiable runners and warn about unsupported overlap.

-    supervisor = get_existing_supervisord_process()
-    worker = get_worker(supervisor, "worker_runner") if supervisor else None
-    if isinstance(worker, dict) and worker.get("statename") in ("STARTING", "RUNNING"):
-        return worker
-
-    for proc in Process.objects.filter(
-        machine=Machine.current(),
-        process_type=Process.TypeChoices.ORCHESTRATOR,
-        status=Process.StatusChoices.RUNNING,
-        pwd=str(data_dir),
-    ).order_by("-created_at"):
-        if proc.is_running:
-            return proc
-    return None
-
-
-def _runner_sort_key(process):
-    return (process.started_at or process.created_at, process.created_at, str(process.id))
-
-
-def live_runner_processes(*, data_dir: str | Path, exclude_id=None):
+    A Process row from another machine is observability only: its PID cannot be
+    checked or signalled here, so it neither joins the local election nor gets
+    mutated. A row for this same Machine from another PID namespace represents
+    a previous sequential container under the supported model; warn, retire the
+    unreachable row, and let the new container continue.
+    """
    from archivebox.machine.models import Machine, Process

    machine = Machine.current()
    Process.cleanup_stale_running(machine=machine)
    qs = Process.objects.filter(
-        machine=machine,
        status=Process.StatusChoices.RUNNING,
        process_type=Process.TypeChoices.ORCHESTRATOR,
        worker_type__in=RUNNER_GATE_WORKER_TYPES,
        pwd=str(data_dir),
    )
-    if exclude_id is not None:
-        qs = qs.exclude(id=exclude_id)
-    return [process for process in qs.order_by("started_at", "created_at").iterator(chunk_size=20) if process.is_running]
+    foreign_machine_exists = qs.exclude(machine=machine).exists()
+    qs = qs.filter(machine=machine)
+    live = []
+    foreign_namespace_ids = []
+    for process in qs.order_by("started_at", "created_at").iterator(chunk_size=20):
+        if not process.shares_pid_namespace:
+            foreign_namespace_ids.append(process.id)
+            continue
+        if process.is_running:
+            live.append(process)
+    if foreign_machine_exists or foreign_namespace_ids:
+        rprint(
+            "[bold yellow]WARNING: Multiple orchestrators sharing a single collection is not officially supported! "
+            "Corruption may occur if you run two ArchiveBox workers on the same collection at once.[/bold yellow]",
+            file=sys.stderr,
+            soft_wrap=True,
+        )
+    if foreign_namespace_ids:
+        now = timezone.now()
+        Process.objects.filter(id__in=foreign_namespace_ids, status=Process.StatusChoices.RUNNING).update(
+            status=Process.StatusChoices.EXITED,
+            exit_code=0,
+            ended_at=now,
+            retry_at=None,
+            modified_at=now,
+        )
+    return live


 def enter_single_runner_gate(command, *, data_dir: str | Path, graceful_timeout: float = 5.0) -> bool:
    """
-    Admit exactly one active runner for this DATA_DIR using Process rows.
+    Admit one active runner for this Machine and DATA_DIR using Process rows.

    The current process is a real OS process while it waits, so we keep its
    Process row RUNNING but mark worker_type=runner_waiting. Only the process
    that wins takeover is promoted to worker_type=worker_runner, which is
-    protected by a partial unique DB constraint. Older runners are terminated
-    and fully waited out before promotion, so the runner work loop never overlaps.
+    protected by a partial unique DB constraint scoped to (Machine, DATA_DIR).
+    Older locally verifiable runners are terminated and fully waited out before
+    promotion, so runner work never overlaps on one machine. Foreign machines
+    are intentionally outside this gate and only produce a warning above.
    """
    from archivebox.machine.models import Process

@@ -251,7 +227,7 @@ def enter_single_runner_gate(command, *, data_dir: str | Path, graceful_timeout:
            )
            runners = live_runner_processes(data_dir=data_dir)

-        newest = max(runners, key=_runner_sort_key)
+        newest = max(runners, key=lambda process: (process.started_at or process.created_at, process.created_at, str(process.id)))
        if newest.id != command.id:
            rprint(
                f"[yellow][*] Newer ArchiveBox runner pid={newest.pid} is taking over; exiting this runner.[/yellow]",
@@ -283,24 +259,9 @@ def enter_single_runner_gate(command, *, data_dir: str | Path, graceful_timeout:
            time.sleep(0.1)


-def standby_until_leader_needed(command, *, process_type: str, data_dir: str | Path, url: str | None = None, interval: float = 2.0) -> None:
-    from archivebox.workers.supervisord_util import reap_foreground_supervisord_process
-
-    announced = False
-    while not command_is_newest(command, process_type=process_type, data_dir=data_dir, url=url):
-        reap_foreground_supervisord_process()
-        if not announced:
-            leader = newest_live_process(process_type=process_type, data_dir=data_dir, url=url)
-            leader_pid = leader.pid if leader else "unknown"
-            rprint(f"[yellow][*] Standing by; newer ArchiveBox process pid={leader_pid} is running the orchestrator and server.[/yellow]")
-            announced = True
-        time.sleep(interval)
-    command.modified_at = timezone.now()
-    command.save(update_fields=["modified_at"])
-
-
 def standby_until_runtime_stack_needed(command, *, data_dir: str | Path, interval: float = 2.0) -> dict[str, object]:
-    from archivebox.workers.supervisord_util import reap_foreground_supervisord_process
+    from archivebox.machine.models import Process
+    from archivebox.workers.supervisord_util import active_supervisord_runtime_components, reap_foreground_supervisord_process

    announced = False
    previous_owner_pid = None
@@ -309,7 +270,16 @@ def standby_until_runtime_stack_needed(command, *, data_dir: str | Path, interva
        if not announced:
            owner = runtime_stack_owner(data_dir=data_dir)
            owner_pid = owner.pid if owner else "unknown"
-            components = runtime_stack_component_label(owner=owner, data_dir=data_dir)
+            try:
+                component_names = list(active_supervisord_runtime_components())
+            except Exception:
+                component_names = []
+            if not component_names and owner is not None:
+                if owner.process_type == Process.TypeChoices.SERVER:
+                    component_names = ["orchestrator", "server"]
+                elif owner.process_type == Process.TypeChoices.ORCHESTRATOR:
+                    component_names = ["orchestrator"]
+            components = ", ".join(dict.fromkeys(component_names)) or "runtime stack"
            previous_owner_pid = owner_pid
            rprint(
                f"[yellow][*] A newer archivebox process took over the {components} "
@@ -323,12 +293,22 @@ def standby_until_runtime_stack_needed(command, *, data_dir: str | Path, interva
    return {"resumed": announced, "previous_owner_pid": previous_owner_pid}


-def standby_until_foreground_runner_needed(command, *, data_dir: str | Path, interval: float = 2.0) -> dict[str, object]:
+def standby_until_foreground_runner_needed(
+    command,
+    *,
+    data_dir: str | Path,
+    interval: float = 2.0,
+    work_is_complete: Callable[[], bool] | None = None,
+) -> dict[str, object]:
    from archivebox.workers.supervisord_util import reap_foreground_supervisord_process

    announced = False
    previous_owner_pid = None
-    while not command_owns_foreground_runner(command, data_dir=data_dir):
+    while True:
+        if work_is_complete is not None and work_is_complete():
+            return {"resumed": announced, "previous_owner_pid": previous_owner_pid, "work_completed": True}
+        if command_owns_foreground_runner(command, data_dir=data_dir):
+            break
        reap_foreground_supervisord_process()
        if not announced:
            owner = foreground_runner_owner(data_dir=data_dir)
@@ -343,4 +323,4 @@ def standby_until_foreground_runner_needed(command, *, data_dir: str | Path, int
        time.sleep(interval)
    command.modified_at = timezone.now()
    command.save(update_fields=["modified_at"])
-    return {"resumed": announced, "previous_owner_pid": previous_owner_pid}
+    return {"resumed": announced, "previous_owner_pid": previous_owner_pid, "work_completed": False}
@@ -1,16 +1,19 @@
+import logging
 import os
+from html import unescape
 from pathlib import Path
 from typing import Any
+from urllib.parse import quote, urlparse

 from abx_plugins.plugins.archivewebpage.replay_preview import is_replay_target as is_archivewebpage_replay_target
 from django import template
-from django.contrib.admin.templatetags.base import InclusionAdminNode
 from django.templatetags.static import static
 from django.utils import timezone
 from django.utils.html import escape
 from django.utils.safestring import mark_safe
 from django.utils.text import Truncator

+from archivebox.config import CONSTANTS
 from archivebox.core.routes_util import (
    build_snapshot_url,
    get_admin_base_url,
@@ -26,10 +29,25 @@ from archivebox.plugins.discovery import (

 register = template.Library()

+
+@register.simple_tag(takes_context=True)
+def plugin_ui(context, plugin: str, slot: str):
+    """Optional plugin UI must never prevent the host page from rendering."""
+    try:
+        source = get_plugin_template(plugin, slot, fallback=False)
+        if not source:
+            return ""
+        return template.engines["django"].from_string(source).render(context.flatten())
+    except Exception:
+        logging.getLogger(__name__).exception("Unable to render optional plugin UI: %s/%s", plugin, slot)
+        return ""
+
+
 _TEXT_PREVIEW_EXTS = (".json", ".jsonl", ".txt", ".csv", ".tsv", ".xml", ".yml", ".yaml", ".md", ".log")
 _IMAGE_PREVIEW_EXTS = (".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".ico", ".avif")
+_STATIC_URL_SAFE = "/@-._~!$&'()*+,;="

-_MEDIA_FILE_EXTS = {
+_VIDEO_FILE_EXTS = {
    ".mp4",
    ".webm",
    ".mkv",
@@ -46,6 +64,9 @@ _MEDIA_FILE_EXTS = {
    ".3gp",
    ".3g2",
    ".ogv",
+}
+
+_AUDIO_FILE_EXTS = {
    ".mp3",
    ".m4a",
    ".aac",
@@ -63,16 +84,30 @@ _MEDIA_FILE_EXTS = {
    ".dts",
 }

+_MEDIA_FILE_EXTS = _VIDEO_FILE_EXTS | _AUDIO_FILE_EXTS
+_BROWSER_VIDEO_FILE_EXTS = {".mp4", ".webm", ".m4v", ".ogv"}
+_BROWSER_AUDIO_FILE_EXTS = {".mp3", ".m4a", ".aac", ".ogg", ".oga", ".opus", ".wav", ".flac"}
+
+
+def _is_web_subdomain_alias(browser_url: str, configured_base_url: str) -> bool:
+    """Hide the informational mismatch banner on the conventional public web alias.
+
+    This is deliberately presentation-only: request routing and the restriction
+    on state-changing requests continue to use the configured BASE_URL.
+    """
+    browser = urlparse(browser_url)
+    configured = urlparse(configured_base_url)
+    return (
+        browser.scheme.lower() == configured.scheme.lower()
+        and browser.hostname == f"web.{configured.hostname}"
+        and browser.port == configured.port
+    )
+

 def _normalize_output_files(output_files: Any) -> dict[str, dict[str, Any]]:
-    if isinstance(output_files, dict):
-        normalized: dict[str, dict[str, Any]] = {}
-        for path, metadata in output_files.items():
-            if not path:
-                continue
-            normalized[str(path)] = dict(metadata) if isinstance(metadata, dict) else {}
-        return normalized
-    return {}
+    from abx_dl.output_files import OutputManifest
+
+    return OutputManifest.from_value(output_files).as_mapping()


 def _snapshot_id(value: Any) -> Any:
@@ -156,15 +191,27 @@ def _list_media_files(result) -> list[dict]:

    for rel_path, size in candidates:
        href = str(Path(result.plugin) / rel_path)
+        suffix = rel_path.suffix.lower()
+        media_type = "video" if suffix in _VIDEO_FILE_EXTS else "audio"
        media_files.append(
            {
                "name": rel_path.name,
                "path": href,
                "size": size,
+                "media_type": media_type,
+                "is_video": media_type == "video",
+                "is_audio": media_type == "audio",
+                "is_browser_playable": suffix in _BROWSER_VIDEO_FILE_EXTS or suffix in _BROWSER_AUDIO_FILE_EXTS,
            },
        )

-    media_files.sort(key=lambda item: item["name"].lower())
+    media_files.sort(
+        key=lambda item: (
+            0 if item["is_video"] and item["is_browser_playable"] else 1 if item["is_audio"] and item["is_browser_playable"] else 2,
+            -int(item.get("size") or 0),
+            item["name"].lower(),
+        ),
+    )
    return media_files


@@ -202,16 +249,70 @@ def _is_root_snapshot_output_path(raw_output_path: str | None) -> bool:
    return normalized in ("", ".", "./", "/", "index.html", "index.json")


-def _build_snapshot_files_url(snapshot_id: str, request=None, config=None) -> str:
-    return build_snapshot_url(str(snapshot_id), "/?files=1", request=request, config=config)
+def _static_snapshot_base_url(context, snapshot) -> str:
+    start_dir = Path(context["STATIC_EXPORT_DIR"])
+    relative_path = os.path.relpath(Path(snapshot.output_dir), start=start_dir).replace(os.sep, "/")
+    if relative_path == ".":
+        return "."
+    return f"./{quote(relative_path, safe=_STATIC_URL_SAFE)}"


-def _build_snapshot_preview_url(snapshot_id: str, path: str = "", request=None, config=None, plugin: str = "") -> str:
+def _snapshot_base_url_for_context(context, snapshot) -> str:
+    if context.get("STATIC_EXPORT"):
+        return _static_snapshot_base_url(context, snapshot)
+    return get_snapshot_base_url(
+        str(_snapshot_id(snapshot)),
+        request=context.get("request"),
+        config=context.get("CONFIG"),
+    )
+
+
+def _snapshot_url_for_context(context, snapshot, path: str = "") -> str:
+    if not context.get("STATIC_EXPORT"):
+        return build_snapshot_url(
+            str(_snapshot_id(snapshot)),
+            path,
+            request=context.get("request"),
+            config=context.get("CONFIG"),
+        )
+
+    base_url = _static_snapshot_base_url(context, snapshot)
+    raw_path = str(path or "")
+    if not raw_path:
+        return base_url
+    path_part, separator, query = raw_path.lstrip("/").partition("?")
+    quoted_path = quote(path_part, safe=_STATIC_URL_SAFE)
+    suffix = f"?{query}" if separator else ""
+    return f"{base_url.rstrip('/')}/{quoted_path}{suffix}"
+
+
+def _build_snapshot_files_url(snapshot_id: str, request=None, config=None, base_url: str | None = None) -> str:
+    return (
+        f"{base_url.rstrip('/')}/index.jsonl"
+        if base_url
+        else build_snapshot_url(str(snapshot_id), "/?files=1", request=request, config=config)
+    )
+
+
+def _build_snapshot_preview_url(
+    snapshot_id: str,
+    path: str = "",
+    request=None,
+    config=None,
+    plugin: str = "",
+    base_url: str | None = None,
+) -> str:
    if path == "about:blank":
        return path
    if _is_root_snapshot_output_path(path):
-        return _build_snapshot_files_url(snapshot_id, request=request, config=config)
-    url = build_snapshot_url(str(snapshot_id), path, request=request, config=config)
+        return _build_snapshot_files_url(snapshot_id, request=request, config=config, base_url=base_url)
+    if base_url:
+        path_part, separator, query = str(path).lstrip("/").partition("?")
+        url = f"{base_url.rstrip('/')}/{quote(path_part, safe=_STATIC_URL_SAFE)}"
+        if separator:
+            url = f"{url}?{query}"
+    else:
+        url = build_snapshot_url(str(snapshot_id), path, request=request, config=config)
    path_parts = Path(path).parts
    plugin = get_plugin_name(plugin) if plugin else (path_parts[0] if len(path_parts) > 1 else "")
    has_plugin_preview = bool(plugin and get_plugin_template(plugin, "full", fallback=False))
@@ -299,6 +400,7 @@ def intcomma(value: int | str | None) -> str:
        return str(value or "")


+@register.inclusion_tag("admin/snapshots_grid.html", takes_context=True, name="snapshots_grid")
 def result_list(context, cl):
    """
    Monkey patched result
@@ -319,17 +421,6 @@ def result_list(context, cl):
    }


-@register.tag(name="snapshots_grid")
-def result_list_tag(parser, token):
-    return InclusionAdminNode(
-        parser,
-        token,
-        func=result_list,
-        template_name="snapshots_grid.html",
-        takes_context=True,
-    )
-
-
 _LOW_DISK_THRESHOLD_GB = 1.0
 _HIGH_MEMORY_THRESHOLD_PCT = 95.0
 _HIGH_LOAD_MULTIPLE = 3  # 15-min loadavg > 3 * cpu_count
@@ -389,6 +480,9 @@ def system_warnings_banner(context):
    after and reuse ``machine.detect.get_host_stats`` (the same function that
    populates the Machine admin page), cached for 30s.
    """
+    if context.get("first_admin_setup"):
+        return {"mode": ""}
+
    config = context.get("CONFIG")
    if config is None:
        from archivebox.config.common import get_config
@@ -398,7 +492,7 @@ def system_warnings_banner(context):
    if not config.BASE_URL:
        return get_setup_wizard_context(context.get("request"), config)
    mismatch = get_base_url_mismatch_context(context.get("request"), config)
-    if mismatch:
+    if mismatch and not _is_web_subdomain_alias(mismatch["browser_url"], mismatch["configured_base_url"]):
        return mismatch
    if config.IS_LOWER_SECURITY_MODE:
        return {"mode": "unsafe"}
@@ -438,16 +532,6 @@ def url_replace(context, **kwargs):
    return dict_.urlencode()


-@register.simple_tag
-def has_real_admin_users() -> bool:
-    """True if any non-``system`` superuser exists. Used by the login page to
-    only show the bootstrap hint (createsuperuser / ADMIN_USERNAME env vars)
-    when the collection still has no real admin."""
-    from django.contrib.auth.models import User
-
-    return User.objects.filter(is_superuser=True).exclude(username="system").exists()
-
-
@register.simple_tag(takes_context=True)
 def admin_base_url(context) -> str:
    return get_admin_base_url(request=context.get("request"), config=context.get("CONFIG"))
@@ -458,23 +542,35 @@ def web_base_url(context) -> str:
    return get_web_base_url(request=context.get("request"), config=context.get("CONFIG"))


+@register.simple_tag(takes_context=True)
+def static_export_root_url(context) -> str:
+    if not context.get("STATIC_EXPORT"):
+        return ""
+    relative_path = os.path.relpath(CONSTANTS.DATA_DIR, start=Path(context["STATIC_EXPORT_DIR"]))
+    relative_path = relative_path.replace(os.sep, "/")
+    return "./" if relative_path == "." else f"{quote(relative_path, safe=_STATIC_URL_SAFE)}/"
+
+
@register.simple_tag(takes_context=True)
 def snapshot_base_url(context, snapshot) -> str:
-    snapshot_id = _snapshot_id(snapshot)
-    return get_snapshot_base_url(str(snapshot_id), request=context.get("request"), config=context.get("CONFIG"))
+    return _snapshot_base_url_for_context(context, snapshot)


@register.simple_tag(takes_context=True)
 def snapshot_url(context, snapshot, path: str = "") -> str:
-    snapshot_id = _snapshot_id(snapshot)
-    return build_snapshot_url(str(snapshot_id), path, request=context.get("request"), config=context.get("CONFIG"))
+    return _snapshot_url_for_context(context, snapshot, path)


@register.simple_tag(takes_context=True)
 def snapshot_archiveresult_url(context, snapshot, plugin: str, filename: str) -> str:
    snapshot_id = str(_snapshot_id(snapshot))
    url_cache = snapshot.__dict__.setdefault("_snapshot_archiveresult_url_cache", {})
-    cache_key = (plugin, filename)
+    cache_key = (
+        plugin,
+        filename,
+        bool(context.get("STATIC_EXPORT")),
+        str(context.get("STATIC_EXPORT_DIR") or ""),
+    )
    if cache_key in url_cache:
        return url_cache[cache_key]

@@ -510,7 +606,7 @@ def snapshot_archiveresult_url(context, snapshot, plugin: str, filename: str) ->
        if not isinstance(file_info, dict) or int(file_info.get("size") or 0) <= 0:
            continue
        output_path = filename if file_info.get("root_relative") else f"{plugin}/{filename}"
-        url_cache[cache_key] = build_snapshot_url(snapshot_id, output_path, request=context.get("request"), config=context.get("CONFIG"))
+        url_cache[cache_key] = _snapshot_url_for_context(context, snapshot, output_path)
        return url_cache[cache_key]

    url_cache[cache_key] = ""
@@ -519,10 +615,7 @@ def snapshot_archiveresult_url(context, snapshot, plugin: str, filename: str) ->

@register.simple_tag(takes_context=True)
 def snapshot_index_row(context, link) -> str:
-    snapshot_id = str(_snapshot_id(link))
-    request = context.get("request")
-    config = context.get("CONFIG")
-    snapshot_base = get_snapshot_base_url(snapshot_id, request=request, config=config)
+    snapshot_base = _snapshot_base_url_for_context(context, link)

    status = getattr(link, "status", None) or "unknown"
    bookmarked_at = getattr(link, "bookmarked_at", None)
@@ -540,7 +633,7 @@ def snapshot_index_row(context, link) -> str:
        title_time = f"Bookmarked: ({timestamp})"

    url = getattr(link, "url", "") or ""
-    title = getattr(link, "title", "") or ""
+    title = unescape(getattr(link, "title", "") or "")
    is_pending = status in {"queued", "started", "backoff"}
    title_text = title or url
    tags_str = link.tags_str() if callable(getattr(link, "tags_str", None)) else getattr(link, "tags_str", "")
@@ -551,22 +644,28 @@ def snapshot_index_row(context, link) -> str:
        tag_cell = '<span class="empty-value">...</span>'

    num_outputs = int(getattr(link, "num_outputs", 0) or 0)
-    icons = link.icons() if callable(getattr(link, "icons", None)) else getattr(link, "icons", "")
+    if context.get("STATIC_EXPORT") and callable(getattr(link, "icons", None)):
+        icons = link.icons(path=quote(link.static_archive_path, safe=_STATIC_URL_SAFE), prefix="./", quote_paths=True)
+    else:
+        icons = link.icons() if callable(getattr(link, "icons", None)) else getattr(link, "icons", "")
    icons_cell = str(icons) if icons else '<span class="empty-value">...</span>'
    archive_size = int(getattr(link, "archive_size", 0) or 0)
    size_cell = file_size(archive_size) if archive_size else '<span class="empty-value">...</span>'
    output_plural = "" if num_outputs == 1 else "s"
+    files_url = _snapshot_url_for_context(context, link, "index.jsonl") if context.get("STATIC_EXPORT") else f"{snapshot_base}/?files=1"

    if is_pending:
-        preview_html = (
-            '<span class="snapshot-preview snapshot-preview-spinner" aria-label="Archiving in progress">'
-            f'<img src="{escape(static("spinner.gif"))}" alt="" decoding="async" loading="lazy">'
-            "</span>"
-        )
+        preview_html = '<span class="snapshot-preview snapshot-preview-spinner" aria-label="Archiving in progress"></span>'
+        if not context.get("STATIC_EXPORT"):
+            preview_html = (
+                '<span class="snapshot-preview snapshot-preview-spinner" aria-label="Archiving in progress">'
+                f'<img src="{escape(static("spinner.gif"))}" alt="" decoding="async" loading="lazy">'
+                "</span>"
+            )
    elif "_public_preview_paths" in link.__dict__:
        preview_paths = list(getattr(link, "_public_preview_paths", []) or [])
        if preview_paths:
-            preview_urls = [build_snapshot_url(snapshot_id, path, request=request, config=config) for path in preview_paths]
+            preview_urls = [_snapshot_url_for_context(context, link, path) for path in preview_paths]
            preview_html = (
                f'<img src="{escape(preview_urls[0])}" '
                f'data-fallbacks="{escape(",".join(preview_urls[1:]))}" '
@@ -596,7 +695,7 @@ def snapshot_index_row(context, link) -> str:
    if "_public_favicon_paths" in link.__dict__:
        favicon_paths = list(getattr(link, "_public_favicon_paths", []) or [])
        if favicon_paths:
-            favicon_urls = [build_snapshot_url(snapshot_id, path, request=request, config=config) for path in favicon_paths]
+            favicon_urls = [_snapshot_url_for_context(context, link, path) for path in favicon_paths]
            favicon_html = (
                f'<img src="{escape(favicon_urls[0])}" '
                f'data-fallbacks="{escape(",".join(favicon_urls[1:]))}" '
@@ -657,7 +756,7 @@ def snapshot_index_row(context, link) -> str:
        </span>
    </td>
    <td class="snapshot-size-cell">
-        <a href="{escape(snapshot_base)}/?files=1" title="View archived files">
+        <a href="{escape(files_url)}" title="View archived file manifest">
            {size_cell}
        </a>
        <small>{num_outputs} output{output_plural}</small>
@@ -677,6 +776,7 @@ def snapshot_preview_url(context, snapshot, path: str = "", result=None) -> str:
        request=context.get("request"),
        config=context.get("CONFIG"),
        plugin=plugin,
+        base_url=_snapshot_base_url_for_context(context, snapshot) if context.get("STATIC_EXPORT") else None,
    )


@@ -716,24 +816,26 @@ def plugin_card(context, result) -> str:

    # Use embed_path() for the display path
    raw_output_path = result.embed_path() or ""
-    output_url = build_snapshot_url(
-        str(result.snapshot_id),
-        raw_output_path or "",
-        request=context.get("request"),
-        config=context.get("CONFIG"),
-    )
+    output_url = _snapshot_url_for_context(context, result.snapshot, raw_output_path or "")

    icon_html = get_plugin_icon(plugin)
    plugin_lower = (plugin or "").lower()
    media_file_count = _count_media_files(result) if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl") else 0
    media_files = _list_media_files(result) if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl") else []
+    if context.get("STATIC_EXPORT") and media_files:
+        media_files = [
+            item
+            for item in media_files
+            if (media_path := Path(str(item.get("path") or "")))
+            and not media_path.is_absolute()
+            and ".." not in media_path.parts
+            and (Path(result.snapshot_dir) / media_path).is_file()
+        ]
+        media_file_count = len(media_files)
    if media_files:
-        snapshot_id = str(result.snapshot_id)
-        request = context.get("request")
-        config = context.get("CONFIG")
        for item in media_files:
            path = item.get("path") or ""
-            item["url"] = build_snapshot_url(snapshot_id, path, request=request, config=config) if path else ""
+            item["url"] = _snapshot_url_for_context(context, result.snapshot, path) if path else ""

    output_lower = (raw_output_path or "").lower()
    force_text_preview = output_lower.endswith(_TEXT_PREVIEW_EXTS)
@@ -811,12 +913,7 @@ def plugin_full(context, result) -> str:
        raw_output_path = result.embed_path() or ""
    if _is_root_snapshot_output_path(raw_output_path):
        return ""
-    output_url = build_snapshot_url(
-        str(result.snapshot_id),
-        raw_output_path,
-        request=context.get("request"),
-        config=context.get("CONFIG"),
-    )
+    output_url = _snapshot_url_for_context(context, result.snapshot, raw_output_path)

    try:
        tpl = template.Template(template_str)
@@ -1,6 +1,5 @@
 __package__ = "archivebox.core"

-import sys
 from importlib.util import find_spec

 from django.conf import settings
@@ -10,7 +9,6 @@ from django.views.generic.base import RedirectView
 from django.http import HttpRequest

 from archivebox.config.constants import CONSTANTS
-from archivebox.config.common import get_config
 from archivebox.misc.serve_static import serve_static

 from archivebox.core.admin_site import archivebox_admin
@@ -30,9 +28,6 @@ from archivebox.progressmonitor.views import live_progress_view
 from archivebox.search.views import public_snapshot_search_stream_view
 from archivebox.opencode.views import opencode_proxy_view

-CONFIG = get_config()
-DEBUG = CONFIG.DEBUG or ("--debug" in sys.argv)
-
 urlpatterns = [
    re_path(r"^static/(?P<path>.*)$", serve_static),
    path("robots.txt", static.serve, {"document_root": CONSTANTS.STATIC_DIR, "path": "robots.txt"}),
@@ -3,7 +3,6 @@ __package__ = "archivebox.core"
 import json
 import os
 import posixpath
-from glob import escape, glob
 from pathlib import Path
 from typing import ClassVar, cast
 from urllib.parse import quote, urlparse
@@ -22,12 +21,11 @@ from django.core.paginator import InvalidPage
 from django.db.models import Case, IntegerField, Q, Value, When
 from django.http import Http404, HttpRequest, HttpResponse, HttpResponseForbidden, QueryDict
 from django.shortcuts import redirect, render
-from django.utils import timezone
 from django.utils.decorators import method_decorator
 from django.utils.html import format_html, format_html_join
 from django.utils.safestring import mark_safe
 from django.views import View
-from django.views.decorators.csrf import csrf_exempt
+from django.views.decorators.csrf import csrf_exempt, csrf_protect
 from django.views.generic import FormView
 from django.views.generic.list import ListView

@@ -55,7 +53,6 @@ from archivebox.core.permissions import (
    can_view_snapshot,
    direct_snapshots_queryset,
    filter_personas_by_permissions,
-    get_snapshot_permissions,
    is_admin_user,
    public_snapshots_queryset,
 )
@@ -70,15 +67,12 @@ from archivebox.core.routes_util import (
    host_matches,
 )
 from archivebox.crawls.models import Crawl
-from archivebox.misc.logging_util import printable_filesize
 from archivebox.misc.paginators import AcceleratedPaginator
 from archivebox.misc.serve_static import serve_static_with_byterange_support
 from archivebox.misc.util import (
    base_url,
    filter_queryset_by_uuid_substring,
-    htmldecode,
    sanitize_html_text,
-    ts_to_date_str,
    urldecode,
    validate_url,
    without_fragment,
@@ -86,7 +80,7 @@ from archivebox.misc.util import (
 from archivebox.plugins.discovery import get_plugin_name, get_plugin_template
 from archivebox.plugins.forms import get_plugin_config_binary_urls
 from archivebox.plugins.views import get_config_definition_link
-from archivebox.progressmonitor.views import live_progress_view, progress_endpoint
+from archivebox.progressmonitor.views import live_progress_view
 from archivebox.search.config import (
    get_search_mode,
    get_search_mode_backend,
@@ -113,18 +107,20 @@ def _find_snapshot_by_ref(snapshot_ref: str) -> Snapshot | None:
    if not lookup:
        return None

+    snapshots = Snapshot.objects.select_related("crawl", "crawl__created_by")
+
    if len(lookup) == 12 and "-" not in lookup:
-        return Snapshot.objects.filter(id__endswith=lookup).order_by("-created_at", "-downloaded_at").first()
+        return snapshots.filter(id__endswith=lookup).order_by("-created_at", "-downloaded_at").first()

    try:
-        return Snapshot.objects.get(pk=lookup)
+        return snapshots.get(pk=lookup)
    except Snapshot.DoesNotExist:
        try:
-            return Snapshot.objects.get(id__startswith=lookup)
+            return snapshots.get(id__startswith=lookup)
        except Snapshot.DoesNotExist:
            return None
        except Snapshot.MultipleObjectsReturned:
-            return Snapshot.objects.filter(id__startswith=lookup).first()
+            return snapshots.filter(id__startswith=lookup).first()


 def _admin_login_redirect_or_forbidden(request: HttpRequest):
@@ -255,6 +251,9 @@ class SnapshotReplayAuthView(View):
 class HomepageView(View):
    def get(self, request):
        request_config = get_request_config(request)
+        if not request_config.BASE_URL:
+            return _admin_login_redirect_or_forbidden(request)
+
        if request.user.is_authenticated and request_config.CONTROL_PLANE_ENABLED:
            return redirect("/admin/core/snapshot/")

@@ -268,7 +267,7 @@ class SnapshotView(View):
    # render static html index from filesystem archive/<timestamp>/index.html

    @staticmethod
-    def find_snapshots_for_url(path: str):
+    def find_snapshots_for_url(path: str, *, allow_fallback: bool = True):
        """Return a queryset of snapshots matching a URL-ish path. URL only — never tries ID matching.

        Use ``find_snapshots_for_id`` separately if you also want to match by snapshot UUID.
@@ -294,7 +293,7 @@ class SnapshotView(View):
        if path.startswith(("http://", "https://")):
            # exact url match (indexed) — fastest path
            qs = Snapshot.objects.filter(_fragmentless_url_query(path))
-            if qs.exists():
+            if not allow_fallback or qs.exists():
                return qs
            normalized = normalized.split("://", 1)[1]

@@ -328,161 +327,11 @@ class SnapshotView(View):

    @staticmethod
    def render_live_index(request, snapshot):
-        TITLE_LOADING_MSG = "Not yet archived..."
-        from archivebox.core.widgets import TagEditorWidget
-
-        # Reuse the middleware-attached config; never re-bootstrap from env + plugin
-        # schemas just to render a snapshot page (that pays ~30ms for no reason).
-        runtime_config = get_request_config(request)
-        snapshot._runtime_config = runtime_config
-        snapshot_permissions = get_snapshot_permissions(snapshot)
-        hidden_card_plugins = {"archivedotorg", "favicon", "title"}
-        outputs = [
-            out
-            for out in snapshot.discover_outputs(include_filesystem_fallback=True)
-            if (out.get("size") or 0) > 0 and out.get("name") not in hidden_card_plugins
-        ]
-        archiveresults = {}
-        for output in outputs:
-            current = archiveresults.get(output["name"])
-            if current is None or (output.get("size") or 0) > (current.get("size") or 0):
-                archiveresults[output["name"]] = output
-        hash_index = snapshot.hashes_index
-        accounted_entries: set[str] = set()
-        for output in outputs:
-            output_name = output.get("name") or ""
-            if output_name:
-                accounted_entries.add(output_name)
-            output_path = output.get("path") or ""
-            if not output_path:
-                continue
-            parts = Path(output_path).parts
-            if parts:
-                accounted_entries.add(parts[0])
-
-        loose_items, failed_items = snapshot.get_detail_page_auxiliary_items(outputs, hidden_card_plugins=hidden_card_plugins)
-        preview_priority = [
-            "singlefile",
-            "screenshot",
-            "wget",
-            "dom",
-            "pdf",
-            "readability",
-        ]
-        preferred_types = tuple(preview_priority)
-        output_order = {result_type: index for index, result_type in enumerate(archiveresults.keys())}
-
-        best_result = {"path": "about:blank", "result": None}
-        for result_type in preferred_types:
-            if result_type in archiveresults:
-                best_result = archiveresults[result_type]
-                break
-
-        related_snapshots_qs = (
-            SnapshotView.find_snapshots_for_url(snapshot.url)
-            .select_related("crawl", "crawl__created_by")
-            .annotate(
-                num_outputs_cached=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.SUCCEEDED),
-                num_failures_cached=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.FAILED),
-            )
+        return render(
+            template_name="core/snapshot.html",
+            request=request,
+            context=snapshot.get_html_details_context(request=request),
        )
-        related_snapshots = list(
-            related_snapshots_qs.exclude(id=snapshot.id).order_by("-bookmarked_at", "-created_at", "-timestamp")[:25],
-        )
-        related_years_map: dict[int, list[Snapshot]] = {}
-        for snap in [snapshot, *related_snapshots]:
-            snap_dt = snap.bookmarked_at or snap.created_at or snap.downloaded_at
-            if not snap_dt:
-                continue
-            related_years_map.setdefault(snap_dt.year, []).append(snap)
-        related_years = []
-        for year, snaps in related_years_map.items():
-            snaps_sorted = sorted(
-                snaps,
-                key=lambda s: s.bookmarked_at or s.created_at or s.downloaded_at or timezone.now(),
-                reverse=True,
-            )
-            related_years.append(
-                {
-                    "year": year,
-                    "latest": snaps_sorted[0],
-                    "snapshots": snaps_sorted,
-                },
-            )
-        related_years.sort(key=lambda item: item["year"], reverse=True)
-
-        warc_path = next(
-            (rel_path for rel_path in hash_index if rel_path.startswith("warc/") and ".warc" in Path(rel_path).name),
-            "warc/",
-        )
-
-        ordered_outputs = sorted(
-            archiveresults.values(),
-            key=lambda r: (
-                preferred_types.index(r["name"]) if r["name"] in preferred_types else len(preferred_types),
-                output_order.get(r["name"], len(output_order)),
-            ),
-        )
-        if best_result["path"] == "about:blank" and ordered_outputs:
-            best_result = ordered_outputs[0]
-        non_compact_outputs = [out for out in ordered_outputs if not out.get("is_compact") and not out.get("is_metadata")]
-        compact_outputs = [out for out in ordered_outputs if out.get("is_compact") or out.get("is_metadata")]
-        tag_widget = TagEditorWidget()
-        output_size = sum(int(out.get("size") or 0) for out in ordered_outputs)
-        has_outputs = bool(ordered_outputs)
-        is_archived = has_outputs or snapshot.status == Snapshot.StatusChoices.SEALED
-        snapshot_status = str(snapshot.status or "").lower()
-        status_label_by_state = {
-            "queued": ("queued", "info"),
-            "started": ("running", "warning"),
-            "paused": ("paused", "default"),
-            "sealed": ("archived", "success"),
-        }
-        if has_outputs and not is_archived:
-            status_label, status_color = ("partial", "warning")
-        elif has_outputs:
-            status_label, status_color = ("archived", "success")
-        else:
-            status_label, status_color = status_label_by_state.get(snapshot_status, ("not yet archived", "danger"))
-
-        context = {
-            "id": str(snapshot.id),
-            "snapshot_id": str(snapshot.id),
-            "progress_endpoint": progress_endpoint("snapshot", snapshot.id),
-            "progress_auto_expand": snapshot_status in {"queued", "started", "paused"},
-            "url": snapshot.url,
-            "archive_path": snapshot.archive_path_from_db,
-            "title": htmldecode(snapshot.resolved_title or (snapshot.base_url if is_archived else TITLE_LOADING_MSG)),
-            "extension": snapshot.extension or "html",
-            "tags": snapshot.tags_str() or "untagged",
-            "size": printable_filesize(output_size) if output_size else "—",
-            "status": status_label,
-            "status_color": status_color,
-            "snapshot_state": snapshot_status,
-            "has_outputs": has_outputs,
-            "snapshot_permissions": snapshot_permissions,
-            "snapshot_permissions_icon": {
-                "public": "👥",
-                "unlisted": "🔗",
-                "private": "🔒",
-            }.get(snapshot_permissions, "👥"),
-            "bookmarked_date": snapshot.bookmarked_date,
-            "downloaded_datestr": snapshot.downloaded_datestr,
-            "num_outputs": snapshot.num_outputs,
-            "num_failures": snapshot.num_failures,
-            "oldest_archive_date": ts_to_date_str(snapshot.oldest_archive_date),
-            "warc_path": warc_path,
-            "archiveresults": [*non_compact_outputs, *compact_outputs],
-            "best_result": best_result,
-            "snapshot": snapshot,  # Pass the snapshot object for template tags
-            "CONFIG": runtime_config,
-            "related_snapshots": related_snapshots,
-            "related_years": related_years,
-            "loose_items": loose_items,
-            "failed_items": failed_items,
-            "title_tags": [{"name": tag.name, "style": tag_widget._tag_style(tag.name)} for tag in snapshot.tags.all().order_by("name")],
-        }
-        return render(template_name="core/snapshot.html", request=request, context=context)

    def get(self, request, path):
        snapshot = None
@@ -572,7 +421,7 @@ class SnapshotView(View):
                        slug,
                    )
                    + snapshot_hrefs
-                    + format_html('</pre><br/>Choose a Snapshot to proceed or go back to the <a href="/" target="_top">Main Index</a>'),
+                    + mark_safe('</pre><br/>Choose a Snapshot to proceed or go back to the <a href="/" target="_top">Main Index</a>'),
                    content_type="text/html",
                    status=404,
                )
@@ -635,8 +484,14 @@ class SnapshotView(View):
                return id_qs
            return SnapshotView.find_snapshots_for_url(slug)

+        snapshots = direct_snapshots_queryset(request, _resolve_snapshots_for_slug(path))
        try:
-            snapshot = direct_snapshots_queryset(request, _resolve_snapshots_for_slug(path)).get()
+            if "://" in path:
+                snapshot = snapshots.order_by("-bookmarked_at").first()
+                if snapshot is None:
+                    raise Snapshot.DoesNotExist
+            else:
+                snapshot = snapshots.get()
        except Snapshot.DoesNotExist:
            return HttpResponse(
                format_html(
@@ -655,7 +510,6 @@ class SnapshotView(View):
                status=404,
            )
        except Snapshot.MultipleObjectsReturned:
-            snapshots = direct_snapshots_queryset(request, _resolve_snapshots_for_slug(path))
            snapshot_hrefs = mark_safe("<br/>").join(
                format_html(
                    '{} <code style="font-size: 0.8em">{}</code> <a href="/{}/index.html"><b><code>{}</code></b></a> {} <b>{}</b>',
@@ -674,12 +528,12 @@ class SnapshotView(View):
                    base_url(path),
                )
                + snapshot_hrefs
-                + format_html('</pre><br/>Choose a Snapshot to proceed or go back to the <a href="/" target="_top">Main Index</a>'),
+                + mark_safe('</pre><br/>Choose a Snapshot to proceed or go back to the <a href="/" target="_top">Main Index</a>'),
                content_type="text/html",
                status=404,
            )

-        target_path = f"/{snapshot.archive_path}/index.html"
+        target_path = build_snapshot_url(str(snapshot.id), "index.html", request=request)
        query = request.META.get("QUERY_STRING")
        if query:
            target_path = f"{target_path}?{query}"
@@ -719,12 +573,13 @@ class SnapshotPathView(View):
            # fuzzy lookup by date + domain/url (most recent)
            username_lookup = "system" if username == "web" else username
            if requested_url:
-                qs = (
+                qs = direct_snapshots_queryset(
+                    request,
                    SnapshotView.find_snapshots_for_url(requested_url)
                    .select_related("crawl", "crawl__created_by")
                    .filter(
                        crawl__created_by__username=username_lookup,
-                    )
+                    ),
                )
            else:
                qs = snapshots_qs.filter(crawl__created_by__username=username_lookup)
@@ -836,26 +691,27 @@ def _resolve_archiveresult_relpath(snapshot: Snapshot, rel_path: str) -> tuple[s

    plugin = parts[0]
    plugin_relpath = posixpath.join(*parts[1:])
-    result = (
-        ArchiveResult.objects.filter(snapshot=snapshot, plugin=plugin, status=ArchiveResult.StatusChoices.SUCCEEDED)
-        .only("plugin", "output_files")
-        .first()
+    results = list(
+        ArchiveResult.objects.filter(
+            snapshot=snapshot,
+            plugin=plugin,
+            status=ArchiveResult.StatusChoices.SUCCEEDED,
+        ).only("plugin", "output_files"),
    )
-    if not result:
+    if not results:
        return rel_path, None
-    if not result.output_files:
-        return rel_path, result

-    output_files = result.output_files or {}
-    for candidate in (plugin_relpath, rel_path):
-        file_info = output_files.get(candidate)
-        if not isinstance(file_info, dict):
-            continue
-        if file_info.get("root_relative"):
-            return candidate, result
-        return rel_path, result
+    for result in results:
+        output_files = result.output_files or {}
+        for candidate in (plugin_relpath, rel_path):
+            file_info = output_files.get(candidate)
+            if not isinstance(file_info, dict):
+                continue
+            if file_info.get("root_relative"):
+                return candidate, result
+            return rel_path, result

-    return rel_path, result
+    return rel_path, results[0]


 def _plugin_full_preview_response(
@@ -909,6 +765,11 @@ def _plugin_full_preview_response(
    response.headers["X-Content-Type-Options"] = "nosniff"
    response.headers["X-ArchiveBox-Security-Mode"] = request.archivebox_config.SERVER_SECURITY_MODE
    response.headers["Referrer-Policy"] = "no-referrer"
+    # Trusted viewers live on the snapshot origin, but the collection UI lives
+    # on web.* (and can be embedded by admin.*). Permit only those configured
+    # origins, not arbitrary sites or other snapshots. This does not grant the
+    # archived document access to the parent frame or its admin session cookies.
+    viewer_origins = " ".join(dict.fromkeys((build_web_url(request=request), build_admin_url(request=request))))
    response.headers["Content-Security-Policy"] = (
        "default-src 'self' data: blob:; "
        "script-src 'self' 'unsafe-inline' 'unsafe-eval' data: blob:; "
@@ -922,116 +783,62 @@ def _plugin_full_preview_response(
        "object-src 'none'; "
        "base-uri 'none'; "
        "form-action 'none'; "
-        "frame-ancestors 'self';"
+        f"frame-ancestors 'self' {viewer_origins};"
    )
    return response


-def _coerce_sort_timestamp(value: str | float | None) -> float:
-    if value is None:
-        return 0.0
-    try:
-        return float(value)
-    except (TypeError, ValueError):
-        return 0.0
-
-
-def _snapshot_sort_key(match_path: str, cache: dict[str, float]) -> tuple[float, str]:
-    parts = Path(match_path).parts
-    date_str = ""
-    snapshot_id = ""
-    try:
-        idx = parts.index("snapshots")
-        date_str = parts[idx + 1]
-        snapshot_id = parts[idx + 3]
-    except (IndexError, ValueError):
-        return (_coerce_sort_timestamp(date_str), match_path)
-
-    if snapshot_id not in cache:
-        snapshot = Snapshot.objects.filter(id=snapshot_id).only("bookmarked_at", "created_at", "downloaded_at", "timestamp").first()
-        if snapshot:
-            snap_dt = snapshot.bookmarked_at or snapshot.created_at or snapshot.downloaded_at
-            cache[snapshot_id] = snap_dt.timestamp() if snap_dt else _coerce_sort_timestamp(snapshot.timestamp)
-        else:
-            cache[snapshot_id] = _coerce_sort_timestamp(date_str)
-
-    return (cache[snapshot_id], match_path)
-
-
-def _snapshot_id_from_replay_path(path: Path) -> str | None:
-    parts = path.parts
-    try:
-        responses_idx = parts.index("responses")
-    except ValueError:
-        return None
-    return parts[responses_idx - 1] if responses_idx > 0 else None
-
-
-def _replay_path_visible(request: HttpRequest, path: Path) -> bool:
-    snapshot_id = _snapshot_id_from_replay_path(path)
-    if not snapshot_id:
-        return False
-    snapshot = Snapshot.objects.filter(id=snapshot_id).select_related("crawl", "crawl__created_by").first()
-    if not snapshot or (not can_view_snapshot(request, snapshot) and not _has_replay_cookie(request, snapshot)):
-        return False
-    request.archivebox_config = get_request_config(request, resolve_plugins=False)
-    return True
-
-
-def _latest_response_match(request: HttpRequest, domain: str, rel_path: str, *, data_root: Path) -> tuple[Path, Path] | None:
-    if not domain or not rel_path:
-        return None
-    domain = domain.split(":", 1)[0].lower()
-    # TODO: optimize by querying output_files in DB instead of globbing filesystem
-    escaped_domain = escape(domain)
-    escaped_path = escape(rel_path)
-    pattern = str(data_root / "*" / "snapshots" / "*" / escaped_domain / "*" / "responses" / escaped_domain / escaped_path)
-    matches = glob(pattern)
-    if not matches:
-        return None
-
-    sort_cache: dict[str, float] = {}
-    best_paths = sorted(matches, key=lambda match_path: _snapshot_sort_key(match_path, sort_cache), reverse=True)
-    best_path = next((Path(match_path) for match_path in best_paths if _replay_path_visible(request, Path(match_path))), None)
-    if best_path is None:
-        return None
-    parts = best_path.parts
-    try:
-        responses_idx = parts.index("responses")
-    except ValueError:
-        return None
-    responses_root = Path(*parts[: responses_idx + 1])
-    rel_to_root = Path(*parts[responses_idx + 1 :])
-    return responses_root, rel_to_root
-
-
-def _latest_responses_root(request: HttpRequest, domain: str, *, data_root: Path) -> Path | None:
+def _visible_response_snapshots_for_domain(request: HttpRequest, domain: str) -> list[Snapshot]:
    if not domain:
-        return None
-    domain = domain.split(":", 1)[0].lower()
-    escaped_domain = escape(domain)
-    pattern = str(data_root / "*" / "snapshots" / "*" / escaped_domain / "*" / "responses" / escaped_domain)
-    matches = glob(pattern)
-    if not matches:
-        return None
-
-    sort_cache: dict[str, float] = {}
-    best_paths = sorted(matches, key=lambda match_path: _snapshot_sort_key(match_path, sort_cache), reverse=True)
-    return next((Path(match_path) for match_path in best_paths if _replay_path_visible(request, Path(match_path))), None)
-
-
-def _latest_snapshot_for_domain(request: HttpRequest, domain: str) -> Snapshot | None:
-    if not domain:
-        return None
+        return []

    requested_domain = domain.split(":", 1)[0].lower()
-    snapshots = direct_snapshots_queryset(
-        request,
-        SnapshotView.find_snapshots_for_url(f"https://{requested_domain}"),
-    ).order_by("-bookmarked_at", "-created_at", "-timestamp")
+    roots = (f"http://{requested_domain}", f"https://{requested_domain}")
+    domain_query = Q(url__in=roots)
+
+    from archivebox.misc.db import is_postgres
+
+    postgres = is_postgres()
+    for root in roots:
+        for separator in ("/", "?", "#"):
+            prefix = f"{root}{separator}"
+            if postgres:
+                domain_query |= Q(url__startswith=prefix)
+            else:
+                domain_query |= Q(url__gte=prefix, url__lt=f"{prefix}\U0010ffff")
+
+    candidates = (
+        Snapshot.objects.filter(domain_query)
+        .select_related("crawl", "crawl__created_by")
+        .order_by("-bookmarked_at", "-created_at", "-timestamp")
+    )
+    return [snapshot for snapshot in candidates if can_view_snapshot(request, snapshot) or _has_replay_cookie(request, snapshot)]
+
+
+def _latest_response_match(
+    snapshots: list[Snapshot],
+    domain: str,
+    rel_path: str,
+) -> tuple[Snapshot, Path, Path] | None:
+    if not domain or not rel_path:
+        return None
+    requested_domain = domain.split(":", 1)[0].lower()
+    rel_to_root = Path(rel_path)
    for snapshot in snapshots:
-        if Snapshot.extract_domain_from_url(snapshot.url).lower() == requested_domain:
-            return snapshot
+        responses_root = Path(snapshot.output_dir) / "responses" / requested_domain
+        if (responses_root / rel_to_root).exists():
+            return snapshot, responses_root, rel_to_root
+    return None
+
+
+def _latest_responses_root(snapshots: list[Snapshot], domain: str) -> tuple[Snapshot, Path] | None:
+    if not domain:
+        return None
+    requested_domain = domain.split(":", 1)[0].lower()
+    for snapshot in snapshots:
+        responses_root = Path(snapshot.output_dir) / "responses" / requested_domain
+        if responses_root.is_dir():
+            return snapshot, responses_root
    return None


@@ -1158,23 +965,22 @@ def _serve_original_domain_replay(request: HttpRequest, domain: str, path: str =
    if rel_path is None:
        raise Http404

-    domain = domain.lower()
-    match = _latest_response_match(request, domain, rel_path, data_root=CONSTANTS.USERS_DIR)
+    domain = domain.split(":", 1)[0].lower()
+    snapshots = _visible_response_snapshots_for_domain(request, domain)
+    match = _latest_response_match(snapshots, domain, rel_path)
    if not match and "." not in Path(rel_path).name:
        index_path = f"{rel_path.rstrip('/')}/index.html"
-        match = _latest_response_match(request, domain, index_path, data_root=CONSTANTS.USERS_DIR)
+        match = _latest_response_match(snapshots, domain, index_path)
    if not match and "." not in Path(rel_path).name:
        html_path = f"{rel_path}.html"
-        match = _latest_response_match(request, domain, html_path, data_root=CONSTANTS.USERS_DIR)
+        match = _latest_response_match(snapshots, domain, html_path)

-    responses_root = match[0] if match else _latest_responses_root(request, domain, data_root=CONSTANTS.USERS_DIR)
+    root_match = (match[0], match[1]) if match else _latest_responses_root(snapshots, domain)
+    responses_root = root_match[1] if root_match else None
    if request_config.USES_SUBDOMAIN_ROUTING:
-        snapshot_id = _snapshot_id_from_replay_path(responses_root) if responses_root else None
-        snapshot = Snapshot.objects.filter(id=snapshot_id).first() if snapshot_id else None
-        if snapshot is None and requested_root_index:
-            snapshot = _latest_snapshot_for_domain(request, domain)
+        snapshot = root_match[0] if root_match else (snapshots[0] if requested_root_index and snapshots else None)
        if snapshot:
-            snapshot_path = f"responses/{match[1]}" if match else path
+            snapshot_path = f"responses/{domain}/{match[2]}" if match else path
            target = build_snapshot_url(str(snapshot.id), snapshot_path, request=request, config=request_config)
            if request.META.get("QUERY_STRING"):
                target = f"{target}?{request.META['QUERY_STRING']}"
@@ -1182,7 +988,7 @@ def _serve_original_domain_replay(request: HttpRequest, domain: str, path: str =

    show_indexes = bool(request.GET.get("files"))
    if match:
-        responses_root, rel_to_root = match
+        _snapshot, responses_root, rel_to_root = match
        response = _serve_responses_path(request, responses_root, str(rel_to_root), show_indexes)
        if response is not None:
            return response
@@ -1193,9 +999,8 @@ def _serve_original_domain_replay(request: HttpRequest, domain: str, path: str =
            return response

    if requested_root_index and not show_indexes:
-        snapshot = _latest_snapshot_for_domain(request, domain)
-        if snapshot:
-            return SnapshotView.render_live_index(request, snapshot)
+        if snapshots:
+            return SnapshotView.render_live_index(request, snapshots[0])

    if request_config.PUBLIC_ADD_VIEW or request.user.is_authenticated:
        target_url = _original_request_url(domain, path, request.META.get("QUERY_STRING", ""))
@@ -1391,33 +1196,20 @@ class PublicIndexView(ListView):
                    return filename
                return f"{result.plugin}/{filename}"

-            for result in (
+            for snapshot_id, plugin, status in (
                ArchiveResult.objects.filter(
                    snapshot_id__in=icons_by_snapshot.keys(),
                )
                .exclude(plugin="")
-                .only("snapshot_id", "plugin", "status", "output_files", "output_str")
+                .values_list("snapshot_id", "plugin", "status")
                .iterator(chunk_size=1000)
            ):
-                snapshot_id = result.snapshot_id
-                plugin = result.plugin
-                status = result.status
                snapshot_key = str(snapshot_id)
                progress = progress_by_snapshot[snapshot_key]
                progress["total"] += 1
                if status == ArchiveResult.StatusChoices.SUCCEEDED:
                    icons_by_snapshot[snapshot_key].add(plugin)
                    progress["succeeded"] += 1
-                    if plugin in preview_candidates:
-                        plugin_rank = preview_plugin_order[plugin]
-                        for filename_rank, filename in enumerate(preview_candidates[plugin]):
-                            output_path = result_output_path(result, filename)
-                            if output_path:
-                                preview_paths_by_snapshot[snapshot_key].append((plugin_rank, filename_rank, output_path))
-                    elif plugin == "favicon":
-                        output_path = result_output_path(result, "favicon.ico")
-                        if output_path:
-                            favicon_paths_by_snapshot[snapshot_key].append(output_path)
                elif status == ArchiveResult.StatusChoices.FAILED:
                    progress["failed"] += 1
                elif status == ArchiveResult.StatusChoices.STARTED:
@@ -1427,6 +1219,27 @@ class PublicIndexView(ListView):
                elif status == ArchiveResult.StatusChoices.NORESULTS:
                    progress["noresults"] += 1

+            for result in (
+                ArchiveResult.objects.filter(
+                    snapshot_id__in=icons_by_snapshot.keys(),
+                    status=ArchiveResult.StatusChoices.SUCCEEDED,
+                    plugin__in=(*preview_candidates, "favicon"),
+                )
+                .only("snapshot_id", "plugin", "output_files")
+                .iterator(chunk_size=1000)
+            ):
+                snapshot_key = str(result.snapshot_id)
+                if result.plugin in preview_candidates:
+                    plugin_rank = preview_plugin_order[result.plugin]
+                    for filename_rank, filename in enumerate(preview_candidates[result.plugin]):
+                        output_path = result_output_path(result, filename)
+                        if output_path:
+                            preview_paths_by_snapshot[snapshot_key].append((plugin_rank, filename_rank, output_path))
+                elif result.plugin == "favicon":
+                    output_path = result_output_path(result, "favicon.ico")
+                    if output_path:
+                        favicon_paths_by_snapshot[snapshot_key].append(output_path)
+
        for snapshot in snapshots:
            snapshot._icons_compact = True
            snapshot._icons_archive_results = icons_by_snapshot.get(str(snapshot.id), set())
@@ -1477,6 +1290,8 @@ class PublicIndexView(ListView):
            return _admin_login_redirect_or_forbidden(self.request)


+# The public web host intentionally has no CSRF cookie. Authenticated POSTs are
+# re-protected in post(); integrations should use the token-authenticated API.
@method_decorator(csrf_exempt, name="dispatch")
 class AddView(UserPassesTestMixin, FormView):
    template_name = "add.html"
@@ -1499,6 +1314,11 @@ class AddView(UserPassesTestMixin, FormView):
    def test_func(self):
        return get_request_config(self.request).PUBLIC_ADD_VIEW or self.request.user.is_authenticated

+    def post(self, request: HttpRequest, *args: object, **kwargs: object):
+        if request.user.is_authenticated:
+            return csrf_protect(super().post)(request, *args, **kwargs)
+        return super().post(request, *args, **kwargs)
+
    def _can_override_crawl_config(self) -> bool:
        user = self.request.user
        return bool(user.is_authenticated and user.is_active and user.is_superuser)
@@ -1532,7 +1352,7 @@ class AddView(UserPassesTestMixin, FormView):
        persona_config_map = {}
        for persona in persona_queryset.order_by("name"):
            effective_config = get_config(persona=persona)
-            effective_config_redacted = get_config(persona=persona, redact_sensitive=True).model_dump(mode="json")
+            effective_config_redacted = redact_sensitive_config(effective_config.model_dump(mode="json"))
            if can_override_crawl_config:
                raw_config = redact_sensitive_config(persona.config or {})
                effective_config_json = effective_config_redacted
@@ -1723,15 +1543,17 @@ class WebAddView(AddView):

        request_host = (request.get_host() or "").lower()
        request_config = get_request_config(request)
-        web_host = get_web_host(config=request_config)
-        admin_host = get_admin_host(config=request_config)
-        if request.user.is_authenticated and not request_config.PUBLIC_ADD_VIEW and host_matches(request_host, web_host):
+        web_host = get_web_host(config=request_config, request=request)
+        admin_host = get_admin_host(config=request_config, request=request)
+        is_web_host = host_matches(request_host, web_host)
+        is_admin_host = host_matches(request_host, admin_host)
+        if request.user.is_authenticated and not request_config.PUBLIC_ADD_VIEW and is_web_host and not is_admin_host:
            return redirect(build_admin_url(request.get_full_path(), request=request))

        if not self.test_func():
-            if host_matches(request_host, web_host):
+            if is_web_host and not is_admin_host:
                return redirect(build_admin_url(request.get_full_path(), request=request))
-            if host_matches(request_host, admin_host):
+            if is_admin_host:
                next_url = quote(request.get_full_path(), safe="/:?=&")
                return redirect(f"{build_admin_url('/admin/login/', request=request)}?next={next_url}")
            return HttpResponse(
@@ -1230,7 +1230,7 @@ class CrawlScheduleAdmin(BaseModelAdmin):
        (
            "Configuration",
            {
-                "fields": ("schedule", "template"),
+                "fields": ("is_enabled", "schedule", "template"),
                "classes": ("card",),
            },
        ),
@@ -1276,7 +1276,21 @@ class CrawlScheduleAdmin(BaseModelAdmin):

    def change_view(self, request, object_id, form_url="", extra_context=None):
        self.request = request
-        return super().change_view(request, object_id, form_url, extra_context)
+        from django.db import connections, router
+
+        connection = connections[router.db_for_write(self.model)]
+        if request.method != "POST" or connection.vendor != "sqlite":
+            return super().change_view(request, object_id, form_url, extra_context)
+        # Reserve the writer before Django's atomic admin form reads. A
+        # deferred read transaction cannot safely upgrade while the runner
+        # writes, even when SQLite's existing busy timeout has time left.
+        connection.ensure_connection()
+        previous_mode = connection.transaction_mode
+        connection.transaction_mode = "IMMEDIATE"
+        try:
+            return super().change_view(request, object_id, form_url, extra_context)
+        finally:
+            connection.transaction_mode = previous_mode

    def add_view(self, request, form_url="", extra_context=None):
        return redirect("/add/#schedule")
@@ -5,11 +5,3 @@ class CrawlsConfig(AppConfig):
    default_auto_field = "django.db.models.BigAutoField"
    name = "archivebox.crawls"
    label = "crawls"
-
-    def ready(self):
-        """Import models to register state machines with the registry"""
-        import sys
-
-        # Skip during makemigrations to avoid premature state machine access
-        if "makemigrations" not in sys.argv:
-            from archivebox.crawls.models import CrawlMachine  # noqa: F401
@@ -19,7 +19,6 @@ from django.core.validators import MaxValueValidator, MinValueValidator
 from django.conf import settings
 from django.urls import reverse_lazy
 from django.utils import timezone
-from statemachine import State, registry
 from archivebox.config.common import rprint as print
 from archivebox.core.permissions import PERMISSIONS_VALUES, normalize_permissions

@@ -32,7 +31,7 @@ from archivebox.base_models.models import (
    ModelWithHealthStats,
    get_or_create_system_user_pk,
 )
-from archivebox.workers.models import RETRY_AT_MAX, ModelWithStateMachine, BaseStateMachine
+from archivebox.workers.models import ModelWithQueue
 from archivebox.crawls.schedule_util import next_run_for_schedule, validate_schedule
 from archivebox.misc.util import parse_date, sanitize_html_text, validate_url, validate_url_length

@@ -91,6 +90,8 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes):

    @property
    def last_run_at(self):
+        if self.kind == "update":
+            return self.modified_at
        latest_crawl = self.crawl_set.order_by("-created_at").first()
        if latest_crawl:
            return latest_crawl.created_at
@@ -106,6 +107,22 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes):
        now = now or timezone.now()
        return self.is_enabled and self.next_run_at <= now

+    @property
+    def kind(self) -> str:
+        return str((self.config or {}).get("SCHEDULE_KIND") or "crawl")
+
+    def dispatch(self, queued_at=None) -> "Crawl | None":
+        """Run maintenance directly or enqueue one ordinary Crawl."""
+        queued_at = queued_at or timezone.now()
+        if self.kind == "update":
+            from archivebox.cli.archivebox_update import run_scheduled_maintenance
+
+            run_scheduled_maintenance()
+            type(self).objects.filter(pk=self.pk).update(modified_at=queued_at)
+            self.modified_at = queued_at
+            return None
+        return self.enqueue(queued_at=queued_at)
+
    def enqueue(self, queued_at=None) -> "Crawl":
        from archivebox.config.common import build_crawl_config_snapshot

@@ -113,10 +130,11 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes):
        template = self.template
        label = template.label or self.label
        persona = template.persona if template.persona_id else None
+        crawl_config = {key: value for key, value in (self.config or {}).items() if key != "SCHEDULE_KIND"}

        return Crawl.objects.create(
            urls=template.urls,
-            config=build_crawl_config_snapshot(persona=persona, overrides=self.config or {}),
+            config=build_crawl_config_snapshot(persona=persona, overrides=crawl_config),
            max_depth=template.max_depth,
            tags_str=template.tags_str,
            persona_id=template.persona_id,
@@ -129,7 +147,7 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes):
        )


-class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWithStateMachine):
+class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWithQueue):
    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
    created_at = models.DateTimeField(default=timezone.now, db_index=True)
    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
@@ -158,16 +176,19 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
    notes = models.TextField(blank=True, null=False, default="")
    schedule = models.ForeignKey(CrawlSchedule, on_delete=models.SET_NULL, null=True, blank=True, editable=True)

-    status = ModelWithStateMachine.StatusField(
-        choices=ModelWithStateMachine.StatusChoices,
-        default=ModelWithStateMachine.StatusChoices.QUEUED,
+    status = ModelWithQueue.StatusField(
+        choices=ModelWithQueue.StatusChoices,
+        default=ModelWithQueue.StatusChoices.QUEUED,
    )
-    retry_at = ModelWithStateMachine.RetryAtField(default=timezone.now)
+    retry_at = ModelWithQueue.RetryAtField(default=timezone.now)

-    state_machine_name = "archivebox.crawls.models.CrawlMachine"
    retry_at_field_name = "retry_at"
    state_field_name = "status"
-    StatusChoices = ModelWithStateMachine.StatusChoices
+    StatusChoices = ModelWithQueue.StatusChoices
+    INITIAL_STATE = StatusChoices.QUEUED
+    ACTIVE_STATE = StatusChoices.STARTED
+    FINAL_STATES = (StatusChoices.SEALED,)
+    FINAL_OR_ACTIVE_STATES = (*FINAL_STATES, ACTIVE_STATE)
    active_state = StatusChoices.STARTED
    delete_after_final_statuses = (StatusChoices.SEALED,)
    RUNNABLE_STATES = (StatusChoices.QUEUED, StatusChoices.STARTED)
@@ -177,17 +198,12 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith

    snapshot_set: models.Manager["Snapshot"]

-    if TYPE_CHECKING:
-
-        @property
-        def sm(self) -> "CrawlMachine": ...
-
    class Meta(
        ModelWithDeleteAfter.Meta,
        ModelWithOutputDir.Meta,
        ModelWithConfig.Meta,
        ModelWithHealthStats.Meta,
-        ModelWithStateMachine.Meta,
+        ModelWithQueue.Meta,
    ):
        app_label = "crawls"
        verbose_name = "Crawl"
@@ -220,7 +236,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
    def resume(self, *, when=None, save: bool = True) -> bool:
        resumed = super().resume(when=when, save=save)
        if resumed and self.pk:
-            from archivebox.core.models import ArchiveResult, Snapshot
+            from archivebox.core.models import Snapshot

            resume_at = when or timezone.now()
            active_snapshots = self.snapshot_set.filter(
@@ -231,7 +247,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
                retry_at=resume_at,
                modified_at=timezone.now(),
            )
-            ArchiveResult.resume_queryset(ArchiveResult.objects.filter(snapshot__crawl=self), when=resume_at)
        return resumed

    def cancel(self) -> None:
@@ -255,7 +270,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith

        now = timezone.now()
        # Cancellation seals the Crawl first, then lets the runner seal each
-        # child Snapshot through its own state machine. Active children that
+        # child Snapshot through its own lifecycle. Active children that
        # are already due need no write; the runner will claim them as-is.
        active_children = self.snapshot_set.filter(
            status__in=Snapshot.OPEN_STATES,
@@ -272,8 +287,8 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith

        now = timezone.now()
        # Parent pause is a scheduler command. Wake child rows only; each
-        # Snapshot runner claim performs the real pause transition and cascades
-        # its own ArchiveResults, keeping request/admin transactions tiny.
+        # Snapshot runner claim performs the real pause transition, keeping
+        # request/admin transactions tiny.
        active_children = self.snapshot_set.filter(
            status__in=Snapshot.RUNNABLE_STATES,
        )
@@ -518,18 +533,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
            return []
        return [url for _raw_line, url in self._iter_url_lines() if url]

-    def has_internal_input_root(self) -> bool:
-        """Return True when Crawl.urls is preserved source text, not the work queue.
-
-        The runner creates a synthetic root snapshot for raw import text so
-        parser hooks use the same Snapshot lifecycle as every other extractor.
-        In that mode the submitted text must remain in Crawl.urls forever;
-        parsed URLs live as child Snapshot rows and should not be appended back.
-        """
-        from archivebox.core.models import Snapshot
-
-        return self.snapshot_set.filter(url=Snapshot.INTERNAL_INPUT_URL, depth=0).exists()
-
    @staticmethod
    def normalize_domain(value: str) -> str:
        candidate = (value or "").strip().lower()
@@ -799,15 +802,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
            "deleted_snapshots": filter_result["deleted_snapshots"],
        }

-    def get_system_task(self) -> str | None:
-        urls = self.get_urls_list()
-        if len(urls) != 1:
-            return None
-        system_url = urls[0].strip().lower()
-        if system_url.startswith("archivebox://"):
-            return system_url
-        return None
-
    def resolve_persona(self):
        from archivebox.personas.models import Persona

@@ -971,14 +965,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith

        if self.status == self.StatusChoices.SEALED:
            return []
-        # Internal-input crawls preserve the submitted text verbatim in
-        # Crawl.urls. The root snapshot's parser hooks are the only supported
-        # path for turning that text into child snapshots, otherwise a later
-        # runner pass could reinterpret plain URL-looking lines as direct
-        # depth-0 work and bypass format-specific metadata parsing.
-        if self.has_internal_input_root():
-            return []
-
        created_snapshots = []
        crawl_tag_names = self.current_tag_names()
        tags_by_name: dict[str, Tag] = {}
@@ -1194,7 +1180,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
                ),
            }
            if tag_names:
-                tag_ids = [Tag.objects.get_or_create(name=tag_name)[0].pk for tag_name in tag_names]
+                tag_ids = [Tag.get_or_create_by_name(tag_name)[0].pk for tag_name in tag_names]
                snapshot.add_tag_ids(tag_ids)

        existing_scope = Snapshot.objects if bool(self._config_value(config, "ONLY_NEW", True)) else self.snapshot_set
@@ -1251,16 +1237,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
        if not created_snapshots:
            return []

-        crawl_urls = {url for _raw_line, url in self._iter_url_lines() if url}
-        new_url_lines = [snapshot.url for snapshot in created_snapshots if snapshot.url not in crawl_urls]
-        # For internal-input crawls, Crawl.urls is the immutable source text.
-        # Child snapshots are the parsed/indexed representation, so appending
-        # discovered URLs here would both duplicate state and destroy the exact
-        # import artifact users submitted through CLI/API/UI.
-        if new_url_lines and not self.has_internal_input_root():
-            self.urls = (self.urls.rstrip() + "\n" + "\n".join(new_url_lines)).lstrip("\n")
-            self.save(update_fields=["urls", "modified_at"])
-
        tag_names_by_url: dict[str, set[str]] = {}
        for snapshot in created_snapshots:
            tag_names = {
@@ -1296,170 +1272,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith

        return created_snapshots

-    def install_declared_binaries(self, binary_names: set[str], machine=None) -> None:
-        """Install crawl-declared binaries through their unified state machine."""
-        from archivebox.crawls.locks import binary_lifecycle_lock
-        from archivebox.machine.models import Binary, Machine
-
-        if not binary_names:
-            return
-
-        machine = machine or Machine.current()
-        binaries = Binary.objects.filter(machine=machine, name__in=binary_names).order_by("name")
-        for binary in binaries:
-            with binary_lifecycle_lock(str(binary.id)):
-                binary.refresh_from_db()
-                if binary.status == Binary.StatusChoices.INSTALLED:
-                    continue
-                binary.update_and_requeue(retry_at=timezone.now())
-                binary.refresh_from_db()
-                binary.tick_claimed(lock_seconds=600)
-
-        unresolved_binaries = list(
-            Binary.objects.filter(
-                machine=machine,
-                name__in=binary_names,
-            )
-            .exclude(
-                status=Binary.StatusChoices.INSTALLED,
-            )
-            .order_by("name"),
-        )
-        if unresolved_binaries:
-            binary_details = ", ".join(
-                f"{binary.name} (status={binary.status}, retry_at={binary.retry_at})" for binary in unresolved_binaries
-            )
-            raise RuntimeError(
-                f"Crawl dependencies failed to install before continuing: {binary_details}",
-            )
-
-    def run(self) -> "Snapshot | None":
-        """
-        Execute this Crawl: run hooks, process JSONL, create snapshots.
-
-        Called by the state machine when entering the 'started' state.
-
-        Returns:
-            The root Snapshot for this crawl, or None for system crawls that don't create snapshots
-        """
-        import time
-        from archivebox.plugins.hooks import run_hook, discover_hooks, process_hook_records
-        from archivebox.config.common import get_config
-        from archivebox.machine.models import Machine
-
-        def get_runtime_config():
-            return get_config(crawl=self).for_crawl_runtime(
-                crawl=self,
-                persona=persona,
-                runtime_overrides=persona_runtime_overrides,
-            )
-
-        system_task = self.get_system_task()
-        if system_task == "archivebox://update":
-            from archivebox.cli.archivebox_update import process_all_db_snapshots
-
-            process_all_db_snapshots()
-            return None
-
-        machine = Machine.current()
-        declared_binary_names: set[str] = set()
-        persona_runtime_overrides: dict[str, str] = {}
-        persona = self.resolve_persona()
-        if persona:
-            base_runtime_config = get_config(crawl=self, persona=persona)
-            chrome_binary = str(base_runtime_config.get("CHROME_BINARY") or "")
-            persona_runtime_overrides = persona.prepare_runtime_for_crawl(
-                crawl=self,
-                chrome_binary=chrome_binary,
-            )
-
-        def run_crawl_hook(hook: Path) -> set[str]:
-            primary_url = next(
-                (line.strip() for line in self.urls.splitlines() if line.strip()),
-                self.urls.strip(),
-            )
-
-            hook_start = time.time()
-            plugin_name = hook.parent.name
-            output_dir = self.output_dir / plugin_name
-            output_dir.mkdir(parents=True, exist_ok=True)
-
-            process = run_hook(
-                hook,
-                output_dir=output_dir,
-                config=get_runtime_config(),
-                crawl_id=str(self.id),
-                source_url=self.urls,
-                url=primary_url,
-                snapshot_id=str(self.id),
-            )
-            hook_elapsed = time.time() - hook_start
-            if hook_elapsed > 0.5:
-                print(f"[yellow]⏱️  Hook {hook.name} took {hook_elapsed:.2f}s[/yellow]")
-
-            if process.status == process.StatusChoices.RUNNING:
-                if process.poll() is None:
-                    return set()
-
-            from archivebox.plugins.hooks import extract_records_from_process
-
-            records = []
-            # A hook can exit before its completed Process metadata is visible.
-            # Give successful hooks a brief chance to flush JSONL stdout into
-            # the Process row before downstream hooks.
-            for delay in (0.0, 0.05, 0.1, 0.25, 0.5):
-                if delay:
-                    time.sleep(delay)
-                records = extract_records_from_process(process)
-                if records:
-                    break
-            if records:
-                print(f"[cyan]📝 Processing {len(records)} records from {hook.name}[/cyan]")
-                for record in records[:3]:
-                    print(f"   Record: type={record.get('type')}, keys={list(record.keys())[:5]}")
-            if system_task:
-                records = [record for record in records if record.get("type") in ("BinaryRequest", "Binary")]
-            overrides = {"crawl": self}
-            stats = process_hook_records(records, overrides=overrides)
-            if stats:
-                print(f"[green]✓ Created: {stats}[/green]")
-
-            hook_binary_names = {
-                str(record.get("name")).strip()
-                for record in records
-                if record.get("type") in ("BinaryRequest", "Binary") and record.get("name")
-            }
-            hook_binary_names.discard("")
-            if hook_binary_names:
-                declared_binary_names.update(hook_binary_names)
-            return hook_binary_names
-
-        hooks = discover_hooks("Crawl", config=get_runtime_config())
-
-        for hook in hooks:
-            hook_binary_names = run_crawl_hook(hook)
-            if hook_binary_names:
-                self.install_declared_binaries(hook_binary_names, machine=machine)
-
-        # Safety check: don't create snapshots if any crawl-declared dependency
-        # is still unresolved after all crawl hooks have run.
-        self.install_declared_binaries(declared_binary_names, machine=machine)
-
-        # Create snapshots from all URLs in self.urls
-        if system_task:
-            leaked_snapshots = self.snapshot_set.all()
-            if leaked_snapshots.exists():
-                leaked_count = leaked_snapshots.count()
-                leaked_snapshots.delete()
-                print(f"[yellow]⚠️  Removed {leaked_count} leaked snapshot(s) created during system crawl {system_task}[/yellow]")
-            return None
-
-        self.create_snapshots_from_urls()
-
-        # Return first snapshot for this crawl (newly created or existing)
-        # This ensures the crawl doesn't seal if snapshots exist, even if they weren't just created
-        return self.snapshot_set.first()
-
    def is_finished(self) -> bool:
        """Check if crawl is finished (all snapshots sealed or no snapshots exist)."""
        from archivebox.core.models import Snapshot
@@ -1483,11 +1295,64 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith

        return True

-    def cleanup(self):
-        """Clean up background hooks and run on_CrawlEnd hooks."""
-        from archivebox.plugins.hooks import run_hook, discover_hooks
+    def can_start(self) -> bool:
+        return bool(self.urls and self.get_urls_list())

-        # Clean up .pid files from output directory
+    def has_finished_snapshots(self) -> bool:
+        from archivebox.core.models import Snapshot
+
+        snapshots = self.snapshot_set.all()
+        return snapshots.exists() and not snapshots.exclude(status=Snapshot.StatusChoices.SEALED).exists()
+
+    def mark_started(self) -> bool:
+        now = timezone.now()
+        updated = self.safe_update(
+            {
+                "status": self.StatusChoices.STARTED,
+                "retry_at": now + timedelta(seconds=2),
+            },
+            extra_filter={"status": self.StatusChoices.QUEUED},
+        )
+        return updated
+
+    def seal(self) -> bool:
+        """Finalize a runner-owned Crawl without dispatching hooks directly."""
+        now = timezone.now()
+        updated = self.safe_update(
+            {
+                "status": self.StatusChoices.SEALED,
+                "retry_at": None,
+                "modified_at": now,
+            },
+            refresh=False,
+            extra_filter={"status__in": (*self.RUNNABLE_STATES, self.StatusChoices.SEALED)},
+        )
+        if not updated:
+            self.refresh_from_db()
+            return False
+        self.status = self.StatusChoices.SEALED
+        self.retry_at = None
+        self.modified_at = now
+        self.schedule_child_snapshots_for_sealing()
+        self.cleanup_runtime()
+        return True
+
+    def advance_lifecycle(self) -> bool:
+        """Advance one explicit lifecycle step after the runner claims this row."""
+        if self.status == self.StatusChoices.PAUSED:
+            return False
+        if self.status == self.StatusChoices.QUEUED:
+            if self.has_finished_snapshots():
+                return self.seal()
+            if not self.can_start():
+                return False
+            return self.mark_started()
+        if self.status == self.StatusChoices.STARTED and self.is_finished():
+            return self.seal()
+        return False
+
+    def cleanup_runtime(self) -> None:
+        """Remove runner-owned runtime artifacts after abx-dl cleanup hooks finish."""
        if self.output_dir.exists():
            for pid_file in self.output_dir.glob("**/*.pid"):
                pid_file.unlink(missing_ok=True)
@@ -1495,207 +1360,3 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
        persona = self.resolve_persona()
        if persona:
            persona.cleanup_runtime_for_crawl(self)
-
-        # Run on_CrawlEnd hooks
-        from archivebox.config.common import get_config
-
-        config = get_config(crawl=self)
-
-        hooks = discover_hooks("CrawlEnd", config=config)
-
-        for hook in hooks:
-            plugin_name = hook.parent.name
-            output_dir = self.output_dir / plugin_name
-            output_dir.mkdir(parents=True, exist_ok=True)
-
-            process = run_hook(
-                hook,
-                output_dir=output_dir,
-                config=config,
-                crawl_id=str(self.id),
-                source_url=self.urls,  # Pass full newline-separated URLs
-            )
-
-            # Log failures but don't block
-            if process.exit_code != 0:
-                print(f"[yellow]⚠️ CrawlEnd hook failed: {hook.name}[/yellow]")
-
-
-# =============================================================================
-# State Machines
-# =============================================================================
-
-
-class CrawlMachine(BaseStateMachine):
-    crawl: Crawl
-
-    """
-    State machine for managing Crawl lifecycle.
-
-    Hook Lifecycle:
-    ┌─────────────────────────────────────────────────────────────┐
-    │ QUEUED State                                                │
-    │  • Waiting for crawl to be ready (has URLs)                 │
-    └─────────────────────────────────────────────────────────────┘
-                            ↓ tick() when can_start()
-    ┌─────────────────────────────────────────────────────────────┐
-    │ STARTED State → enter_started()                             │
-    │  1. crawl.run()                                             │
-    │     • discover_hooks('Crawl') → finds all crawl hooks       │
-    │     • For each hook:                                        │
-    │       - run_hook(script, output_dir, ...)                   │
-    │       - Parse JSONL from hook output                        │
-    │       - process_hook_records() → creates Snapshots          │
-    │     • create_snapshots_from_urls() → from self.urls field   │
-    │                                                              │
-    │  2. Snapshots process independently with their own          │
-    │     state machines (see SnapshotMachine)                    │
-    └─────────────────────────────────────────────────────────────┘
-                            ↓ tick() when is_finished()
-    ┌─────────────────────────────────────────────────────────────┐
-    │ SEALED State → enter_sealed()                               │
-    │  • cleanup() → runs on_CrawlEnd hooks, kills background     │
-    │  • Set retry_at=None (no more processing)                   │
-    └─────────────────────────────────────────────────────────────┘
-    """
-
-    model_attr_name = "crawl"
-
-    # States
-    queued = State(value=Crawl.StatusChoices.QUEUED, initial=True)
-    started = State(value=Crawl.StatusChoices.STARTED)
-    paused = State(value=Crawl.StatusChoices.PAUSED)
-    sealed = State(value=Crawl.StatusChoices.SEALED, final=True)
-
-    # Tick Event (polled by workers)
-    tick = (
-        queued.to(sealed, cond="has_finished_snapshots")
-        | queued.to.itself(unless="can_start")
-        | queued.to(started, cond="can_start")
-        | started.to(sealed, cond="is_finished")
-        | paused.to.itself()
-    )
-
-    # Manual event (triggered by last Snapshot sealing, or by direct
-    # index-only/bg creation when every requested URL is rejected before any
-    # Snapshot rows exist).
-    seal = queued.to(sealed) | started.to(sealed) | paused.to(sealed)
-    pause_requested = queued.to(paused) | started.to(paused)
-    resume_requested = paused.to(queued)
-
-    def can_start(self) -> bool:
-        if not self.crawl.urls:
-            print(f"[red]⚠️ Crawl {self.crawl.id} cannot start: no URLs[/red]")
-            return False
-        urls_list = self.crawl.get_urls_list()
-        if not urls_list:
-            print(f"[red]⚠️ Crawl {self.crawl.id} cannot start: no valid URLs in urls field[/red]")
-            return False
-        return True
-
-    def is_finished(self) -> bool:
-        """Check if all Snapshots for this crawl are finished."""
-        return self.crawl.is_finished()
-
-    def has_finished_snapshots(self) -> bool:
-        """A queued crawl with only final Snapshot rows was interrupted before sealing."""
-        from archivebox.core.models import Snapshot
-
-        snapshots = self.crawl.snapshot_set.all()
-        return snapshots.exists() and not snapshots.exclude(status=Snapshot.StatusChoices.SEALED).exists()
-
-    @queued.enter
-    def enter_queued(self):
-        self.crawl.update_and_requeue(
-            retry_at=timezone.now(),
-            status=Crawl.StatusChoices.QUEUED,
-        )
-
-    @started.enter
-    def enter_started(self):
-        import sys
-
-        print(f"[cyan]🔄 CrawlMachine.enter_started() - creating snapshots for {self.crawl.id}[/cyan]", file=sys.stderr)
-
-        try:
-            # Run the crawl - runs hooks, processes JSONL, creates snapshots
-            first_snapshot = self.crawl.run()
-
-            if first_snapshot:
-                print(
-                    f"[cyan]🔄 Created {self.crawl.snapshot_set.count()} snapshot(s), first: {first_snapshot.url}[/cyan]",
-                    file=sys.stderr,
-                )
-                # Update status to STARTED
-                # Set retry_at to near future so tick() can poll and check is_finished()
-                self.crawl.update_and_requeue(
-                    retry_at=timezone.now() + timedelta(seconds=2),
-                    status=Crawl.StatusChoices.STARTED,
-                )
-            else:
-                # No snapshots (system crawl that only runs setup hooks)
-                print("[cyan]🔄 No snapshots created, sealing crawl immediately[/cyan]", file=sys.stderr)
-                # Seal immediately since there's no work to do
-                self.seal()
-
-        except Exception as e:
-            print(f"[red]⚠️ Crawl {self.crawl.id} failed to start: {e}[/red]")
-            import traceback
-
-            traceback.print_exc()
-            raise
-
-    @paused.enter
-    def enter_paused(self):
-        paused = self.crawl.safe_update(
-            {
-                "retry_at": RETRY_AT_MAX,
-                "status": Crawl.StatusChoices.PAUSED,
-            },
-            extra_filter={"status__in": Crawl.RUNNABLE_STATES},
-        )
-        if paused:
-            self.crawl.schedule_child_snapshots_for_pause()
-
-    @sealed.enter
-    def enter_sealed(self):
-        now = timezone.now()
-        self.crawl.status = Crawl.StatusChoices.SEALED
-        self.crawl.retry_at = None
-        # Guard: never seal a row that a concurrent writer flipped to PAUSED.
-        # Sealing is idempotent (SEALED→SEALED is a no-op rewrite), so
-        # status__in covers both the QUEUED/STARTED→SEALED transition and the
-        # rare re-entry case.
-        updated = self.crawl.safe_update(
-            {
-                "status": Crawl.StatusChoices.SEALED,
-                "retry_at": None,
-                "modified_at": now,
-            },
-            refresh=False,
-            extra_filter={
-                "status__in": [
-                    Crawl.StatusChoices.QUEUED,
-                    Crawl.StatusChoices.STARTED,
-                    Crawl.StatusChoices.SEALED,
-                ],
-            },
-        )
-        if not updated:
-            self.crawl.refresh_from_db()
-            return
-        self.crawl.modified_at = now
-
-        self.crawl.schedule_child_snapshots_for_sealing()
-        # Clean up background hooks and run on_CrawlEnd hooks after the final
-        # state is visible so cleanup projectors cannot resurrect the crawl.
-        self.crawl.cleanup()
-
-
-# =============================================================================
-# Register State Machines
-# =============================================================================
-
-# Manually register state machines with python-statemachine registry
-# (normally auto-discovered from statemachines.py, but we define them here for clarity)
-registry.register(CrawlMachine)
@@ -10,14 +10,6 @@ class MachineConfig(AppConfig):
    label = "machine"  # Explicit label for migrations
    verbose_name = "Machine Info"

-    def ready(self):
-        """Import models to register state machines with the registry"""
-        import sys
-
-        # Skip during makemigrations to avoid premature state machine access
-        if "makemigrations" not in sys.argv:
-            from archivebox.machine import models  # noqa: F401
-

 def register_admin(admin_site):
    from archivebox.machine.admin import register_admin
@@ -12,8 +12,6 @@ from archivebox.uuid_compat import CompactUUIDField, uuid7
 from datetime import timedelta, datetime
 from typing import TYPE_CHECKING, Any, cast

-from statemachine import State, registry
-
 from django.db import IntegrityError, transaction
 from django.db import models
 from django.db.models import Q, QuerySet
@@ -23,7 +21,7 @@ from django.utils.functional import cached_property
 from archivebox.config import CONSTANTS
 from archivebox.config.common import rprint
 from archivebox.base_models.models import ModelWithDeleteAfter, ModelWithHealthStats, normalize_config_json_values
-from archivebox.workers.models import BaseStateMachine, ModelWithStateMachine
+from archivebox.workers.models import ModelWithQueue
 from .detect import get_host_guid, get_os_info, get_vm_info, get_host_network, get_host_stats

 _psutil: Any | None = None
@@ -54,6 +52,15 @@ PROCESS_RECHECK_INTERVAL = 60  # Re-validate every 60 seconds
 PID_REUSE_WINDOW = timedelta(hours=24)  # Max age for considering a PID match valid
 PROCESS_TIMEOUT_GRACE = timedelta(seconds=30)  # Extra margin before force-cleaning timed-out RUNNING rows
 START_TIME_TOLERANCE = 5.0  # Seconds tolerance for start time matching
+PROCESS_PID_NAMESPACE_KEY = "_ARCHIVEBOX_PID_NAMESPACE"
+
+
+def get_current_pid_namespace() -> str:
+    """Return the OS PID namespace that makes numeric PIDs meaningful."""
+    try:
+        return os.readlink("/proc/self/ns/pid")
+    except OSError:
+        return f"host:{socket.gethostname()}"


 def _default_exit_code_for_unowned_process(process_type: str) -> int:
@@ -220,6 +227,9 @@ class Machine(ModelWithHealthStats):
            _CURRENT_MACHINE = None
        if _CURRENT_MACHINE:
            if timezone.now() < _CURRENT_MACHINE.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL):
+                # Reconciliation can save() and clear the shared cache; keep
+                # this call's instance so we never return None after that save.
+                machine = _CURRENT_MACHINE
                # One-time-per-process reconciliation between ArchiveBox.conf
                # and Machine.config. Fast-path: bool check + early-return when
                # the sync has already run, so the cached-machine return path
@@ -227,16 +237,21 @@ class Machine(ModelWithHealthStats):
                try:
                    from archivebox.config.collection import sync_machine_and_file

-                    sync_machine_and_file(_CURRENT_MACHINE)
+                    sync_machine_and_file(machine)
                except Exception:
                    pass
-                return _CURRENT_MACHINE
+                return machine
            else:
                _CURRENT_MACHINE = None

        host_guid = get_host_guid()
+        # save() intentionally invalidates _CURRENT_MACHINE so config edits
+        # made through other instances become visible in this process. Host
+        # refresh (after seven days), sanitation, and reconciliation can all
+        # save here, so the working instance must not live only in that cache.
+        # Otherwise a routine refresh leaves startup dereferencing None.
        try:
-            _CURRENT_MACHINE = cls.objects.get(guid=host_guid)
+            machine = cls.objects.get(guid=host_guid)
        except cls.DoesNotExist:
            config = {}
            try:
@@ -246,7 +261,7 @@ class Machine(ModelWithHealthStats):
                config = _coerce_from_str_dict(file_config)
            except Exception:
                config = {}
-            _CURRENT_MACHINE = cls.objects.create(
+            machine = cls.objects.create(
                guid=host_guid,
                hostname=socket.gethostname(),
                config=config,
@@ -255,15 +270,15 @@ class Machine(ModelWithHealthStats):
                stats=get_host_stats(),
            )
        else:
-            if timezone.now() >= _CURRENT_MACHINE.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL):
+            if timezone.now() >= machine.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL):
                for key, value in {
                    "hostname": socket.gethostname(),
                    **get_os_info(),
                    **get_vm_info(),
                    "stats": get_host_stats(),
                }.items():
-                    setattr(_CURRENT_MACHINE, key, value)
-                _CURRENT_MACHINE.save(
+                    setattr(machine, key, value)
+                machine.save(
                    update_fields=[
                        "hostname",
                        "hw_in_docker",
@@ -280,16 +295,19 @@ class Machine(ModelWithHealthStats):
                        "modified_at",
                    ],
                )
-        machine = cls._sanitize_config(_CURRENT_MACHINE)
+        machine = cls._sanitize_config(machine)
        # Same one-time sync as the cached-return path. Triggers here on the
        # very first ``Machine.current()`` call in a process before any
        # cached return can occur.
        try:
            from archivebox.config.collection import sync_machine_and_file

-            sync_machine_and_file(_CURRENT_MACHINE)
+            sync_machine_and_file(machine)
        except Exception:
            pass
+        # Publish only after all save-capable steps; publishing earlier would
+        # let those steps invalidate the cache we are in the middle of filling.
+        _CURRENT_MACHINE = machine
        return machine

    @classmethod
@@ -506,19 +524,21 @@ class BinaryManager(models.Manager):
        )


-class Binary(ModelWithHealthStats, ModelWithStateMachine):
+class Binary(ModelWithHealthStats, ModelWithQueue):
    """
    Tracks a binary on a specific machine.

-    Simple state machine with 2 states:
+    Simple queue lifecycle with 2 states:
    - queued: Binary needs to be installed
    - installed: Binary installed successfully (abspath, version, sha256 populated)

    Installation is synchronous during queued→installed transition.
    If installation fails, Binary stays in queued with retry_at set for later retry.

-    State machine calls run(), which emits an abxpkg BinaryRequestEvent through
-    the ArchiveBox runner and installs the binary using the specified providers.
+    BinaryService claims queued rows with a conditional update, then run()
+    emits an abxpkg BinaryRequestEvent and persists the resolved installation.
+    The database row is the only lifecycle state; there is intentionally no
+    second in-memory state machine to reconcile after a worker interruption.
    """

    class StatusChoices(models.TextChoices):
@@ -557,9 +577,9 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
    version = models.CharField(max_length=32, default="", null=False, blank=True)
    sha256 = models.CharField(max_length=64, default="", null=False, blank=True)

-    # State machine fields
-    status = ModelWithStateMachine.StatusField(choices=StatusChoices.choices, default=StatusChoices.QUEUED, max_length=16)
-    retry_at = ModelWithStateMachine.RetryAtField(
+    # Durable queue lifecycle fields
+    status = ModelWithQueue.StatusField(choices=StatusChoices.choices, default=StatusChoices.QUEUED, max_length=16)
+    retry_at = ModelWithQueue.RetryAtField(
        default=timezone.now,
        help_text="When to retry this binary installation",
    )
@@ -570,18 +590,16 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):

    machine_id: uuid.UUID

-    state_machine_name: str | None = "archivebox.machine.models.BinaryMachine"
+    INITIAL_STATE = StatusChoices.QUEUED
+    ACTIVE_STATE = StatusChoices.QUEUED
+    FINAL_STATES = (StatusChoices.INSTALLED,)
+    FINAL_OR_ACTIVE_STATES = (*FINAL_STATES, ACTIVE_STATE)
    active_state: str = StatusChoices.QUEUED
    warn_on_save_outside_runner = False

    objects = BinaryManager()  # pyright: ignore[reportIncompatibleVariableOverride]

-    if TYPE_CHECKING:
-
-        @property
-        def sm(self) -> BinaryMachine: ...
-
-    class Meta(ModelWithHealthStats.Meta, ModelWithStateMachine.Meta):
+    class Meta(ModelWithHealthStats.Meta, ModelWithQueue.Meta):
        app_label = "machine"
        verbose_name = "Binary"
        verbose_name_plural = "Binaries"
@@ -595,6 +613,10 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
        """A binary is valid if it has a resolved path and is marked installed."""
        return bool(self.abspath) and self.status == self.StatusChoices.INSTALLED

+    @property
+    def can_install(self) -> bool:
+        return bool(self.name and self.binproviders)
+
    @cached_property
    def binary_info(self) -> dict:
        """Return info about the binary."""
@@ -739,11 +761,43 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):

        run_binary(str(self.id))

+    def install(self) -> bool:
+        """Run one synchronous installation attempt for a claimed Binary."""
+        if self.status == self.StatusChoices.INSTALLED:
+            return True
+        if not self.can_install:
+            return False
+
+        rprint(f"[cyan]      🔄 installing {self.name}[/cyan]", file=sys.stderr)
+        self.run()
+        self.refresh_from_db()
+        if self.status != self.StatusChoices.INSTALLED:
+            self.update_and_requeue(
+                retry_at=timezone.now() + timedelta(seconds=300),
+                status=self.StatusChoices.QUEUED,
+            )
+            self.increment_health_stats(success=False)
+            raise RuntimeError(f"Binary {self.name} installation failed")
+
+        self.update_and_requeue(retry_at=None, status=self.StatusChoices.INSTALLED)
+        self.increment_health_stats(success=True)
+        return True
+
+    def advance_lifecycle(self) -> bool:
+        """Advance the explicit binary lifecycle after its queue row is claimed."""
+        return self.install()
+
+    def install_claimed(self, *, lock_seconds: int = 600) -> bool:
+        if not self.claim_processing_lock(lock_seconds=lock_seconds):
+            return False
+        self.refresh_from_db()
+        return self.advance_lifecycle()
+
    def cleanup(self):
        """
        Clean up background binary installation hooks.

-        Called by state machine if needed (not typically used for binaries
+        Called after an installation attempt if needed (not typically used for binaries
        since installations are foreground, but included for consistency).
        """

@@ -900,7 +954,10 @@ class ProcessManager(models.Manager):
            started_at__gte=timezone.now() - PID_REUSE_WINDOW,
        ).order_by("-started_at")

+        current_pid_namespace = get_current_pid_namespace()
        for candidate in candidates:
+            if candidate.env.get(PROCESS_PID_NAMESPACE_KEY) not in (None, current_pid_namespace):
+                continue
            # Validate start time matches (within tolerance)
            if candidate.started_at:
                db_start_time = candidate.started_at.timestamp()
@@ -941,12 +998,14 @@ class Process(ModelWithDeleteAfter, models.Model):
    One Process can optionally be associated with an ArchiveResult (via OneToOne),
    but Process can also exist standalone for internal operations.

-    Follows the unified state machine pattern:
+    Follows the unified process lifecycle:
    - queued: Process ready to launch
    - running: Process actively executing
    - exited: Process completed (check exit_code for success/failure)

-    State machine calls launch() to spawn the process and monitors its lifecycle.
+    Direct Process methods and the abx-dl event projector own these transitions.
+    Keeping the DB row as the only lifecycle state makes interrupted subprocess
+    recovery observable without reconciling a second in-memory state machine.
    """

    class StatusChoices(models.TextChoices):
@@ -1105,7 +1164,7 @@ class Process(ModelWithDeleteAfter, models.Model):
    # Reverse relation to ArchiveResult (OneToOne from AR side)
    # archiveresult: OneToOneField defined on ArchiveResult model

-    # State machine fields
+    # Durable process lifecycle fields
    status = models.CharField(
        max_length=16,
        choices=StatusChoices.choices,
@@ -1126,7 +1185,6 @@ class Process(ModelWithDeleteAfter, models.Model):
    children: models.Manager[Process]
    archiveresult: ArchiveResult

-    state_machine_name: str = "archivebox.machine.models.ProcessMachine"
    delete_after_final_statuses = (StatusChoices.EXITED,)

    objects = ProcessManager()  # pyright: ignore[reportIncompatibleVariableOverride]
@@ -1144,6 +1202,11 @@ class Process(ModelWithDeleteAfter, models.Model):
            models.Index(fields=["machine", "status", "process_type"], name="mach_proc_running_idx"),
        ]
        constraints = [
+            # This is deliberately machine-scoped. It prevents two locally
+            # verifiable runners from working the same collection, while not
+            # claiming to coordinate independent hosts that share a database.
+            # Cross-machine work ownership belongs to short Crawl/Snapshot CAS
+            # claims so PostgreSQL deployments can support that model later.
            models.UniqueConstraint(
                fields=["machine", "pwd"],
                condition=Q(status="running", process_type="orchestrator", worker_type="worker_runner"),
@@ -1291,8 +1354,8 @@ class Process(ModelWithDeleteAfter, models.Model):
        """
        Compare-and-swap update for short Process scheduler writes.

-        Process is not a ModelWithStateMachine subclass yet, but its
-        state-machine methods still need the same modified_at CAS behavior as
+        Process is not a ModelWithQueue subclass, but its scheduler methods
+        still need the same modified_at CAS behavior as
        Crawl/Snapshot/Binary without falling back to save().
        """
        values = dict(update_fields)
@@ -1328,6 +1391,10 @@ class Process(ModelWithDeleteAfter, models.Model):
        updates = ["status", "retry_at", "modified_at"]
        self.status = self.StatusChoices.RUNNING
        self.retry_at = None
+        pid_namespace = get_current_pid_namespace()
+        if self.env.get(PROCESS_PID_NAMESPACE_KEY) != pid_namespace:
+            self.env = {**self.env, PROCESS_PID_NAMESPACE_KEY: pid_namespace}
+            updates.append("env")
        if process_type is not None and self.process_type != process_type:
            self.process_type = process_type
            updates.append("process_type")
@@ -1346,7 +1413,7 @@ class Process(ModelWithDeleteAfter, models.Model):
        self.save(update_fields=updates)

    def heartbeat(self) -> None:
-        """Touch modified_at so standby/leader selection can see this parent is alive."""
+        """Keep a long-lived watcher visible in recent-process monitoring."""
        self.save(update_fields=["modified_at"])

    def mark_exited(self, *, exit_code: int = 0) -> None:
@@ -1426,17 +1493,18 @@ class Process(ModelWithDeleteAfter, models.Model):
        # Try to find existing Process for this PID on this machine
        # Filter by: machine + PID + RUNNING + recent + start time matches
        if os_start_time:
-            existing = (
-                cls.objects.filter(
-                    machine=machine,
-                    pid=current_pid,
-                    status=cls.StatusChoices.RUNNING,
-                    started_at__gte=timezone.now() - PID_REUSE_WINDOW,
-                )
-                .order_by("-started_at")
-                .first()
-            )
+            candidates = cls.objects.filter(
+                machine=machine,
+                pid=current_pid,
+                status=cls.StatusChoices.RUNNING,
+                started_at__gte=timezone.now() - PID_REUSE_WINDOW,
+            ).order_by("-started_at")

+            current_pid_namespace = get_current_pid_namespace()
+            existing = next(
+                (candidate for candidate in candidates if candidate.env.get(PROCESS_PID_NAMESPACE_KEY) in (None, current_pid_namespace)),
+                None,
+            )
            if existing and existing.started_at:
                db_start_time = existing.started_at.timestamp()
                if abs(db_start_time - os_start_time) < START_TIME_TOLERANCE:
@@ -1472,6 +1540,7 @@ class Process(ModelWithDeleteAfter, models.Model):
            parent=parent,
            process_type=process_type,
            cmd=cmd,
+            env={PROCESS_PID_NAMESPACE_KEY: get_current_pid_namespace()},
            pwd=os.getcwd(),
            pid=current_pid,
            started_at=started_at,
@@ -1514,7 +1583,10 @@ class Process(ModelWithDeleteAfter, models.Model):
            started_at__gte=timezone.now() - PID_REUSE_WINDOW,
        ).order_by("-started_at")

+        current_pid_namespace = get_current_pid_namespace()
        for candidate in candidates:
+            if candidate.env.get(PROCESS_PID_NAMESPACE_KEY) not in (None, current_pid_namespace):
+                continue
            if candidate.started_at:
                db_start_time = candidate.started_at.timestamp()
                time_diff = abs(db_start_time - os_parent_start)
@@ -1579,14 +1651,20 @@ class Process(ModelWithDeleteAfter, models.Model):
        """
        cleaned = 0

-        stale = cls.objects.filter(status=cls.StatusChoices.RUNNING)
+        # Top-level commands own their lifecycle and may run in another Docker
+        # PID namespace while sharing this database. Their takeover helpers
+        # validate and finalize them; local psutil cannot do so safely.
+        stale = cls.objects.filter(status=cls.StatusChoices.RUNNING).exclude(
+            process_type__in=(cls.TypeChoices.SERVER, cls.TypeChoices.ADD, cls.TypeChoices.UPDATE),
+        )
        if machine is not None:
            stale = stale.filter(machine=machine)

        # Recovery can run against damaged DB state; stream rows so a large
        # stale Process backlog cannot be materialized in memory at once.
        for proc in stale.iterator(chunk_size=100):
-            if proc.poll() is not None:
+            shares_pid_namespace = proc.shares_pid_namespace
+            if shares_pid_namespace and proc.poll() is not None:
                cleaned += 1
                continue

@@ -1601,7 +1679,7 @@ class Process(ModelWithDeleteAfter, models.Model):
            # Check if too old (PID definitely reused)
            if not is_stale and proc.started_at and proc.started_at < timezone.now() - PID_REUSE_WINDOW:
                is_stale = True
-            elif not is_stale and PSUTIL_AVAILABLE and proc.pid is not None:
+            elif not is_stale and shares_pid_namespace and PSUTIL_AVAILABLE and proc.pid is not None:
                # Check if OS process still exists with matching start time
                try:
                    os_proc = psutil.Process(proc.pid)
@@ -1666,6 +1744,11 @@ class Process(ModelWithDeleteAfter, models.Model):
    # Validated psutil access via .proc property
    # =========================================================================

+    @property
+    def shares_pid_namespace(self) -> bool:
+        recorded_namespace = self.env.get(PROCESS_PID_NAMESPACE_KEY)
+        return recorded_namespace in (None, get_current_pid_namespace())
+
    @property
    def proc(self) -> psutil.Process | None:
        """
@@ -1687,6 +1770,9 @@ class Process(ModelWithDeleteAfter, models.Model):
        if not PSUTIL_AVAILABLE:
            return None

+        if not self.shares_pid_namespace:
+            return None
+
        # Can't get psutil.Process if we don't have a PID
        if not self.pid:
            return None
@@ -1733,6 +1819,9 @@ class Process(ModelWithDeleteAfter, models.Model):
        More reliable than checking status field since it validates
        the actual OS process exists and matches our record.
        """
+        if not self.shares_pid_namespace:
+            return self.status == self.StatusChoices.RUNNING
+
        proc = self.proc
        if proc is None:
            return False
@@ -2082,6 +2171,9 @@ class Process(ModelWithDeleteAfter, models.Model):
        Returns:
            True if killed successfully, False otherwise
        """
+        if not self.shares_pid_namespace:
+            return False
+
        # Use validated psutil.Process to ensure we're killing the right process
        proc = self.proc
        if proc is None:
@@ -2187,8 +2279,8 @@ class Process(ModelWithDeleteAfter, models.Model):
        """
        Gracefully terminate process: SIGTERM → wait → SIGKILL.

-        This consolidates the scattered SIGTERM/SIGKILL logic from:
-        - crawls/models.py Crawl.cleanup()
+        This consolidates SIGTERM/SIGKILL logic used by:
+        - workers/management/commands/runner_watch.py
        - workers/pid_utils.py stop_worker()
        - supervisord_util.py stop_existing_supervisord_process()

@@ -2251,8 +2343,8 @@ class Process(ModelWithDeleteAfter, models.Model):
        Uses parallel polling approach - sends SIGTERM to all processes at once,
        then polls all simultaneously with individual deadline tracking.

-        This consolidates the scattered child-killing logic from:
-        - crawls/models.py Crawl.cleanup() os.killpg()
+        This consolidates child-killing logic used by:
+        - core/takeover_util.py
        - supervisord_util.py stop_existing_supervisord_process()

        Args:
@@ -2265,6 +2357,9 @@ class Process(ModelWithDeleteAfter, models.Model):
        import time
        import os

+        if not self.shares_pid_namespace:
+            return 0
+
        killed_count = 0
        used_sigkill = False
        proc = self.proc
@@ -2549,206 +2644,3 @@ class Process(ModelWithDeleteAfter, models.Model):
        if cleaned:
            rprint(f"[yellow]🧹 Cleaned up {cleaned} orphaned worker/hook process record(s)[/yellow]")
        return cleaned
-
-
-# =============================================================================
-# Binary State Machine
-# =============================================================================
-
-
-class BinaryMachine(BaseStateMachine):
-    """
-    State machine for managing Binary installation lifecycle.
-
-    Simple 2-state machine:
-    ┌─────────────────────────────────────────────────────────────┐
-    │ QUEUED State                                                │
-    │  • Binary needs to be installed                             │
-    └─────────────────────────────────────────────────────────────┘
-                            ↓ tick() when can_install()
-                            ↓ Synchronous installation during transition
-    ┌─────────────────────────────────────────────────────────────┐
-    │ INSTALLED State                                             │
-    │  • Binary installed (abspath, version, sha256 set)          │
-    │  • Health stats incremented                                 │
-    └─────────────────────────────────────────────────────────────┘
-
-    If installation fails, Binary stays in QUEUED with retry_at bumped.
-    """
-
-    model_attr_name = "binary"
-    binary: Binary
-
-    # States
-    queued = State(value=Binary.StatusChoices.QUEUED, initial=True)
-    installed = State(value=Binary.StatusChoices.INSTALLED, final=True)
-
-    # Tick Event - install happens during transition
-    tick = queued.to.itself(unless="can_install") | queued.to(installed, cond="can_install", on="on_install")
-
-    def can_install(self) -> bool:
-        """Check if binary installation can start."""
-        return bool(self.binary.name and self.binary.binproviders)
-
-    @queued.enter
-    def enter_queued(self):
-        """Binary is queued for installation."""
-        self.binary.update_and_requeue(
-            retry_at=timezone.now(),
-            status=Binary.StatusChoices.QUEUED,
-        )
-
-    def on_install(self):
-        """Called during queued→installed transition. Runs installation synchronously."""
-        import sys
-
-        rprint(f"[cyan]      🔄 BinaryMachine.on_install() - installing {self.binary.name}[/cyan]", file=sys.stderr)
-
-        # Run installation hooks (synchronous, updates abspath/version/sha256 and sets status)
-        self.binary.run()
-
-        # Check if installation succeeded by looking at updated status
-        # Note: Binary.run() updates self.binary.status internally but doesn't refresh our reference
-        self.binary.refresh_from_db()
-
-        if self.binary.status != Binary.StatusChoices.INSTALLED:
-            # Installation failed - abort transition, stay in queued
-            rprint(f"[red]      ❌ BinaryMachine - {self.binary.name} installation failed, retrying later[/red]", file=sys.stderr)
-
-            # Bump retry_at to try again later
-            self.binary.update_and_requeue(
-                retry_at=timezone.now() + timedelta(seconds=300),  # Retry in 5 minutes
-                status=Binary.StatusChoices.QUEUED,  # Ensure we stay queued
-            )
-
-            # Increment health stats for failure
-            self.binary.increment_health_stats(success=False)
-
-            # Abort the transition - this will raise an exception and keep us in queued
-            raise Exception(f"Binary {self.binary.name} installation failed")
-
-        rprint(f"[cyan]      ✅ BinaryMachine - {self.binary.name} installed successfully[/cyan]", file=sys.stderr)
-
-    @installed.enter
-    def enter_installed(self):
-        """Binary installed successfully."""
-        self.binary.update_and_requeue(
-            retry_at=None,
-            status=Binary.StatusChoices.INSTALLED,
-        )
-
-        # Increment health stats
-        self.binary.increment_health_stats(success=True)
-
-
-# =============================================================================
-# Process State Machine
-# =============================================================================
-
-
-class ProcessMachine(BaseStateMachine):
-    """
-    State machine for managing Process (OS subprocess) lifecycle.
-
-    Process Lifecycle:
-    ┌─────────────────────────────────────────────────────────────┐
-    │ QUEUED State                                                │
-    │  • Process ready to launch, waiting for resources           │
-    └─────────────────────────────────────────────────────────────┘
-                            ↓ tick() when can_start()
-    ┌─────────────────────────────────────────────────────────────┐
-    │ RUNNING State → enter_running()                             │
-    │  1. process.launch()                                        │
-    │     • Spawn subprocess with cmd, pwd, env, timeout          │
-    │     • Set pid, started_at                                   │
-    │     • Process runs in background or foreground              │
-    │  2. Monitor process completion                              │
-    │     • Check exit code when process completes                │
-    └─────────────────────────────────────────────────────────────┘
-                            ↓ tick() checks is_exited()
-    ┌─────────────────────────────────────────────────────────────┐
-    │ EXITED State                                                │
-    │  • Process completed (exit_code set)                        │
-    │  • Health stats incremented                                 │
-    │  • stdout/stderr captured                                   │
-    └─────────────────────────────────────────────────────────────┘
-
-    Note: This is a simpler state machine than ArchiveResult.
-    Process is just about execution lifecycle. ArchiveResult handles
-    the archival-specific logic (status, output parsing, etc.).
-    """
-
-    model_attr_name = "process"
-    process: Process
-
-    # States
-    queued = State(value=Process.StatusChoices.QUEUED, initial=True)
-    running = State(value=Process.StatusChoices.RUNNING)
-    exited = State(value=Process.StatusChoices.EXITED, final=True)
-
-    # Tick Event - transitions based on conditions
-    tick = (
-        queued.to.itself(unless="can_start")
-        | queued.to(running, cond="can_start")
-        | running.to.itself(unless="is_exited")
-        | running.to(exited, cond="is_exited")
-    )
-
-    # Additional events (for explicit control)
-    launch = queued.to(running)
-    kill = running.to(exited)
-
-    def can_start(self) -> bool:
-        """Check if process can start (has cmd and machine)."""
-        return bool(self.process.cmd and self.process.machine)
-
-    def is_exited(self) -> bool:
-        """Check if process has exited (exit_code is set)."""
-        return self.process.exit_code is not None
-
-    @queued.enter
-    def enter_queued(self):
-        """Process is queued for execution."""
-        self.process.update_and_requeue(
-            retry_at=timezone.now(),
-            status=Process.StatusChoices.QUEUED,
-        )
-
-    @running.enter
-    def enter_running(self):
-        """Start process execution."""
-        # Lock the process while it runs
-        self.process.update_and_requeue(
-            retry_at=timezone.now() + timedelta(seconds=self.process.timeout),
-            status=Process.StatusChoices.RUNNING,
-            started_at=timezone.now(),
-        )
-
-        # Launch the subprocess
-        # NOTE: This is a placeholder - actual launch logic would
-        # be implemented based on how hooks currently spawn processes
-        # For now, Process is a data model that tracks execution metadata
-        # The actual subprocess spawning is still handled by run_hook()
-
-        # Mark as immediately exited for now (until we refactor run_hook)
-        # In the future, this would actually spawn the subprocess
-        self.process.exit_code = 0  # Placeholder
-        self.process.save()
-
-    @exited.enter
-    def enter_exited(self):
-        """Process has exited."""
-        self.process.update_and_requeue(
-            retry_at=None,
-            status=Process.StatusChoices.EXITED,
-            ended_at=timezone.now(),
-        )
-
-
-# =============================================================================
-# State Machine Registration
-# =============================================================================
-
-# Manually register state machines with python-statemachine registry
-registry.register(BinaryMachine)
-registry.register(ProcessMachine)
@@ -48,6 +48,11 @@ def _exit_on_migration_interrupt():
            os.write(sys.stderr.fileno(), _migration_interrupt_message().encode())
        except Exception:
            pass
+        # Django's migration executor can catch or delay normal exceptions while
+        # unwinding transactions. Use the real process exit path after printing
+        # the recovery command so Ctrl+C/SIGTERM during auto-migrations does not
+        # leave `archivebox server` apparently hung after the user asked it to
+        # stop. Migrations are atomic, so this does not record partial progress.
        os._exit(130)

    try:
@@ -250,7 +255,7 @@ def check_data_dir_permissions(config=None, **config_kwargs):
            "    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#permissions]https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#permissions[/link]",
        )
        STDERR.print(
-            "    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#puid--pgid]https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#puid--pgid[/link]",
+            "    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#puid--pgid]https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#puid--pgid[/link]",
        )
        STDERR.print(
            "    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#filesystem-doesnt-support-fsync-eg-network-mounts]https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#filesystem-doesnt-support-fsync-eg-network-mounts[/link]",
@@ -31,11 +31,22 @@ from archivebox.misc.util import enforce_types
 # ``CONSTANTS.DATABASE_FILE`` directly.


-def is_postgres() -> bool:
-    """True if DATABASE_ENGINE selects postgres (sqlite is the default)."""
-    from archivebox.config.common import get_config
+_IS_POSTGRES: bool | None = None

-    return (get_config().DATABASE_ENGINE or "sqlite").strip().lower().startswith("postgres")
+
+def is_postgres(config: Any | None = None) -> bool:
+    """True if this process started with the PostgreSQL database backend."""
+    global _IS_POSTGRES
+
+    if _IS_POSTGRES is None:
+        if config is None:
+            from archivebox.config.common import get_config
+
+            config = get_config()
+
+        _IS_POSTGRES = (config.DATABASE_ENGINE or "sqlite").strip().lower().startswith("postgres")
+
+    return _IS_POSTGRES


 def postgres_db_params() -> dict[str, str]:
@@ -369,29 +380,40 @@ def log_sqlite_lock_holders(console: Any, *, db_path: Path = CONSTANTS.DATABASE_
 def sqlite_lock_error(error: BaseException) -> bool:
    from django.db import OperationalError as DjangoOperationalError

-    return isinstance(error, (SQLiteOperationalError, DjangoOperationalError)) and "database is locked" in str(error).lower()
+    message = str(error).lower()
+    return isinstance(error, (SQLiteOperationalError, DjangoOperationalError)) and (
+        "database is locked" in message or "database table is locked" in message
+    )


 def retry_sqlite_locks(action: Callable[[], Any], *, label: str, stderr: TextIO | None = None) -> Any:
+    from django.conf import settings
    from django.db import OperationalError, connections
    from rich.console import Console

    console = Console(file=stderr or None, stderr=stderr is None)
+    started_at = time.monotonic()
+    retry_timeout = settings.CONFIG.SQLITE_LOCK_RETRY_TIMEOUT
+    retry_interval = settings.CONFIG.SQLITE_LOCK_RETRY_INTERVAL
    while True:
        try:
            return action()
-        except OperationalError as err:
-            if "database is locked" not in str(err).lower():
-                raise
-        except SQLiteOperationalError as err:
+        except (OperationalError, SQLiteOperationalError) as err:
            if not sqlite_lock_error(err):
                raise
+            if retry_timeout:
+                remaining = retry_timeout - (time.monotonic() - started_at)
+                if remaining <= 0:
+                    raise
+                sleep_for = min(retry_interval, remaining)
+            else:
+                sleep_for = retry_interval

        connections.close_all()
-        console.print(f"[yellow][*] SQLite database is locked while {label}; retrying in 5s...[/yellow]")
+        console.print(f"[yellow][*] SQLite database is locked while {label}; retrying in {sleep_for:g}s...[/yellow]")
        log_sqlite_lock_holders(console)
        with console.status("[yellow]Waiting for SQLite database lock to clear...[/yellow]", spinner="dots"):
-            time.sleep(5.0)
+            time.sleep(sleep_for)


@contextmanager
@@ -1,260 +0,0 @@
-# Bootable hashing helpers (file content + mtime/size cache, mime detection).
-# MUST NOT import archivebox.config, archivebox.core, or Django — stdlib only.
-
-import hashlib
-import mimetypes
-from functools import lru_cache
-from pathlib import Path
-from collections.abc import Callable
-from datetime import datetime
-
-
-@lru_cache(maxsize=1024)
-def _cached_file_hash(filepath: str, size: int, mtime: float) -> str:
-    """Internal function to calculate file hash with cache key based on path, size and mtime."""
-    sha256_hash = hashlib.sha256()
-
-    with open(filepath, "rb") as f:
-        for chunk in iter(lambda: f.read(4096), b""):
-            sha256_hash.update(chunk)
-
-    return sha256_hash.hexdigest()
-
-
-@lru_cache(maxsize=10)
-def hash_file(file_path: Path, pwd: Path | None = None) -> str:
-    """Calculate SHA256 hash of a file with caching based on path, size and mtime."""
-    pwd = Path(pwd) if pwd else None
-    file_path = Path(file_path)
-    if not file_path.is_absolute():
-        file_path = pwd / file_path if pwd else file_path.absolute()
-
-    abs_path = file_path.resolve()
-    stat_info = abs_path.stat()
-
-    return _cached_file_hash(
-        str(abs_path),
-        stat_info.st_size,
-        stat_info.st_mtime,
-    )
-
-
-@lru_cache(maxsize=10)
-def get_dir_hashes(dir_path: Path, pwd: Path | None = None, filter_func: Callable | None = None, max_depth: int = -1) -> dict[str, str]:
-    """Calculate SHA256 hashes for all files and directories recursively."""
-    pwd = Path(pwd) if pwd else None
-    dir_path = Path(dir_path)
-    if not dir_path.is_absolute():
-        dir_path = pwd / dir_path if pwd else dir_path.absolute()
-
-    if not dir_path.is_dir():
-        raise ValueError(f"Not a directory: {dir_path}")
-    if max_depth < -1:
-        raise ValueError(f"max_depth must be >= -1, got {max_depth}")
-
-    # Get all files recursively
-    all_files = get_dir_entries(
-        dir_path,
-        pwd=pwd,
-        recursive=True,
-        include_files=True,
-        include_dirs=False,
-        filter_func=filter_func,
-    )
-
-    hashes: dict[str, str] = {}
-    hashable_summary = []
-
-    # Calculate hashes for all files
-    for subfile in all_files:
-        subfile_path = dir_path / subfile
-        sha256_hash = hash_file(subfile_path)
-        hashes[subfile] = sha256_hash
-        hashable_summary.append(f"{sha256_hash}  ./{subfile}")
-
-    # Calculate hashes for all directories
-    subdirs = get_dir_entries(
-        dir_path,
-        pwd=pwd,
-        recursive=True,
-        include_files=False,
-        include_dirs=True,
-        include_hidden=False,
-        filter_func=filter_func,
-        max_depth=max_depth,
-    )
-
-    for subdir in subdirs:
-        subdir_path = dir_path / subdir
-        subdir_hashes = get_dir_hashes(
-            subdir_path,
-            filter_func=filter_func,
-            max_depth=0,
-        )
-        hashes[subdir] = subdir_hashes["."]
-
-    # Filter results by max_depth
-    if max_depth >= 0:
-        hashes = {path: value for path, value in hashes.items() if len(Path(path).parts) <= max_depth + 1}
-
-    # Calculate root directory hash
-    hashable_summary.sort()
-    root_sha256 = hashlib.sha256("\n".join(hashable_summary).encode()).hexdigest()
-    hashes["."] = root_sha256
-
-    return hashes
-
-
-@lru_cache(maxsize=128)
-def get_dir_entries(
-    dir_path: Path,
-    pwd: Path | None = None,
-    recursive: bool = True,
-    include_files: bool = True,
-    include_dirs: bool = True,
-    include_hidden: bool = False,
-    filter_func: Callable | None = None,
-    max_depth: int = -1,
-) -> tuple[str, ...]:
-    """Get filtered list of directory entries."""
-    pwd = Path(pwd) if pwd else None
-    dir_path = Path(dir_path)
-    if not dir_path.is_absolute():
-        dir_path = pwd / dir_path if pwd else dir_path.absolute()
-
-    results = []
-
-    def process_path(path: Path, depth: int):
-        if not include_hidden and path.name.startswith("."):
-            return False
-        if max_depth >= 0 and depth > max_depth:
-            return False
-        if filter_func:
-            info = {
-                "abspath": str(path.absolute()),
-                "relpath": str(path.relative_to(dir_path)),
-            }
-            if not filter_func(info):
-                return False
-        return True
-
-    for path in dir_path.rglob("*") if recursive else dir_path.glob("*"):
-        current_depth = len(path.relative_to(dir_path).parts)
-
-        if path.is_file() and include_files and process_path(path, current_depth):
-            results.append(str(path.relative_to(dir_path)))
-        elif path.is_dir() and include_dirs and process_path(path, current_depth):
-            results.append(str(path.relative_to(dir_path)))
-
-        if not recursive:
-            break
-
-    return tuple(sorted(results))  # Make immutable for caching
-
-
-@lru_cache(maxsize=1024)
-def get_dir_sizes(dir_path: Path, pwd: Path | None = None, **kwargs) -> dict[str, int]:
-    """Calculate sizes for all files and directories recursively."""
-    sizes: dict[str, int] = {}
-    hashes = get_dir_hashes(dir_path, pwd=pwd, **kwargs)
-    dir_path = Path(dir_path)
-
-    for path_key in hashes:
-        full_path = dir_path / path_key
-        if full_path.is_file():
-            sizes[path_key] = full_path.stat().st_size
-        else:
-            total = 0
-            for file_path in full_path.rglob("*"):
-                if file_path.is_file() and not file_path.name.startswith("."):
-                    total += file_path.stat().st_size
-            sizes[path_key + "/"] = total
-
-    return sizes
-
-
-@lru_cache(maxsize=10)
-def get_dir_info(dir_path: Path, pwd: Path | None = None, filter_func: Callable | None = None, max_depth: int = -1) -> dict:
-    """Get detailed information about directory contents including hashes and sizes."""
-    pwd = Path(pwd) if pwd else None
-    dir_path = Path(dir_path)
-    if not dir_path.is_absolute():
-        dir_path = pwd / dir_path if pwd else dir_path.absolute()
-
-    hashes = get_dir_hashes(dir_path, pwd=pwd, filter_func=filter_func, max_depth=max_depth)
-    sizes = get_dir_sizes(str(dir_path), pwd=pwd, filter_func=filter_func, max_depth=max_depth)
-
-    num_total_subpaths = sum(1 for name in hashes if name != ".")
-    details = {}
-
-    for filename, sha256_hash in sorted(hashes.items()):
-        abs_path = (dir_path / filename).resolve()
-        stat_info = abs_path.stat()
-        num_subpaths = sum(1 for p in hashes if p.startswith(filename + "/"))
-        is_dir = abs_path.is_dir()
-        if is_dir:
-            mime_type = "inode/directory"
-            basename = abs_path.name
-            extension = ""
-            num_bytes = sizes[filename + "/"]
-            if filename == ".":
-                num_subpaths = num_total_subpaths
-            else:
-                filename += "/"
-                num_subpaths = num_subpaths
-        else:  # is_file
-            num_subpaths = None
-            mime_type = mimetypes.guess_type(str(abs_path))[0]
-            extension = abs_path.suffix
-            basename = abs_path.name.rsplit(extension, 1)[0]
-            num_bytes = sizes[filename]
-
-        details[filename] = {
-            "basename": basename,
-            "mime_type": mime_type,
-            "extension": extension,
-            "num_subpaths": num_subpaths,
-            "num_bytes": num_bytes,
-            "hash_sha256": sha256_hash,
-            "created_at": datetime.fromtimestamp(stat_info.st_ctime).isoformat(),
-            "modified_at": datetime.fromtimestamp(stat_info.st_mtime).isoformat(),
-        }
-
-        if filter_func and not filter_func(details[filename]):
-            del details[filename]
-
-    return details
-
-
-if __name__ == "__main__":
-    import json
-
-    dir_info = get_dir_info(Path("."), max_depth=6)
-    with open(".hashes.json", "w") as f:
-        json.dump(dir_info, f, indent=4)
-    print("Wrote .hashes.json")
-
-# Example output:
-# {
-#     ".": {
-#         "basename": "misc",
-#         "mime_type": "inode/directory",
-#         "extension": "",
-#         "num_subpaths": 25,
-#         "num_bytes": 214677,
-#         "hash_sha256": "addfacf88b2ff6b564846415fb7b21dcb7e63ee4e911bc0aec255ee354958530",
-#         "created_at": "2024-12-04T00:08:38.537449",
-#         "modified_at": "2024-12-04T00:08:38.537449"
-#     },
-#     "__init__.py": {
-#         "basename": "__init__",
-#         "mime_type": "text/x-python",
-#         "extension": ".py",
-#         "num_subpaths": null,
-#         "num_bytes": 32,
-#         "hash_sha256": "b0e5e7ff17db3b60535cf664282787767c336e3e203a43e21b6326c6fe457551",
-#         "created_at": "2024-10-08T00:51:41.001359",
-#         "modified_at": "2024-10-08T00:51:41.001359"
-#     },
-#     ...
-# }
@@ -32,7 +32,6 @@ import sys
 import json
 from typing import Any, TextIO
 from collections.abc import Iterable, Iterator
-from pathlib import Path


 # Type constants for JSONL records
@@ -120,19 +119,6 @@ def read_stdin(stream: TextIO | None = None) -> Iterator[dict[str, Any]]:
            yield record


-def read_file(path: Path) -> Iterator[dict[str, Any]]:
-    """
-    Read JSONL or plain URLs from a file.
-
-    Yields parsed records as dicts.
-    """
-    with open(path) as f:
-        for line in f:
-            record = parse_line(line)
-            if record:
-                yield record
-
-
 def read_args_or_stdin(args: Iterable[str], stream: TextIO | None = None) -> Iterator[dict[str, Any]]:
    """
    Read from CLI arguments if provided, otherwise from stdin.
@@ -15,16 +15,14 @@ from multiprocessing import Process
 from pathlib import Path

 from datetime import datetime, timezone
-from typing import Any, Optional, IO, TYPE_CHECKING, cast
-from collections.abc import Iterable
+from typing import Any, Optional, TYPE_CHECKING, cast

 if TYPE_CHECKING:
    from archivebox.core.models import Snapshot

 from rich import print
-from rich.panel import Panel

-from archivebox.config import CONSTANTS, VERSION
+from archivebox.config import CONSTANTS
 from archivebox.config.common import get_config
 from archivebox.misc.util import enforce_types
 from archivebox.misc.logging import ANSI
@@ -136,17 +134,6 @@ def progress_bar(seconds: int, prefix: str = "", ANSI: dict[str, str] = ANSI, co
        print()


-def log_cli_command(subcommand: str, subcommand_args: Iterable[str] = (), stdin: str | IO | None = None, pwd: str = "."):
-    args = " ".join(subcommand_args)
-    version_msg = "[dark_magenta]\\[{now}][/dark_magenta] [dark_red]ArchiveBox[/dark_red] [dark_goldenrod]v{VERSION}[/dark_goldenrod]: [green4]archivebox [green3]{subcommand}[green2] {args}[/green2]".format(
-        now=datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S"),
-        VERSION=VERSION,
-        subcommand=subcommand,
-        args=args,
-    )
-    print(Panel(version_msg), file=sys.stderr)
-
-
 def log_list_started(filter_patterns: list[str] | None, filter_type: str):
    print(f"[green][*] Finding links in the archive index matching these {filter_type} patterns:[/]")
    print("    {}".format(" ".join(filter_patterns or ())))
@@ -33,6 +33,26 @@ from archivebox.misc.logging_util import printable_filesize
 _HASHES_CACHE: dict[Path, tuple[float, dict[str, str]]] = {}
 IMG_SRC_ATTR_RE = re.compile(r'(<img\b[^>]*?\s(?:src|data-src)=["\'])([^"\']+)(["\'])', re.IGNORECASE)
 TRANSFORMED_HTML_PREVIEW_STYLE = """<style id="archivebox-static-html-preview-style">
+html {
+    width: 100%;
+    min-width: 100%;
+    background: #fff;
+}
+body {
+    box-sizing: border-box;
+    width: min(100%, 72rem);
+    max-width: none;
+    min-height: 100vh;
+    margin: 0 auto;
+    padding: clamp(1rem, 3vw, 2rem);
+    background: #fff;
+    color: #111827;
+    font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
+    line-height: 1.55;
+}
+body > * {
+    max-width: 100%;
+}
 img:not([width]):not([height]) {
    max-width: min(100%, 12rem);
    max-height: 12rem;
@@ -302,11 +322,19 @@ def _render_directory_index(request, path: str, fullpath: Path) -> HttpResponse:
    if zip_query:
        zip_url = f"{zip_url}?{zip_query.urlencode()}"

+    directory_query = "?files=1" if request.GET.get("files") else ""
+    path_suffix = f"{path.strip('/')}/" if path.strip("/") else ""
+    snapshot_page_url = request.path
+    if path_suffix and snapshot_page_url.endswith(path_suffix):
+        snapshot_page_url = snapshot_page_url[: -len(path_suffix)]
+
    context = {
        "directory": f"{path}/",
        "file_list": file_list,
        "entries": entries,
        "zip_url": zip_url,
+        "directory_query": directory_query,
+        "snapshot_page_url": snapshot_page_url,
    }
    return HttpResponse(template.render(context))

@@ -476,12 +504,7 @@ def _render_image_preview_document(image_url: str, title: str) -> str:
 </html>"""


-def _responses_path_for_html_image(
-    snapshot_root: Path,
-    html_rel_path: str,
-    image_url: str,
-    page_url: str | None,
-) -> str | None:
+def _encoded_responses_image_url(image_url: str, page_url: str | None) -> str | None:
    raw_url = str(image_url or "").strip()
    if not raw_url or raw_url.startswith(("#", "data:", "blob:", "about:", "javascript:")):
        return None
@@ -490,23 +513,40 @@ def _responses_path_for_html_image(
    if not absolute_url.startswith(("http://", "https://")):
        return None

-    responses_root = snapshot_root / "responses"
-    if not responses_root.is_dir():
-        return None
+    return quote(absolute_url, safe="").replace("%", "_")

-    encoded_url = quote(absolute_url, safe="").replace("%", "_")
-    best_match = None
-    for candidate in responses_root.rglob(f"*__GET__{encoded_url}*"):
+
+def _index_responses_paths_for_html_images(
+    snapshot_root: Path,
+    html_rel_path: str,
+    encoded_urls: set[str],
+) -> dict[str, str]:
+    responses_root = snapshot_root / "responses"
+    if not encoded_urls or not responses_root.is_dir():
+        return {}
+
+    best_matches: dict[str, str] = {}
+    image_matches: set[str] = set()
+    image_suffixes = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".ico"}
+    for candidate in responses_root.rglob("*"):
        if not candidate.is_file():
            continue
        try:
            rel_path = candidate.relative_to(snapshot_root)
        except ValueError:
            continue
-        best_match = posixpath.relpath(rel_path.as_posix(), start=posixpath.dirname(html_rel_path) or ".")
-        if candidate.suffix.lower() in {".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".ico"}:
-            return best_match
-    return best_match
+        relative_match = posixpath.relpath(rel_path.as_posix(), start=posixpath.dirname(html_rel_path) or ".")
+        candidate_name = candidate.name
+        is_image = candidate.suffix.lower() in image_suffixes
+        for encoded_url in encoded_urls:
+            if encoded_url in image_matches or f"__GET__{encoded_url}" not in candidate_name:
+                continue
+            best_matches[encoded_url] = relative_match
+            if is_image:
+                # Preserve the old rglob behavior: the first image match wins,
+                # otherwise the last matching response file is used.
+                image_matches.add(encoded_url)
+    return best_matches


 def _rewrite_html_image_sources_to_responses(
@@ -515,11 +555,24 @@ def _rewrite_html_image_sources_to_responses(
    html_rel_path: str,
    page_url: str | None,
 ) -> tuple[str, int]:
+    encoded_urls_by_src: dict[str, str | None] = {}
+    for match in IMG_SRC_ATTR_RE.finditer(html_text):
+        image_url = html.unescape(match.group(2))
+        if image_url not in encoded_urls_by_src:
+            encoded_urls_by_src[image_url] = _encoded_responses_image_url(image_url, page_url)
+
+    response_paths = _index_responses_paths_for_html_images(
+        snapshot_root,
+        html_rel_path,
+        {encoded_url for encoded_url in encoded_urls_by_src.values() if encoded_url},
+    )
    rewrites = 0

    def replace_src(match: re.Match[str]) -> str:
        nonlocal rewrites
-        local_path = _responses_path_for_html_image(snapshot_root, html_rel_path, html.unescape(match.group(2)), page_url)
+        image_url = html.unescape(match.group(2))
+        encoded_url = encoded_urls_by_src.get(image_url)
+        local_path = response_paths.get(encoded_url or "")
        if not local_path:
            return match.group(0)
        rewrites += 1
@@ -857,6 +910,9 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
                )

    content_type, encoding = mimetypes.guess_type(str(fullpath))
+    preserve_plain_text = fullpath.suffix.lower() in {".log", ".sh"}
+    if preserve_plain_text:
+        content_type = "text/plain"
    content_type = content_type or "application/octet-stream"
    # Add charset for text-like types (best guess), but don't override the type.
    is_text_like = content_type.startswith("text/") or content_type in {
@@ -967,7 +1023,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_

    # Heuristic fix: some archived HTML outputs are stored with HTML-escaped markup
    # or markdown sources. If so, render sensibly.
-    if content_type.startswith(("text/plain", "text/html")):
+    if not preserve_plain_text and content_type.startswith(("text/plain", "text/html")):
        try:
            max_unescape_size = 10 * 1024 * 1024  # 10MB cap to avoid heavy memory use
            if statobj.st_size <= max_unescape_size:
@@ -2,94 +2,11 @@ from typing import Any, cast
 from collections.abc import Callable

 import json
-import ast
-import inspect
 import toml
 import re
-import configparser

 from pathlib import Path, PosixPath

-from pydantic.json_schema import GenerateJsonSchema
-from pydantic_core import to_jsonable_python
-
-JSONValue = str | bool | int | None | list["JSONValue"]
-
-TOML_HEADER = "# Converted from INI to TOML format: https://toml.io/en/\n\n"
-
-
-def load_ini_value(val: str) -> JSONValue:
-    """Convert lax INI values into strict TOML-compliant (JSON) values"""
-    if val.lower() in ("true", "yes", "1"):
-        return True
-    if val.lower() in ("false", "no", "0"):
-        return False
-    if val.isdigit():
-        return int(val)
-
-    try:
-        return ast.literal_eval(val)
-    except Exception:
-        pass
-
-    try:
-        return json.loads(val)
-    except Exception:
-        pass
-
-    return val
-
-
-def convert(ini_str: str) -> str:
-    """Convert a string of INI config into its TOML equivalent (warning: strips comments)"""
-
-    config = configparser.ConfigParser()
-    setattr(config, "optionxform", str)  # capitalize key names
-    config.read_string(ini_str)
-
-    # Initialize an empty dictionary to store the TOML representation
-    toml_dict = {}
-
-    # Iterate over each section in the INI configuration
-    for section in config.sections():
-        toml_dict[section] = {}
-
-        # Iterate over each key-value pair in the section
-        for key, value in config.items(section):
-            parsed_value = load_ini_value(value)
-
-            # Convert the parsed value to its TOML-compatible JSON representation
-            toml_dict[section.upper()][key.upper()] = json.dumps(parsed_value)
-
-    # Build the TOML string
-    toml_str = TOML_HEADER
-    for section, items in toml_dict.items():
-        toml_str += f"[{section}]\n"
-        for key, value in items.items():
-            toml_str += f"{key} = {value}\n"
-        toml_str += "\n"
-
-    return toml_str.strip()
-
-
-class JSONSchemaWithLambdas(GenerateJsonSchema):
-    """
-    Encode lambda functions in default values properly.
-    Usage:
-    >>> json.dumps(value, encoder=JSONSchemaWithLambdas())
-    """
-
-    def encode_default(self, dft: Any) -> Any:
-        config = self._config
-        if isinstance(dft, Callable):
-            return "{{lambda " + inspect.getsource(dft).split("=lambda ")[-1].strip()[:-1] + "}}"
-        return to_jsonable_python(
-            dft,
-            timedelta_mode=config.ser_json_timedelta,
-            bytes_mode=config.ser_json_bytes,
-            serialize_unknown=True,
-        )
-

 def better_toml_dump_str(val: Any) -> str:
    try:
@@ -192,7 +192,10 @@ def validate_url_length(url: str) -> str:


 def validate_url(url: str) -> str:
-    url = validate_url_length((url or "").strip())
+    url = (url or "").strip()
+    if "\n" in url or "\r" in url:
+        raise ValueError("URL must be a single line.")
+    url = validate_url_length(url)
    parsed = urlparse(url)
    if parsed.scheme.lower() not in ("http", "https") or not parsed.hostname:
        raise ValueError("URL must start with http:// or https:// and include a hostname.")
@@ -1,6 +0,0 @@
-from django.apps import AppConfig
-
-
-class OpencodeConfig(AppConfig):
-    name = "archivebox.opencode"
-    label = "opencode_plugin"
@@ -1,211 +0,0 @@
-{% extends "admin/base.html" %}
-{% load i18n %}
-
-{% block title %}Agent{% endblock %}
-
-{% block breadcrumbs %}
-    <div class="breadcrumbs">
-        <a href="{% url 'admin:index' %}">{% trans 'Home' %}</a>
-        &rsaquo; Agent
-    </div>
-{% endblock %}
-
-{% block extrastyle %}
-    {{ block.super }}
-    <style>
-        body.opencode-agent #content {
-            padding: 0;
-        }
-        body.opencode-agent #content-main {
-            height: calc(100vh - 150px);
-            min-height: 560px;
-        }
-        .opencode-agent-error {
-            margin: 24px;
-            color: #0f172a;
-            font: 14px -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
-        }
-        .opencode-agent-error code {
-            color: #0369a1;
-        }
-        .opencode-agent-frame {
-            width: 100%;
-            height: 100%;
-            border: 0;
-            display: block;
-        }
-        .opencode-agent-shell {
-            position: relative;
-            height: 100%;
-        }
-        .opencode-agent-welcome {
-            position: absolute;
-            inset: 0;
-            z-index: 20;
-            display: flex;
-            align-items: center;
-            justify-content: center;
-            padding: 24px;
-            background: rgba(0, 0, 0, 0.48);
-            color: #111827;
-        }
-        .opencode-agent-welcome[hidden] {
-            display: none;
-        }
-        .opencode-agent-welcome-panel {
-            width: min(720px, 100%);
-            max-height: min(720px, calc(100vh - 190px));
-            overflow: auto;
-            border-radius: 8px;
-            border: 1px solid #d1d5db;
-            background: Canvas;
-            color: CanvasText;
-            box-shadow: 0 20px 60px rgba(0, 0, 0, 0.32);
-            padding: 24px;
-            font: 14px/1.5 -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
-        }
-        .opencode-agent-welcome-panel h1 {
-            margin: 0 0 10px;
-            font-size: 22px;
-            line-height: 1.25;
-        }
-        .opencode-agent-welcome-panel p {
-            margin: 0 0 14px;
-        }
-        .opencode-agent-welcome-panel ul {
-            margin: 0 0 16px 20px;
-            padding: 0;
-        }
-        .opencode-agent-welcome-panel li {
-            margin: 0 0 8px;
-        }
-        .opencode-agent-welcome-warning {
-            margin: 16px 0;
-            padding: 12px 14px;
-            border: 1px solid #f59e0b;
-            border-radius: 6px;
-            background: color-mix(in srgb, #f59e0b 12%, Canvas);
-        }
-        .opencode-agent-welcome-actions {
-            display: flex;
-            justify-content: flex-end;
-            gap: 8px;
-        }
-        .opencode-agent-welcome button {
-            cursor: pointer;
-            border: 1px solid #2563eb;
-            border-radius: 6px;
-            background: #2563eb;
-            color: #fff;
-            padding: 8px 14px;
-            font-weight: 600;
-        }
-        @media (max-width: 700px) {
-            .opencode-agent-welcome {
-                padding: 12px;
-            }
-            .opencode-agent-welcome-panel {
-                max-height: calc(100vh - 170px);
-                padding: 18px;
-            }
-        }
-    </style>
-{% endblock %}
-
-{% block bodyclass %}{{ block.super }} opencode-agent{% endblock %}
-
-{% block content %}
-    <div id="content-main">
-        {% if error %}
-            <div class="opencode-agent-error">
-                {{ error }}
-                {% if command %}<br><br><code>{{ command }}</code>{% endif %}
-            </div>
-        {% else %}
-            <script>
-                (() => {
-                    const workdir = "{{ workdir|escapejs }}";
-                    const recentSessionId = "{{ recent_session_id|escapejs }}";
-                    const merge = (left, right) => {
-                        if (!left || typeof left !== "object" || Array.isArray(left)) return right;
-                        const out = {...left};
-                        for (const [key, value] of Object.entries(right)) {
-                            out[key] = value && typeof value === "object" && !Array.isArray(value) ? merge(out[key], value) : value;
-                        }
-                        return out;
-                    };
-                    const seed = (key, value) => {
-                        try {
-                            localStorage.setItem(key, JSON.stringify(merge(JSON.parse(localStorage.getItem(key) || "{}"), value)));
-                        } catch {
-                            localStorage.setItem(key, JSON.stringify(value));
-                        }
-                    };
-                    const setJSON = (key, value) => localStorage.setItem(key, JSON.stringify(value));
-                    const page = {
-                        activeProject: workdir,
-                        activeWorkspace: workdir,
-                        gettingStartedDismissed: true,
-                        workspaceExpanded: {[workdir]: true},
-                    };
-                    if (recentSessionId) {
-                        page.lastProjectSession = {[workdir]: {directory: workdir, id: recentSessionId, at: Date.now()}};
-                    }
-                    seed("opencode.global.dat:layout.page", page);
-                    seed("opencode.global.dat:layout", {
-                        sidebar: {opened: true, width: 280},
-                        fileTree: {opened: true, width: 280, tab: "all"},
-                    });
-                    const serverKey = `${location.origin}/admin/agent/opencode`;
-                    setJSON("opencode.global.dat:server", {
-                        list: [],
-                        projects: {
-                            [serverKey]: [{worktree: workdir, expanded: true}],
-                        },
-                        lastProject: {
-                            [serverKey]: workdir,
-                        },
-                    });
-                    localStorage.setItem("opencode.settings.dat:defaultServerUrl", serverKey);
-                })();
-            </script>
-            <div class="opencode-agent-shell">
-                <iframe src="{{ proxy_url }}" title="OpenCode Agent" class="opencode-agent-frame"></iframe>
-                <div class="opencode-agent-welcome" id="opencode-agent-welcome" hidden>
-                    <div class="opencode-agent-welcome-panel" role="dialog" aria-modal="true" aria-labelledby="opencode-agent-welcome-title">
-                        <h1 id="opencode-agent-welcome-title">ArchiveBox AI Agent</h1>
-                        <p>This agent can work directly with your ArchiveBox collection. It can inspect the crawl database, create and monitor crawls, run maintenance operations, answer research questions, query archived data, and help with complex collection workflows.</p>
-                        <p>Example prompts:</p>
-                        <ul>
-                            <li>Archive this list of URLs with depth 0, then report which ones failed and why.</li>
-                            <li>Find snapshots from the last month that failed PDF or screenshot extraction and retry them.</li>
-                            <li>Create a constrained crawl for this site, avoid login/privacy/sitemap URLs, and watch the logs as it runs.</li>
-                            <li>Summarize what my collection contains about this topic and link to the best saved pages.</li>
-                        </ul>
-                        <div class="opencode-agent-welcome-warning">
-                            The agent has unrestricted access to this collection and can make destructive edits. Review requests carefully. Permission settings can be changed from the OpenCode gear icon in the lower left.
-                        </div>
-                        <div class="opencode-agent-welcome-actions">
-                            <button type="button" id="opencode-agent-welcome-dismiss">Start using Agent</button>
-                        </div>
-                    </div>
-                </div>
-            </div>
-            <script>
-                (() => {
-                    const key = "archivebox.opencode.agentWelcomeDismissed.v1";
-                    const welcome = document.getElementById("opencode-agent-welcome");
-                    const dismiss = document.getElementById("opencode-agent-welcome-dismiss");
-                    if (!welcome || !dismiss) return;
-                    if (localStorage.getItem(key) !== "1") {
-                        welcome.hidden = false;
-                    }
-                    dismiss.addEventListener("click", () => {
-                        localStorage.setItem(key, "1");
-                        welcome.hidden = true;
-                    });
-                })();
-            </script>
-        {% endif %}
-    </div>
-{% endblock %}
@@ -1,716 +1,140 @@
-from __future__ import annotations
+"""Django adapter for the optional plugin; never import its runtime at startup."""

-import atexit
-import base64
-import os
-import re
-import shutil
-import subprocess
-import threading
-import time
-from pathlib import Path
-from typing import Any
-from urllib.parse import urljoin, urlsplit
+import logging
+from io import BytesIO

-import httpx
-import requests
-from abx_plugins.plugins import opencode as opencode_plugin
-from archivebox.config import CONSTANTS
-from archivebox.config.common import get_config
-from archivebox.core.routes_util import build_admin_url, get_api_base_url, get_base_url
-from django.http import (
-    Http404,
-    HttpRequest,
-    HttpResponse,
-    HttpResponseForbidden,
-    StreamingHttpResponse,
-)
-from django.shortcuts import redirect, render
+from asgiref.sync import sync_to_async
+
+from django.contrib.auth.middleware import AuthenticationMiddleware
+from django.contrib.auth.views import redirect_to_login
+from django.contrib.sessions.middleware import SessionMiddleware
+from django.core.exceptions import PermissionDenied
+from django.core.handlers.asgi import ASGIRequest
+from django.db import close_old_connections
+from django.http import Http404, HttpResponse, HttpResponseForbidden, StreamingHttpResponse
+from django.template import engines
+from django.urls import Resolver404, resolve
 from django.views.decorators.csrf import csrf_exempt

+from archivebox.config import CONSTANTS
+from archivebox.config.common import get_request_config
+from archivebox.core.middleware import ReverseProxyAuthMiddleware
+from archivebox.core.routes_util import build_admin_url, get_admin_host, get_api_base_url, get_base_url, host_matches
+from archivebox.plugins.discovery import get_plugin_template

-_PROCESS: subprocess.Popen | None = None
-_PROCESS_LOCK = threading.Lock()
-_PROXY_PREFIX = "/admin/agent/opencode"
-_PROXY_PREFIX_REGEX = _PROXY_PREFIX.replace("/", r"\/")
-_PROXY_PREFIX_NO_SLASH_REGEX = _PROXY_PREFIX.lstrip("/").replace("/", r"\/")
-_CONFIG_PATH = Path(opencode_plugin.__file__).with_name("config.json")
-
-_TEXT_CONTENT_TYPES = (
-    "text/",
-    "application/javascript",
-    "application/json",
-    "application/x-javascript",
-)
-_HOP_BY_HOP_HEADERS = {
-    "connection",
-    "keep-alive",
-    "proxy-authenticate",
-    "proxy-authorization",
-    "te",
-    "trailers",
-    "transfer-encoding",
-    "upgrade",
-}
-_ARCHIVEBOX_SKILL = """---
-name: archivebox
-description: Use ArchiveBox's CLI and local REST API from an ArchiveBox collection.
---
-
-You are running inside an ArchiveBox collection directory.
-
- ArchiveBox collection directory: {archivebox_data_dir}
- ArchiveBox BASE_URL: {archivebox_base_url}
- ArchiveBox Admin URL: {archivebox_admin_url}
- ArchiveBox REST API URL: {archivebox_api_url}
- Prefer the `archivebox` CLI for authenticated changes, e.g. `archivebox add`, `archivebox schedule`, `archivebox update`, and `archivebox shell`.
- Run ArchiveBox CLI commands from the ArchiveBox collection directory above.
- Get command help with `archivebox list --help`, `archivebox add --help`, `archivebox schedule --help`, etc. Do not use `archivebox help <command>`.
- Use `--depth=0` by default. Only use recursive crawling when the user explicitly asks for it; use `--depth=1` when you need pages one hop out.
- Before any recursive crawl, constrain scope with ArchiveBox config such as `CRAWL_MAX_URLS`, `CRAWL_MAX_SIZE`, `SNAPSHOT_MAX_*`, `URL_ALLOWLIST`, `URL_DENYLIST`, and related limits.
- Respect the configured `archivebox config --get ONLY_NEW` behavior unless the user explicitly says otherwise. Remind users that expected crawl URLs can be skipped when the collection already contains snapshots with the same URL.
- Always audit newly discovered crawl URLs before letting a crawl run broadly. Treat junk URLs such as privacy policies, legal pages, tag archives, sitemap files, feeds, login/logout URLs, and other low-value boilerplate as unwanted unless the user explicitly asked to archive them.
- Always watch crawl output and logs as the crawl progresses, and correct errors early instead of waiting until the crawl finishes.
- If a crawl contains bad URLs, pause it, edit the crawl's `urls` field to remove them, delete any unneeded snapshots already created under that crawl, then resume the crawl.
- Use `archivebox shell -c '...'` or `archivebox shell <<'PY' ... PY` for Django ORM work. Shell Plus prints an import banner first; keep stderr visible while debugging.
- Use full ArchiveBox module paths in shell code: `from archivebox.crawls.models import Crawl, CrawlSchedule` and `from archivebox.core.models import Snapshot, ArchiveResult`.
- If a model/field/relation is unclear, inspect `_meta.fields` before guessing, e.g. `archivebox shell -c "from archivebox.crawls.models import Crawl; print([f.name for f in Crawl._meta.fields])"`.
- Use `archivebox config --get BASE_URL` only to verify the configured base URL; prefer the seeded URLs above for API/admin requests.
- Use `$ARCHIVEBOX_API_URL` for REST API inspection when helpful. Do not assume admin session cookies authenticate API subdomain requests; prefer CLI/shell for authenticated mutations unless the admin provides or asks you to create an API token.
- Discover REST endpoints from `${{ARCHIVEBOX_API_URL}}v1/openapi.json`; crawl endpoints live under `/api/v1/crawls/`, snapshots under `/api/v1/core/`.
- Do not bypass ArchiveBox auth, expose API keys, or modify config unless the admin explicitly asks.
- After creating crawls or snapshots, report the crawl/snapshot IDs and the exact command or API request used.
-"""
+_LOGGER = logging.getLogger(__name__)


-def _stop_owned_process(process: subprocess.Popen | None = None) -> None:
-    global _PROCESS
-    owned_process = process or _PROCESS
-    if owned_process is None:
-        return
-    if owned_process.poll() is None:
-        owned_process.terminate()
-        try:
-            owned_process.wait(timeout=5)
-        except subprocess.TimeoutExpired:
-            owned_process.kill()
-            owned_process.wait()
-    if _PROCESS is owned_process:
-        _PROCESS = None
+def _runtime_settings(request, config):
+    from abx_plugins.plugins.opencode import runtime

-
-atexit.register(_stop_owned_process)
-
-
-def _machine_config() -> dict[str, Any]:
-    resolved = get_config()
-    return dict(resolved.model_dump(mode="json"))
-
-
-def _archivebox_data_dir_default() -> Path:
-    return Path(CONSTANTS.DATA_DIR)
-
-
-def _archivebox_route_urls(request: HttpRequest, route_config) -> tuple[str, str, str]:
-    base_url = get_base_url(
-        request=request,
-        config=route_config,
-    ).rstrip("/")
-    admin_url = build_admin_url(
-        "/admin/",
-        request=request,
-        config=route_config,
-    ).rstrip("/")
-    api_url = f"{get_api_base_url(request=request, config=route_config).rstrip('/')}/api/"
-    return base_url, admin_url, api_url
-
-
-def _config_value(config: dict, key: str, default):
-    value = config.get(key, default)
-    if value in (None, ""):
-        return default
-    return value
-
-
-def _opencode_enabled(config: dict) -> bool:
-    value = config.get("OPENCODE_ENABLED", False)
-    if isinstance(value, bool):
-        return value
-    return str(value).strip().lower() in {"1", "true", "yes", "on"}
-
-
-def _require_enabled(config: dict) -> None:
-    if not _opencode_enabled(config):
-        raise Http404
-
-
-def _require_superuser(request: HttpRequest):
-    user = getattr(request, "user", None)
-    if user is None:
-        return redirect(f"/admin/login/?next={request.get_full_path()}")
-    if (
-        bool(getattr(user, "is_authenticated", False))
-        and bool(getattr(user, "is_active", False))
-        and bool(getattr(user, "is_superuser", False))
-    ):
-        return None
-    if bool(getattr(user, "is_authenticated", False)):
-        return HttpResponseForbidden(
-            b"ArchiveBox agent access requires a superuser account.",
-        )
-    return redirect(f"/admin/login/?next={request.get_full_path()}")
-
-
-def _origin_allowed(request: HttpRequest, path: str | None = None) -> bool:
-    if request.method in {"GET", "HEAD", "OPTIONS", "TRACE"}:
-        return True
-
-    expected_host = request.get_host()
-    pty_connect = bool(
-        path and path.startswith("pty/") and path.endswith("/connect-token"),
-    )
-    if pty_connect:
-        return True
-
-    origin = _request_header(request, "Origin")
-    if origin:
-        return _same_host(origin, expected_host)
-
-    referer = _request_header(request, "Referer")
-    if referer:
-        return _same_host(referer, expected_host)
-
-    fetch_site = _request_header(request, "Sec-Fetch-Site")
-    if fetch_site:
-        return fetch_site in {"same-origin", "same-site", "none"}
-
-    return False
-
-
-def _same_host(value: str, expected_host: str) -> bool:
-    parsed = urlsplit(value)
-    return parsed.scheme in {"http", "https"} and parsed.netloc == expected_host
-
-
-def _settings(config: dict) -> dict:
-    host = str(_config_value(config, "OPENCODE_HOST", "127.0.0.1"))
-    port = int(_config_value(config, "OPENCODE_PORT", 4096))
-    default_data_dir = _archivebox_data_dir_default()
-    workdir = Path(
-        str(_config_value(config, "OPENCODE_WORKDIR", default_data_dir)),
-    ).expanduser()
-    opencode_dir = Path(
-        str(_config_value(config, "OPENCODE_STATE_DIR", workdir / "opencode")),
-    ).expanduser()
-    binary = str(_config_value(config, "OPENCODE_BINARY", "opencode"))
-    timeout = int(_config_value(config, "OPENCODE_TIMEOUT", 30))
-    return {
-        "host": host,
-        "port": port,
-        "origin": f"http://{host}:{port}",
-        "workdir": workdir,
-        "opencode_dir": opencode_dir,
-        "config_home": opencode_dir / "config",
-        "data_home": opencode_dir / "data",
-        "state_home": opencode_dir / "state",
-        "cache_home": opencode_dir / "cache",
-        "home": opencode_dir / "home",
-        "binary": binary,
-        "config": config,
-        "timeout": timeout,
-    }
-
-
-def _resolve_binary(binary: str, config: dict) -> tuple[Any, Any, dict[str, str]]:
-    try:
-        from abxpkg import BinProvider
-        from abx_plugins.plugins.base.utils import load_required_binary_from_config
-
-        binary_environ = os.environ.copy()
-        lib_dir = config.get("ABXPKG_LIB_DIR")
-        if lib_dir:
-            binary_environ["ABXPKG_LIB_DIR"] = str(lib_dir)
-        loaded_dependencies = [
-            load_required_binary_from_config(
-                required_binary,
-                _CONFIG_PATH,
-                global_config=config,
-                environ=binary_environ,
-                install=False,
-            )
-            for required_binary in (
-                str(config.get("NODE_BINARY") or "node"),
-                str(config.get("GIT_BINARY") or "git"),
-                binary,
-            )
-        ]
-    except Exception as err:
-        raise RuntimeError(
-            f"OpenCode dependency is not installed from required_binaries: {err}",
-        ) from err
-
-    if any(not loaded.loaded_abspath for loaded in loaded_dependencies):
-        raise RuntimeError(
-            "OpenCode dependency is not installed from required_binaries.",
-        )
-
-    providers = [loaded.loaded_binprovider for loaded in loaded_dependencies if loaded.loaded_binprovider is not None]
-    binary_env = BinProvider.build_exec_env(
-        providers=providers,
-        base_env=binary_environ,
-    )
-    return (
-        loaded_dependencies[-1],
-        loaded_dependencies[1],
-        binary_env,
-    )
-
-
-def _project_route(workdir: Path, session_id: str = "") -> str:
-    encoded = base64.b64encode(str(workdir.resolve()).encode()).decode()
-    encoded = encoded.replace("+", "-").replace("/", "_").rstrip("=")
-    route = f"{_PROXY_PREFIX}/{encoded}/session"
-    return f"{route}/{session_id}" if session_id else route
-
-
-def _ensure_project_files(settings: dict) -> None:
-    workdir = settings["workdir"].resolve()
-    workdir.mkdir(parents=True, exist_ok=True)
-    git_marker = workdir / ".git" / "not-a-git"
-    if git_marker.exists():
-        # Current OpenCode hangs on the legacy fake marker, so remove only
-        # that invalid shape before initializing the real worktree.
-        shutil.rmtree(git_marker.parent)
-
-    editable_skill_path = settings["opencode_dir"] / "SKILL.md"
-    editable_skill_path.parent.mkdir(parents=True, exist_ok=True)
-    if not editable_skill_path.exists():
-        editable_skill_path.write_text(
-            _ARCHIVEBOX_SKILL.format(
-                archivebox_data_dir=workdir,
-                archivebox_base_url=settings.get("archivebox_base_url", ""),
-                archivebox_admin_url=settings.get("archivebox_admin_url", ""),
-                archivebox_api_url=settings.get("archivebox_api_url", ""),
-            ),
-        )
-
-    opencode_skill_path = settings["config_home"] / "opencode" / "skills" / "archivebox" / "SKILL.md"
-    opencode_skill_path.parent.mkdir(parents=True, exist_ok=True)
-    if opencode_skill_path.resolve() != editable_skill_path.resolve():
-        if opencode_skill_path.exists() or opencode_skill_path.is_symlink():
-            opencode_skill_path.unlink()
-        opencode_skill_path.symlink_to(editable_skill_path)
-
-
-def _ensure_default_session(settings: dict) -> str:
-    workdir = settings["workdir"].resolve()
-    params = {"directory": str(workdir)}
-    timeout = settings["timeout"]
-    project = requests.post(
-        f"{settings['origin']}/project/git/init",
-        params=params,
-        timeout=timeout,
-    )
-    project.raise_for_status()
-    project_data = project.json()
-    if Path(str(project_data.get("worktree") or "/")).resolve() != workdir:
-        raise RuntimeError(
-            f"OpenCode initialized the wrong project worktree: {project_data.get('worktree')!r}",
-        )
-
-    sessions = requests.get(
-        f"{settings['origin']}/session",
-        params={**params, "roots": "true", "limit": 55},
-        timeout=timeout,
-    )
-    sessions.raise_for_status()
-    session_data = sessions.json()
-    if not isinstance(session_data, list):
-        raise RuntimeError("OpenCode returned an invalid project session list.")
-    for session_data_item in session_data:
-        if not isinstance(session_data_item, dict):
-            continue
-        session_id = str(session_data_item.get("id") or "")
-        session_directory = session_data_item.get("directory")
-        if session_id and session_directory and Path(str(session_directory)).resolve() == workdir:
-            return session_id
-
-    session = requests.post(
-        f"{settings['origin']}/session",
-        params=params,
-        json={},
-        timeout=timeout,
-    )
-    session.raise_for_status()
-    session_data = session.json()
-    session_id = str(session_data.get("id") or "")
-    session_directory = session_data.get("directory")
-    if not session_id or not session_directory or Path(str(session_directory)).resolve() != workdir:
-        raise RuntimeError(
-            "OpenCode did not create a session for the requested worktree.",
-        )
-    return session_id
-
-
-def _recent_session_id(settings: dict) -> str:
-    workdir = str(settings["workdir"].resolve())
-    try:
-        response = requests.get(
-            f"{settings['origin']}/session",
-            params={"directory": workdir, "roots": "true", "limit": 1},
-            timeout=settings["timeout"],
-        )
-        response.raise_for_status()
-        sessions = response.json()
-        if sessions:
-            return str(sessions[0].get("id") or "")
-    except requests.RequestException:
-        pass
-    return ""
-
-
-def _health(settings: dict) -> bool:
-    try:
-        response = requests.get(
-            f"{settings['origin']}/global/health",
-            timeout=2,
-        )
-        return response.status_code == 200
-    except requests.RequestException:
-        return False
-
-
-def _ensure_opencode(settings: dict) -> tuple[bool, str]:
-    global _PROCESS
-    started_process: subprocess.Popen | None = None
-    workdir = settings["workdir"].resolve()
-    try:
-        binary, git_binary, binary_env = _resolve_binary(
-            settings["binary"],
-            settings["config"],
-        )
-    except RuntimeError as err:
-        return False, str(err)
-
-    env = {
-        **os.environ,
-        **binary_env,
-        "ARCHIVEBOX_BASE_URL": str(settings.get("archivebox_base_url", "")),
-        "ARCHIVEBOX_ADMIN_URL": str(settings.get("archivebox_admin_url", "")),
-        "ARCHIVEBOX_API_URL": str(settings.get("archivebox_api_url", "")),
-        "BROWSER": "false",
-        "GIT_CEILING_DIRECTORIES": str(workdir),
-        "HOME": str(settings["home"]),
-        "OPENCODE_DISABLE_PROJECT_CONFIG": "true",
-        "XDG_CONFIG_HOME": str(settings["config_home"]),
-        "XDG_DATA_HOME": str(settings["data_home"]),
-        "XDG_STATE_HOME": str(settings["state_home"]),
-        "XDG_CACHE_HOME": str(settings["cache_home"]),
-    }
-
-    with _PROCESS_LOCK:
-        settings["workdir"].mkdir(parents=True, exist_ok=True)
-        settings["config_home"].mkdir(parents=True, exist_ok=True)
-        settings["data_home"].mkdir(parents=True, exist_ok=True)
-        settings["state_home"].mkdir(parents=True, exist_ok=True)
-        settings["cache_home"].mkdir(parents=True, exist_ok=True)
-        settings["home"].mkdir(parents=True, exist_ok=True)
-        _ensure_project_files(settings)
-
-        if not (workdir / ".git").exists():
-            try:
-                git_init = git_binary.exec(
-                    cmd=("init", "--quiet"),
-                    cwd=workdir,
-                    env=env,
-                    timeout=settings["timeout"],
-                )
-            except (AssertionError, OSError, subprocess.SubprocessError) as err:
-                return False, f"OpenCode project initialization failed: {err}"
-            if git_init.returncode != 0:
-                output = (git_init.stderr or git_init.stdout or "").strip()
-                return (
-                    False,
-                    f"OpenCode project initialization failed: {output or f'git exited with {git_init.returncode}'}",
-                )
-
-        if _health(settings):
-            try:
-                _ensure_default_session(settings)
-            except (requests.RequestException, RuntimeError, ValueError) as err:
-                return False, f"OpenCode project initialization failed: {err}"
-            return True, ""
-
-        binary_abspath = binary.loaded_abspath
-        if binary.loaded_binprovider is not None:
-            binary_abspath = binary.loaded_binprovider._exec_bin_abspath(
-                Path(binary.loaded_abspath),
-            )
-        cmd = [
-            str(binary_abspath),
-            "serve",
-            "--hostname",
-            settings["host"],
-            "--port",
-            str(settings["port"]),
-        ]
-        try:
-            _PROCESS = subprocess.Popen(
-                cmd,
-                cwd=workdir,
-                env=env,
-                stdin=subprocess.DEVNULL,
-                stdout=subprocess.DEVNULL,
-                stderr=subprocess.DEVNULL,
-                start_new_session=True,
-            )
-            started_process = _PROCESS
-        except FileNotFoundError:
-            return False, f"OpenCode binary not found: {settings['binary']}"
-
-        deadline = time.monotonic() + settings["timeout"]
-        while time.monotonic() < deadline:
-            if _health(settings):
-                try:
-                    _ensure_default_session(settings)
-                except (requests.RequestException, RuntimeError, ValueError) as err:
-                    _stop_owned_process(started_process)
-                    return False, f"OpenCode project initialization failed: {err}"
-                return True, ""
-            if started_process and started_process.poll() is not None:
-                if _PROCESS is started_process:
-                    _PROCESS = None
-                return False, "OpenCode exited before the web server became ready."
-            time.sleep(0.25)
-
-        _stop_owned_process(started_process)
-        return False, "Timed out waiting for OpenCode to start."
-
-
-def agent_view(request: HttpRequest):
-    config = _machine_config()
-    _require_enabled(config)
-    auth_response = _require_superuser(request)
-    if auth_response:
-        return auth_response
-
-    settings = _settings(config)
+    settings = runtime._settings(config, CONSTANTS.DATA_DIR)
    route_config = request.__dict__.get("archivebox_config")
-    base_url, admin_url, api_url = _archivebox_route_urls(request, route_config)
-    settings["archivebox_base_url"] = base_url
-    settings["archivebox_admin_url"] = admin_url
-    settings["archivebox_api_url"] = api_url
-    ok, error = _ensure_opencode(settings)
-    from archivebox.core.admin_site import archivebox_admin
-
-    recent_session_id = _recent_session_id(settings) if ok else ""
-    context = {
-        **archivebox_admin.each_context(request),
-        "title": "Agent",
-        "error": "" if ok else error,
-        "command": f"{settings['binary']} serve --hostname {settings['host']} --port {settings['port']}" if error else "",
-        # OpenCode 1.17+ keeps durable sessions on the explicit
-        # /<dirBase64>/session/<sessionId> route. We still seed localStorage
-        # below because the sidebar state uses it, but the iframe itself must
-        # open the durable session URL so a fresh browser does not land on the
-        # transient new-session route and appear to have lost prior sessions.
-        "proxy_url": _project_route(settings["workdir"], recent_session_id),
-        "workdir": str(settings["workdir"].resolve()),
-        "recent_session_id": recent_session_id,
-    }
-    return render(
-        request,
-        "opencode/agent.html",
-        context,
-        status=200 if ok else 502,
+    settings.update(
+        archivebox_base_url=get_base_url(request=request, config=route_config).rstrip("/"),
+        archivebox_admin_url=build_admin_url("/admin/", request=request, config=route_config).rstrip("/"),
+        archivebox_api_url=f"{get_api_base_url(request=request, config=route_config).rstrip('/')}/api/",
    )
+    return runtime, settings


-def _proxy_url(settings: dict, path: str | None) -> str:
-    rel = "/" if not path else f"/{path}"
-    return urljoin(settings["origin"], rel)
+def _dispatch(request, path=None):
+    try:
+        # Middleware already merged server defaults and live Machine overrides.
+        # Re-resolving every extractor's config here stalls the UI request burst.
+        config = get_request_config(request).model_dump(mode="json")
+        if not config.get("OPENCODE_ENABLED", False):
+            raise Http404
+        if not request.user.is_authenticated:
+            return redirect_to_login(request.get_full_path(), login_url="/admin/login/")
+        if not request.user.is_active or not request.user.is_superuser:
+            return HttpResponseForbidden("Agent access requires a superuser account.")
+
+        runtime, settings = _runtime_settings(request, config)
+        if path is None:
+            from archivebox.core.admin_site import archivebox_admin
+
+            context = {**archivebox_admin.each_context(request), **runtime.agent_context(settings)}
+            source = get_plugin_template("opencode", "agent", fallback=False)
+            if source is None:
+                raise RuntimeError("Agent template unavailable")
+            return HttpResponse(engines["django"].from_string(source).render(context, request))
+
+        if not runtime._origin_allowed(request.method, request.get_host(), request.headers):
+            return HttpResponseForbidden("Cross-origin agent requests are blocked.")
+        status, headers, body = runtime.proxy(
+            settings,
+            request.method,
+            path,
+            tuple((key, value) for key, values in request.GET.lists() for value in values),
+            request.headers,
+            request.body,
+        )
+        response_type = HttpResponse if isinstance(body, bytes) else StreamingHttpResponse
+        response = response_type(body, status=status, headers=headers)
+        response.xframe_options_exempt = True
+        response.headers["X-Frame-Options"] = "SAMEORIGIN"
+        response.headers["Content-Security-Policy"] = "frame-ancestors 'self'"
+        return response
+    except Http404:
+        raise
+    except Exception:
+        # Optional-service boundary, including imports and template rendering.
+        _LOGGER.exception("Optional AI service failed")
+        return HttpResponse("AI service unavailable. See server logs.", status=503, content_type="text/plain")


-def _request_header(request: HttpRequest, name: str) -> str | None:
-    meta = getattr(request, "META", {})
-    if name == "Content-Type":
-        value = meta.get("CONTENT_TYPE")
-    elif name == "Content-Length":
-        value = meta.get("CONTENT_LENGTH")
-    else:
-        value = meta.get(f"HTTP_{name.upper().replace('-', '_')}")
-    return str(value) if value else None
-
-
-def _request_headers(request: HttpRequest, settings: dict) -> dict[str, str]:
-    forwarded = {}
-    for key in ("Accept", "Accept-Language", "Content-Type", "Range", "User-Agent"):
-        value = _request_header(request, key)
-        if value:
-            forwarded[key] = value
-    return forwarded
-
-
-def _request_params(request: HttpRequest) -> tuple[tuple[str, str], ...]:
-    if hasattr(request.GET, "lists"):
-        return tuple((key, str(value)) for key, values in request.GET.lists() for value in values)
-    return tuple((key, str(value)) for key, value in dict(request.GET).items())
-
-
-async def _event_chunks(request: HttpRequest, settings: dict, path: str | None):
-    timeout = httpx.Timeout(settings["timeout"], read=None)
-    url = _proxy_url(settings, path)
-    method = request.method or "GET"
-    async with httpx.AsyncClient(timeout=timeout, follow_redirects=False) as client:
-        async with client.stream(
-            method,
-            url,
-            params=_request_params(request),
-            headers=_request_headers(request, settings),
-        ) as upstream:
-            async for chunk in upstream.aiter_raw(chunk_size=512):
-                yield chunk
-
-
-def _rewrite_text(body: bytes, settings: dict) -> bytes:
-    text = body.decode("utf-8", errors="replace")
-    text = text.replace(settings["origin"], _PROXY_PREFIX)
-    text = text.replace("location.origin", f'location.origin+"{_PROXY_PREFIX}"')
-    text = text.replace(
-        "k(k5,{get component(){return t.router??Az},",
-        f'k(k5,{{base:"{_PROXY_PREFIX}",get component(){{return t.router??Az}},',
-    )
-    text = text.replace('"/assets/', f'"{_PROXY_PREFIX}/assets/')
-    text = text.replace("'/assets/", f"'{_PROXY_PREFIX}/assets/")
-    proxy_path = rf'(\1.replace(/^{_PROXY_PREFIX_REGEX}(?=\/|$)/,"")||"/")'
-    text = re.sub(r"\b(window\.location\.pathname)\b", proxy_path, text)
-    text = re.sub(r"(?<![.\w])(location\.pathname)\b", proxy_path, text)
-    text = text.replace(
-        'window.history.replaceState(nz(o),"",r):window.history.pushState(o,"",r)',
-        (
-            f'window.history.replaceState(nz(o),"",r.startsWith("{_PROXY_PREFIX}")?r:r.startsWith("/")?"{_PROXY_PREFIX}"+r:r):'
-            f'window.history.pushState(o,"",r.startsWith("{_PROXY_PREFIX}")?r:r.startsWith("/")?"{_PROXY_PREFIX}"+r:r)'
-        ),
-    )
-    text = text.replace(
-        'const BL="modulepreload",UL=function(t){return"/"+t}',
-        f'const BL="modulepreload",UL=function(t){{return"{_PROXY_PREFIX}/"+t}}',
-    )
-    text = re.sub(
-        rf"""(?P<prefix>\b(?:href|src|action)=["'])/(?!{_PROXY_PREFIX_NO_SLASH_REGEX}(?:/|$))""",
-        rf"\g<prefix>{_PROXY_PREFIX}/",
-        text,
-    )
-    text = re.sub(
-        rf"""(?P<prefix>\b(?:fetch|EventSource)\(["'])/(?!{_PROXY_PREFIX_NO_SLASH_REGEX}(?:/|$))""",
-        rf"\g<prefix>{_PROXY_PREFIX}/",
-        text,
-    )
-    text = re.sub(
-        rf"""(?P<prefix>\burl\(["']?)/(?!{_PROXY_PREFIX_NO_SLASH_REGEX}(?:/|$))""",
-        rf"\g<prefix>{_PROXY_PREFIX}/",
-        text,
-    )
-    return text.encode("utf-8")
-
-
-def _response_headers(upstream: requests.Response, settings: dict) -> dict[str, str]:
-    headers = {}
-    for key, value in upstream.headers.items():
-        lower = key.lower()
-        if lower in _HOP_BY_HOP_HEADERS or lower in {
-            "content-length",
-            "content-encoding",
-            "x-frame-options",
-        }:
-            continue
-        if lower == "location":
-            if value.startswith(settings["origin"]):
-                value = value.replace(settings["origin"], _PROXY_PREFIX, 1)
-            elif value.startswith("/"):
-                value = f"{_PROXY_PREFIX}{value}"
-        headers[key] = value
-    return headers
+def agent_view(request):
+    return _dispatch(request)


@csrf_exempt
-def opencode_proxy_view(request: HttpRequest, path: str | None = None):
-    config = _machine_config()
-    _require_enabled(config)
-    auth_response = _require_superuser(request)
-    if auth_response:
-        return auth_response
-    if not _origin_allowed(request, path):
-        return HttpResponseForbidden(
-            b"Cross-origin OpenCode agent requests are blocked.",
-        )
+def opencode_proxy_view(request, path=None):
+    return _dispatch(request, path=path or "")

-    settings = _settings(config)
-    route_config = request.__dict__.get("archivebox_config")
-    base_url, admin_url, api_url = _archivebox_route_urls(request, route_config)
-    settings["archivebox_base_url"] = base_url
-    settings["archivebox_admin_url"] = admin_url
-    settings["archivebox_api_url"] = api_url
-    ok, error = _ensure_opencode(settings)
-    if not ok:
-        return HttpResponse(
-            error.encode(),
-            status=502,
-            content_type="text/plain; charset=utf-8",
-        )
-
-    if request.method == "GET" and (path or "").endswith("/event"):
-        response = StreamingHttpResponse(
-            _event_chunks(request, settings, path),
-            content_type="text/event-stream",
-        )
-        response.headers["Cache-Control"] = "no-store"
-        response.headers["X-Accel-Buffering"] = "no"
-        return response

+def _websocket_context(scope):
+    close_old_connections()
    try:
-        method = request.method or "GET"
-        upstream = requests.request(
-            method,
-            _proxy_url(settings, path),
-            params=_request_params(request),
-            data=request.body if method not in {"GET", "HEAD"} else None,
-            headers=_request_headers(request, settings),
-            stream=True,
-            timeout=(settings["timeout"], None),
-            allow_redirects=False,
-        )
-    except requests.RequestException as err:
-        return HttpResponse(
-            str(err).encode(),
-            status=502,
-            content_type="text/plain; charset=utf-8",
+        match = resolve(scope["path"])
+        if match.url_name != "opencode-proxy":
+            raise PermissionDenied
+        request = ASGIRequest(
+            {**scope, "type": "http", "method": "GET", "scheme": "https" if scope.get("scheme") == "wss" else "http"},
+            BytesIO(),
        )
+        route_config = get_request_config(request, resolve_plugins=False)
+        request.archivebox_config = route_config
+        if not route_config.CONTROL_PLANE_ENABLED:
+            raise PermissionDenied
+        if (
+            route_config.USES_SUBDOMAIN_ROUTING
+            and route_config.BASE_URL
+            and not host_matches(request.get_host(), get_admin_host(config=route_config, request=request))
+        ):
+            raise PermissionDenied
+        SessionMiddleware(_dispatch).process_request(request)
+        AuthenticationMiddleware(_dispatch).process_request(request)
+        ReverseProxyAuthMiddleware(_dispatch).process_request(request)
+        config = route_config.model_dump(mode="json")
+        if not config.get("OPENCODE_ENABLED") or not request.user.is_active or not request.user.is_superuser:
+            raise PermissionDenied
+        runtime, settings = _runtime_settings(request, config)
+        if not request.headers.get("Origin") or not runtime._origin_allowed("POST", request.get_host(), request.headers):
+            raise PermissionDenied
+        return runtime, settings, match.kwargs.get("path", "")
+    finally:
+        close_old_connections()

-    content_type = upstream.headers.get("Content-Type", "")
-    is_event_stream = content_type.startswith("text/event-stream")
-    is_text = not is_event_stream and any(content_type.startswith(prefix) for prefix in _TEXT_CONTENT_TYPES)
-    headers = _response_headers(upstream, settings)
-    if is_text:
-        body = _rewrite_text(upstream.content, settings)
-        response = HttpResponse(
-            body,
-            status=upstream.status_code,
-            content_type=content_type or "text/plain; charset=utf-8",
-        )
-    elif is_event_stream:
-        response = StreamingHttpResponse(
-            upstream.iter_lines(chunk_size=1),
-            status=upstream.status_code,
-            content_type=content_type or "text/event-stream",
-        )
-    else:
-        response = StreamingHttpResponse(
-            upstream.iter_content(chunk_size=64 * 1024),
-            status=upstream.status_code,
-            content_type=content_type or "application/octet-stream",
-        )
-    for key, value in headers.items():
-        response.headers[key] = value
-    response.headers["Cache-Control"] = "no-store"
-    return response
+
+async def websocket_view(scope, receive, send):
+    if (await receive())["type"] != "websocket.connect":
+        return
+    try:
+        runtime, settings, path = await sync_to_async(_websocket_context)(scope)
+        await runtime.websocket_proxy(settings, path, scope.get("query_string", b""), scope.get("subprotocols", []), receive, send)
+    except (PermissionDenied, Resolver404, Http404):
+        await send({"type": "websocket.close", "code": 1008})
+    except Exception:
+        _LOGGER.exception("Optional AI WebSocket failed")
+        await send({"type": "websocket.close", "code": 1011})
@@ -9,11 +9,11 @@
 *   COOKIES_OUTPUT_FILE         Optional output path for Netscape cookies.txt
 *   AUTH_STORAGE_OUTPUT_FILE    Optional output path for auth.json
 *   CHROME_BINARY               Absolute browser path resolved by abxpkg
+ *   CHROME_SANDBOX              Set to false when the Linux sandbox is unavailable
 *   NODE_MODULES_DIR            node_modules path resolved by abxpkg
 */

 const fs = require('fs');
-const os = require('os');
 const path = require('path');

 const pluginsDir = process.env.ARCHIVEBOX_ABX_PLUGINS_DIR || process.env.ABX_PLUGINS_DIR;
@@ -26,7 +26,7 @@ const baseUtils = require(path.join(pluginsDir, 'base', 'utils.js'));
 baseUtils.ensureNodeModuleResolution(module);

 const chromeUtils = require(path.join(pluginsDir, 'chrome', 'chrome_utils.js'));
-const puppeteer = require('puppeteer-core');
+const puppeteer = require('puppeteer');

 function cookieToNetscape(cookie) {
    let domain = cookie.domain;
@@ -115,38 +115,28 @@ async function openBrowser() {
        throw new Error(`User data directory does not exist: ${userDataDir}`);
    }

-    const outputDir = fs.mkdtempSync(path.join(os.tmpdir(), 'abx-browser-state-'));
    const binary = process.env.CHROME_BINARY;
    if (!binary || !path.isAbsolute(binary) || !fs.existsSync(binary)) {
        throw new Error('CHROME_BINARY must be an absolute executable path resolved by abxpkg');
    }

-    const launched = await chromeUtils.launchChromium({
-        binary,
-        outputDir,
+    // The ordinary archiving launcher uses a mock keychain. An import must
+    // instead let the original signed browser decrypt its own profile cookies.
+    const sandboxDisabled = ['false', '0', 'no', 'off'].includes(
+        String(process.env.CHROME_SANDBOX || '').trim().toLowerCase(),
+    );
+    const browser = await puppeteer.launch({
+        executablePath: binary,
        userDataDir,
        headless: true,
-        killZombies: false,
+        ignoreDefaultArgs: ['--use-mock-keychain', '--password-store=basic'],
+        args: ['--no-first-run', '--disable-sync', '--disable-background-networking',
+               '--profile-directory=Default', ...(process.platform === 'linux' && (process.getuid?.() === 0 || sandboxDisabled) ? ['--no-sandbox'] : [])],
    });
-
-    if (!launched.success) {
-        throw new Error(launched.error || 'Chrome launch failed');
-    }
-
-    const browser = await chromeUtils.connectToBrowserEndpoint(puppeteer, launched.cdpUrl, { defaultViewport: null });
-
    return {
        browser,
        async cleanup() {
-            try {
-                await browser.disconnect();
-            } catch (error) {}
-            try {
-                await chromeUtils.killChrome(launched.pid, outputDir);
-            } catch (error) {}
-            try {
-                fs.rmSync(outputDir, { recursive: true, force: true });
-            } catch (error) {}
+            await browser.close();
        },
        sourceDescription: userDataDir,
    };
@@ -11,6 +11,9 @@ import json
 import os
 import platform
 import shutil
+import sqlite3
+import time
+from http.cookiejar import MozillaCookieJar
 import subprocess
 import sys
 import tempfile
@@ -42,25 +45,27 @@ BROWSER_PROFILE_DIR_NAMES = (
 )

 VOLATILE_PROFILE_COPY_PATTERNS = (
+    # Chromium/macOS atomic-write staging files (e.g.
+    # .com.brave.Browser.TransportSecurity.Ze7UBU) disappear after rename.
+    ".*.??????",
    "Cache",
    "Code Cache",
    "GPUCache",
    "ShaderCache",
-    "Service Worker",
-    "GCM Store",
-    "chrome-extension_*",
-    "DNR Extension Rules",
-    "Extension Rules",
-    "Extension Scripts",
-    "Extension State",
-    "Local Extension Settings",
-    "*.log",
    "Crashpad",
    "BrowserMetrics",
    "BrowserMetrics-spare.pma",
+    "RunningChromeVersion",
+    "DevToolsActivePort",
    "SingletonLock",
    "SingletonSocket",
    "SingletonCookie",
+    "Sessions",
+    "Sessions_Encrypted",
+    "Current Session",
+    "Current Tabs",
+    "Last Session",
+    "Last Tabs",
 )

 PERSONA_PROFILE_DIR_CANDIDATES = (
@@ -201,6 +206,7 @@ class PersonaImportResult:
    source: PersonaImportSource
    profile_copied: bool = False
    cookies_imported: bool = False
+    cookie_count: int = 0
    storage_captured: bool = False
    user_agent_imported: bool = False
    warnings: list[str] = field(default_factory=list)
@@ -218,20 +224,16 @@ def get_chrome_user_data_dir() -> Path | None:
    if system == "Darwin":
        candidates = [
            home / "Library" / "Application Support" / "Google" / "Chrome",
-            home / "Library" / "Application Support" / "Chromium",
        ]
    elif system == "Linux":
        candidates = [
            home / ".config" / "google-chrome",
-            home / ".config" / "chromium",
            home / ".config" / "chrome",
-            home / "snap" / "chromium" / "common" / "chromium",
        ]
    elif system == "Windows":
        local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
        candidates = [
            local_app_data / "Google" / "Chrome" / "User Data",
-            local_app_data / "Chromium" / "User Data",
        ]
    else:
        candidates = []
@@ -243,6 +245,21 @@ def get_chrome_user_data_dir() -> Path | None:
    return None


+def get_chromium_user_data_dir() -> Path | None:
+    """Select Chromium's own profile even when Google Chrome is installed."""
+    home = Path.home()
+    if platform.system() == "Darwin":
+        candidates = [home / "Library/Application Support/Chromium"]
+    elif platform.system() == "Windows":
+        candidates = [Path(os.environ.get("LOCALAPPDATA", home / "AppData/Local")) / "Chromium/User Data"]
+    else:
+        candidates = [
+            Path(os.environ.get("XDG_CONFIG_HOME", home / ".config")) / "chromium",
+            home / "snap/chromium/common/chromium",
+        ]
+    return next((path for path in candidates if path.is_dir() and _list_profile_names(path)), None)
+
+
 def get_brave_user_data_dir() -> Path | None:
    """Get the default Brave user data directory for the current platform."""
    system = platform.system()
@@ -303,7 +320,7 @@ def get_edge_user_data_dir() -> Path | None:

 BROWSER_PROFILE_FINDERS = {
    "chrome": get_chrome_user_data_dir,
-    "chromium": get_chrome_user_data_dir,
+    "chromium": get_chromium_user_data_dir,
    "brave": get_brave_user_data_dir,
    "edge": get_edge_user_data_dir,
 }
@@ -419,7 +436,10 @@ def resolve_browser_import_source(browser: str, profile_dir: str | None = None)

    user_data_dir = BROWSER_PROFILE_FINDERS[browser]()
    if not user_data_dir:
-        raise ValueError(f"Could not find {browser} profile directory")
+        raise ValueError(
+            f"Could not find {browser} profile directory. Use --source /path/to/browser-data. "
+            "For Docker on macOS/Windows, import on the host into the shared data directory first.",
+        )

    chosen_profile = profile_dir or pick_default_profile_dir(user_data_dir)
    if not chosen_profile:
@@ -444,6 +464,8 @@ def resolve_browser_profile_source(
        resolved_root = resolved_root.resolve()
    if not resolved_root.exists():
        raise ValueError(f"Profile root does not exist: {resolved_root}")
+    if Path(profile_dir).name != profile_dir or profile_dir in {".", ".."}:
+        raise ValueError("Profile must be a directory name within the source browser root.")
    if not profile_dir.strip():
        raise ValueError("Profile directory name cannot be empty.")

@@ -476,6 +498,17 @@ def resolve_custom_import_source(raw_value: str, profile_dir: str | None = None)
        raise ValueError(f"Custom browser path does not exist: {source_path}")

    explicit_profile = profile_dir.strip() if profile_dir else ""
+    for profile_root in PERSONA_PROFILE_DIR_CANDIDATES:
+        root = source_path / profile_root
+        if root.is_dir():
+            selected = explicit_profile or pick_default_profile_dir(root)
+            if selected:
+                return resolve_browser_profile_source(
+                    browser="persona",
+                    source_name=source_path.name,
+                    user_data_dir=root.resolve(),
+                    profile_dir=selected,
+                )
    if _looks_like_profile_dir(source_path):
        if explicit_profile and explicit_profile != source_path.name:
            raise ValueError("Profile name does not match the provided profile directory path.")
@@ -508,6 +541,46 @@ def pick_default_profile_dir(user_data_dir: Path) -> str | None:
    return profiles[0]


+def resolve_source_browser_binary(source: PersonaImportSource) -> str:
+    """Use the originating browser so OS-protected cookies use the correct keychain."""
+    if source.browser_binary:
+        return str(Path(source.browser_binary).expanduser().resolve())
+    names = {
+        "chrome": ("Google Chrome", "google-chrome"),
+        "chromium": ("Chromium", "chromium"),
+        "brave": ("Brave Browser", "brave-browser"),
+        "edge": ("Microsoft Edge", "microsoft-edge"),
+    }
+    app, executable = names.get(source.browser, ("", ""))
+    candidates = []
+    if platform.system() == "Darwin" and app:
+        candidates = [
+            Path("/Applications") / f"{app}.app/Contents/MacOS/{app}",
+            Path.home() / f"Applications/{app}.app/Contents/MacOS/{app}",
+        ]
+    elif platform.system() == "Windows":
+        relative = {
+            "chrome": "Google/Chrome/Application/chrome.exe",
+            "edge": "Microsoft/Edge/Application/msedge.exe",
+            "brave": "BraveSoftware/Brave-Browser/Application/brave.exe",
+            "chromium": "Chromium/Application/chrome.exe",
+        }.get(source.browser)
+        if relative:
+            candidates = [
+                Path(os.environ[root]) / relative for root in ("LOCALAPPDATA", "PROGRAMFILES", "PROGRAMFILES(X86)") if os.environ.get(root)
+            ]
+    for candidate in candidates:
+        if candidate.is_file() and os.access(candidate, os.X_OK):
+            return str(candidate)
+    if executable and (found := shutil.which(executable)):
+        return found
+    raise ValueError(
+        f"Source {source.browser} browser executable not found. Use --browser-binary with the originating Chromium browser. "
+        "For Docker, run persona create --import on the browser's host into the shared data directory first: "
+        "the container cannot decrypt cookies using the host's macOS Keychain or Windows credentials.",
+    )
+
+
 def import_persona_from_source(
    persona: Persona,
    source: PersonaImportSource,
@@ -516,67 +589,182 @@ def import_persona_from_source(
    import_cookies: bool = True,
    capture_storage: bool = False,
 ) -> PersonaImportResult:
+    # Stage all work before replacing an existing, working identity.
    persona.ensure_dirs()
    result = PersonaImportResult(source=source)
+    expected_cookies = 0
+    browser_binary = None
+    native_cookie_browsers = {"chrome", "chromium", "brave", "edge", "vivaldi", "opera", "opera_gx"}
+    # browser-cookie3 needs a desktop D-Bus session on Linux. In a headless
+    # environment, let the originating browser decode its own profile instead.
+    has_native_cookie_decoder = platform.system() != "Linux" or bool(os.environ.get("DBUS_SESSION_BUS_ADDRESS"))
+    native_cookie_import = source.kind == "browser-profile" and source.browser in native_cookie_browsers and has_native_cookie_decoder
+    portable_import = source.kind == "browser-profile" and source.browser == "persona"
+    if source.kind == "browser-profile" and not portable_import and (import_cookies or capture_storage):
+        if not native_cookie_import:
+            browser_binary = resolve_source_browser_binary(source)
+        for cookie_db in (source.profile_path / "Network" / "Cookies", source.profile_path / "Cookies"):
+            if cookie_db.is_file():
+                with sqlite3.connect(f"{cookie_db.as_uri()}?mode=ro", uri=True) as conn:
+                    expected_cookies = conn.execute(
+                        "SELECT COUNT(*) FROM cookies WHERE expires_utc = 0 OR expires_utc > ?",
+                        (int((time.time() + 11644473600) * 1_000_000),),
+                    ).fetchone()[0]
+                break
+    with tempfile.TemporaryDirectory(prefix=".import-", dir=persona.path.parent) as tmp:
+        stage = Path(tmp)
+        staged_profile = stage / "chrome_profile"
+        if source.kind == "browser-profile":
+            assert source.user_data_dir and source.profile_path
+            if copy_profile or import_cookies or capture_storage:
+                staged_profile.mkdir()
+                # Only the selected profile belongs to this persona. Normalize its
+                # name so downstream Chromium launches always select it.
+                copy_browser_user_data_dir(source.profile_path, staged_profile / "Default")
+                local_state = source.user_data_dir / "Local State"
+                if local_state.exists():
+                    state = json.loads(local_state.read_text())
+                    profile_state = state.setdefault("profile", {})
+                    profile_state["last_used"] = "Default"
+                    profile_state["last_active_profiles"] = ["Default"]
+                    info = profile_state.get("info_cache", {}).get(source.profile_dir)
+                    profile_state["info_cache"] = {"Default": info} if info else {}
+                    (staged_profile / "Local State").write_text(json.dumps(state))
+                persona.cleanup_chrome_profile(staged_profile)
+                result.profile_copied = copy_profile
+        elif copy_profile:
+            result.warnings.append(
+                "CDP imports capture cookies and open-tab storage, not browser settings. Use --source for a full profile import.",
+            )

-    persona_chrome_dir = Path(persona.CHROME_USER_DATA_DIR)
-    cookies_file = persona.path / "cookies.txt"
-    auth_file = persona.path / "auth.json"
-
-    launch_user_data_dir: Path | None = None
-
-    if source.kind == "browser-profile":
-        if copy_profile and source.user_data_dir:
-            resolved_source_root = source.user_data_dir.resolve()
-            resolved_persona_root = persona_chrome_dir.resolve()
-            if resolved_source_root == resolved_persona_root:
-                result.warnings.append(
-                    "Skipped profile copy because the selected source is already this persona's chrome_profile directory.",
-                )
+        if import_cookies or capture_storage:
+            if portable_import:
+                export_dir = source.user_data_dir.parent
+                if not all((export_dir / name).is_file() for name in ("auth.json", "cookies.txt")):
+                    raise ValueError("This persona has no portable cookie export. Re-import it on the original browser host first.")
+                auth_payload = json.loads((export_dir / "auth.json").read_text())
+                if not isinstance(auth_payload, dict) or not isinstance(auth_payload.get("cookies"), list):
+                    raise ValueError("Invalid persona auth.json: expected a cookies list.")
+                for name in ("auth.json", "cookies.txt"):
+                    shutil.copy2(export_dir / name, stage / name)
+                success, message = True, ""
+            elif native_cookie_import:
+                auth_payload = export_profile_cookies(source, staged_profile, stage)
+                success, message = True, ""
            else:
-                copy_browser_user_data_dir(resolved_source_root, resolved_persona_root)
-                persona.cleanup_chrome_profile(resolved_persona_root)
-                result.profile_copied = True
-            launch_user_data_dir = resolved_persona_root
-        else:
-            launch_user_data_dir = source.user_data_dir
-    elif copy_profile:
-        result.warnings.append(
-            "Profile copying is only available for local Chromium profile paths. CDP imports can only pull cookies and open-tab storage.",
-        )
-
-    if not import_cookies and not capture_storage:
-        return result
-
-    if source.kind == "cdp":
-        export_success, auth_payload, export_message = export_browser_state(
-            cdp_url=source.cdp_url,
-            cookies_output_file=cookies_file if import_cookies else None,
-            auth_output_file=auth_file if capture_storage else None,
-        )
-    else:
-        export_success, auth_payload, export_message = export_browser_state(
-            user_data_dir=launch_user_data_dir,
-            profile_dir=source.profile_dir,
-            chrome_binary=source.browser_binary,
-            cookies_output_file=cookies_file if import_cookies else None,
-            auth_output_file=auth_file if capture_storage else None,
-        )
-
-    if not export_success:
-        result.warnings.append(export_message or "Browser import failed.")
-        return result
-
-    if import_cookies and cookies_file.exists():
-        result.cookies_imported = True
-    if capture_storage and auth_file.exists():
-        result.storage_captured = True
-    if _apply_imported_user_agent(persona, auth_payload):
-        result.user_agent_imported = True
+                # Custom browsers and headless Linux use the originating browser.
+                # Desktop imports use their OS decoder without launching or probing.
+                launch_profile = stage / "export_profile"
+                if source.kind == "browser-profile":
+                    copy_browser_user_data_dir(staged_profile, launch_profile)
+                success, auth_payload, message = export_browser_state(
+                    user_data_dir=launch_profile if source.kind == "browser-profile" else None,
+                    cdp_url=source.cdp_url,
+                    profile_dir="Default" if source.kind == "browser-profile" else None,
+                    chrome_binary=browser_binary,
+                    cookies_output_file=stage / "cookies.txt" if import_cookies else None,
+                    auth_output_file=stage / "auth.json",
+                )
+            if not success:
+                raise ValueError(message or "Browser state export failed; persona was not replaced.")
+            if expected_cookies and not (auth_payload or {}).get("cookies"):
+                raise ValueError(
+                    "The source has cookies but the browser exported none. Run the import on the source host "
+                    "with the original browser and unlock its OS keychain; the existing persona was not replaced.",
+                )
+            if import_cookies:
+                result.cookies_imported = True
+                result.cookie_count = len((auth_payload or {}).get("cookies", []))
+            if capture_storage:
+                result.storage_captured = True
+            # Keep full CDP cookie attributes (sameSite, httpOnly, partition keys)
+            # as well as cookies.txt for non-browser extractors.
+            result.user_agent_imported = _apply_imported_user_agent(persona, auth_payload)

+        if result.profile_copied:
+            target = Path(persona.CHROME_USER_DATA_DIR)
+            if target.exists():
+                shutil.rmtree(target)
+            staged_profile.rename(target)
+        for filename in ("cookies.txt", "auth.json"):
+            exported = stage / filename
+            if exported.exists():
+                exported.chmod(0o600)
+                exported.replace(persona.path / filename)
    return result


+def export_profile_cookies(source: PersonaImportSource, staged_profile: Path, output: Path) -> dict:
+    """Decode with the host keychain, never by starting the user's browser."""
+    import browser_cookie3
+
+    payload = {"TYPE": "auth", "cookies": [], "localStorage": {}, "sessionStorage": {}}
+    jar = MozillaCookieJar(str(output / "cookies.txt"))
+    assert source.profile_path
+    for relative in (Path("Network/Cookies"), Path("Cookies")):
+        original = source.profile_path / relative
+        if not original.is_file():
+            continue
+        database = staged_profile / "Default" / relative
+        for suffix in ("", "-wal", "-shm"):
+            database.with_name(database.name + suffix).unlink(missing_ok=True)
+        # SQLite backup includes committed WAL data even while the browser is open.
+        with sqlite3.connect(f"{original.as_uri()}?mode=ro", uri=True) as reader:
+            with sqlite3.connect(database) as writer:
+                reader.backup(writer)
+        try:
+            cookies = getattr(browser_cookie3, source.browser)(
+                cookie_file=str(database),
+                key_file=str(staged_profile / "Local State"),
+            )
+        except Exception as err:
+            raise ValueError(
+                "Could not decrypt browser cookies. Run the import as your desktop user on the browser host "
+                "with its OS keychain unlocked. Mounting macOS/Windows browser files into Linux does not provide "
+                "the decryption keys. The existing persona was not replaced.",
+            ) from err
+        with sqlite3.connect(database) as connection:
+            connection.row_factory = sqlite3.Row
+            columns = {row[1] for row in connection.execute("PRAGMA table_info(cookies)")}
+            metadata_columns = [
+                name
+                for name in ("host_key", "path", "name", "samesite", "top_frame_site_key", "has_cross_site_ancestor")
+                if name in columns
+            ]
+            rows = {
+                (row["host_key"], row["path"], row["name"]): dict(row)
+                for row in connection.execute("SELECT " + ", ".join(metadata_columns) + " FROM cookies")
+            }
+        for cookie in cookies:
+            if cookie.is_expired():
+                continue
+            jar.set_cookie(cookie)
+            item = {
+                "name": cookie.name,
+                "value": cookie.value,
+                "domain": cookie.domain,
+                "path": cookie.path,
+                "secure": bool(cookie.secure),
+                "httpOnly": cookie.has_nonstandard_attr("HTTPOnly"),
+            }
+            if cookie.expires:
+                item["expires"] = cookie.expires
+            row = rows.get((cookie.domain, cookie.path, cookie.name), {})
+            same_site = {0: "None", 1: "Lax", 2: "Strict"}.get(row.get("samesite"))
+            if same_site:
+                item["sameSite"] = same_site
+            if row.get("top_frame_site_key"):
+                item["partitionKey"] = {
+                    "topLevelSite": row["top_frame_site_key"],
+                    "hasCrossSiteAncestor": bool(row.get("has_cross_site_ancestor", False)),
+                }
+            payload["cookies"].append(item)
+        break
+    jar.save(ignore_discard=True, ignore_expires=False)
+    (output / "auth.json").write_text(json.dumps(payload) + "\n")
+    return payload
+
+
 def copy_browser_user_data_dir(source_dir: Path, destination_dir: Path) -> None:
    destination_dir.parent.mkdir(parents=True, exist_ok=True)
    shutil.rmtree(destination_dir, ignore_errors=True)
@@ -611,21 +799,27 @@ def export_browser_state(
    chrome_config = chrome_plugin_dir / "chrome" / "config.json"

    env = os.environ.copy()
-    if chrome_binary:
-        env["CHROME_BINARY"] = str(chrome_binary)
-    dependency_env = subprocess.run(
-        [
-            str(Path(sys.executable).with_name("abxpkg")),
-            "env",
-            "--install",
-            "--json",
-            f"--lib={get_config().ABXPKG_LIB_DIR}",
-            f"--deps-from={chrome_config}:required_binaries",
-        ],
-        capture_output=True,
-        text=True,
-        env=env,
-    )
+    dependency_config = json.loads(chrome_config.read_text())
+    dependency_config["required_binaries"] = [dep for dep in dependency_config["required_binaries"] if dep["name"] != "{CHROME_BINARY}"]
+    dependency_file = tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False)
+    with dependency_file:
+        json.dump(dependency_config, dependency_file)
+    try:
+        dependency_env = subprocess.run(
+            [
+                str(Path(sys.executable).with_name("abxpkg")),
+                "env",
+                "--install",
+                "--json",
+                f"--lib={get_config().ABXPKG_LIB_DIR}",
+                f"--deps-from={dependency_file.name}:required_binaries",
+            ],
+            capture_output=True,
+            text=True,
+            env=env,
+        )
+    finally:
+        Path(dependency_file.name).unlink()
    if dependency_env.returncode != 0:
        return False, None, dependency_env.stderr.strip() or "abxpkg could not resolve browser export dependencies."
    try:
@@ -639,6 +833,9 @@ def export_browser_state(
    if not node_projection.is_symlink() or not os.access(node_projection, os.X_OK):
        return False, None, f"abxpkg did not resolve Node.js into {node_projection}."
    env.update({str(key): str(value) for key, value in resolved_env.items()})
+    if chrome_binary:
+        env["CHROME_BINARY"] = chrome_binary
+    env["NODE_MODULES_DIR"] = str(abxpkg_lib_dir / "pnpm" / "packages" / "chrome" / "node_modules")
    env["NODE_BINARY"] = str(node_projection)
    env["ARCHIVEBOX_ABX_PLUGINS_DIR"] = str(chrome_plugin_dir)

@@ -865,7 +1062,7 @@ def _apply_imported_user_agent(persona: Persona, auth_payload: dict | None) -> b
    if not auth_payload:
        return False

-    user_agent = str(auth_payload.get("user_agent") or "").strip()
+    user_agent = str(auth_payload.get("user_agent") or "").strip().replace("HeadlessChrome/", "Chrome/")
    if not user_agent:
        return False

@@ -45,17 +45,23 @@ VOLATILE_PROFILE_DIR_NAMES = {
    "Code Cache",
    "GPUCache",
    "ShaderCache",
-    "Service Worker",
-    "GCM Store",
    "Crashpad",
    "BrowserMetrics",
+    "Sessions",
+    "Sessions_Encrypted",
 }

 VOLATILE_PROFILE_FILE_NAMES = {
    "BrowserMetrics-spare.pma",
+    "RunningChromeVersion",
+    "DevToolsActivePort",
    "SingletonCookie",
    "SingletonLock",
    "SingletonSocket",
+    "Current Session",
+    "Current Tabs",
+    "Last Session",
+    "Last Tabs",
 }


@@ -211,7 +217,7 @@ class Persona(ModelWithConfig):
                cleaned = True

        for path in profile_paths():
-            if not path.match("*.log"):
+            if path.name not in {"chrome.log", "chrome_debug.log"}:
                continue
            try:
                path.unlink()
@@ -297,6 +303,10 @@ class Persona(ModelWithConfig):
            else:
                runtime_profile_dir.mkdir(parents=True, exist_ok=True)

+            for filename in ("cookies.txt", "auth.json"):
+                source = self.path / filename
+                if source.is_file():
+                    shutil.copy2(source, runtime_root / filename)
            runtime_downloads_dir.mkdir(parents=True, exist_ok=True)
            self.cleanup_chrome_profile(runtime_profile_dir)

@@ -313,6 +323,11 @@ class Persona(ModelWithConfig):
            # derivation centralized in the Chrome plugin helpers.
            "PERSONAS_DIR": str(runtime_root.parent),
            "ACTIVE_PERSONA": self.name,
+            **{
+                key: str(runtime_root / filename)
+                for key, filename in (("COOKIES_FILE", "cookies.txt"), ("AUTH_STORAGE_FILE", "auth.json"))
+                if (runtime_root / filename).is_file()
+            },
        }

    def prepare_runtime_for_snapshot(self, snapshot, chrome_binary: str = "") -> dict[str, str]:
@@ -329,6 +344,12 @@ class Persona(ModelWithConfig):
        else:
            runtime_profile_dir.mkdir(parents=True, exist_ok=True)

+        for filename in ("cookies.txt", "auth.json"):
+            source = self.runtime_root_for_crawl(snapshot.crawl) / filename
+            if not source.is_file():
+                source = self.path / filename
+            if source.is_file():
+                shutil.copy2(source, runtime_root / filename)
        runtime_downloads_dir.mkdir(parents=True, exist_ok=True)
        self.cleanup_chrome_profile(runtime_profile_dir)

@@ -343,6 +364,11 @@ class Persona(ModelWithConfig):
            # Chrome hooks and ArchiveBox-driven hooks resolve paths the same way.
            "PERSONAS_DIR": str(runtime_root.parent),
            "ACTIVE_PERSONA": self.name,
+            **{
+                key: str(runtime_root / filename)
+                for key, filename in (("COOKIES_FILE", "cookies.txt"), ("AUTH_STORAGE_FILE", "auth.json"))
+                if (runtime_root / filename).is_file()
+            },
        }

    def cleanup_runtime_for_crawl(self, crawl) -> None:
@@ -0,0 +1,61 @@
+#!/usr/bin/env node
+
+const fs = require('fs');
+const path = require('path');
+const pluginsDir = process.env.ARCHIVEBOX_ABX_PLUGINS_DIR;
+const {ensureNodeModuleResolution} = require(path.join(pluginsDir, 'base', 'utils.js'));
+ensureNodeModuleResolution(module);
+const installShutdownHandler = require(path.join(pluginsDir, 'base', 'daemon_lifecycle.js')).captureShutdownSignals();
+const {
+    acquireSessionLock,
+    ensureChromeSession,
+    getChromeSessionOptionsFromConfig,
+    getChromeLaunchPrerequisites,
+    connectToBrowserEndpoint,
+    closeBrowserInChromeSession,
+} = require(path.join(pluginsDir, 'chrome', 'chrome_utils.js'));
+
+async function main() {
+    const config = JSON.parse(fs.readFileSync(0, 'utf8'));
+    const outputDir = path.join(path.dirname(config.CHROME_USER_DATA_DIR), '.browser');
+    fs.mkdirSync(outputDir, {recursive: true});
+    const release = await acquireSessionLock(path.join(outputDir, 'launch.lock'));
+    let session;
+    const {puppeteer, binary} = getChromeLaunchPrerequisites();
+    try {
+        session = await ensureChromeSession({
+            ...getChromeSessionOptionsFromConfig(config),
+            outputDir, puppeteer, binary,
+        });
+    } finally {
+        release();
+    }
+    const browser = await connectToBrowserEndpoint(puppeteer, session.cdpUrl, {defaultViewport: null});
+    installShutdownHandler(async () => {
+        if (!session.reusedExisting) {
+            await closeBrowserInChromeSession({
+                outputDir, puppeteer, cdpUrl: session.cdpUrl, pid: session.pid, processIsLocal: true,
+            });
+        }
+        await browser.disconnect();
+        process.exit(0);
+    });
+    const pages = await browser.pages();
+    const page = pages[0] || await browser.newPage();
+    const windowSession = await page.createCDPSession();
+    const {windowId} = await windowSession.send('Browser.getWindowForTarget');
+    await windowSession.send('Browser.setWindowBounds', {windowId, bounds: {windowState: 'maximized'}});
+    await windowSession.detach();
+    await page.bringToFront();
+    console.log(`Browser ready for persona ${config.ACTIVE_PERSONA || 'Default'}`);
+    if (session.reusedExisting) {
+        await browser.disconnect();
+        return;
+    }
+    await new Promise(resolve => browser.once('disconnected', resolve));
+}
+
+main().catch(error => {
+    console.error(error.message);
+    process.exit(1);
+});
@@ -1,12 +1,12 @@
 __package__ = "archivebox.plugins"

-import json
 from collections.abc import Iterable
 from functools import lru_cache
 from pathlib import Path
 from typing import Any, Protocol, TypedDict

 from abx_plugins import get_plugins_dir
+from abx_dl.catalog import PluginCatalog, PluginConfigResolver
 from django.utils.safestring import mark_safe

 from archivebox.config.constants import CONSTANTS
@@ -29,18 +29,18 @@ USER_PLUGINS_DIR = CONSTANTS.USER_PLUGINS_DIR


 def iter_plugin_dirs() -> list[Path]:
-    """Iterate over all built-in and user plugin directories."""
-    plugin_dirs: list[Path] = []
+    """Return the exact plugin directories exposed by the shared catalog."""
+    return [plugin.path for plugin in get_plugin_catalog().values()]

-    for base_dir in (BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR):
-        if not base_dir.exists():
-            continue

-        for plugin_dir in base_dir.iterdir():
-            if plugin_dir.is_dir() and not plugin_dir.name.startswith("_"):
-                plugin_dirs.append(plugin_dir)
+@lru_cache(maxsize=1)
+def get_plugin_catalog() -> PluginCatalog:
+    return PluginCatalog.discover(extra_plugin_dirs=[USER_PLUGINS_DIR], runtime="archivebox")

-    return plugin_dirs
+
+@lru_cache(maxsize=1)
+def get_plugin_config_resolver() -> PluginConfigResolver:
+    return PluginConfigResolver(get_plugin_catalog())


@lru_cache(maxsize=1)
@@ -52,25 +52,7 @@ def get_plugins() -> list[str]:
    or a standardized templates/icon.html asset. This includes non-extractor
    plugins such as binary providers and shared base plugins.
    """
-    plugins = []
-
-    for plugin_dir in iter_plugin_dirs():
-        has_hooks = any(plugin_dir.glob("on_*__*.*"))
-        has_config = (plugin_dir / "config.json").exists()
-        has_icon = (plugin_dir / "templates" / "icon.html").exists()
-        if has_hooks or has_config or has_icon:
-            plugins.append(plugin_dir.name)
-
-    return sorted(set(plugins))
-
-
-def get_plugin_models():
-    from abx_dl.models import discover_plugins
-
-    plugins = {}
-    for base_dir in (BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR):
-        plugins.update(discover_plugins(plugins_dir=base_dir, runtime="archivebox"))
-    return plugins
+    return sorted(get_plugin_catalog())


 def get_plugin_name(plugin: str) -> str:
@@ -99,91 +81,17 @@ def get_enabled_plugins(config: ConfigLookup | None = None, **config_kwargs: Any

        config = get_config(**config_kwargs)

-    enabled = []
-    disabled = []
-    for plugin in get_plugins():
-        plugin_config = get_plugin_special_config(plugin, config)
-        if plugin_config["enabled"]:
-            enabled.append(plugin)
-        else:
-            disabled.append(plugin)
-
-    from abx_dl.models import filter_plugins
-
-    return list(filter_plugins(get_plugin_models(), enabled, include_providers=True, disabled_names=disabled))
+    return get_plugin_config_resolver().enabled_plugin_names_from_flat(dict(config.items()))


-def discover_plugins_that_provide_interface(
-    module_name: str,
-    required_attrs: list[str],
-    plugin_prefix: str | None = None,
-) -> dict[str, Any]:
-    """
-    Discover plugins that provide a specific Python module with required interface.
-
-    This enables dynamic plugin discovery for features like search backends,
-    storage backends, etc. without hardcoding imports.
-    """
-    import importlib.util
-
-    backends = {}
-
-    for base_dir in (BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR):
-        if not base_dir.exists():
-            continue
-
-        for plugin_dir in base_dir.iterdir():
-            if not plugin_dir.is_dir():
-                continue
-
-            plugin_name = plugin_dir.name
-            if plugin_prefix and not plugin_name.startswith(plugin_prefix):
-                continue
-
-            module_path = plugin_dir / f"{module_name}.py"
-            if not module_path.exists():
-                continue
-
-            try:
-                spec = importlib.util.spec_from_file_location(
-                    f"archivebox.dynamic_plugins.{plugin_name}.{module_name}",
-                    module_path,
-                )
-                if spec is None or spec.loader is None:
-                    continue
-
-                module = importlib.util.module_from_spec(spec)
-                spec.loader.exec_module(module)
-
-                if not all(attr in vars(module) for attr in required_attrs):
-                    continue
-
-                if plugin_prefix:
-                    backend_name = plugin_name[len(plugin_prefix) :]
-                else:
-                    backend_name = plugin_name
-
-                backends[backend_name] = module
-
-            except Exception:
-                continue
-
-    return backends
-
-
-def get_search_backends() -> dict[str, Any]:
-    """
-    Discover all available search backend plugins.
-
-    Search backends must provide a search.py module with:
-        - search(query: str) -> List[str]  (returns snapshot IDs)
-        - flush(snapshot_ids: Iterable[str]) -> None
-    """
-    return discover_plugins_that_provide_interface(
-        module_name="search",
-        required_attrs=["search", "flush"],
-        plugin_prefix="search_backend_",
-    )
+def get_search_backends():
+    """Return plugins that declare both standalone search commands."""
+    catalog = get_plugin_catalog()
+    return {
+        plugin.name.removeprefix("search_backend_"): plugin
+        for plugin in catalog.values()
+        if catalog.command(plugin.name, "search") is not None and catalog.command(plugin.name, "flush") is not None
+    }


@lru_cache(maxsize=1)
@@ -196,33 +104,7 @@ def discover_plugin_configs() -> dict[str, dict[str, Any]]:
    schemas are plugin package metadata, not live user config; runtime values
    still come from env/db config at each callsite.
    """
-    configs = {}
-
-    for plugin_dir in iter_plugin_dirs():
-        config_path = plugin_dir / "config.json"
-        if not config_path.exists():
-            continue
-
-        try:
-            with open(config_path) as f:
-                schema = json.load(f)
-
-            if not isinstance(schema, dict):
-                continue
-            if schema.get("type") != "object":
-                continue
-            if "properties" not in schema:
-                continue
-
-            configs[plugin_dir.name] = schema
-
-        except (json.JSONDecodeError, OSError) as e:
-            import sys
-
-            print(f"Warning: Failed to load config.json from {plugin_dir.name}: {e}", file=sys.stderr)
-            continue
-
-    return configs
+    return get_plugin_config_resolver().schemas


 def get_plugin_special_config(plugin_name: str, config: ConfigLookup, _visited: set[str] | None = None) -> PluginSpecialConfig:
@@ -234,26 +116,7 @@ def get_plugin_special_config(plugin_name: str, config: ConfigLookup, _visited:
        - {PLUGIN}_TIMEOUT: Plugin-specific timeout (fallback to TIMEOUT, default 300)
        - {PLUGIN}_BINARY: Primary binary path (default to plugin_name)
    """
-    plugin_upper = plugin_name.upper()
-
-    enabled_key = f"{plugin_upper}_ENABLED"
-    enabled = config.get(enabled_key)
-    if enabled is None:
-        enabled = True
-    elif isinstance(enabled, str):
-        enabled = enabled.lower() not in ("false", "0", "no", "")
-
-    timeout_key = f"{plugin_upper}_TIMEOUT"
-    timeout = config.get(timeout_key) or config.get("TIMEOUT", 300)
-
-    binary_key = f"{plugin_upper}_BINARY"
-    binary = config.get(binary_key, plugin_name)
-
-    return {
-        "enabled": bool(enabled),
-        "timeout": int(timeout),
-        "binary": str(binary),
-    }
+    return get_plugin_config_resolver().runtime_settings(plugin_name, dict(config.items()))


 DEFAULT_TEMPLATES = {
@@ -295,11 +158,11 @@ def get_plugin_template(plugin: str, template_name: str, fallback: bool = True)
    if base_name in ("yt-dlp", "youtube-dl"):
        base_name = "ytdlp"

-    for plugin_dir in iter_plugin_dirs():
-        if plugin_dir.name == base_name or plugin_dir.name.endswith(f"_{base_name}"):
-            template_path = plugin_dir / "templates" / f"{template_name}.html"
-            if template_path.exists():
-                return template_path.read_text()
+    catalog = get_plugin_catalog()
+    if base_name in catalog:
+        template_path = catalog.template_path(base_name, template_name)
+        if template_path is not None:
+            return template_path.read_text()

    if fallback:
        return DEFAULT_TEMPLATES.get(template_name, "")
@@ -11,133 +11,19 @@ from django.utils.html import format_html

 from archivebox.config import CONSTANTS_CONFIG
 from archivebox.config.common import ArchiveBoxConfig, get_config
-from archivebox.plugins.discovery import discover_plugin_configs, get_plugin_icon, get_plugins
+from archivebox.plugins.discovery import discover_plugin_configs, get_plugin_catalog, get_plugin_icon, get_plugins


 PLUGIN_CONFIG_FIELD_PREFIX = "plugin_config__"
-PLUGIN_GROUP_DEFINITIONS = (
-    (
-        "main_plugins",
-        "Main",
-        "",
-        "",
-        "",
-        (
-            "dom",
-            "screenshot",
-            "pdf",
-            "singlefile",
-            "wget",
-            "archivedotorg",
-            "chrome_mhtml",
-            "archivewebpage",
-        ),
-    ),
-    (
-        "page_setup_plugins",
-        "Page Setup",
-        "",
-        "",
-        "",
-        (
-            "chrome",
-            "infiniscroll",
-            "modalcloser",
-            "ublock",
-            "istilldontcareaboutcookies",
-            "twocaptcha",
-            "claudechrome",
-        ),
-    ),
-    (
-        "media_plugins",
-        "Media",
-        "",
-        "",
-        "",
-        (
-            "staticfile",
-            "responses",
-            "chrome_screencast",
-            "ytdlp",
-            "gallerydl",
-            "git",
-        ),
-    ),
-    (
-        "text_plugins",
-        "Text",
-        "",
-        "",
-        "",
-        (
-            "readability",
-            "htmltotext",
-            "defuddle",
-            "forumdl",
-            "mercury",
-            "trafilatura",
-            "liteparse",
-            "opendataloader",
-            "papersdl",
-        ),
-    ),
-    (
-        "metadata_plugins",
-        "Metadata",
-        "",
-        "",
-        "",
-        (
-            "title",
-            "favicon",
-            "headers",
-            "redirects",
-            "accessibility",
-            "consolelog",
-            "sslcerts",
-            "dns",
-            "seo",
-            "hashes",
-        ),
-    ),
-    (
-        "postprocessing_plugins",
-        "Postprocessing",
-        "",
-        "",
-        "",
-        (
-            "parse_dom_outlinks",
-            "parse_html_urls",
-            "parse_jsonl_urls",
-            "parse_netscape_urls",
-            "parse_rss_urls",
-            "parse_txt_urls",
-            "claudecode",
-            "claudecodecleanup",
-            "claudecodeextract",
-        ),
-    ),
+PLUGIN_GROUPS = (
+    ("main", "main_plugins", "Main"),
+    ("page_setup", "page_setup_plugins", "Page Setup"),
+    ("media", "media_plugins", "Media"),
+    ("text", "text_plugins", "Text"),
+    ("metadata", "metadata_plugins", "Metadata"),
+    ("postprocessing", "postprocessing_plugins", "Postprocessing"),
+    ("other", "other_plugins", "Other"),
 )
-HIDDEN_PLUGIN_CONFIG_UI_PLUGINS = {
-    "apt",
-    "base",
-    "bash",
-    "brew",
-    "cargo",
-    "chromewebstore",
-    "env",
-    "media",
-    "npm",
-    "opencode",
-    "pip",
-    "puppeteer",
-    "search_backend_ripgrep",
-    "search_backend_sonic",
-    "search_backend_sqlite",
-    "ssl",
-}
 TIMEOUT_INPUT_PATTERN = r"(0|[1-9][0-9]*|[0-9]+(?:\.[0-9]+)?\s*(?:s|sec|secs|second|seconds|m|min|mins|minute|minutes|h|hr|hrs|hour|hours))"


@@ -255,26 +141,21 @@ class PluginConfigFormMixin:
    allow_crawl_execution_config_fields = True

    def build_plugin_groups(self, runtime_config: Mapping[str, Any] | None = None) -> None:
-        all_plugins = get_plugins()
+        catalog = get_plugin_catalog()
+        all_plugins = set(catalog)
        plugin_configs = discover_plugin_configs()
        runtime_config = runtime_config or get_config()
        self.plugin_config_binary_urls = get_plugin_config_binary_urls(runtime_config)
-        grouped_plugins = set().union(*(group[-1] for group in PLUGIN_GROUP_DEFINITIONS))
-        other_plugins = tuple(sorted(set(all_plugins) - grouped_plugins - HIDDEN_PLUGIN_CONFIG_UI_PLUGINS))
+        grouped_plugins = catalog.groups()

-        for field_name, *_rest, plugin_names in PLUGIN_GROUP_DEFINITIONS:
+        group_specs = []
+        for category, field_name, title in PLUGIN_GROUPS:
+            plugin_names = tuple(plugin.name for plugin in grouped_plugins.get(category, []))
+            group_specs.append((field_name, title, "", "", "", plugin_names))
            if field_name in self.fields:
                get_choice_field(self, field_name).choices = [
                    (p, get_plugin_choice_label(p, plugin_configs)) for p in plugin_names if p in all_plugins
                ]
-
-        if "other_plugins" in self.fields:
-            get_choice_field(self, "other_plugins").choices = [(p, get_plugin_choice_label(p, plugin_configs)) for p in other_plugins]
-
-        group_specs = (
-            *PLUGIN_GROUP_DEFINITIONS,
-            ("other_plugins", "Other", "", "", "", other_plugins),
-        )
        binary_url_lookup = _build_required_binary_url_lookup(plugin_configs, runtime_config)
        self.plugin_groups = [
            {
@@ -395,7 +276,7 @@ class PluginConfigFormMixin:
        elif is_sensitive:
            input_type = "password"
        else:
-            input_value = "" if is_sensitive else str(current_value)
+            input_value = "" if current_value is None else str(current_value)

        return {
            "key": config_key,
@@ -409,7 +290,7 @@ class PluginConfigFormMixin:
            "default": _jsonish(default_display),
            "current": "configured"
            if is_sensitive and current_value
-            else (str(current_value) if "string" in schema_types else _jsonish(current_value)),
+            else (("" if current_value is None else str(current_value)) if "string" in schema_types else _jsonish(current_value)),
            "current_url": self.plugin_config_binary_urls.get(config_key, "") if str(config_key).endswith("_BINARY") else "",
            "is_sensitive": is_sensitive,
            "minimum": prop_schema.get("minimum"),
@@ -461,6 +342,8 @@ class PluginConfigFormMixin:
                    continue

                base_value = effective_config.get(config_key, prop_schema.get("default", ""))
+                if base_value is None and coerced_value == "":
+                    continue
                if _same_config_value(coerced_value, base_value):
                    continue

@@ -1,147 +1,32 @@
+"""ArchiveBox adapters around the framework-free abx-dl plugin runtime.
+
+Discovery and execution are owned by abx-dl. ArchiveBox keeps only the small
+Django projection adapter and its application-specific URL-output reader.
 """
-Hook discovery and execution helpers for ArchiveBox plugins.

-ArchiveBox no longer drives plugin execution itself during normal crawls.
-`abx-dl` owns the live runtime and emits typed bus events; ArchiveBox mainly:
-
- discovers hook files for inspection / docs / legacy direct execution helpers
- executes individual hook scripts when explicitly requested
- parses hook stdout JSONL records into ArchiveBox models when needed
-
-Hook-backed event families are discovered from filenames like:
-    on_CrawlSetup__*
-    on_Snapshot__*
-
-Internal bus event names are normalized to the corresponding
-`on_{EventFamily}__*` prefix by a simple string transform. If no scripts exist
-for that prefix, discovery returns `[]`.
-
-Directory structure:
-    abx_plugins/plugins/<plugin_name>/on_<Event>__<hook_name>.<ext>     (built-in package)
-    data/custom_plugins/<plugin_name>/on_<Event>__<hook_name>.<ext>     (user)
-
-Hook contract:
-    Input:  --url=<url> (and other --key=value args)
-    Output: JSONL records to stdout, files to $PWD
-    Exit:   0 = success, non-zero = failure
-
-Execution order:
-    - Hooks are named with two-digit prefixes (00-99) and sorted lexicographically by filename
-    - Foreground hooks run sequentially in that order
-    - Background hooks (.bg suffix) run concurrently and do not block foreground progress
-    - After all foreground hooks complete, background hooks receive SIGTERM and must finalize
-
-Hook naming convention:
-    on_{EventFamily}__{run_order}_{description}[.bg].{ext}
-
-API:
-    discover_hooks(event)     -> List[Path]     Find hook scripts for a hook-backed event family
-    run_hook(script, ...)     -> Process        Execute a hook script directly
-    is_background_hook(name)  -> bool           Check if hook is background (.bg suffix)
-"""
+from __future__ import annotations

 __package__ = "archivebox.plugins"

-import json
-import os
-import sys
-from collections.abc import Mapping
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Optional, Protocol, TypeGuard, runtime_checkable
+from typing import Any
+
+from abx_dl.models import parse_hook_filename

-from archivebox.config.constants import CONSTANTS
-from archivebox.config.version import VERSION
 from archivebox.misc.util import fix_url_from_markdown, sanitize_extracted_url
-from archivebox.plugins.discovery import (
-    BUILTIN_PLUGINS_DIR,
-    USER_PLUGINS_DIR,
-    ConfigLookup,
-    get_enabled_plugins,
-    get_plugin_special_config,
-)
-
-if TYPE_CHECKING:
-    from archivebox.machine.models import Process
-
-
-@runtime_checkable
-class ConfigDump(Protocol):
-    def as_dict(self) -> dict[str, Any]: ...
-
-
-def _has_config_dump(config: object) -> TypeGuard[ConfigDump]:
-    return isinstance(config, ConfigDump)
-
-
-def _config_to_overrides(config: ConfigLookup | Mapping[str, Any] | None) -> dict[str, Any]:
-    if config is None:
-        return {}
-    if _has_config_dump(config):
-        return dict(config.as_dict())
-    return dict(config.items())
-
-
-# =============================================================================
-# Hook Step Extraction
-# =============================================================================
+from archivebox.plugins.discovery import ConfigLookup, get_enabled_plugins, get_plugin_catalog


 def is_background_hook(hook_name: str) -> bool:
-    """
-    Check if a hook is a background hook (doesn't block foreground progression).
-
-    Background hooks have '.bg.' in their filename before the extension.
-
-    Args:
-        hook_name: Hook filename (e.g., 'on_Snapshot__10_chrome_tab.daemon.bg.js')
-
-    Returns:
-        True if background hook, False if foreground.
-
-    Examples:
-        is_background_hook('on_Snapshot__10_chrome_tab.daemon.bg.js') -> True
-        is_background_hook('on_Snapshot__50_wget.py') -> False
-        is_background_hook('on_Snapshot__63_media.finite.bg.py') -> True
-    """
-    return ".bg." in hook_name or "__background" in hook_name
+    parsed = parse_hook_filename(Path(hook_name).name)
+    return bool(parsed and parsed[2])


 def normalize_hook_event_name(event_name: str) -> str | None:
-    """
-    Normalize a hook event family or event class name to its on_* prefix.
-
-    Examples:
-        CrawlSetupEvent -> CrawlSetup
-        SnapshotEvent -> Snapshot
-        BinaryEvent -> Binary
-        CrawlCleanupEvent -> CrawlCleanup
-    """
    normalized = str(event_name or "").strip()
    if not normalized:
        return None
-
-    if normalized.endswith("Event"):
-        return normalized[:-5] or None
-    return normalized
-
-
-def _model_output_dir_from_child_path(path: Path, marker: str) -> Path | None:
-    """
-    Infer the model output dir from a model dir or one of its plugin subdirs.
-
-    Current ArchiveBox snapshot/crawl dirs are:
-        .../{snapshots,crawls}/YYYYMMDD/domain/uuid[/plugin]
-    """
-    parts = path.resolve().parts
-    try:
-        marker_index = parts.index(marker)
-    except ValueError:
-        return None
-
-    model_end_index = marker_index + 4
-    if len(parts) < model_end_index:
-        return None
-    return Path(*parts[:model_end_index])
+    return normalized.removesuffix("Event") or None


 def discover_hooks(
@@ -150,500 +35,40 @@ def discover_hooks(
    config: ConfigLookup | None = None,
    **config_kwargs: Any,
 ) -> list[Path]:
-    """
-    Find all hook scripts for an event family.
-
-    Searches both built-in and user plugin directories.
-    Filters out hooks from disabled plugins by default (respects USE_/SAVE_ flags).
-    Returns scripts sorted alphabetically by filename for deterministic execution order.
-
-    Hook naming convention uses numeric prefixes to control order:
-        on_Snapshot__10_title.py        # runs first
-        on_Snapshot__15_singlefile.py   # runs second
-        on_Snapshot__26_readability.py  # runs later (depends on singlefile)
-
-    Args:
-        event_name: Hook event family or event class name.
-            Examples: 'CrawlSetupEvent', 'Snapshot'.
-            Event names are normalized by stripping a trailing `Event`.
-            If no matching `on_{EventFamily}__*` scripts exist, returns [].
-        filter_disabled: If True, skip hooks from disabled plugins (default: True)
-        config: Optional pre-merged config dict from get_config().
-        **config_kwargs: Scope/override args forwarded to get_config() when config is not supplied.
-
-    Returns:
-        Sorted list of hook script paths from enabled plugins only.
-
-    Examples:
-        # With proper config context (recommended):
-        from archivebox.config.common import get_config
-        config = get_config(crawl=my_crawl, snapshot=my_snapshot)
-        discover_hooks('Snapshot', config=config)
-        # Returns: [Path('.../on_Snapshot__10_title.py'), ...] (wget excluded if SAVE_WGET=False)
-
-        # Without config (uses global defaults):
-        discover_hooks('Snapshot')
-        # Returns: [Path('.../on_Snapshot__10_title.py'), ...]
-
-        # Show all plugins regardless of enabled status:
-        discover_hooks('Snapshot', filter_disabled=False)
-        # Returns: [Path('.../on_Snapshot__10_title.py'), ..., Path('.../on_Snapshot__50_wget.py')]
-    """
-    hook_event_name = normalize_hook_event_name(event_name)
-    if not hook_event_name:
+    """Return the exact catalog hooks used by abx-dl, in execution order."""
+    normalized = normalize_hook_event_name(event_name)
+    if not normalized or normalized == "BinaryRequest":
        return []
-    if hook_event_name == "BinaryRequest":
-        return []
-
-    hooks = []
-
-    for base_dir in (BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR):
-        if not base_dir.exists():
-            continue
-
-        # Search for hook scripts in all subdirectories
-        for ext in ("sh", "py", "js"):
-            pattern = f"*/on_{hook_event_name}__*.{ext}"
-            hooks.extend(base_dir.glob(pattern))
-
-            # Also check for hooks directly in the plugins directory
-            pattern_direct = f"on_{hook_event_name}__*.{ext}"
-            hooks.extend(base_dir.glob(pattern_direct))
-
+    names = None
    if filter_disabled:
-        # Get merged config if not provided (lazy import to avoid circular dependency)
        if config is None:
            from archivebox.config.common import get_config

            config = get_config(**config_kwargs)
-
-        enabled_plugins = set(get_enabled_plugins(config=config))
-        enabled_hooks = []
-
-        for hook in hooks:
-            # Get plugin name from parent directory
-            # e.g., abx_plugins/plugins/wget/on_Snapshot__50_wget.py -> 'wget'
-            plugin_name = hook.parent.name
-
-            # Check if this is a plugin directory (not the root plugins dir)
-            if hook.parent.resolve() in (BUILTIN_PLUGINS_DIR.resolve(), USER_PLUGINS_DIR.resolve()):
-                # Hook is in root plugins directory, not a plugin subdir
-                # Include it by default (no filtering for non-plugin hooks)
-                enabled_hooks.append(hook)
-                continue
-
-            if plugin_name in enabled_plugins:
-                enabled_hooks.append(hook)
-
-        hooks = enabled_hooks
-
-    # Sort by filename (not full path) to ensure numeric prefix ordering works
-    # e.g., on_Snapshot__10_title.py sorts before on_Snapshot__26_readability.py
-    return sorted(set(hooks), key=lambda p: p.name)
-
-
-def run_hook(
-    script: Path,
-    output_dir: Path,
-    config: ConfigLookup | Mapping[str, Any] | None = None,
-    timeout: int | None = None,
-    parent: Optional["Process"] = None,
-    **kwargs: Any,
-) -> "Process":
-    """
-    Execute a hook script with the given arguments using Process model.
-
-    This is the low-level hook executor that creates a Process record and
-    uses Process.launch() for subprocess management.
-
-    Config is passed to hooks via environment variables. Crawl/snapshot callers
-    should pass the runtime config produced by for_crawl_runtime().
-
-    Args:
-        script: Path to the hook script (.sh, .py, or .js)
-        output_dir: Working directory for the script (where output files go)
-        config: Optional runtime config dict from for_crawl_runtime().
-                If omitted, pass scope/override args using kwargs prefixed with config_.
-        timeout: Maximum execution time in seconds
-                 If None, auto-detects from PLUGINNAME_TIMEOUT config (fallback to TIMEOUT, default 300)
-        parent: Optional parent Process (for tracking worker->hook hierarchy)
-        **kwargs: Arguments passed to the script as --key=value
-
-    Returns:
-        Process model instance (use process.exit_code, process.stdout, process.get_records())
-
-    Example:
-        from archivebox.config.common import get_config
-        config = get_config(crawl=my_crawl, snapshot=my_snapshot).for_crawl_runtime(crawl=my_crawl, snapshot=my_snapshot)
-        process = run_hook(hook_path, output_dir, config=config, url=url, snapshot_id=id)
-        if process.status == 'exited':
-            records = process.get_records()  # Get parsed JSONL output
-    """
-    from archivebox.machine.models import Process, Machine, NetworkInterface
-    from archivebox.config.common import (
-        ArchiveBoxConfig,
-        _archivebox_config_input_names,
-        get_config,
-        normalize_runtime_config,
-        _plugin_enabled_config_keys,
-    )
-
-    config_scope = {key.removeprefix("config_"): kwargs.pop(key) for key in list(kwargs) if key.startswith("config_")}
-    config_overrides = _config_to_overrides(config)
-    explicit_override_keys = set(config_overrides)
-    resolved_config = get_config(overrides=config_overrides, **config_scope)
-    hook_config = normalize_runtime_config(
-        resolved_config.for_crawl_runtime(runtime_overrides=config_overrides),
-        json_safe=False,
-    )
-    hook_config.update(normalize_runtime_config(config_overrides, json_safe=False))
-    plugin_enabled_keys = set(_plugin_enabled_config_keys().values())
-    if plugin_enabled_keys.intersection(hook_config):
-        for enabled_key in plugin_enabled_keys:
-            hook_config.setdefault(enabled_key, False)
-
-    # Auto-detect timeout from plugin config if not explicitly provided
-    if timeout is None:
-        plugin_name = script.parent.name
-        plugin_config = get_plugin_special_config(plugin_name, resolved_config)
-        timeout = plugin_config["timeout"]
-    if timeout:
-        timeout = min(int(timeout), int(CONSTANTS.MAX_HOOK_RUNTIME_SECONDS))
-
-    # Get current machine
-    machine = Machine.current()
-    iface = NetworkInterface.current(refresh=True)
-    machine = iface.machine
-
-    # Auto-detect parent process if not explicitly provided
-    # This enables automatic hierarchy tracking: Worker -> Hook
-    if parent is None:
-        try:
-            parent = Process.current()
-        except Exception:
-            # If Process.current() fails (e.g., not in a worker context), leave parent as None
-            pass
-
-    if not script.is_file():
-        raise FileNotFoundError(f"Hook script not found: {script}")
-
-    # ArchiveBox resolves hook dependencies before execution and exports the
-    # resolved *_BINARY env values below. Run Python hooks in the current Python
-    # environment so per-snapshot hook execution does not repeatedly pay the
-    # abxpkg script bootstrap cost from the standalone shebang.
-    # Shell/JS hooks still dispatch through abxpkg-resolved interpreters.
-    ext = script.suffix.lower()
-    if ext == ".sh":
-        bash_projection = Path(hook_config["ABXPKG_LIB_DIR"]).expanduser() / "env" / "bin" / "bash"
-        if not bash_projection.is_symlink() or not os.access(bash_projection, os.X_OK):
-            raise RuntimeError(f"Bash must be resolved by abxpkg into {bash_projection}")
-        cmd = [str(bash_projection), str(script)]
-    elif ext == ".py":
-        cmd = [sys.executable, str(script)]
-    elif ext == ".js":
-        node_projection = Path(hook_config["ABXPKG_LIB_DIR"]).expanduser() / "env" / "bin" / "node"
-        if not node_projection.is_symlink() or not os.access(node_projection, os.X_OK):
-            raise RuntimeError(f"Node.js must be resolved by abxpkg into {node_projection}")
-        hook_config["NODE_BINARY"] = str(node_projection)
-        cmd = [str(node_projection), str(script)]
-    else:
-        # Try to execute directly (assumes shebang)
-        cmd = [str(script)]
-
-    # Build CLI arguments from kwargs
-    for key, value in kwargs.items():
-        # Skip keys that start with underscore (internal parameters)
-        if key.startswith("_"):
-            continue
-
-        arg_key = f"--{key.replace('_', '-')}"
-        if isinstance(value, bool):
-            if value:
-                cmd.append(arg_key)
-        elif value is not None and value != "":
-            # JSON-encode complex values, use str for simple ones
-            # Skip empty strings to avoid --key= which breaks argument parsers
-            if isinstance(value, (dict, list)):
-                cmd.append(f"{arg_key}={json.dumps(value)}")
-            else:
-                # Ensure value is converted to string and strip whitespace
-                str_value = str(value).strip()
-                if str_value:  # Only add if non-empty after stripping
-                    cmd.append(f"{arg_key}={str_value}")
-
-    # Set up environment with base paths
-    env = os.environ.copy()
-    archivebox_config_input_names = _archivebox_config_input_names()
-    for key in archivebox_config_input_names:
-        env.pop(key, None)
-    env.pop("PLUGINS", None)
-    env["DATA_DIR"] = str(CONSTANTS.DATA_DIR)
-    env["LIBRARY_VERSION"] = VERSION
-    env.setdefault("MACHINE_ID", os.environ.get("MACHINE_ID", CONSTANTS.MACHINE_ID))
-    snap_dir = hook_config.get("SNAP_DIR") or _model_output_dir_from_child_path(output_dir, CONSTANTS.SNAPSHOTS_DIR_NAME)
-    crawl_dir = hook_config.get("CRAWL_DIR") or _model_output_dir_from_child_path(output_dir, CONSTANTS.CRAWLS_DIR_NAME)
-    if snap_dir:
-        env["SNAP_DIR"] = str(snap_dir)
-    if crawl_dir:
-        env["CRAWL_DIR"] = str(crawl_dir)
-
-    # Export the runtime library root; abx-dl/abxpkg own executable lookup env.
-    lib_dir = hook_config.get("ABXPKG_LIB_DIR")
-    if lib_dir:
-        env["ABXPKG_LIB_DIR"] = str(lib_dir)
-
-    # Set Node.js module resolution paths.
-    # NODE_PATH may be a path list, but NODE_MODULES_DIR is a single canonical directory.
-    node_modules_dir = hook_config.get("NODE_MODULES_DIR")
-    if lib_dir and "ABXPKG_LIB_DIR" in explicit_override_keys and "NODE_MODULES_DIR" not in explicit_override_keys:
-        node_modules_dir = Path(lib_dir) / "pnpm" / "packages" / "chrome" / "node_modules"
-    elif not node_modules_dir and lib_dir:
-        node_modules_dir = Path(lib_dir) / "pnpm" / "packages" / "chrome" / "node_modules"
-
-    node_path_parts = [part for part in str(hook_config.get("NODE_PATH") or "").split(os.pathsep) if part]
-    if node_modules_dir:
-        node_modules_dir = Path(node_modules_dir)
-        node_modules_dir.mkdir(parents=True, exist_ok=True)
-        node_modules_dir_str = str(node_modules_dir)
-        env["NODE_MODULES_DIR"] = node_modules_dir_str
-        env["NODE_MODULE_DIR"] = node_modules_dir_str
-        if node_modules_dir_str not in node_path_parts:
-            node_path_parts.append(node_modules_dir_str)
-    if node_path_parts:
-        env["NODE_PATH"] = os.pathsep.join(node_path_parts)
-
-    # Export all config values to environment (already merged by get_config())
-    # Skip keys we've already handled specially above (PATH, ABXPKG_LIB_DIR, NODE_PATH, etc.)
-    SKIP_KEYS = {
-        "PATH",
-        "ABXPKG_LIB_DIR",
-        "NODE_PATH",
-        "NODE_MODULES_DIR",
-        "NODE_MODULE_DIR",
-        "DATA_DIR",
-        "MACHINE_ID",
-        "SNAP_DIR",
-        "CRAWL_DIR",
-    }
-    canonical_config_keys = set(ArchiveBoxConfig.model_fields)
-    for key, value in hook_config.items():
-        if key in SKIP_KEYS:
-            continue  # Already handled specially above, don't overwrite
-        if key in archivebox_config_input_names and key not in canonical_config_keys:
-            continue
-        if value is None:
-            continue
-        elif isinstance(value, bool):
-            env[key] = "true" if value else "false"
-        elif isinstance(value, (list, dict)):
-            env[key] = json.dumps(value)
-        else:
-            env[key] = str(value)
-
-    # Create output directory if needed
-    output_dir.mkdir(parents=True, exist_ok=True)
-
-    # Detect if this is a background hook.
-    # Background hooks use the .bg. filename marker.
-    # Old convention: __background in stem (for backwards compatibility)
-    is_background = ".bg." in script.name or "__background" in script.stem
-
-    try:
-        # Create Process record
-        process = Process.objects.create(
-            machine=machine,
-            iface=iface,
-            parent=parent,
-            process_type=Process.TypeChoices.HOOK,
-            pwd=str(output_dir),
-            cmd=cmd,
-            timeout=timeout,
-        )
-
-        # Copy the env dict we already built (includes os.environ + all customizations)
-        process.env = env.copy()
-        process.hydrate_binary_from_context(plugin_name=script.parent.name, hook_path=str(script))
-
-        # Save env before launching
-        process.save()
-
-        # Launch subprocess using Process.launch()
-        process.launch(background=is_background)
-
-        # Return Process object (caller can use process.exit_code, process.stdout, process.get_records())
-        return process
-
-    except Exception as e:
-        # Create a failed Process record for exceptions
-        process = Process.objects.create(
-            machine=machine,
-            iface=iface,
-            process_type=Process.TypeChoices.HOOK,
-            pwd=str(output_dir),
-            cmd=cmd,
-            timeout=timeout,
-            status=Process.StatusChoices.EXITED,
-            exit_code=1,
-            stderr=f"Failed to run hook: {type(e).__name__}: {e}",
-        )
-        return process
-
-
-def extract_records_from_process(process: "Process") -> list[dict[str, Any]]:
-    """
-    Extract JSONL records from a Process's stdout.
-
-    Adds plugin metadata to each record.
-
-    Args:
-        process: Process model instance with stdout captured
-
-    Returns:
-        List of parsed JSONL records with plugin metadata
-    """
-    records = process.get_records()
-    if not records:
-        return []
-
-    # Extract plugin metadata from process.pwd and the shipped hook path in cmd.
-    # Python hooks execute directly through their shebang, while JS and shell
-    # hooks have an interpreter in cmd[0].
-    plugin_name = Path(process.pwd).name if process.pwd else "unknown"
-    plugin_hook = next((str(arg) for arg in process.cmd if Path(str(arg)).name.startswith("on_")), "")
-    hook_name = Path(plugin_hook).name if plugin_hook else "unknown"
-
-    for record in records:
-        # Add plugin metadata to record
-        record.setdefault("plugin", plugin_name)
-        record.setdefault("hook_name", hook_name)
-        record.setdefault("plugin_hook", plugin_hook)
-
-    return records
+        names = get_enabled_plugins(config=config)
+    return [hook.path for _plugin, hook in get_plugin_catalog().hooks(normalized, names=names)]


 def collect_urls_from_plugins(snapshot_dir: Path) -> list[dict[str, Any]]:
-    """
-    Collect all urls.jsonl entries from parser plugin output subdirectories.
-
-    Each parser plugin outputs urls.jsonl to its own subdir:
-        snapshot_dir/parse_rss_urls/urls.jsonl
-        snapshot_dir/parse_html_urls/urls.jsonl
-        etc.
-
-    This is not special handling - urls.jsonl is just a normal output file.
-    This utility collects them all for the crawl system.
-    """
-    urls = []
-
-    # Look in each immediate subdirectory for urls.jsonl
+    """Read the durable urls.jsonl interface emitted by parser plugins."""
+    urls: list[dict[str, Any]] = []
    if not snapshot_dir.exists():
        return urls

+    from archivebox.machine.models import Process
+
    for subdir in snapshot_dir.iterdir():
-        if not subdir.is_dir():
-            continue
-
        urls_file = subdir / "urls.jsonl"
-        if not urls_file.exists():
+        if not subdir.is_dir() or not urls_file.is_file():
            continue
-
        try:
-            from archivebox.machine.models import Process
-
-            text = urls_file.read_text()
-            for entry in Process.parse_records_from_text(text):
-                if entry.get("url"):
-                    entry["url"] = sanitize_extracted_url(fix_url_from_markdown(str(entry["url"]).strip()))
-                    if not entry["url"]:
-                        continue
-                    # Track which parser plugin found this URL
+            for entry in Process.parse_records_from_text(urls_file.read_text()):
+                if not entry.get("url"):
+                    continue
+                entry["url"] = sanitize_extracted_url(fix_url_from_markdown(str(entry["url"]).strip()))
+                if entry["url"]:
                    entry["plugin"] = subdir.name
                    urls.append(entry)
-        except Exception:
-            pass
-
+        except (OSError, UnicodeError):
+            continue
    return urls
-
-
-# =============================================================================
-# Hook Result Processing Helpers
-# =============================================================================
-
-
-def process_hook_records(records: list[dict[str, Any]], overrides: dict[str, Any] | None = None) -> dict[str, int]:
-    """
-    Process JSONL records emitted by hook stdout.
-
-    This handles hook-emitted record types such as Snapshot, Tag, and Binary.
-    It does not process internal bus lifecycle events, since those
-    are not emitted as JSONL records by hook subprocesses.
-
-    Args:
-        records: List of JSONL record dicts from result['records']
-        overrides: Dict with 'snapshot', 'crawl', 'dependency', 'created_by_id', etc.
-
-    Returns:
-        Dict with counts by record type
-    """
-    stats = {}
-    overrides = overrides or {}
-
-    for record in records:
-        record_type = record.get("type")
-        if not record_type:
-            continue
-
-        # Skip ArchiveResult records (they update the calling ArchiveResult, not create new ones)
-        if record_type == "ArchiveResult":
-            continue
-
-        try:
-            # Dispatch to appropriate model's from_json() method
-            if record_type == "Snapshot":
-                from archivebox.core.models import Snapshot
-
-                if record.get("url"):
-                    record = {
-                        **record,
-                        "url": sanitize_extracted_url(fix_url_from_markdown(str(record["url"]).strip())),
-                    }
-                    if not record["url"]:
-                        continue
-
-                # Check if discovered snapshot exceeds crawl max_depth
-                snapshot_depth = record.get("depth", 0)
-                crawl = overrides.get("crawl")
-                if crawl and snapshot_depth > crawl.max_depth:
-                    # Skip - this URL was discovered but exceeds max crawl depth
-                    continue
-
-                obj = Snapshot.from_json(record.copy(), overrides)
-                if obj:
-                    stats["Snapshot"] = stats.get("Snapshot", 0) + 1
-
-            elif record_type == "Tag":
-                from archivebox.core.models import Tag
-
-                obj = Tag.from_json(record.copy(), overrides)
-                if obj:
-                    stats["Tag"] = stats.get("Tag", 0) + 1
-
-            elif record_type == "Binary":
-                from archivebox.machine.models import Binary
-
-                obj = Binary.from_json(record.copy(), overrides)
-                if obj:
-                    stats[record_type] = stats.get(record_type, 0) + 1
-
-            else:
-                import sys
-
-                print(f"Warning: Unknown record type '{record_type}' from hook output", file=sys.stderr)
-
-        except Exception as e:
-            import sys
-
-            print(f"Warning: Failed to create {record_type}: {e}", file=sys.stderr)
-            continue
-
-    return stats
@@ -0,0 +1,54 @@
+"""Non-blocking, database-only collection totals for the progress monitor."""
+
+import logging
+from threading import Lock, Thread
+from time import time
+
+from django.core.cache import cache
+from django.db import close_old_connections, connections
+from django.db.models import Count, Sum
+
+
+_refresh_lock = Lock()
+
+
+def collection_summary(user):
+    user_id = None if user.is_superuser else user.pk
+    key = f"progress-collection:{'all' if user_id is None else user_id}"
+    summary = cache.get(key)
+    # Even an indexed SUM scans the index. Never make an HTTP request wait for
+    # that scan on a large collection. Serve stale totals while one worker refreshes;
+    # a single non-blocking lock prevents a queue of scans from concurrent clients.
+    if (summary is None or time() - summary["sampled_at"] >= 60) and _refresh_lock.acquire(blocking=False):
+        # Throttle failures too: a broken/locked database must not trigger work
+        # and log output on every sidebar poll. This cache is local to each worker.
+        if not cache.add(f"{key}:refresh", True, timeout=60):
+            _refresh_lock.release()
+            return summary
+
+        def refresh():
+            try:
+                close_old_connections()
+                from archivebox.core.models import Snapshot
+
+                snapshots = Snapshot.objects.all()
+                if user_id is not None:
+                    snapshots = snapshots.filter(crawl__created_by_id=user_id)
+                # COUNT(*) + SUM(output_size) can use Snapshot's covering size index.
+                # Snapshot.output_size is maintained when ArchiveResult sizes change.
+                totals = snapshots.aggregate(snapshots=Count("*"), bytes=Sum("output_size"))
+                totals["bytes"] = totals["bytes"] or 0
+                totals["sampled_at"] = time()
+                cache.set(key, totals, timeout=3600)
+            except Exception:
+                logging.getLogger(__name__).exception("Could not refresh collection totals")
+            finally:
+                connections.close_all()
+                _refresh_lock.release()
+
+        try:
+            Thread(target=refresh, name="collection-summary", daemon=True).start()
+        except RuntimeError:
+            _refresh_lock.release()
+            logging.getLogger(__name__).exception("Could not start collection totals refresh")
+    return summary
@@ -30,12 +30,16 @@ def progress_endpoint(scope: Literal["crawl", "snapshot"] | None = None, object_
@lru_cache(maxsize=1)
 def _live_progress_plugin_names() -> tuple[frozenset[str], frozenset[str]]:
    plugin_configs = discover_plugin_configs()
+    indexing_plugin_names = frozenset(
+        plugin_name
+        for plugin_name, plugin_config in plugin_configs.items()
+        if {"search", "flush"}.issubset(plugin_config.get("commands", {}))
+    )
    download_plugin_names = frozenset(
        plugin_name
        for plugin_name, plugin_config in plugin_configs.items()
-        if plugin_config.get("output_mimetypes") and not plugin_name.startswith("search_backend_")
+        if plugin_config.get("output_mimetypes") and plugin_name not in indexing_plugin_names
    )
-    indexing_plugin_names = frozenset(plugin_name for plugin_name in plugin_configs if plugin_name.startswith("search_backend_"))
    return download_plugin_names, indexing_plugin_names


@@ -938,6 +942,10 @@ def live_progress_view(request):
            "queued_crawls_hidden": queued_crawls_hidden,
            "server_time": timezone.now().isoformat(),
        }
+        if is_admin and not snapshot_id_filter and not crawl_id_filter and request.GET.get("collection") == "1":
+            from archivebox.progressmonitor.collection import collection_summary
+
+            payload["collection"] = collection_summary(request.user)
        try:
            import ujson

@@ -1,7 +1,7 @@
 __package__ = "archivebox.search"

+import json
 import os
-from contextlib import contextmanager
 from typing import Any

 from archivebox.config.common import get_config
@@ -10,29 +10,21 @@ from archivebox.config.common import get_config
 _search_backends_cache: dict | None = None


-@contextmanager
-def search_backend_env(config: dict[str, Any] | None = None, **config_kwargs: Any):
-    """Temporarily expose resolved search config through os.environ for backend code."""
+def search_backend_command_env(config: dict[str, Any] | None = None, **config_kwargs: Any) -> dict[str, str]:
+    """Serialize resolved application config for a standalone plugin command."""
    config = config or get_config(**config_kwargs)
-    updates = {}
+    env = os.environ.copy()
    for key, value in config.items():
        key = str(key)
-        if not (key.startswith("SEARCH_BACKEND_") or key.endswith("_BINARY")):
-            continue
        if value is None:
            continue
-        if isinstance(value, (str, int, float, bool, os.PathLike)):
-            updates[key] = str(value)
-    previous = {key: os.environ.get(key) for key in updates}
-    os.environ.update(updates)
-    try:
-        yield
-    finally:
-        for key, value in previous.items():
-            if value is None:
-                os.environ.pop(key, None)
-            else:
-                os.environ[key] = value
+        if isinstance(value, bool):
+            env[key] = "true" if value else "false"
+        elif isinstance(value, (dict, list, tuple)):
+            env[key] = json.dumps(value)
+        elif isinstance(value, (str, int, float, os.PathLike)):
+            env[key] = str(value)
+    return env


 def normalize_search_backend_name(backend_name: str | None) -> str:
@@ -41,7 +33,7 @@ def normalize_search_backend_name(backend_name: str | None) -> str:


 def get_available_backends() -> dict:
-    """Discover search backend plugin modules and cache them in memory."""
+    """Discover search-capable plugins and cache their catalog entries."""
    global _search_backends_cache

    if _search_backends_cache is None:
@@ -53,7 +45,7 @@ def get_available_backends() -> dict:


 def get_backend(config: dict[str, Any] | None = None, **config_kwargs: Any) -> Any:
-    """Resolve the configured search backend module."""
+    """Resolve the configured search-capable plugin."""
    config = config or get_config(**config_kwargs)
    backend_name = normalize_search_backend_name(config.SEARCH_BACKEND_ENGINE)
    backends = get_available_backends()
@@ -1,5 +1,6 @@
 __package__ = "archivebox.search"

+from subprocess import CalledProcessError
 from typing import Any

 from django.db import connection
@@ -8,7 +9,7 @@ from django.db.models import Case, IntegerField, Q, QuerySet, Value, When
 from archivebox.config.common import get_config
 from archivebox.misc.logging import stderr
 from archivebox.misc.util import enforce_types
-from archivebox.search.backends import get_available_backends, get_backend, normalize_search_backend_name, search_backend_env
+from archivebox.search.backends import get_available_backends, get_backend, normalize_search_backend_name, search_backend_command_env
 from archivebox.search.config import get_search_mode, get_search_mode_backend, get_search_mode_base


@@ -220,6 +221,7 @@ def iter_query_search_ids(
    search_mode: str | None = None,
    config: dict[str, Any] | None = None,
    max_results: int | None = None,
+    stop_event=None,
    **config_kwargs: Any,
 ):
    """Yield snapshot IDs from configured search backend modules."""
@@ -266,6 +268,8 @@ def iter_query_search_ids(
    seen: set[str] = set()
    try:
        for backend_name in backend_names:
+            if stop_event is not None and stop_event.is_set():
+                return
            backend = backends[backend_name]
            try:
                if backend_name == "sonic":
@@ -278,11 +282,22 @@ def iter_query_search_ids(
                    # --csv/--json. Keep daemon diagnostics on stderr.
                    with redirect_stdout(sys.stderr):
                        ensure_daemon_stack(reason="search query")
-                with search_backend_env(config=config):
-                    if backend_name == "ripgrep":
-                        ids = backend.iter_search(query, search_mode=search_mode_base)
-                    else:
-                        ids = backend.search(query)
+                from abx_dl.execution import iter_plugin_command
+                from archivebox.config.constants import CONSTANTS
+                from archivebox.plugins.discovery import get_plugin_catalog
+
+                command = get_plugin_catalog().command(backend.name, "search")
+                if command is None:
+                    raise RuntimeError(f'Plugin "{backend.name}" does not expose a search command')
+                ids = iter_plugin_command(
+                    command,
+                    arguments={"query": query, "search_mode": search_mode_base},
+                    env=search_backend_command_env(config=config),
+                    cwd=CONSTANTS.DATA_DIR,
+                    timeout=max(1, int(config.get("TIMEOUT", 60))) * 4,
+                    stop_event=stop_event,
+                )
+                try:
                    for snapshot_id in ids:
                        if snapshot_id in seen:
                            continue
@@ -290,21 +305,24 @@ def iter_query_search_ids(
                        yield snapshot_id
                        if max_results and len(seen) >= max_results:
                            return
+                finally:
+                    ids.close()
                successful_backends += 1
            except Exception as err:
                errors.append(err)
                if not fallback_enabled:
                    raise
+        if not successful_backends and errors and search_mode_base == "deep":
+            raise errors[0]
    except Exception as err:
        stderr()
        stderr(
            f"[X] The search backend threw an exception={err}:",
            color="red",
        )
+        if isinstance(err, CalledProcessError) and err.stderr:
+            stderr(err.stderr.rstrip(), color="red")
        raise
-    else:
-        if not successful_backends and errors and search_mode_base == "deep":
-            raise errors[0]


@enforce_types
@@ -318,8 +336,22 @@ def flush_search_index(snapshots: QuerySet, config: dict[str, Any] | None = None
    snapshot_pks = [str(pk) for pk in snapshots.values_list("pk", flat=True)]

    try:
-        with search_backend_env(config=config):
-            backend.flush(snapshot_pks)
+        from abx_dl.execution import iter_plugin_command
+        from archivebox.config.constants import CONSTANTS
+        from archivebox.plugins.discovery import get_plugin_catalog
+
+        command = get_plugin_catalog().command(backend.name, "flush")
+        if command is None:
+            raise RuntimeError(f'Plugin "{backend.name}" does not expose a flush command')
+        list(
+            iter_plugin_command(
+                command,
+                stdin=snapshot_pks,
+                env=search_backend_command_env(config=config),
+                cwd=CONSTANTS.DATA_DIR,
+                timeout=max(1, int(config.get("TIMEOUT", 60))) * 4,
+            ),
+        )
    except Exception as err:
        stderr()
        stderr(
@@ -6,7 +6,7 @@ import json
 import threading
 import time
 from copy import copy
-from queue import Full, Queue
+from queue import Empty, Full, Queue
 from urllib.parse import urlsplit
 from uuid import UUID

@@ -254,15 +254,18 @@ def iter_filtered_search_result_ids(iterator, queryset, *, flush_max_delay=0.05)
        yield from flush_batch()


-def iter_search_result_ids(query, base_queryset, *, search_mode, config):
+def iter_search_result_ids(query, base_queryset, *, search_mode, config, stop_event=None):
    """Yield filtered Snapshot IDs from the selected search provider."""
    search_mode_base = get_search_mode_base(search_mode, config=config)
    provider = (
        iter_meta_search_ids(query, base_queryset)
        if search_mode_base == "meta"
-        else iter_query_search_ids(query, search_mode=search_mode, config=config)
+        else iter_query_search_ids(query, search_mode=search_mode, config=config, stop_event=stop_event)
    )
-    yield from iter_filtered_search_result_ids(provider, base_queryset)
+    try:
+        yield from iter_filtered_search_result_ids(provider, base_queryset)
+    finally:
+        provider.close()


 def snapshot_search_stream_response(query, base_queryset, *, search_mode, config, cache_key, thread_name):
@@ -300,7 +303,7 @@ def snapshot_search_stream_response(query, base_queryset, *, search_mode, config
            iterator = None
            try:
                close_old_connections()
-                iterator = iter_search_result_ids(query, base_queryset, search_mode=search_mode, config=config)
+                iterator = iter_search_result_ids(query, base_queryset, search_mode=search_mode, config=config, stop_event=stop_event)
                for snapshot_id in iterator:
                    if stop_event.is_set():
                        break
@@ -326,10 +329,13 @@ def snapshot_search_stream_response(query, base_queryset, *, search_mode, config
                emit(None)

        threading.Thread(target=run_search, name=thread_name, daemon=True).start()
-        yield f"0{stream_padding}\n"
        try:
+            yield f"0{stream_padding}\n"
            while True:
-                item = await asyncio.to_thread(queue.get)
+                try:
+                    item = await asyncio.to_thread(queue.get, True, 0.1)
+                except Empty:
+                    continue
                if item is None:
                    break
                if isinstance(item, BaseException):
@@ -1,5 +1,5 @@
 from .archive_result_service import ArchiveResultService
-from .binary_service import ArchiveBoxBinaryService, ArchiveBoxDBBinaryCacheBackend
+from .binary_service import ArchiveBoxBinaryService
 from .crawl_service import CrawlService
 from .machine_service import MachineService
 from .process_service import ProcessService
@@ -10,7 +10,6 @@ from .tag_service import TagService
 __all__ = [
    "ArchiveResultService",
    "ArchiveBoxBinaryService",
-    "ArchiveBoxDBBinaryCacheBackend",
    "CrawlService",
    "MachineService",
    "ProcessService",
@@ -2,24 +2,19 @@ from __future__ import annotations

 import asyncio
 import inspect
-import json
 import os
-import signal
 import sys
 import time
-from collections import defaultdict
-from collections.abc import Iterable
 from contextlib import contextmanager
 from functools import wraps
 from pathlib import Path
-from typing import Any, Protocol, runtime_checkable
+from typing import Any

 from asgiref.sync import sync_to_async
-from django.db import IntegrityError
 from django.utils import timezone

-from abx_dl.events import PROCESS_EXIT_SKIPPED, ArchiveResultEvent, ProcessCompletedEvent, ProcessStartedEvent, SnapshotEvent
-from abx_dl.output_files import guess_mimetype
+from abx_dl.events import ArchiveResultEvent, ProcessStartedEvent
+from abx_dl.output_files import OutputManifest
 from abx_dl.services.base import BaseService

 from .process_service import parse_event_datetime
@@ -71,129 +66,15 @@ def _perf_span(label: str):
        print(f"PERF_TRACE label={label} ms={elapsed_ms:.3f}", file=sys.stderr, flush=True)


-@runtime_checkable
-class ModelDumpable(Protocol):
-    def model_dump(self) -> dict[str, Any]: ...
-
-
-def _collect_output_metadata(plugin_dir: Path) -> tuple[dict[str, dict], int, str]:
-    exclude_names = {"stdout.log", "stderr.log", "process.pid", "hook.pid", "listener.pid"}
-    output_files: dict[str, dict] = {}
-    mime_sizes: dict[str, int] = defaultdict(int)
-    total_size = 0
-
-    if not plugin_dir.exists():
-        return output_files, total_size, ""
-
-    for file_path in plugin_dir.rglob("*"):
-        if not file_path.is_file():
-            continue
-        if ".hooks" in file_path.parts:
-            continue
-        if file_path.name in exclude_names:
-            continue
-        try:
-            stat = file_path.stat()
-        except OSError:
-            continue
-        mime_type = guess_mimetype(file_path) or "application/octet-stream"
-        relative_path = str(file_path.relative_to(plugin_dir))
-        output_files[relative_path] = {
-            "extension": file_path.suffix.lower().lstrip("."),
-            "mimetype": mime_type,
-            "size": stat.st_size,
-        }
-        mime_sizes[mime_type] += stat.st_size
-        total_size += stat.st_size
-
-    output_mimetypes = ",".join(mime for mime, _size in sorted(mime_sizes.items(), key=lambda item: item[1], reverse=True))
-    return output_files, total_size, output_mimetypes
-
-
-def _coerce_output_file_size(value: Any) -> int:
-    try:
-        return max(int(value or 0), 0)
-    except (TypeError, ValueError):
-        return 0
-
-
-def _normalize_output_files(raw_output_files: Any) -> dict[str, dict]:
-    def _enrich_metadata(path: str, metadata: dict[str, Any]) -> dict[str, Any]:
-        normalized = dict(metadata)
-        if "extension" not in normalized:
-            normalized["extension"] = Path(path).suffix.lower().lstrip(".")
-        if "mimetype" not in normalized:
-            guessed = guess_mimetype(path)
-            if guessed:
-                normalized["mimetype"] = guessed
-        return normalized
-
-    if raw_output_files is None:
-        return {}
-
-    if isinstance(raw_output_files, str):
-        try:
-            raw_output_files = json.loads(raw_output_files)
-        except json.JSONDecodeError:
-            return {}
-
-    if isinstance(raw_output_files, dict):
-        normalized: dict[str, dict] = {}
-        for path, metadata in raw_output_files.items():
-            if not path:
-                continue
-            metadata_dict = dict(metadata) if isinstance(metadata, dict) else {}
-            metadata_dict.pop("path", None)
-            normalized[str(path)] = _enrich_metadata(str(path), metadata_dict)
-        return normalized
-
-    if not isinstance(raw_output_files, Iterable):
-        return {}
-
-    normalized: dict[str, dict] = {}
-    for item in raw_output_files:
-        if isinstance(item, str):
-            normalized[item] = _enrich_metadata(item, {})
-            continue
-        if isinstance(item, ModelDumpable):
-            item = item.model_dump()
-        if not isinstance(item, dict):
-            continue
-        path = str(item.get("path") or "").strip()
-        if not path:
-            continue
-        normalized[path] = _enrich_metadata(path, {key: value for key, value in item.items() if key != "path" and value not in (None, "")})
-
-    return normalized
-
-
-def _has_structured_output_metadata(output_files: dict[str, dict]) -> bool:
-    return any(any(key in metadata for key in ("extension", "mimetype", "size")) for metadata in output_files.values())
-
-
-def _summarize_output_files(output_files: dict[str, dict]) -> tuple[int, str]:
-    mime_sizes: dict[str, int] = defaultdict(int)
-    total_size = 0
-
-    for metadata in output_files.values():
-        if not isinstance(metadata, dict):
-            continue
-        size = _coerce_output_file_size(metadata.get("size"))
-        mimetype = str(metadata.get("mimetype") or "").strip()
-        total_size += size
-        if mimetype and size:
-            mime_sizes[mimetype] += size
-
-    output_mimetypes = ",".join(mime for mime, _size in sorted(mime_sizes.items(), key=lambda item: item[1], reverse=True))
-    return total_size, output_mimetypes
+def _manifest_metadata(manifest: OutputManifest) -> tuple[dict[str, dict], int, str]:
+    return manifest.as_mapping(), manifest.total_size, ",".join(manifest.mimetypes)


 def _resolve_output_metadata(raw_output_files: Any, plugin_dir: Path) -> tuple[dict[str, dict], int, str]:
-    normalized_output_files = _normalize_output_files(raw_output_files)
-    if normalized_output_files and _has_structured_output_metadata(normalized_output_files):
-        output_size, output_mimetypes = _summarize_output_files(normalized_output_files)
-        return normalized_output_files, output_size, output_mimetypes
-    return _collect_output_metadata(plugin_dir)
+    manifest = OutputManifest.from_value(raw_output_files)
+    if manifest.files and any(output_file.size for output_file in manifest.files):
+        return _manifest_metadata(manifest)
+    return _manifest_metadata(OutputManifest.scan(plugin_dir, containment_root=plugin_dir.parent))


 def _normalize_status(status: str) -> str:
@@ -238,37 +119,6 @@ def _should_update_snapshot_title(current_title: str, next_title: str, *, snapsh
    return len(next_title) > len(current)


-def _status_for_process_without_archive_result(event: ProcessCompletedEvent) -> str:
-    if event.exit_code == PROCESS_EXIT_SKIPPED:
-        return "skipped"
-    if event.exit_code in {128 + signal.SIGHUP, 128 + signal.SIGINT, 128 + signal.SIGTERM}:
-        # This fallback only runs when a snapshot hook exited before emitting a
-        # structured ArchiveResult. A polite shutdown signal means the runner
-        # was interrupted during ownership transfer, not that the extractor
-        # produced a durable negative result. Keep the hook queued so the next
-        # runner can retry the exact work item instead of sealing in a transient
-        # process-lifecycle failure.
-        return "queued"
-    if event.exit_code != 0:
-        return "failed"
-    return "noresults"
-
-
-def _iter_archiveresult_records(stdout: str) -> list[dict]:
-    records: list[dict] = []
-    for raw_line in stdout.splitlines():
-        line = raw_line.strip()
-        if not line.startswith("{"):
-            continue
-        try:
-            record = json.loads(line)
-        except json.JSONDecodeError:
-            continue
-        if record.get("type") == "ArchiveResult":
-            records.append(record)
-    return records
-
-
@_perf_trace("archivebox.ArchiveResultService._save_archiveresult_event_sync")
 def _save_archiveresult_event_to_db(
    event: ArchiveResultEvent,
@@ -333,28 +183,13 @@ def _save_archiveresult_event_to_db(
        if event.error:
            defaults["notes"] = event.error

-    with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_lookup"):
-        result = ArchiveResult.objects.filter(
-            snapshot=snapshot,
-            plugin=event.plugin,
-            hook_name=event.hook_name,
-        ).first()
-    if result is None:
-        try:
-            with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_create"):
-                result = ArchiveResult.objects.create(
-                    snapshot=snapshot,
-                    plugin=event.plugin,
-                    hook_name=event.hook_name,
-                    **defaults,
-                )
-        except IntegrityError:
-            with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_get_after_integrity"):
-                result = ArchiveResult.objects.get(
-                    snapshot=snapshot,
-                    plugin=event.plugin,
-                    hook_name=event.hook_name,
-                )
+    with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_get_or_create"):
+        result, _created = ArchiveResult.get_or_create_by_hook(
+            snapshot,
+            event.plugin,
+            event.hook_name,
+            defaults=defaults,
+        )

    with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.diff_fields"):
        update_fields = []
@@ -366,13 +201,6 @@ def _save_archiveresult_event_to_db(
        with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_update"):
            result.save(update_fields=[*update_fields, "modified_at"])

-    if result.status == ArchiveResult.StatusChoices.QUEUED:
-        # ArchiveResult has no retry_at column. If a shutdown/takeover projects
-        # a killed hook back to QUEUED, wake the parent Snapshot/Crawl so the
-        # next runner retries that exact hook instead of waiting on a stale
-        # active-state lease.
-        snapshot.update_and_requeue(retry_at=timezone.now())
-
    if result.status in (ArchiveResult.StatusChoices.SUCCEEDED, ArchiveResult.StatusChoices.NORESULTS):
        with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.title_update"):
            title_output_str = result.output_str if result.status == ArchiveResult.StatusChoices.SUCCEEDED else ""
@@ -396,36 +224,45 @@ def _save_archiveresult_event_to_db(


 def mark_archiveresult_started(event: ProcessStartedEvent, *, snapshot_id: str, process_id: str) -> None:
-    """Advance an existing queued hook row after its OS process is persisted."""
-    from archivebox.core.models import ArchiveResult
+    """Project a running abx-dl hook after its OS process is persisted."""
+    from archivebox.core.models import ArchiveResult, Snapshot

    started_at = parse_event_datetime(event.start_ts)
    if started_at is None:
        raise ValueError("ProcessStartedEvent.start_ts is required")
-    ArchiveResult.objects.filter(
-        snapshot_id=snapshot_id,
-        plugin=event.plugin_name,
-        hook_name=event.hook_name,
-        status=ArchiveResult.StatusChoices.QUEUED,
-    ).update(
-        status=ArchiveResult.StatusChoices.STARTED,
-        start_ts=started_at,
-        end_ts=None,
-        process_id=process_id,
-        modified_at=timezone.now(),
+    snapshot = Snapshot.objects.filter(id=snapshot_id).first()
+    if snapshot is None:
+        return
+    result, _created = ArchiveResult.get_or_create_by_hook(
+        snapshot,
+        event.plugin_name,
+        event.hook_name,
+        defaults={
+            "status": ArchiveResult.StatusChoices.STARTED,
+            "start_ts": started_at,
+            "end_ts": None,
+            "process_id": process_id,
+        },
    )
+    if result.start_ts is not None and started_at <= result.start_ts:
+        return
+    result.status = ArchiveResult.StatusChoices.STARTED
+    result.start_ts = started_at
+    result.end_ts = None
+    result.process_id = process_id
+    result.save(update_fields=["status", "start_ts", "end_ts", "process_id", "modified_at"])


 class ArchiveResultService(BaseService):
-    LISTENS_TO = [ArchiveResultEvent, ProcessCompletedEvent]
+    """Project abx-dl ArchiveResult facts into Django models."""
+
+    LISTENS_TO = [ArchiveResultEvent]
    EMITS = []

    def __init__(self, bus):
-        self._completed_process_event_ids: set[str] = set()
        self._save_locks: dict[tuple[str, str, str], asyncio.Lock] = {}
        super().__init__(bus)
        self.bus.on(ArchiveResultEvent, self.on_ArchiveResultEvent__save_to_db)
-        self.bus.on(ProcessCompletedEvent, self.on_ProcessCompletedEvent__save_to_db)

    @_perf_trace("archivebox.ArchiveResultService.on_ArchiveResultEvent__save_to_db")
    async def on_ArchiveResultEvent__save_to_db(self, event: ArchiveResultEvent) -> None:
@@ -441,64 +278,3 @@ class ArchiveResultService(BaseService):
        lock = self._save_locks.setdefault(key, asyncio.Lock())
        async with lock:
            await sync_to_async(_save_archiveresult_event_to_db, thread_sensitive=True)(event, process_started)
-
-    @_perf_trace("archivebox.ArchiveResultService.on_ProcessCompletedEvent__save_to_db")
-    async def on_ProcessCompletedEvent__save_to_db(self, event: ProcessCompletedEvent) -> None:
-        if event.event_id in self._completed_process_event_ids:
-            return
-        self._completed_process_event_ids.add(event.event_id)
-
-        if not event.hook_name.startswith("on_Snapshot"):
-            return
-        with _perf_span("archivebox.ArchiveResultService.on_ProcessCompletedEvent.find_snapshot_event"):
-            snapshot_event = await self.bus.find(
-                SnapshotEvent,
-                past=True,
-                future=False,
-                where=lambda candidate: self.bus.event_is_child_of(event, candidate),
-            )
-        if snapshot_event is None:
-            return
-
-        with _perf_span("archivebox.ArchiveResultService.on_ProcessCompletedEvent.parse_stdout_records"):
-            records = _iter_archiveresult_records(event.stdout)
-        if records:
-            if len(records) > 1:
-                raise RuntimeError(
-                    f"Hook {event.plugin_name}:{event.hook_name} emitted {len(records)} ArchiveResult records; expected exactly one",
-                )
-            for record in records:
-                record_status = _normalize_status(record.get("status") or "")
-                record_failed = record_status == "failed" or (not record_status and event.exit_code not in (0, PROCESS_EXIT_SKIPPED))
-                with _perf_span("archivebox.ArchiveResultService.on_ProcessCompletedEvent.emit_archive_result_record"):
-                    await event.emit(
-                        ArchiveResultEvent(
-                            snapshot_id=record.get("snapshot_id") or snapshot_event.snapshot_id,
-                            plugin=record.get("plugin") or event.plugin_name,
-                            hook_name=record.get("hook_name") or event.hook_name,
-                            status=record_status,
-                            output_str=record.get("output_str") or "",
-                            output_json=record.get("output_json") if isinstance(record.get("output_json"), dict) else None,
-                            output_files=event.output_files,
-                            start_ts=event.start_ts,
-                            end_ts=event.end_ts,
-                            error=record.get("error") or (event.stderr if record_failed else ""),
-                        ),
-                    ).now()
-            return
-
-        process_failed = _status_for_process_without_archive_result(event) == "failed"
-        with _perf_span("archivebox.ArchiveResultService.on_ProcessCompletedEvent.emit_archive_result_fallback"):
-            await event.emit(
-                ArchiveResultEvent(
-                    snapshot_id=snapshot_event.snapshot_id,
-                    plugin=event.plugin_name,
-                    hook_name=event.hook_name,
-                    status=_status_for_process_without_archive_result(event),
-                    output_str=event.stderr if process_failed else "",
-                    output_files=event.output_files,
-                    start_ts=event.start_ts,
-                    end_ts=event.end_ts,
-                    error=event.stderr if process_failed else "",
-                ),
-            ).now()
@@ -9,152 +9,12 @@ from typing import Any
 from asgiref.sync import sync_to_async
 from django.utils import timezone

-from abxpkg import Binary as AbxBinary
-from abxpkg import BinProvider, PROVIDER_CLASS_BY_NAME
 from abxpkg.binary_service import BinaryEvent, BinaryRequestEvent
 from abxpkg.config import load_derived_cache
 from abxbus import BaseEvent, EventBus
 from abx_dl.services.base import BaseService


-class ArchiveBoxDBBinaryCacheBackend:
-    """ArchiveBox machine.Binary projection backend for abxpkg BinaryCacheService."""
-
-    async def get(self, request: BinaryRequestEvent) -> AbxBinary | None:
-        from archivebox.config.common import get_config
-        from archivebox.machine.models import Binary, Machine, _canonical_binary_name
-
-        machine = await sync_to_async(Machine.current, thread_sensitive=True)()
-        binary_name = _canonical_binary_name(request.name)
-        if not binary_name:
-            return None
-
-        persisted_overrides = _persisted_overrides_for_request(request)
-        native_overrides = request.overrides or {}
-        requested_provider_names = _provider_names(request.binproviders)
-        await sync_to_async(get_config, thread_sensitive=True)()
-
-        installed_qs = (
-            Binary.objects.filter(machine=machine, name=binary_name, status=Binary.StatusChoices.INSTALLED)
-            .exclude(abspath="")
-            .exclude(abspath__isnull=True)
-            .order_by("-modified_at")
-        )
-        async for installed in installed_qs:
-            installed_path = Path(installed.abspath).expanduser().resolve(strict=False)
-            if not await sync_to_async(installed_path.exists, thread_sensitive=True)():
-                await _mark_binary_queued(installed)
-                continue
-            if persisted_overrides and installed.overrides != persisted_overrides:
-                await _mark_binary_queued(installed)
-                continue
-
-            provider_name = (installed.binprovider or installed.binproviders.split(",", 1)[0]).strip()
-            if provider_name and provider_name not in requested_provider_names:
-                await _mark_binary_queued(installed)
-                continue
-
-            provider = _provider_for_name(provider_name, installed.name, native_overrides)
-            binary_env = BinProvider.build_exec_env(providers=[provider], base_env={}) if provider is not None else {}
-            provider_names = _provider_names(installed.binproviders or request.binproviders or "env")
-            return AbxBinary.model_validate(
-                {
-                    "name": request.name,
-                    "description": request.description,
-                    "binproviders": _providers_for_names(provider_names),
-                    "overrides": native_overrides,
-                    "loaded_binprovider": provider,
-                    "loaded_abspath": installed.abspath,
-                    "loaded_version": installed.version or None,
-                    "loaded_sha256": installed.sha256 or None,
-                    "env": binary_env,
-                },
-            )
-
-        existing = await Binary.objects.filter(machine=machine, name=binary_name).order_by("-modified_at").afirst()
-        if existing is None:
-            await Binary.objects.acreate(
-                machine=machine,
-                name=binary_name,
-                binproviders=_binproviders_to_str(request.binproviders),
-                overrides=persisted_overrides,
-                status=Binary.StatusChoices.QUEUED,
-            )
-        else:
-            changed = False
-            requested_binproviders = _binproviders_to_str(request.binproviders)
-            if requested_binproviders and existing.binproviders != requested_binproviders:
-                existing.binproviders = requested_binproviders
-                changed = True
-            if persisted_overrides and existing.overrides != persisted_overrides:
-                existing.overrides = persisted_overrides
-                changed = True
-            if existing.status != Binary.StatusChoices.QUEUED:
-                existing.status = Binary.StatusChoices.QUEUED
-                existing.retry_at = None
-                changed = True
-            if changed:
-                await existing.asave(update_fields=["binproviders", "overrides", "status", "retry_at", "modified_at"])
-        return None
-
-    async def set(self, request: BinaryRequestEvent | None, binary: AbxBinary) -> None:
-        from archivebox.machine.models import Binary, Machine, _canonical_binary_name
-
-        machine = await sync_to_async(Machine.current, thread_sensitive=True)()
-        binary_name = _canonical_binary_name(binary.name)
-        if not binary_name:
-            return
-        request_context = request.extra_context if request is not None else {}
-        binary_id = str(request_context.get("binary_id") or "")
-        if binary_id:
-            existing = await Binary.objects.filter(id=binary_id).afirst()
-        else:
-            existing = None
-        if existing is None:
-            existing, _created = await Binary.objects.aget_or_create(
-                machine=machine,
-                name=binary_name,
-                defaults={"status": Binary.StatusChoices.QUEUED},
-            )
-
-        existing.abspath = str(binary.loaded_abspath or "")
-        if binary.loaded_version:
-            existing.version = str(binary.loaded_version)
-        if binary.loaded_sha256:
-            existing.sha256 = str(binary.loaded_sha256)
-        existing.binproviders = _binproviders_to_str(
-            request.binproviders if request is not None else [provider.name for provider in binary.binproviders],
-        )
-        if binary.loaded_binprovider is not None:
-            existing.binprovider = binary.loaded_binprovider.name
-        existing.overrides = _persisted_overrides_for_request(request) if request is not None else binary.overrides
-        existing.status = Binary.StatusChoices.INSTALLED
-        existing.retry_at = None
-        await existing.asave(
-            update_fields=["abspath", "version", "sha256", "binproviders", "binprovider", "overrides", "status", "retry_at", "modified_at"],
-        )
-
-    async def invalidate(self, request: BinaryRequestEvent, binary: AbxBinary, reason: str) -> None:
-        from archivebox.machine.models import Binary, Machine, _canonical_binary_name
-
-        machine = await sync_to_async(Machine.current, thread_sensitive=True)()
-        binary_name = _canonical_binary_name(request.name)
-        if not binary_name:
-            return
-        installed = (
-            await Binary.objects.filter(machine=machine, name=binary_name, status=Binary.StatusChoices.INSTALLED)
-            .exclude(abspath="")
-            .exclude(abspath__isnull=True)
-            .order_by("-modified_at")
-            .afirst()
-        )
-        if installed is None:
-            return
-        installed.status = Binary.StatusChoices.QUEUED
-        installed.retry_at = None
-        await installed.asave(update_fields=["status", "retry_at", "modified_at"])
-
-
 class ArchiveBoxBinaryService(BaseService):
    """Preserve ArchiveBox's legacy Binary Process rows around abxpkg requests."""

@@ -503,38 +363,6 @@ def _binproviders_to_str(binproviders: str | list[str] | None) -> str:
    return ",".join(_provider_names(binproviders))


-def _providers_for_names(names: list[str]) -> list[BinProvider]:
-    providers: list[BinProvider] = []
-    for name in names:
-        provider_class = PROVIDER_CLASS_BY_NAME.get(name)
-        if provider_class is not None:
-            providers.append(provider_class())
-    return providers
-
-
-def _provider_for_name(provider_name: str, binary_name: str, overrides: dict[str, Any] | None) -> BinProvider | None:
-    provider_class = PROVIDER_CLASS_BY_NAME.get(provider_name)
-    if provider_class is None:
-        return None
-    provider = provider_class()
-    provider_overrides = overrides.get(provider_name) if isinstance(overrides, dict) else None
-    if isinstance(provider_overrides, dict):
-        provider = provider.get_provider_with_overrides(
-            overrides={binary_name: provider_overrides},
-        )
-    return provider
-
-
-async def _mark_binary_queued(binary) -> None:
-    from archivebox.machine.models import Binary
-
-    if binary.status == Binary.StatusChoices.QUEUED:
-        return
-    binary.status = Binary.StatusChoices.QUEUED
-    binary.retry_at = None
-    await binary.asave(update_fields=["status", "retry_at", "modified_at"])
-
-
 def _persisted_overrides_for_request(request: BinaryRequestEvent | None) -> dict[str, Any]:
    if request is None:
        return {}
@@ -43,10 +43,10 @@ def normalize_process_env(env: dict) -> dict:
        if is_sensitive_config_key(key) or (key in config_input_names and key not in allowed_config_keys):
            normalized.pop(key, None)
    if selected_plugins:
-        from abx_dl.models import discover_plugins, filter_plugins
        from archivebox.config.common import _plugin_enabled_config_keys
+        from archivebox.plugins.discovery import get_plugin_catalog

-        selected_plugins = set(filter_plugins(discover_plugins(runtime="archivebox"), sorted(selected_plugins), include_providers=True))
+        selected_plugins = set(get_plugin_catalog().select(sorted(selected_plugins)))
        for plugin_name, enabled_key in _plugin_enabled_config_keys().items():
            normalized.setdefault(enabled_key, "True" if plugin_name in selected_plugins else "False")
    return normalized
@@ -100,6 +100,7 @@ class ProcessService(BaseService):
            process = await Process.objects.acreate(
                machine=iface.machine,
                iface=iface,
+                parent_id=None,
                process_type=process_type,
                worker_type=worker_type,
                pwd=event.output_dir,
@@ -133,6 +134,7 @@ class ProcessService(BaseService):
            hook_path=event.hook_path,
        )
        await Process.objects.filter(id=process.id).aupdate(
+            parent_id=process.parent_id,
            pwd=process.pwd,
            cmd=process.cmd,
            env=process.env,
@@ -219,6 +221,7 @@ class ProcessService(BaseService):
            await Process.objects.acreate(
                machine=iface.machine,
                iface=iface,
+                parent_id=None,
                process_type=process_type,
                worker_type=worker_type,
                pwd=event.output_dir,
@@ -239,6 +242,7 @@ class ProcessService(BaseService):
        updates = {
            "machine_id": iface.machine_id,
            "iface_id": iface.id,
+            "parent_id": process.parent_id,
            "pwd": event.output_dir,
            "env": process_env,
            "pid": event.pid or process.pid,
@@ -0,0 +1,91 @@
+"""Admission signals, not estimates of how much RAM an extractor will need."""
+
+import math
+import os
+import time
+from pathlib import Path
+
+
+RESOURCE_RECHECK_SECONDS = 2.0
+# Scheduling policy, not a minimum-RAM claim: stop increasing extraction
+# load when all non-idle work loses at least one second out of ten to memory
+# stalls. Require both recent and rolling pressure so transient reclaim does
+# not indefinitely starve otherwise viable work on small hosts.
+MEMORY_FULL_STALL_PERCENT = 10.0
+
+
+class ResourceAdmission:
+    def __init__(self):
+        self._memory_stalls: dict[str, tuple[int, float]] = {}
+        self._checked_at = float("-inf")
+        self._reason: str | None = None
+
+    @staticmethod
+    def cgroup_paths() -> list[Path]:
+        root = Path("/sys/fs/cgroup")
+        paths = [root]
+        try:
+            for line in Path("/proc/self/cgroup").read_text().splitlines():
+                if line.startswith("0::"):
+                    current = root / line[3:].lstrip("/")
+                    if current.is_dir() and current.is_relative_to(root):
+                        paths = [current, *[parent for parent in current.parents if parent.is_relative_to(root)]]
+                    break
+        except OSError:
+            pass
+        return paths
+
+    def snapshot_limit(self, configured: int) -> int:
+        cpus = len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else (os.cpu_count() or 1)
+        for path in self.cgroup_paths():
+            try:
+                quota, period = (path / "cpu.max").read_text().split()
+                if quota != "max":
+                    cpus = min(cpus, max(1, math.ceil(int(quota) / int(period))))
+            except (OSError, ValueError, ZeroDivisionError):
+                continue
+        # CPU quota limits parallel admission, not forward progress. A host
+        # with a fractional CPU must still be able to run one snapshot.
+        return max(1, min(configured, cpus))
+
+    def observe_memory_stalls(self, source: str, total: int, avg10: float, sampled_at: float) -> bool:
+        previous = self._memory_stalls.get(source)
+        self._memory_stalls[source] = (total, sampled_at)
+        if previous is None:
+            return avg10 >= MEMORY_FULL_STALL_PERCENT
+        previous_total, previous_time = previous
+        elapsed = sampled_at - previous_time
+        if elapsed <= 0 or total < previous_total:
+            return False
+        stalled_percent = (total - previous_total) / (elapsed * 1_000_000) * 100
+        return avg10 >= MEMORY_FULL_STALL_PERCENT and stalled_percent >= MEMORY_FULL_STALL_PERCENT
+
+    def memory_pressure(self) -> str | None:
+        now = time.monotonic()
+        if now - self._checked_at < RESOURCE_RECHECK_SECONDS:
+            return self._reason
+        self._checked_at = now
+        reasons = []
+        cgroups = self.cgroup_paths()
+        for path in cgroups:
+            try:
+                current = int((path / "memory.current").read_text())
+                for name in ("memory.high", "memory.max"):
+                    limit = (path / name).read_text().strip()
+                    if limit != "max" and current >= int(limit):
+                        reasons.append(f"{name} reached")
+            except (OSError, ValueError):
+                continue
+        for path in [Path("/proc/pressure/memory"), *[p / "memory.pressure" for p in cgroups]]:
+            try:
+                full = next(line for line in path.read_text().splitlines() if line.startswith("full "))
+                fields = dict(field.split("=", 1) for field in full.split()[1:])
+                if self.observe_memory_stalls(str(path), int(fields["total"]), float(fields["avg10"]), now):
+                    reasons.append("memory stalls")
+            except (OSError, ValueError, KeyError, StopIteration):
+                continue
+        self._reason = ", ".join(dict.fromkeys(reasons)) or None
+        return self._reason
+
+
+resource_admission = ResourceAdmission()
@@ -1,12 +1,10 @@
 from __future__ import annotations

-import sys
+import asyncio
 from pathlib import Path

 from asgiref.sync import sync_to_async
 from django.utils import timezone
-from django.core.exceptions import ValidationError
-from rich import print as rprint
 from abx_dl.events import SnapshotCompletedEvent, SnapshotEvent
 from abx_dl.limits import CrawlLimitState
 from abx_dl.services.base import BaseService
@@ -44,6 +42,9 @@ def project_discovered_snapshots(snapshot_id: str) -> list:
 def finalize_completed_snapshot(
    snapshot_id: str,
    *,
+    owned_retry_at,
+    was_sealed: bool,
+    consumed_retry_plugins: list[str] | None = None,
    output_dir=None,
    crawl_limit_stop_reason: str | None = None,
 ) -> None:
@@ -73,13 +74,40 @@ def finalize_completed_snapshot(
            modified_at=timezone.now(),
        )

-    if snapshot.status == Snapshot.StatusChoices.QUEUED:
-        snapshot.sm.tick()
-        snapshot.refresh_from_db()
-    if snapshot.status == Snapshot.StatusChoices.STARTED and snapshot.is_finished_processing():
-        snapshot.sm.seal()
+    # SnapshotCompletedEvent is abx-dl's authoritative signal that the complete
+    # snapshot hook sequence (including cleanup) finished. ArchiveResult rows
+    # are projections of that work, never prerequisites used to decide whether
+    # the Snapshot may seal.
+    if not was_sealed and snapshot.status == Snapshot.StatusChoices.STARTED and snapshot.retry_at == owned_retry_at:
+        snapshot.seal()
        snapshot.refresh_from_db()

+    consumed_plugins = sorted({str(name).strip() for name in (consumed_retry_plugins or []) if str(name).strip()})
+    if consumed_plugins and snapshot.status == Snapshot.StatusChoices.SEALED:
+        for _attempt in range(3):
+            current = Snapshot.objects.only("status", "retry_at", "config").get(pk=snapshot.pk)
+            pending_plugins = sorted(
+                {str(name).strip() for name in (current.config or {}).get("RETRY_PLUGINS", []) if str(name).strip()},
+            )
+            if pending_plugins != consumed_plugins:
+                break
+            owned_filter = {"retry_at": owned_retry_at} if was_sealed else {"retry_at__isnull": True}
+            updates: dict[str, object] = {
+                "config": {key: value for key, value in current.config.items() if key != "RETRY_PLUGINS"},
+                "modified_at": timezone.now(),
+            }
+            if was_sealed:
+                updates["retry_at"] = None
+            updated = Snapshot.objects.filter(
+                pk=current.pk,
+                status=Snapshot.StatusChoices.SEALED,
+                config=current.config,
+                **owned_filter,
+            ).update(**updates)
+            if updated:
+                snapshot.refresh_from_db()
+                break
+
    snapshot.write_index_jsonl(output_dir=output_dir)


@@ -100,6 +128,8 @@ class SnapshotService(BaseService):

    def __init__(self, bus, *, crawl_id: str):
        self.crawl_id = crawl_id
+        self._run_ownership: dict[str, tuple[str, object, bool, list[str]]] = {}
+        self._ownership_lock = asyncio.Lock()
        super().__init__(bus)
        self.bus.on(SnapshotEvent, self.on_SnapshotEvent)
        self.bus.on(SnapshotCompletedEvent, self.on_SnapshotCompletedEvent)
@@ -112,33 +142,49 @@ class SnapshotService(BaseService):
        if snapshot is not None:
            if snapshot.is_paused:
                return
+            was_sealed = snapshot.status == Snapshot.StatusChoices.SEALED
            if snapshot.status == Snapshot.StatusChoices.QUEUED:
-                if not await snapshot.archiveresult_set.aexists():
-                    from archivebox.services.runner import snapshot_hooks_for_pending_archiveresults
-
-                    hooks = await sync_to_async(snapshot_hooks_for_pending_archiveresults, thread_sensitive=True)(snapshot)
-                    await sync_to_async(snapshot.create_pending_archiveresults, thread_sensitive=True)(hooks=hooks)
-                try:
-                    await sync_to_async(snapshot.sm.tick, thread_sensitive=True)()
-                except ValidationError as err:
-                    if "ArchiveBox cannot archive its own admin, web, api, or snapshot URLs." not in str(err):
-                        raise
-                    await Snapshot.objects.filter(id=snapshot.id).aupdate(
-                        status=Snapshot.StatusChoices.SEALED,
-                        retry_at=None,
-                        modified_at=timezone.now(),
-                    )
-                    rprint(
-                        f"[red][X] Refusing to archive ArchiveBox internal URL for security: {snapshot.url}[/red]",
-                        file=sys.stderr,
-                    )
-                    return
+                await sync_to_async(snapshot.advance_lifecycle, thread_sensitive=True)()
                await sync_to_async(snapshot.refresh_from_db, thread_sensitive=True)()
-            elif snapshot.status != Snapshot.StatusChoices.STARTED:
+            elif snapshot.status not in (Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.SEALED):
                return
-            if snapshot.status != Snapshot.StatusChoices.STARTED:
-                return
-            await sync_to_async(snapshot.ensure_crawl_symlink, thread_sensitive=True)()
+            if snapshot.status == Snapshot.StatusChoices.STARTED:
+                await sync_to_async(snapshot.ensure_crawl_symlink, thread_sensitive=True)()
+            retry_plugins = [str(name).strip() for name in (snapshot.config or {}).get("RETRY_PLUGINS", []) if str(name).strip()]
+            async with self._ownership_lock:
+                self._run_ownership[str(event.snapshot_id)] = (str(event.event_id), snapshot.retry_at, was_sealed, retry_plugins)
+
+    async def renew_lease(self, snapshot_id: str, lease_until) -> bool | None:
+        from archivebox.core.models import Snapshot
+
+        async with self._ownership_lock:
+            ownership = self._run_ownership.get(str(snapshot_id))
+            if ownership is None or ownership[2]:
+                return None
+            event_id, owned_retry_at, was_sealed, retry_plugins = ownership
+            updated = await Snapshot.objects.filter(
+                id=snapshot_id,
+                status=Snapshot.StatusChoices.STARTED,
+                retry_at=owned_retry_at,
+            ).aupdate(
+                retry_at=lease_until,
+                modified_at=timezone.now(),
+            )
+            if updated:
+                self._run_ownership[str(snapshot_id)] = (event_id, lease_until, was_sealed, retry_plugins)
+            return bool(updated)

    async def on_SnapshotCompletedEvent(self, event: SnapshotCompletedEvent) -> None:
-        await sync_to_async(finalize_completed_snapshot, thread_sensitive=True)(event.snapshot_id)
+        snapshot_id = str(event.snapshot_id)
+        async with self._ownership_lock:
+            ownership = self._run_ownership.get(snapshot_id)
+            if ownership is None or ownership[0] != str(event.event_parent_id):
+                return
+            self._run_ownership.pop(snapshot_id, None)
+        _, owned_retry_at, was_sealed, retry_plugins = ownership
+        await sync_to_async(finalize_completed_snapshot, thread_sensitive=True)(
+            event.snapshot_id,
+            owned_retry_at=owned_retry_at,
+            was_sealed=was_sealed,
+            consumed_retry_plugins=retry_plugins,
+        )
@@ -1,5 +1,7 @@
 from __future__ import annotations

+from asgiref.sync import sync_to_async
+
 from abx_dl.events import TagEvent
 from abx_dl.services.base import BaseService

@@ -13,10 +15,10 @@ class TagService(BaseService):
        self.bus.on(TagEvent, self.on_TagEvent__save_to_db)

    async def on_TagEvent__save_to_db(self, event: TagEvent) -> None:
-        from archivebox.core.models import Snapshot, SnapshotTag, Tag
+        from archivebox.core.models import Snapshot, Tag

        snapshot = await Snapshot.objects.filter(id=event.snapshot_id).afirst()
        if snapshot is None:
            return
-        tag, _ = await Tag.objects.aget_or_create(name=event.name)
-        await SnapshotTag.objects.aget_or_create(snapshot=snapshot, tag=tag)
+        tag, _ = await sync_to_async(Tag.get_or_create_by_name, thread_sensitive=True)(event.name)
+        await sync_to_async(snapshot.add_tag_ids, thread_sensitive=True)([tag.pk])
@@ -155,13 +155,13 @@ Each action dict may set:
 </style>
 {% endblock %}

-{# Render the toolbar in object-tools (OUTSIDE the main <form>). It must not be
-   inside the form: the toolbar contains its own <form> elements (the Re-Archive
+{# Render the toolbar before block.super (OUTSIDE the main <form>). It must not
+   be inside the form: the toolbar contains its own <form> elements (Re-Archive
   dropdown, django-object-actions, crawl pause/resume) and nested forms are
   invalid HTML — the browser closes the outer form early and ejects the
   fieldsets, breaking the card layout and every field widget. The Save button
   re-associates with the main form via the HTML form="…" attribute. #}
-{% block object-tools %}
+{% block content %}
 {% if not is_popup %}
 <div class="archivebox-toolbar" role="toolbar" aria-label="Actions">
  {% if original and has_change_permission %}
@@ -254,6 +254,7 @@ Each action dict may set:
  {% endif %}
 </div>
 {% endif %}
+{{ block.super }}
 {% endblock %}

 {% block submit_buttons_top %}{% endblock %}
@@ -1339,20 +1339,36 @@
                display: inline-block;
            }

-            .action-rearchive-select,
-            .action-permissions-select {
+            /* The native select receives the actual click, including in WKWebView. */
+            body.change-list #changelist .actions .action-rearchive-select,
+            body.change-list #changelist .actions .action-permissions-select {
                position: absolute;
-                left: 0;
-                bottom: 0;
-                width: 1px;
-                height: 1px;
+                inset: 0;
+                width: 100%;
+                height: 100%;
                opacity: 0;
-                pointer-events: none;
-                border: 0;
-                padding: 0;
+                cursor: pointer;
+                z-index: 1;
                margin: 0;
            }

+            body.change-list #changelist .action-rearchive-wrapper > .button,
+            body.change-list #changelist .action-permissions-wrapper > .button {
+                padding: 8px 16px !important;
+                width: max-content;
+                min-width: max-content;
+                flex: 0 0 auto;
+                gap: 6px;
+                white-space: nowrap;
+            }
+
+            .action-rearchive-wrapper:focus-within,
+            .action-permissions-wrapper:focus-within {
+                outline: 2px solid #2563eb;
+                outline-offset: 2px;
+                border-radius: 6px;
+            }
+
            /* Object count */
            .actions .action-counter {
                color: #64748b;
@@ -1960,22 +1976,10 @@
                            .addClass('action-rearchive-select')
                            .attr('aria-label', 'Re-Archive')
                            .appendTo(wrapper)
-                        $('<button>')
-                            .attr('type', 'button')
+                        $('<span>')
+                            .attr('aria-hidden', 'true')
                            .addClass('button')
                            .html('Re-Archive <span aria-hidden="true">▾</span>')
-                            .click(function (e) {
-                                e.preventDefault()
-                                e.stopPropagation()
-                                const el = select[0]
-                                if (typeof el.showPicker === 'function') {
-                                    try { el.showPicker() } catch (err) { el.focus(); el.click() }
-                                } else {
-                                    el.focus()
-                                    el.click()
-                                }
-                                return false
-                            })
                            .appendTo(wrapper)
                        $('<option></option>').attr('value', '').attr('hidden', true).appendTo(select)
                        rearchiveActions.forEach(function (action_value) {
@@ -2021,23 +2025,11 @@
                            .addClass('action-permissions-select')
                            .attr('aria-label', 'Permissions')
                            .appendTo(wrapper)
-                        $('<button>')
-                            .attr('type', 'button')
+                        $('<span>')
+                            .attr('aria-hidden', 'true')
                            .attr('name', action_type)
                            .addClass('button')
                            .text(this.text)
-                            .click(function (e) {
-                                e.preventDefault()
-                                e.stopPropagation()
-                                const el = select[0]
-                                if (typeof el.showPicker === 'function') {
-                                    try { el.showPicker() } catch (err) { el.focus(); el.click() }
-                                } else {
-                                    el.focus()
-                                    el.click()
-                                }
-                                return false
-                            })
                            .appendTo(wrapper)
                        $('<option></option>').attr('value', '').attr('hidden', true).appendTo(select)
                        ;[
@@ -2556,5 +2548,6 @@
            }
        </script>
        <script src="{% static 'admin-inline-tags.js' %}"></script>
+        {% include "includes/output_delete_controls.html" %}
    </body>
 </html>