Keep macOS process inspection failures out of SQLite recovery

Machine.save intentionally invalidates the process cache so config edits are visible immediately. Keep a local instance through host refresh, sanitation, and file reconciliation, then publish the completed instance to the cache. Explain why invalidation must remain and why reading the global after save can return None.

Add a real CLI regression that ages persisted machine rows, runs archivebox status, and verifies successful refresh without changing machine identities. Confirmed failing before the fix; 13 targeted tests and Ruff checks pass.

.github/workflows/deploy-publicsite.yml

@@ -10,7 +10,6 @@ on:
       - bin/collect_ui_screenshots.*
       - bin/generate_ui_screenshot_gallery.py
       - bin/take_screenshot.js
-      - docs/Screenshots.md
       - pyproject.toml
       - uv.lock
       - .github/workflows/deploy-publicsite.yml

.github/workflows/docker.yml

@@ -357,7 +357,7 @@ jobs:
           "$SUDO_BINARY" "$MKDIR_BINARY" -p /etc/exports.d
           "$SUDO_BINARY" "$CHMOD_BINARY" 0777 "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
           "$SUDO_BINARY" "$BASH_BINARY" -c \
-            "printf '%s *(rw,sync,no_subtree_check,no_root_squash,insecure)\\n' '$NFS_SOURCE' > /etc/exports.d/archivebox-ci.exports"
+            "printf '%s *(rw,sync,no_subtree_check,root_squash,insecure)\\n' '$NFS_SOURCE' > /etc/exports.d/archivebox-ci.exports"
           "$SUDO_BINARY" "$BASH_BINARY" -c \
             "printf '\\n[archivebox-ci]\\npath = %s\\nbrowseable = yes\\nread only = no\\nguest ok = yes\\nguest only = yes\\nforce user = nobody\\ncreate mask = 0777\\ndirectory mask = 0777\\n' '$SMB_SOURCE' >> /etc/samba/smb.conf"
 

.github/workflows/docs.yml

@@ -99,9 +99,10 @@ jobs:
           if [[ '${{ matrix.environment }}' != macos ]]; then
             dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docs_binaries")
           fi
-          if [[ '${{ matrix.environment }}' == ubuntu ]]; then
+          if [[ '${{ matrix.name }}' == ubuntu-AGENTS || '${{ matrix.name }}' == ubuntu-skills-archivebox-SKILL || '${{ matrix.name }}' == ubuntu-docs-Configuration || '${{ matrix.name }}' == ubuntu-docs-Setting-up-Authentication ]]; then
             dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:required_binaries")
-          elif [[ '${{ matrix.environment }}' == docker ]]; then
+          fi
+          if [[ '${{ matrix.environment }}' == docker ]]; then
             dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries")
           elif [[ '${{ matrix.environment }}' == macos ]]; then
             dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-macos-brew.json:required_binaries")

.github/workflows/duplicate-issue-detection.yml

@@ -1,96 +0,0 @@
-name: Duplicate Issue Detection
-
-on:
-  issues:
-    types: [opened]
-
-jobs:
-  check-duplicates:
-    runs-on: ubuntu-24.04
-    permissions:
-      contents: read
-      issues: write
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 1
-
-      - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: "3.13"
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-
-      - name: Resolve OpenCode and workflow tools through abxpkg
-        env:
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-        run: |
-          set -Eeuo pipefail
-          uv sync --no-cache --locked --no-sources --no-install-project
-          OPENCODE_CONFIG="$(uv run --no-cache --no-sync --no-sources python -c 'from abx_plugins import get_plugins_dir; print(get_plugins_dir() / "opencode" / "config.json")')"
-          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
-          export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
-          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
-          echo "$ABXPKG_LIB_DIR/env/bin" >> "$GITHUB_PATH"
-          opencode_env="$(
-            uv run --no-cache --no-sync --no-sources abxpkg env \
-              --install \
-              --json \
-              --lib="$ABXPKG_LIB_DIR" \
-              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \
-              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:github_binaries" \
-              --deps-from="$OPENCODE_CONFIG:required_binaries"
-          )"
-          jq_binary="$ABXPKG_LIB_DIR/env/bin/jq"
-          "$jq_binary" -r 'to_entries[] | "\(.key)=\(.value)"' \
-            <<< "$opencode_env" >> "$GITHUB_ENV"
-
-          opencode_binary="$ABXPKG_LIB_DIR/env/bin/opencode"
-          echo "OPENCODE_BINARY=$opencode_binary" >> "$GITHUB_ENV"
-          test -L "$opencode_binary"
-          test -x "$opencode_binary"
-
-      - name: Check for duplicate issues
-        env:
-          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          OPENCODE_PERMISSION: |
-            {
-              "bash": {
-                "gh issue*": "allow",
-                "gh pr*": "allow",
-                "*": "deny"
-              },
-              "webfetch": "allow"
-            }
-        run: |
-          "$OPENCODE_BINARY" run -m anthropic/claude-haiku-4-5 "A new issue has been created:
-
-          Issue number:
-          ${{ github.event.issue.number }}
-
-          Lookup this issue and search through existing issues and PRs (excluding #${{ github.event.issue.number }}) in this repository to find any potential duplicates of this new issue.
-          Consider:
-          1. Similar titles or descriptions
-          2. Same error messages or symptoms
-          3. Related functionality or components
-          4. Similar feature requests
-
-          If you find any potential duplicates, please comment on the new issue with:
-          - A brief explanation of why it might be a duplicate
-          - Links to the potentially duplicate issues or PRs
-          - A suggestion to check those issues first
-
-          Use this format for the comment:
-          'This issue might be a duplicate of existing issues. Please check:
-          - #[issue_number]: [brief description of similarity]
-
-          Feel free to ignore if none of these address your specific case.'
-
-          If no clear duplicates are found, do not comment."

.pre-commit-config.yaml

@@ -64,4 +64,4 @@ repos:
         args: ["--maxkb=600"]
       - id: name-tests-test
         args: ["--pytest-test-first"]
-        exclude: ^archivebox/tests/(data/|fixtures\.py$|migrations_helpers\.py$)
+        exclude: ^archivebox/tests/(data/|fixtures\.py$|migrations_helpers\.py$|import_helpers\.py$)

AGENTS.md

@@ -15,6 +15,13 @@ ArchiveBox is the full self-hosted web archiving app. Keep this repo on the `dev
 - Trace root causes from observed behavior. Do not paper over failures with retries, wider timeouts, broad fallbacks, or looser assertions.
 - Read `README.md` for the full setup, CLI, Docker, API, and release surface.
 
+## Concurrency Contract
+
+- A collection has one orchestrator at a time. Local PID/process checks may warn about obvious same-machine duplicates, but must not claim to enforce ownership across machines or shared filesystems.
+- SQLite remains supported with concurrent short writes from CLI and server processes. Keep the SQLite database on a local filesystem, never NFS/SMB, and use short autocommit/CAS updates instead of long transactions or database locks.
+- Network calls, hook execution, filesystem migrations, and other long work belong in the orchestrator. Never hold a database transaction or lock across that work.
+- PostgreSQL and shared data directories are the path to future multi-machine scheduling. Coordinate that work through database-backed per-crawl/per-snapshot claims; do not introduce file leases or timer-based orchestrator election.
+
 ## Development Setup
 
 ```bash

Dockerfile

@@ -10,10 +10,10 @@
 #       --build-context abx-plugins=../abx-plugins \
 #       -t archivebox/abx-dl:dev
 #   docker buildx build . -f Dockerfile \
-#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:1.12.225 \
+#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:1.12.274 \
 #       -t archivebox:multistage
 
-ARG ABX_DL_IMAGE=archivebox/abx-dl:1.12.225
+ARG ABX_DL_IMAGE=archivebox/abx-dl:1.12.274
 
 FROM archivebox/sonic:1.4.9 AS sonic
 FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base
@@ -102,7 +102,7 @@ echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests
 rm -f /etc/apt/apt.conf.d/docker-clean
 apt-get update -qq
 apt-get install -qq -y --no-install-recommends \
-    build-essential gcc libldap2-dev libsasl2-dev libssl-dev
+    build-essential gcc libldap2-dev libsasl2-dev libssl-dev x11-utils
 # Extend abx-dl's existing venv instead of clearing it. Clearing and later
 # copying a complete replacement over the base duplicates every venv byte in
 # the final overlay history even when most packages are unchanged.

README.md

@@ -1,5 +1,5 @@
 <div align="center" style="text-align: center; width: 100%">
-<img src="https://archivebox.io/icon.png" height="90px"/>
+<img style="height: 90px" src="https://archivebox.io/icon.png"/>
 <h1>ArchiveBox<br/><sub>Open-source self-hosted web archiving.</sub></h1>
 
 <br/>
@@ -80,8 +80,6 @@ docker compose up -d --wait                                                # ini
 <br/>
 # Option B: Or use it as a plain Docker container:
 mkdir -p ~/archivebox/data && cd ~/archivebox/data
-docker run --rm -it -v "$PWD:/data" archivebox/archivebox:dev init
-docker run --rm -it -v "$PWD:/data" archivebox/archivebox:dev install
 docker run -d --name archivebox -v "$PWD:/data" -p 8000:8000 archivebox/archivebox:dev
 # open http://admin.archivebox.localhost:8000 to finish setup
 # docker run -it -v $PWD:/data archivebox/archivebox:dev add 'https://example.com'
@@ -110,7 +108,7 @@ curl -fsSL 'https://get.archivebox.io' | bash
 
 <div align="center" style="text-align: center">
 <br/><br/>
-<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/5a7d95f2-6977-4de6-9f08-42851a1fe1d2" height="70px" alt="bookshelf graphic"> &nbsp; <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/b2765a33-0d1e-4019-a1db-920c7e00e20e" height="75px" alt="logo" align="top"/> &nbsp; <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/5a7d95f2-6977-4de6-9f08-42851a1fe1d2" height="70px" alt="bookshelf graphic">
+<img style="height: 70px" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/5a7d95f2-6977-4de6-9f08-42851a1fe1d2" alt="bookshelf graphic"> &nbsp; <img style="height: 75px" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/b2765a33-0d1e-4019-a1db-920c7e00e20e" alt="logo" align="top"/> &nbsp; <img style="height: 70px" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/5a7d95f2-6977-4de6-9f08-42851a1fe1d2" alt="bookshelf graphic">
 <br/><br/>
 <small><a href="https://demo.archivebox.io">Demo</a> | <a href="#screenshots">Screenshots</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a></small>
 <br/>
@@ -154,7 +152,7 @@ curl -fsSL 'https://get.archivebox.io' | bash
 #### ✳️&nbsp; Easy Setup
 
 <details>
-<summary><b><img src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" height="28px" align="top"/> <code>docker-compose</code></b>  (macOS/Linux) &nbsp; <b>👈&nbsp; recommended</b> &nbsp; <i>(click to expand)</i></summary>
+<summary><b><img style="height: 28px" src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" align="top"/> <code>docker-compose</code></b>  (macOS/Linux) &nbsp; <b>👈&nbsp; recommended</b> &nbsp; <i>(click to expand)</i></summary>
 <br/>
 <i>👍 Docker Compose is recommended for the easiest install/update UX + best security + all <a href="#dependencies">extras</a> out-of-the-box.</i>
 <br/><br/>
@@ -183,21 +181,20 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 </details>
 
 <details>
-<summary><b><img src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" height="28px" align="top"/> <code>docker run</code></b>  (macOS/Linux)</summary>
+<summary><b><img style="height: 28px" src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" align="top"/> <code>docker run</code></b>  (macOS/Linux)</summary>
 <br/>
 <ol>
 <li>Install <a href="https://docs.docker.com/get-docker/">Docker</a> on your system (if not already installed).</li>
-<li>Create a new empty directory and initialize your collection (can be anywhere).
+<li>Create a new empty directory and start the server, which initializes the collection automatically (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
-docker run --rm -v $PWD:/data -it archivebox/archivebox:dev init
-docker run --rm -v $PWD:/data -it archivebox/archivebox:dev install
+docker run -d --name archivebox -v $PWD:/data -p 8000:8000 archivebox/archivebox:dev
 </code></pre>
 </li>
-<li>Optional: Start the server, then open <code>/admin/</code> on the hostname or IP used to reach ArchiveBox (local example: <a href="http://admin.archivebox.localhost:8000/admin/">http://admin.archivebox.localhost:8000/admin/</a>) to create the first admin. If <code>BASE_URL</code> is not configured yet, continue through the web setup wizard.
-<pre lang="bash"><code style="white-space: pre-line">docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox:dev
+<li>Open <code>/admin/</code> on the hostname or IP used to reach ArchiveBox (local example: <a href="http://admin.archivebox.localhost:8000/admin/">http://admin.archivebox.localhost:8000/admin/</a>) to create the first admin. If <code>BASE_URL</code> is not configured yet, continue through the web setup wizard.
+<pre lang="bash"><code style="white-space: pre-line">
 # completely optional, CLI can always be used without running a server
-# docker run -v $PWD:/data -it archivebox/archivebox:dev [subcommand] [--help]
-docker run -v $PWD:/data -it archivebox/archivebox:dev help
+# docker exec archivebox archivebox [subcommand] [--help]
+docker exec archivebox archivebox help
 </code></pre>
 <i>For more info, see <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-a-docker--docker-compose-setup-%EF%B8%8F">Install: Docker Compose</a> in the Wiki. ➡️</i>
 </li>
@@ -208,7 +205,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 </details>
 
 <details>
-<summary><b><img src="https://user-images.githubusercontent.com/511499/117456282-08665e80-af16-11eb-91a1-8102eff54091.png" alt="curl sh uv install shortcut" height="28px" align="top"/> <code>bash</code> uv install shortcut</b>  (macOS/Linux/BSD)</summary>
+<summary><b><img style="height: 28px" src="https://user-images.githubusercontent.com/511499/117456282-08665e80-af16-11eb-91a1-8102eff54091.png" alt="curl sh uv install shortcut" align="top"/> <code>bash</code> uv install shortcut</b>  (macOS/Linux/BSD)</summary>
 <br/>
 <ol>
 <li>Run the shortcut for the <code>uv</code> install method. It installs <code>uv</code> first when needed.
@@ -231,7 +228,7 @@ See <a href="https://docs.sweeting.me/s/against-curl-sh">"Against curl | sh as a
 
 
 <details>
-<summary><b><img src="https://user-images.githubusercontent.com/511499/117447613-ba4c5d80-af0b-11eb-8f89-1d98e31b6a79.png" alt="uv" height="28px" align="top"/> <code>uv</code></b> (macOS/Linux/BSD)</summary>
+<summary><b><img style="height: 28px" src="https://user-images.githubusercontent.com/511499/117447613-ba4c5d80-af0b-11eb-8f89-1d98e31b6a79.png" alt="uv" align="top"/> <code>uv</code></b> (macOS/Linux/BSD)</summary>
 <br/>
 <ol>
 
@@ -265,7 +262,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 
 
 <details>
-<summary><b><img src="https://user-images.githubusercontent.com/511499/117448075-49597580-af0c-11eb-91ba-f34fff10096b.png" alt="aptitude" height="28px" align="top"/> <code>apt</code></b> (Ubuntu/Debian/etc.)</summary>
+<summary><b><img style="height: 28px" src="https://user-images.githubusercontent.com/511499/117448075-49597580-af0c-11eb-91ba-f34fff10096b.png" alt="aptitude" align="top"/> <code>apt</code></b> (Ubuntu/Debian/etc.)</summary>
 <br/>
 <ol>
 <li>Add the third-party ArchiveBox apt repo and install <code>archivebox</code>.
@@ -298,7 +295,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 </details>
 
 <details>
-<summary><b><img src="https://user-images.githubusercontent.com/511499/117447803-f2ec3700-af0b-11eb-87d3-671d114f011d.png" alt="homebrew" height="28px" align="top"/> <code>brew</code></b> (macOS and Linux)</summary>
+<summary><b><img style="height: 28px" src="https://user-images.githubusercontent.com/511499/117447803-f2ec3700-af0b-11eb-87d3-671d114f011d.png" alt="homebrew" align="top"/> <code>brew</code></b> (macOS and Linux)</summary>
 <br/>
 Run Homebrew as your normal non-root user on both macOS and Linux; do not use <code>sudo brew</code>.
 <ol>
@@ -332,7 +329,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 </details>
 
 <details>
-<summary><img src="https://user-images.githubusercontent.com/511499/118077361-f0616580-b381-11eb-973c-ee894a3349fb.png" alt="Arch" height="28px" align="top"/> <code>pacman</code> / <img src="https://user-images.githubusercontent.com/511499/118077946-29e6a080-b383-11eb-94f0-d4871da08c3f.png" alt="FreeBSD" height="28px" align="top"/> <code>pkg</code> / <img src="https://user-images.githubusercontent.com/511499/118077861-002d7980-b383-11eb-86a7-5936fad9190f.png" alt="Nix" height="28px" align="top"/> <code>nix</code> (Arch/FreeBSD/NixOS/more)</summary>
+<summary><img style="height: 28px" src="https://user-images.githubusercontent.com/511499/118077361-f0616580-b381-11eb-973c-ee894a3349fb.png" alt="Arch" align="top"/> <code>pacman</code> / <img style="height: 28px" src="https://user-images.githubusercontent.com/511499/118077946-29e6a080-b383-11eb-94f0-d4871da08c3f.png" alt="FreeBSD" align="top"/> <code>pkg</code> / <img style="height: 28px" src="https://user-images.githubusercontent.com/511499/118077861-002d7980-b383-11eb-86a7-5936fad9190f.png" alt="Nix" align="top"/> <code>nix</code> (Arch/FreeBSD/NixOS/more)</summary>
 <br/>
 
 > *Warning: These are contributed by external volunteers and may lag behind the official `uv` and Docker channels.*
@@ -353,7 +350,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for usage examples using the CLI, W
 #### 🎗&nbsp; Other Options
 
 <details>
-<summary><b><img src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" height="28px" align="top"/> <code>docker</code> + <img src="https://user-images.githubusercontent.com/511499/117447263-4316c980-af0b-11eb-928d-eaf1292ac646.png" alt="Electron" height="28px" align="top"/> <code>electron</code> Desktop App</b> (macOS/Linux/Windows)</summary>
+<summary><b><img style="height: 28px" src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" align="top"/> <code>docker</code> + <img style="height: 28px" src="https://user-images.githubusercontent.com/511499/117447263-4316c980-af0b-11eb-928d-eaf1292ac646.png" alt="Electron" align="top"/> <code>electron</code> Desktop App</b> (macOS/Linux/Windows)</summary>
 <br/>
 <ol>
 <li>Install <a href="https://docs.docker.com/get-docker/">Docker</a> on your system (if not already installed).</li>
@@ -372,7 +369,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for usage examples using the CLI, W
 </details>
 
 <details>
-<summary><img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/0c46e949-00fe-49c8-a613-ee14501c014c" alt="Self-hosting Platforms" height="28px" align="top"/><b> TrueNAS / UNRAID / YunoHost / Cloudron / etc.</b> (self-hosting solutions)</summary>
+<summary><img style="height: 28px" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/0c46e949-00fe-49c8-a613-ee14501c014c" alt="Self-hosting Platforms" align="top"/><b> TrueNAS / UNRAID / YunoHost / Cloudron / etc.</b> (self-hosting solutions)</summary>
 <br/>
 
 > *Warning: These are contributed by external volunteers and may lag behind the official Docker and `uv` channels.*
@@ -396,26 +393,26 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for usage examples using the CLI, W
 </details>
 
 <details>
-<summary><img src="https://user-images.githubusercontent.com/511499/117448723-1663b180-af0d-11eb-837f-d43959227810.png" alt="paid" height="27px" align="top"/> Third-party paid hosting options (cloud VPS)</summary>
+<summary><img style="height: 27px" src="https://user-images.githubusercontent.com/511499/117448723-1663b180-af0d-11eb-837f-d43959227810.png" alt="paid" align="top"/> Third-party paid hosting options (cloud VPS)</summary>
 <br/>
 <ul>
-<li><a href="https://elest.io/open-source/archivebox"><img src="https://img.shields.io/badge/Managed_Hosting-Elest.io-%23193f7e.svg?style=flat" height="22px"/></a></li>
-<li><a href="https://www.stellarhosted.com/archivebox/"><img src="https://img.shields.io/badge/Semi_Managed_Hosting-StellarHosted.com-%23193f7e.svg?style=flat" height="22px"/></a> (USD $29-250/mo, <a href="https://www.stellarhosted.com/archivebox/#pricing">pricing</a>)</li>
-<li><a href="https://www.pikapods.com/pods?run=archivebox"><img src="https://img.shields.io/badge/Semi_Managed_Hosting-PikaPods.com-%2343a047.svg?style=flat" height="22px"/></a> (from USD $2.6/mo)</li>
+<li><a href="https://elest.io/open-source/archivebox"><img style="height: 22px" src="https://img.shields.io/badge/Managed_Hosting-Elest.io-%23193f7e.svg?style=flat"/></a></li>
+<li><a href="https://www.stellarhosted.com/archivebox/"><img style="height: 22px" src="https://img.shields.io/badge/Semi_Managed_Hosting-StellarHosted.com-%23193f7e.svg?style=flat"/></a> (USD $29-250/mo, <a href="https://www.stellarhosted.com/archivebox/#pricing">pricing</a>)</li>
+<li><a href="https://www.pikapods.com/pods?run=archivebox"><img style="height: 22px" src="https://img.shields.io/badge/Semi_Managed_Hosting-PikaPods.com-%2343a047.svg?style=flat"/></a> (from USD $2.6/mo)</li>
 <li><a href="https://m.do.co/c/cbc4c0c17840">
- <img src="https://img.shields.io/badge/Unmanaged_VPS-DigitalOcean.com-%232f7cf7.svg?style=flat" height="22px"/>
+ <img style="height: 22px" src="https://img.shields.io/badge/Unmanaged_VPS-DigitalOcean.com-%232f7cf7.svg?style=flat"/>
 </a> (USD $5-50+/mo, <a href="https://m.do.co/c/cbc4c0c17840">🎗&nbsp; referral link</a>, <a href="https://www.digitalocean.com/community/tutorials/how-to-install-and-use-docker-compose-on-ubuntu-20-04">instructions</a>)</li>
 <li><a href="https://www.vultr.com/?ref=7130289">
- <img src="https://img.shields.io/badge/Unmanaged_VPS-Vultr.com-%232337a8.svg?style=flat" height="22px"/>
+ <img style="height: 22px" src="https://img.shields.io/badge/Unmanaged_VPS-Vultr.com-%232337a8.svg?style=flat"/>
 </a> (USD $2.5-50+/mo, <a href="https://www.vultr.com/?ref=7130289">🎗&nbsp; referral link</a>, <a href="https://www.vultr.com/docs/install-docker-compose-on-ubuntu-20-04">instructions</a>)</li>
 <li><a href="https://fly.io/">
- <img src="https://img.shields.io/badge/Unmanaged_App-Fly.io-%239a2de6.svg?style=flat" height="22px"/>
+ <img style="height: 22px" src="https://img.shields.io/badge/Unmanaged_App-Fly.io-%239a2de6.svg?style=flat"/>
 </a> (USD $10-50+/mo, <a href="https://fly.io/docs/hands-on/start/">instructions</a>)</li>
 <li><a href="https://railway.app/template/2Vvhmy">
- <img src="https://img.shields.io/badge/Unmanaged_App-Railway-%23A11BE6.svg?style=flat" height="22px"/>
+ <img style="height: 22px" src="https://img.shields.io/badge/Unmanaged_App-Railway-%23A11BE6.svg?style=flat"/>
 </a> (USD $0-5+/mo)</li>
-<li><a href="https://aws.amazon.com/marketplace/pp/Linnovate-Open-Source-Innovation-Support-For-Archi/B08RVW6MJ2"><img src="https://img.shields.io/badge/Unmanaged_VPS-AWS-%23ee8135.svg?style=flat" height="22px"/></a> (USD $60-200+/mo)</li>
-<li><a href="https://azuremarketplace.microsoft.com/en-us/marketplace/apps/meanio.archivebox?ocid=gtmrewards_whatsnewblog_archivebox_vol118"><img src="https://img.shields.io/badge/Unmanaged_VPS-Azure-%237cb300.svg?style=flat" height="22px"/></a> (USD $60-200+/mo)</li>
+<li><a href="https://aws.amazon.com/marketplace/pp/Linnovate-Open-Source-Innovation-Support-For-Archi/B08RVW6MJ2"><img style="height: 22px" src="https://img.shields.io/badge/Unmanaged_VPS-AWS-%23ee8135.svg?style=flat"/></a> (USD $60-200+/mo)</li>
+<li><a href="https://azuremarketplace.microsoft.com/en-us/marketplace/apps/meanio.archivebox?ocid=gtmrewards_whatsnewblog_archivebox_vol118"><img style="height: 22px" src="https://img.shields.io/badge/Unmanaged_VPS-Azure-%237cb300.svg?style=flat"/></a> (USD $60-200+/mo)</li>
 <br/>
 <sub><i>Referral links marked 🎗 provide $5-10 of free credit for new users and help pay for our <a href="https://demo.archivebox.io">demo server</a> hosting costs.</i></sub>
 </ul>
@@ -429,7 +426,7 @@ For more discussion on third-party hosting options see here: <a href="https://gi
 #### ➡️&nbsp; Next Steps
 
 - Import URLs from some of the supported [Input Formats](#input-formats) or view the supported [Output Formats](#output-formats)...
-- (Optional) Create a persona and import browser cookies to archive logged-in sites: `archivebox persona create --import=chrome personal`
+- (Optional) [Import browser cookies and settings into a persona](docs/Chromium-Install.md#import-an-existing-browser-profile).
 - Tweak your UI or archiving behavior [Configuration](#configuration), read about some of the [Caveats](#caveats), or [Troubleshoot](https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting)
 - Read about the [Dependencies](#dependencies) used for archiving, the [Upgrading Process](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives), or the [Archive Layout](#archive-layout) on disk...
 - Or check out our full [Documentation](#documentation) or [Community Wiki](#internet-archiving-ecosystem)...
@@ -457,11 +454,6 @@ docker compose run --rm archivebox help
 # equivalent: docker run -it -v $PWD:/data archivebox/archivebox:dev [subcommand] [--help]
 docker run -it -v $PWD:/data archivebox/archivebox:dev help
 
-# optional: import your browser cookies into a persona for logged-in archiving
-archivebox persona create --import=chrome personal
-# supported: chrome/chromium/brave/edge (Chromium-based only)
-# use --profile to target a specific profile (e.g. Default, Profile 1)
-# re-running import merges/dedupes cookies.txt (by domain/path/name) but replaces chrome_user_data
 ```
 
 #### ArchiveBox Subcommands
@@ -473,7 +465,7 @@ archivebox persona create --import=chrome personal
 
 <br/>
 <details>
-<summary><img src="https://user-images.githubusercontent.com/511499/117456282-08665e80-af16-11eb-91a1-8102eff54091.png" alt="curl sh automatic setup script" height="22px" align="top"/> <b>CLI Usage Examples: non-Docker</b></summary>
+<summary><img style="height: 22px" src="https://user-images.githubusercontent.com/511499/117456282-08665e80-af16-11eb-91a1-8102eff54091.png" alt="curl sh automatic setup script" align="top"/> <b>CLI Usage Examples: non-Docker</b></summary>
 <br/>
 <pre lang="bash"><code style="white-space: pre-line">
 # make sure you have uv-installed ArchiveBox and it's available in your $PATH first
@@ -492,17 +484,15 @@ archivebox help              # get list of archivebox subcommands that can be ru
 <br/>
 
 <details>
-<summary><img src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" height="22px" align="top"/> <b>CLI Usage Examples: Docker Compose</b></summary>
+<summary><img style="height: 22px" src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" align="top"/> <b>CLI Usage Examples: Docker Compose</b></summary>
 <br/>
 <pre lang="bash"><code style="white-space: pre-line">
 # make sure you have `docker-compose.yml` from the Quickstart instructions first
 <br/>
-# docker compose run --rm archivebox [subcommand] [--help]
-docker compose run --rm archivebox init
-docker compose run --rm archivebox install
-docker compose run --rm archivebox version
-docker compose run --rm archivebox help
-docker compose run --rm archivebox add 'https://example.com'
+# docker compose exec archivebox archivebox [subcommand] [--help]
+docker compose exec archivebox archivebox version
+docker compose exec archivebox archivebox help
+docker compose exec archivebox archivebox add 'https://example.com'
 # to start webserver: docker compose up
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage">Usage: Docker Compose CLI</a> wiki. ➡️</i>
@@ -511,18 +501,15 @@ docker compose run --rm archivebox add 'https://example.com'
 <br/>
 
 <details>
-<summary><img src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" height="22px" align="top"/> <b>CLI Usage Examples: Docker</b></summary>
+<summary><img style="height: 22px" src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" align="top"/> <b>CLI Usage Examples: Docker</b></summary>
 <br/>
 <pre lang="bash"><code style="white-space: pre-line">
-# make sure you create and cd into in a new empty directory first  
+# make sure the `archivebox` server container from the Quickstart is running first
 <br/>
-# docker run -it -v $PWD:/data archivebox/archivebox:dev [subcommand] [--help]
-docker run -v $PWD:/data -it archivebox/archivebox:dev init
-docker run -v $PWD:/data -it archivebox/archivebox:dev install
-docker run -v $PWD:/data -it archivebox/archivebox:dev version
-docker run -v $PWD:/data -it archivebox/archivebox:dev help
-docker run -v $PWD:/data -it archivebox/archivebox:dev add 'https://example.com'
-# to start webserver: docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox:dev
+# docker exec archivebox archivebox [subcommand] [--help]
+docker exec archivebox archivebox version
+docker exec archivebox archivebox help
+docker exec archivebox archivebox add 'https://example.com'
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage-1">Usage: Docker CLI</a> wiki. ➡️</i>
 </details>
@@ -626,19 +613,19 @@ docker run -it -v $PWD:/data archivebox/archivebox:dev add --depth=1 'https://ex
 ##  Input Formats: How to pass URLs into ArchiveBox for saving
 
 
-- <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/ff20d251-5347-4b85-ae9b-83037d0ac01e" height="28px"/> <b>From the official <a href="https://github.com/ArchiveBox/archivebox-extension">ArchiveBox Browser Extension</a></b>  
+- <img style="height: 28px" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/ff20d251-5347-4b85-ae9b-83037d0ac01e"/> <b>From the official <a href="https://github.com/ArchiveBox/archivebox-extension">ArchiveBox Browser Extension</a></b><br>
   <i>Provides realtime archiving of browsing history or selected pages from Chrome/Chromium/Firefox browsers.</i>
 
-- <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/64078483-21d7-4eb1-aa6e-9ad55afe45b8" height="22px"/> From manual imports of URLs from RSS, JSON, CSV, TXT, SQL, HTML, Markdown, etc. files  
+- <img style="height: 22px" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/64078483-21d7-4eb1-aa6e-9ad55afe45b8"/> From manual imports of URLs from RSS, JSON, CSV, TXT, SQL, HTML, Markdown, etc. files<br>
   <i>ArchiveBox supports injesting URLs in [any text-based format](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Import-a-list-of-URLs-from-a-text-file).</i>
 
-- <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/32b494e6-4de1-4984-8d88-dc02f18e5c34" height="22px"/> From manually exported [browser history](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) or [browser bookmarks](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) (in Netscape format)  
+- <img style="height: 22px" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/32b494e6-4de1-4984-8d88-dc02f18e5c34"/> From manually exported [browser history](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) or [browser bookmarks](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) (in Netscape format)<br>
   <i>Instructions: <a href="https://support.google.com/chrome/answer/96816?hl=en">Chrome</a>, <a href="https://support.mozilla.org/en-US/kb/export-firefox-bookmarks-to-backup-or-transfer">Firefox</a>, <a href="https://github.com/ArchiveBox/ArchiveBox/assets/511499/24ad068e-0fa6-41f4-a7ff-4c26fc91f71a">Safari</a>, <a href="https://support.microsoft.com/en-us/help/211089/how-to-import-and-export-the-internet-explorer-favorites-folder-to-a-32-bit-version-of-windows">IE</a>, <a href="https://help.opera.com/en/latest/features/#bookmarks:~:text=Click%20the%20import/-,export%20button,-on%20the%20bottom">Opera</a>, <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive">and more...</a></i>
 
-- <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/4f7bd318-265c-4235-ad25-38be89946b12" height="22px"/> From URLs visited through a [MITM Proxy](https://mitmproxy.org/) with [`archivebox-proxy`](https://github.com/ArchiveBox/archivebox-proxy)  
+- <img style="height: 22px" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/4f7bd318-265c-4235-ad25-38be89946b12"/> From URLs visited through a [MITM Proxy](https://mitmproxy.org/) with [`archivebox-proxy`](https://github.com/ArchiveBox/archivebox-proxy)<br>
   <i>Provides [realtime archiving](https://github.com/ArchiveBox/ArchiveBox/issues/577) of all traffic from any device going through the proxy.</i>
 
-- <img src="https://getpocket.com/favicon.ico" height="22px"/> From bookmarking services or social media (e.g. Twitter bookmarks, Reddit saved posts, etc.)  
+- <img style="height: 22px" src="https://getpocket.com/favicon.ico"/> From bookmarking services or social media (e.g. Twitter bookmarks, Reddit saved posts, etc.)<br>
   <i>Instructions: <a href="https://getpocket.com/export">Pocket</a>, <a href="https://pinboard.in/export/">Pinboard</a>, <a href="https://www.instapaper.com/user">Instapaper</a>, <a href="https://shaarli.readthedocs.io/en/master/Usage/#importexport">Shaarli</a>, <a href="https://www.groovypost.com/howto/howto/export-delicious-bookmarks-xml/">Delicious</a>, <a href="https://github.com/csu/export-saved-reddit">Reddit Saved</a>, <a href="https://doc.wallabag.org/en/user/import/wallabagv2.html">Wallabag</a>, <a href="http://help.unmark.it/import-export">Unmark.it</a>, <a href="https://www.addictivetips.com/web/onetab-save-close-all-chrome-tabs-to-restore-export-or-import/">OneTab</a>, <a href="https://github.com/ArchiveBox/ArchiveBox/issues/648">Firefox Sync</a>, <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive">and more...</a></i>
 
 
@@ -940,7 +927,7 @@ archivebox manage createsuperuser
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive">Wiki: Publishing Your Archive</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview">Wiki: Security Overview</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile">Wiki: Chromium Install (Setting Up a User Profile)</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Personas">Wiki: Personas (browser profiles and cookies)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#import-an-existing-browser-profile">Wiki: Chrome profiles and cookies</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#default_persona">Wiki: Configuration (<code>DEFAULT_PERSONA</code>)</a></li>
 </ul>
 
@@ -994,7 +981,7 @@ For various reasons, many large sites (Reddit, Twitter, Cloudflare, etc.) active
 
 <ul>
 <li>Set <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#curl_user_agent"><code>CHROME_USER_AGENT</code>, <code>WGET_USER_AGENT</code>, <code>CURL_USER_AGENT</code></a> to impersonate a real browser (by default, ArchiveBox reveals that it's a bot when using the default user agent settings)</li>
-<li>Set up a logged-in browser session for archiving by <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Personas">importing a dedicated browser profile into a persona</a></li>
+<li>Set up a logged-in browser session for archiving by <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#import-an-existing-browser-profile">importing a dedicated browser profile into a persona</a></li>
 <li>Rewrite your URLs before archiving to swap in alternative frontends that are more bot-friendly e.g.<br>
 <code>reddit.com/some/url</code> -&gt; <code>teddit.net/some/url</code>: <a href="https://github.com/mendel5/alternative-front-ends">https://github.com/mendel5/alternative-front-ends</a></li>
 </ul>
@@ -1022,7 +1009,7 @@ Each re-archive creates a distinct Snapshot row for the same URL:
 archivebox add --no-only-new 'https://example.com'
 </code></pre>
 
-The <img src="https://user-images.githubusercontent.com/511499/115942091-73c02300-a476-11eb-958e-5c1fc04da488.png" alt="Re-Snapshot Button" height="24px"/> button in the Admin UI performs the same explicit re-archive.
+The <img style="height: 24px" src="https://user-images.githubusercontent.com/511499/115942091-73c02300-a476-11eb-958e-5c1fc04da488.png" alt="Re-Snapshot Button"/> button in the Admin UI performs the same explicit re-archive.
 
 <h4>Learn More</h4>
 
@@ -1049,8 +1036,8 @@ Because ArchiveBox is designed to ingest a large volume of URLs with multiple co
 </li>
 <li><strong>Don't store large collections on older filesystems like EXT3/FAT</strong> as they may not be able to handle more than 50k directory entries in the <code>data/archive/</code> folder.
 </li>
-<li><strong>Try to keep the <code>data/index.sqlite3</code> file on local drive (not a network mount)</strong> or SSD for maximum performance, however the <code>data/archive/</code> folder can be on a network mount or slower HDD.</li>
-<li>If using Docker or NFS/SMB/FUSE for the <code>data/archive/</code> folder, make sure the mounted data directory is writable by its intended owner and consider disabling <a href="https://github.com/ArchiveBox/ArchiveBox/issues/1304"><code>root_squash</code></a> on your fileshare server.
+<li><strong>Keep the <code>data/index.sqlite3</code> file on a reliable local drive (not a network mount)</strong> or SSD when using SQLite; the <code>data/archive/</code> folder can be on a network mount or slower HDD.</li>
+<li>If using Docker or NFS/SMB/FUSE for the <code>data/archive/</code> folder, configure the server-side UID/GID mapping or ACL so ArchiveBox's non-root user can create and remove files. <code>root_squash</code> is compatible when the selected non-root identity has write access; do not rely on container root to override server-side permissions.
 </li>
 </ul>
 
@@ -1341,18 +1328,15 @@ archivebox server 0.0.0.0:8000
 
 ```bash
 # Optional: develop via docker by mounting the code dir into the container
-# if you edit e.g. ./archivebox/core/models.py on the docker host, runserver
+# if you edit e.g. ./archivebox/core/models/snapshots.py on the docker host, runserver
 # inside the container will reload and pick up your changes
 ./bin/build_docker.sh dev
 
-docker run -it -v $PWD/data:/data archivebox/archivebox:dev init
-docker run -it -v $PWD/data:/data archivebox/archivebox:dev install
-
 # Run the development server w/ autoreloading (but no bg workers)
-docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server --debug --reload 0.0.0.0:8000
+docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server --init --debug --reload 0.0.0.0:8000
 
 # Run the production server (with bg workers but no autoreloading)
-docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server
+docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server --init
 
 # (remove the --reload flag and add the --nothreading flag when profiling with the django debug toolbar)
 # When using --reload, make sure any files you create can be read by the user in the Docker container, eg with 'chmod a+rX'.
@@ -1413,7 +1397,7 @@ services:
 
 # or with plain Docker:
 docker build -t archivebox:dev https://github.com/ArchiveBox/ArchiveBox.git#dev
-docker run -it -v $PWD:/data archivebox:dev init
+docker run -it -v $PWD:/data -p 8000:8000 archivebox:dev
 
 # or with uv:
 uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'

archivebox/api/admin.py

@@ -8,7 +8,7 @@ from signal_webhooks.admin import WebhookAdmin, WebhookModelForm
 from signal_webhooks.settings import webhook_settings
 from signal_webhooks.utils import get_webhook_model, model_from_reference
 
-from archivebox.base_models.admin import BaseModelAdmin
+from archivebox.base_models.admin import card_fieldset, BaseModelAdmin
 
 from archivebox.api.models import APIToken
 
@@ -25,27 +25,9 @@ class APITokenAdmin(BaseModelAdmin):
     search_fields = ("id", "created_by__username", "token")
 
     fieldsets = (
-        (
-            "Token",
-            {
-                "fields": ("token", "expires"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Owner",
-            {
-                "fields": ("created_by",),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Timestamps",
-            {
-                "fields": ("created_at", "modified_at"),
-                "classes": ("card",),
-            },
-        ),
+        card_fieldset("Token", ("token", "expires")),
+        card_fieldset("Owner", ("created_by",)),
+        card_fieldset("Timestamps", ("created_at", "modified_at")),
     )
 
     list_filter = ("created_by",)
@@ -73,41 +55,11 @@ class CustomWebhookAdmin(WebhookAdmin, BaseModelAdmin):
     readonly_fields = _webhook_fields("created_at", "modified_at", *WebhookAdmin.readonly_fields)
 
     fieldsets = (
-        (
-            "Webhook",
-            {
-                "fields": _webhook_fields("name", "signal", "ref", "endpoint", "headers", "keep_last_response"),
-                "classes": ("card", "wide"),
-            },
-        ),
-        (
-            "Authentication",
-            {
-                "fields": _webhook_fields("auth_token"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Status",
-            {
-                "fields": _webhook_fields("enabled", "last_success", "last_failure", "last_response"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Owner",
-            {
-                "fields": _webhook_fields("created_by"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Timestamps",
-            {
-                "fields": _webhook_fields("created_at", "modified_at"),
-                "classes": ("card",),
-            },
-        ),
+        card_fieldset("Webhook", _webhook_fields("name", "signal", "ref", "endpoint", "headers", "keep_last_response"), wide=True),
+        card_fieldset("Authentication", _webhook_fields("auth_token")),
+        card_fieldset("Status", _webhook_fields("enabled", "last_success", "last_failure", "last_response")),
+        card_fieldset("Owner", _webhook_fields("created_by")),
+        card_fieldset("Timestamps", _webhook_fields("created_at", "modified_at")),
     )
 
     def lookup_allowed(self, lookup: str, value: str, request: HttpRequest | None = None) -> bool:

archivebox/api/schemas.py

@@ -0,0 +1,17 @@
+"""Fields shared by responses for user-owned model objects."""
+
+from ninja import Schema
+
+
+class OwnedObjectSchema(Schema):
+    created_by_id: str
+    created_by_username: str
+
+    @staticmethod
+    def resolve_created_by_id(obj):
+        return str(obj.created_by.pk)
+
+    @staticmethod
+    def resolve_created_by_username(obj) -> str:
+        user = obj.created_by
+        return user.username if isinstance(user.username, str) else str(user)

archivebox/api/v1_cli.py

@@ -117,38 +117,48 @@ def snapshot_filter_kwargs(args: SnapshotFilterCommandSchema, *, default_filter_
     return kwargs
 
 
+def cli_response(request, result, *, result_format="json", success=True, errors=()):
+    """Use one response envelope for CLI-backed endpoints, including captured output."""
+    payload = {"success": success, "errors": list(errors), "result": result, "result_format": result_format}
+    for stream_name in ("stdout", "stderr"):
+        stream = request.__dict__.get(stream_name)
+        payload[stream_name] = ansi_to_html(stream.getvalue().strip()) if isinstance(stream, StringIO) else ""
+    return payload
+
+
+def command_config(args) -> dict[str, object] | None:
+    """Translate shared legacy add/schedule flags into runtime config overrides."""
+    if args.update or args.overwrite:
+        return {"ONLY_NEW": False}
+    return {"ONLY_NEW": bool(args.only_new)} if args.only_new is not None else None
+
+
 @router.post("/add", response=CLICommandResponseSchema, summary="archivebox add [args] [urls]")
 def cli_add(request: HttpRequest, args: AddCommandSchema):
     from archivebox.cli.archivebox_add import add
     from archivebox.misc.util import validate_url
 
-    config_overrides: dict[str, object] = {}
-    if args.only_new is not None:
-        config_overrides["ONLY_NEW"] = bool(args.only_new)
-    if args.update or args.overwrite:
-        config_overrides["ONLY_NEW"] = False
     submitted_urls: str | list[str] = args.urls
     if len(args.urls) == 1:
         try:
             validate_url(args.urls[0])
         except ValueError:
             submitted_urls = args.urls[0]
+    else:
+        submitted_urls = []
+        for url in args.urls:
+            try:
+                submitted_urls.append(validate_url(url))
+            except ValueError:
+                continue
+        if not submitted_urls:
+            raise HttpError(400, "No valid URLs were submitted")
     crawl, snapshots = add(
         urls=submitted_urls,
-        snapshot_ids=args.snapshot_ids,
-        tag=args.tag,
-        depth=args.depth,
-        max_urls=args.max_urls,
-        crawl_max_size=args.crawl_max_size,
-        crawl_timeout=args.crawl_timeout,
-        snapshot_max_size=args.snapshot_max_size,
-        index_only=args.index_only,
-        plugins=args.plugins,
-        persona=args.persona,
-        parser=args.parser,
+        **args.model_dump(exclude={"urls", "only_new", "update", "overwrite"}),
         bg=True,  # Always run in background for API calls
         created_by_id=request.user.pk,
-        config=config_overrides or None,
+        config=command_config(args),
     )
 
     snapshot_ids = [str(snapshot_id) for snapshot_id in snapshots.values_list("id", flat=True)]
@@ -156,19 +166,9 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
         "crawl_id": str(crawl.id),
         "num_snapshots": len(snapshot_ids),
         "snapshot_ids": snapshot_ids,
-        "queued_urls": args.urls,
-    }
-    stdout = request.__dict__.get("stdout")
-    stderr = request.__dict__.get("stderr")
-
-    return {
-        "success": True,
-        "errors": [],
-        "result": result_payload,
-        "result_format": "json",
-        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
-        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
+        "queued_urls": args.urls if isinstance(submitted_urls, str) else submitted_urls,
     }
+    return cli_response(request, result_payload, result_format="json")
 
 
 @router.post("/update", response=CLICommandResponseSchema, summary="archivebox update [args] [filter_patterns]")
@@ -195,55 +195,20 @@ def cli_update(request: HttpRequest, args: UpdateCommandSchema):
         ]
 
     update(**update_kwargs)
-    stdout = request.__dict__.get("stdout")
-    stderr = request.__dict__.get("stderr")
-    return {
-        "success": True,
-        "errors": [],
-        "result": {
-            "matched_count": len(matched_snapshot_ids),
-            "snapshot_ids": matched_snapshot_ids,
-        }
-        if is_filtered_update
-        else None,
-        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
-        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
-    }
+    return cli_response(
+        request,
+        {"matched_count": len(matched_snapshot_ids), "snapshot_ids": matched_snapshot_ids} if is_filtered_update else None,
+        result_format="str",
+    )
 
 
 @router.post("/schedule", response=CLICommandResponseSchema, summary="archivebox schedule [args] [import_path]")
 def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
     from archivebox.cli.archivebox_schedule import schedule
 
-    config_overrides: dict[str, object] = {}
-    if args.only_new is not None:
-        config_overrides["ONLY_NEW"] = bool(args.only_new)
-    if args.update or args.overwrite:
-        config_overrides["ONLY_NEW"] = False
-    result = schedule(
-        import_path=args.import_path,
-        add=args.add,
-        show=args.show,
-        foreground=args.foreground,
-        run_all=args.run_all,
-        quiet=args.quiet,
-        clear=args.clear,
-        every=args.every,
-        tag=args.tag,
-        depth=args.depth,
-        config=config_overrides or None,
-    )
+    result = schedule(**args.model_dump(exclude={"only_new", "update", "overwrite"}), config=command_config(args))
 
-    stdout = request.__dict__.get("stdout")
-    stderr = request.__dict__.get("stderr")
-    return {
-        "success": True,
-        "errors": [],
-        "result": result,
-        "result_format": "json",
-        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
-        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
-    }
+    return cli_response(request, result, result_format="json")
 
 
 @router.post("/search", response=CLICommandResponseSchema, summary="archivebox search [args] [filter_patterns]")
@@ -278,16 +243,7 @@ def cli_search(request: HttpRequest, args: ListCommandSchema):
     else:
         result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500))
 
-    stdout = request.__dict__.get("stdout")
-    stderr = request.__dict__.get("stderr")
-    return {
-        "success": True,
-        "errors": [],
-        "result": result,
-        "result_format": result_format,
-        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
-        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
-    }
+    return cli_response(request, result, result_format=result_format)
 
 
 @router.post("/remove", response=CLICommandResponseSchema, summary="archivebox remove [args] [filter_patterns]")
@@ -308,13 +264,10 @@ def cli_remove(request: HttpRequest, args: RemoveCommandSchema):
         snapshots=snapshots_to_remove,
         timeout=timeout,
     )
-    stdout = request.__dict__.get("stdout")
-    stderr = request.__dict__.get("stderr")
-    return {
-        "success": bool(result["success"]),
-        "errors": [str(result["error"])] if result["error"] else [],
-        "result": result,
-        "result_format": "json",
-        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
-        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
-    }
+    return cli_response(
+        request,
+        result,
+        result_format="json",
+        success=bool(result["success"]),
+        errors=[str(result["error"])] if result["error"] else [],
+    )

archivebox/api/v1_core/__init__.py

@@ -0,0 +1,47 @@
+"""Core REST routes, composed from entity-specific routers."""
+
+from typing import Union
+
+from django.db.models import Model
+from django.http import HttpRequest
+from django.shortcuts import redirect
+from ninja import Router
+from ninja.errors import HttpError
+
+from archivebox.api.v1_crawls import CrawlSchema, get_crawl
+
+from . import archiveresults, snapshots, tags
+from .archiveresults import get_archiveresult
+from .pagination import CustomPagination as CustomPagination
+from .schemas import ArchiveResultSchema, SnapshotSchema, TagSchema
+from .snapshots import get_snapshot
+from .tags import get_tag
+
+router = Router(tags=["Core Models"])
+router.add_router("", archiveresults.router)
+router.add_router("", snapshots.router)
+router.add_router("", tags.router)
+
+
+@router.get(
+    "/any/{id}",
+    response=Union[SnapshotSchema, ArchiveResultSchema, TagSchema, CrawlSchema],
+    url_name="get_any",
+    summary="Get any object by its ID",
+)
+def get_any(request: HttpRequest, id: str):
+    """Get any object by its ID (e.g. snapshot, archiveresult, tag, crawl, etc.)."""
+    setattr(request, "with_snapshots", False)
+    setattr(request, "with_archiveresults", False)
+
+    for getter in (get_snapshot, get_archiveresult, get_tag, get_crawl):
+        try:
+            response = getter(request, id)
+            if isinstance(response, Model):
+                return redirect(
+                    f"/api/v1/{response._meta.app_label}/{response._meta.model_name}/{response.pk}?{request.META['QUERY_STRING']}",
+                )
+        except Exception:
+            pass
+
+    raise HttpError(404, "Object with given ID not found")

archivebox/api/v1_core/archiveresults.py

@@ -0,0 +1,310 @@
+import json
+import mimetypes
+import re
+from pathlib import Path, PurePosixPath
+from typing import Any
+
+from django.core.files.storage import FileSystemStorage
+from django.http import HttpRequest
+from django.http.multipartparser import MultiPartParser, MultiPartParserError
+from ninja import Form, Query, Router, UploadedFile
+from ninja.errors import HttpError
+from ninja.pagination import paginate
+
+from archivebox.core.models import ArchiveResult, Snapshot
+from archivebox.misc.db import uuid_ref_query as _uuid_ref_query
+
+from .lookup import _get_snapshot_by_ref
+from .pagination import CustomPagination
+from .schemas import ArchiveResultFilterSchema, ArchiveResultSchema
+
+router = Router(tags=["Core Models"])
+
+ARCHIVERESULT_UPLOAD_HOOK_NAME = Snapshot.BROWSER_EXTENSION_UPLOAD_HOOK_NAME
+ARCHIVERESULT_UPLOAD_PLUGIN_RE = re.compile(r"^[A-Za-z0-9_.-]{1,32}$")
+
+
+@router.get("/archiveresults", response=list[ArchiveResultSchema], url_name="get_archiveresult")
+@paginate(CustomPagination)
+def get_archiveresults(request: HttpRequest, filters: Query[ArchiveResultFilterSchema]):
+    """List all ArchiveResult entries matching these filters."""
+    queryset = filters.filter(ArchiveResult.objects.all())
+    if filters.search or filters.snapshot_tag:
+        return queryset.distinct()
+    return queryset
+
+
+@router.get("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema, url_name="get_archiveresult")
+def get_archiveresult(request: HttpRequest, archiveresult_id: str):
+    """Get a specific ArchiveResult by id."""
+    return ArchiveResult.objects.get(_uuid_ref_query("id", archiveresult_id))
+
+
+def _normalize_uploaded_archiveresult_plugin(plugin: str) -> str:
+    normalized = str(plugin or "").strip().strip("/")
+    if not ARCHIVERESULT_UPLOAD_PLUGIN_RE.fullmatch(normalized):
+        raise HttpError(400, "Invalid ArchiveResult plugin name")
+    return normalized
+
+
+def _normalize_uploaded_archiveresult_output_path(output_path: str, *, filename: str) -> str:
+    raw_path = str(output_path or filename or "").strip().replace("\\", "/")
+    if not raw_path:
+        raise HttpError(400, "ArchiveResult output path is required")
+
+    path = PurePosixPath(raw_path)
+    if path.is_absolute() or any(part in ("", ".", "..") for part in path.parts):
+        raise HttpError(400, "Invalid ArchiveResult output path")
+
+    return str(path)
+
+
+def _parse_archiveresult_output_json(output_json: str | None) -> dict[str, Any] | None:
+    if not output_json:
+        return None
+    try:
+        parsed = json.loads(output_json)
+    except json.JSONDecodeError as err:
+        raise HttpError(400, "ArchiveResult output_json must be valid JSON") from err
+    if parsed is None:
+        return None
+    if not isinstance(parsed, dict):
+        raise HttpError(400, "ArchiveResult output_json must be a JSON object")
+    return parsed
+
+
+def _get_archiveresult_upload_data(request: HttpRequest):
+    cached = request.__dict__.get("_archiveresult_upload_data")
+    if cached is not None:
+        return cached
+
+    if request.method.upper() == "PATCH" and request.content_type.startswith("multipart/"):
+        try:
+            data = MultiPartParser(request.META, request, request.upload_handlers, request.encoding).parse()
+        except MultiPartParserError as err:
+            raise HttpError(400, f"Invalid ArchiveResult multipart upload: {err}") from err
+    else:
+        data = (request.POST, request.FILES)
+
+    setattr(request, "_archiveresult_upload_data", data)
+    return data
+
+
+def _get_archiveresult_upload_files(request: HttpRequest, *, allow_empty: bool = False) -> list[UploadedFile]:
+    _post, request_files = _get_archiveresult_upload_data(request)
+    files = [*request_files.getlist("files"), *request_files.getlist("file")]
+    if not files and not allow_empty:
+        raise HttpError(400, "At least one ArchiveResult file is required")
+    return files
+
+
+def _get_archiveresult_upload_form_values(request: HttpRequest, *field_names: str) -> list[str]:
+    request_post, _files = _get_archiveresult_upload_data(request)
+    values: list[str] = []
+    for field_name in field_names:
+        values.extend(str(value) for value in request_post.getlist(field_name))
+    if len(values) == 1:
+        value = values[0].strip()
+        if value.startswith("["):
+            try:
+                parsed = json.loads(value)
+            except json.JSONDecodeError:
+                parsed = None
+            if isinstance(parsed, list):
+                return [str(item) for item in parsed]
+    return values
+
+
+def _get_archiveresult_upload_form_value(request: HttpRequest, *field_names: str) -> str:
+    request_post, _files = _get_archiveresult_upload_data(request)
+    for field_name in field_names:
+        value = request_post.get(field_name)
+        if value is not None:
+            return str(value)
+    return ""
+
+
+def _parse_archiveresult_upload_int(value: str, field_name: str, *, default: int | None = None) -> int:
+    if value == "" and default is not None:
+        return default
+    try:
+        parsed = int(value)
+    except (TypeError, ValueError) as err:
+        raise HttpError(400, f"ArchiveResult {field_name} must be an integer") from err
+    if parsed < 0:
+        raise HttpError(400, f"ArchiveResult {field_name} must be non-negative")
+    return parsed
+
+
+def _write_archiveresult_files(
+    request: HttpRequest,
+    snapshot: Snapshot,
+    plugin_name: str,
+    *,
+    existing_output_files: dict[str, dict[str, Any]] | None = None,
+    allow_empty: bool = False,
+) -> dict[str, dict[str, Any]]:
+    files = _get_archiveresult_upload_files(request, allow_empty=allow_empty)
+    output_paths = _get_archiveresult_upload_form_values(request, "output_paths", "output_path")
+    mime_types = _get_archiveresult_upload_form_values(request, "mime_types", "mime_type")
+    chunk_output_path = _get_archiveresult_upload_form_value(request, "chunk_output_path")
+
+    snapshot_dir = snapshot.output_dir
+    plugin_dir = snapshot_dir / plugin_name
+    storage = FileSystemStorage(location=str(plugin_dir))
+    output_files = dict(existing_output_files or {})
+
+    if not files:
+        return output_files
+
+    if chunk_output_path:
+        if len(files) != 1:
+            raise HttpError(400, "Exactly one ArchiveResult file chunk is required")
+
+        uploaded_file = files[0]
+        relative_output_path = _normalize_uploaded_archiveresult_output_path(
+            chunk_output_path,
+            filename=uploaded_file.name,
+        )
+        chunk_index = _parse_archiveresult_upload_int(
+            _get_archiveresult_upload_form_value(request, "chunk_index"),
+            "chunk_index",
+        )
+        chunk_count = _parse_archiveresult_upload_int(
+            _get_archiveresult_upload_form_value(request, "chunk_count"),
+            "chunk_count",
+        )
+        chunk_offset = _parse_archiveresult_upload_int(
+            _get_archiveresult_upload_form_value(request, "chunk_offset"),
+            "chunk_offset",
+        )
+        chunk_total_size = _parse_archiveresult_upload_int(
+            _get_archiveresult_upload_form_value(request, "chunk_total_size"),
+            "chunk_total_size",
+        )
+
+        if chunk_count < 1:
+            raise HttpError(400, "ArchiveResult chunk_count must be at least 1")
+        if chunk_index >= chunk_count:
+            raise HttpError(400, "ArchiveResult chunk_index must be less than chunk_count")
+        if chunk_total_size and chunk_offset > chunk_total_size:
+            raise HttpError(400, "ArchiveResult chunk_offset cannot exceed chunk_total_size")
+
+        if chunk_index == 0 and chunk_offset == 0 and storage.exists(relative_output_path):
+            storage.delete(relative_output_path)
+
+        current_size = storage.size(relative_output_path) if storage.exists(relative_output_path) else 0
+        if current_size != chunk_offset:
+            raise HttpError(
+                409,
+                f"ArchiveResult chunk offset mismatch for {relative_output_path}: expected {current_size}, got {chunk_offset}",
+            )
+
+        Path(storage.path(relative_output_path)).parent.mkdir(parents=True, exist_ok=True)
+        with storage.open(relative_output_path, "ab") as destination:
+            for chunk in uploaded_file.chunks():
+                destination.write(chunk)
+
+        size = storage.size(relative_output_path)
+        upload_complete = chunk_index + 1 == chunk_count
+        if upload_complete and size != chunk_total_size:
+            raise HttpError(
+                409,
+                f"ArchiveResult chunk size mismatch for {relative_output_path}: expected {chunk_total_size}, got {size}",
+            )
+
+        guessed_mime = mimetypes.guess_type(relative_output_path)[0]
+        output_mime_type = (mime_types[0] if mime_types else "") or uploaded_file.content_type or guessed_mime or "application/octet-stream"
+        output_files[relative_output_path] = {
+            "extension": PurePosixPath(relative_output_path).suffix.lower().lstrip("."),
+            "mimetype": output_mime_type,
+            "size": size,
+            "upload": {
+                "chunked": True,
+                "chunk_index": chunk_index,
+                "chunk_count": chunk_count,
+                "chunks_received": chunk_index + 1,
+                "complete": upload_complete,
+            },
+        }
+
+        return output_files
+
+    for index, uploaded_file in enumerate(files):
+        relative_output_path = _normalize_uploaded_archiveresult_output_path(
+            output_paths[index] if index < len(output_paths) else "",
+            filename=uploaded_file.name,
+        )
+        if storage.exists(relative_output_path):
+            storage.delete(relative_output_path)
+        saved_output_path = storage.save(relative_output_path, uploaded_file)
+        size = storage.size(saved_output_path)
+        guessed_mime = mimetypes.guess_type(saved_output_path)[0]
+        output_mime_type = (
+            (mime_types[index] if index < len(mime_types) else "")
+            or uploaded_file.content_type
+            or guessed_mime
+            or "application/octet-stream"
+        )
+        output_files[saved_output_path] = {
+            "extension": PurePosixPath(saved_output_path).suffix.lower().lstrip("."),
+            "mimetype": output_mime_type,
+            "size": size,
+        }
+
+    return output_files
+
+
+@router.post(
+    "/archiveresults",
+    response=ArchiveResultSchema,
+    url_name="create_archiveresult",
+)
+def create_archiveresult(
+    request: HttpRequest,
+    snapshot_id: str = Form(...),
+    plugin: str = Form(...),
+    output_str: str = Form(""),
+    hook_name: str = Form(ARCHIVERESULT_UPLOAD_HOOK_NAME),
+    status: str = Form(str(ArchiveResult.StatusChoices.SUCCEEDED)),
+    output_json: str = Form(""),
+):
+    """Create or update an ArchiveResult with one or more output files."""
+    snapshot = _get_snapshot_by_ref(snapshot_id)
+    plugin_name = _normalize_uploaded_archiveresult_plugin(plugin)
+    result = ArchiveResult.objects.filter(
+        snapshot=snapshot,
+        plugin=plugin_name,
+        hook_name=hook_name or ARCHIVERESULT_UPLOAD_HOOK_NAME,
+    ).first()
+    result = result or ArchiveResult(snapshot=snapshot, plugin=plugin_name, hook_name=hook_name or ARCHIVERESULT_UPLOAD_HOOK_NAME)
+    metadata = {"output_str": output_str, "status": status, "output_json": _parse_archiveresult_output_json(output_json)}
+    files = _write_archiveresult_files(request, snapshot, plugin_name, allow_empty=True)
+    try:
+        return result.apply_upload(files, metadata=metadata, replace_metadata=True)
+    except ArchiveResult.UploadConflict as err:
+        raise HttpError(409, str(err)) from err
+
+
+@router.patch("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema, url_name="patch_archiveresult")
+def patch_archiveresult(
+    request: HttpRequest,
+    archiveresult_id: str,
+):
+    """Append or replace files on an existing ArchiveResult."""
+    result = ArchiveResult.objects.select_related("snapshot__crawl__created_by").get(_uuid_ref_query("id", archiveresult_id))
+    uploaded_output_files = _write_archiveresult_files(
+        request,
+        result.snapshot,
+        result.plugin,
+    )
+    output_str = _get_archiveresult_upload_form_value(request, "output_str")
+    status = _get_archiveresult_upload_form_value(request, "status")
+    output_json = _get_archiveresult_upload_form_value(request, "output_json")
+    metadata = {key: value for key, value in {"output_str": output_str, "status": status}.items() if value}
+    if output_json:
+        metadata["output_json"] = _parse_archiveresult_output_json(output_json)
+    try:
+        return result.apply_upload(uploaded_output_files, metadata=metadata)
+    except ArchiveResult.UploadConflict as err:
+        raise HttpError(409, str(err)) from err

archivebox/api/v1_core/lookup.py

@@ -0,0 +1,12 @@
+from django.db.models import Q
+
+from archivebox.core.models import Snapshot
+from archivebox.misc.db import uuid_ref_query as _uuid_ref_query
+
+
+def _get_snapshot_by_ref(snapshot_id: str):
+    queryset = Snapshot.objects.select_related("crawl__created_by")
+    try:
+        return queryset.get(_uuid_ref_query("id", snapshot_id) | Q(timestamp__startswith=snapshot_id))
+    except Snapshot.DoesNotExist:
+        return queryset.get(_uuid_ref_query("id", snapshot_id))

archivebox/api/v1_core/pagination.py

@@ -0,0 +1,40 @@
+import math
+from typing import Any
+
+from django.http import HttpRequest
+from ninja.pagination import PaginationBase
+
+
+class CustomPagination(PaginationBase):
+    class Input(PaginationBase.Input):
+        limit: int = 200
+        offset: int = 0
+        page: int = 0
+
+    class Output(PaginationBase.Output):
+        count: int
+        total_items: int
+        total_pages: int
+        page: int
+        limit: int
+        offset: int
+        num_items: int
+        items: list[Any]
+
+    def paginate_queryset(self, queryset, pagination: Input, request: HttpRequest, **params):
+        limit = min(pagination.limit, 500)
+        offset = pagination.offset or (pagination.page * limit)
+        total = queryset.values("pk").distinct().count() if queryset.query.distinct else queryset.count()
+        total_pages = math.ceil(total / limit)
+        current_page = math.ceil(offset / (limit + 1))
+        items = queryset[offset : offset + limit]
+        return {
+            "count": total,
+            "total_items": total,
+            "total_pages": total_pages,
+            "page": current_page,
+            "limit": limit,
+            "offset": offset,
+            "num_items": len(items),
+            "items": items,
+        }

archivebox/api/v1_core/schemas.py

@@ -0,0 +1,301 @@
+from collections import defaultdict
+from datetime import datetime
+from typing import Annotated, Any
+from uuid import UUID
+
+from django.db.models import Q
+from archivebox.api.schemas import OwnedObjectSchema
+
+from ninja import FilterLookup, FilterSchema, Schema
+
+from archivebox.core.models import ArchiveResult, Snapshot
+
+
+class MinimalArchiveResultSchema(OwnedObjectSchema):
+    TYPE: str = "core.models.ArchiveResult"
+    id: UUID
+    created_at: datetime | None
+    modified_at: datetime | None
+    status: str
+    retry_at: datetime | None = None
+    plugin: str
+    hook_name: str
+    process_id: UUID | None
+    cmd_version: str | None
+    cmd: list[str] | None
+    pwd: str | None
+    output_str: str
+    output_json: dict[str, Any] | None
+    output_files: dict[str, dict[str, Any]] | None
+    output_size: int
+    output_mimetypes: str
+    start_ts: datetime | None
+    end_ts: datetime | None
+
+    @staticmethod
+    def resolve_output_files(obj):
+        return obj.output_file_map()
+
+    @staticmethod
+    def resolve_output_mimetypes(obj) -> str:
+        mime_sizes: dict[str, int] = defaultdict(int)
+        for metadata in obj.output_file_map().values():
+            if not isinstance(metadata, dict):
+                continue
+            mimetype = str(metadata.get("mimetype") or "").strip()
+            try:
+                size = max(int(metadata.get("size") or 0), 0)
+            except (TypeError, ValueError):
+                size = 0
+            if mimetype and size:
+                mime_sizes[mimetype] += size
+        if mime_sizes:
+            return ",".join(mime for mime, _size in sorted(mime_sizes.items(), key=lambda item: item[1], reverse=True))
+        return obj.output_mimetypes or ""
+
+
+class ArchiveResultSchema(MinimalArchiveResultSchema):
+    TYPE: str = "core.models.ArchiveResult"
+    snapshot_id: UUID
+    snapshot_timestamp: str
+    snapshot_url: str
+    snapshot_tags: list[str]
+
+    @staticmethod
+    def resolve_snapshot_timestamp(obj):
+        return obj.snapshot.timestamp
+
+    @staticmethod
+    def resolve_snapshot_url(obj):
+        return obj.snapshot.url
+
+    @staticmethod
+    def resolve_snapshot_id(obj):
+        return obj.snapshot_id
+
+    @staticmethod
+    def resolve_snapshot_tags(obj):
+        return sorted(tag.name for tag in obj.snapshot.tags.all())
+
+
+class ArchiveResultFilterSchema(FilterSchema):
+    id: Annotated[str | None, FilterLookup(["id__startswith", "snapshot__id__startswith", "snapshot__timestamp__startswith"])] = None
+    search: Annotated[
+        str | None,
+        FilterLookup(
+            [
+                "snapshot__url__icontains",
+                "snapshot__title__icontains",
+                "snapshot__tags__name__icontains",
+                "plugin",
+                "output_str__icontains",
+                "id__startswith",
+                "snapshot__id__startswith",
+                "snapshot__timestamp__startswith",
+            ],
+        ),
+    ] = None
+    snapshot_id: Annotated[str | None, FilterLookup(["snapshot__id__startswith", "snapshot__timestamp__startswith"])] = None
+    snapshot_url: Annotated[str | None, FilterLookup("snapshot__url__icontains")] = None
+    snapshot_tag: Annotated[str | None, FilterLookup("snapshot__tags__name__icontains")] = None
+    status: Annotated[str | None, FilterLookup("status")] = None
+    output_str: Annotated[str | None, FilterLookup("output_str__icontains")] = None
+    plugin: Annotated[str | None, FilterLookup("plugin__icontains")] = None
+    hook_name: Annotated[str | None, FilterLookup("hook_name__icontains")] = None
+    process_id: Annotated[str | None, FilterLookup("process__id__startswith")] = None
+    cmd: Annotated[str | None, FilterLookup("cmd__0__icontains")] = None
+    pwd: Annotated[str | None, FilterLookup("pwd__icontains")] = None
+    cmd_version: Annotated[str | None, FilterLookup("cmd_version")] = None
+    created_at: Annotated[datetime | None, FilterLookup("created_at")] = None
+    created_at__gte: Annotated[datetime | None, FilterLookup("created_at__gte")] = None
+    created_at__lt: Annotated[datetime | None, FilterLookup("created_at__lt")] = None
+
+
+class SnapshotSchema(OwnedObjectSchema):
+    TYPE: str = "core.models.Snapshot"
+    id: UUID
+    created_at: datetime
+    modified_at: datetime
+    status: str
+    retry_at: datetime | None
+    bookmarked_at: datetime
+    downloaded_at: datetime | None
+    url: str
+    tags: list[str]
+    title: str | None
+    timestamp: str
+    archive_path: str
+    archive_size: int
+    output_size: int
+    num_archiveresults: int
+    archiveresults: list[MinimalArchiveResultSchema]
+
+    @staticmethod
+    def resolve_tags(obj):
+        return sorted(tag.name for tag in obj.tags.all())
+
+    @staticmethod
+    def resolve_archive_size(obj):
+        return int(obj.archive_size or 0)
+
+    @staticmethod
+    def resolve_output_size(obj):
+        return SnapshotSchema.resolve_archive_size(obj)
+
+    @staticmethod
+    def resolve_num_archiveresults(obj, context):
+        return obj.archiveresult_set.all().distinct().count()
+
+    @staticmethod
+    def resolve_archiveresults(obj, context):
+        if bool(context["request"].__dict__.get("with_archiveresults", False)):
+            return obj.archiveresult_set.all().distinct()
+        return ArchiveResult.objects.none()
+
+
+class SnapshotUpdateSchema(Schema):
+    action: str | None = None
+    status: str | None = None
+    retry_at: datetime | None = None
+    tags: list[str] | None = None
+
+
+class SnapshotCreateSchema(Schema):
+    url: str
+    crawl_id: str | None = None
+    depth: int = 0
+    title: str | None = None
+    tags: list[str] | None = None
+    status: str | None = None
+
+
+class SnapshotDeleteResponseSchema(Schema):
+    success: bool
+    snapshot_id: str
+    crawl_id: str
+    deleted_count: int
+
+
+class SnapshotFilterSchema(FilterSchema):
+    id: Annotated[str | None, FilterLookup(["id__istartswith", "id__iendswith", "timestamp__startswith"])] = None
+    created_by_id: Annotated[str | None, FilterLookup("crawl__created_by_id")] = None
+    created_by_username: Annotated[str | None, FilterLookup("crawl__created_by__username__icontains")] = None
+    created_at__gte: Annotated[datetime | None, FilterLookup("created_at__gte")] = None
+    created_at__lt: Annotated[datetime | None, FilterLookup("created_at__lt")] = None
+    created_at: Annotated[datetime | None, FilterLookup("created_at")] = None
+    modified_at: Annotated[datetime | None, FilterLookup("modified_at")] = None
+    modified_at__gte: Annotated[datetime | None, FilterLookup("modified_at__gte")] = None
+    modified_at__lt: Annotated[datetime | None, FilterLookup("modified_at__lt")] = None
+    search: str | None = None
+    search_mode: str | None = None
+    status: str | None = None
+    url: Annotated[str | None, FilterLookup("url")] = None
+    tag: Annotated[str | None, FilterLookup("tags__name")] = None
+    title: Annotated[str | None, FilterLookup("title__icontains")] = None
+    timestamp: Annotated[str | None, FilterLookup("timestamp__startswith")] = None
+    bookmarked_at__gte: Annotated[datetime | None, FilterLookup("bookmarked_at__gte")] = None
+    bookmarked_at__lt: Annotated[datetime | None, FilterLookup("bookmarked_at__lt")] = None
+
+    def filter_search(self, value: str | None) -> Q:
+        return Q()
+
+    def filter_search_mode(self, value: str | None) -> Q:
+        return Q()
+
+    def filter_status(self, value: str | None) -> Q:
+        return Q()
+
+
+class TagSchema(OwnedObjectSchema):
+    TYPE: str = "core.models.Tag"
+    id: int
+    modified_at: datetime
+    created_at: datetime
+    name: str
+    num_snapshots: int
+    snapshots: list[SnapshotSchema]
+
+    @staticmethod
+    def resolve_num_snapshots(obj, context):
+        return obj.snapshot_set.all().distinct().count()
+
+    @staticmethod
+    def resolve_snapshots(obj, context):
+        if bool(context["request"].__dict__.get("with_snapshots", False)):
+            return obj.snapshot_set.all().distinct()
+        return Snapshot.objects.none()
+
+
+class TagAutocompleteSchema(Schema):
+    tags: list[dict]
+
+
+class TagCreateSchema(Schema):
+    name: str
+
+
+class TagCreateResponseSchema(Schema):
+    success: bool
+    tag_id: int
+    tag_name: str
+    created: bool
+
+
+class TagSearchSnapshotSchema(Schema):
+    id: str
+    title: str
+    url: str
+    favicon_url: str
+    admin_url: str
+    archive_url: str
+    downloaded_at: str | None = None
+
+
+class TagSearchCardSchema(Schema):
+    id: int
+    name: str
+    slug: str
+    num_snapshots: int
+    filter_url: str
+    edit_url: str
+    export_urls_url: str
+    export_jsonl_url: str
+    rename_url: str
+    delete_url: str
+    snapshots: list[TagSearchSnapshotSchema]
+
+
+class TagSearchResponseSchema(Schema):
+    tags: list[TagSearchCardSchema]
+    sort: str
+    created_by: str
+    year: str
+    has_snapshots: str
+
+
+class TagUpdateSchema(Schema):
+    name: str
+
+
+class TagUpdateResponseSchema(Schema):
+    success: bool
+    tag_id: int
+    tag_name: str
+
+
+class TagDeleteResponseSchema(Schema):
+    success: bool
+    tag_id: int
+    deleted_count: int
+
+
+class TagSnapshotRequestSchema(Schema):
+    snapshot_id: str
+    tag_name: str | None = None
+    tag_id: int | None = None
+
+
+class TagSnapshotResponseSchema(Schema):
+    success: bool
+    tag_id: int
+    tag_name: str

archivebox/api/v1_core/snapshots.py

@@ -0,0 +1,356 @@
+from datetime import datetime, time
+from html import unescape
+from typing import Any
+
+from django.contrib.auth import get_user_model
+from django.contrib.auth.models import User
+from django.core.exceptions import ValidationError
+from django.db import IntegrityError
+from django.db.models import Q
+from django.http import HttpRequest, HttpResponse
+from django.utils import timezone
+from django.utils.dateparse import parse_date, parse_datetime
+from django.utils.feedgenerator import Rss201rev2Feed
+from ninja import Query, Router
+from ninja.errors import HttpError
+from ninja.pagination import paginate
+
+from archivebox.api.v1_crawls import get_crawl_by_ref
+from archivebox.core.models import Snapshot
+from archivebox.core.routes_util import build_web_url
+from archivebox.core.snapshot_status import filter_snapshots_by_status, normalize_snapshot_status
+from archivebox.crawls.locks import crawl_lifecycle_lock
+from archivebox.crawls.models import Crawl
+from archivebox.misc.util import filter_queryset_by_uuid_substring, validate_url_length
+from archivebox.search.config import get_search_mode, get_search_mode_backend
+from archivebox.search.query import apply_snapshot_search
+
+from .lookup import _get_snapshot_by_ref
+from .pagination import CustomPagination
+from .schemas import SnapshotCreateSchema, SnapshotDeleteResponseSchema, SnapshotFilterSchema, SnapshotSchema, SnapshotUpdateSchema
+
+router = Router(tags=["Core Models"])
+
+
+def normalize_tag_list(tags: list[str] | None = None) -> list[str]:
+    return [tag.strip() for tag in (tags or []) if tag and tag.strip()]
+
+
+def _parse_rss_before(before: str | None) -> datetime:
+    if not before:
+        return timezone.now()
+
+    value = before.strip()
+    parsed_dt = None
+
+    if len(value) == 8 and value.isdigit():
+        parsed_date = datetime.strptime(value, "%Y%m%d").date()
+    else:
+        parsed_dt = parse_datetime(value)
+        parsed_date = None if parsed_dt else parse_date(value)
+
+    if parsed_dt is None:
+        if parsed_date is None:
+            raise HttpError(400, "before must be an ISO datetime, YYYY-MM-DD, or YYYYMMDD")
+        parsed_dt = datetime.combine(parsed_date, time.max)
+
+    if timezone.is_naive(parsed_dt):
+        parsed_dt = timezone.make_aware(parsed_dt, timezone.get_current_timezone())
+    return parsed_dt
+
+
+def _filter_snapshots_for_rss(
+    *,
+    crawl_id: str = "",
+    created_by: str = "",
+    before: str | None = None,
+    limit: int = 50,
+):
+    limit = max(1, min(int(limit or 50), 500))
+    before_dt = _parse_rss_before(before)
+    queryset = (
+        Snapshot.objects.select_related("crawl__created_by")
+        .prefetch_related("tags")
+        .only(
+            "id",
+            "url",
+            "title",
+            "timestamp",
+            "bookmarked_at",
+            "created_at",
+            "modified_at",
+            "fs_version",
+            "crawl_id",
+            "crawl__id",
+            "crawl__created_by_id",
+            "crawl__created_by__id",
+            "crawl__created_by__username",
+        )
+        .filter(bookmarked_at__lte=before_dt)
+    )
+    crawl_id = crawl_id.strip()
+    if crawl_id:
+        matching_crawl_pks = list(filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).values_list("pk", flat=True)[:100])
+        queryset = queryset.filter(crawl_id__in=matching_crawl_pks)
+
+    created_by = created_by.strip()
+    if created_by:
+        created_by_query = Q(crawl__created_by__username__iexact=created_by)
+        user_model = get_user_model()
+        try:
+            prepared_pk = user_model._meta.pk.get_prep_value(created_by)
+        except (TypeError, ValueError, ValidationError):
+            prepared_pk = None
+        if prepared_pk not in (None, ""):
+            created_by_query |= Q(crawl__created_by_id=prepared_pk)
+        queryset = queryset.filter(created_by_query)
+
+    return queryset.order_by("-bookmarked_at", "-created_at", "-id")[:limit]
+
+
+def _snapshots_rss_response(
+    request: HttpRequest,
+    *,
+    snapshots,
+    title: str = "ArchiveBox Snapshots",
+) -> HttpResponse:
+    web_base_url = build_web_url("/", request=request).rstrip("/")
+    feed_query = request.GET.copy()
+    for sensitive_param in ("api_key", "token", "password"):
+        feed_query.pop(sensitive_param, None)
+    feed_path = request.path
+    feed_url = request.build_absolute_uri(f"{feed_path}?{feed_query.urlencode()}" if feed_query else feed_path)
+
+    feed = Rss201rev2Feed(
+        title=title,
+        link=build_web_url("/public/", request=request),
+        description="Recently added ArchiveBox snapshots.",
+        language="en",
+        feed_url=feed_url,
+    )
+
+    for snapshot in snapshots:
+        archived_url = build_web_url(f"/{snapshot.archive_path_from_db}", request=request)
+        tags = [tag.name for tag in snapshot.tags.all()]
+        crawl_user = snapshot.crawl.created_by if snapshot.crawl_id else None
+        description = f"Original URL: {snapshot.url}\nArchived snapshot: {archived_url}"
+        feed.add_item(
+            title=unescape(snapshot.title or snapshot.url),
+            link=archived_url or web_base_url,
+            description=description,
+            unique_id=str(snapshot.id),
+            unique_id_is_permalink=False,
+            pubdate=snapshot.bookmarked_at or snapshot.created_at,
+            updateddate=snapshot.modified_at,
+            author_name=crawl_user.username if crawl_user else None,
+            categories=tags,
+        )
+
+    return HttpResponse(feed.writeString("utf-8"), content_type="application/rss+xml; charset=utf-8")
+
+
+@router.get("/snapshots", response=list[SnapshotSchema], url_name="get_snapshots")
+@paginate(CustomPagination)
+def get_snapshots(request: HttpRequest, filters: Query[SnapshotFilterSchema], with_archiveresults: bool = False):
+    """List all Snapshot entries matching these filters."""
+    setattr(request, "with_archiveresults", with_archiveresults)
+    try:
+        queryset = filter_snapshots_by_status(Snapshot.objects.all(), filters.status)
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
+    queryset = filters.filter(queryset).distinct()
+    query = (filters.search or "").strip()
+    if not query:
+        return queryset
+
+    runtime_config = request.archivebox_config
+    search_mode = get_search_mode(filters.search_mode, config=runtime_config)
+    try:
+        return apply_snapshot_search(
+            queryset,
+            query,
+            search_mode=search_mode,
+            config=runtime_config,
+            include_id_matches=True,
+        )
+    except Exception:
+        if get_search_mode_backend(search_mode, config=runtime_config):
+            return queryset.none()
+        return apply_snapshot_search(queryset, query, search_mode="meta", config=runtime_config, include_id_matches=True)
+
+
+@router.get("/snapshots.rss", url_name="get_snapshots_rss")
+def get_snapshots_rss(
+    request: HttpRequest,
+    crawl_id: str = "",
+    created_by: str = "",
+    limit: int = 50,
+    before: str | None = None,
+):
+    """Return matching snapshots as an RSS feed, newest first."""
+    snapshots = _filter_snapshots_for_rss(
+        crawl_id=crawl_id,
+        created_by=created_by,
+        limit=limit,
+        before=before,
+    )
+    return _snapshots_rss_response(request, snapshots=snapshots)
+
+
+@router.get("/snapshot/{snapshot_id}", response=SnapshotSchema, url_name="get_snapshot")
+def get_snapshot(request: HttpRequest, snapshot_id: str, with_archiveresults: bool = True):
+    """Get a specific Snapshot by id."""
+    setattr(request, "with_archiveresults", with_archiveresults)
+    return _get_snapshot_by_ref(snapshot_id)
+
+
+@router.post("/snapshots", response=SnapshotSchema, url_name="create_snapshot")
+def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
+    tags = normalize_tag_list(data.tags)
+    try:
+        status = normalize_snapshot_status(data.status)
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
+    if not data.url.strip():
+        raise HttpError(400, "URL is required")
+    try:
+        validate_url_length(data.url.strip())
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
+    if data.depth not in (0, 1, 2, 3, 4):
+        raise HttpError(400, "depth must be between 0 and 4")
+
+    if data.crawl_id:
+        crawl = get_crawl_by_ref(data.crawl_id)
+        crawl_tags = normalize_tag_list(crawl.tags_str.split(","))
+        tags = tags or crawl_tags
+    else:
+        crawl = Crawl.objects.create(
+            urls=data.url,
+            max_depth=max(data.depth, 0),
+            tags_str=",".join(tags),
+            status=Crawl.StatusChoices.QUEUED,
+            retry_at=timezone.now(),
+            created_by=request.user if isinstance(request.user, User) else None,
+        )
+
+    # Browser uploads must not wait behind the runner's crawl-wide lifecycle
+    # lock. The unique insert recovery and CAS update below are the request-side
+    # coordination boundary for this idempotent metadata sync.
+    snapshot = Snapshot.objects.filter(url=data.url, crawl=crawl).first()
+    if snapshot is None:
+        try:
+            snapshot = Snapshot.objects.create(
+                url=data.url,
+                crawl=crawl,
+                depth=data.depth,
+                title=data.title,
+                timestamp=str(timezone.now().timestamp()),
+                status=status or Snapshot.StatusChoices.QUEUED,
+                retry_at=timezone.now(),
+            )
+        except IntegrityError:
+            snapshot = Snapshot.objects.filter(url=data.url, crawl=crawl).first()
+            if snapshot is None:
+                raise
+
+    for _attempt in range(3):
+        updates: dict[str, Any] = {}
+        if data.title is not None and snapshot.title != data.title:
+            updates["title"] = data.title
+        if status is not None and snapshot.status != status:
+            updates["status"] = status
+        if not updates or snapshot.safe_update(updates, extra_filter={"modified_at": snapshot.modified_at}):
+            break
+    else:
+        raise HttpError(409, "Snapshot changed while metadata was being updated")
+
+    if tags:
+        snapshot.save_tags(
+            tags,
+            created_by=request.user if isinstance(request.user, User) else None,
+        )
+
+    try:
+        snapshot.ensure_crawl_symlink()
+    except Exception:
+        pass
+
+    setattr(request, "with_archiveresults", False)
+    return snapshot
+
+
+@router.patch("/snapshot/{snapshot_id}", response=SnapshotSchema, url_name="patch_snapshot")
+def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateSchema):
+    """Update a snapshot (e.g., set status=sealed to cancel queued work)."""
+    snapshot = _get_snapshot_by_ref(snapshot_id)
+    crawl_id = str(snapshot.crawl_id)
+
+    payload = data.dict(exclude_unset=True)
+    update_fields = ["modified_at"]
+    action = payload.pop("action", None)
+    tags = payload.pop("tags", None)
+
+    if action:
+        if action == "pause":
+            snapshot.pause()
+            setattr(request, "with_archiveresults", False)
+            return snapshot
+        if action in ("resume", "unpause"):
+            snapshot.resume()
+            setattr(request, "with_archiveresults", False)
+            return snapshot
+        if action == "cancel":
+            snapshot.cancel()
+            setattr(request, "with_archiveresults", False)
+            return snapshot
+        raise HttpError(400, f"Invalid action: {action}")
+
+    with crawl_lifecycle_lock(crawl_id):
+        snapshot = _get_snapshot_by_ref(snapshot_id)
+        if "status" in payload:
+            try:
+                snapshot.status = normalize_snapshot_status(payload["status"])
+            except ValueError as err:
+                raise HttpError(400, str(err)) from err
+            if snapshot.status == Snapshot.StatusChoices.SEALED and "retry_at" not in payload:
+                snapshot.retry_at = None
+            update_fields.append("status")
+
+        if "retry_at" in payload:
+            snapshot.retry_at = payload["retry_at"]
+            update_fields.append("retry_at")
+
+        if tags is not None:
+            snapshot.save_tags(
+                normalize_tag_list(tags),
+                created_by=request.user if isinstance(request.user, User) else None,
+            )
+
+        if payload.get("status") == Snapshot.StatusChoices.SEALED:
+            snapshot.cancel()
+        else:
+            snapshot.save(update_fields=update_fields)
+    setattr(request, "with_archiveresults", False)
+    return snapshot
+
+
+@router.delete("/snapshot/{snapshot_id}", response=SnapshotDeleteResponseSchema, url_name="delete_snapshot")
+def delete_snapshot(request: HttpRequest, snapshot_id: str):
+    snapshot = get_snapshot(request, snapshot_id, with_archiveresults=False)
+    snapshot_id_str = str(snapshot.id)
+    crawl_id_str = str(snapshot.crawl.pk)
+    snapshot.cancel()
+
+    from archivebox.services.runner import run_pending_crawls
+
+    with crawl_lifecycle_lock(crawl_id_str):
+        run_pending_crawls(crawl_id=crawl_id_str, daemon=False)
+        snapshot = get_snapshot(request, snapshot_id_str, with_archiveresults=False)
+        deleted_count, _ = snapshot.delete()
+    return {
+        "success": True,
+        "snapshot_id": snapshot_id_str,
+        "crawl_id": crawl_id_str,
+        "deleted_count": deleted_count,
+    }

archivebox/api/v1_core/tags.py

@@ -0,0 +1,285 @@
+from django.core.exceptions import ValidationError
+from django.db.models import Q
+from django.http import HttpRequest, HttpResponse
+from ninja import Router
+from ninja.errors import HttpError
+from ninja.pagination import paginate
+
+from archivebox.api.auth import authenticated_user_from_request
+from archivebox.config.common import get_config
+from archivebox.core.models import Snapshot, Tag
+from archivebox.core.permissions import public_snapshots_queryset
+from archivebox.core.tag_util import (
+    add_snapshot_counts,
+    build_tag_cards,
+    export_tag_snapshots_jsonl,
+    export_tag_urls,
+    get_matching_tags,
+    get_tag_by_ref,
+    normalize_created_by_filter,
+    normalize_created_year_filter,
+    normalize_has_snapshots_filter,
+    normalize_tag_sort,
+)
+
+from .pagination import CustomPagination
+from .schemas import (
+    TagAutocompleteSchema,
+    TagCreateResponseSchema,
+    TagCreateSchema,
+    TagDeleteResponseSchema,
+    TagSchema,
+    TagSearchResponseSchema,
+    TagSnapshotRequestSchema,
+    TagSnapshotResponseSchema,
+    TagUpdateResponseSchema,
+    TagUpdateSchema,
+)
+
+router = Router(tags=["Core Models"])
+
+
+@router.get("/tags", response=list[TagSchema], url_name="get_tags")
+@paginate(CustomPagination)
+def get_tags(request: HttpRequest):
+    setattr(request, "with_snapshots", False)
+    setattr(request, "with_archiveresults", False)
+    return get_matching_tags()
+
+
+@router.get("/tag/{tag_id}", response=TagSchema, url_name="get_tag")
+def get_tag(request: HttpRequest, tag_id: str, with_snapshots: bool = True):
+    setattr(request, "with_snapshots", with_snapshots)
+    setattr(request, "with_archiveresults", False)
+    try:
+        return get_tag_by_ref(tag_id)
+    except (Tag.DoesNotExist, ValidationError):
+        raise HttpError(404, "Tag not found")
+
+
+def _get_snapshot_for_tag_edit(snapshot_ref: str) -> Snapshot:
+    snapshot_ref = str(snapshot_ref or "").strip().lower()
+    if not snapshot_ref:
+        raise HttpError(400, "Snapshot id is required")
+
+    snapshot_qs = Snapshot.objects.only("id")
+    is_full_uuid = len(snapshot_ref.replace("-", "")) == 32 and all(char in "0123456789abcdef-" for char in snapshot_ref)
+    if is_full_uuid:
+        try:
+            return snapshot_qs.get(pk=snapshot_ref.replace("-", ""))
+        except (Snapshot.DoesNotExist, ValueError):
+            pass
+
+    if len(snapshot_ref) >= 14:
+        try:
+            return snapshot_qs.get(timestamp=snapshot_ref)
+        except Snapshot.DoesNotExist:
+            pass
+        except Snapshot.MultipleObjectsReturned:
+            snapshot = snapshot_qs.filter(timestamp=snapshot_ref).first()
+            if snapshot is not None:
+                return snapshot
+
+    try:
+        return snapshot_qs.get(Q(id__startswith=snapshot_ref) | Q(timestamp__startswith=snapshot_ref))
+    except Snapshot.DoesNotExist:
+        raise HttpError(404, "Snapshot not found") from None
+    except Snapshot.MultipleObjectsReturned:
+        snapshot = snapshot_qs.filter(Q(id__startswith=snapshot_ref) | Q(timestamp__startswith=snapshot_ref)).first()
+        if snapshot is None:
+            raise HttpError(404, "Snapshot not found")
+        return snapshot
+
+
+@router.get("/tags/search/", response=TagSearchResponseSchema, url_name="search_tags")
+def search_tags(
+    request: HttpRequest,
+    q: str = "",
+    sort: str = "created_desc",
+    created_by: str = "",
+    year: str = "",
+    has_snapshots: str = "all",
+):
+    """Return detailed tag cards for admin/live-search UIs."""
+    normalized_sort = normalize_tag_sort(sort)
+    normalized_created_by = normalize_created_by_filter(created_by)
+    normalized_year = normalize_created_year_filter(year)
+    normalized_has_snapshots = normalize_has_snapshots_filter(has_snapshots)
+    return {
+        "tags": build_tag_cards(
+            query=q,
+            request=request,
+            preview_limit=0,
+            sort=normalized_sort,
+            created_by=normalized_created_by,
+            year=normalized_year,
+            has_snapshots=normalized_has_snapshots,
+        ),
+        "sort": normalized_sort,
+        "created_by": normalized_created_by,
+        "year": normalized_year,
+        "has_snapshots": normalized_has_snapshots,
+    }
+
+
+def _public_tag_listing_enabled() -> bool:
+    return get_config().PUBLIC_INDEX
+
+
+def _request_has_tag_autocomplete_access(request: HttpRequest) -> bool:
+    if authenticated_user_from_request(request):
+        return True
+
+    return _public_tag_listing_enabled()
+
+
+@router.get("/tags/autocomplete/", response=TagAutocompleteSchema, url_name="tags_autocomplete", auth=None)
+def tags_autocomplete(request: HttpRequest, q: str = ""):
+    """Return tags matching the query for autocomplete."""
+    if not _request_has_tag_autocomplete_access(request):
+        raise HttpError(401, "Authentication required")
+
+    public_only = not request.user.is_authenticated and not request.__dict__.get("_api_token")
+    queryset = get_matching_tags(q)
+    public_snapshots = public_snapshots_queryset(Snapshot.objects.all())
+    if public_only:
+        queryset = queryset.filter(snapshot_set__id__in=public_snapshots.values("id")).distinct()
+    tags = list(queryset[: 50 if not q else 20])
+    add_snapshot_counts(tags, snapshot_queryset=public_snapshots if public_only else None)
+
+    return {
+        "tags": [{"id": tag.pk, "name": tag.name, "num_snapshots": tag.__dict__.get("num_snapshots", 0)} for tag in tags],
+    }
+
+
+@router.post("/tags/create/", response=TagCreateResponseSchema, url_name="tags_create")
+def tags_create(request: HttpRequest, data: TagCreateSchema):
+    """Create a new tag or return existing one."""
+    try:
+        tag, created = Tag.get_or_create_by_name(
+            data.name,
+            created_by=request.user if request.user.is_authenticated else None,
+        )
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
+
+    return {
+        "success": True,
+        "tag_id": tag.pk,
+        "tag_name": tag.name,
+        "created": created,
+    }
+
+
+@router.post("/tag/{tag_id}/rename", response=TagUpdateResponseSchema, url_name="rename_tag")
+def rename_tag(request: HttpRequest, tag_id: int, data: TagUpdateSchema):
+    try:
+        tag = get_tag_by_ref(tag_id).rename(data.name)
+    except Tag.DoesNotExist as err:
+        raise HttpError(404, "Tag not found") from err
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
+
+    return {
+        "success": True,
+        "tag_id": tag.pk,
+        "tag_name": tag.name,
+    }
+
+
+@router.delete("/tag/{tag_id}", response=TagDeleteResponseSchema, url_name="delete_tag")
+def delete_tag(request: HttpRequest, tag_id: int):
+    try:
+        tag = get_tag_by_ref(tag_id)
+    except Tag.DoesNotExist as err:
+        raise HttpError(404, "Tag not found") from err
+
+    deleted_count, _ = tag.delete()
+    return {
+        "success": True,
+        "tag_id": int(tag_id),
+        "deleted_count": deleted_count,
+    }
+
+
+@router.get("/tag/{tag_id}/urls.txt", url_name="tag_urls_export")
+def tag_urls_export(request: HttpRequest, tag_id: int):
+    try:
+        tag = get_tag_by_ref(tag_id)
+    except Tag.DoesNotExist as err:
+        raise HttpError(404, "Tag not found") from err
+
+    response = HttpResponse(export_tag_urls(tag), content_type="text/plain; charset=utf-8")
+    response["Content-Disposition"] = f'attachment; filename="tag-{tag.slug}-urls.txt"'
+    return response
+
+
+@router.get("/tag/{tag_id}/snapshots.jsonl", url_name="tag_snapshots_export")
+def tag_snapshots_export(request: HttpRequest, tag_id: int):
+    try:
+        tag = get_tag_by_ref(tag_id)
+    except Tag.DoesNotExist as err:
+        raise HttpError(404, "Tag not found") from err
+
+    response = HttpResponse(export_tag_snapshots_jsonl(tag), content_type="application/x-ndjson; charset=utf-8")
+    response["Content-Disposition"] = f'attachment; filename="tag-{tag.slug}-snapshots.jsonl"'
+    return response
+
+
+@router.post("/tags/add-to-snapshot/", response=TagSnapshotResponseSchema, url_name="tags_add_to_snapshot")
+def tags_add_to_snapshot(request: HttpRequest, data: TagSnapshotRequestSchema):
+    """Add a tag to a snapshot. Creates the tag if it doesn't exist."""
+    snapshot = _get_snapshot_for_tag_edit(data.snapshot_id)
+
+    # Get or create the tag
+    if data.tag_name:
+        try:
+            tag, _ = Tag.get_or_create_by_name(
+                data.tag_name,
+                created_by=request.user if request.user.is_authenticated else None,
+            )
+        except ValueError as err:
+            raise HttpError(400, str(err)) from err
+    elif data.tag_id:
+        try:
+            tag = get_tag_by_ref(data.tag_id)
+        except Tag.DoesNotExist:
+            raise HttpError(404, "Tag not found")
+    else:
+        raise HttpError(400, "Either tag_name or tag_id is required")
+
+    snapshot.add_tag_ids([tag.pk])
+
+    return {
+        "success": True,
+        "tag_id": tag.pk,
+        "tag_name": tag.name,
+    }
+
+
+@router.post("/tags/remove-from-snapshot/", response=TagSnapshotResponseSchema, url_name="tags_remove_from_snapshot")
+def tags_remove_from_snapshot(request: HttpRequest, data: TagSnapshotRequestSchema):
+    """Remove a tag from a snapshot."""
+    snapshot = _get_snapshot_for_tag_edit(data.snapshot_id)
+
+    # Get the tag
+    if data.tag_id:
+        try:
+            tag = Tag.objects.get(pk=data.tag_id)
+        except Tag.DoesNotExist:
+            raise HttpError(404, "Tag not found")
+    elif data.tag_name:
+        try:
+            tag = Tag.objects.get(name__iexact=data.tag_name.strip())
+        except Tag.DoesNotExist:
+            raise HttpError(404, "Tag not found")
+    else:
+        raise HttpError(400, "Either tag_name or tag_id is required")
+
+    snapshot.remove_tag_ids([tag.pk])
+
+    return {
+        "success": True,
+        "tag_id": tag.pk,
+        "tag_name": tag.name,
+    }

archivebox/api/v1_crawls.py

@@ -1,6 +1,5 @@
 __package__ = "archivebox.api"
 
-import json
 from pathlib import Path
 from uuid import UUID
 from datetime import datetime
@@ -8,9 +7,10 @@ from django.http import FileResponse, HttpRequest
 from django.shortcuts import redirect
 from django.utils import timezone
 
-from django.contrib.auth import get_user_model
 from django.contrib.auth.models import User
 
+from archivebox.api.schemas import OwnedObjectSchema
+
 from ninja import Router, Schema
 from ninja.errors import HttpError
 
@@ -23,22 +23,20 @@ from archivebox.core.permissions import (
 )
 from archivebox.config.common import get_config
 from archivebox.crawls.models import Crawl
-from archivebox.misc.util import filter_queryset_by_uuid_substring
+from archivebox.misc.util import filter_queryset_by_uuid_substring, validate_url
 
 from .auth import API_AUTH_METHODS, authenticated_user_from_request
 
 router = Router(tags=["Crawl Models"], auth=API_AUTH_METHODS)
 
 
-class CrawlSchema(Schema):
+class CrawlSchema(OwnedObjectSchema):
     TYPE: str = "crawls.models.Crawl"
 
     id: UUID
 
     modified_at: datetime
     created_at: datetime
-    created_by_id: str
-    created_by_username: str
 
     status: str
     retry_at: datetime | None
@@ -49,17 +47,6 @@ class CrawlSchema(Schema):
     tags_str: str
     config: dict
 
-    @staticmethod
-    def resolve_created_by_id(obj):
-        return str(obj.created_by_id)
-
-    @staticmethod
-    def resolve_created_by_username(obj):
-        user_model = get_user_model()
-        user = user_model.objects.get(id=obj.created_by_id)
-        username = user.username
-        return username if isinstance(username, str) else str(user)
-
     @staticmethod
     def resolve_config(obj):
         # Redact credential values so REST responses can never leak the raw
@@ -113,7 +100,10 @@ def get_crawls(request: HttpRequest):
 
 @router.post("/crawls", response=CrawlSchema, url_name="create_crawl")
 def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
-    urls = [url.strip() for url in data.urls if url and url.strip()]
+    try:
+        urls = [validate_url(url) for url in data.urls if url and url.strip()]
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
     if not urls:
         raise HttpError(400, "At least one URL is required")
     if data.max_depth not in (0, 1, 2, 3, 4):
@@ -121,9 +111,10 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
 
     tags = normalize_tag_list(data.tags, data.tags_str)
     config = dict(data.config or {})
+    config.setdefault("PARSER", "url_list")
     config.setdefault("PERMISSIONS", str(get_config().PERMISSIONS))
     crawl = Crawl.objects.create(
-        urls="\n".join(json.dumps({"type": "CrawlSeed", "url": url, "depth": 0}, separators=(",", ":")) for url in urls),
+        urls="\n".join(urls),
         max_depth=data.max_depth,
         tags_str=",".join(tags),
         label=data.label,

archivebox/api/v1_personas.py

@@ -7,6 +7,8 @@ from uuid import UUID
 
 from django.db.models import Q
 from django.http import HttpRequest
+from archivebox.api.schemas import OwnedObjectSchema
+
 from ninja import Router, Schema
 from ninja.pagination import paginate
 from pydantic import Field
@@ -36,23 +38,13 @@ class PersonaSyncSchema(Schema):
     auth_json: dict[str, Any] = Field(default_factory=dict)
 
 
-class PersonaSchema(Schema):
+class PersonaSchema(OwnedObjectSchema):
     TYPE: str = "personas.models.Persona"
     id: UUID
     name: str
     created_at: datetime
-    created_by_id: str
-    created_by_username: str
     config: dict[str, Any] | None
 
-    @staticmethod
-    def resolve_created_by_id(obj):
-        return str(obj.created_by.pk)
-
-    @staticmethod
-    def resolve_created_by_username(obj) -> str:
-        return obj.created_by.username
-
     @staticmethod
     def resolve_config(obj):
         # Redact credential values so REST responses don't leak the raw

archivebox/base_models/admin.py

@@ -13,10 +13,16 @@ from django.db import DatabaseError, models
 from django.forms.renderers import BaseRenderer
 from django.http import HttpRequest, QueryDict
 from django.urls import path, register_converter
+from django.utils.html import format_html
 from django.utils.safestring import SafeString, mark_safe
 from django_object_actions import DjangoObjectActions
 
 
+def card_fieldset(title: str | None, fields: tuple, *, wide: bool = False, **options):
+    """Build a Django fieldset using the shared card layout."""
+    return title, {"fields": fields, "classes": ("card", "wide") if wide else ("card",), **options}
+
+
 class HexUUIDConverter:
     """URL path converter that canonicalizes UUIDs to their 32-char hex form.
 
@@ -81,6 +87,11 @@ class KeyValueWidget(forms.Widget):
             options: dict[str, ConfigOption] = {}
             for key, metadata in config_field_metadata().items():
                 option_type = metadata.get("type", "string")
+                # Core fields expose Python type names; plugins use JSON Schema.
+                # The editor should apply the same typed validation to both.
+                type_names = {"bool": "boolean", "int": "integer", "float": "number", "str": "string", "dict": "object", "list": "array"}
+                if isinstance(option_type, str):
+                    option_type = type_names.get(option_type, option_type)
                 option: ConfigOption = {
                     "plugin": str(metadata.get("plugin", "archivebox")),
                     "type": cast(str | list[str], option_type if isinstance(option_type, (str, list)) else str(option_type)),
@@ -118,650 +129,36 @@ class KeyValueWidget(forms.Widget):
         attrs: Mapping[str, str] | None = None,
         renderer: BaseRenderer | None = None,
     ) -> SafeString:
-        data = self._parse_value(value)
+        from django.template.loader import render_to_string
 
         widget_id = attrs.get("id", name) if attrs else name
-        config_options = self._get_config_options()
-
-        # Build datalist options
-        datalist_options = "\n".join(
-            f'<option value="{self._escape(key)}">{self._escape(opt["description"][:60] or opt["type"])}</option>'
-            for key, opt in sorted(config_options.items())
+        options = self._get_config_options()
+        rows = [
+            mark_safe(self._render_row(widget_id, key, val if isinstance(val, str) else json.dumps(val)))
+            for key, val in self._parse_value(value).items()
+        ]
+        empty_row = mark_safe(self._render_row(widget_id, "", ""))
+        # JSON is embedded in a script, so escape HTML delimiters without changing data.
+        metadata = json.dumps(options).translate({ord("<"): r"\u003C", ord(">"): r"\u003E", ord("&"): r"\u0026"})
+        return mark_safe(
+            render_to_string(
+                "admin/widgets/config.html",
+                {
+                    "name": name,
+                    "widget_id": widget_id,
+                    "datalist_options": mark_safe(
+                        "\n".join(
+                            f'<option value="{self._escape(key)}">{self._escape(opt["description"][:60] or opt["type"])}</option>'
+                            for key, opt in sorted(options.items())
+                        ),
+                    ),
+                    "config_meta_json": metadata,
+                    "rows": [*rows, empty_row],
+                    "empty_row": empty_row,
+                },
+            ),
         )
 
-        # Build config metadata as JSON for JS
-        config_meta_json = json.dumps(config_options)
-
-        html = f'''
-        <div id="{widget_id}_container" class="key-value-editor" style="width: 100%; max-width: none;">
-            <datalist id="{widget_id}_keys">
-                {datalist_options}
-            </datalist>
-            <div id="{widget_id}_rows" class="key-value-rows">
-        '''
-
-        # Render existing key-value pairs
-        for key, val in data.items():
-            val_str = json.dumps(val) if not isinstance(val, str) else val
-            html += self._render_row(widget_id, key, val_str)
-
-        # Always add one empty row for new entries
-        html += self._render_row(widget_id, "", "")
-
-        html += f'''
-            </div>
-            <div style="display: flex; gap: 8px; align-items: center; margin-top: 8px;">
-                <button type="button" onclick="addKeyValueRow_{widget_id}()"
-                        style="padding: 4px 12px; cursor: pointer; background: #417690; color: white; border: none; border-radius: 4px;">
-                    + Add Row
-                </button>
-            </div>
-            <input type="hidden" name="{name}" id="{widget_id}" value="">
-            <script>
-                (function() {{
-                    var configMeta_{widget_id} = {config_meta_json};
-                    var rowCounter_{widget_id} = 0;
-
-                    function stringifyValue_{widget_id}(value) {{
-                        return typeof value === 'string' ? value : JSON.stringify(value);
-                    }}
-
-                    function getTypes_{widget_id}(meta) {{
-                        if (!meta || meta.type === undefined || meta.type === null) {{
-                            return [];
-                        }}
-                        return Array.isArray(meta.type) ? meta.type : [meta.type];
-                    }}
-
-                    function getMetaForKey_{widget_id}(key) {{
-                        if (!key) {{
-                            return null;
-                        }}
-
-                        var explicitMeta = configMeta_{widget_id}[key];
-                        if (explicitMeta) {{
-                            return Object.assign({{ key: key }}, explicitMeta);
-                        }}
-
-                        if (key.endsWith('_BINARY')) {{
-                            return {{
-                                key: key,
-                                plugin: 'custom',
-                                type: 'string',
-                                default: '',
-                                description: 'Path to binary executable',
-                            }};
-                        }}
-
-                        if (isRegexConfigKey_{widget_id}(key)) {{
-                            return {{
-                                key: key,
-                                plugin: 'custom',
-                                type: 'string',
-                                default: '',
-                                description: 'Regex pattern list',
-                            }};
-                        }}
-
-                        return null;
-                    }}
-
-                    function describeMeta_{widget_id}(meta) {{
-                        if (!meta) {{
-                            return '';
-                        }}
-
-                        var details = '';
-                        if (Array.isArray(meta.enum) && meta.enum.length) {{
-                            details = 'Allowed: ' + meta.enum.map(stringifyValue_{widget_id}).join(', ');
-                        }} else {{
-                            var types = getTypes_{widget_id}(meta);
-                            if (types.length) {{
-                                details = 'Expected: ' + types.join(' or ');
-                            }}
-                        }}
-
-                        if (meta.minimum !== undefined || meta.maximum !== undefined) {{
-                            var bounds = [];
-                            if (meta.minimum !== undefined) bounds.push('min ' + meta.minimum);
-                            if (meta.maximum !== undefined) bounds.push('max ' + meta.maximum);
-                            details += (details ? ' ' : '') + '(' + bounds.join(', ') + ')';
-                        }}
-
-                        return [meta.description || '', details].filter(Boolean).join(' ');
-                    }}
-
-                    function getExampleInput_{widget_id}(key, meta) {{
-                        var types = getTypes_{widget_id}(meta);
-                        if (key.endsWith('_BINARY')) {{
-                            return 'Example: wget or /usr/bin/wget';
-                        }}
-                        if (key.endsWith('_ARGS_EXTRA') || key.endsWith('_ARGS')) {{
-                            return 'Example: ["--extra-arg"]';
-                        }}
-                        if (types.includes('array')) {{
-                            return 'Example: ["value"]';
-                        }}
-                        if (types.includes('object')) {{
-                            return 'Example: {{"key": "value"}}';
-                        }}
-                        return '';
-                    }}
-
-                    function isRegexConfigKey_{widget_id}(key) {{
-                        return key === 'URL_ALLOWLIST' ||
-                            key === 'URL_DENYLIST' ||
-                            key.endsWith('_PATTERN') ||
-                            key.includes('REGEX');
-                    }}
-
-                    function isSimpleFilterPattern_{widget_id}(pattern) {{
-                        return /^[\\w.*:-]+$/.test(pattern);
-                    }}
-
-                    function validateRegexPattern_{widget_id}(pattern) {{
-                        if (!pattern || isSimpleFilterPattern_{widget_id}(pattern)) {{
-                            return '';
-                        }}
-
-                        try {{
-                            new RegExp(pattern);
-                        }} catch (error) {{
-                            return error && error.message ? error.message : 'Invalid regex';
-                        }}
-                        return '';
-                    }}
-
-                    function validateRegexConfig_{widget_id}(key, raw, typeName) {{
-                        if (typeName === 'object') {{
-                            var parsed;
-                            try {{
-                                parsed = JSON.parse(raw);
-                            }} catch (error) {{
-                                return {{ ok: false, value: raw, message: 'Must be valid JSON' }};
-                            }}
-                            if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {{
-                                return {{ ok: false, value: parsed, message: 'Must be a JSON object' }};
-                            }}
-                            for (var regexKey in parsed) {{
-                                var objectRegexError = validateRegexPattern_{widget_id}(regexKey);
-                                if (objectRegexError) {{
-                                    return {{ ok: false, value: parsed, message: 'Invalid regex key "' + regexKey + '": ' + objectRegexError }};
-                                }}
-                            }}
-                            return {{ ok: true, value: parsed, message: '' }};
-                        }}
-
-                        var patterns = raw.split(/[\\n,]+/).map(function(pattern) {{
-                            return pattern.trim();
-                        }}).filter(Boolean);
-                        for (var i = 0; i < patterns.length; i++) {{
-                            var regexError = validateRegexPattern_{widget_id}(patterns[i]);
-                            if (regexError) {{
-                                return {{ ok: false, value: raw, message: 'Invalid regex "' + patterns[i] + '": ' + regexError }};
-                            }}
-                        }}
-                        return {{ ok: true, value: raw, message: '' }};
-                    }}
-
-                    function validateBinaryValue_{widget_id}(raw) {{
-                        if (!raw) {{
-                            return {{ ok: true, value: raw, message: '' }};
-                        }}
-
-                        if (/['"`]/.test(raw)) {{
-                            return {{ ok: false, value: raw, message: 'Binary paths cannot contain quotes' }};
-                        }}
-
-                        if (/[;&|<>$(){{}}\\[\\]!]/.test(raw)) {{
-                            return {{ ok: false, value: raw, message: 'Binary paths can only be a binary name or absolute path' }};
-                        }}
-
-                        if (raw.startsWith('/')) {{
-                            if (/^[A-Za-z0-9_./+\\- ]+$/.test(raw)) {{
-                                return {{ ok: true, value: raw, message: '' }};
-                            }}
-                            return {{ ok: false, value: raw, message: 'Absolute paths may only contain path-safe characters' }};
-                        }}
-
-                        if (/^[A-Za-z0-9_.+-]+$/.test(raw)) {{
-                            return {{ ok: true, value: raw, message: '' }};
-                        }}
-
-                        return {{ ok: false, value: raw, message: 'Enter a binary name like wget or an absolute path like /usr/bin/wget' }};
-                    }}
-
-                    function parseValue_{widget_id}(raw) {{
-                        try {{
-                            if (raw === 'true') return true;
-                            if (raw === 'false') return false;
-                            if (raw === 'null') return null;
-                            if (raw !== '' && !isNaN(raw)) return Number(raw);
-                            if ((raw.startsWith('{{') && raw.endsWith('}}')) ||
-                                (raw.startsWith('[') && raw.endsWith(']')) ||
-                                (raw.startsWith('"') && raw.endsWith('"'))) {{
-                                return JSON.parse(raw);
-                            }}
-                        }} catch (error) {{
-                            return raw;
-                        }}
-                        return raw;
-                    }}
-
-                    function sameValue_{widget_id}(left, right) {{
-                        return left === right || JSON.stringify(left) === JSON.stringify(right);
-                    }}
-
-                    function parseTypedValue_{widget_id}(raw, typeName, meta) {{
-                        var numberValue;
-                        var parsed;
-
-                        if (typeName && meta && meta.key && isRegexConfigKey_{widget_id}(meta.key)) {{
-                            return validateRegexConfig_{widget_id}(meta.key, raw, typeName);
-                        }}
-
-                        if (typeName === 'string' && meta && meta.key && meta.key.endsWith('_BINARY')) {{
-                            return validateBinaryValue_{widget_id}(raw);
-                        }}
-
-                        if (typeName === 'string') {{
-                            if (meta.pattern) {{
-                                try {{
-                                    if (!(new RegExp(meta.pattern)).test(raw)) {{
-                                        return {{ ok: false, value: raw, message: 'Must match pattern ' + meta.pattern }};
-                                    }}
-                                }} catch (error) {{}}
-                            }}
-                            return {{ ok: true, value: raw, message: '' }};
-                        }}
-
-                        if (typeName === 'integer') {{
-                            if (!/^-?\\d+$/.test(raw)) {{
-                                return {{ ok: false, value: raw, message: 'Must be an integer' }};
-                            }}
-                            numberValue = Number(raw);
-                            if (meta.minimum !== undefined && numberValue < meta.minimum) {{
-                                return {{ ok: false, value: numberValue, message: 'Must be at least ' + meta.minimum }};
-                            }}
-                            if (meta.maximum !== undefined && numberValue > meta.maximum) {{
-                                return {{ ok: false, value: numberValue, message: 'Must be at most ' + meta.maximum }};
-                            }}
-                            return {{ ok: true, value: numberValue, message: '' }};
-                        }}
-
-                        if (typeName === 'number') {{
-                            if (raw === '' || isNaN(raw)) {{
-                                return {{ ok: false, value: raw, message: 'Must be a number' }};
-                            }}
-                            numberValue = Number(raw);
-                            if (meta.minimum !== undefined && numberValue < meta.minimum) {{
-                                return {{ ok: false, value: numberValue, message: 'Must be at least ' + meta.minimum }};
-                            }}
-                            if (meta.maximum !== undefined && numberValue > meta.maximum) {{
-                                return {{ ok: false, value: numberValue, message: 'Must be at most ' + meta.maximum }};
-                            }}
-                            return {{ ok: true, value: numberValue, message: '' }};
-                        }}
-
-                        if (typeName === 'boolean') {{
-                            var lowered = raw.toLowerCase();
-                            if (lowered === 'true' || raw === '1') return {{ ok: true, value: true, message: '' }};
-                            if (lowered === 'false' || raw === '0') return {{ ok: true, value: false, message: '' }};
-                            return {{ ok: false, value: raw, message: 'Must be true or false' }};
-                        }}
-
-                        if (typeName === 'null') {{
-                            return raw === 'null'
-                                ? {{ ok: true, value: null, message: '' }}
-                                : {{ ok: false, value: raw, message: 'Must be null' }};
-                        }}
-
-                        if (typeName === 'array' || typeName === 'object') {{
-                            try {{
-                                parsed = JSON.parse(raw);
-                            }} catch (error) {{
-                                return {{ ok: false, value: raw, message: 'Must be valid JSON' }};
-                            }}
-
-                            if (typeName === 'array' && Array.isArray(parsed)) {{
-                                return {{ ok: true, value: parsed, message: '' }};
-                            }}
-                            if (typeName === 'object' && parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {{
-                                return {{ ok: true, value: parsed, message: '' }};
-                            }}
-
-                            return {{
-                                ok: false,
-                                value: parsed,
-                                message: typeName === 'array' ? 'Must be a JSON array' : 'Must be a JSON object',
-                            }};
-                        }}
-
-                        return {{ ok: true, value: parseValue_{widget_id}(raw), message: '' }};
-                    }}
-
-                    function validateValueAgainstMeta_{widget_id}(raw, meta) {{
-                        if (!meta || raw === '') {{
-                            return {{ state: 'neutral', value: raw, message: '' }};
-                        }}
-
-                        var enumValues = Array.isArray(meta.enum) ? meta.enum : [];
-                        var types = getTypes_{widget_id}(meta);
-                        if (!types.length) {{
-                            types = ['string'];
-                        }}
-
-                        var error = 'Invalid value';
-                        for (var i = 0; i < types.length; i++) {{
-                            var candidate = parseTypedValue_{widget_id}(raw, types[i], meta);
-                            if (!candidate.ok) {{
-                                error = candidate.message || error;
-                                continue;
-                            }}
-                            if (enumValues.length && !enumValues.some(function(enumValue) {{
-                                return sameValue_{widget_id}(enumValue, candidate.value) || stringifyValue_{widget_id}(enumValue) === raw;
-                            }})) {{
-                                error = 'Must be one of: ' + enumValues.map(stringifyValue_{widget_id}).join(', ');
-                                continue;
-                            }}
-                            return {{ state: 'valid', value: candidate.value, message: '' }};
-                        }}
-
-                        return {{ state: 'invalid', value: raw, message: error }};
-                    }}
-
-                    function ensureRowId_{widget_id}(row) {{
-                        if (!row.dataset.rowId) {{
-                            row.dataset.rowId = String(rowCounter_{widget_id}++);
-                        }}
-                        return row.dataset.rowId;
-                    }}
-
-                    function setRowHelp_{widget_id}(row) {{
-                        var keyInput = row.querySelector('.kv-key');
-                        var help = row.querySelector('.kv-help');
-                        if (!keyInput || !help) {{
-                            return;
-                        }}
-
-                        var key = keyInput.value.trim();
-                        if (!key) {{
-                            help.textContent = '';
-                            return;
-                        }}
-
-                        var meta = getMetaForKey_{widget_id}(key);
-                        if (meta) {{
-                            var extra = isRegexConfigKey_{widget_id}(key)
-                                ? ((meta.type === 'object' || (Array.isArray(meta.type) && meta.type.includes('object')))
-                                    ? ' Expected: JSON object with regex keys.'
-                                    : ' Expected: valid regex.')
-                                : '';
-                            var example = getExampleInput_{widget_id}(key, meta);
-                            help.textContent = [describeMeta_{widget_id}(meta) + extra, example].filter(Boolean).join(' ');
-                        }} else {{
-                            help.textContent = 'Custom key';
-                        }}
-                    }}
-
-                    function configureValueInput_{widget_id}(row) {{
-                        var keyInput = row.querySelector('.kv-key');
-                        var valueInput = row.querySelector('.kv-value');
-                        var datalist = row.querySelector('.kv-value-options');
-                        if (!keyInput || !valueInput || !datalist) {{
-                            return;
-                        }}
-
-                        var rowId = ensureRowId_{widget_id}(row);
-                        datalist.id = '{widget_id}_value_options_' + rowId;
-
-                        var meta = getMetaForKey_{widget_id}(keyInput.value.trim());
-                        var enumValues = Array.isArray(meta && meta.enum) ? meta.enum : [];
-                        var types = getTypes_{widget_id}(meta);
-                        if (!enumValues.length && types.includes('boolean')) {{
-                            enumValues = ['True', 'False'];
-                        }}
-                        if (enumValues.length) {{
-                            datalist.innerHTML = enumValues.map(function(enumValue) {{
-                                return '<option value="' + stringifyValue_{widget_id}(enumValue).replace(/"/g, '&quot;') + '"></option>';
-                            }}).join('');
-                            valueInput.setAttribute('list', datalist.id);
-                        }} else {{
-                            datalist.innerHTML = '';
-                            valueInput.removeAttribute('list');
-                        }}
-                    }}
-
-                    function setValueValidationState_{widget_id}(input, state, message) {{
-                        if (!input) {{
-                            return;
-                        }}
-
-                        if (state === 'valid') {{
-                            input.style.borderColor = '#2da44e';
-                            input.style.boxShadow = '0 0 0 1px rgba(45, 164, 78, 0.18)';
-                            input.style.backgroundColor = '#f6ffed';
-                        }} else if (state === 'invalid') {{
-                            input.style.borderColor = '#cf222e';
-                            input.style.boxShadow = '0 0 0 1px rgba(207, 34, 46, 0.18)';
-                            input.style.backgroundColor = '#fff8f8';
-                        }} else {{
-                            input.style.borderColor = '#ccc';
-                            input.style.boxShadow = 'none';
-                            input.style.backgroundColor = '';
-                        }}
-                        input.title = message || '';
-                    }}
-
-                    function applyValueValidation_{widget_id}(row) {{
-                        var keyInput = row.querySelector('.kv-key');
-                        var valueInput = row.querySelector('.kv-value');
-                        if (!keyInput || !valueInput) {{
-                            return;
-                        }}
-
-                        var key = keyInput.value.trim();
-                        if (!key) {{
-                            setValueValidationState_{widget_id}(valueInput, 'neutral', '');
-                            return;
-                        }}
-
-                        var meta = getMetaForKey_{widget_id}(key);
-                        if (!meta) {{
-                            setValueValidationState_{widget_id}(valueInput, 'neutral', '');
-                            return;
-                        }}
-
-                        var validation = validateValueAgainstMeta_{widget_id}(valueInput.value.trim(), meta);
-                        setValueValidationState_{widget_id}(valueInput, validation.state, validation.message);
-                    }}
-
-                    function coerceValueForStorage_{widget_id}(key, raw) {{
-                        var meta = getMetaForKey_{widget_id}(key);
-                        if (!meta) {{
-                            return parseValue_{widget_id}(raw);
-                        }}
-
-                        var validation = validateValueAgainstMeta_{widget_id}(raw, meta);
-                        return validation.state === 'valid' ? validation.value : raw;
-                    }}
-
-                    function initializeRows_{widget_id}() {{
-                        var container = document.getElementById('{widget_id}_rows');
-                        container.querySelectorAll('.key-value-row').forEach(function(row) {{
-                            ensureRowId_{widget_id}(row);
-                            configureValueInput_{widget_id}(row);
-                            setRowHelp_{widget_id}(row);
-                            applyValueValidation_{widget_id}(row);
-                        }});
-                    }}
-
-                    function updateHiddenField_{widget_id}() {{
-                        var container = document.getElementById('{widget_id}_rows');
-                        var rows = container.querySelectorAll('.key-value-row');
-                        var result = {{}};
-                        rows.forEach(function(row) {{
-                            var keyInput = row.querySelector('.kv-key');
-                            var valInput = row.querySelector('.kv-value');
-                            if (keyInput && valInput && keyInput.value.trim()) {{
-                                var key = keyInput.value.trim();
-                                var val = valInput.value.trim();
-                                result[key] = coerceValueForStorage_{widget_id}(key, val);
-                            }}
-                        }});
-                        document.getElementById('{widget_id}').value = JSON.stringify(result);
-                    }}
-
-                    window.addKeyValueRow_{widget_id} = function() {{
-                        var container = document.getElementById('{widget_id}_rows');
-                        var newRow = document.createElement('div');
-                        newRow.className = 'key-value-row';
-                        newRow.style.cssText = 'margin-bottom: 6px;';
-                        newRow.innerHTML = '<div class="kv-inputs" style="display: flex; gap: 8px; align-items: center;">' +
-                            '<input type="text" class="kv-key" placeholder="KEY" list="{widget_id}_keys" ' +
-                            'style="flex: 1; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">' +
-                            '<input type="text" class="kv-value" placeholder="value" ' +
-                            'style="flex: 2; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">' +
-                            '<datalist class="kv-value-options"></datalist>' +
-                            '<button type="button" onclick="removeKeyValueRow_{widget_id}(this)" ' +
-                            'style="padding: 4px 10px; cursor: pointer; background: #ba2121; color: white; border: none; border-radius: 4px; font-weight: bold;">−</button>' +
-                            '</div>' +
-                            '<div class="kv-help" style="margin-top: 4px; font-size: 11px; color: #666; font-style: italic;"></div>';
-                        container.appendChild(newRow);
-                        ensureRowId_{widget_id}(newRow);
-                        configureValueInput_{widget_id}(newRow);
-                        setRowHelp_{widget_id}(newRow);
-                        applyValueValidation_{widget_id}(newRow);
-                        updateHiddenField_{widget_id}();
-                        newRow.querySelector('.kv-key').focus();
-                    }};
-
-                    window.removeKeyValueRow_{widget_id} = function(btn) {{
-                        var row = btn.closest('.key-value-row');
-                        row.remove();
-                        updateHiddenField_{widget_id}();
-                    }};
-
-                    window.updateHiddenField_{widget_id} = updateHiddenField_{widget_id};
-
-                    function configRowForKey_{widget_id}(key) {{
-                        var container = document.getElementById('{widget_id}_rows');
-                        if (!container) {{
-                            return null;
-                        }}
-                        var match = null;
-                        container.querySelectorAll('.key-value-row').forEach(function(row) {{
-                            if (match) {{ return; }}
-                            var keyInput = row.querySelector('.kv-key');
-                            if (keyInput && keyInput.value.trim() === key) {{
-                                match = row;
-                            }}
-                        }});
-                        if (!match) {{
-                            window.addKeyValueRow_{widget_id}();
-                            var rows = container.querySelectorAll('.key-value-row');
-                            match = rows[rows.length - 1];
-                            var keyInput = match.querySelector('.kv-key');
-                            if (keyInput) {{
-                                keyInput.value = key;
-                                keyInput.dispatchEvent(new Event('input', {{ bubbles: true }}));
-                            }}
-                        }}
-                        return match;
-                    }}
-
-                    function prefillConfigFromQuery_{widget_id}() {{
-                        var params = new URLSearchParams(window.location.search);
-                        var consumedConfigKeys = [];
-                        params.forEach(function(value, key) {{
-                            if (!/^[A-Z][A-Z0-9_]*$/.test(key) || !configMeta_{widget_id}[key]) {{
-                                return;
-                            }}
-                            consumedConfigKeys.push(key);
-                            var match = configRowForKey_{widget_id}(key);
-                            var valueInput = match && match.querySelector('.kv-value');
-                            if (valueInput) {{
-                                valueInput.value = value;
-                                valueInput.dispatchEvent(new Event('input', {{ bubbles: true }}));
-                            }}
-                        }});
-                        updateHiddenField_{widget_id}();
-                        if (consumedConfigKeys.length) {{
-                            var cleanUrl = new URL(window.location.href);
-                            consumedConfigKeys.forEach(function(key) {{ cleanUrl.searchParams.delete(key); }});
-                            window.history.replaceState(null, '', cleanUrl.pathname + cleanUrl.search + cleanUrl.hash);
-                        }}
-                    }}
-
-                    function focusConfigKeyFromHash_{widget_id}() {{
-                        // Deep-link affordance: ``…/change/#SOME_KEY`` jumps directly
-                        // to (or creates) the matching row in this editor. Used by
-                        // the setup wizard and live-config detail pages.
-                        var hash = (window.location.hash || '').replace(/^#/, '').trim();
-                        if (!hash || !/^[A-Z][A-Z0-9_]*$/.test(hash)) {{
-                            return;
-                        }}
-                        var match = configRowForKey_{widget_id}(hash);
-                        if (!match) {{
-                            return;
-                        }}
-                        match.scrollIntoView({{ behavior: 'smooth', block: 'center' }});
-                        var prevOutline = match.style.outline;
-                        match.style.outline = '2px solid #f59e0b';
-                        match.style.outlineOffset = '2px';
-                        match.style.transition = 'outline 1.2s ease-out';
-                        setTimeout(function() {{
-                            match.style.outline = prevOutline || 'none';
-                        }}, 1400);
-                        var valueInput = match.querySelector('.kv-value');
-                        if (valueInput) {{
-                            valueInput.focus();
-                            try {{ valueInput.setSelectionRange(valueInput.value.length, valueInput.value.length); }} catch (e) {{}}
-                        }}
-                    }}
-
-                    // Initialize on load
-                    document.addEventListener('DOMContentLoaded', function() {{
-                        initializeRows_{widget_id}();
-                        prefillConfigFromQuery_{widget_id}();
-                        updateHiddenField_{widget_id}();
-                        focusConfigKeyFromHash_{widget_id}();
-                    }});
-                    // Also run immediately in case DOM is already ready
-                    if (document.readyState !== 'loading') {{
-                        initializeRows_{widget_id}();
-                        prefillConfigFromQuery_{widget_id}();
-                        updateHiddenField_{widget_id}();
-                        focusConfigKeyFromHash_{widget_id}();
-                    }}
-
-                    window.addEventListener('hashchange', focusConfigKeyFromHash_{widget_id});
-
-                    // Update on any input change
-                    var rowsEl_{widget_id} = document.getElementById('{widget_id}_rows');
-
-                    rowsEl_{widget_id}.addEventListener('input', function(event) {{
-                        var row = event.target.closest('.key-value-row');
-                        if (!row) {{
-                            return;
-                        }}
-
-                        if (event.target.classList.contains('kv-key')) {{
-                            configureValueInput_{widget_id}(row);
-                            setRowHelp_{widget_id}(row);
-                        }}
-
-                        if (event.target.classList.contains('kv-key') || event.target.classList.contains('kv-value')) {{
-                            applyValueValidation_{widget_id}(row);
-                            updateHiddenField_{widget_id}();
-                        }}
-                    }});
-                }})();
-            </script>
-        </div>
-        '''
-        return mark_safe(html)
-
     def _render_row(self, widget_id: str, key: str, value: str) -> str:
         from archivebox.config.common import is_sensitive_config_key
 
@@ -872,6 +269,28 @@ class BaseModelAdmin(DjangoObjectActions, admin.ModelAdmin):
     show_search_mode_selector = False
     change_form_template = "admin/archivebox_change_form.html"
 
+    @admin.display(description="Health", ordering="health")
+    def health_display(self, obj):
+        h = obj.health
+        color = "green" if h >= 80 else "orange" if h >= 50 else "red"
+        return format_html('<span style="color: {};">{}</span>', color, h)
+
+    def get_ordering_fields(self, request):
+        ordering = request.GET.get("o")
+        if not ordering:
+            return set()
+        fields = set()
+        for part in ordering.split("."):
+            if not part:
+                continue
+            try:
+                idx = abs(int(part)) - 1
+            except ValueError:
+                continue
+            if 0 <= idx < len(self.list_display):
+                fields.add(self.list_display[idx])
+        return fields
+
     def get_admin_toolbar_actions(self, request, obj):
         """Return extra action button dicts for the shared change-form toolbar.
 

archivebox/base_models/models.py

@@ -4,7 +4,7 @@ __package__ = "archivebox.base_models"
 
 import json
 import shutil
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 from archivebox.uuid_compat import CompactUUIDField, uuid7
 from pathlib import Path
@@ -23,6 +23,10 @@ from django_stubs_ext.db.models import TypedModelMeta
 from archivebox.config import CONSTANTS
 
 
+if TYPE_CHECKING:
+    from archivebox.config.common import ArchiveBoxBaseConfig
+
+
 def normalize_config_json_values(config: Any) -> Any:
     if not isinstance(config, dict):
         return config
@@ -55,17 +59,6 @@ def get_or_create_system_user_pk(username="system"):
     return user.pk
 
 
-class AutoDateTimeField(models.DateTimeField):
-    """DateTimeField that automatically updates on save (legacy compatibility)."""
-
-    def pre_save(self, model_instance, add):
-        if add or self.attname not in model_instance.__dict__ or not model_instance.__dict__[self.attname]:
-            value = timezone.now()
-            setattr(model_instance, self.attname, value)
-            return value
-        return super().pre_save(model_instance, add)
-
-
 class ModelWithUUID(models.Model):
     id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
@@ -138,6 +131,7 @@ class ModelWithHealthStats(models.Model):
 class ModelWithConfig(models.Model):
     """Mixin for models with a JSON config field."""
 
+    _runtime_config: "ArchiveBoxBaseConfig"
     config = models.JSONField(default=dict, null=True, blank=True, editable=True)
 
     class Meta(TypedModelMeta):

archivebox/cli/archivebox_add.py

@@ -6,7 +6,6 @@ __command__ = "archivebox add"
 
 import sys
 import os
-import json
 from pathlib import Path
 
 from typing import Any, TYPE_CHECKING
@@ -71,17 +70,7 @@ def add(
     created_by_id: int | None = None,
     config: dict[str, Any] | None = None,
 ) -> tuple[Crawl, QuerySet[Snapshot]]:
-    """Add a new URL or list of URLs to your archive.
-
-    The flow is:
-    1. Save URLs to sources file
-    2. Create Crawl with URLs and max_depth
-    3. Crawl runner creates Snapshots from Crawl URLs (depth=0)
-    4. Crawl runner runs parser extractors on root snapshots
-    5. Parser extractors output to urls.jsonl
-    6. URLs are added to Crawl.urls and child Snapshots are created
-    7. Repeat until max_depth is reached
-    """
+    """Add a URL list or imported URL document to a new Crawl."""
 
     from rich import print
 
@@ -125,12 +114,12 @@ def add(
     created_by_id = created_by_id or get_or_create_system_user_pk()
     started_at = timezone.now()
 
-    use_internal_input_root = isinstance(urls, str)
-    source_text = (
-        urls
-        if use_internal_input_root
-        else "\n".join(json.dumps({"type": "CrawlSeed", "url": str(url), "depth": 0}, separators=(",", ":")) for url in urls)
-    )
+    if isinstance(urls, str):
+        source_text = urls
+    else:
+        from archivebox.misc.util import validate_url
+
+        source_text = "\n".join(validate_url(str(url)) for url in urls)
 
     # 2. Create a new Crawl with inline URLs
     # Foreground add must claim runner ownership before publishing runnable
@@ -187,12 +176,7 @@ def add(
     try:
         crawl = Crawl.objects.create(
             urls=source_text,
-            # Stdin/import text gets an extra hop because the synthetic
-            # archivebox://internal root lives at depth 0 and parser-discovered
-            # URLs land at depth 1; direct URL args become the depth=0 input
-            # snapshots themselves so --depth=N matches the deepest hop the user
-            # asked for.
-            max_depth=depth + 1 if use_internal_input_root else depth,
+            max_depth=depth,
             tags_str=tag,
             persona_id=persona_obj.id,
             label=f"{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]",
@@ -213,9 +197,8 @@ def add(
     print(f"[green]\\[+] Created Crawl {crawl.id} with max_depth={depth}[/green]")
     print(f"    [dim]First URL: {first_url}[/dim]")
 
-    # 3. The CrawlMachine will create Snapshots from all URLs when started
-    #    Parser extractors run on snapshots and discover more URLs
-    #    Discovered URLs become child Snapshots (depth+1)
+    # The runner parses Crawl.urls after claiming the Crawl, then persists the
+    # resulting depth-0 Snapshot facts. Recursive discoveries start at depth 1.
 
     if index_only:
         print("[yellow]\\[*] Index-only mode - URLs queued, runner not started[/yellow]")
@@ -421,13 +404,13 @@ def main(**kwargs):
         if only_new is not None:
             kwargs["config"] = {"ONLY_NEW": bool(only_new)}
         if extract:
-            from abx_dl.models import discover_plugins, plugins_matching_output
+            from archivebox.plugins.discovery import get_plugin_catalog
 
-            all_plugins = discover_plugins()
+            catalog = get_plugin_catalog()
             tokens = [token.strip() for token in extract.split(",") if token.strip()]
-            plugin_names = {name.lower(): name for name in all_plugins}
+            plugin_names = {name.lower(): name for name in catalog}
             selected = [plugin_names[token.lower()] for token in tokens if token.lower() in plugin_names]
-            selected += plugins_matching_output(all_plugins, tokens)
+            selected += catalog.matching_output(tokens)
             if not selected:
                 raise click.UsageError(f"No plugins found matching extract types: {extract}")
             existing = [token.strip() for token in (kwargs.get("plugins") or "").split(",") if token.strip()]

archivebox/cli/archivebox_archiveresult.py

@@ -6,13 +6,13 @@ archivebox archiveresult <action> [args...] [--filters]
 Manage ArchiveResult records (plugin extraction results).
 
 Actions:
-    create  - Create ArchiveResults for Snapshots (queue extractions)
+    create  - Emit plugin extraction request records for Snapshots
     list    - List ArchiveResults as JSONL (with optional filters)
     update  - Update ArchiveResults from stdin JSONL
     delete  - Delete ArchiveResults from stdin JSONL
 
 Examples:
-    # Create ArchiveResults for snapshots (queue for extraction)
+    # Emit extraction requests; `archivebox run` schedules their parent snapshots
     archivebox snapshot list --status=queued | archivebox archiveresult create
     archivebox archiveresult create --plugin=screenshot --snapshot-id=<uuid>
 
@@ -20,9 +20,6 @@ Examples:
     archivebox archiveresult list --status=failed
     archivebox archiveresult list --plugin=screenshot --status=succeeded
 
-    # Update (reset failed extractions to queued)
-    archivebox archiveresult list --status=failed | archivebox archiveresult update --status=queued
-
     # Delete
     archivebox archiveresult list --plugin=singlefile | archivebox archiveresult delete --yes
 
@@ -67,7 +64,7 @@ def create_archiveresults(
     Reads Snapshot records from stdin and emits ArchiveResult request JSONL.
     Pass-through: Non-Snapshot/ArchiveResult records are output unchanged.
     If --plugin is specified, only emits requests for that plugin.
-    Otherwise, emits one request for each enabled Snapshot plugin.
+    Otherwise, emits requests for all enabled snapshot hooks.
 
     Exit codes:
         0: Success
@@ -75,7 +72,7 @@ def create_archiveresults(
     """
     from archivebox.config.common import get_config
     from archivebox.plugins.hooks import discover_hooks
-    from archivebox.misc.jsonl import read_stdin, write_record, TYPE_SNAPSHOT, TYPE_ARCHIVERESULT
+    from archivebox.misc.jsonl import read_stdin, write_record, TYPE_SNAPSHOT
     from archivebox.core.models import Snapshot
 
     is_tty = sys.stdout.isatty()
@@ -102,26 +99,9 @@ def create_archiveresults(
         for record in records:
             record_type = record.get("type", "")
 
-            if record_type == TYPE_SNAPSHOT:
-                # Pass through the Snapshot record itself
+            if record_type:
                 pass_through_records.append(record)
-                if record.get("id"):
-                    snapshot_ids.append(record["id"])
-
-            elif record_type == TYPE_ARCHIVERESULT:
-                # ArchiveResult records: pass through if they have an id
-                if record.get("id"):
-                    pass_through_records.append(record)
-                # If no id, we could create it, but for now just pass through
-                else:
-                    pass_through_records.append(record)
-
-            elif record_type:
-                # Other typed records (Crawl, Tag, etc): pass through
-                pass_through_records.append(record)
-
-            elif record.get("id"):
-                # Untyped record with id - assume it's a snapshot ID
+            if record.get("id") and record_type in ("", TYPE_SNAPSHOT):
                 snapshot_ids.append(record["id"])
 
         # Output pass-through records first
@@ -144,19 +124,20 @@ def create_archiveresults(
 
     created_count = 0
     for snapshot in snapshots:
-        if plugin:
+        config = get_config(crawl=snapshot.crawl, snapshot=snapshot)
+        hooks = [
+            hook
+            for hook in discover_hooks("Snapshot", filter_disabled=not plugin, config=config)
+            if not plugin or hook.parent.name == plugin
+        ]
+        for hook_path in hooks:
+            hook_name = hook_path.stem
+            plugin_name = hook_path.parent.name
             if not is_tty:
-                write_record(build_archiveresult_request(snapshot.id, plugin, status=status))
+                write_record(build_archiveresult_request(snapshot.id, plugin_name, hook_name=hook_name, status=status))
             created_count += 1
-        else:
-            config = get_config(crawl=snapshot.crawl, snapshot=snapshot)
-            hooks = discover_hooks("Snapshot", filter_disabled=True, config=config)
-            for plugin_name in dict.fromkeys(hook_path.parent.name for hook_path in hooks):
-                if not is_tty:
-                    write_record(build_archiveresult_request(snapshot.id, plugin_name, status=status))
-                created_count += 1
 
-    rprint(f"[green]Created {created_count} archive result request records[/green]", file=sys.stderr)
+    rprint(f"[green]Created {created_count} extraction request records[/green]", file=sys.stderr)
     return 0
 
 
@@ -171,48 +152,20 @@ def list_archiveresults(
     snapshot_id: str | None = None,
     limit: int | None = None,
 ) -> int:
-    """
-    List ArchiveResults as JSONL with optional filters.
-
-    Exit codes:
-        0: Success (even if no results)
-    """
-    from archivebox.misc.jsonl import write_record
+    """List archive results as JSONL, or formatted rows in a terminal."""
     from archivebox.core.models import ArchiveResult
+    from archivebox.cli.cli_util import list_records, format_status
 
-    is_tty = sys.stdout.isatty()
-
-    queryset = ArchiveResult.objects.all().order_by("-start_ts")
-
-    # Apply filters
-    filter_kwargs = {
-        "status": status,
-        "plugin": plugin,
-        "snapshot_id": snapshot_id,
-    }
-    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
-
-    count = 0
-    for result in queryset:
-        if is_tty:
-            status_color = {
-                "queued": "yellow",
-                "started": "blue",
-                "succeeded": "green",
-                "failed": "red",
-                "skipped": "dim",
-                "noresults": "dim",
-                "backoff": "magenta",
-            }.get(result.status, "dim")
-            rprint(
-                f"[{status_color}]{result.status:10}[/{status_color}] {result.plugin:15} [dim]{result.id}[/dim] {result.snapshot.url[:40]}",
-            )
-        else:
-            write_record(result.to_json())
-        count += 1
-
-    rprint(f"[dim]Listed {count} archive results[/dim]", file=sys.stderr)
-    return 0
+    queryset = apply_filters(
+        ArchiveResult.objects.order_by("-start_ts"),
+        {"status": status, "plugin": plugin, "snapshot_id": snapshot_id},
+        limit=limit,
+    )
+    return list_records(
+        queryset,
+        plural="archive results",
+        render=lambda result: f"{format_status(result.status, 10)} {result.plugin:15} [dim]{result.id}[/dim] {result.snapshot.url[:40]}",
+    )
 
 
 # =============================================================================
@@ -220,56 +173,17 @@ def list_archiveresults(
 # =============================================================================
 
 
-def update_archiveresults(
-    status: str | None = None,
-) -> int:
-    """
-    Update ArchiveResults from stdin JSONL.
-
-    Reads ArchiveResult records from stdin and applies updates.
-    Uses PATCH semantics - only specified fields are updated.
-
-    Exit codes:
-        0: Success
-        1: No input or error
-    """
-    from archivebox.misc.jsonl import read_stdin, write_record
+def update_archiveresults(status: str | None = None) -> int:
+    """Apply supplied fields to each JSONL-selected ArchiveResult."""
     from archivebox.core.models import ArchiveResult
+    from archivebox.cli.cli_util import update_records
 
-    is_tty = sys.stdout.isatty()
+    def update(archiveresult):
+        if status:
+            archiveresult.status = status
+        archiveresult.save()
 
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    updated_count = 0
-    for record in records:
-        result_id = record.get("id")
-        if not result_id:
-            continue
-
-        try:
-            result = ArchiveResult.objects.get(id=result_id)
-
-            # Apply updates from CLI flags
-            if status:
-                if status == ArchiveResult.StatusChoices.QUEUED:
-                    result.reset_for_retry()
-                else:
-                    result.status = status
-                    result.save(update_fields=["status", "modified_at"])
-            updated_count += 1
-
-            if not is_tty:
-                write_record(result.to_json())
-
-        except ArchiveResult.DoesNotExist:
-            rprint(f"[yellow]ArchiveResult not found: {result_id}[/yellow]", file=sys.stderr)
-            continue
-
-    rprint(f"[green]Updated {updated_count} archive results[/green]", file=sys.stderr)
-    return 0
+    return update_records(ArchiveResult, update, plural="archive results")
 
 
 # =============================================================================
@@ -278,52 +192,19 @@ def update_archiveresults(
 
 
 def delete_archiveresults(yes: bool = False, dry_run: bool = False) -> int:
-    """
-    Delete ArchiveResults from stdin JSONL.
-
-    Requires --yes flag to confirm deletion.
-
-    Exit codes:
-        0: Success
-        1: No input or missing --yes flag
-    """
-    from archivebox.misc.jsonl import read_stdin
+    """Delete archive results selected by stdin JSONL; --yes confirms, --dry-run previews."""
+    from archivebox.cli.cli_util import delete_records
     from archivebox.core.models import ArchiveResult
 
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    result_ids = [r.get("id") for r in records if r.get("id")]
-
-    if not result_ids:
-        rprint("[yellow]No valid archive result IDs in input[/yellow]", file=sys.stderr)
-        return 1
-
-    results = ArchiveResult.objects.filter(id__in=result_ids)
-    count = results.count()
-
-    if count == 0:
-        rprint("[yellow]No matching archive results found[/yellow]", file=sys.stderr)
-        return 0
-
-    if dry_run:
-        rprint(f"[yellow]Would delete {count} archive results (dry run)[/yellow]", file=sys.stderr)
-        for result in results[:10]:
-            rprint(f"  [dim]{result.id}[/dim] {result.plugin} {result.snapshot.url[:40]}", file=sys.stderr)
-        if count > 10:
-            rprint(f"  ... and {count - 10} more", file=sys.stderr)
-        return 0
-
-    if not yes:
-        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
-        return 1
-
-    # Perform deletion
-    deleted_count, _ = results.delete()
-    rprint(f"[green]Deleted {deleted_count} archive results[/green]", file=sys.stderr)
-    return 0
+    return delete_records(
+        ArchiveResult,
+        label="archive result",
+        plural="archive results",
+        preview=lambda obj: f"[dim]{obj.id}[/dim] {obj.plugin} {obj.snapshot.url[:40]}",
+        yes=yes,
+        dry_run=dry_run,
+        preview_limit=10,
+    )
 
 
 # =============================================================================
@@ -341,9 +222,9 @@ def main():
 @click.option("--snapshot-id", help="Snapshot ID to create results for")
 @click.option("--plugin", "-p", help="Plugin name (e.g., screenshot, singlefile)")
 @click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
-def create_cmd(snapshot_id: str | None, plugin: str | None, status: str):
-    """Create ArchiveResults for Snapshots from stdin JSONL."""
-    sys.exit(create_archiveresults(snapshot_id=snapshot_id, plugin=plugin, status=status))
+def create_cmd(**kwargs):
+    """Emit Snapshot plugin extraction requests as JSONL."""
+    sys.exit(create_archiveresults(**kwargs))
 
 
 @main.command("list")
@@ -351,36 +232,24 @@ def create_cmd(snapshot_id: str | None, plugin: str | None, status: str):
 @click.option("--plugin", "-p", help="Filter by plugin name")
 @click.option("--snapshot-id", help="Filter by snapshot ID")
 @click.option("--limit", "-n", type=int, help="Limit number of results")
-def list_cmd(
-    status: str | None,
-    plugin: str | None,
-    snapshot_id: str | None,
-    limit: int | None,
-):
+def list_cmd(**kwargs):
     """List ArchiveResults as JSONL."""
-    sys.exit(
-        list_archiveresults(
-            status=status,
-            plugin=plugin,
-            snapshot_id=snapshot_id,
-            limit=limit,
-        ),
-    )
+    sys.exit(list_archiveresults(**kwargs))
 
 
 @main.command("update")
 @click.option("--status", "-s", help="Set status")
-def update_cmd(status: str | None):
+def update_cmd(**kwargs):
     """Update ArchiveResults from stdin JSONL."""
-    sys.exit(update_archiveresults(status=status))
+    sys.exit(update_archiveresults(**kwargs))
 
 
 @main.command("delete")
 @click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
 @click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(yes: bool, dry_run: bool):
+def delete_cmd(**kwargs):
     """Delete ArchiveResults from stdin JSONL."""
-    sys.exit(delete_archiveresults(yes=yes, dry_run=dry_run))
+    sys.exit(delete_archiveresults(**kwargs))
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_binary.py

@@ -114,37 +114,20 @@ def list_binaries(
     version__icontains: str | None = None,
     limit: int | None = None,
 ) -> int:
-    """
-    List Binaries as JSONL with optional filters.
-
-    Exit codes:
-        0: Success (even if no results)
-    """
-    from archivebox.misc.jsonl import write_record
+    """List binaries as JSONL, or formatted rows in a terminal."""
     from archivebox.machine.models import Binary
+    from archivebox.cli.cli_util import list_records
 
-    is_tty = sys.stdout.isatty()
-
-    queryset = Binary.objects.all().order_by("name", "-modified_at", "-created_at")
-
-    # Apply filters
-    filter_kwargs = {
-        "name": name,
-        "abspath__icontains": abspath__icontains,
-        "version__icontains": version__icontains,
-    }
-    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
-
-    count = 0
-    for binary in queryset:
-        if is_tty:
-            rprint(f"[cyan]{binary.name:20}[/cyan] [dim]{binary.version:15}[/dim] {binary.abspath}")
-        else:
-            write_record(binary.to_json())
-        count += 1
-
-    rprint(f"[dim]Listed {count} binaries[/dim]", file=sys.stderr)
-    return 0
+    queryset = apply_filters(
+        Binary.objects.order_by("name", "-modified_at", "-created_at"),
+        {"name": name, "abspath__icontains": abspath__icontains, "version__icontains": version__icontains},
+        limit=limit,
+    )
+    return list_records(
+        queryset,
+        plural="binaries",
+        render=lambda binary: f"[cyan]{binary.name:20}[/cyan] [dim]{binary.version:15}[/dim] {binary.abspath}",
+    )
 
 
 # =============================================================================
@@ -152,57 +135,19 @@ def list_binaries(
 # =============================================================================
 
 
-def update_binaries(
-    version: str | None = None,
-    abspath: str | None = None,
-) -> int:
-    """
-    Update Binaries from stdin JSONL.
-
-    Reads Binary records from stdin and applies updates.
-    Uses PATCH semantics - only specified fields are updated.
-
-    Exit codes:
-        0: Success
-        1: No input or error
-    """
-    from archivebox.misc.jsonl import read_stdin, write_record
+def update_binaries(version: str | None = None, abspath: str | None = None) -> int:
+    """Apply supplied fields to each JSONL-selected Binary."""
     from archivebox.machine.models import Binary
+    from archivebox.cli.cli_util import update_records
 
-    is_tty = sys.stdout.isatty()
+    def update(binary):
+        if version:
+            binary.version = version
+        if abspath:
+            binary.abspath = abspath
+        binary.save()
 
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    updated_count = 0
-    for record in records:
-        binary_id = record.get("id")
-        if not binary_id:
-            continue
-
-        try:
-            binary = Binary.objects.get(id=binary_id)
-
-            # Apply updates from CLI flags
-            if version:
-                binary.version = version
-            if abspath:
-                binary.abspath = abspath
-
-            binary.save()
-            updated_count += 1
-
-            if not is_tty:
-                write_record(binary.to_json())
-
-        except Binary.DoesNotExist:
-            rprint(f"[yellow]Binary not found: {binary_id}[/yellow]", file=sys.stderr)
-            continue
-
-    rprint(f"[green]Updated {updated_count} binaries[/green]", file=sys.stderr)
-    return 0
+    return update_records(Binary, update, plural="binaries")
 
 
 # =============================================================================
@@ -211,50 +156,18 @@ def update_binaries(
 
 
 def delete_binaries(yes: bool = False, dry_run: bool = False) -> int:
-    """
-    Delete Binaries from stdin JSONL.
-
-    Requires --yes flag to confirm deletion.
-
-    Exit codes:
-        0: Success
-        1: No input or missing --yes flag
-    """
-    from archivebox.misc.jsonl import read_stdin
+    """Delete binaries selected by stdin JSONL; --yes confirms, --dry-run previews."""
+    from archivebox.cli.cli_util import delete_records
     from archivebox.machine.models import Binary
 
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    binary_ids = [r.get("id") for r in records if r.get("id")]
-
-    if not binary_ids:
-        rprint("[yellow]No valid binary IDs in input[/yellow]", file=sys.stderr)
-        return 1
-
-    binaries = Binary.objects.filter(id__in=binary_ids)
-    count = binaries.count()
-
-    if count == 0:
-        rprint("[yellow]No matching binaries found[/yellow]", file=sys.stderr)
-        return 0
-
-    if dry_run:
-        rprint(f"[yellow]Would delete {count} binaries (dry run)[/yellow]", file=sys.stderr)
-        for binary in binaries:
-            rprint(f"  {binary.name} {binary.abspath}", file=sys.stderr)
-        return 0
-
-    if not yes:
-        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
-        return 1
-
-    # Perform deletion
-    deleted_count, _ = binaries.delete()
-    rprint(f"[green]Deleted {deleted_count} binaries[/green]", file=sys.stderr)
-    return 0
+    return delete_records(
+        Binary,
+        label="binary",
+        plural="binaries",
+        preview=lambda obj: f"{obj.name} {obj.abspath}",
+        yes=yes,
+        dry_run=dry_run,
+    )
 
 
 # =============================================================================
@@ -272,9 +185,9 @@ def main():
 @click.option("--name", "-n", required=True, help="Binary name (e.g., chrome, wget)")
 @click.option("--abspath", "-p", required=True, help="Absolute path to binary")
 @click.option("--version", "-v", default="", help="Binary version")
-def create_cmd(name: str, abspath: str, version: str):
+def create_cmd(**kwargs):
     """Create/register a Binary."""
-    sys.exit(create_binary(name=name, abspath=abspath, version=version))
+    sys.exit(create_binary(**kwargs))
 
 
 @main.command("list")
@@ -282,37 +195,25 @@ def create_cmd(name: str, abspath: str, version: str):
 @click.option("--abspath__icontains", help="Filter by path contains")
 @click.option("--version__icontains", help="Filter by version contains")
 @click.option("--limit", type=int, help="Limit number of results")
-def list_cmd(
-    name: str | None,
-    abspath__icontains: str | None,
-    version__icontains: str | None,
-    limit: int | None,
-):
+def list_cmd(**kwargs):
     """List Binaries as JSONL."""
-    sys.exit(
-        list_binaries(
-            name=name,
-            abspath__icontains=abspath__icontains,
-            version__icontains=version__icontains,
-            limit=limit,
-        ),
-    )
+    sys.exit(list_binaries(**kwargs))
 
 
 @main.command("update")
 @click.option("--version", "-v", help="Set version")
 @click.option("--abspath", "-p", help="Set path")
-def update_cmd(version: str | None, abspath: str | None):
+def update_cmd(**kwargs):
     """Update Binaries from stdin JSONL."""
-    sys.exit(update_binaries(version=version, abspath=abspath))
+    sys.exit(update_binaries(**kwargs))
 
 
 @main.command("delete")
 @click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
 @click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(yes: bool, dry_run: bool):
+def delete_cmd(**kwargs):
     """Delete Binaries from stdin JSONL."""
-    sys.exit(delete_binaries(yes=yes, dry_run=dry_run))
+    sys.exit(delete_binaries(**kwargs))
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_crawl.py

@@ -177,43 +177,20 @@ def list_crawls(
     max_depth: int | None = None,
     limit: int | None = None,
 ) -> int:
-    """
-    List Crawls as JSONL with optional filters.
-
-    Exit codes:
-        0: Success (even if no results)
-    """
-    from archivebox.misc.jsonl import write_record
+    """List crawls as JSONL, or formatted rows in a terminal."""
     from archivebox.crawls.models import Crawl
+    from archivebox.cli.cli_util import list_records, format_status
 
-    is_tty = sys.stdout.isatty()
-
-    queryset = Crawl.objects.all().order_by("-created_at")
-
-    # Apply filters
-    filter_kwargs = {
-        "status": status,
-        "urls__icontains": urls__icontains,
-        "max_depth": max_depth,
-    }
-    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
-
-    count = 0
-    for crawl in queryset:
-        if is_tty:
-            status_color = {
-                "queued": "yellow",
-                "started": "blue",
-                "sealed": "green",
-            }.get(crawl.status, "dim")
-            url_preview = crawl.urls[:50].replace("\n", " ")
-            rprint(f"[{status_color}]{crawl.status:8}[/{status_color}] [dim]{crawl.id}[/dim] {url_preview}...")
-        else:
-            write_record(crawl.to_json())
-        count += 1
-
-    rprint(f"[dim]Listed {count} crawls[/dim]", file=sys.stderr)
-    return 0
+    queryset = apply_filters(
+        Crawl.objects.order_by("-created_at"),
+        {"status": status, "urls__icontains": urls__icontains, "max_depth": max_depth},
+        limit=limit,
+    )
+    return list_records(
+        queryset,
+        plural="crawls",
+        render=lambda crawl: f"{format_status(crawl.status, 8)} [dim]{crawl.id}[/dim] {crawl.urls[:50].replace(chr(10), chr(32))}...",
+    )
 
 
 # =============================================================================
@@ -221,73 +198,26 @@ def list_crawls(
 # =============================================================================
 
 
-def update_crawls(
-    status: str | None = None,
-    max_depth: int | None = None,
-) -> int:
-    """
-    Update Crawls from stdin JSONL.
-
-    Reads Crawl records from stdin and applies updates.
-    Uses PATCH semantics - only specified fields are updated.
-
-    Exit codes:
-        0: Success
-        1: No input or error
-    """
+def update_crawls(status: str | None = None, max_depth: int | None = None) -> int:
+    """Update JSONL-selected crawls through their lifecycle operations."""
     from django.utils import timezone
-
-    from archivebox.misc.jsonl import read_stdin, write_record
     from archivebox.crawls.models import Crawl
+    from archivebox.cli.cli_util import update_records, update_record_status
 
-    is_tty = sys.stdout.isatty()
+    def update(crawl):
+        if status:
+            try:
+                update_record_status(crawl, status)
+            except ValueError as err:
+                rprint(f"[red]{err}[/red]", file=sys.stderr)
+                return False
+        if max_depth is not None:
+            crawl.safe_update({"max_depth": max_depth, "modified_at": timezone.now()}, refresh=False)
+            crawl.max_depth = max_depth
+        elif not status:
+            crawl.safe_update({"modified_at": timezone.now()}, refresh=False)
 
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    updated_count = 0
-    for record in records:
-        crawl_id = record.get("id")
-        if not crawl_id:
-            continue
-
-        try:
-            crawl = Crawl.objects.get(id=crawl_id)
-
-            if status:
-                if status not in Crawl.StatusChoices.values:
-                    rprint(f"[red]Invalid crawl status: {status}[/red]", file=sys.stderr)
-                    continue
-                if status == Crawl.StatusChoices.SEALED:
-                    crawl.cancel()
-                elif status == Crawl.StatusChoices.PAUSED:
-                    crawl.pause()
-                elif status == Crawl.StatusChoices.QUEUED:
-                    if crawl.status == Crawl.StatusChoices.PAUSED:
-                        crawl.resume()
-                    else:
-                        crawl.update_and_requeue(status=Crawl.StatusChoices.QUEUED, retry_at=timezone.now())
-                elif status == Crawl.StatusChoices.STARTED:
-                    crawl.update_and_requeue(status=Crawl.StatusChoices.STARTED, retry_at=timezone.now())
-            if max_depth is not None:
-                crawl.safe_update({"max_depth": max_depth, "modified_at": timezone.now()}, refresh=False)
-                crawl.max_depth = max_depth
-            elif not status:
-                crawl.safe_update({"modified_at": timezone.now()}, refresh=False)
-            updated_count += 1
-
-            if not is_tty:
-                crawl.refresh_from_db()
-                write_record(crawl.to_json())
-
-        except Crawl.DoesNotExist:
-            rprint(f"[yellow]Crawl not found: {crawl_id}[/yellow]", file=sys.stderr)
-            continue
-
-    rprint(f"[green]Updated {updated_count} crawls[/green]", file=sys.stderr)
-    return 0
+    return update_records(Crawl, update, plural="crawls", refresh=True)
 
 
 # =============================================================================
@@ -296,51 +226,18 @@ def update_crawls(
 
 
 def delete_crawls(yes: bool = False, dry_run: bool = False) -> int:
-    """
-    Delete Crawls from stdin JSONL.
-
-    Requires --yes flag to confirm deletion.
-
-    Exit codes:
-        0: Success
-        1: No input or missing --yes flag
-    """
-    from archivebox.misc.jsonl import read_stdin
+    """Delete crawls selected by stdin JSONL; --yes confirms, --dry-run previews."""
+    from archivebox.cli.cli_util import delete_records
     from archivebox.crawls.models import Crawl
 
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    crawl_ids = [r.get("id") for r in records if r.get("id")]
-
-    if not crawl_ids:
-        rprint("[yellow]No valid crawl IDs in input[/yellow]", file=sys.stderr)
-        return 1
-
-    crawls = Crawl.objects.filter(id__in=crawl_ids)
-    count = crawls.count()
-
-    if count == 0:
-        rprint("[yellow]No matching crawls found[/yellow]", file=sys.stderr)
-        return 0
-
-    if dry_run:
-        rprint(f"[yellow]Would delete {count} crawls (dry run)[/yellow]", file=sys.stderr)
-        for crawl in crawls:
-            url_preview = crawl.urls[:50].replace("\n", " ")
-            rprint(f"  [dim]{crawl.id}[/dim] {url_preview}...", file=sys.stderr)
-        return 0
-
-    if not yes:
-        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
-        return 1
-
-    # Perform deletion
-    deleted_count, _ = crawls.delete()
-    rprint(f"[green]Deleted {deleted_count} crawls[/green]", file=sys.stderr)
-    return 0
+    return delete_records(
+        Crawl,
+        label="crawl",
+        plural="crawls",
+        preview=lambda obj: f"[dim]{obj.id}[/dim] {obj.urls[:50].replace(chr(10), chr(32))}...",
+        yes=yes,
+        dry_run=dry_run,
+    )
 
 
 # =============================================================================
@@ -369,37 +266,25 @@ def create_cmd(urls: tuple, depth: int, tag: str, status: str):
 @click.option("--urls__icontains", help="Filter by URLs contains")
 @click.option("--max-depth", type=int, help="Filter by max depth")
 @click.option("--limit", "-n", type=int, help="Limit number of results")
-def list_cmd(
-    status: str | None,
-    urls__icontains: str | None,
-    max_depth: int | None,
-    limit: int | None,
-):
+def list_cmd(**kwargs):
     """List Crawls as JSONL."""
-    sys.exit(
-        list_crawls(
-            status=status,
-            urls__icontains=urls__icontains,
-            max_depth=max_depth,
-            limit=limit,
-        ),
-    )
+    sys.exit(list_crawls(**kwargs))
 
 
 @main.command("update")
 @click.option("--status", "-s", help="Set status")
 @click.option("--max-depth", type=int, help="Set max depth")
-def update_cmd(status: str | None, max_depth: int | None):
+def update_cmd(**kwargs):
     """Update Crawls from stdin JSONL."""
-    sys.exit(update_crawls(status=status, max_depth=max_depth))
+    sys.exit(update_crawls(**kwargs))
 
 
 @main.command("delete")
 @click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
 @click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(yes: bool, dry_run: bool):
+def delete_cmd(**kwargs):
     """Delete Crawls from stdin JSONL."""
-    sys.exit(delete_crawls(yes=yes, dry_run=dry_run))
+    sys.exit(delete_crawls(**kwargs))
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_extract.py

@@ -1,95 +1,135 @@
 #!/usr/bin/env python3
 
-"""
-archivebox extract [snapshot_ids...] [--plugins=NAMES]
-
-Run plugins on Snapshots. Accepts snapshot IDs as arguments, from stdin, or via JSONL.
-
-Input formats:
-    - Snapshot UUIDs (one per line)
-    - JSONL: {"type": "Snapshot", "id": "...", "url": "..."}
-    - JSONL: {"type": "ArchiveResult", "snapshot_id": "...", "plugin": "..."}
-
-Output (JSONL):
-    {"type": "ArchiveResult", "id": "...", "snapshot_id": "...", "plugin": "...", "status": "..."}
-
-Examples:
-    # Extract specific snapshot
-    archivebox extract 01234567-89ab-cdef-0123-456789abcdef
-
-    # Pipe from snapshot command
-    archivebox snapshot https://example.com | archivebox extract
-
-    # Run specific plugins only
-    archivebox extract --plugins=screenshot,singlefile 01234567-89ab-cdef-0123-456789abcdef
-
-    # Chain commands
-    archivebox crawl https://example.com | archivebox snapshot | archivebox extract
-"""
+"""Run abx-dl snapshot hooks for existing ArchiveBox snapshots."""
 
 __package__ = "archivebox.cli"
 __command__ = "archivebox extract"
 
 import sys
 from collections import defaultdict
-from itertools import product
+from contextlib import redirect_stdout
 
 import rich_click as click
 
 
-def process_archiveresult_by_id(archiveresult_id: str) -> int:
-    """
-    Re-run extraction for a single ArchiveResult by ID.
+def _resolve_requests(records: list[dict], plugins: str) -> tuple[dict[str, set[str]], set[str]]:
+    """Resolve CLI records to snapshot-level execution requests.
 
-    ArchiveResults are projected status rows, not queued work items. Re-running
-    a single result means resetting that row and queueing its parent snapshot
-    through the shared crawl runner with the corresponding plugin selected.
+    ArchiveResult input is accepted as a convenient reference to its parent
+    Snapshot and plugin. It is never reset or converted into queued work.
     """
+    from archivebox.misc.db import uuid_ref_query
+    from archivebox.core.models import ArchiveResult, Snapshot
+    from archivebox.misc.jsonl import TYPE_ARCHIVERESULT
+
+    explicit_plugins = {name.strip() for name in plugins.split(",") if name.strip()}
+    requested: dict[str, set[str]] = defaultdict(set)
+    missing: set[str] = set()
+
+    for record in records:
+        record_type = record.get("type")
+        record_id = str(record.get("id") or "")
+
+        if record_type == TYPE_ARCHIVERESULT and record.get("snapshot_id"):
+            requested[str(record["snapshot_id"])].update(explicit_plugins or {str(record.get("plugin") or "")})
+            requested[str(record["snapshot_id"])].discard("")
+            continue
+
+        if record_type == TYPE_ARCHIVERESULT:
+            result = ArchiveResult.objects.filter(uuid_ref_query("id", record_id)).only("snapshot_id", "plugin").first()
+            if result is not None:
+                requested[str(result.snapshot_id)].update(explicit_plugins or {result.plugin})
+                continue
+
+        snapshot_id = str(record.get("snapshot_id") or record_id or "")
+        snapshot = Snapshot.objects.filter(id=snapshot_id).only("id").first() if snapshot_id else None
+        if snapshot is None and record.get("url"):
+            snapshot = Snapshot.objects.filter(url=record["url"]).order_by("-created_at").only("id").first()
+        # Bare UUID CLI arguments are parsed as Snapshot records because the
+        # input layer cannot know which model owns them. Preserve the public
+        # convenience of passing an ArchiveResult ID by trying that reference
+        # only after the Snapshot lookup misses.
+        if snapshot is None and record_id and not record.get("url"):
+            result = ArchiveResult.objects.filter(uuid_ref_query("id", record_id)).only("snapshot_id", "plugin").first()
+            if result is not None:
+                requested[str(result.snapshot_id)].update(explicit_plugins or {result.plugin})
+                continue
+        if snapshot is None:
+            missing.add(snapshot_id or str(record.get("url") or ""))
+            continue
+        requested[str(snapshot.id)].update(explicit_plugins)
+
+    return requested, missing
+
+
+def _run_snapshot_requests(requested: dict[str, set[str]], *, wait: bool, show_progress: bool) -> int:
+    from django.utils import timezone
     from rich import print as rprint
-    from archivebox.core.models import ArchiveResult
-    from archivebox.api.v1_core import _uuid_ref_query
+
+    from archivebox.core.models import Snapshot
     from archivebox.services.runner import run_crawl
 
-    try:
-        archiveresult = ArchiveResult.objects.get(_uuid_ref_query("id", archiveresult_id))
-    except ArchiveResult.DoesNotExist:
-        rprint(f"[red]ArchiveResult {archiveresult_id} not found[/red]", file=sys.stderr)
+    snapshots = {str(snapshot.id): snapshot for snapshot in Snapshot.objects.filter(id__in=requested).select_related("crawl")}
+    if not snapshots:
+        rprint("[red]No snapshots to process[/red]", file=sys.stderr)
         return 1
 
-    rprint(f"[blue]Extracting {archiveresult.plugin} for {archiveresult.snapshot.url}[/blue]", file=sys.stderr)
+    from archivebox.config.common import get_config
+    from archivebox.plugins.discovery import get_enabled_plugins
 
-    try:
-        archiveresult.reset_for_retry()
-        snapshot = archiveresult.snapshot
-        snapshot.queue_for_extraction()
-        crawl = snapshot.crawl
-        if not crawl.claim_processing_lock(lock_seconds=10):
-            rprint(
-                f"[yellow]Crawl {crawl.id} is already owned by another runner[/yellow]",
-                file=sys.stderr,
-            )
-            return 1
+    for snapshot_id, plugin_names in requested.items():
+        snapshot = snapshots.get(snapshot_id)
+        if snapshot is not None and not plugin_names:
+            plugin_names.update(get_enabled_plugins(config=get_config(crawl=snapshot.crawl, snapshot=snapshot)))
 
-        run_crawl(str(snapshot.crawl_id), snapshot_ids=[str(snapshot.id)], selected_plugins=[archiveresult.plugin])
-        archiveresult.refresh_from_db()
+    if wait and any("search_backend_sonic" in plugin_names for plugin_names in requested.values()):
+        from archivebox.core.takeover_util import ensure_daemon_stack
 
-        if archiveresult.status == ArchiveResult.StatusChoices.SUCCEEDED:
-            print(f"[green]Extraction succeeded: {archiveresult.output_str}[/green]")
-            return 0
-        elif archiveresult.status == ArchiveResult.StatusChoices.NORESULTS:
-            print(f"[dim]Extraction completed with no results: {archiveresult.output_str}[/dim]")
-            return 0
-        elif archiveresult.status == ArchiveResult.StatusChoices.FAILED:
-            print(f"[red]Extraction failed: {archiveresult.output_str}[/red]", file=sys.stderr)
-            return 1
-        else:
-            # Still in progress or backoff - not a failure
-            print(f"[yellow]Extraction status: {archiveresult.status}[/yellow]")
-            return 0
+        with redirect_stdout(sys.stderr):
+            ensure_daemon_stack(reason="Sonic snapshot indexing")
 
-    except Exception as e:
-        print(f"[red]Extraction error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
-        return 1
+    # Explicit extraction resumes open/paused snapshots at the snapshot level.
+    # Sealed snapshots stay sealed during targeted maintenance backfills.
+    if wait:
+        for snapshot in snapshots.values():
+            if snapshot.status != Snapshot.StatusChoices.SEALED:
+                snapshot.update_and_requeue(
+                    status=Snapshot.StatusChoices.QUEUED,
+                    retry_at=timezone.now(),
+                )
+
+    if not wait:
+        for snapshot_id, plugin_names in requested.items():
+            snapshot = snapshots.get(snapshot_id)
+            if snapshot is None:
+                continue
+            if plugin_names:
+                snapshot.schedule_plugin_run(plugin_names, when=timezone.now())
+            elif snapshot.status == Snapshot.StatusChoices.SEALED:
+                snapshot.update_and_requeue(retry_at=timezone.now())
+            else:
+                snapshot.update_and_requeue(
+                    status=Snapshot.StatusChoices.QUEUED,
+                    retry_at=timezone.now(),
+                )
+        if show_progress:
+            rprint(f"[blue]Queued {len(snapshots)} snapshots for extraction[/blue]", file=sys.stderr)
+        return 0
+
+    grouped: dict[tuple[str, tuple[str, ...]], list[str]] = defaultdict(list)
+    for snapshot_id, plugin_names in requested.items():
+        snapshot = snapshots.get(snapshot_id)
+        if snapshot is not None:
+            grouped[(str(snapshot.crawl_id), tuple(sorted(plugin_names)))].append(snapshot_id)
+
+    for (crawl_id, plugin_names), snapshot_ids in grouped.items():
+        run_crawl(
+            crawl_id,
+            snapshot_ids=sorted(snapshot_ids),
+            selected_plugins=list(plugin_names) or None,
+            show_progress=show_progress,
+        )
+    return 0
 
 
 def run_plugins(
@@ -99,356 +139,59 @@ def run_plugins(
     wait: bool = True,
     emit_results: bool = True,
     show_progress: bool = True,
-    preserve_queued: bool = False,
 ) -> int:
-    """
-    Run plugins on Snapshots from input.
-
-    Reads Snapshot IDs or JSONL from args/stdin, runs plugins, outputs JSONL.
-
-    Exit codes:
-        0: Success
-        1: Failure
-    """
+    """Execute selected plugins through the snapshot-level runner."""
     from rich import print as rprint
-    from django.utils import timezone
 
-    from archivebox.misc.jsonl import (
-        read_args_or_stdin,
-        write_record,
-        TYPE_SNAPSHOT,
-        TYPE_ARCHIVERESULT,
-    )
     from archivebox.core.models import Snapshot
-    from archivebox.core.models import ArchiveResult
-    from archivebox.services.runner import run_crawl
+    from archivebox.misc.jsonl import read_args_or_stdin, write_record
 
-    is_tty = sys.stdout.isatty()
-
-    # Parse comma-separated plugins list once (reused in creation and filtering)
-    plugins_list = [p.strip() for p in plugins.split(",") if p.strip()] if plugins else []
-
-    # Parse stdin/args exactly once per CLI invocation.
-    # `main()` may already have consumed stdin to distinguish Snapshot input from
-    # ArchiveResult IDs; if so, it must pass the parsed records through here
-    # instead of asking this helper to reread an already-drained pipe.
     if records is None:
         records = list(read_args_or_stdin(args))
-
     if not records:
         rprint("[yellow]No snapshots provided. Pass snapshot IDs as arguments or via stdin.[/yellow]", file=sys.stderr)
         return 1
 
-    # Gather snapshot IDs and optional plugin constraints to process
-    snapshot_ids = set()
-    requested_plugins_by_snapshot: dict[str, set[str]] = defaultdict(set)
-    for record in records:
-        record_type = record.get("type")
-
-        if record_type == TYPE_SNAPSHOT:
-            snapshot_id = record.get("id")
-            if snapshot_id:
-                snapshot_ids.add(str(snapshot_id))
-            elif record.get("url"):
-                # Look up by URL (get most recent if multiple exist)
-                snap = Snapshot.objects.filter(url=record["url"]).order_by("-created_at").first()
-                if snap:
-                    snapshot_ids.add(str(snap.id))
-                else:
-                    rprint(f"[yellow]Snapshot not found for URL: {record['url']}[/yellow]", file=sys.stderr)
-
-        elif record_type == TYPE_ARCHIVERESULT:
-            snapshot_id = record.get("snapshot_id")
-            if snapshot_id:
-                snapshot_ids.add(str(snapshot_id))
-                plugin_name = record.get("plugin")
-                if plugin_name and not plugins_list:
-                    requested_plugins_by_snapshot[str(snapshot_id)].add(str(plugin_name))
-
-        elif "id" in record:
-            # Assume it's a snapshot ID
-            snapshot_ids.add(str(record["id"]))
-
-    if not snapshot_ids:
-        rprint("[red]No valid snapshot IDs found in input[/red]", file=sys.stderr)
+    requested, missing = _resolve_requests(records, plugins)
+    for value in sorted(missing):
+        rprint(f"[yellow]Snapshot or ArchiveResult not found: {value}[/yellow]", file=sys.stderr)
+    if not requested:
         return 1
 
-    existing_snapshots = list(Snapshot.objects.filter(id__in=snapshot_ids).values_list("id", "crawl_id"))
-    existing_snapshot_ids = {str(snapshot_id) for snapshot_id, _crawl_id in existing_snapshots}
-    existing_crawl_ids = {str(crawl_id) for _snapshot_id, crawl_id in existing_snapshots}
-    missing_snapshot_ids = sorted(str(snapshot_id) for snapshot_id in snapshot_ids - existing_snapshot_ids)
-    for snapshot_id in missing_snapshot_ids:
-        rprint(f"[yellow]Snapshot {snapshot_id} not found[/yellow]", file=sys.stderr)
-
-    # Queue only the target plugin rows. Bulk updates keep large reindex runs
-    # from doing one SELECT+UPDATE per snapshot/plugin before hooks even start.
-    requested_pairs: set[tuple[str, str]] = set()
-    if plugins_list:
-        requested_pairs.update((snapshot_id, plugin_name) for snapshot_id, plugin_name in product(existing_snapshot_ids, plugins_list))
-    else:
-        requested_pairs.update(
-            (snapshot_id, plugin_name)
-            for snapshot_id, plugin_names in requested_plugins_by_snapshot.items()
-            if snapshot_id in existing_snapshot_ids
-            for plugin_name in plugin_names
-        )
-    queued_rows: set[tuple[str, str]] = set()
-    if preserve_queued and requested_pairs:
-        queued_rows = {
-            (str(snapshot_id), plugin_name)
-            for snapshot_id, plugin_name in ArchiveResult.objects.filter(
-                snapshot_id__in=existing_snapshot_ids,
-                plugin__in={plugin_name for _snapshot_id, plugin_name in requested_pairs},
-                status=ArchiveResult.StatusChoices.QUEUED,
-            ).values_list("snapshot_id", "plugin")
-        }
-    rows_to_queue = requested_pairs - queued_rows
-
-    reset_fields = {
-        "status": ArchiveResult.StatusChoices.QUEUED,
-        "hook_name": "",
-        "output_str": "",
-        "output_json": None,
-        "output_files": {},
-        "output_size": 0,
-        "output_mimetypes": "",
-        "start_ts": None,
-        "end_ts": None,
-        "modified_at": timezone.now(),
-    }
-    if rows_to_queue and plugins_list:
-        rows_to_reset_by_plugin: dict[str, set[str]] = defaultdict(set)
-        for snapshot_id, plugin_name in rows_to_queue:
-            rows_to_reset_by_plugin[plugin_name].add(snapshot_id)
-        for plugin_name, plugin_snapshot_ids in rows_to_reset_by_plugin.items():
-            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name).update(
-                **reset_fields,
-            )
-    elif rows_to_queue and requested_plugins_by_snapshot:
-        snapshot_ids_by_plugin: dict[str, set[str]] = defaultdict(set)
-        for snapshot_id, plugin_name in rows_to_queue:
-            snapshot_ids_by_plugin[plugin_name].add(snapshot_id)
-        for plugin_name, plugin_snapshot_ids in snapshot_ids_by_plugin.items():
-            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name).update(
-                **reset_fields,
-            )
-    existing_rows = (
-        {
-            (str(snapshot_id), plugin_name)
-            for snapshot_id, plugin_name in ArchiveResult.objects.filter(
-                snapshot_id__in=existing_snapshot_ids,
-                plugin__in={plugin_name for _snapshot_id, plugin_name in rows_to_queue},
-            ).values_list("snapshot_id", "plugin")
-        }
-        if rows_to_queue
-        else set()
-    )
-    missing_rows = rows_to_queue - existing_rows
-    if missing_rows:
-        ArchiveResult.objects.bulk_create(
-            [
-                ArchiveResult(
-                    snapshot_id=snapshot_id,
-                    plugin=plugin_name,
-                    hook_name="",
-                    status=ArchiveResult.StatusChoices.QUEUED,
-                )
-                for snapshot_id, plugin_name in sorted(missing_rows)
-            ],
-            batch_size=500,
-        )
-
-    processed_count = len(existing_snapshot_ids)
-    queue_at = timezone.now()
-    if existing_snapshot_ids:
-        if requested_pairs:
-            # Search indexing on a sealed Snapshot is the only targeted plugin
-            # allowed to bypass the normal lifecycle. Every other requested
-            # plugin requeues its Snapshot through the unified state machine.
-            affected_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name in rows_to_queue}
-            if preserve_queued and queued_rows:
-                queued_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name in queued_rows}
-                affected_snapshot_ids.update(
-                    str(snapshot_id)
-                    for snapshot_id in Snapshot.objects.filter(id__in=queued_snapshot_ids)
-                    .filter(retry_at__gt=queue_at)
-                    .values_list("id", flat=True)
-                )
-                affected_snapshot_ids.update(
-                    str(snapshot_id)
-                    for snapshot_id in Snapshot.objects.filter(id__in=queued_snapshot_ids, retry_at__isnull=True).values_list(
-                        "id",
-                        flat=True,
-                    )
-                )
-            requested_plugins_by_id: dict[str, set[str]] = defaultdict(set)
-            for snapshot_id, plugin_name in requested_pairs:
-                requested_plugins_by_id[snapshot_id].add(plugin_name)
-            for snapshot in Snapshot.objects.filter(id__in=affected_snapshot_ids).only("id", "status", "modified_at"):
-                # Guard the read-time status so we never bump retry_at on a
-                # row that's been re-queued / started by a concurrent runner.
-                plugin_names = requested_plugins_by_id.get(str(snapshot.id), set())
-                sealed_search_backfill = (
-                    snapshot.status == Snapshot.StatusChoices.SEALED
-                    and plugin_names
-                    and all(plugin_name.startswith("search_backend_") for plugin_name in plugin_names)
-                )
-                if sealed_search_backfill:
-                    snapshot.safe_update(
-                        {"retry_at": queue_at, "modified_at": queue_at},
-                        refresh=False,
-                        extra_filter={"status": snapshot.status},
-                    )
-                else:
-                    snapshot.update_and_requeue(
-                        status=Snapshot.StatusChoices.QUEUED,
-                        retry_at=queue_at,
-                        current_step=0,
-                    )
-        else:
-            # No plugin rows were requested, so this is a full snapshot retry.
-            for snapshot in Snapshot.objects.filter(id__in=existing_snapshot_ids).only("id", "status", "retry_at", "modified_at"):
-                snapshot.safe_update(
-                    {
-                        "status": Snapshot.StatusChoices.QUEUED,
-                        "retry_at": queue_at,
-                        "current_step": 0,
-                        "modified_at": queue_at,
-                    },
-                    refresh=False,
-                    extra_filter={"status": snapshot.status},
-                )
-    if existing_crawl_ids and not requested_pairs:
-        from archivebox.crawls.models import Crawl
-
-        for crawl in Crawl.objects.filter(id__in=existing_crawl_ids).only("id", "status", "retry_at", "modified_at"):
-            update_fields = {
-                "retry_at": queue_at,
-                "modified_at": queue_at,
-            }
-            if crawl.status != Crawl.StatusChoices.STARTED:
-                update_fields["status"] = Crawl.StatusChoices.QUEUED
-            crawl.safe_update(
-                update_fields,
-                refresh=False,
-                extra_filter={"status": crawl.status},
-            )
-
-    if processed_count == 0:
-        rprint("[red]No snapshots to process[/red]", file=sys.stderr)
-        return 1
-
-    if show_progress:
-        rprint(f"[blue]Queued {processed_count} snapshots for extraction[/blue]", file=sys.stderr)
-
-    # Run orchestrator if --wait (default)
-    if wait:
-        if show_progress:
-            rprint("[blue]Running plugins...[/blue]", file=sys.stderr)
-        snapshot_ids_by_crawl: dict[str, set[str]] = defaultdict(set)
-        for snapshot_id, crawl_id in existing_snapshots:
-            snapshot_ids_by_crawl[str(crawl_id)].add(str(snapshot_id))
-
-        for crawl_id, crawl_snapshot_ids in snapshot_ids_by_crawl.items():
-            from archivebox.crawls.models import Crawl
-
-            crawl = Crawl.objects.get(id=crawl_id)
-            if not crawl.claim_processing_lock(lock_seconds=10):
-                rprint(
-                    f"[yellow]Crawl {crawl_id} is already owned by another runner[/yellow]",
-                    file=sys.stderr,
-                )
-                return 1
-            selected_plugins = (
-                plugins_list
-                or sorted(
-                    {plugin for snapshot_id in crawl_snapshot_ids for plugin in requested_plugins_by_snapshot.get(str(snapshot_id), set())},
-                )
-                or None
-            )
-            run_crawl(
-                crawl_id,
-                snapshot_ids=sorted(crawl_snapshot_ids),
-                selected_plugins=selected_plugins,
-                show_progress=show_progress,
-            )
-
-    if not emit_results:
-        return 0
-
-    # Output results as JSONL (when piped) or human-readable (when TTY)
-    for snapshot_id in snapshot_ids:
-        try:
-            snapshot = Snapshot.objects.get(id=snapshot_id)
-            results = snapshot.archiveresult_set.all()
-            if plugins_list:
-                results = results.filter(plugin__in=plugins_list)
-
-            for result in results:
-                if is_tty:
-                    status_color = {
-                        "succeeded": "green",
-                        "failed": "red",
-                        "skipped": "yellow",
-                    }.get(result.status, "dim")
-                    rprint(
-                        f"  [{status_color}]{result.status}[/{status_color}] {result.plugin} → {result.output_str or ''}",
-                        file=sys.stderr,
-                    )
-                else:
-                    write_record(result.to_json())
-        except Snapshot.DoesNotExist:
-            continue
+    exit_code = _run_snapshot_requests(requested, wait=wait, show_progress=show_progress)
+    if exit_code or not emit_results:
+        return exit_code
 
+    is_tty = sys.stdout.isatty()
+    for snapshot in Snapshot.objects.filter(id__in=requested):
+        results = snapshot.archiveresult_set.all()
+        requested_plugins = requested[str(snapshot.id)]
+        if requested_plugins:
+            results = results.filter(plugin__in=requested_plugins)
+        for result in results:
+            if is_tty:
+                color = {"succeeded": "green", "failed": "red", "skipped": "yellow"}.get(result.status, "dim")
+                rprint(f"  [{color}]{result.status}[/{color}] {result.plugin} → {result.output_str or ''}", file=sys.stderr)
+            else:
+                write_record(result.to_json())
     return 0
 
 
-def is_archiveresult_id(value: str) -> bool:
-    """Check if value resolves to an ArchiveResult ID."""
-    from archivebox.core.models import ArchiveResult
-    from archivebox.api.v1_core import _uuid_ref_query
-
-    return ArchiveResult.objects.filter(_uuid_ref_query("id", value)).exists()
-
-
 @click.command()
-@click.option("--plugins", "--plugin", "-p", default="", help="Comma-separated list of plugins to run (e.g., screenshot,singlefile)")
+@click.option("--plugins", "--plugin", "-p", default="", help="Comma-separated list of plugins to run")
 @click.option("--wait/--no-wait", default=True, help="Wait for plugins to complete (default: wait)")
 @click.argument("args", nargs=-1)
 def main(plugins: str, wait: bool, args: tuple):
-    """Run plugins on Snapshots, or process existing ArchiveResults by ID"""
+    """Run plugins on Snapshots; ArchiveResult IDs select their parent plugin."""
     from archivebox.misc.jsonl import read_args_or_stdin
 
-    # Read all input
     records = list(read_args_or_stdin(args))
-
     if not records:
         from rich import print as rprint
 
         rprint("[yellow]No Snapshot IDs or ArchiveResult IDs provided. Pass as arguments or via stdin.[/yellow]", file=sys.stderr)
         sys.exit(1)
-
-    # Check if input looks like existing ArchiveResult IDs to process
-    all_are_archiveresult_ids = all(is_archiveresult_id(r.get("id") or r.get("url", "")) for r in records)
-
-    if all_are_archiveresult_ids:
-        # Process existing ArchiveResults by ID
-        from rich import print as rprint
-
-        exit_code = 0
-        for record in records:
-            archiveresult_id = record.get("id") or record.get("url")
-            if not isinstance(archiveresult_id, str):
-                rprint(f"[red]Invalid ArchiveResult input: {record}[/red]", file=sys.stderr)
-                exit_code = 1
-                continue
-            result = process_archiveresult_by_id(archiveresult_id)
-            if result != 0:
-                exit_code = result
-        sys.exit(exit_code)
-    else:
-        # Default behavior: run plugins on Snapshots from input
-        sys.exit(run_plugins(args, records=records, plugins=plugins, wait=wait))
+    sys.exit(run_plugins(args, records=records, plugins=plugins, wait=wait))
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_install.py

@@ -17,12 +17,12 @@ def _resolve_install_targets(
     requested_names: tuple[str, ...],
 ) -> tuple[list[str], list[str]]:
     """Resolve plugin names and declared binary aliases, leaving unknown tokens raw."""
-    from abx_dl.models import discover_plugins, filter_plugins
+    from archivebox.plugins.discovery import get_plugin_catalog
 
-    plugins = discover_plugins(runtime="archivebox")
-    plugin_names_by_lower = {plugin_name.lower(): plugin_name for plugin_name in plugins}
+    catalog = get_plugin_catalog()
+    plugin_names_by_lower = {plugin_name.lower(): plugin_name for plugin_name in catalog}
     plugin_names_by_binary_alias: dict[str, set[str]] = {}
-    for plugin_name, plugin in plugins.items():
+    for plugin_name, plugin in catalog.items():
         for required_binary in plugin.config.required_binaries:
             aliases = {required_binary.name.lower()}
             template_match = re.fullmatch(r"\{([A-Za-z_][A-Za-z0-9_]*)\}", required_binary.name)
@@ -47,14 +47,14 @@ def _resolve_install_targets(
         else:
             raw_binary_names.append(name)
 
-    selected_plugins = filter_plugins(plugins, requested_plugins, include_providers=True) if requested_plugins else {}
+    selected_plugins = catalog.select(requested_plugins) if requested_plugins else catalog.select([])
     selected_plugin_names = {name.lower() for name in selected_plugins}
     raw_binary_names = [name for name in raw_binary_names if name.lower() not in selected_plugin_names]
     return sorted(selected_plugins), sorted(set(raw_binary_names))
 
 
 def _install_raw_binary_names(binary_names: list[str], binproviders: str) -> None:
-    """Install user-requested standalone binaries through the Binary state machine."""
+    """Install user-requested standalone binaries through the Binary lifecycle."""
     from django.utils import timezone
 
     from archivebox.machine.models import Binary, Machine, _canonical_binary_name
@@ -179,7 +179,7 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo
         run_install(plugin_names=install_plugin_names or None)
 
     if raw_binary_names:
-        print(f"[+] Running direct binary installer via ArchiveBox binary state machine: {', '.join(raw_binary_names)}")
+        print(f"[+] Running direct binary installer via ArchiveBox binary lifecycle: {', '.join(raw_binary_names)}")
         print()
         _install_raw_binary_names(raw_binary_names, binproviders)
 

archivebox/cli/archivebox_machine.py

@@ -25,7 +25,6 @@ __command__ = "archivebox machine"
 import sys
 
 import rich_click as click
-from rich import print as rprint
 
 from archivebox.cli.cli_util import apply_filters
 
@@ -35,41 +34,21 @@ from archivebox.cli.cli_util import apply_filters
 # =============================================================================
 
 
-def list_machines(
-    hostname__icontains: str | None = None,
-    os_platform: str | None = None,
-    limit: int | None = None,
-) -> int:
-    """
-    List Machines as JSONL with optional filters.
-
-    Exit codes:
-        0: Success (even if no results)
-    """
-    from archivebox.misc.jsonl import write_record
+def list_machines(hostname__icontains: str | None = None, os_platform: str | None = None, limit: int | None = None) -> int:
+    """List machines as JSONL, or formatted rows in a terminal."""
     from archivebox.machine.models import Machine
+    from archivebox.cli.cli_util import list_records
 
-    is_tty = sys.stdout.isatty()
-
-    queryset = Machine.objects.all().order_by("-created_at")
-
-    # Apply filters
-    filter_kwargs = {
-        "hostname__icontains": hostname__icontains,
-        "os_platform": os_platform,
-    }
-    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
-
-    count = 0
-    for machine in queryset:
-        if is_tty:
-            rprint(f"[cyan]{machine.hostname:30}[/cyan] [dim]{machine.os_platform:10}[/dim] {machine.id}")
-        else:
-            write_record(machine.to_json())
-        count += 1
-
-    rprint(f"[dim]Listed {count} machines[/dim]", file=sys.stderr)
-    return 0
+    queryset = apply_filters(
+        Machine.objects.order_by("-created_at"),
+        {"hostname__icontains": hostname__icontains, "os_platform": os_platform},
+        limit=limit,
+    )
+    return list_records(
+        queryset,
+        plural="machines",
+        render=lambda machine: f"[cyan]{machine.hostname:30}[/cyan] [dim]{machine.os_platform:10}[/dim] {machine.id}",
+    )
 
 
 # =============================================================================
@@ -87,15 +66,9 @@ def main():
 @click.option("--hostname__icontains", help="Filter by hostname contains")
 @click.option("--os-platform", help="Filter by OS platform")
 @click.option("--limit", "-n", type=int, help="Limit number of results")
-def list_cmd(hostname__icontains: str | None, os_platform: str | None, limit: int | None):
+def list_cmd(**kwargs):
     """List Machines as JSONL."""
-    sys.exit(
-        list_machines(
-            hostname__icontains=hostname__icontains,
-            os_platform=os_platform,
-            limit=limit,
-        ),
-    )
+    sys.exit(list_machines(**kwargs))
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_persona.py

@@ -27,164 +27,19 @@ Examples:
 __package__ = "archivebox.cli"
 __command__ = "archivebox persona"
 
-import os
 import sys
 import shutil
-import platform
 from pathlib import Path
+from dataclasses import replace
 from collections.abc import Iterable
 
 import rich_click as click
 from rich import print as rprint
 
-from archivebox.cli.cli_util import apply_filters
+from archivebox.cli.cli_util import apply_filters, list_records
 from archivebox.personas import importers as persona_importers
 
 
-# =============================================================================
-# Browser Profile Locations
-# =============================================================================
-
-
-def get_chrome_user_data_dir() -> Path | None:
-    """Get the default Chrome user data directory for the current platform."""
-    system = platform.system()
-    home = Path.home()
-
-    if system == "Darwin":  # macOS
-        candidates = [
-            home / "Library" / "Application Support" / "Google" / "Chrome",
-            home / "Library" / "Application Support" / "Chromium",
-        ]
-    elif system == "Linux":
-        candidates = [
-            home / ".config" / "google-chrome",
-            home / ".config" / "chromium",
-            home / ".config" / "chrome",
-            home / "snap" / "chromium" / "common" / "chromium",
-        ]
-    elif system == "Windows":
-        local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
-        candidates = [
-            local_app_data / "Google" / "Chrome" / "User Data",
-            local_app_data / "Chromium" / "User Data",
-        ]
-    else:
-        candidates = []
-
-    for candidate in candidates:
-        if candidate.exists() and (candidate / "Default").exists():
-            return candidate
-
-    return None
-
-
-def get_brave_user_data_dir() -> Path | None:
-    """Get the default Brave user data directory for the current platform."""
-    system = platform.system()
-    home = Path.home()
-
-    if system == "Darwin":
-        candidates = [
-            home / "Library" / "Application Support" / "BraveSoftware" / "Brave-Browser",
-        ]
-    elif system == "Linux":
-        candidates = [
-            home / ".config" / "BraveSoftware" / "Brave-Browser",
-        ]
-    elif system == "Windows":
-        local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
-        candidates = [
-            local_app_data / "BraveSoftware" / "Brave-Browser" / "User Data",
-        ]
-    else:
-        candidates = []
-
-    for candidate in candidates:
-        if candidate.exists() and (candidate / "Default").exists():
-            return candidate
-
-    return None
-
-
-def get_edge_user_data_dir() -> Path | None:
-    """Get the default Edge user data directory for the current platform."""
-    system = platform.system()
-    home = Path.home()
-
-    if system == "Darwin":
-        candidates = [
-            home / "Library" / "Application Support" / "Microsoft Edge",
-        ]
-    elif system == "Linux":
-        candidates = [
-            home / ".config" / "microsoft-edge",
-            home / ".config" / "microsoft-edge-beta",
-            home / ".config" / "microsoft-edge-dev",
-        ]
-    elif system == "Windows":
-        local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
-        candidates = [
-            local_app_data / "Microsoft" / "Edge" / "User Data",
-        ]
-    else:
-        candidates = []
-
-    for candidate in candidates:
-        if candidate.exists() and (candidate / "Default").exists():
-            return candidate
-
-    return None
-
-
-BROWSER_PROFILE_FINDERS = {
-    "chrome": get_chrome_user_data_dir,
-    "chromium": get_chrome_user_data_dir,  # Same locations
-    "brave": get_brave_user_data_dir,
-    "edge": get_edge_user_data_dir,
-}
-
-CHROMIUM_BROWSERS = {"chrome", "chromium", "brave", "edge"}
-
-
-# =============================================================================
-# Cookie Extraction via CDP
-# =============================================================================
-
-# =============================================================================
-# Validation Helpers
-# =============================================================================
-
-
-def validate_persona_name(name: str) -> tuple[bool, str]:
-    """
-    Validate persona name to prevent path traversal attacks.
-
-    Returns:
-        (is_valid, error_message): tuple indicating if name is valid
-    """
-    if not name or not name.strip():
-        return False, "Persona name cannot be empty"
-
-    # Check for path separators
-    if "/" in name or "\\" in name:
-        return False, "Persona name cannot contain path separators (/ or \\)"
-
-    # Check for parent directory references
-    if ".." in name:
-        return False, "Persona name cannot contain parent directory references (..)"
-
-    # Check for hidden files/directories
-    if name.startswith("."):
-        return False, "Persona name cannot start with a dot (.)"
-
-    # Ensure name doesn't contain null bytes or other dangerous chars
-    if "\x00" in name or "\n" in name or "\r" in name:
-        return False, "Persona name contains invalid characters"
-
-    return True, ""
-
-
 def ensure_path_within_personas_dir(persona_path: Path) -> bool:
     """
     Verify that a persona path is within PERSONAS_DIR.
@@ -218,6 +73,8 @@ def create_personas(
     names: Iterable[str],
     import_from: str | None = None,
     profile: str | None = None,
+    source: str | None = None,
+    browser_binary: str | None = None,
 ) -> int:
     """
     Create Personas from names.
@@ -239,25 +96,26 @@ def create_personas(
         rprint("[yellow]No persona names provided. Pass names as arguments.[/yellow]", file=sys.stderr)
         return 1
 
-    # Validate import source if specified
-    source_profile_dir = None
+    import_source = None
+    if source and not import_from:
+        rprint("[red]--source requires --import (the source browser name).[/red]", file=sys.stderr)
+        return 1
     if import_from:
-        import_from = import_from.lower()
-        if import_from not in BROWSER_PROFILE_FINDERS:
-            rprint(f"[red]Unknown browser: {import_from}[/red]", file=sys.stderr)
-            rprint(f"[dim]Supported browsers: {', '.join(BROWSER_PROFILE_FINDERS.keys())}[/dim]", file=sys.stderr)
+        try:
+            if source:
+                import_source = persona_importers.resolve_custom_import_source(source, profile_dir=profile)
+                if import_source.browser != "persona":
+                    import_source = replace(import_source, browser=import_from.lower(), browser_binary=browser_binary)
+            elif import_from.startswith(("http://", "https://", "ws://", "wss://")) or Path(import_from).expanduser().is_absolute():
+                import_source = persona_importers.resolve_custom_import_source(import_from)
+            else:
+                import_source = persona_importers.resolve_browser_import_source(import_from, profile_dir=profile)
+            if browser_binary:
+                import_source = replace(import_source, browser_binary=browser_binary)
+        except ValueError as err:
+            rprint(f"[red]{err}[/red]", file=sys.stderr)
             return 1
 
-        source_profile_dir = BROWSER_PROFILE_FINDERS[import_from]()
-        if not source_profile_dir:
-            rprint(f"[red]Could not find {import_from} profile directory[/red]", file=sys.stderr)
-            return 1
-
-        rprint(f"[dim]Found {import_from} profile: {source_profile_dir}[/dim]", file=sys.stderr)
-
-        if profile is None and (source_profile_dir / "Default").exists():
-            profile = "Default"
-
     created_count = 0
     for name in name_list:
         name = name.strip()
@@ -275,16 +133,16 @@ def create_personas(
         if created:
             persona.ensure_dirs()
             created_count += 1
-            rprint(f"[green]Created persona: {name}[/green]", file=sys.stderr)
+
         else:
             rprint(f"[dim]Persona already exists: {name}[/dim]", file=sys.stderr)
 
         cookies_file = Path(persona.path) / "cookies.txt"
 
         # Import browser profile if requested
-        if import_from in CHROMIUM_BROWSERS and source_profile_dir is not None:
+        if import_source is not None:
             try:
-                import_source = persona_importers.resolve_browser_import_source(import_from, profile_dir=profile)
+                rprint(f"[dim]Importing {import_source.display_label}; copying settings and exporting cookies...[/dim]", file=sys.stderr)
                 import_result = persona_importers.import_persona_from_source(
                     persona,
                     import_source,
@@ -292,30 +150,27 @@ def create_personas(
                     import_cookies=True,
                     capture_storage=False,
                 )
-            except Exception as e:
+            except (Exception, KeyboardInterrupt) as e:
+                if created:
+                    shutil.rmtree(persona.path, ignore_errors=True)
+                    persona.delete()
                 rprint(f"[red]Failed to import browser profile: {e}[/red]", file=sys.stderr)
                 return 1
 
             if import_result.profile_copied:
                 rprint("[green]Copied browser profile to persona[/green]", file=sys.stderr)
             if import_result.cookies_imported:
-                rprint(f"[green]Extracted cookies to {cookies_file}[/green]", file=sys.stderr)
+                rprint(f"[green]Extracted {import_result.cookie_count} cookies to {cookies_file}[/green]", file=sys.stderr)
             elif not import_result.profile_copied:
                 rprint("[yellow]Could not import cookies automatically.[/yellow]", file=sys.stderr)
 
             for warning in import_result.warnings:
                 rprint(f"[yellow]{warning}[/yellow]", file=sys.stderr)
 
+        if created:
+            rprint(f"[green]Created persona: {name}[/green]", file=sys.stderr)
         if not is_tty:
-            write_record(
-                {
-                    "id": str(persona.id),
-                    "name": persona.name,
-                    "path": str(persona.path),
-                    "CHROME_USER_DATA_DIR": persona.CHROME_USER_DATA_DIR,
-                    "COOKIES_FILE": persona.COOKIES_FILE,
-                },
-            )
+            write_record(persona.to_json())
 
     rprint(f"[green]Created {created_count} new persona(s)[/green]", file=sys.stderr)
     return 0
@@ -337,41 +192,20 @@ def list_personas(
     Exit codes:
         0: Success (even if no results)
     """
-    from archivebox.misc.jsonl import write_record
     from archivebox.personas.models import Persona
 
-    is_tty = sys.stdout.isatty()
+    queryset = apply_filters(
+        Persona.objects.order_by("name"),
+        {"name": name, "name__icontains": name__icontains},
+        limit=limit,
+    )
 
-    queryset = Persona.objects.all().order_by("name")
+    def render(persona):
+        cookies = "[green]✓[/green]" if persona.COOKIES_FILE else "[dim]✗[/dim]"
+        chrome = "[green]✓[/green]" if Path(persona.CHROME_USER_DATA_DIR).exists() else "[dim]✗[/dim]"
+        return f"[cyan]{persona.name:20}[/cyan] cookies:{cookies} chrome:{chrome} [dim]{persona.path}[/dim]"
 
-    # Apply filters
-    filter_kwargs = {
-        "name": name,
-        "name__icontains": name__icontains,
-    }
-    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
-
-    count = 0
-    for persona in queryset:
-        cookies_status = "[green]✓[/green]" if persona.COOKIES_FILE else "[dim]✗[/dim]"
-        chrome_status = "[green]✓[/green]" if Path(persona.CHROME_USER_DATA_DIR).exists() else "[dim]✗[/dim]"
-
-        if is_tty:
-            rprint(f"[cyan]{persona.name:20}[/cyan] cookies:{cookies_status} chrome:{chrome_status} [dim]{persona.path}[/dim]")
-        else:
-            write_record(
-                {
-                    "id": str(persona.id),
-                    "name": persona.name,
-                    "path": str(persona.path),
-                    "CHROME_USER_DATA_DIR": persona.CHROME_USER_DATA_DIR,
-                    "COOKIES_FILE": persona.COOKIES_FILE,
-                },
-            )
-        count += 1
-
-    rprint(f"[dim]Listed {count} persona(s)[/dim]", file=sys.stderr)
-    return 0
+    return list_records(queryset, plural="persona(s)", render=render)
 
 
 # =============================================================================
@@ -435,13 +269,7 @@ def update_personas(name: str | None = None) -> int:
             updated_count += 1
 
             if not is_tty:
-                write_record(
-                    {
-                        "id": str(persona.id),
-                        "name": persona.name,
-                        "path": str(persona.path),
-                    },
-                )
+                write_record(persona.to_json())
 
         except Persona.DoesNotExist:
             rprint(f"[yellow]Persona not found: {persona_id or old_name}[/yellow]", file=sys.stderr)
@@ -544,35 +372,119 @@ def main():
 
 @main.command("create")
 @click.argument("names", nargs=-1)
-@click.option("--import", "import_from", help="Import profile from browser (chrome, chromium, brave, edge)")
+@click.option("--import", "import_from", help="Import from chrome, chromium, brave, edge, a persona directory, or a live CDP URL")
+@click.option("--source", help="Source browser user-data directory or exact profile path (including Docker mounts)")
+@click.option(
+    "--browser-binary",
+    type=click.Path(exists=True, dir_okay=False),
+    help="Source Chromium browser executable; required for other Chromium-based browsers",
+)
 @click.option("--profile", help="Profile directory name under the user data dir (e.g. Default, Profile 1)")
-def create_cmd(names: tuple, import_from: str | None, profile: str | None):
+def create_cmd(names: tuple, import_from: str | None, profile: str | None, source: str | None, browser_binary: str | None):
     """Create Personas, optionally importing from a browser profile."""
-    sys.exit(create_personas(names, import_from=import_from, profile=profile))
+    sys.exit(create_personas(names, import_from=import_from, profile=profile, source=source, browser_binary=browser_binary))
 
 
 @main.command("list")
 @click.option("--name", help="Filter by exact name")
 @click.option("--name__icontains", help="Filter by name contains")
 @click.option("--limit", "-n", type=int, help="Limit number of results")
-def list_cmd(name: str | None, name__icontains: str | None, limit: int | None):
+def list_cmd(**kwargs):
     """List Personas as JSONL."""
-    sys.exit(list_personas(name=name, name__icontains=name__icontains, limit=limit))
+    sys.exit(list_personas(**kwargs))
+
+
+@main.command("open")
+@click.argument("name", default="Default")
+def open_cmd(name: str):
+    """Open a persona's browser, including its imported logins."""
+    import json
+    import os
+    import subprocess
+
+    import psutil
+
+    from abx_plugins import get_plugins_dir
+    from archivebox.config.common import get_config
+    from archivebox.personas.models import Persona
+
+    valid, reason = persona_importers.validate_persona_name(name)
+    if not valid:
+        raise click.ClickException(reason)
+    persona = Persona.get_or_create_named(name)
+    persona.ensure_dirs()
+    config = get_config(persona=persona)
+    if os.environ.get("DISPLAY"):
+        from archivebox.config import CONSTANTS
+
+        for process in psutil.process_iter():
+            try:
+                args = process.cmdline()
+                profile = next((arg.split("=", 1)[1] for arg in args if arg.startswith("--user-data-dir=")), None)
+                if (
+                    not profile
+                    or any(arg.startswith(("--type=", "--headless")) for arg in args)
+                    or not Path(profile).resolve().is_relative_to(CONSTANTS.DATA_DIR.resolve())
+                    or process.environ().get("DISPLAY") != os.environ["DISPLAY"]
+                ):
+                    continue
+                rprint("An ArchiveBox browser is already open on this display; waiting for it to close.")
+                process.wait()
+            except (psutil.NoSuchProcess, psutil.AccessDenied):
+                continue
+    plugins_dir = Path(get_plugins_dir()).resolve()
+    env = os.environ.copy()
+    dependencies = subprocess.run(
+        [
+            str(Path(sys.executable).with_name("abxpkg")),
+            "env",
+            "--install",
+            "--json",
+            f"--lib={config.ABXPKG_LIB_DIR}",
+            f"--deps-from={plugins_dir / 'chrome' / 'config.json'}:required_binaries",
+        ],
+        capture_output=True,
+        text=True,
+        env=env,
+    )
+    if dependencies.returncode:
+        raise click.ClickException(dependencies.stderr.strip())
+    env.update({key: str(value) for key, value in json.loads(dependencies.stdout).items()})
+    env["ARCHIVEBOX_ABX_PLUGINS_DIR"] = str(plugins_dir)
+    env["ACTIVE_PERSONA"] = persona.name
+    env["PERSONAS_DIR"] = str(persona.path.parent)
+    env["NODE_MODULES_DIR"] = str(config.ABXPKG_LIB_DIR / "pnpm" / "packages" / "chrome" / "node_modules")
+    payload = config.model_dump(mode="json")
+    payload.update(
+        ACTIVE_PERSONA=persona.name,
+        CHROME_HEADLESS=False,
+        CHROME_USER_DATA_DIR=str(persona.CHROME_USER_DATA_DIR),
+        AUTH_STORAGE_FILE=str(persona.AUTH_STORAGE_FILE or ""),
+        COOKIES_FILE=str(persona.COOKIES_FILE or ""),
+    )
+    script = Path(__file__).parent.parent / "personas" / "open_browser.js"
+    result = subprocess.run(
+        [env.get("NODE_BINARY") or "node", str(script)],
+        input=json.dumps(payload),
+        text=True,
+        env=env,
+    )
+    sys.exit(result.returncode)
 
 
 @main.command("update")
 @click.option("--name", "-n", help="Set new name")
-def update_cmd(name: str | None):
+def update_cmd(**kwargs):
     """Update Personas from stdin JSONL."""
-    sys.exit(update_personas(name=name))
+    sys.exit(update_personas(**kwargs))
 
 
 @main.command("delete")
 @click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
 @click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(yes: bool, dry_run: bool):
+def delete_cmd(**kwargs):
     """Delete Personas from stdin JSONL."""
-    sys.exit(delete_personas(yes=yes, dry_run=dry_run))
+    sys.exit(delete_personas(**kwargs))
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_pluginmap.py

@@ -48,7 +48,7 @@ def pluginmap(
     Show the current abx-dl event phases and their associated plugin hooks.
 
     This command reflects the new bus-driven runtime, not the legacy ArchiveBox
-    state-machine executor. Event names are normalized to hook prefixes by
+    event runtime. Event names are normalized to hook prefixes by
     stripping a trailing `Event`, then ArchiveBox checks whether any matching
     `on_{EventFamily}__*` scripts actually exist.
     """

archivebox/cli/archivebox_process.py

@@ -28,7 +28,6 @@ __command__ = "archivebox process"
 import sys
 
 import rich_click as click
-from rich import print as rprint
 
 from archivebox.cli.cli_util import apply_filters
 
@@ -38,46 +37,20 @@ from archivebox.cli.cli_util import apply_filters
 # =============================================================================
 
 
-def list_processes(
-    binary_name: str | None = None,
-    machine_id: str | None = None,
-    limit: int | None = None,
-) -> int:
-    """
-    List Processes as JSONL with optional filters.
-
-    Exit codes:
-        0: Success (even if no results)
-    """
-    from archivebox.misc.jsonl import write_record
+def list_processes(binary_name: str | None = None, machine_id: str | None = None, limit: int | None = None) -> int:
+    """List processes as JSONL, or exit codes and binary names in a terminal."""
     from archivebox.machine.models import Process
+    from archivebox.cli.cli_util import list_records
 
-    is_tty = sys.stdout.isatty()
+    def render(process):
+        name = process.binary.name if process.binary else "unknown"
+        exit_code = process.exit_code if process.exit_code is not None else "?"
+        color = "green" if process.exit_code == 0 else "red" if process.exit_code else "yellow"
+        return f"[{color}]exit={exit_code:3}[/{color}] [cyan]{name:15}[/cyan] [dim]{process.id}[/dim]"
 
-    queryset = Process.objects.all().select_related("binary", "machine").order_by("-started_at", "-created_at")
-
-    # Apply filters
-    filter_kwargs = {}
-    if binary_name:
-        filter_kwargs["binary__name"] = binary_name
-    if machine_id:
-        filter_kwargs["machine_id"] = machine_id
-
-    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
-
-    count = 0
-    for process in queryset:
-        if is_tty:
-            binary_name_str = process.binary.name if process.binary else "unknown"
-            exit_code = process.exit_code if process.exit_code is not None else "?"
-            status_color = "green" if process.exit_code == 0 else "red" if process.exit_code else "yellow"
-            rprint(f"[{status_color}]exit={exit_code:3}[/{status_color}] [cyan]{binary_name_str:15}[/cyan] [dim]{process.id}[/dim]")
-        else:
-            write_record(process.to_json())
-        count += 1
-
-    rprint(f"[dim]Listed {count} processes[/dim]", file=sys.stderr)
-    return 0
+    queryset = Process.objects.select_related("binary", "machine").order_by("-started_at", "-created_at")
+    filters = {"binary__name": binary_name or None, "machine_id": machine_id or None}
+    return list_records(apply_filters(queryset, filters, limit=limit), plural="processes", render=render)
 
 
 # =============================================================================
@@ -95,15 +68,9 @@ def main():
 @click.option("--binary-name", "-b", help="Filter by binary name")
 @click.option("--machine-id", "-m", help="Filter by machine ID")
 @click.option("--limit", "-n", type=int, help="Limit number of results")
-def list_cmd(binary_name: str | None, machine_id: str | None, limit: int | None):
+def list_cmd(**kwargs):
     """List Processes as JSONL."""
-    sys.exit(
-        list_processes(
-            binary_name=binary_name,
-            machine_id=machine_id,
-            limit=limit,
-        ),
-    )
+    sys.exit(list_processes(**kwargs))
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_remove.py

@@ -4,14 +4,17 @@ __package__ = "archivebox.cli"
 __command__ = "archivebox remove"
 
 import time
+from contextlib import nullcontext
 from pathlib import Path
 from collections.abc import Iterable
 
 import rich_click as click
+from rich.console import Console
 
 from django.db.models import QuerySet
 
 from archivebox.config import CONSTANTS
+from archivebox.config.common import get_config
 from archivebox.config.django import setup_django
 from archivebox.misc.util import enforce_types, docstring
 from archivebox.misc.checks import check_data_folder
@@ -20,7 +23,6 @@ from archivebox.misc.logging_util import (
     log_list_finished,
     log_removal_started,
     log_removal_finished,
-    TimedProgress,
 )
 from archivebox.cli.archivebox_snapshot import snapshot_filter_options
 
@@ -56,16 +58,14 @@ def remove(
     pattern_list = list(filter_patterns)
 
     log_list_started(pattern_list or None, filter_type)
-    timer = TimedProgress(360, prefix="      ")
-    try:
+    progress = Console().status("Finding matching snapshots...") if get_config().SHOW_PROGRESS else nullcontext()
+    with progress:
         if snapshots is None:
             snapshots = Snapshot.objects.order_by("-created_at").search(**filter_kwargs)
         # Freeze the target set up-front so a concurrent daemon writing new
         # snapshots can't extend the deletion under us, and so the cursor isn't
         # held open across the per-row deletes below.
         snapshot_pks = list(snapshots.values_list("pk", flat=True))
-    finally:
-        timer.end()
 
     if not snapshot_pks:
         log_removal_finished(0, 0)

archivebox/cli/archivebox_run.py

@@ -72,8 +72,8 @@ def process_stdin_records() -> int:
 
     Outputs JSONL of all processed records (for chaining).
 
-    Handles any record type: Crawl, Snapshot, ArchiveResult.
-    Auto-cascades: Crawl → Snapshots → ArchiveResults.
+    Handles Crawl and Snapshot work records. ArchiveResult records are accepted
+    as references to their parent Snapshot and plugin.
 
     Returns exit code (0 = success, 1 = error).
     """
@@ -90,6 +90,7 @@ def process_stdin_records() -> int:
     )
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.core.models import Snapshot, ArchiveResult
+    from archivebox.misc.db import uuid_ref_query
     from archivebox.crawls.models import Crawl
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
     from archivebox.machine.models import Binary
@@ -116,15 +117,7 @@ def process_stdin_records() -> int:
 
         try:
             if record_type == TYPE_CRAWL:
-                if record_id:
-                    # Existing crawl - re-queue
-                    try:
-                        crawl = Crawl.objects.get(id=record_id)
-                    except Crawl.DoesNotExist:
-                        crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
-                else:
-                    # New crawl - create it
-                    crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
+                crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
 
                 if crawl:
                     crawl.update_and_requeue(
@@ -137,14 +130,8 @@ def process_stdin_records() -> int:
                     queued_count += 1
 
             elif record_type == TYPE_SNAPSHOT or (record.get("url") and not record_type):
-                if record_id:
-                    # Existing snapshot - re-queue
-                    try:
-                        snapshot = Snapshot.objects.get(id=record_id)
-                    except Snapshot.DoesNotExist:
-                        snapshot = Snapshot.from_json(record, overrides={"created_by_id": created_by_id})
-                else:
-                    # New snapshot - create it
+                snapshot = Snapshot.objects.filter(id=record_id).first() if record_id else None
+                if snapshot is None:
                     snapshot = Snapshot.from_json(record, overrides={"created_by_id": created_by_id})
 
                 if snapshot:
@@ -156,33 +143,15 @@ def process_stdin_records() -> int:
                     queued_count += 1
 
             elif record_type == TYPE_ARCHIVERESULT:
-                if record_id:
-                    # Existing archiveresult - re-queue
-                    try:
-                        archiveresult = ArchiveResult.objects.get(id=record_id)
-                    except ArchiveResult.DoesNotExist:
-                        archiveresult = None
-                else:
-                    archiveresult = None
-
-                snapshot_id = record.get("snapshot_id")
-                plugin_name = record.get("plugin")
-                snapshot = None
-                if archiveresult:
-                    if archiveresult.status in [
-                        ArchiveResult.StatusChoices.FAILED,
-                        ArchiveResult.StatusChoices.SKIPPED,
-                        ArchiveResult.StatusChoices.NORESULTS,
-                        ArchiveResult.StatusChoices.BACKOFF,
-                    ]:
-                        archiveresult.reset_for_retry()
-                    snapshot = archiveresult.snapshot
-                    plugin_name = plugin_name or archiveresult.plugin
-                elif snapshot_id:
-                    try:
-                        snapshot = Snapshot.objects.get(id=snapshot_id)
-                    except Snapshot.DoesNotExist:
-                        snapshot = None
+                snapshot_id = str(record.get("snapshot_id") or "")
+                plugin_name = str(record.get("plugin") or "")
+                archiveresult = None
+                if not snapshot_id and record_id:
+                    archiveresult = ArchiveResult.objects.filter(uuid_ref_query("id", str(record_id))).select_related("snapshot").first()
+                    if archiveresult:
+                        snapshot_id = str(archiveresult.snapshot_id)
+                        plugin_name = plugin_name or archiveresult.plugin
+                snapshot = Snapshot.objects.filter(id=snapshot_id).first() if snapshot_id else None
 
                 if snapshot:
                     snapshot.queue_for_extraction()
@@ -190,16 +159,12 @@ def process_stdin_records() -> int:
                     snapshot_ids_by_crawl[crawl_id].add(str(snapshot.id))
                     if plugin_name:
                         plugin_names_by_crawl[crawl_id].add(str(plugin_name))
-                    output_records.append(record if not archiveresult else archiveresult.to_json())
+                    output_records.append(archiveresult.to_json() if archiveresult else record)
                     queued_count += 1
 
             elif record_type in {TYPE_BINARYREQUEST, TYPE_BINARY}:
-                if record_id:
-                    try:
-                        binary = Binary.objects.get(id=record_id)
-                    except Binary.DoesNotExist:
-                        binary = Binary.from_json(record)
-                else:
+                binary = Binary.objects.filter(id=record_id).first() if record_id else None
+                if binary is None:
                     binary = Binary.from_json(record)
 
                 if binary:
@@ -248,7 +213,6 @@ def process_stdin_records() -> int:
                     crawl_id,
                     snapshot_ids=None if crawl_id in full_crawl_ids else sorted(snapshot_ids_by_crawl[crawl_id]),
                     selected_plugins=None if crawl_id in run_all_plugins_for_crawl else sorted(plugin_names_by_crawl[crawl_id]),
-                    selected_plugins_are_explicit=False,
                 )
     return 0
 
@@ -257,7 +221,6 @@ def run_runner(
     daemon: bool = False,
     crawl_id: str | None = None,
     maintenance_only: bool = False,
-    maintenance_batch_size: int | None = None,
 ) -> int:
     """
     Run the background runner loop.
@@ -324,7 +287,6 @@ def run_runner(
                 crawl_id=crawl_id,
                 maintenance_only=maintenance_only,
                 interactive_interrupts=interactive_interrupts,
-                **({"maintenance_batch_size": maintenance_batch_size} if maintenance_batch_size else {}),
             )
         return 0
     except KeyboardInterrupt:
@@ -349,13 +311,6 @@ def run_runner(
 @click.option("--snapshot-id", help="Run one snapshot through its crawl")
 @click.option("--binary-id", help="Run one queued binary install directly on the bus")
 @click.option("--maintenance-only", is_flag=True, help="Only process sealed Snapshot maintenance and search-index backfills")
-@click.option(
-    "--maintenance-batch-size",
-    type=int,
-    default=None,
-    hidden=True,
-    help="Limit queued maintenance snapshots claimed per scheduler tick",
-)
 @click.option("--no-stdin", is_flag=True, hidden=True, help="Run the scheduler even when stdin is not a TTY")
 def main(
     daemon: bool,
@@ -363,7 +318,6 @@ def main(
     snapshot_id: str,
     binary_id: str,
     maintenance_only: bool,
-    maintenance_batch_size: int | None,
     no_stdin: bool,
 ):
     """
@@ -388,7 +342,7 @@ def main(
                 ),
                 foreground_parent_watchdog(enabled=False),
             ):
-                sys.exit(run_runner(daemon=True, maintenance_only=maintenance_only, maintenance_batch_size=maintenance_batch_size))
+                sys.exit(run_runner(daemon=True, maintenance_only=maintenance_only))
         except KeyboardInterrupt:
             sys.exit(0)
 
@@ -417,17 +371,16 @@ def main(
                     daemon=False,
                     crawl_id=crawl_id,
                     maintenance_only=maintenance_only,
-                    maintenance_batch_size=maintenance_batch_size,
                 ),
             )
 
         if maintenance_only:
-            sys.exit(run_runner(daemon=daemon, maintenance_only=True, maintenance_batch_size=maintenance_batch_size))
+            sys.exit(run_runner(daemon=daemon, maintenance_only=True))
 
         if not no_stdin and not sys.stdin.isatty():
             sys.exit(process_stdin_records())
         else:
-            sys.exit(run_runner(daemon=daemon, maintenance_only=maintenance_only, maintenance_batch_size=maintenance_batch_size))
+            sys.exit(run_runner(daemon=daemon, maintenance_only=maintenance_only))
 
 
 def run_snapshot_worker(snapshot_id: str) -> int:

archivebox/cli/archivebox_schedule.py

@@ -37,6 +37,7 @@ def schedule(
         "created_schedule_ids": [],
         "disabled_count": 0,
         "run_all_enqueued": 0,
+        "run_all_maintained": 0,
         "active_schedule_ids": [],
     }
 
@@ -57,12 +58,12 @@ def schedule(
 
         created_by_id = get_or_create_system_user_pk()
         is_update_schedule = not import_path
-        template_urls = import_path or "archivebox://update"
+        template_urls = import_path or ""
         template_label = (f"Scheduled import: {template_urls}" if import_path else "Scheduled ArchiveBox update")[:64]
         template_notes = (
             f"Created by archivebox schedule for {template_urls}"
             if import_path
-            else "Created by archivebox schedule to queue recurring archivebox://update maintenance crawls."
+            else "Created by archivebox schedule to run recurring ArchiveBox maintenance."
         )
 
         template = Crawl.objects.create(
@@ -76,7 +77,6 @@ def schedule(
             retry_at=None,
             config={
                 "DEPTH": 0 if is_update_schedule else depth,
-                "SCHEDULE_KIND": "update" if is_update_schedule else "crawl",
                 # Caller-supplied overrides (e.g. {"ONLY_NEW": False}) win over the
                 # template defaults. Anything left unset falls through to the
                 # standard config stack at crawl-resolution time.
@@ -86,6 +86,7 @@ def schedule(
         crawl_schedule = CrawlSchedule.objects.create(
             template=template,
             schedule=schedule_str,
+            config={**template.config, "SCHEDULE_KIND": "update" if is_update_schedule else "crawl"},
             is_enabled=True,
             label=template_label,
             notes=template_notes,
@@ -119,12 +120,19 @@ def schedule(
 
     if run_all:
         enqueued = 0
+        maintained = 0
         now = timezone.now()
         for scheduled_crawl in schedules:
-            scheduled_crawl.enqueue(queued_at=now)
-            enqueued += 1
+            if scheduled_crawl.dispatch(queued_at=now) is None:
+                maintained += 1
+            else:
+                enqueued += 1
         result["run_all_enqueued"] = enqueued
+        result["run_all_maintained"] = maintained
         print(f"[green]\\[*] Enqueued {enqueued} scheduled crawl(s) immediately.[/green]")
+        if maintained:
+            run_pending_crawls(maintenance_only=True)
+            print(f"[green]\\[*] Ran {maintained} scheduled maintenance update(s).[/green]")
         if enqueued:
             print(
                 "[yellow]\\[*] Start `archivebox server`, `archivebox run --daemon`, or `archivebox schedule --foreground` to process the queued crawls.[/yellow]",
@@ -171,7 +179,7 @@ def schedule(
 @click.option("--clear", is_flag=True, help="Disable all currently enabled schedules")
 @click.option("--show", is_flag=True, help="Print all currently enabled schedules")
 @click.option("--foreground", "-f", is_flag=True, help="Run the global crawl runner in the foreground (no crontab required)")
-@click.option("--run-all", is_flag=True, help="Enqueue all enabled schedules immediately and process them once")
+@click.option("--run-all", is_flag=True, help="Dispatch all enabled schedules immediately")
 @click.argument("import_path", required=False)
 @docstring(schedule.__doc__)
 def main(**kwargs):

archivebox/cli/archivebox_snapshot.py

@@ -144,15 +144,7 @@ def create_snapshots(
                     write_record(record)
 
                 # Input is a Crawl - get or create it, then create Snapshots for its URLs
-                crawl = None
-                crawl_id = record.get("id")
-                if crawl_id:
-                    try:
-                        crawl = Crawl.objects.get(id=crawl_id)
-                    except Crawl.DoesNotExist:
-                        crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
-                else:
-                    crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
+                crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
 
                 if not crawl:
                     continue
@@ -174,7 +166,7 @@ def create_snapshots(
                         "depth": depth,
                         "status": status,
                     }
-                    snapshot = Snapshot.from_json(snapshot_record, overrides={"created_by_id": created_by_id})
+                    snapshot = Snapshot.from_json(snapshot_record, overrides={"crawl": crawl, "created_by_id": created_by_id})
                     if snapshot:
                         created_snapshots.append(snapshot)
                         if not is_tty:
@@ -368,77 +360,29 @@ def list_snapshots(
 # =============================================================================
 
 
-def update_snapshots(
-    status: str | None = None,
-    tag: str | None = None,
-) -> int:
-    """
-    Update Snapshots from stdin JSONL.
-
-    Reads Snapshot records from stdin and applies updates.
-    Uses PATCH semantics - only specified fields are updated.
-
-    Exit codes:
-        0: Success
-        1: No input or error
-    """
+def update_snapshots(status: str | None = None, tag: str | None = None) -> int:
+    """Update JSONL-selected snapshots through their lifecycle operations."""
     from django.utils import timezone
-
-    from archivebox.misc.jsonl import read_stdin, write_record
     from archivebox.core.models import Snapshot
+    from archivebox.cli.cli_util import update_records, update_record_status
 
-    is_tty = sys.stdout.isatty()
+    def update(snapshot):
+        if status:
+            try:
+                update_record_status(snapshot, status)
+            except ValueError as err:
+                rprint(f"[red]{err}[/red]", file=sys.stderr)
+                return False
+        if tag:
+            from archivebox.core.models import Tag
 
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
+            tag_obj, _ = Tag.objects.get_or_create(name=tag)
+            snapshot.tags.add(tag_obj)
+            snapshot.safe_update({"modified_at": timezone.now()}, refresh=False)
+        if not status and not tag:
+            snapshot.safe_update({"modified_at": timezone.now()}, refresh=False)
 
-    updated_count = 0
-    for record in records:
-        snapshot_id = record.get("id")
-        if not snapshot_id:
-            continue
-
-        try:
-            snapshot = Snapshot.objects.get(id=snapshot_id)
-
-            if status:
-                if status not in Snapshot.StatusChoices.values:
-                    rprint(f"[red]Invalid snapshot status: {status}[/red]", file=sys.stderr)
-                    continue
-                if status == Snapshot.StatusChoices.SEALED:
-                    snapshot.cancel()
-                elif status == Snapshot.StatusChoices.PAUSED:
-                    snapshot.pause()
-                elif status == Snapshot.StatusChoices.QUEUED:
-                    if snapshot.status == Snapshot.StatusChoices.PAUSED:
-                        snapshot.resume()
-                    else:
-                        snapshot.update_and_requeue(status=Snapshot.StatusChoices.QUEUED, retry_at=timezone.now())
-                elif status == Snapshot.StatusChoices.STARTED:
-                    snapshot.update_and_requeue(status=Snapshot.StatusChoices.STARTED, retry_at=timezone.now())
-            if tag:
-                from archivebox.core.models import Tag
-
-                tag_obj, _ = Tag.objects.get_or_create(name=tag)
-                snapshot.tags.add(tag_obj)
-                snapshot.safe_update({"modified_at": timezone.now()}, refresh=False)
-
-            if not status and not tag:
-                snapshot.safe_update({"modified_at": timezone.now()}, refresh=False)
-            updated_count += 1
-
-            if not is_tty:
-                snapshot.refresh_from_db()
-                write_record(snapshot.to_json())
-
-        except Snapshot.DoesNotExist:
-            rprint(f"[yellow]Snapshot not found: {snapshot_id}[/yellow]", file=sys.stderr)
-            continue
-
-    rprint(f"[green]Updated {updated_count} snapshots[/green]", file=sys.stderr)
-    return 0
+    return update_records(Snapshot, update, plural="snapshots", refresh=True)
 
 
 # =============================================================================
@@ -447,50 +391,18 @@ def update_snapshots(
 
 
 def delete_snapshots(yes: bool = False, dry_run: bool = False) -> int:
-    """
-    Delete Snapshots from stdin JSONL.
-
-    Requires --yes flag to confirm deletion.
-
-    Exit codes:
-        0: Success
-        1: No input or missing --yes flag
-    """
-    from archivebox.misc.jsonl import read_stdin
+    """Delete snapshots selected by stdin JSONL; --yes confirms, --dry-run previews."""
+    from archivebox.cli.cli_util import delete_records
     from archivebox.core.models import Snapshot
 
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    snapshot_ids = [r.get("id") for r in records if r.get("id")]
-
-    if not snapshot_ids:
-        rprint("[yellow]No valid snapshot IDs in input[/yellow]", file=sys.stderr)
-        return 1
-
-    snapshots = Snapshot.objects.filter(id__in=snapshot_ids)
-    count = snapshots.count()
-
-    if count == 0:
-        rprint("[yellow]No matching snapshots found[/yellow]", file=sys.stderr)
-        return 0
-
-    if dry_run:
-        rprint(f"[yellow]Would delete {count} snapshots (dry run)[/yellow]", file=sys.stderr)
-        for snapshot in snapshots:
-            rprint(f"  [dim]{snapshot.id}[/dim] {snapshot.url[:60]}", file=sys.stderr)
-        return 0
-
-    if not yes:
-        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
-        return 1
-
-    # Perform deletion
-    deleted_count, _ = snapshots.delete()
-    rprint(f"[green]Deleted {deleted_count} snapshots[/green]", file=sys.stderr)
-    return 0
+    return delete_records(
+        Snapshot,
+        label="snapshot",
+        plural="snapshots",
+        preview=lambda obj: f"[dim]{obj.id}[/dim] {obj.url[:60]}",
+        yes=yes,
+        dry_run=dry_run,
+    )
 
 
 # =============================================================================
@@ -525,17 +437,17 @@ def list_cmd(**kwargs):
 @main.command("update")
 @click.option("--status", "-s", help="Set status")
 @click.option("--tag", "-t", help="Add tag")
-def update_cmd(status: str | None, tag: str | None):
+def update_cmd(**kwargs):
     """Update Snapshots from stdin JSONL."""
-    sys.exit(update_snapshots(status=status, tag=tag))
+    sys.exit(update_snapshots(**kwargs))
 
 
 @main.command("delete")
 @click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
 @click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(yes: bool, dry_run: bool):
+def delete_cmd(**kwargs):
     """Delete Snapshots from stdin JSONL."""
-    sys.exit(delete_snapshots(yes=yes, dry_run=dry_run))
+    sys.exit(delete_snapshots(**kwargs))
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_tag.py

@@ -70,7 +70,7 @@ def create_tags(names: Iterable[str]) -> int:
         if not name:
             continue
 
-        tag, created = Tag.objects.get_or_create(name=name)
+        tag, created = Tag.get_or_create_by_name(name)
 
         if not is_tty:
             write_record(tag.to_json())
@@ -90,42 +90,17 @@ def create_tags(names: Iterable[str]) -> int:
 # =============================================================================
 
 
-def list_tags(
-    name: str | None = None,
-    name__icontains: str | None = None,
-    limit: int | None = None,
-) -> int:
-    """
-    List Tags as JSONL with optional filters.
-
-    Exit codes:
-        0: Success (even if no results)
-    """
-    from archivebox.misc.jsonl import write_record
+def list_tags(name: str | None = None, name__icontains: str | None = None, limit: int | None = None) -> int:
+    """List tags as JSONL, or formatted rows in a terminal."""
     from archivebox.core.models import Tag
+    from archivebox.cli.cli_util import list_records
 
-    is_tty = sys.stdout.isatty()
-
-    queryset = Tag.objects.all().order_by("name")
-
-    # Apply filters
-    filter_kwargs = {
-        "name": name,
-        "name__icontains": name__icontains,
-    }
-    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
-
-    count = 0
-    for tag in queryset:
-        snapshot_count = tag.snapshot_set.count()
-        if is_tty:
-            rprint(f"[cyan]{tag.name:30}[/cyan] [dim]({snapshot_count} snapshots)[/dim]")
-        else:
-            write_record(tag.to_json())
-        count += 1
-
-    rprint(f"[dim]Listed {count} tags[/dim]", file=sys.stderr)
-    return 0
+    queryset = apply_filters(Tag.objects.order_by("name"), {"name": name, "name__icontains": name__icontains}, limit=limit)
+    return list_records(
+        queryset,
+        plural="tags",
+        render=lambda tag: f"[cyan]{tag.name:30}[/cyan] [dim]({tag.snapshot_set.count()} snapshots)[/dim]",
+    )
 
 
 # =============================================================================
@@ -134,56 +109,15 @@ def list_tags(
 
 
 def update_tags(name: str | None = None) -> int:
-    """
-    Update Tags from stdin JSONL.
-
-    Reads Tag records from stdin and applies updates.
-    Uses PATCH semantics - only specified fields are updated.
-
-    Exit codes:
-        0: Success
-        1: No input or error
-    """
-    from archivebox.misc.jsonl import read_stdin, write_record
+    """Apply supplied fields to each JSONL-selected Tag."""
     from archivebox.core.models import Tag
+    from archivebox.cli.cli_util import update_records
 
-    is_tty = sys.stdout.isatty()
+    def update(tag):
+        if name:
+            tag.rename(name)
 
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    updated_count = 0
-    for record in records:
-        tag_id = record.get("id")
-        old_name = record.get("name")
-
-        if not tag_id and not old_name:
-            continue
-
-        try:
-            if tag_id:
-                tag = Tag.objects.get(id=tag_id)
-            else:
-                tag = Tag.objects.get(name=old_name)
-
-            # Apply updates from CLI flags
-            if name:
-                tag.name = name
-                tag.save()
-
-            updated_count += 1
-
-            if not is_tty:
-                write_record(tag.to_json())
-
-        except Tag.DoesNotExist:
-            rprint(f"[yellow]Tag not found: {tag_id or old_name}[/yellow]", file=sys.stderr)
-            continue
-
-    rprint(f"[green]Updated {updated_count} tags[/green]", file=sys.stderr)
-    return 0
+    return update_records(Tag, update, plural="tags", by_name=True)
 
 
 # =============================================================================
@@ -192,65 +126,19 @@ def update_tags(name: str | None = None) -> int:
 
 
 def delete_tags(yes: bool = False, dry_run: bool = False) -> int:
-    """
-    Delete Tags from stdin JSONL.
-
-    Requires --yes flag to confirm deletion.
-
-    Exit codes:
-        0: Success
-        1: No input or missing --yes flag
-    """
-    from archivebox.misc.jsonl import read_stdin
+    """Delete tags selected by stdin JSONL; --yes confirms, --dry-run previews."""
+    from archivebox.cli.cli_util import delete_records
     from archivebox.core.models import Tag
 
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    # Collect tag IDs or names
-    tag_ids = []
-    tag_names = []
-    for r in records:
-        if r.get("id"):
-            tag_ids.append(r["id"])
-        elif r.get("name"):
-            tag_names.append(r["name"])
-
-    if not tag_ids and not tag_names:
-        rprint("[yellow]No valid tag IDs or names in input[/yellow]", file=sys.stderr)
-        return 1
-
-    from django.db.models import Q
-
-    query = Q()
-    if tag_ids:
-        query |= Q(id__in=tag_ids)
-    if tag_names:
-        query |= Q(name__in=tag_names)
-
-    tags = Tag.objects.filter(query)
-    count = tags.count()
-
-    if count == 0:
-        rprint("[yellow]No matching tags found[/yellow]", file=sys.stderr)
-        return 0
-
-    if dry_run:
-        rprint(f"[yellow]Would delete {count} tags (dry run)[/yellow]", file=sys.stderr)
-        for tag in tags:
-            rprint(f"  {tag.name}", file=sys.stderr)
-        return 0
-
-    if not yes:
-        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
-        return 1
-
-    # Perform deletion
-    deleted_count, _ = tags.delete()
-    rprint(f"[green]Deleted {deleted_count} tags[/green]", file=sys.stderr)
-    return 0
+    return delete_records(
+        Tag,
+        label="tag",
+        plural="tags",
+        preview=lambda obj: obj.name,
+        yes=yes,
+        dry_run=dry_run,
+        by_name=True,
+    )
 
 
 # =============================================================================
@@ -275,24 +163,24 @@ def create_cmd(names: tuple):
 @click.option("--name", help="Filter by exact name")
 @click.option("--name__icontains", help="Filter by name contains")
 @click.option("--limit", "-n", type=int, help="Limit number of results")
-def list_cmd(name: str | None, name__icontains: str | None, limit: int | None):
+def list_cmd(**kwargs):
     """List Tags as JSONL."""
-    sys.exit(list_tags(name=name, name__icontains=name__icontains, limit=limit))
+    sys.exit(list_tags(**kwargs))
 
 
 @main.command("update")
 @click.option("--name", "-n", help="Set new name")
-def update_cmd(name: str | None):
+def update_cmd(**kwargs):
     """Update Tags from stdin JSONL."""
-    sys.exit(update_tags(name=name))
+    sys.exit(update_tags(**kwargs))
 
 
 @main.command("delete")
 @click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
 @click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(yes: bool, dry_run: bool):
+def delete_cmd(**kwargs):
     """Delete Tags from stdin JSONL."""
-    sys.exit(delete_tags(yes=yes, dry_run=dry_run))
+    sys.exit(delete_tags(**kwargs))
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_update.py

@@ -77,107 +77,71 @@ def reindex_snapshots(
 ) -> dict[str, Any]:
     from archivebox.cli.archivebox_extract import run_plugins
     from archivebox.core.models import ArchiveResult, Snapshot
-    from abx_dl.models import discover_plugins
+    from django.db.models import Exists, OuterRef
 
     # Search backfill is the one maintenance hook allowed to execute without
     # reopening a Snapshot. Restrict that exception to already-sealed rows;
-    # every open lifecycle state remains owned by the normal state machine.
+    # every open lifecycle state remains owned by the normal runner lifecycle.
     snapshots = snapshots.filter(status=Snapshot.StatusChoices.SEALED)
 
-    stats: dict[str, Any] = {"processed": 0, "requested": 0, "queued": 0, "skipped_queued": 0, "reindexed": 0, "snapshot_ids": []}
-    records: list[dict[str, str]] = []
-    plugins_by_name = discover_plugins(runtime="archivebox")
-    required_hooks_by_plugin = {
-        plugin_name: frozenset(hook.name for hook in plugins_by_name[plugin_name].filter_hooks("Snapshot"))
-        for plugin_name in search_plugins
-        if plugin_name in plugins_by_name
-    }
+    stats: dict[str, Any] = {"processed": 0, "requested": 0, "queued": 0, "reindexed": 0, "snapshot_ids": []}
+    print(f"[*] Reindexing missing search indexes with: {', '.join(search_plugins)}")
 
-    total = snapshots.count()
-    print(f"[*] Reindexing {total} snapshots with search plugins: {', '.join(search_plugins)}")
-
-    def run_batch() -> None:
-        if not records:
-            return
-        if wait_for_turn:
-            wait_for_turn()
-        batch_records = list(records)
-        snapshot_ids = {record["snapshot_id"] for record in batch_records}
-        plugin_names = {record["plugin"] for record in batch_records}
-        queued_rows = {
-            (str(snapshot_id), plugin_name, hook_name)
-            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
-                snapshot_id__in=snapshot_ids,
-                plugin__in=plugin_names,
-                status=ArchiveResult.StatusChoices.QUEUED,
-            ).values_list("snapshot_id", "plugin", "hook_name")
-        }
-        records_to_queue = []
-        for record in batch_records:
-            snapshot_id = record["snapshot_id"]
-            plugin_name = record["plugin"]
-            required_hooks = required_hooks_by_plugin.get(plugin_name, frozenset())
-            if required_hooks and all((snapshot_id, plugin_name, hook_name) in queued_rows for hook_name in required_hooks):
-                stats["skipped_queued"] += 1
-                continue
-            records_to_queue.append(record)
-        if not records_to_queue:
-            print(
-                f"    [{stats['processed']}/{total}] Already queued {len(batch_records)} index jobs",
-            )
-            records.clear()
-            return
-        # `archivebox update --index-only` intentionally breaks the usual
-        # "runner discovers work" rule by inserting synthetic queued
-        # ArchiveResult rows for search backends. run_plugins() keeps this as
-        # statement-sized UPDATE/bulk_create work, then bumps Snapshot.retry_at
-        # so the orchestrator owns actual hook execution. Paused snapshots stay
-        # PAUSED; run_due_snapshot restores retry_at=MAX after targeted rows
-        # finish.
-        exit_code = run_plugins(
-            args=(),
-            records=records_to_queue,
-            wait=False,
-            emit_results=False,
-            show_progress=False,
-            preserve_queued=True,
+    completed_statuses = [ArchiveResult.StatusChoices.SUCCEEDED, ArchiveResult.StatusChoices.NORESULTS]
+    for plugin_name in search_plugins:
+        completed_result = ArchiveResult.objects.filter(
+            snapshot_id=OuterRef("pk"),
+            plugin=plugin_name,
+            status__in=completed_statuses,
         )
-        if exit_code != 0:
-            raise SystemExit(exit_code)
-        stats["queued"] += len(records_to_queue)
-        print(
-            f"    [{stats['processed']}/{total}] Queued {len(records_to_queue)} index jobs for orchestrator",
-        )
-        records.clear()
-
-    for snapshot in snapshots.select_related("crawl").paged_iterator(chunk_size=batch_size):
-        try:
-            stats["processed"] += 1
-
-            if _get_snapshot_crawl(snapshot) is None:
-                continue
-
+        candidates = snapshots.annotate(has_completed_index=Exists(completed_result)).filter(has_completed_index=False).order_by("id")
+        after_id = None
+        while True:
+            if wait_for_turn:
+                wait_for_turn()
+            page = candidates.filter(id__gt=after_id) if after_id is not None else candidates
+            batch = list(page.select_related(None).only("id", "timestamp")[:batch_size])
+            if not batch:
+                break
+            after_id = batch[-1].id
+            records = [{"type": "Snapshot", "id": str(snapshot.id)} for snapshot in batch]
+            stats["processed"] += len(batch)
+            stats["requested"] += len(batch)
             if collect_ids:
-                stats["snapshot_ids"].append(str(snapshot.id))
-            for plugin_name in search_plugins:
-                records.append(
-                    {
-                        "type": "ArchiveResult",
-                        "snapshot_id": str(snapshot.id),
-                        "plugin": plugin_name,
-                    },
-                )
-                stats["requested"] += 1
-            if len(records) >= batch_size:
-                run_batch()
-        except KeyboardInterrupt as err:
-            err.archivebox_resume = snapshot.timestamp
-            raise
-
-    run_batch()
+                stats["snapshot_ids"].extend(str(snapshot.id) for snapshot in batch)
+            exit_code = run_plugins(
+                args=(),
+                records=records,
+                plugins=plugin_name,
+                wait=True,
+                emit_results=False,
+                show_progress=False,
+            )
+            if exit_code != 0:
+                raise SystemExit(exit_code)
+            stats["reindexed"] += len(batch)
+            print(f"    [{plugin_name}] indexed {stats['reindexed']} missing snapshots")
     return stats
 
 
+def run_scheduled_maintenance(*, batch_size: int = 500) -> dict[str, Any]:
+    """Queue stale filesystem rows and backfill missing search facts."""
+    from archivebox.core.models import Snapshot
+
+    filesystem_stats = process_all_db_snapshots(batch_size=batch_size)
+    search_plugins = _get_search_indexing_plugins()
+    search_stats = (
+        reindex_snapshots(
+            Snapshot.objects.filter(fs_version=Snapshot._fs_current_version()),
+            search_plugins=search_plugins,
+            batch_size=batch_size,
+        )
+        if search_plugins
+        else {"processed": 0, "requested": 0, "queued": 0, "reindexed": 0, "snapshot_ids": []}
+    )
+    return {"filesystem": filesystem_stats, "search": search_stats}
+
+
 @enforce_types
 def update(
     filter_patterns: Iterable[str] = (),
@@ -203,9 +167,9 @@ def update(
     Update snapshots: migrate old dirs, reconcile DB, and re-queue for archiving.
 
     Three-phase operation (without filters):
-    - Phase 1: Drain old archive/ dirs by moving to new fs location (0.8.x → 0.9.x)
-    - Phase 2: O(n) scan over entire DB from most recent to least recent
-    - No orphan scans needed (trust 1:1 mapping between DB and filesystem after phase 1)
+    - Phase 1: Drain legacy archive/ directories into the current layout
+    - Phase 2: Select only stale fs_version rows through the indexed column
+    - Phase 3: Run queued snapshot-level filesystem maintenance until idle
 
     With filters: Only phase 2 (DB query), no filesystem operations.
     Without filters: All phases (full update).
@@ -232,7 +196,6 @@ def update(
     from archivebox.core.takeover_util import (
         command_owns_foreground_runner,
         current_command,
-        foreground_runner_owner,
         standby_until_foreground_runner_needed,
     )
     from archivebox.workers.supervisord_util import run_runner_worker, stop_own_supervisord_process
@@ -266,21 +229,20 @@ def update(
                 continue
             raise SystemExit(exit_code)
 
-    is_filtered_update = any(
-        (
-            filter_patterns,
-            status,
-            url__icontains,
-            url__istartswith,
-            tag,
-            crawl_id,
-            limit,
-            sort,
-            search,
-            before,
-            after,
-        ),
-    )
+    filters = {
+        "filter_patterns": filter_patterns,
+        "status": status,
+        "url__icontains": url__icontains,
+        "url__istartswith": url__istartswith,
+        "tag": tag,
+        "crawl_id": crawl_id,
+        "limit": limit,
+        "sort": sort,
+        "search": search,
+        "before": before,
+        "after": after,
+    }
+    is_filtered_update = any(filters.values())
     touched_snapshot_ids: set[str] = set()
     exit_code = 0
 
@@ -291,49 +253,20 @@ def update(
             while True:
                 do_migrate = migrate_only or not index_only
                 do_index = index_only or not migrate_only
-                do_run_until_idle = do_migrate or do_index
-                ran_post_migrate_runner = False
-                full_update_empty = False
-                maintenance_work_queued = False
-                runner_work_queued = False
 
                 if do_migrate:
-                    if (
-                        filter_patterns
-                        or status
-                        or url__icontains
-                        or url__istartswith
-                        or tag
-                        or crawl_id
-                        or limit
-                        or sort
-                        or search
-                        or before
-                        or after
-                    ):
+                    if is_filtered_update:
                         print("[*] Processing filtered snapshots from database...")
                         stats = process_filtered_snapshots(
-                            filter_patterns=filter_patterns,
+                            **filters,
                             filter_type=filter_type,
-                            status=status,
-                            url__icontains=url__icontains,
-                            url__istartswith=url__istartswith,
-                            tag=tag,
-                            crawl_id=crawl_id,
-                            limit=limit,
-                            sort=sort,
-                            search=search,
-                            before=before,
-                            after=after,
                             resume=resume,
                             batch_size=batch_size,
-                            queue_for_archiving=do_run_until_idle,
+                            queue_for_archiving=True,
                             wait_for_turn=wait_for_turn,
                         )
                         print_stats(stats)
                         touched_snapshot_ids.update(stats.get("snapshot_ids", []))
-                        maintenance_work_queued = stats.get("queued", 0) > 0
-                        runner_work_queued = runner_work_queued or maintenance_work_queued
                     else:
                         stats_combined = {"phase1": {}, "phase2": {}}
 
@@ -343,162 +276,44 @@ def update(
                             batch_size=batch_size,
                         )
 
-                        print("[*] Phase 2: Processing all database snapshots (most recent first)...")
+                        print("[*] Phase 2: Selecting database snapshots with stale filesystem versions...")
                         stats_combined["phase2"] = process_all_db_snapshots(
                             batch_size=batch_size,
                             resume=resume,
                             wait_for_turn=wait_for_turn,
                         )
                         print_combined_stats(stats_combined)
-                        full_update_empty = (
-                            stats_combined["phase1"].get("processed", 0) == 0 and stats_combined["phase2"].get("snapshots", 0) == 0
-                        )
-                        maintenance_work_queued = any(
-                            (
-                                stats_combined["phase1"].get("queued", 0),
-                                stats_combined["phase2"].get("queued", 0),
-                                stats_combined["phase2"].get("crawls_sealed", 0),
-                            ),
-                        )
-                        runner_work_queued = runner_work_queued or maintenance_work_queued
-
-                    if do_run_until_idle:
-                        # Filesystem migration is maintenance on existing
-                        # Snapshot rows: Snapshot.save() moves archive/<ts> to
-                        # the current output_dir and preserves the lifecycle
-                        # status. Drain those retry_at ticks before queuing
-                        # search backfill below. Otherwise the sealed search
-                        # runner branch correctly sees queued ArchiveResult
-                        # rows first, runs the targeted plugins, and may leave
-                        # the fs_version maintenance tick hidden behind that
-                        # plugin work until another update pass.
-                        if full_update_empty:
-                            print("[*] No snapshots or legacy archive directories found; skipping filesystem maintenance runner.")
-                        elif not maintenance_work_queued:
-                            print("[*] No filesystem maintenance work queued; skipping filesystem maintenance runner.")
-                        else:
-                            print("[*] Phase 3: Running filesystem maintenance until idle...")
-                        if full_update_empty:
-                            pass
-                        elif not maintenance_work_queued:
-                            pass
-                        elif is_filtered_update:
-                            if not touched_snapshot_ids:
-                                print("[*] No matching snapshots queued work for the runner.")
-                            for snapshot_id in sorted(touched_snapshot_ids):
-                                run_scoped_runner("--snapshot-id", snapshot_id)
-                        else:
-                            run_scoped_runner("--maintenance-only", "--maintenance-batch-size", str(batch_size))
-                        ran_post_migrate_runner = True
-
-                if do_index:
-                    if full_update_empty:
-                        print("[*] No snapshots found; skipping search indexing backfill.")
-                    else:
-                        search_plugins = _get_search_indexing_plugins()
-                        if not search_plugins:
-                            print("[*] No search indexing plugins are available, nothing to backfill.")
-                        else:
-                            snapshots = _build_filtered_snapshots_queryset(
-                                filter_patterns=filter_patterns,
-                                filter_type=filter_type,
-                                status=status,
-                                url__icontains=url__icontains,
-                                url__istartswith=url__istartswith,
-                                tag=tag,
-                                crawl_id=crawl_id,
-                                limit=limit,
-                                sort=sort,
-                                search=search,
-                                before=before,
-                                after=after,
-                                resume=resume,
-                            )
-                            from django.db.models import Exists, OuterRef, Q
-                            from django.utils import timezone
-                            from archivebox.core.models import ArchiveResult, Snapshot
-
-                            scoped_snapshot_ids = snapshots.order_by().values("id") if is_filtered_update else None
-                            queued_index_results = ArchiveResult.objects.filter(
-                                status=ArchiveResult.StatusChoices.QUEUED,
-                                plugin__in=search_plugins,
-                            )
-                            if scoped_snapshot_ids is not None:
-                                queued_index_results = queued_index_results.filter(snapshot_id__in=scoped_snapshot_ids)
-
-                            if queued_index_results.exists():
-                                runner_work_queued = True
-                                now = timezone.now()
-                                queued_result_for_snapshot = queued_index_results.filter(snapshot_id=OuterRef("pk"))
-                                snapshots_to_wake = (
-                                    Snapshot.objects.filter(status=Snapshot.StatusChoices.SEALED)
-                                    .annotate(
-                                        has_queued_index_result=Exists(queued_result_for_snapshot),
-                                    )
-                                    .filter(
-                                        has_queued_index_result=True,
-                                    )
-                                    .filter(
-                                        Q(retry_at__isnull=True) | Q(retry_at__gt=now),
-                                    )
-                                )
-                                if scoped_snapshot_ids is not None:
-                                    snapshots_to_wake = snapshots_to_wake.filter(id__in=scoped_snapshot_ids)
-                                woken_count = snapshots_to_wake.update(
-                                    retry_at=now,
-                                    modified_at=now,
-                                )
-                                print(
-                                    "[*] Existing queued search index jobs found; "
-                                    f"skipping backfill scan and waking {woken_count} snapshot(s) for the runner.",
-                                )
-                            else:
-                                collect_index_ids = (
-                                    is_filtered_update
-                                    or foreground_runner_owner(
-                                        data_dir=CONSTANTS.DATA_DIR,
-                                        exclude_id=command.id,
-                                    )
-                                    is not None
-                                )
-                                stats = reindex_snapshots(
-                                    snapshots,
-                                    search_plugins=search_plugins,
-                                    batch_size=batch_size,
-                                    collect_ids=collect_index_ids,
-                                    wait_for_turn=wait_for_turn,
-                                )
-                                print_index_stats(stats)
-                                touched_snapshot_ids.update(stats.get("snapshot_ids", []))
-                                runner_work_queued = runner_work_queued or stats["queued"] > 0
-
-                if do_run_until_idle and (do_index or not ran_post_migrate_runner):
-                    # Search/index backfill intentionally queues targeted
-                    # ArchiveResult rows without reopening sealed snapshots.
-                    # This second runner pass drains those plugin
-                    # rows after filesystem maintenance has had its own turn.
-                    # For a normal unfiltered `archivebox update`, keep the
-                    # historical final pass broad enough to resume genuinely
-                    # queued/interrupted crawl work after maintenance is done.
-                    if full_update_empty:
-                        print("[*] No snapshots found; skipping queued/interrupted crawl runner.")
-                    elif not runner_work_queued:
-                        print("[*] No queued/interrupted crawl work found; skipping queued/interrupted crawl runner.")
-                    else:
-                        print("[*] Phase 3: Running queued/interrupted crawl work until idle...")
-                    if full_update_empty:
-                        pass
-                    elif not runner_work_queued:
-                        pass
-                    elif touched_snapshot_ids and is_filtered_update:
-                        if not touched_snapshot_ids:
-                            print("[*] No matching snapshots queued work for the runner.")
+                    # The due selectors are indexed and cheap when empty, so
+                    # always drain them instead of preceding the runner with
+                    # whole-table counts merely to decide whether to call it.
+                    print("[*] Phase 3: Running filesystem maintenance until idle...")
+                    if is_filtered_update:
                         for snapshot_id in sorted(touched_snapshot_ids):
                             run_scoped_runner("--snapshot-id", snapshot_id)
+                    elif migrate_only:
+                        run_scoped_runner("--maintenance-only")
                     else:
-                        run_scoped_runner(
-                            *(["--maintenance-only", "--maintenance-batch-size", str(batch_size)] if index_only or migrate_only else []),
+                        run_scoped_runner()
+
+                if do_index:
+                    search_plugins = _get_search_indexing_plugins()
+                    if not search_plugins:
+                        print("[*] No search indexing plugins are available, nothing to backfill.")
+                    else:
+                        snapshots = _build_filtered_snapshots_queryset(
+                            **filters,
+                            filter_type=filter_type,
+                            resume=resume,
                         )
+                        stats = reindex_snapshots(
+                            snapshots,
+                            search_plugins=search_plugins,
+                            batch_size=batch_size,
+                            collect_ids=is_filtered_update,
+                            wait_for_turn=wait_for_turn,
+                        )
+                        print_index_stats(stats)
+                        touched_snapshot_ids.update(stats.get("snapshot_ids", []))
 
                 if not continuous:
                     break
@@ -514,32 +329,17 @@ def update(
             resume_cmd.append("--migrate-only")
         if index_only:
             resume_cmd.append("--index-only")
-        if batch_size != 500:
-            resume_cmd.extend(["--batch-size", str(batch_size)])
-        if exact_resume or resume:
-            resume_cmd.extend(["--resume", str(exact_resume or resume)])
-        if before is not None:
-            resume_cmd.extend(["--before", str(before)])
-        if after is not None:
-            resume_cmd.extend(["--after", str(after)])
-        if filter_type != "exact":
-            resume_cmd.extend(["--filter-type", filter_type])
-        if status:
-            resume_cmd.extend(["--status", status])
-        if url__icontains:
-            resume_cmd.extend(["--url__icontains", url__icontains])
-        if url__istartswith:
-            resume_cmd.extend(["--url__istartswith", url__istartswith])
-        if tag:
-            resume_cmd.extend(["--tag", tag])
-        if crawl_id:
-            resume_cmd.extend(["--crawl-id", crawl_id])
-        if limit:
-            resume_cmd.extend(["--limit", str(limit)])
-        if sort:
-            resume_cmd.extend(["--sort", sort])
-        if search:
-            resume_cmd.extend(["--search", search])
+        resume_options = {
+            "batch-size": batch_size if batch_size != 500 else None,
+            "resume": exact_resume or resume,
+            "filter-type": filter_type if filter_type != "exact" else None,
+            **{key: value for key, value in filters.items() if key != "filter_patterns"},
+        }
+        for name, value in resume_options.items():
+            # Zero is a meaningful date boundary, but remains an unset limit.
+            if value or (name in {"before", "after"} and value is not None):
+                option = "crawl-id" if name == "crawl_id" else name
+                resume_cmd.extend([f"--{option}", str(value)])
         resume_cmd.extend(str(pattern) for pattern in filter_patterns)
         print("\n[red][X] archivebox update interrupted.[/red]")
         print("[yellow]Hint: resume this idempotent update with:[/yellow]")
@@ -737,18 +537,8 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500
 
 
 def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, wait_for_turn=None) -> dict[str, int]:
-    """
-    O(n) scan over entire DB from most recent to least recent.
-
-    For each snapshot:
-      1. Reconcile index.json with DB (merge titles, tags, archive results)
-      2. Mark migrated snapshots sealed unless explicitly re-queued elsewhere
-
-    No orphan detection needed - we trust 1:1 mapping between DB and filesystem
-    after Phase 1 has drained all old archive/ directories.
-    """
+    """Queue only snapshots whose indexed filesystem version is stale."""
     from archivebox.core.models import Snapshot
-    from archivebox.crawls.models import Crawl
     from django.db.models import Q
     from django.utils import timezone
 
@@ -761,128 +551,35 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
         "sealed": 0,
         "crawls_sealed": 0,
     }
-    current_fs_version = Snapshot._fs_current_version()
-
-    queryset = Snapshot.objects.all()
+    queryset = Snapshot.objects.filter(fs_version__in=Snapshot._FS_VERSION_MIGRATION_PATHS)
     if resume:
         queryset = queryset.filter(timestamp__lte=resume)
-    total = queryset.count()
-    stats["snapshots"] = total
-    print(f"[*] Processing {total} snapshots from database (most recent first)...")
-
-    def update_in_batches(rows, *, label: str, **updates) -> int:
-        updated = 0
-        checked = 0
-        while True:
-            if wait_for_turn:
-                wait_for_turn()
-            batch = list(rows.only("id", "modified_at").order_by("-timestamp")[:batch_size])
-            if not batch:
-                if updated:
-                    print(f"    [{label}] complete: {updated} rows updated")
-                return updated
-            checked += len(batch)
-            print(f"    [{label}] updating next {len(batch)} rows (seen {checked})...")
-            for snapshot in batch:
-                # This maintenance scan intentionally bypasses save(); it is
-                # only normalizing scheduler fields, and Snapshot.save() may
-                # do filesystem migration work that belongs in the runner.
-                # Guard each single-row UPDATE with modified_at so stale scan
-                # pages cannot overwrite newer runner/admin writes.
-                updated += int(
-                    snapshot.safe_update(
-                        updates,
-                        refresh=False,
-                        extra_filter={"modified_at": snapshot.modified_at},
-                    ),
-                )
-            print(f"    [{label}] updated {updated} rows so far")
-
-    now = timezone.now()
-    updated_rows = update_in_batches(
-        queryset.exclude(
-            status__in=[
-                Snapshot.StatusChoices.QUEUED,
-                Snapshot.StatusChoices.STARTED,
-                Snapshot.StatusChoices.PAUSED,
-                Snapshot.StatusChoices.SEALED,
-            ],
-        ),
-        label="snapshot status normalization",
-        status=Snapshot.StatusChoices.SEALED,
-        retry_at=None,
-        modified_at=now,
-    )
-    stats["sealed"] += updated_rows
-    stats["updated_db"] += updated_rows
-
-    fs_version_rows = queryset.exclude(fs_version=current_fs_version).filter(Q(retry_at__isnull=True) | Q(retry_at__gt=now))
-    stale_batch = []
-
-    def queue_stale_fs_batch() -> None:
-        if not stale_batch:
-            return
+    initial_now = timezone.now()
+    rows_to_wake = queryset.filter(Q(retry_at__isnull=True) | Q(retry_at__gt=initial_now))
+    after_id = None
+    while True:
         if wait_for_turn:
             wait_for_turn()
+        page = rows_to_wake.filter(id__gt=after_id) if after_id is not None else rows_to_wake
+        batch = list(page.only("id", "fs_version", "modified_at").order_by("id")[:batch_size])
+        if not batch:
+            break
+        after_id = batch[-1].id
         now = timezone.now()
-        # Do not bump fs_version here. The orchestrator calls Snapshot.save(),
-        # which performs the idempotent filesystem migration and commits the new
-        # fs_version in the same serialized worker path as normal crawls.
         updated = 0
-        for snapshot in stale_batch:
-            # Each row gets its own short autocommit UPDATE because this scan
-            # can touch millions of snapshots while a server is also alive.
-            # The modified_at predicate is the CAS guard: if the runner or
-            # admin changed the snapshot after paged_iterator read it, skip it
-            # and let the newer state decide whether migration is still due.
+        for snapshot in batch:
             updated += int(
                 snapshot.safe_update(
-                    {
-                        "retry_at": now,
-                        "modified_at": now,
-                    },
+                    {"retry_at": now, "modified_at": now},
                     refresh=False,
                     extra_filter={"fs_version": snapshot.fs_version},
                 ),
             )
-        stats["processed"] += len(stale_batch)
+        stats["processed"] += len(batch)
         stats["updated_db"] += updated
         stats["queued"] += updated
-        print(f"    [{stats['processed']}/{total}] Queued {updated} filesystem migrations for orchestrator...")
-        stale_batch.clear()
-
-    for snapshot in (
-        fs_version_rows.only("id", "crawl_id", "timestamp", "fs_version", "modified_at")
-        .order_by("-timestamp")
-        .paged_iterator(chunk_size=batch_size)
-    ):
-        try:
-            stale_batch.append(snapshot)
-            if len(stale_batch) >= batch_size:
-                queue_stale_fs_batch()
-        except KeyboardInterrupt as err:
-            err.archivebox_resume = snapshot.timestamp
-            raise
-    queue_stale_fs_batch()
-
-    now = timezone.now()
-    # Crawls with no open child snapshots are already finished. Seal them here
-    # instead of waking the foreground runner; otherwise migration/update can
-    # accidentally re-enter full crawl execution for historical rows.
-    stats["crawls_sealed"] = (
-        Crawl.objects.filter(
-            status__in=Crawl.RUNNABLE_STATES,
-        )
-        .exclude(
-            snapshot_set__status__in=Snapshot.OPEN_STATES,
-        )
-        .update(
-            status=Crawl.StatusChoices.SEALED,
-            retry_at=None,
-            modified_at=now,
-        )
-    )
-    stats["updated_db"] += stats["crawls_sealed"]
+        print(f"    Queued {stats['queued']} stale filesystem snapshots so far...")
+    stats["snapshots"] = stats["processed"]
     return stats
 
 
@@ -942,8 +639,6 @@ def process_filtered_snapshots(
             stats["snapshot_ids"].append(str(snapshot.id))
             update_values = {}
             updated = 0
-            if not isinstance(snapshot.current_step, int):
-                update_values["current_step"] = 0
             if queue_for_archiving:
                 update_values.update(
                     {
@@ -1029,10 +724,9 @@ def print_index_stats(stats: dict[str, Any]) -> None:
 
     print(f"""
 [green]Search Reindex Complete[/green]
-  Scanned rows:      {stats["processed"]}
-  Requested jobs:    {stats.get("requested", stats["queued"])}
-  Queued index jobs: {stats["queued"]}
-  Already queued:    {stats.get("skipped_queued", 0)}
+  Missing index runs: {stats["processed"]}
+  Requested runs:    {stats.get("requested", 0)}
+  Indexed snapshots: {stats.get("reindexed", 0)}
 """)
 
 

archivebox/cli/archivebox_version.py

@@ -47,13 +47,7 @@ def _format_binary_abspath(
                 continue
 
             relative_str = relative.as_posix()
-            if prefix == "./":
-                return "." if not relative_str else f"./{relative_str}"
-            if prefix == "ABXPKG_LIB_DIR/":
-                return "ABXPKG_LIB_DIR" if not relative_str else f"ABXPKG_LIB_DIR/{relative_str}"
-            if prefix == "PERSONAS_DIR/":
-                return "PERSONAS_DIR" if not relative_str else f"PERSONAS_DIR/{relative_str}"
-            return "~" if not relative_str else f"~/{relative_str}"
+            return prefix + relative_str if relative_str else prefix.rstrip("/")
 
     return normalized.as_posix()
 
@@ -61,22 +55,9 @@ def _format_binary_abspath(
 def _render_binary_abspath(abspath: str):
     from rich.text import Text
 
-    if abspath.startswith("ABXPKG_LIB_DIR/"):
-        return Text.assemble(("ABXPKG_LIB_DIR", "bright_blue"), (abspath.removeprefix("ABXPKG_LIB_DIR"), "green"))
-    if abspath == "ABXPKG_LIB_DIR":
-        return Text("ABXPKG_LIB_DIR", style="bright_blue")
-    if abspath.startswith("PERSONAS_DIR/"):
-        return Text.assemble(("PERSONAS_DIR", "medium_purple"), (abspath.removeprefix("PERSONAS_DIR"), "green"))
-    if abspath == "PERSONAS_DIR":
-        return Text("PERSONAS_DIR", style="medium_purple")
-    if abspath.startswith("~/"):
-        return Text.assemble(("~", "cyan"), (abspath.removeprefix("~"), "green"))
-    if abspath == "~":
-        return Text("~", style="cyan")
-    if abspath.startswith("./"):
-        return Text.assemble((".", "cyan"), (abspath.removeprefix("."), "green"))
-    if abspath == ".":
-        return Text(".", style="cyan")
+    for prefix, color in (("ABXPKG_LIB_DIR", "bright_blue"), ("PERSONAS_DIR", "medium_purple"), ("~", "cyan"), (".", "cyan")):
+        if abspath == prefix or abspath.startswith(prefix + "/"):
+            return Text.assemble((prefix, color), (abspath.removeprefix(prefix), "green"))
     return Text(abspath, style="green")
 
 
@@ -201,9 +182,6 @@ def version(
     else:
         requested_names = {name for name in (binaries or ()) if name}
 
-    def binary_is_requested(logical_name: str, actual_name: str, display_name: str) -> bool:
-        return not requested_names or bool({logical_name, actual_name, display_name} & requested_names)
-
     def plugin_may_have_requested_binary(plugin) -> bool:
         if not requested_names:
             return True
@@ -262,18 +240,16 @@ def version(
 
     prnt("[pale_green1][i] Binary Dependencies:[/pale_green1]")
     failures = []
-    seen_failures: set[str] = set()
     seen_rows: set[tuple[str, str, str, str]] = set()
 
-    from archivebox.plugins.discovery import get_enabled_plugins
+    from archivebox.plugins.discovery import get_enabled_plugins, get_plugin_catalog
     from abx_dl.config import get_required_binary_requests
     from abx_dl.dependencies import resolve_binary_requests
-    from abx_dl.models import discover_plugins, filter_plugins
     from abx_dl.orchestrator import create_bus
     from abxpkg.binary_service import BinaryEvent, BinaryService
 
-    plugins = discover_plugins(runtime="archivebox")
-    enabled_plugins = filter_plugins(plugins, get_enabled_plugins(config=config), include_providers=True)
+    plugins = get_plugin_catalog()
+    enabled_plugins = plugins.select(get_enabled_plugins(config=config))
     enabled_plugin_names = set(enabled_plugins)
     runtime_config = normalize_runtime_config(config.for_crawl(), json_safe=False)
     derived_config: dict[str, object] = {}
@@ -299,10 +275,10 @@ def version(
             prnt("", f"[yellow]Warning: Could not query collection binary records; resolving through abxpkg: {e}[/yellow]")
 
     declared_binary_specs: dict[str, dict[str, object]] = {}
+    plugin_binaries = []
     for plugin_name, plugin in plugins.items():
         if not plugin_may_have_requested_binary(plugin):
             continue
-        plugin_requested = bool(requested_names)
         plugin_enabled = plugin_name in enabled_plugin_names
         binary_records = get_required_binary_requests(
             plugin,
@@ -319,8 +295,7 @@ def version(
                 else actual_name
             )
             display_name = logical_name
-            if not plugin_requested and not binary_is_requested(logical_name, actual_name, display_name):
-                continue
+            plugin_binaries.append((plugin_name, plugin_enabled, display_name, binary_record))
             if not plugin_enabled and not requested_names:
                 continue
             if _binary_record_matches_runtime(db_binaries.get(logical_name), config.ABXPKG_LIB_DIR):
@@ -343,156 +318,81 @@ def version(
         loaded_binaries = asyncio.run(resolve_declared_binaries())
 
     rows: list[dict[str, object]] = []
-    any_rows = False
     any_available = False
     compact_paths = console.is_terminal
-    live_enabled = console.is_terminal
-    live_cm = Live(binary_dependency_table(rows), console=console, refresh_per_second=8) if live_enabled else None
+    live_cm = Live(binary_dependency_table(rows), console=console, refresh_per_second=8) if console.is_terminal else None
 
-    def emit_row(row: dict[str, object]) -> None:
-        rows.append(row)
+    def emit_binary(plugin_name, plugin_enabled, display_name, binary, *, dim=False):
+        nonlocal any_available
+        valid = binary is not None
+        abspath = binary.abspath if valid else ""
+        version_str = str(binary.version or "unknown")[:15] if valid else "-"
+        provider = str(binary.binprovider or "env")[:8] if valid else "-"
+        any_available |= valid
+        if not valid and plugin_enabled and display_name not in failures:
+            failures.append(display_name)
+        row_key = _binary_row_dedupe_key(
+            display_name=display_name,
+            valid=valid,
+            version=version_str,
+            provider=provider,
+            abspath=abspath,
+        )
+        if row_key in seen_rows:
+            return
+        seen_rows.add(row_key)
+        rendered_path = "[grey53]not installed[/grey53]"
+        if valid:
+            display_path = (
+                _format_binary_abspath(
+                    abspath,
+                    pwd=Path.cwd(),
+                    lib_dir=config.ABXPKG_LIB_DIR,
+                    personas_dir=CONSTANTS.PERSONAS_DIR,
+                    home=Path.home(),
+                )
+                if compact_paths
+                else abspath
+            )
+            rendered_path = _render_binary_abspath(display_path) if compact_paths else display_path
+        rows.append(
+            {
+                "plugin": plugin_name,
+                "status": binary_dependency_status(enabled=plugin_enabled, valid=valid),
+                "binary": display_name,
+                "version": version_str,
+                "provider": provider,
+                "path": rendered_path,
+                "style": "dim" if dim or not plugin_enabled else "",
+            },
+        )
         if live_cm is not None:
             live_cm.update(binary_dependency_table(rows), refresh=True)
 
     if live_cm is not None:
         live_cm.start()
     try:
-        for plugin_name, plugin in plugins.items():
-            if not plugin_may_have_requested_binary(plugin):
-                continue
-            plugin_requested = bool(requested_names)
-            plugin_enabled = plugin_name in enabled_plugin_names
-            binary_records = get_required_binary_requests(
-                plugin,
-                plugin.config.required_binaries,
-                overrides=runtime_config,
-                derived_overrides=derived_config,
-                run_output_dir=CONSTANTS.DATA_DIR,
-            )
-            for binary_record in binary_records:
-                actual_name = str(binary_record["name"])
-                logical_name = (
-                    Path(actual_name).expanduser().name
-                    if ("/" in actual_name or "\\" in actual_name or actual_name.startswith("~"))
-                    else actual_name
-                )
-                display_name = logical_name
-                if not plugin_requested and not binary_is_requested(logical_name, actual_name, display_name):
+        for plugin_name, plugin_enabled, display_name, binary_record in plugin_binaries:
+            installed = db_binaries.get(display_name) if db_available else None
+            if not _binary_record_matches_runtime(installed, config.ABXPKG_LIB_DIR):
+                # Resolve only enabled or explicitly requested dependencies. Cold-loading
+                # every optional provider makes status and migration checks needlessly slow.
+                if not plugin_enabled and not requested_names:
                     continue
-
-                installed = db_binaries.get(logical_name) if db_available else None
-                if _binary_record_matches_runtime(installed, config.ABXPKG_LIB_DIR):
-                    abspath = installed.abspath
-                    version_str = (installed.version or "unknown")[:15]
-                    provider = (installed.binprovider or "env")[:8]
-                    valid = True
-                elif not plugin_enabled and not requested_names:
-                    # `archivebox version` is expected to verify the active runtime, not
-                    # cold-load every optional plugin provider. Migration and status
-                    # checks often run with PLUGINS narrowed to a tiny set; resolving
-                    # disabled plugin binaries there can spend most of the command on
-                    # providers the current collection will never execute.
-                    continue
-                else:
-                    loaded = loaded_binaries[json.dumps(binary_record, sort_keys=True, default=str)]
-                    abspath = loaded.abspath if loaded is not None else ""
-                    version_str = str(loaded.version or "unknown")[:15] if loaded is not None else "unknown"
-                    provider = str(loaded.binprovider or "env")[:8] if loaded is not None else "env"
-                    valid = loaded is not None
-
-                any_rows = True
-                if valid:
-                    display_path = (
-                        _format_binary_abspath(
-                            abspath,
-                            pwd=Path.cwd(),
-                            lib_dir=config.ABXPKG_LIB_DIR,
-                            personas_dir=CONSTANTS.PERSONAS_DIR,
-                            home=Path.home(),
-                        )
-                        if compact_paths
-                        else abspath
-                    )
-                    rendered_path = _render_binary_abspath(display_path) if compact_paths else display_path
-                    any_available = True
-                else:
-                    rendered_path = "[grey53]not installed[/grey53]"
-                    if plugin_enabled and display_name not in seen_failures:
-                        failures.append(display_name)
-                        seen_failures.add(display_name)
-
-                row_key = _binary_row_dedupe_key(
-                    display_name=display_name,
-                    valid=valid,
-                    version=version_str if valid else "-",
-                    provider=provider if valid else "-",
-                    abspath=abspath,
-                )
-                if row_key in seen_rows:
-                    continue
-                seen_rows.add(row_key)
-
-                emit_row(
-                    {
-                        "plugin": plugin_name,
-                        "status": binary_dependency_status(enabled=plugin_enabled, valid=valid),
-                        "binary": display_name,
-                        "version": version_str if valid else "-",
-                        "provider": provider if valid else "-",
-                        "path": rendered_path,
-                        "style": "" if plugin_enabled else "dim",
-                    },
-                )
+                installed = loaded_binaries[json.dumps(binary_record, sort_keys=True, default=str)]
+            emit_binary(plugin_name, plugin_enabled, display_name, installed)
 
         if db_available:
             for binary_name, installed in db_binaries.items():
-                if requested_names and binary_name not in requested_names:
-                    continue
-                abspath = installed.abspath
-                version_str = (installed.version or "unknown")[:15]
-                provider = (installed.binprovider or "env")[:8]
-                row_key = _binary_row_dedupe_key(
-                    display_name=binary_name,
-                    valid=True,
-                    version=version_str,
-                    provider=provider,
-                    abspath=abspath,
-                )
-                if row_key in seen_rows:
-                    continue
-                seen_rows.add(row_key)
-
-                display_path = (
-                    _format_binary_abspath(
-                        abspath,
-                        pwd=Path.cwd(),
-                        lib_dir=config.ABXPKG_LIB_DIR,
-                        personas_dir=CONSTANTS.PERSONAS_DIR,
-                        home=Path.home(),
-                    )
-                    if compact_paths
-                    else abspath
-                )
-                emit_row(
-                    {
-                        "plugin": "(database)",
-                        "status": binary_dependency_status(enabled=True, valid=True),
-                        "binary": binary_name,
-                        "version": version_str,
-                        "provider": provider,
-                        "path": _render_binary_abspath(display_path) if compact_paths else display_path,
-                        "style": "dim",
-                    },
-                )
-                any_rows = True
-                any_available = True
+                if not requested_names or binary_name in requested_names:
+                    emit_binary("(database)", True, binary_name, installed, dim=True)
     finally:
         if live_cm is not None:
             live_cm.stop()
         else:
             prnt(binary_dependency_table(rows))
 
-    if not any_rows:
+    if not rows:
         prnt("", "[grey53]No required binaries declared for discovered plugins.[/grey53]")
     elif not any_available:
         prnt("", "[grey53]No binaries detected. Run [green]archivebox install[/green] to detect dependencies.[/grey53]")

archivebox/cli/cli_util.py

@@ -42,3 +42,139 @@ def apply_filters(queryset, filter_kwargs: dict, limit: int | None = None):
         queryset = queryset[:limit]
 
     return queryset
+
+
+def delete_records(
+    model,
+    *,
+    label: str,
+    plural: str,
+    preview,
+    yes: bool,
+    dry_run: bool,
+    preview_limit: int | None = None,
+    by_name: bool = False,
+) -> int:
+    """Delete a JSONL selection with one confirmation and dry-run contract.
+
+    Keep queryset deletion so Django's cascades and model cleanup signals run.
+    Persona deletion has additional filesystem policy and stays with that command.
+    """
+    import sys
+
+    from django.db.models import Q
+    from rich import print as rprint
+
+    from archivebox.misc.jsonl import read_stdin
+
+    records = list(read_stdin())
+    if not records:
+        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+        return 1
+    ids = [record["id"] for record in records if record.get("id")]
+    names = [record["name"] for record in records if by_name and not record.get("id") and record.get("name")]
+    if not ids and not names:
+        rprint(f"[yellow]No valid {label} IDs{' or names' if by_name else ''} in input[/yellow]", file=sys.stderr)
+        return 1
+    query = Q(id__in=ids)
+    if names:
+        query |= Q(name__in=names)
+    queryset = model.objects.filter(query)
+    count = queryset.count()
+    if not count:
+        rprint(f"[yellow]No matching {plural} found[/yellow]", file=sys.stderr)
+        return 0
+    if dry_run:
+        rprint(f"[yellow]Would delete {count} {plural} (dry run)[/yellow]", file=sys.stderr)
+        for obj in queryset[:preview_limit] if preview_limit is not None else queryset:
+            rprint(f"  {preview(obj)}", file=sys.stderr)
+        if preview_limit is not None and count > preview_limit:
+            rprint(f"  ... and {count - preview_limit} more", file=sys.stderr)
+        return 0
+    if not yes:
+        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
+        return 1
+    deleted_count, _ = queryset.delete()
+    rprint(f"[green]Deleted {deleted_count} {plural}[/green]", file=sys.stderr)
+    return 0
+
+
+def update_record_status(record, status: str) -> None:
+    """Route CLI status changes through the model's lifecycle operations."""
+    from django.utils import timezone
+
+    if status not in record.StatusChoices.values:
+        raise ValueError(f"Invalid {record._meta.model_name} status: {status}")
+    transitions = {"sealed": record.cancel, "paused": record.pause}
+    if status == "queued" and record.is_paused:
+        record.resume()
+    elif transition := transitions.get(status):
+        transition()
+    else:
+        record.update_and_requeue(status=status, retry_at=timezone.now())
+
+
+def list_records(queryset, *, plural: str, render) -> int:
+    """Stream JSONL when piped; render rows and a count in a terminal."""
+    import sys
+    from rich import print as rprint
+    from archivebox.misc.jsonl import write_record
+
+    is_tty = sys.stdout.isatty()
+    count = 0
+    for record in queryset:
+        if is_tty:
+            rprint(render(record))
+        else:
+            write_record(record.to_json())
+        count += 1
+    rprint(f"[dim]Listed {count} {plural}[/dim]", file=sys.stderr)
+    return 0
+
+
+def format_status(status: str, width: int) -> str:
+    colors = {
+        "queued": "yellow",
+        "started": "blue",
+        "sealed": "green",
+        "succeeded": "green",
+        "failed": "red",
+        "backoff": "magenta",
+    }
+    color = colors.get(status, "dim")
+    return f"[{color}]{status:{width}}[/{color}]"
+
+
+def update_records(model, update, *, plural: str, by_name: bool = False, refresh: bool = False) -> int:
+    """Apply a command's PATCH operation to each JSONL-selected model.
+
+    Returning False from the operation skips that row. Model operations stay in
+    their commands so lifecycle changes and ordinary field saves remain explicit.
+    """
+    import sys
+    from rich import print as rprint
+    from archivebox.misc.jsonl import read_stdin, write_record
+
+    records = list(read_stdin())
+    if not records:
+        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+        return 1
+    is_tty = sys.stdout.isatty()
+    count = 0
+    for record in records:
+        key = "id" if record.get("id") else "name" if by_name and record.get("name") else None
+        if key is None:
+            continue
+        try:
+            obj = model.objects.get(**{key: record[key]})
+            if update(obj) is False:
+                continue
+            count += 1
+            if not is_tty:
+                if refresh:
+                    obj.refresh_from_db()
+                write_record(obj.to_json())
+        except model.DoesNotExist:
+            rprint(f"[yellow]{model.__name__} not found: {record[key]}[/yellow]", file=sys.stderr)
+    rprint(f"[green]Updated {count} {plural}[/green]", file=sys.stderr)
+    return 0

archivebox/config/collection.py

@@ -17,7 +17,7 @@ CONFIG_FILE_HEADER = (
     "# You can add options here manually in INI format, or automatically by running:\n"
     "#    archivebox config --set KEY=VALUE\n"
     "#\n"
-    "# This file is kept in sync 1:1 with Machine.config in the index DB —\n"
+    "# This file is kept in sync 1:1 with the current Machine.config in the DB —\n"
     "# editing either side propagates to the other. ``archivebox init`` reads\n"
     "# this file on startup; the admin Machine.config editor writes both.\n"
     "#\n"
@@ -160,7 +160,7 @@ def _coerce_from_str_dict(file_config: dict[str, str]) -> dict[str, Any]:
     so they round-trip through INI's string-only storage. When reading the
     file back into ``Machine.config`` (a JSONField that holds native types)
     those strings have to be decoded — otherwise downstream consumers like
-    ``_emit_machine_config`` → ``MachineEvent`` → abx-dl see a JSON string
+    ``MachineEvent`` → abx-dl see a JSON string
     where they expect a dict and raise ``TypeError``.
     Declared fields go through pydantic-settings' own ``field_is_complex`` /
     ``prepare_field_value`` so they're decoded per annotation. Undeclared

archivebox/config/common.py

@@ -16,9 +16,9 @@ from pathlib import Path
 from typing import Any, ClassVar, cast
 from urllib.parse import quote, urlparse
 
-from abx_plugins.plugins.base.utils import BASE_CONFIG_PATH, build_config_model, resolve_plugin_configs
+from abx_plugins.plugins.base.utils import build_config_model
 from django.db import DatabaseError
-from pydantic import BaseModel, Field, PrivateAttr, create_model, field_validator, model_validator
+from pydantic import BaseModel, Field, PrivateAttr, TypeAdapter, create_model, field_validator, model_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
 from rich.console import Console
 
@@ -441,6 +441,7 @@ class ArchivingConfig(BaseConfigSet):
         default="",
         description="Comma-separated plugin selection for this run. Empty means use enabled plugin defaults.",
     )
+    PARSER: str = Field(default="auto", description="Parser used to read the submitted Crawl input document.")
 
     ONLY_NEW: bool = Field(default=True)
     INDEX_ONLY: bool = Field(default=False)
@@ -460,7 +461,8 @@ class ArchivingConfig(BaseConfigSet):
     USER_AGENT: str = Field(
         default=f"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)",
     )
-    COOKIES_FILE: Path | None = Field(default=None)
+    COOKIES_FILE: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
+    AUTH_STORAGE_FILE: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
 
     URL_DENYLIST: str = Field(default=r"\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$", alias="URL_BLACKLIST")
     URL_ALLOWLIST: str | None = Field(default=None, alias="URL_WHITELIST")
@@ -601,13 +603,9 @@ def _explicit_plugin_enabled_keys(config: Mapping[str, object]) -> set[str]:
 
 
 def _discover_plugin_config_schemas() -> PluginSchemaDocuments:
-    from archivebox.plugins.discovery import discover_plugin_configs
+    from archivebox.plugins.discovery import get_plugin_config_resolver
 
-    schemas: PluginSchemaDocuments = {}
-    if BASE_CONFIG_PATH.exists():
-        schemas["base"] = json.loads(BASE_CONFIG_PATH.read_text())
-    schemas.update(discover_plugin_configs())
-    return schemas
+    return get_plugin_config_resolver().schemas
 
 
 def _plugin_config_properties(plugin_schemas: PluginSchemaDocuments) -> dict[str, dict[str, Any]]:
@@ -856,13 +854,11 @@ class ArchiveBoxBaseConfig(
             disabled_plugins = [plugin_name for plugin_name, enabled_key in enabled_config_keys.items() if not getattr(self, enabled_key)]
         selected_plugin_roots = plugin_names
         if selected_plugin_roots:
-            from abx_dl.models import discover_plugins, filter_plugins
+            from archivebox.plugins.discovery import get_plugin_catalog
 
             selected_plugins = set(
-                filter_plugins(
-                    discover_plugins(runtime="archivebox"),
+                get_plugin_catalog().select(
                     sorted(selected_plugin_roots),
-                    include_providers=True,
                     disabled_names=disabled_plugins,
                 ),
             )
@@ -1195,7 +1191,6 @@ def get_config(
 
     explicit_plugin_enabled_keys: set[str] = set()
     if resolve_plugins:
-        plugin_schemas = {plugin_name: schema for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items() if isinstance(schema, dict)}
         plugin_global_config = {key: str(value) if isinstance(value, Path) else value for key, value in config_data.items()}
         crawl_selected_plugins = crawl_config_base and bool(_normalize_plugins_config_value(dict(crawl.config or {}).get("PLUGINS")))
         # A frozen crawl-level PLUGINS selector is the exact extractor set for
@@ -1219,8 +1214,9 @@ def get_config(
                 **_plugin_user_config(_plugin_input_config(file_config)),
                 **plugin_user_config,
             }
-        plugin_sections = resolve_plugin_configs(
-            plugin_schemas,
+        from archivebox.plugins.discovery import get_plugin_config_resolver
+
+        plugin_sections = get_plugin_config_resolver().resolve(
             global_config=plugin_global_config,
             user_config=plugin_user_config,
             environ={},
@@ -1258,7 +1254,7 @@ def get_config(
             config = config.model_copy(update={"SERVER_SECURITY_MODE": "safe-subdomains-fullreplay"})
     for key in explicit_plugin_enabled_keys:
         if key in config_data:
-            setattr(config, key, config_data[key])
+            setattr(config, key, TypeAdapter(bool).validate_python(config_data[key]))
     if config.PLUGINS:
         config._derive_plugin_enabled_config(respect_current_enabled=True)
     if redact_sensitive:

archivebox/core/admin_archiveresults.py

@@ -1,7 +1,5 @@
 __package__ = "archivebox.core"
 
-import html
-import json
 import os
 import shlex
 from functools import reduce
@@ -16,16 +14,16 @@ from django.core.exceptions import PermissionDenied, SuspiciousOperation, Valida
 from django.db.models import Count, Min, Prefetch, Q, Subquery, TextField, Window
 from django.db.models.functions import Cast
 from django.shortcuts import redirect
-from django.urls import resolve, reverse
-from django.utils import timezone
+from django.template.loader import render_to_string
+from django.urls import reverse
 from django.utils.html import format_html
 from django.utils.safestring import mark_safe
 from django.utils.text import smart_split
 
-from archivebox.base_models.admin import BaseModelAdmin
+from archivebox.base_models.admin import card_fieldset, BaseModelAdmin
 from archivebox.core.models import ArchiveResult, Snapshot
 from archivebox.core.routes_util import build_snapshot_url
-from archivebox.core.widgets import InlineTagEditorWidget
+from archivebox.core.widgets import InlineTagEditorWidget, render_copy_block
 from archivebox.machine.env_util import env_to_shell_exports
 from archivebox.misc.logging_util import printable_filesize
 from archivebox.misc.paginators import AcceleratedPaginator
@@ -116,302 +114,58 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None, can_del
     }
 
     rows = []
-    delete_url = html.escape(reverse("admin:core_archiveresult_changelist"), quote=True)
     for idx, result in enumerate(results):
         status = result.status or "queued"
         color, bg = status_colors.get(status, ("#6b7280", "#f3f4f6"))
-        output_files = result.output_files or {}
-        if isinstance(output_files, (dict, list, tuple, set)):
-            output_file_count = len(output_files)
-        elif isinstance(output_files, str):
-            try:
-                parsed = json.loads(output_files)
-                output_file_count = len(parsed) if isinstance(parsed, (dict, list, tuple, set)) else 0
-            except (TypeError, ValueError):
-                output_file_count = 0
-        else:
-            output_file_count = 0
-        output_size = int(result.output_size or 0)
-        output_size_display = html.escape(printable_filesize(output_size))
-
-        # Get plugin icon
-        icon = get_plugin_icon(result.plugin)
-
-        # Keep each timestamp component intact while allowing one wrap between them.
-        if result.end_ts:
-            end_time = (
-                f'<span style="white-space: nowrap;">{result.end_ts:%Y-%m-%d}</span>'
-                f'<wbr> <span style="white-space: nowrap;">{result.end_ts:%H:%M:%S}</span>'
-            )
-        else:
-            end_time = "-"
-
         process = result.process_record
-        process_display = "-"
-        if process:
-            process_url = html.escape(reverse("admin:machine_process_change", args=[process.id]), quote=True)
-            process_label = html.escape(get_process_link_label(process), quote=True)
-            process_display = f'''
-                <a href="{process_url}"
-                   style="color: #2563eb; text-decoration: none; font-family: ui-monospace, monospace; font-size: 12px;"
-                   title="View process">{process_label}</a>
-            '''
-
-        machine_display = "-"
-        if process and process.machine_id:
-            machine_url = html.escape(reverse("admin:machine_machine_change", args=[process.machine_id]), quote=True)
-            machine_label = html.escape(str(process.machine.hostname or ""), quote=True)
-            machine_display = f'''
-                <a href="{machine_url}"
-                   style="color: #2563eb; text-decoration: none; font-size: 12px;"
-                   title="View machine">{machine_label}</a>
-            '''
-
-        # Truncate output for display
-        full_output_raw = result.output_str_for_display() or "-"
-        output_display_raw = full_output_raw[:60]
-        if len(full_output_raw) > 60:
-            output_display_raw += "..."
-        full_output = html.escape(full_output_raw)
-        output_display = html.escape(output_display_raw)
-
-        display_cmd = build_abx_dl_display_command(result)
-        replay_cmd = build_abx_dl_replay_command(result, config=config)
-        cmd_str_escaped = html.escape(display_cmd)
-        cmd_attr = html.escape(replay_cmd, quote=True)
-
-        # Build output link - use embed_path() which checks output_files first
+        output = result.output_str_for_display() or "-"
         embed_path = result.embed_path()
-        snapshot_id = str(result.snapshot_id)
-        if embed_path and result.status == "succeeded":
-            output_link = build_snapshot_url(snapshot_id, embed_path, config=config)
-        else:
-            output_link = build_snapshot_url(snapshot_id, "", config=config)
-        output_link_attr = html.escape(output_link, quote=True)
-
-        # Get version - try cmd_version field
-        version = html.escape(str(result.cmd_version if result.cmd_version else "-"), quote=True)
-        plugin_text = html.escape(str(result.plugin or ""), quote=True)
-        status_text = html.escape(str(status), quote=True)
-        pwd_text = html.escape(str(result.pwd or "-"), quote=True)
-
-        # Unique ID for this row's expandable output
-        row_id = f"output_{idx}_{str(result.id)[:8]}"
-        delete_button = ""
-        if can_delete:
-            delete_button = f'''
-                <button type="button" data-archive-result-ids="{result.id}" data-delete-url="{delete_url}"
-                        title="Delete this output">×</button>
-            '''
-
-        rows.append(f'''
-            <tr data-output-size="{output_size}" style="border-bottom: 1px solid #f1f5f9; transition: background 0.15s;" onmouseover="this.style.background='#f8fafc'" onmouseout="this.style.background='transparent'">
-                <td style="padding: 10px 12px; white-space: nowrap;">
-                    <a href="{reverse("admin:core_archiveresult_change", args=[result.id])}"
-                       style="color: #2563eb; text-decoration: none; font-family: ui-monospace, monospace; font-size: 11px;"
-                       title="View/edit archive result">
-                        <code>{str(result.id)[-8:]}</code>
-                    </a>
-                </td>
-                <td style="padding: 10px 12px; white-space: nowrap;">
-                    <span style="display: inline-block; padding: 3px 10px; border-radius: 12px;
-                                 font-size: 11px; font-weight: 600; text-transform: uppercase;
-                                 color: {color}; background: {bg};">{status_text}</span>
-                </td>
-                <td style="padding: 10px 12px; white-space: nowrap; font-size: 20px;" title="{plugin_text}">
-                    {icon}
-                </td>
-                <td class="archive-results-plugin" style="padding: 10px 12px; font-weight: 500; color: #334155;">
-                        <a href="{output_link_attr}" target="_blank"
-                           style="color: #334155; text-decoration: none;"
-                       title="View output fullscreen"
-                       onmouseover="this.style.color='#2563eb'; this.style.textDecoration='underline';"
-                       onmouseout="this.style.color='#334155'; this.style.textDecoration='none';">
-                        {plugin_text}
-                    </a>
-                </td>
-                <td class="archive-results-output" style="padding: 10px 12px; max-width: 280px;">
-                    <span onclick="document.getElementById('{row_id}').open = !document.getElementById('{row_id}').open"
-                          style="display: block; max-width: 280px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
-                                 color: #2563eb; text-decoration: none; font-family: ui-monospace, monospace; font-size: 12px; cursor: pointer;"
-                          title="Click to expand full output">
-                        {output_display}
-                    </span>
-                </td>
-                <td class="archive-results-files" style="padding: 10px 12px; color: #64748b; font-size: 12px; text-align: right;">
-                    {output_file_count}
-                </td>
-                <td class="archive-results-size" style="padding: 10px 12px; color: #64748b; font-size: 12px; text-align: right; white-space: nowrap;">
-                    {output_size_display}
-                </td>
-                <td class="archive-results-completed" style="padding: 10px 12px; color: #64748b; font-size: 12px;">
-                    {end_time}
-                </td>
-                <td style="padding: 10px 12px; white-space: nowrap;">
-                    {process_display}
-                </td>
-                <td style="padding: 10px 12px; white-space: nowrap;">
-                    {machine_display}
-                </td>
-                <td style="padding: 10px 12px; white-space: nowrap; font-family: ui-monospace, monospace; font-size: 11px; color: #64748b;">
-                    {version}
-                </td>
-                <td class="archive-results-actions-cell">
-                    <div class="archive-results-actions">
-                        <a href="{output_link_attr}" target="_blank"
-                           title="View output">📄</a>
-                        <a href="{reverse("admin:core_archiveresult_change", args=[result.id])}"
-                           title="Edit">✏️</a>
-                        {delete_button}
-                    </div>
-                </td>
-            </tr>
-            <tr style="border-bottom: 1px solid #e2e8f0;">
-                <td colspan="12" style="padding: 0 12px 10px 12px;">
-                    <details id="{row_id}" style="margin: 0;">
-                        <summary style="cursor: pointer; font-size: 11px; color: #94a3b8; user-select: none;">
-                            Details &amp; Output
-                        </summary>
-                        <div style="margin-top: 8px; padding: 10px; background: #f8fafc; border: 1px solid #e2e8f0; border-radius: 6px; max-height: 200px; overflow: auto;">
-                            <div style="font-size: 11px; color: #64748b; margin-bottom: 8px;">
-                                <span style="margin-right: 16px;"><b>ID:</b> <code>{result.id!s}</code></span>
-                                <span style="margin-right: 16px;"><b>Version:</b> <code>{version}</code></span>
-                                <span style="margin-right: 16px;"><b>PWD:</b> <code>{pwd_text}</code></span>
-                            </div>
-                            <div style="font-size: 11px; color: #64748b; margin-bottom: 8px;">
-                                <b>Output:</b>
-                            </div>
-                            <pre style="margin: 0; padding: 8px; background: #1e293b; border-radius: 4px; color: #e2e8f0; font-size: 12px; white-space: pre-wrap; word-break: break-all; max-height: 120px; overflow: auto;">{full_output}</pre>
-                            <div style="font-size: 11px; color: #64748b; margin-top: 8px;">
-                                <b>Command:</b>
-                            </div>
-                            <div style="position: relative; margin: 0; padding: 8px 56px 8px 8px; background: #1e293b; border-radius: 4px;">
-                                <button type="button"
-                                        data-command="{cmd_attr}"
-                                        onclick="(function(btn){{var text=btn.dataset.command||''; if(navigator.clipboard&&navigator.clipboard.writeText){{navigator.clipboard.writeText(text);}} else {{var ta=document.createElement('textarea'); ta.value=text; document.body.appendChild(ta); ta.select(); document.execCommand('copy'); document.body.removeChild(ta);}}}})(this); return false;"
-                                        style="position: absolute; top: 6px; right: 6px; padding: 2px 8px; border: 0; border-radius: 4px; background: #334155; color: #e2e8f0; font-size: 11px; cursor: pointer;">
-                                    Copy
-                                </button>
-                                <code title="{cmd_attr}" style="display: block; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; color: #e2e8f0; font-size: 11px;">{cmd_str_escaped}</code>
-                            </div>
-                        </div>
-                    </details>
-                </td>
-            </tr>
-        ''')
-
-    total_count = results[0]._inline_total_count
-    footer = ""
-    if total_count > limit:
-        footer = f"""
-            <tr data-output-footer>
-                <td colspan="12" style="padding: 12px; text-align: center; color: #64748b; font-size: 13px; background: #f8fafc;">
-                    Showing {limit} of {total_count} results &nbsp;
-                    <a href="/admin/core/archiveresult/?snapshot__id__exact={results[0].snapshot_id if results else ""}"
-                       style="color: #2563eb;">View all →</a>
-                </td>
-            </tr>
-        """
-
-    return mark_safe(f"""
-        <div style="border: 1px solid #e2e8f0; border-radius: 8px; overflow-x: auto; overflow-y: hidden; background: #fff; width: 100%;">
-            <table class="archive-results-table" style="width: 100%; min-width: 1100px; border-collapse: collapse; font-size: 14px;">
-                <thead>
-                    <tr style="background: #f8fafc; border-bottom: 2px solid #e2e8f0;">
-                        <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Details</th>
-                        <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Status</th>
-                        <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; width: 32px;"></th>
-                        <th class="archive-results-plugin" style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Plugin</th>
-                        <th class="archive-results-output" style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Output</th>
-                        <th class="archive-results-files" style="padding: 10px 12px; text-align: right; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Files</th>
-                        <th class="archive-results-size" style="padding: 10px 12px; text-align: right; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">
-                            <button type="button" data-output-size-sort style="all: unset; cursor: pointer;">Size ↕</button>
-                        </th>
-                        <th class="archive-results-completed" style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Completed</th>
-                        <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Process</th>
-                        <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Machine</th>
-                        <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Version</th>
-                        <th class="archive-results-actions-cell" style="text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Actions</th>
-                    </tr>
-                </thead>
-                <tbody>
-                    {"".join(rows)}
-                    {footer}
-                </tbody>
-            </table>
-        </div>
-    """)
-
-
-class ArchiveResultInline(admin.TabularInline):
-    name = "Archive Results"
-    model = ArchiveResult
-    parent_model = Snapshot
-    extra = 0
-    sort_fields = ("end_ts", "plugin", "output_str", "status", "cmd_version")
-    readonly_fields = ("id", "result_id", "completed", "command", "version")
-    fields = ("start_ts", "end_ts", *readonly_fields, "plugin", "cmd", "cmd_version", "pwd", "status", "output_str")
-    ordering = ("end_ts",)
-    show_change_link = True
-
-    def get_parent_object_from_request(self, request):
-        resolved = resolve(request.path_info)
-        try:
-            return self.parent_model.objects.get(pk=resolved.kwargs["object_id"])
-        except (self.parent_model.DoesNotExist, ValidationError):
-            return None
-
-    @admin.display(
-        description="Completed",
-        ordering="end_ts",
-    )
-    def completed(self, obj):
-        return format_html('<p style="white-space: nowrap">{}</p>', obj.end_ts.strftime("%Y-%m-%d %H:%M:%S"))
-
-    def result_id(self, obj):
-        return format_html(
-            '<a href="{}"><code style="font-size: 10px">[{}]</code></a>',
-            reverse("admin:core_archiveresult_change", args=(obj.id,)),
-            str(obj.id)[:8],
+        rows.append(
+            {
+                "result": result,
+                "status": status,
+                "color": color,
+                "bg": bg,
+                "icon": get_plugin_icon(result.plugin),
+                "file_count": result.output_file_stats()[0],
+                "output_size": int(result.output_size or 0),
+                "output_size_display": printable_filesize(int(result.output_size or 0)),
+                "end_date": result.end_ts.strftime("%Y-%m-%d") if result.end_ts else "",
+                "end_time": result.end_ts.strftime("%H:%M:%S") if result.end_ts else "",
+                "process": process,
+                "process_url": reverse("admin:machine_process_change", args=[process.id]) if process else "",
+                "process_label": get_process_link_label(process) if process else "",
+                "machine_url": reverse("admin:machine_machine_change", args=[process.machine_id]) if process and process.machine_id else "",
+                "output": output,
+                "output_preview": output[:60] + ("..." if len(output) > 60 else ""),
+                "display_command": build_abx_dl_display_command(result),
+                "replay_command": build_abx_dl_replay_command(result, config=config),
+                "output_url": build_snapshot_url(
+                    str(result.snapshot_id),
+                    embed_path if embed_path and status == "succeeded" else "",
+                    config=config,
+                ),
+                "change_url": reverse("admin:core_archiveresult_change", args=[result.id]),
+                "version": result.cmd_version or "-",
+                "pwd": str(result.pwd or "-"),
+                "dom_id": f"output_{idx}_{str(result.id)[:8]}",
+                "short_id": str(result.id)[-8:],
+            },
         )
 
-    def command(self, obj):
-        return format_html("<small><code>{}</code></small>", " ".join(obj.cmd or []))
-
-    def version(self, obj):
-        return format_html("<small><code>{}</code></small>", obj.cmd_version or "-")
-
-    def get_formset(self, request, obj=None, **kwargs):
-        formset = super().get_formset(request, obj, **kwargs)
-        snapshot = self.get_parent_object_from_request(request)
-        base_fields = formset.form.base_fields
-        snapshot_output_dir = str(snapshot.output_dir) if snapshot else ""
-
-        # formset.form.base_fields['id'].widget = formset.form.base_fields['id'].hidden_widget()
-
-        # default values for new entries
-        base_fields["status"].initial = "succeeded"
-        base_fields["start_ts"].initial = timezone.now()
-        base_fields["end_ts"].initial = timezone.now()
-        base_fields["cmd_version"].initial = "-"
-        base_fields["pwd"].initial = snapshot_output_dir
-        base_fields["cmd"].initial = '["-"]'
-        base_fields["output_str"].initial = "Manually recorded cmd output..."
-
-        if obj is not None:
-            # hidden values for existing entries and new entries
-            base_fields["start_ts"].widget = base_fields["start_ts"].hidden_widget()
-            base_fields["end_ts"].widget = base_fields["end_ts"].hidden_widget()
-            base_fields["cmd"].widget = base_fields["cmd"].hidden_widget()
-            base_fields["pwd"].widget = base_fields["pwd"].hidden_widget()
-            base_fields["cmd_version"].widget = base_fields["cmd_version"].hidden_widget()
-        return formset
-
-    def get_readonly_fields(self, request, obj=None):
-        if obj is not None:
-            return self.readonly_fields
-        else:
-            return []
+    return mark_safe(
+        render_to_string(
+            "admin/core/archiveresult/inline.html",
+            {
+                "rows": rows,
+                "limit": limit,
+                "total_count": results[0]._inline_total_count,
+                "snapshot_id": results[0].snapshot_id,
+                "can_delete": can_delete,
+                "delete_url": reverse("admin:core_archiveresult_changelist"),
+            },
+        ),
+    )
 
 
 class ArchiveResultAdmin(BaseModelAdmin):
@@ -459,40 +213,14 @@ class ArchiveResultAdmin(BaseModelAdmin):
     autocomplete_fields = ("snapshot",)
 
     fieldsets = (
-        (
-            "Snapshot",
-            {
-                "fields": ("snapshot", "snapshot_info", "tags_str", "admin_actions"),
-                "classes": ("card", "wide"),
-            },
-        ),
-        (
-            "Plugin",
-            {
-                "fields": ("plugin_with_icon", "process_link", "status"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Timing",
-            {
-                "fields": ("start_ts", "end_ts", "created_at", "modified_at"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Command",
-            {
-                "fields": ("cmd", "cmd_str", "cmd_version", "pwd"),
-                "classes": ("card",),
-            },
-        ),
-        (
+        card_fieldset("Snapshot", ("snapshot", "snapshot_info", "tags_str", "admin_actions"), wide=True),
+        card_fieldset("Plugin", ("plugin_with_icon", "process_link", "status")),
+        card_fieldset("Timing", ("start_ts", "end_ts", "created_at", "modified_at")),
+        card_fieldset("Command", ("cmd", "cmd_str", "cmd_version", "pwd")),
+        card_fieldset(
             "Output",
-            {
-                "fields": ("output_str", "output_json", "output_files", "output_size", "output_mimetypes", "output_summary"),
-                "classes": ("card", "wide"),
-            },
+            ("output_str", "output_json", "output_files", "output_size", "output_mimetypes", "output_summary"),
+            wide=True,
         ),
     )
 
@@ -588,18 +316,7 @@ class ArchiveResultAdmin(BaseModelAdmin):
             self.list_per_page = saved_list_per_page
 
     def get_queryset(self, request):
-        ordering = request.GET.get("o")
-        ordering_fields = set()
-        if ordering:
-            for part in ordering.split("."):
-                if not part:
-                    continue
-                try:
-                    idx = abs(int(part)) - 1
-                except ValueError:
-                    continue
-                if 0 <= idx < len(self.list_display):
-                    ordering_fields.add(self.list_display[idx])
+        ordering_fields = self.get_ordering_fields(request)
 
         qs = (
             super()
@@ -800,26 +517,9 @@ class ArchiveResultAdmin(BaseModelAdmin):
 
     @admin.display(description="Command")
     def cmd_str(self, result):
-        request = self.request
-        display_cmd = build_abx_dl_display_command(result)
-        replay_cmd = build_abx_dl_replay_command(result, config=request.archivebox_config)
-        return format_html(
-            """
-            <div style="position: relative; width: 100%; max-width: 100%; overflow: hidden; box-sizing: border-box;">
-                <button type="button"
-                        data-command="{}"
-                        onclick="(function(btn){{var text=btn.dataset.command||''; if(navigator.clipboard&&navigator.clipboard.writeText){{navigator.clipboard.writeText(text);}} else {{var ta=document.createElement('textarea'); ta.value=text; document.body.appendChild(ta); ta.select(); document.execCommand('copy'); document.body.removeChild(ta);}}}})(this); return false;"
-                        style="position: absolute; top: 6px; right: 6px; z-index: 1; padding: 2px 8px; border: 0; border-radius: 4px; background: #e2e8f0; color: #334155; font-size: 11px; cursor: pointer;">
-                    Copy
-                </button>
-                <code title="{}" style="display: block; width: 100%; max-width: 100%; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; padding: 8px 56px 8px 8px; background: #f8fafc; border: 1px solid #e2e8f0; border-radius: 6px; font-size: 11px; box-sizing: border-box;">
-                    {}
-                </code>
-            </div>
-            """,
-            replay_cmd,
-            replay_cmd,
-            display_cmd,
+        return render_copy_block(
+            build_abx_dl_display_command(result),
+            copy_text=build_abx_dl_replay_command(result, config=self.request.archivebox_config),
         )
 
     def output_display(self, result):

archivebox/core/admin_snapshots.py

@@ -1,6 +1,8 @@
 __package__ = "archivebox.core"
 
 import json
+from copy import copy
+from urllib.parse import urlencode
 from functools import lru_cache
 from types import SimpleNamespace
 
@@ -9,11 +11,12 @@ from django.urls import path, reverse
 from django.shortcuts import get_object_or_404, redirect
 from django.http import JsonResponse, HttpResponseBadRequest, HttpResponseNotAllowed
 from django.utils import timezone
-from django.utils.html import format_html, format_html_join
+from django.utils.html import format_html
 from django.utils.safestring import mark_safe
 from django.db.models import Q, Count, Exists, F, OuterRef, Prefetch
 from django import forms
 from django.template import Template, RequestContext
+from django.template.loader import render_to_string
 from django.contrib.admin.helpers import ActionForm
 
 from archivebox.config.common import get_config
@@ -23,11 +26,11 @@ from archivebox.misc.logging_util import printable_filesize
 from archivebox.search.admin import SearchResultsAdminMixin, SearchResultsChangeList
 from archivebox.search.views import admin_snapshot_search_stream_view
 from archivebox.core.routes_util import build_snapshot_url, build_web_url
-from archivebox.core.tag_util import get_or_create_tag
 from archivebox.plugins.hooks import discover_hooks
 from archivebox.plugins.discovery import get_plugin_icon, get_plugin_name, get_plugins
 
-from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin
+from archivebox.core.widgets import render_permissions_badge, render_snapshot_progress
+from archivebox.base_models.admin import card_fieldset, BaseModelAdmin, ConfigEditorMixin
 
 from archivebox.core.models import Tag, Snapshot, ArchiveResult
 from archivebox.crawls.models import Crawl
@@ -38,7 +41,6 @@ from archivebox.core.permissions import (
     PERMISSIONS_CHOICES,
     PERMISSIONS_META,
     get_snapshot_permissions,
-    normalize_permissions,
 )
 from archivebox.core.widgets import TagEditorWidget, InlineTagEditorWidget
 
@@ -230,10 +232,12 @@ class SnapshotResultHealthListFilter(admin.SimpleListFilter):
 
 
 class SnapshotChangeList(SearchResultsChangeList):
+    snapshot_status_choices = Snapshot.StatusChoices.choices
+
     def __init__(self, request, *args, **kwargs):
         super().__init__(request, *args, **kwargs)
         resolver_name = request.resolver_match.url_name
-        self.embedded_changelist = request.GET.get("_embedded") == "crawl"
+        self.embedded_changelist = request.GET.get("_embedded") in {"crawl", "snapshot"}
         self.snapshot_is_grid_view = not self.embedded_changelist and (
             resolver_name == "grid" or request.path.rstrip("/").endswith("/grid")
         )
@@ -297,7 +301,7 @@ class SnapshotChangeList(SearchResultsChangeList):
 
     def get_results(self, request):
         super().get_results(request)
-        if request.GET.get("_embedded") == "crawl":
+        if request.GET.get("_embedded") in {"crawl", "snapshot"}:
             self.full_result_count = self.result_count
             self.show_full_result_count = True
         self._attach_archiveresult_summaries()
@@ -346,19 +350,8 @@ class SnapshotAdminForm(forms.ModelForm):
 
             # Parse and save tags from tags_editor
             tags_str = self.cleaned_data.get("tags_editor", "")
-            if tags_str:
-                tag_names = [name.strip() for name in tags_str.split(",") if name.strip()]
-                tags = []
-                for name in tag_names:
-                    tag, _ = Tag.objects.get_or_create(
-                        name__iexact=name,
-                        defaults={"name": name},
-                    )
-                    tag = Tag.objects.filter(name__iexact=name).first() or tag
-                    tags.append(tag)
-                instance.tags.set(tags)
-            else:
-                instance.tags.clear()
+            tag_names = [name.strip() for name in tags_str.split(",") if name.strip()]
+            instance.save_tags(tag_names)
 
         return instance
 
@@ -367,6 +360,16 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     form = SnapshotAdminForm
     raw_id_fields = ("crawl", "parent_snapshot")
     list_select_related = ()
+
+    def save_related(self, request, form, formsets, change):
+        super().save_related(request, form, formsets, change)
+        tags_str = form.cleaned_data.get("tags_editor", "")
+        tag_names = [name.strip() for name in tags_str.split(",") if name.strip()]
+        form.instance.save_tags(
+            tag_names,
+            created_by=request.user if request.user.is_authenticated else None,
+        )
+
     list_display = (
         "permissions_badge",
         "created_at",
@@ -412,70 +415,19 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
                 "classes": ("card", "actions-card"),
             },
         ),
-        (
-            "Snapshot",
-            {
-                "fields": ("snapshot_summary",),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "URL",
-            {
-                "fields": (("url_favicon", "url"), ("title", "tags_badges")),
-                "classes": ("card", "wide"),
-            },
-        ),
-        (
-            "Tags",
-            {
-                "fields": ("tags_editor", "permissions_config"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Status",
-            {
-                "fields": ("status", "retry_at"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Timestamps",
-            {
-                "fields": ("bookmarked_at", "created_at", "modified_at", "downloaded_at"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Relations",
-            {
-                "fields": ("crawl",),
-                "classes": ("card",),
-            },
-        ),
-        (
+        card_fieldset("Snapshot", ("snapshot_summary",)),
+        card_fieldset("URL", (("url_favicon", "url"), ("title", "tags_badges")), wide=True),
+        card_fieldset("Tags", ("tags_editor", "permissions_config")),
+        card_fieldset("Status", ("status", "retry_at")),
+        card_fieldset("Timestamps", ("bookmarked_at", "created_at", "modified_at", "downloaded_at")),
+        card_fieldset("Relations", ("crawl",)),
+        card_fieldset(
             "Config",
-            {
-                "fields": ("config",),
-                "description": '<span style="display:block; margin:-4px 0 6px; font-size:11px; line-height:1.35; color:#94a3b8;">Uses <code>Crawl.config</code> by default. Only set per-snapshot overrides here when needed.</span>',
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Files",
-            {
-                "fields": ("output_dir",),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Archive Results",
-            {
-                "fields": ("archiveresults_list",),
-                "classes": ("card", "wide"),
-            },
+            ("config",),
+            description='<span style="display:block; margin:-4px 0 6px; font-size:11px; line-height:1.35; color:#94a3b8;">Uses <code>Crawl.config</code> by default. Only set per-snapshot overrides here when needed.</span>',
         ),
+        card_fieldset("Files", ("output_dir",)),
+        card_fieldset("Archive Results", ("archiveresults_list",), wide=True),
     )
 
     ordering = ["-created_at"]
@@ -521,7 +473,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     def changelist_view(self, request, extra_context=None):
         self.request = request
         saved_list_per_page = self.list_per_page
-        embedded_changelist = request.GET.get("_embedded") == "crawl"
+        embedded_changelist = request.GET.get("_embedded") in {"crawl", "snapshot"}
         if embedded_changelist:
             try:
                 requested_per_page = int(request.GET.get("per_page", "200"))
@@ -542,6 +494,27 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         finally:
             self.list_per_page = saved_list_per_page
 
+    def render_embedded_changelist(self, request, *, filters, title, default_search_mode=None):
+        """Reuse snapshot search, rows, pagination, and actions in parent admin pages."""
+        changelist_path = reverse(f"{self.admin_site.name}:core_snapshot_changelist")
+        full_url = f"{changelist_path}?{urlencode(filters)}"
+        changelist_request = copy(request)
+        changelist_request.method = "GET"
+        changelist_request.path = changelist_path
+        changelist_request.GET = request.GET.copy()
+        if default_search_mode:
+            changelist_request.GET.setdefault("search_mode", default_search_mode)
+        changelist_request.GET.update({**filters, "_embedded": "snapshot", "per_page": "200"})
+        changelist_request.POST = request.POST.copy()
+        changelist_request.POST.clear()
+        response = self.changelist_view(changelist_request)
+        context = {
+            **response.context_data,
+            "snapshot_changelist_url": full_url,
+            "snapshot_changelist_title": title,
+        }
+        return mark_safe(render_to_string("admin/core/snapshot/embedded_changelist.html", context, request=request))
+
     def get_actions(self, request):
         actions = super().get_actions(request)
         if not actions:
@@ -550,7 +523,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         return actions
 
     def lookup_allowed(self, lookup, value, request=None):
-        if lookup in {"crawl__id__exact", "crawl_id__exact", "crawl_id"}:
+        if lookup in {"crawl__id__exact", "crawl_id__exact", "crawl_id", "crawl__schedule__id__exact"}:
             return True
         return super().lookup_allowed(lookup, value, request=request)
 
@@ -651,7 +624,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     def get_queryset(self, request):
         self.request = request
-        ordering_fields = self._get_ordering_fields(request)
+        ordering_fields = self.get_ordering_fields(request)
         needs_files_sort = "files" in ordering_fields
         needs_tags_sort = "tags_inline" in ordering_fields
         is_change_view = request.resolver_match.url_name == "core_snapshot_change"
@@ -715,45 +688,10 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         permissions = obj.__dict__.get("snapshot_permissions")
         if permissions is None:
             permissions = obj.permissions
-        permissions = normalize_permissions(permissions)
-        icon, label, fg, bg = SNAPSHOT_PERMISSION_META[permissions]
-        menu_items = format_html_join(
-            "",
-            (
-                '<button type="button" class="snapshot-permissions-menu-item{}" data-permissions="{}">'
-                '<span class="snapshot-permissions-icon" aria-hidden="true" style="color:{}; background:{};">{}</span>'
-                "<span>{}</span>"
-                "</button>"
-            ),
-            (
-                (
-                    " is-active" if choice_value == permissions else "",
-                    choice_value,
-                    choice_fg,
-                    choice_bg,
-                    choice_icon,
-                    choice_label,
-                )
-                for choice_value, choice_label in PERMISSIONS_CHOICES
-                for choice_icon, _choice_title, choice_fg, choice_bg in [SNAPSHOT_PERMISSION_META[choice_value]]
-            ),
-        )
-        return format_html(
-            '<span class="snapshot-permissions-quick" data-current-permissions="{}" data-permissions-url="{}">'
-            '<button type="button" class="snapshot-permissions-button snapshot-permissions-{}" title="{}" aria-label="Change snapshot permissions: {}" aria-expanded="false">'
-            '<span class="snapshot-permissions-icon" aria-hidden="true" style="color:{}; background:{};">{}</span>'
-            "</button>"
-            '<span class="snapshot-permissions-menu" role="menu" hidden>{}</span>'
-            "</span>",
+        return render_permissions_badge(
             permissions,
-            reverse(f"{self.admin_site.name}:core_snapshot_set_permissions", args=[obj.pk]),
-            permissions,
-            label,
-            label,
-            fg,
-            bg,
-            icon,
-            menu_items,
+            url=reverse(f"{self.admin_site.name}:core_snapshot_set_permissions", args=[obj.pk]),
+            object_name="snapshot",
         )
 
     @admin.display(description="Imported Timestamp")
@@ -853,23 +791,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
             obj.pk,
         )
 
-    def status_info(self, obj):
-        request = self.request
-        config = request.archivebox_config
-        favicon_url = build_snapshot_url(str(obj.id), "favicon.ico", request=request, config=config)
-        return format_html(
-            """
-            Archived: {} ({} files {}) &nbsp; &nbsp;
-            Favicon: <img src="{}" style="height: 20px"/> &nbsp; &nbsp;
-            Extension: {} &nbsp; &nbsp;
-            """,
-            "✅" if obj.is_archived else "❌",
-            obj.num_outputs,
-            self.size(obj) or "0kb",
-            favicon_url,
-            obj.extension or "-",
-        )
-
     @admin.display(description="Archive Results")
     def archiveresults_list(self, obj):
         request = self.request
@@ -1190,36 +1111,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
         # For started snapshots, show progress bar
         if obj.status == "started" and stats["total"] > 0:
-            percent = stats["percent"]
-            running = stats["running"]
-            succeeded = stats["succeeded"]
-            failed = stats["failed"]
-
-            return format_html(
-                """<div style="min-width: 90px;">
-                    <div style="display: flex; align-items: center; gap: 6px; margin-bottom: 4px;">
-                        <span class="snapshot-progress-spinner"></span>
-                        <span style="font-size: 11px; color: #64748b;">{}/{} hooks</span>
-                    </div>
-                    <div style="background: #e2e8f0; border-radius: 4px; height: 6px; overflow: hidden;">
-                        <div style="background: linear-gradient(90deg, #10b981 0%, #10b981 {}%, #ef4444 {}%, #ef4444 {}%, #3b82f6 {}%, #3b82f6 100%);
-                                    width: {}%; height: 100%; transition: width 0.3s;"></div>
-                    </div>
-                    <div style="font-size: 10px; color: #94a3b8; margin-top: 2px;">
-                        ✓{} ✗{} ⏳{}
-                    </div>
-                </div>""",
-                succeeded + failed + stats["skipped"],
-                stats["total"],
-                int(succeeded / stats["total"] * 100) if stats["total"] else 0,
-                int(succeeded / stats["total"] * 100) if stats["total"] else 0,
-                int((succeeded + failed) / stats["total"] * 100) if stats["total"] else 0,
-                int((succeeded + failed) / stats["total"] * 100) if stats["total"] else 0,
-                percent,
-                succeeded,
-                failed,
-                running,
-            )
+            return render_snapshot_progress(stats)
 
         # For other statuses, show simple badge
         return format_html(
@@ -1267,48 +1159,12 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         )
 
     def _get_progress_stats(self, obj):
-        cached_stats = obj.__dict__.get("_admin_progress_stats")
-        if cached_stats is not None:
-            return cached_stats
-
-        results = self._get_prefetched_results(obj)
-        if results is None:
-            stats = obj.get_progress_stats()
-            expected_total = self._get_expected_hook_total(obj)
-            total = max(stats["total"], expected_total)
-            completed = stats["succeeded"] + stats["failed"] + stats.get("skipped", 0) + stats.get("noresults", 0)
-            stats["total"] = total
-            stats["pending"] = max(total - completed - stats["running"], 0)
-            stats["percent"] = int((completed / total * 100) if total > 0 else 0)
-            obj._admin_progress_stats = stats
-            return stats
-
-        expected_total = self._get_expected_hook_total(obj)
-        observed_total = len(results)
-        total = max(observed_total, expected_total)
-        succeeded = sum(1 for r in results if r.status == "succeeded")
-        failed = sum(1 for r in results if r.status == "failed")
-        running = sum(1 for r in results if r.status == "started")
-        skipped = sum(1 for r in results if r.status == "skipped")
-        noresults = sum(1 for r in results if r.status == "noresults")
-        pending = max(total - succeeded - failed - running - skipped - noresults, 0)
-        completed = succeeded + failed + skipped + noresults
-        percent = int((completed / total * 100) if total > 0 else 0)
-        is_sealed = obj.status not in (obj.StatusChoices.QUEUED, obj.StatusChoices.STARTED, obj.StatusChoices.PAUSED)
-        stats = {
-            "total": total,
-            "succeeded": succeeded,
-            "failed": failed,
-            "running": running,
-            "pending": pending,
-            "skipped": skipped,
-            "noresults": noresults,
-            "percent": percent,
-            "output_size": obj.output_size or 0,
-            "is_sealed": is_sealed,
-        }
-        obj._admin_progress_stats = stats
-        return stats
+        if "_admin_progress_stats" not in obj.__dict__:
+            obj._admin_progress_stats = obj.get_progress_stats(
+                results=self._get_prefetched_results(obj),
+                expected_total=self._get_expected_hook_total(obj),
+            )
+        return obj._admin_progress_stats
 
     def _get_prefetched_results(self, obj):
         if "_admin_output_results" in obj.__dict__:
@@ -1364,22 +1220,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
             return list(prefetched_cache["tags"])
         return None
 
-    def _get_ordering_fields(self, request):
-        ordering = request.GET.get("o")
-        if not ordering:
-            return set()
-        fields = set()
-        for part in ordering.split("."):
-            if not part:
-                continue
-            try:
-                idx = abs(int(part)) - 1
-            except ValueError:
-                continue
-            if 0 <= idx < len(self.list_display):
-                fields.add(self.list_display[idx])
-        return fields
-
     @admin.display(
         description="Original URL",
         ordering="url",
@@ -1391,12 +1231,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
             obj.url[:128],
         )
 
-    @admin.display(description="Health", ordering="health")
-    def health_display(self, obj):
-        h = obj.health
-        color = "green" if h >= 80 else "orange" if h >= 50 else "red"
-        return format_html('<span style="color: {};">{}</span>', color, h)
-
     def grid_view(self, request, extra_context=None):
         extra_context = extra_context or {}
         extra_context["snapshot_is_grid_view"] = True
@@ -1500,7 +1334,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         tag_names = [name.strip() for name in tags_str.split(",") if name.strip()]
         tags = []
         for name in tag_names:
-            tag, _ = get_or_create_tag(
+            tag, _ = Tag.get_or_create_by_name(
                 name,
                 created_by=request.user if request.user.is_authenticated else None,
             )

archivebox/core/admin_tags.py

@@ -11,7 +11,7 @@ from django.urls import reverse
 from django.utils.html import format_html
 from django.utils.safestring import mark_safe
 
-from archivebox.base_models.admin import BaseModelAdmin
+from archivebox.base_models.admin import card_fieldset, BaseModelAdmin
 from archivebox.core.models import SnapshotTag, Tag
 from archivebox.core.tag_util import (
     TAG_HAS_SNAPSHOTS_CHOICES,
@@ -69,44 +69,14 @@ class TagAdmin(BaseModelAdmin):
     ordering = ["name", "id"]
 
     fieldsets = (
-        (
-            "Tag",
-            {
-                "fields": ("name",),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Metadata",
-            {
-                "fields": ("id", "created_by", "created_at", "modified_at"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Recent Snapshots",
-            {
-                "fields": ("snapshots",),
-                "classes": ("card", "wide"),
-            },
-        ),
+        card_fieldset("Tag", ("name",)),
+        card_fieldset("Metadata", ("id", "created_by", "created_at", "modified_at")),
+        card_fieldset("Recent Snapshots", ("snapshots",), wide=True),
     )
 
     add_fieldsets = (
-        (
-            "Tag",
-            {
-                "fields": ("name",),
-                "classes": ("card", "wide"),
-            },
-        ),
-        (
-            "Metadata",
-            {
-                "fields": ("created_by",),
-                "classes": ("card",),
-            },
-        ),
+        card_fieldset("Tag", ("name",), wide=True),
+        card_fieldset("Metadata", ("created_by",)),
     )
 
     def get_fieldsets(self, request: HttpRequest, obj: Tag | None = None):

archivebox/core/apps.py

@@ -10,7 +10,6 @@ class CoreConfig(AppConfig):
 
     def ready(self):
         """Register the archivebox.core.admin_site as the main django admin site"""
-        import sys
         from django.utils.autoreload import DJANGO_AUTORELOAD_ENV
 
         from archivebox.core.admin_site import register_admin_site
@@ -28,11 +27,6 @@ class CoreConfig(AppConfig):
 
         pre_save.connect(truncate_overlong_charfields, dispatch_uid="archivebox_truncate_overlong_charfields")
 
-        # Import models to register state machines with the registry
-        # Skip during makemigrations to avoid premature state machine access
-        if "makemigrations" not in sys.argv:
-            from archivebox.core import models  # noqa: F401
-
         def _should_prepare_runtime() -> bool:
             if os.environ.get("ARCHIVEBOX_RUNSERVER") == "1":
                 if os.environ.get("ARCHIVEBOX_AUTORELOAD") == "1":

archivebox/core/asgi.py

@@ -13,64 +13,12 @@ from archivebox.config.django import setup_django
 
 setup_django(check_db=True)
 
-
-def _patch_thread_sensitive_context_shutdown() -> None:
-    """Stop ``ThreadSensitiveContext.__aexit__`` from blocking the daphne loop.
-
-    Django 6.0's ASGIHandler wraps every request in ``async with
-    ThreadSensitiveContext():`` (django/core/handlers/asgi.py:169). On exit
-    asgiref calls ``executor.shutdown()`` with the default ``wait=True``
-    (asgiref/sync.py:148), which is a *synchronous* ``Thread.join()`` inside
-    an async function — so it blocks the daphne event loop until the
-    executor's worker thread exits.
-
-    That's normally fine because the request handler has already awaited
-    every ``sync_to_async`` it submitted, so the worker is idle and dies as
-    soon as the shutdown sentinel reaches it. The blocking turns into a
-    problem when a client disconnects mid-request:
-
-    * ``SyncToAsync.__call__`` shields the executor work with
-      ``await asyncio.shield(exec_coro)`` (asgiref/sync.py:506) so that the
-      sync DB call doesn't get torn down halfway through.
-    * On cancellation it calls ``exec_coro.cancel()`` (line 522) which only
-      flips the asyncio ``Future`` to cancelled — the underlying thread
-      keeps running the SQL query.
-    * Control unwinds to ``__aexit__`` while the orphaned thread is still
-      mid-query. ``shutdown(wait=True)`` then blocks the event loop until
-      that orphan finishes.
-
-    Under heavy SQLite contention (the load-test scenario that surfaced
-    this on cabbage) those orphan threads can take 30 seconds each waiting
-    for write locks, and the daphne loop is single-threaded — so every
-    such orphan stalls every other in-flight request, healthchecks time
-    out, and the container goes ``unhealthy``.
-
-    Switching to ``shutdown(wait=False)`` queues the sentinel and returns
-    immediately; the worker thread still exits cleanly once its current
-    task finishes, and asgiref's ``WeakKeyDictionary`` releases the
-    executor as soon as the request's context is GC'd. No per-request
-    teardown guarantee is lost — there was no caller relying on it.
-    """
-    from asgiref import sync as _asgiref_sync
-
-    original_aexit = _asgiref_sync.ThreadSensitiveContext.__aexit__
-
-    async def __aexit__(self, exc, value, tb):  # type: ignore[no-redef]
-        if not self.token:
-            return
-        executor = _asgiref_sync.SyncToAsync.context_to_thread_executor.pop(self, None)
-        if executor is not None:
-            executor.shutdown(wait=False)
-        _asgiref_sync.SyncToAsync.thread_sensitive_context.reset(self.token)
-
-    # Idempotent: only patch once even if asgi.py is reloaded.
-    if getattr(original_aexit, "_archivebox_patched", False):
-        return
-    __aexit__._archivebox_patched = True  # type: ignore[attr-defined]
-    _asgiref_sync.ThreadSensitiveContext.__aexit__ = __aexit__
+django_application = get_asgi_application()
 
 
-_patch_thread_sensitive_context_shutdown()
+async def application(scope, receive, send):
+    if scope["type"] == "websocket":
+        from archivebox.opencode.views import websocket_view
 
-# Standard Django ASGI application (no websockets/channels needed)
-application = get_asgi_application()
+        return await websocket_view(scope, receive, send)
+    return await django_application(scope, receive, send)

archivebox/core/forms.py

@@ -9,12 +9,13 @@ from archivebox.base_models.admin import KeyValueWidget
 from archivebox.config.common import get_config, parse_delete_after
 from archivebox.core.permissions import PERMISSIONS_CHOICES, PERMISSIONS_PUBLIC, filter_personas_by_permissions
 from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget
+from archivebox.crawls.forms import URLFiltersField
 from archivebox.crawls.schedule_util import validate_schedule
 from archivebox.misc.util import URL_REGEX, find_all_urls, parse_filesize_to_bytes
 from archivebox.personas.models import Persona
-from archivebox.plugins.discovery import get_plugins
+from archivebox.plugins.discovery import get_plugin_catalog
 from archivebox.plugins.forms import (
-    PLUGIN_GROUP_DEFINITIONS,
+    PLUGIN_GROUPS,
     TIMEOUT_INPUT_PATTERN,
     PluginConfigFormMixin,
     get_choice_field,
@@ -90,6 +91,18 @@ def edit_string_for_tag_names(tags) -> str:
     return ", ".join(sorted(names))
 
 
+class FileSizeField(forms.CharField):
+    def to_python(self, value):
+        raw_value = super().to_python(value)
+        try:
+            size = parse_filesize_to_bytes(raw_value) if raw_value else 0
+        except ValueError as err:
+            raise forms.ValidationError(str(err)) from err
+        if size < 0:
+            raise forms.ValidationError(f"{self.label} must be 0 or a positive number of bytes.")
+        return size
+
+
 class AddLinkForm(PluginConfigFormMixin, forms.Form):
     allow_crawl_execution_config_fields = False
 
@@ -136,7 +149,7 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             },
         ),
     )
-    crawl_max_size = forms.CharField(
+    crawl_max_size = FileSizeField(
         label="Max crawl size",
         required=False,
         initial="0",
@@ -169,7 +182,7 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             },
         ),
     )
-    snapshot_max_size = forms.CharField(
+    snapshot_max_size = FileSizeField(
         label="Max snapshot size",
         required=False,
         initial="0",
@@ -210,7 +223,7 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             },
         ),
     )
-    url_filters = forms.Field(
+    url_filters = URLFiltersField(
         label="URL allowlist / denylist",
         required=False,
         widget=URLFiltersWidget(source_selector='textarea[name="url"]'),
@@ -333,10 +346,9 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
         if self.can_override_crawl_config:
             self.build_plugin_groups(get_config(persona=selected_persona) if selected_persona else get_config())
         else:
-            all_plugins = get_plugins()
-            for field_name, *_rest, plugin_names in PLUGIN_GROUP_DEFINITIONS:
-                get_choice_field(self, field_name).choices = [(p, p) for p in all_plugins if p in plugin_names]
-            get_choice_field(self, "other_plugins").choices = [(p, p) for p in all_plugins]
+            grouped_plugins = get_plugin_catalog().groups()
+            for category, field_name, _title in PLUGIN_GROUPS:
+                get_choice_field(self, field_name).choices = [(plugin.name, plugin.name) for plugin in grouped_plugins.get(category, [])]
             self.plugin_groups = []
 
     def clean(self):
@@ -348,23 +360,7 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             cleaned_data["config"] = {}
             return cleaned_data
 
-        # Combine all plugin groups into single list
-        all_selected_plugins = []
-        for field in [
-            "main_plugins",
-            "page_setup_plugins",
-            "media_plugins",
-            "text_plugins",
-            "metadata_plugins",
-            "postprocessing_plugins",
-            "other_plugins",
-        ]:
-            selected = cleaned_data.get(field)
-            if isinstance(selected, list):
-                all_selected_plugins.extend(selected)
-
-        # Store combined list for easy access
-        cleaned_data["plugins"] = all_selected_plugins
+        cleaned_data["plugins"] = [plugin for _category, field, _title in PLUGIN_GROUPS for plugin in cleaned_data.get(field, [])]
 
         plugin_config_overrides = self.clean_plugin_config_overrides(get_config(persona=cleaned_data.get("persona")))
         custom_config = cleaned_data.get("config") or {}
@@ -381,34 +377,10 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             raise forms.ValidationError("Enter at least one valid URL.")
         return value
 
-    def clean_url_filters(self):
-        from archivebox.crawls.models import Crawl
-
-        value = self.cleaned_data.get("url_filters") or {}
-        return {
-            "allowlist": "\n".join(Crawl.split_filter_patterns(value.get("allowlist", ""))),
-            "denylist": "\n".join(Crawl.split_filter_patterns(value.get("denylist", ""))),
-            "same_domain_only": bool(value.get("same_domain_only")),
-            "subpaths_only": bool(value.get("subpaths_only")),
-            "only_new": bool(value.get("only_new")),
-        }
-
     def clean_max_urls(self):
         value = self.cleaned_data.get("max_urls")
         return int(value or 0)
 
-    def clean_crawl_max_size(self):
-        raw_value = str(self.cleaned_data.get("crawl_max_size") or "").strip()
-        if not raw_value:
-            return 0
-        try:
-            value = parse_filesize_to_bytes(raw_value)
-        except ValueError as err:
-            raise forms.ValidationError(str(err))
-        if value < 0:
-            raise forms.ValidationError("Max crawl size must be 0 or a positive number of bytes.")
-        return value
-
     def clean_crawl_timeout(self):
         return self._clean_timeout_seconds(self.cleaned_data.get("crawl_timeout"), "Max crawl time", blank_value=0)
 
@@ -440,18 +412,6 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             raise forms.ValidationError(f"{field_label} must be 0 or greater than 10 seconds.")
         return value
 
-    def clean_snapshot_max_size(self):
-        raw_value = str(self.cleaned_data.get("snapshot_max_size") or "").strip()
-        if not raw_value:
-            return 0
-        try:
-            value = parse_filesize_to_bytes(raw_value)
-        except ValueError as err:
-            raise forms.ValidationError(str(err))
-        if value < 0:
-            raise forms.ValidationError("Max snapshot size must be 0 or a positive number of bytes.")
-        return value
-
     def clean_delete_after(self):
         raw_value = str(self.cleaned_data.get("delete_after") or "0").strip() or "0"
         try:

archivebox/core/middleware.py

@@ -89,8 +89,9 @@ def AdminCookieIsolationMiddleware(get_response):
         response = get_response(request)
 
         if request.path == "/admin" or request.path.startswith("/admin/"):
-            response.headers["X-Frame-Options"] = "DENY"
-            response.headers["Content-Security-Policy"] = "frame-ancestors 'none'"
+            if not getattr(response, "xframe_options_exempt", False):
+                response.headers["X-Frame-Options"] = "DENY"
+                response.headers["Content-Security-Policy"] = "frame-ancestors 'none'"
 
         config = request.__dict__.get("archivebox_config")
         if config is None or config.SERVER_SECURITY_MODE == "auto":

archivebox/core/migrations/0052_unique_archiveresult_per_snapshot_plugin.py

@@ -1,71 +0,0 @@
-from django.db import migrations, models
-from django.db.models import Sum
-
-
-def consolidate_archiveresults_per_plugin(apps, schema_editor):
-    ArchiveResult = apps.get_model("core", "ArchiveResult")
-    Snapshot = apps.get_model("core", "Snapshot")
-    duplicate_groups = ArchiveResult.objects.values("snapshot_id", "plugin").annotate(count=models.Count("id")).filter(count__gt=1)
-
-    affected_snapshot_ids = set()
-    for group in duplicate_groups.iterator(chunk_size=200):
-        rows = list(
-            ArchiveResult.objects.filter(
-                snapshot_id=group["snapshot_id"],
-                plugin=group["plugin"],
-            ).order_by("created_at", "id"),
-        )
-        winner = max(
-            rows,
-            key=lambda row: (
-                bool(row.output_files),
-                int(row.output_size or 0),
-                row.modified_at,
-                str(row.id),
-            ),
-        )
-        output_files = {}
-        mimetypes = set()
-        for row in rows:
-            output_files.update(row.output_files or {})
-            mimetypes.update(part.strip() for part in (row.output_mimetypes or "").split(",") if part.strip())
-
-        winner.output_files = output_files
-        winner.output_size = max(
-            sum(int(metadata.get("size") or 0) for metadata in output_files.values() if isinstance(metadata, dict)),
-            *(int(row.output_size or 0) for row in rows),
-        )
-        winner.output_mimetypes = ",".join(sorted(mimetypes))
-        winner.start_ts = min((row.start_ts for row in rows if row.start_ts), default=None)
-        winner.end_ts = max((row.end_ts for row in rows if row.end_ts), default=None)
-        winner.save(update_fields=["output_files", "output_size", "output_mimetypes", "start_ts", "end_ts"])
-        ArchiveResult.objects.filter(id__in=[row.id for row in rows if row.id != winner.id]).delete()
-        affected_snapshot_ids.add(group["snapshot_id"])
-
-    for snapshot_id in affected_snapshot_ids:
-        total = ArchiveResult.objects.filter(snapshot_id=snapshot_id).aggregate(total=Sum("output_size"))["total"] or 0
-        Snapshot.objects.filter(id=snapshot_id).update(output_size=total)
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("core", "0051_postgres_url_pattern_ops_index"),
-    ]
-
-    operations = [
-        migrations.RemoveConstraint(
-            model_name="archiveresult",
-            name="unique_archiveresult_per_snapshot_hook",
-        ),
-        migrations.RunPython(
-            consolidate_archiveresults_per_plugin,
-            reverse_code=migrations.RunPython.noop,
-        ),
-        migrations.AddConstraint(
-            model_name="archiveresult",
-            constraint=models.UniqueConstraint(
-                fields=("snapshot", "plugin"),
-                name="unique_archiveresult_per_snapshot_plugin",
-            ),
-        ),
-    ]

archivebox/core/migrations/0053_alter_archiveresult_options.py

@@ -5,7 +5,7 @@ from django.db import migrations
 
 class Migration(migrations.Migration):
     dependencies = [
-        ("core", "0052_unique_archiveresult_per_snapshot_plugin"),
+        ("core", "0051_postgres_url_pattern_ops_index"),
     ]
 
     operations = [

archivebox/core/migrations/0054_snapshot_fs_version_index.py

@@ -0,0 +1,26 @@
+# Generated by Django 6.1 on 2026-09-02
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0053_alter_archiveresult_options"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="snapshot",
+            name="current_step",
+        ),
+        migrations.AlterField(
+            model_name="snapshot",
+            name="fs_version",
+            field=models.CharField(
+                db_index=True,
+                default="0.9.0",
+                help_text='Filesystem version of this snapshot (e.g., "0.7.0", "0.8.0", "0.9.4").',
+                max_length=10,
+            ),
+        ),
+    ]

archivebox/core/models/__init__.py

@@ -0,0 +1,17 @@
+"""Core persistence models, grouped by entity and queryset responsibility."""
+
+from .archiveresults import ArchiveResult
+from .querysets import SnapshotManager, SnapshotQuerySet, UngroupedSubquery
+from .snapshots import Snapshot, SnapshotMigrationError
+from .tags import SnapshotTag, Tag
+
+__all__ = [
+    "ArchiveResult",
+    "Snapshot",
+    "SnapshotManager",
+    "SnapshotMigrationError",
+    "SnapshotQuerySet",
+    "SnapshotTag",
+    "Tag",
+    "UngroupedSubquery",
+]

archivebox/core/models/querysets.py

@@ -0,0 +1,361 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from typing import Any, ClassVar
+
+from django.core.exceptions import FieldDoesNotExist
+from django.db import models
+
+from archivebox.config import CONSTANTS
+from archivebox.config.common import get_config
+from archivebox.crawls.models import Crawl
+from archivebox.misc.system import atomic_write
+from archivebox.misc.util import (
+    to_json,
+)
+
+
+class UngroupedSubquery(models.Subquery):
+    """Scalar subquery that should not be copied into the outer GROUP BY."""
+
+    def get_group_by_cols(self):
+        return []
+
+
+class SnapshotQuerySet(models.QuerySet):
+    """Custom QuerySet for Snapshot model with export methods that persist through .filter() etc."""
+
+    def bulk_create(self, objs, *args, **kwargs):
+        objs = list(objs)
+        missing_crawl_ids = set()
+        from archivebox.core.permissions import PERMISSIONS_VALUES
+
+        for obj in objs:
+            if isinstance(obj, self.model):
+                config = dict(obj.config or {})
+                permission = str(config.get("PERMISSIONS") or "").strip().lower()
+                if permission not in PERMISSIONS_VALUES and obj.crawl_id:
+                    crawl = getattr(obj, "crawl", None)
+                    if not getattr(crawl, "permissions", None):
+                        missing_crawl_ids.add(str(obj.crawl_id))
+
+        crawl_permissions_by_id = {}
+        if missing_crawl_ids:
+            crawl_permissions_by_id = {
+                str(crawl_id): permissions
+                for crawl_id, permissions in Crawl.objects.filter(pk__in=missing_crawl_ids).values_list("pk", "permissions")
+            }
+
+        from archivebox.misc.db import truncate_overlong_charfields
+
+        for obj in objs:
+            if isinstance(obj, self.model):
+                obj.ensure_permissions_config(crawl_permissions=crawl_permissions_by_id.get(str(obj.crawl_id)))
+                # bulk_create bypasses pre_save, so clamp CharFields here too
+                # (e.g. page titles) to stay within postgres VARCHAR limits.
+                truncate_overlong_charfields(obj)
+        return super().bulk_create(objs, *args, **kwargs)
+
+    def paged_iterator(self, chunk_size: int = 500):
+        """
+        Iterate snapshots using bounded keyset pages instead of one streaming cursor.
+
+        Django's iterator(chunk_size=...) still keeps a single SQLite SELECT
+        cursor open until the full queryset is exhausted. That is fine for
+        read-only exports, but update/migration code does filesystem work and
+        writes while iterating; a long-lived read cursor there can stretch lock
+        waits across thousands of rows. This respects the queryset's existing
+        filters, order_by(), select_related(), and prefetch_related() state; if
+        no ordering is defined, it falls back to primary-key order.
+        """
+        pk_field = self.model._meta.pk.name
+        raw_ordering = tuple(self.query.order_by or self.model._meta.ordering or (pk_field,))
+
+        ordering = []
+        for term in raw_ordering:
+            if not isinstance(term, str) or term == "?":
+                ordering = []
+                break
+            field_name = term.removeprefix("-")
+            field_name = pk_field if field_name == "pk" else field_name
+            ordering.append(f"-{field_name}" if term.startswith("-") else field_name)
+
+        ordered_field_names = [term.removeprefix("-") for term in ordering]
+        unique_field_names = {pk_field, *(field.name for field in self.model._meta.fields if field.unique)}
+        try:
+            use_keyset = any(name in unique_field_names for name in ordered_field_names) and not any(
+                self.model._meta.get_field(name).null for name in ordered_field_names
+            )
+        except (AttributeError, FieldDoesNotExist):
+            use_keyset = False
+        # Expressions, nullable/related fields and non-unique orderings retain
+        # Django's ordering through bounded offset pages.
+        if not use_keyset:
+            offset = 0
+            while batch := list(self[offset : offset + chunk_size]):
+                yield from batch
+                offset += chunk_size
+            return
+
+        last_values = None
+        value_field_names = tuple(dict.fromkeys([*ordered_field_names, pk_field]))
+        while True:
+            batch_qs = self.order_by(*ordering)
+            if last_values is not None:
+                page_filter = models.Q()
+                for idx, term in enumerate(ordering):
+                    descending = term.startswith("-")
+                    field_name = term[1:] if descending else term
+                    prefix = {ordered_field_names[i]: last_values[i] for i in range(idx)}
+                    comparison = "lt" if descending else "gt"
+                    page_filter |= models.Q(**prefix, **{f"{field_name}__{comparison}": last_values[idx]})
+                batch_qs = batch_qs.filter(page_filter)
+
+            batch_rows = list(batch_qs.values_list(*value_field_names)[:chunk_size])
+            if not batch_rows:
+                break
+
+            pk_idx = value_field_names.index(pk_field)
+            snapshot_ids = [row[pk_idx] for row in batch_rows]
+            snapshots_by_id = {snapshot.pk: snapshot for snapshot in self.filter(pk__in=snapshot_ids).order_by()}
+
+            for row in batch_rows:
+                snapshot_id = row[pk_idx]
+                snapshot = snapshots_by_id.get(snapshot_id)
+                if snapshot is not None:
+                    yield snapshot
+
+            last_values = batch_rows[-1][: len(ordered_field_names)]
+
+    # =========================================================================
+    # Filtering Methods
+    # =========================================================================
+
+    FILTER_TYPES: ClassVar[dict[str, Any]] = {
+        "exact": lambda pattern: models.Q(url=pattern),
+        "substring": lambda pattern: models.Q(url__icontains=pattern),
+        "regex": lambda pattern: models.Q(url__iregex=pattern),
+        "domain": lambda pattern: (
+            models.Q(url__istartswith=f"http://{pattern}")
+            | models.Q(url__istartswith=f"https://{pattern}")
+            | models.Q(url__istartswith=f"ftp://{pattern}")
+        ),
+        "tag": lambda pattern: models.Q(tags__name=pattern),
+        "timestamp": lambda pattern: models.Q(timestamp=pattern),
+    }
+    FILTER_TYPE_CHOICES = tuple(FILTER_TYPES)
+    FILTER_ARG_KEYS = (
+        "after",
+        "before",
+        "filter_type",
+        "filter_patterns",
+        "status",
+        "url__icontains",
+        "url__istartswith",
+        "tag",
+        "crawl_id",
+        "limit",
+        "sort",
+        "search",
+    )
+    SPECIAL_FILTER_ARG_KEYS = frozenset({"filter_patterns", "filter_type", "query", "search", "tag", "before", "after", "limit", "sort"})
+
+    def filter_by_patterns(self, patterns: list[str], filter_type: str = "exact") -> SnapshotQuerySet:
+        """Filter snapshots by URL patterns using specified filter type"""
+        from archivebox.misc.logging import stderr
+
+        q_filter = models.Q()
+        for pattern in patterns:
+            try:
+                q_filter = q_filter | self.FILTER_TYPES[filter_type](pattern)
+            except KeyError:
+                stderr()
+                stderr(f"[X] Got invalid pattern for --filter-type={filter_type}:", color="red")
+                stderr(f"    {pattern}")
+                raise SystemExit(2)
+        return self.filter(q_filter)
+
+    def search(self, **kwargs) -> SnapshotQuerySet:
+
+        from archivebox.core.snapshot_status import filter_snapshots_by_status
+        from archivebox.search.query import apply_snapshot_search
+
+        queryset = self
+        filter_patterns = tuple(str(pattern) for pattern in kwargs.get("filter_patterns") or ())
+        filter_type = kwargs.get("filter_type") or "substring"
+        query = kwargs.get("query")
+        if isinstance(query, (list, tuple)):
+            query = " ".join(str(part) for part in query)
+        query = (query or (" ".join(filter_patterns) if kwargs.get("search") else "")).strip()
+
+        field_names = {field.name for field in self.model._meta.get_fields()}
+        field_names.update(field.attname for field in self.model._meta.fields)
+        field_filters = {
+            key: value
+            for key, value in kwargs.items()
+            if value is not None and key not in self.SPECIAL_FILTER_ARG_KEYS and key.split("__", 1)[0] in field_names
+        }
+        status = field_filters.pop("status", None)
+        queryset = filter_snapshots_by_status(queryset, status)
+        if field_filters:
+            queryset = queryset.filter(**field_filters)
+        if kwargs.get("tag"):
+            queryset = queryset.filter(tags__name__iexact=kwargs["tag"])
+        if kwargs.get("before") is not None:
+            queryset = queryset.filter(bookmarked_at__lt=datetime.fromtimestamp(float(kwargs["before"]), tz=UTC))
+        if kwargs.get("after") is not None:
+            queryset = queryset.filter(bookmarked_at__gt=datetime.fromtimestamp(float(kwargs["after"]), tz=UTC))
+
+        if query:
+            queryset = apply_snapshot_search(
+                queryset,
+                query,
+                search_mode=kwargs.get("search"),
+                ordering=("-created_at",) if not kwargs.get("sort") else None,
+                max_results=kwargs.get("limit"),
+                skip_backend_when_metadata_satisfies_limit=True,
+                include_metadata_for_forced_backend=True,
+            )
+        elif filter_patterns:
+            queryset = queryset.filter_by_patterns(list(filter_patterns), filter_type)
+
+        if kwargs.get("sort"):
+            queryset = queryset.order_by(kwargs["sort"])
+        elif not queryset.query.order_by:
+            queryset = queryset.order_by("-created_at")
+
+        limit = kwargs.get("limit")
+        if limit is not None and limit > 0:
+            queryset = queryset[:limit]
+
+        return queryset
+
+    # =========================================================================
+    # Export Methods
+    # =========================================================================
+
+    def to_json(self, with_headers: bool = False) -> str:
+        """Generate JSON index from snapshots"""
+        import sys
+        from datetime import datetime
+
+        from archivebox.config import VERSION
+
+        config = get_config()
+
+        MAIN_INDEX_HEADER = (
+            {
+                "info": "This is an index of site data archived by ArchiveBox: The self-hosted web archive.",
+                "schema": "archivebox.index.json",
+                "copyright_info": config.FOOTER_INFO,
+                "meta": {
+                    "project": "ArchiveBox",
+                    "version": VERSION,
+                    "git_sha": VERSION,
+                    "website": "https://ArchiveBox.io",
+                    "docs": "https://github.com/ArchiveBox/ArchiveBox/wiki",
+                    "source": "https://github.com/ArchiveBox/ArchiveBox",
+                    "issues": "https://github.com/ArchiveBox/ArchiveBox/issues",
+                    "dependencies": {},
+                },
+            }
+            if with_headers
+            else {}
+        )
+
+        snapshot_dicts = [s.to_dict(extended=True, static_export=True) for s in self.iterator(chunk_size=500)]
+
+        if with_headers:
+            output = {
+                **MAIN_INDEX_HEADER,
+                "num_links": len(snapshot_dicts),
+                "updated": datetime.now(UTC),
+                "last_run_cmd": sys.argv,
+                "links": snapshot_dicts,
+            }
+        else:
+            output = snapshot_dicts
+        return to_json(output, indent=4, sort_keys=True)
+
+    def to_csv(self, cols: list[str] | None = None, header: bool = True, separator: str = ",", ljust: int = 0) -> str:
+        """Generate CSV output from snapshots"""
+        cols = cols or ["timestamp", "is_archived", "url"]
+        header_str = separator.join(col.ljust(ljust) for col in cols) if header else ""
+        row_strs = (s.to_csv(cols=cols, ljust=ljust, separator=separator) for s in self.iterator(chunk_size=500))
+        return "\n".join((header_str, *row_strs))
+
+    def to_html(self, with_headers: bool = True) -> str:
+        """Generate main index HTML from snapshots"""
+        from datetime import datetime
+
+        from django.template.loader import render_to_string
+
+        from archivebox.config import VERSION
+        from archivebox.config.version import get_COMMIT_HASH
+
+        config = get_config()
+
+        template = "static_index.html" if with_headers else "minimal_index.html"
+        snapshot_list = list(self.iterator(chunk_size=500))
+        manifest_records = []
+        for snapshot in snapshot_list:
+            outputs = snapshot.discover_outputs(include_filesystem_fallback=True)
+            output_paths = [str(output.get("path") or "") for output in outputs]
+            snapshot._public_preview_paths = [
+                path for preferred in ("screenshot/screenshot.png", "screenshot.png") for path in output_paths if path == preferred
+            ]
+            snapshot._public_favicon_paths = [path for path in output_paths if path in ("favicon/favicon.ico", "favicon.ico")]
+            snapshot.write_html_details()
+            if with_headers:
+                # Use the same portable schema as the JSON export. Rendering
+                # above has already populated result-count caches, archive_size
+                # reuses the sealed output_size field, and tags are prefetched.
+                manifest_records.append(snapshot.to_dict(extended=True, static_export=True))
+
+        if with_headers:
+            manifest = "".join(f"{to_json(record, indent=None, sort_keys=True)}\n" for record in manifest_records)
+            atomic_write(str(CONSTANTS.DATA_DIR / CONSTANTS.JSONL_INDEX_FILENAME), manifest)
+
+        return render_to_string(
+            template,
+            {
+                "version": VERSION,
+                "git_sha": get_COMMIT_HASH() or VERSION,
+                "num_links": str(len(snapshot_list)),
+                "date_updated": datetime.now(UTC).strftime("%Y-%m-%d"),
+                "time_updated": datetime.now(UTC).strftime("%Y-%m-%d %H:%M"),
+                "links": snapshot_list,
+                "FOOTER_INFO": config.FOOTER_INFO,
+                "STATIC_EXPORT": True,
+                "STATIC_EXPORT_DIR": CONSTANTS.DATA_DIR,
+            },
+        )
+
+
+class SnapshotManager(models.Manager.from_queryset(SnapshotQuerySet)):  # ty: ignore[unsupported-base]
+    """Manager for Snapshot model - uses SnapshotQuerySet for chainable methods"""
+
+    def filter(self, *args, **kwargs):
+        domain = kwargs.pop("domain", None)
+        qs = super().filter(*args, **kwargs)
+        if domain:
+            qs = qs.filter(url__icontains=f"://{domain}")
+        return qs
+
+    def get_queryset(self):
+        # Don't prefetch by default - it causes "too many open files" during bulk operations
+        # Views/templates can add .prefetch_related('tags', 'archiveresult_set') where needed
+        return super().get_queryset()
+
+    # =========================================================================
+    # Import Methods
+    # =========================================================================
+
+    def remove(self, atomic: bool = False) -> tuple:
+        """Remove snapshots from the database"""
+        from django.db import transaction
+
+        if atomic:
+            with transaction.atomic():
+                return self.get_queryset().delete()
+        return self.get_queryset().delete()

archivebox/core/models/tags.py

@@ -0,0 +1,150 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Any, ClassVar
+
+from django.conf import settings
+from django.db import IntegrityError, models
+from django.urls import reverse_lazy
+from django.utils import timezone
+from django.utils.text import slugify
+
+from archivebox.base_models.models import (
+    ModelWithUUID,
+    get_or_create_system_user_pk,
+)
+from archivebox.misc.util import (
+    sanitize_html_text,
+)
+
+if TYPE_CHECKING:
+    from .snapshots import Snapshot
+
+
+class Tag(ModelWithUUID):
+    # Keep AutoField for compatibility with main branch migrations
+    # Don't use UUIDField here - requires complex FK transformation
+    id = models.AutoField(primary_key=True, serialize=False, verbose_name="ID")
+    created_by = models.ForeignKey(
+        settings.AUTH_USER_MODEL,
+        on_delete=models.CASCADE,
+        default=get_or_create_system_user_pk,
+        null=True,
+        related_name="tag_set",
+    )
+    created_at = models.DateTimeField(default=timezone.now, db_index=True, null=True)
+    modified_at = models.DateTimeField(auto_now=True)
+    name = models.CharField(unique=True, blank=False, max_length=100)
+
+    @classmethod
+    def get_or_create_by_name(
+        cls,
+        name: str,
+        *,
+        defaults: Mapping[str, Any] | None = None,
+        created_by=None,
+    ) -> tuple[Tag, bool]:
+        name = cls.normalize_name(name)
+        defaults = dict(defaults or {})
+        if created_by is not None:
+            defaults["created_by"] = created_by
+        tag = cls.objects.filter(name__iexact=name).first()
+        if tag:
+            return tag, False
+        try:
+            return cls.objects.create(name=name, **(defaults or {})), True
+        except IntegrityError:
+            tag = cls.objects.filter(name__iexact=name).first()
+            if tag is None:
+                raise
+            return tag, False
+
+    @staticmethod
+    def normalize_name(name: str) -> str:
+        name = sanitize_html_text(name).strip()
+        if not name:
+            raise ValueError("Tag name is required")
+        return name
+
+    def rename(self, name: str) -> Tag:
+        name = self.normalize_name(name)
+        existing = type(self).objects.filter(name__iexact=name).exclude(pk=self.pk).first()
+        if existing:
+            raise ValueError(f'Tag "{existing.name}" already exists')
+        if self.name != name:
+            self.name = name
+            self.save()
+        return self
+
+    snapshot_set: models.Manager[Snapshot]
+
+    class Meta(ModelWithUUID.Meta):
+        app_label = "core"
+        verbose_name = "Tag"
+        verbose_name_plural = "Tags"
+
+    def __str__(self):
+        return self.name
+
+    def save(self, *args, **kwargs):
+        update_fields = kwargs.get("update_fields")
+        if update_fields is None or "name" in update_fields:
+            self.name = sanitize_html_text(self.name).strip()
+        super().save(*args, **kwargs)
+
+    @property
+    def slug(self) -> str:
+        """ASCII-safe slugified form of the tag name (derived, not stored)."""
+        return slugify(self.name or "") or "tag"
+
+    @property
+    def api_url(self) -> str:
+        return str(reverse_lazy("api-1:get_tag", args=[self.id]))
+
+    def to_json(self) -> dict:
+        """
+        Convert Tag model instance to a JSON-serializable dict.
+        """
+        from archivebox.config import VERSION
+
+        return {
+            "type": "Tag",
+            "schema_version": VERSION,
+            "id": str(self.id),
+            "name": self.name,
+        }
+
+    @staticmethod
+    def from_json(record: dict[str, Any], overrides: dict[str, Any] | None = None):
+        """
+        Create/update Tag from JSON dict.
+
+        Args:
+            record: JSON dict with 'name' field
+            overrides: Optional dict with 'snapshot' to auto-attach tag
+
+        Returns:
+            Tag instance or None
+        """
+        name = record.get("name")
+        if not name:
+            return None
+
+        tag, _ = Tag.get_or_create_by_name(name)
+
+        # Auto-attach to snapshot if in overrides
+        if overrides and "snapshot" in overrides and tag:
+            overrides["snapshot"].add_tag_ids([tag.pk])
+
+        return tag
+
+
+class SnapshotTag(models.Model):
+    id = models.AutoField(primary_key=True)
+    snapshot = models.ForeignKey("Snapshot", db_column="snapshot_id", on_delete=models.CASCADE, to_field="id")
+    tag = models.ForeignKey(Tag, db_column="tag_id", on_delete=models.CASCADE, to_field="id")
+
+    class Meta:
+        app_label = "core"
+        db_table = "core_snapshot_tags"
+        unique_together: ClassVar[list[tuple[str, str]]] = [("snapshot", "tag")]

archivebox/core/recovery_util.py

@@ -10,10 +10,15 @@ def _is_signal_interrupted_exit(exit_code: int | None) -> bool:
     return exit_code is not None and (exit_code < 0 or exit_code >= 128)
 
 
+def _canonical_hook_name(hook_name: str) -> str:
+    hook_name = Path(hook_name).name
+    return Path(hook_name).stem if Path(hook_name).suffix in {".py", ".js", ".sh"} else hook_name
+
+
 def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str | None = None) -> dict[str, int]:
     from archivebox.crawls.models import Crawl
     from archivebox.core.models import ArchiveResult, Snapshot
-    from archivebox.services.archive_result_service import _collect_output_metadata
+    from abx_dl.output_files import OutputManifest
     from archivebox.machine.models import Process
     from django.core.exceptions import ValidationError
     from django.db.models import Exists, OuterRef, Q, Subquery, Value
@@ -31,9 +36,7 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
         "crawls_queued_without_retry_at": 0,
         "snapshots_queued_without_retry_at": 0,
         "snapshots_sealed_with_extension_uploads_only": 0,
-        "archiveresults_backoff": 0,
-        "snapshots_queued_plugin_rows_waiting_on_stale_lease": 0,
-        "archiveresults_started_without_running_process": 0,
+        "archiveresults_interrupted_without_running_process": 0,
         "archiveresults_missing_for_orphaned_hook_processes": 0,
         "snapshots_started_without_running_results": 0,
         "crawls_started_with_due_snapshots": 0,
@@ -41,10 +44,10 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
         "crawls_started_without_active_snapshots": 0,
     }
 
-    running_archiveresults = ArchiveResult.objects.filter(
-        snapshot_id=OuterRef("pk"),
-        status=ArchiveResult.StatusChoices.STARTED,
-        process__status=Process.StatusChoices.RUNNING,
+    running_hook_processes = Process.objects.filter(
+        archiveresult__snapshot_id=OuterRef("pk"),
+        process_type=Process.TypeChoices.HOOK,
+        status=Process.StatusChoices.RUNNING,
     )
     active_child_snapshots = Snapshot.objects.filter(
         crawl_id=OuterRef("pk"),
@@ -104,49 +107,18 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
         retry_at=now,
         modified_at=now,
     )
-    backoff_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.BACKOFF, **result_filter)
     orphaned_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.STARTED, **result_filter).exclude(
         process__status=Process.StatusChoices.RUNNING,
     )
-    # ArchiveResult has no retry_at scheduler. Wake only the parent Snapshots
-    # for result rows we are about to repair, then requeue those exact rows.
-    # Using subqueries keeps million-row recovery in SQLite instead of building
-    # Python ID lists or scanning all sealed snapshots.
-    Snapshot.objects.filter(
-        id__in=backoff_results.values("snapshot_id"),
-        status__in=[Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED],
-        retry_at__isnull=True,
-    ).update(retry_at=now, modified_at=now)
+    # ArchiveResult rows are projections, never work items. Close interrupted
+    # projections as failed and wake their parent Snapshot so abx-dl can replay
+    # the snapshot-level sequence. Indexed subqueries keep this bounded.
     Snapshot.objects.filter(
         id__in=orphaned_results.values("snapshot_id"),
-        status__in=[Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED],
-        retry_at__isnull=True,
+        status=Snapshot.StatusChoices.STARTED,
     ).update(retry_at=now, modified_at=now)
-    cleaned["archiveresults_backoff"] = backoff_results.update(status=ArchiveResult.StatusChoices.QUEUED, modified_at=now)
-    # Targeted plugin rows on final/paused Snapshots are scheduled through the
-    # parent Snapshot.retry_at. retry_at=NULL is the normal idle marker for a
-    # sealed Snapshot and must not be interpreted as queued work just because
-    # old/synthetic ArchiveResult rows exist. If takeover kills the runner
-    # after it leases the Snapshot but before queued ArchiveResult rows finish,
-    # the rows remain QUEUED while retry_at sits in the future. Recovery runs
-    # only after this runner has won the single-runner gate, so it can safely
-    # unlock those stale plugin leases for immediate processing instead of
-    # waiting out the previous owner's full lock timeout.
-    queued_plugin_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.QUEUED, **result_filter)
-    cleaned["snapshots_queued_plugin_rows_waiting_on_stale_lease"] = (
-        Snapshot.objects.filter(
-            id__in=queued_plugin_results.values("snapshot_id"),
-            status__in=[Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED],
-        )
-        .filter(retry_at__gt=now)
-        .update(retry_at=now, modified_at=now)
-    )
-    # Impossible state repair: STARTED ArchiveResults without a live Process
-    # have no owner left to emit completion. Requeue only the result row; the
-    # snapshot/crawl schedulers will pick up normal retry processing.
-    cleaned["archiveresults_started_without_running_process"] = orphaned_results.update(
-        status=ArchiveResult.StatusChoices.QUEUED,
-        process=None,
+    cleaned["archiveresults_interrupted_without_running_process"] = orphaned_results.update(
+        status=ArchiveResult.StatusChoices.FAILED,
         modified_at=now,
     )
     orphaned_hook_processes = Process.objects.filter(
@@ -168,14 +140,23 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
             orphaned_hook_processes = orphaned_hook_processes.filter(snapshot_pwd_filter)
         else:
             orphaned_hook_processes = orphaned_hook_processes.none()
-    for process in orphaned_hook_processes.only("id", "pwd", "cmd", "process_type", "status", "started_at", "ended_at").order_by(
-        "-started_at",
-        "-id",
-    ):
+    for process in orphaned_hook_processes.only(
+        "id",
+        "pwd",
+        "cmd",
+        "process_type",
+        "status",
+        "exit_code",
+        "stdout",
+        "stderr",
+        "started_at",
+        "ended_at",
+    ).order_by("-started_at", "-id"):
         hook_script_name = process.hook_script_name
         if not hook_script_name or not process.pwd:
             continue
         plugin_dir = Path(process.pwd)
+        hook_name = _canonical_hook_name(hook_script_name)
         if crawl_snapshot_ids is not None and plugin_dir.parent.name not in crawl_snapshot_ids:
             continue
         try:
@@ -187,42 +168,44 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
             continue
         if snapshot is None:
             continue
-        result, created = ArchiveResult.objects.get_or_create(
-            snapshot=snapshot,
-            plugin=plugin_dir.name,
+        result, created = ArchiveResult.get_or_create_by_hook(
+            snapshot,
+            plugin_dir.name,
+            hook_name,
             defaults={
-                "hook_name": Path(hook_script_name).stem,
-                "status": ArchiveResult.StatusChoices.QUEUED,
+                "status": ArchiveResult.StatusChoices.FAILED,
             },
         )
-        if created or result.status == ArchiveResult.StatusChoices.QUEUED:
+        process_is_newer = bool(process.started_at and (result.start_ts is None or process.started_at >= result.start_ts))
+        if created or process_is_newer:
             requeue_snapshot = False
             # A runner can die after the hook Process exits but before the
             # ProcessCompletedEvent projector links/finalizes ArchiveResult.
-            # Reconstruct the plugin row from its newest durable Process row.
-            output_files, output_size, output_mimetypes = _collect_output_metadata(plugin_dir)
+            # Reconstruct only that exact hook row from the durable Process row.
+            manifest = OutputManifest.scan(plugin_dir, containment_root=snapshot.output_dir)
+            output_files = manifest.as_mapping()
+            output_size = manifest.total_size
+            output_mimetypes = ",".join(manifest.mimetypes)
             emitted_records = [
                 record
                 for record in Process.parse_records_from_text(process.stdout or "")
                 if record.get("type") == "ArchiveResult"
                 and (record.get("plugin") or plugin_dir.name) == plugin_dir.name
-                and (record.get("hook_name") or Path(hook_script_name).stem) == Path(hook_script_name).stem
+                and _canonical_hook_name(str(record.get("hook_name") or hook_name)) == hook_name
             ]
             emitted_result = emitted_records[-1] if emitted_records else {}
-            result.hook_name = Path(hook_script_name).stem
             result.process = process
             result.start_ts = process.started_at
             result.end_ts = process.ended_at
             if _is_signal_interrupted_exit(process.exit_code):
-                # The owning runner died or was asked to stop while the hook was
-                # still active. Keep the work item queued so takeover retries the
-                # same hook; treating an unknown signal exit as success would
-                # silently skip unfinished side effects.
+                # The Process is a durable fact; record the interruption as a
+                # failure and wake the parent Snapshot for replay.
                 result.output_files = {}
                 result.output_size = 0
                 result.output_mimetypes = ""
                 result.output_str = ""
-                result.status = ArchiveResult.StatusChoices.QUEUED
+                result.output_json = None
+                result.status = ArchiveResult.StatusChoices.FAILED
                 requeue_snapshot = True
             else:
                 result.output_files = output_files
@@ -246,7 +229,6 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
                 )
             result.save(
                 update_fields=[
-                    "hook_name",
                     "process",
                     "start_ts",
                     "end_ts",
@@ -263,7 +245,6 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
                 Snapshot.objects.filter(id=snapshot.id).update(retry_at=now, modified_at=now)
         if created:
             cleaned["archiveresults_missing_for_orphaned_hook_processes"] += 1
-            Snapshot.objects.filter(id=snapshot.id).update(retry_at=now, modified_at=now)
     started_snapshots = Snapshot.objects.filter(status=Snapshot.StatusChoices.STARTED).filter(
         Q(retry_at__isnull=True) | Q(retry_at__gt=now),
         **snapshot_filter,
@@ -277,8 +258,8 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
     # We only unlock scheduling; normal Snapshot runner code owns the next
     # transition and side effects.
     cleaned["snapshots_started_without_running_results"] = (
-        started_snapshots.annotate(has_running_results=Exists(running_archiveresults))
-        .filter(has_running_results=False)
+        started_snapshots.annotate(has_running_process=Exists(running_hook_processes))
+        .filter(has_running_process=False)
         .update(
             retry_at=now,
             modified_at=now,
@@ -287,7 +268,7 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
 
     # Broken lock repair: STARTED + retry_at=NULL is an orphaned ownership
     # lease. Recovery only unlocks scheduling; the runner owns any subsequent
-    # state-machine transition, including sealing rows whose children/results
+    # lifecycle transition, including sealing rows whose children/results
     # are already final.
     recoverable_started_crawls = Crawl.objects.filter(status=Crawl.StatusChoices.STARTED).filter(
         Q(retry_at__isnull=True) | Q(retry_at__gt=now),
@@ -335,13 +316,9 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
             "Finishing {count} browser-extension Snapshot(s) that received uploaded files before server extractors started "
             "(uploaded and server-created results will remain together on the same Snapshot)."
         ),
-        "archiveresults_backoff": (
-            "Retrying {count} extractor result(s) that were waiting to retry "
-            "(ArchiveBox may have been interrupted before it was able to try them again; affected outputs will be retried)."
-        ),
-        "archiveresults_started_without_running_process": (
-            "Retrying {count} extractor result(s) that were interrupted before finishing "
-            "(ArchiveBox may have been interrupted before it was able to save their final status; partial files will be overwritten with fresh results upon retry)."
+        "archiveresults_interrupted_without_running_process": (
+            "Closing {count} interrupted extractor result projection(s) "
+            "(their parent Snapshots are resumed through the normal snapshot-level runner)."
         ),
         "snapshots_started_without_running_results": (
             "Resuming {count} Snapshot(s) that were interrupted before finishing "

archivebox/core/routes_util.py

@@ -60,13 +60,6 @@ def _csrf_trusted_origins(config) -> list[str]:
     return seen
 
 
-def _allowed_hosts(config) -> set[str]:
-    raw = (config.ALLOWED_HOSTS or "").strip()
-    if not raw:
-        return set()
-    return {entry.strip().lower() for entry in raw.split(",") if entry.strip() and entry.strip() != "*"}
-
-
 def derive_base_url_from_csrf(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
     """Pick a single CSRF_TRUSTED_ORIGINS entry to act as the implicit BASE_URL.
 

archivebox/core/settings.py

@@ -76,7 +76,6 @@ INSTALLED_APPS = [
     "archivebox.crawls",  # handles Crawl and CrawlSchedule models and management (depends on core)
     "archivebox.progressmonitor",  # live progress endpoint and admin monitor template
     "archivebox.api",  # Django-Ninja-based Rest API interfaces, config, APIToken model, etc.
-    "archivebox.opencode",
     # 3rd-party apps from PyPI that need to be loaded last
     "admin_data_views",  # handles rendering some convenient automatic read-only views of data in Django admin
     "django_extensions",  # provides Django Debug Toolbar (and other non-debug helpers)

archivebox/core/tag_util.py

@@ -5,7 +5,6 @@ from collections import defaultdict
 from typing import Any
 from urllib.parse import unquote
 
-from django.contrib.auth.models import User
 from django.db.models import Count, Exists, F, OuterRef, QuerySet
 from django.db.models.functions import Lower
 from django.http import HttpRequest
@@ -147,41 +146,6 @@ def get_tag_by_ref(tag_ref: str | int) -> Tag:
     return Tag.objects.get(name__iexact=decoded)
 
 
-def get_or_create_tag(name: str, created_by: User | None = None) -> tuple[Tag, bool]:
-    normalized_name = normalize_tag_name(name)
-    if not normalized_name:
-        raise ValueError("Tag name is required")
-
-    existing = Tag.objects.filter(name__iexact=normalized_name).first()
-    if existing:
-        return existing, False
-
-    tag = Tag.objects.create(
-        name=normalized_name,
-        created_by=created_by,
-    )
-    return tag, True
-
-
-def rename_tag(tag: Tag, name: str) -> Tag:
-    normalized_name = normalize_tag_name(name)
-    if not normalized_name:
-        raise ValueError("Tag name is required")
-
-    existing = Tag.objects.filter(name__iexact=normalized_name).exclude(pk=tag.pk).first()
-    if existing:
-        raise ValueError(f'Tag "{existing.name}" already exists')
-
-    if tag.name != normalized_name:
-        tag.name = normalized_name
-        tag.save()
-    return tag
-
-
-def delete_tag(tag: Tag) -> tuple[int, dict[str, int]]:
-    return tag.delete()
-
-
 def export_tag_urls(tag: Tag) -> str:
     urls = tag.snapshot_set.order_by("-downloaded_at", "-created_at", "-pk").values_list("url", flat=True)
     return "\n".join(urls)
@@ -257,10 +221,10 @@ def build_tag_card(tag: Tag, snapshot_previews: list[dict[str, Any]] | None = No
         "num_snapshots": count,
         "filter_url": f"/admin/core/snapshot/?tags__id__exact={tag.pk}",
         "edit_url": f"/admin/core/tag/{tag.pk}/change/",
-        "export_urls_url": f"/api/v1/core/tag/{tag.pk}/urls.txt",
-        "export_jsonl_url": f"/api/v1/core/tag/{tag.pk}/snapshots.jsonl",
-        "rename_url": f"/api/v1/core/tag/{tag.pk}/rename",
-        "delete_url": f"/api/v1/core/tag/{tag.pk}/",
+        "export_urls_url": reverse("api-1:tag_urls_export", args=[tag.pk]),
+        "export_jsonl_url": reverse("api-1:tag_snapshots_export", args=[tag.pk]),
+        "rename_url": reverse("api-1:rename_tag", args=[tag.pk]),
+        "delete_url": reverse("api-1:delete_tag", args=[tag.pk]),
         "snapshots": snapshot_previews or [],
     }
 

archivebox/core/takeover_util.py

@@ -1,3 +1,17 @@
+"""Coordinate local foreground processes without pretending to provide a distributed lock.
+
+ArchiveBox wants one orchestrator per collection, but Process rows can only prove
+liveness for PIDs visible on the current machine/PID namespace. We therefore
+enforce one active runner per ``(Machine, DATA_DIR)`` locally, retire stale rows
+from sequential containers on that machine, and only warn about rows owned by a
+different machine. Foreign-machine rows must never block progress or be killed:
+multi-machine coordination belongs in the Crawl/Snapshot CAS claim layer, not in
+process takeover.
+
+These helpers only hand local supervisord/runner ownership between CLI parents.
+They must not hold database transactions or filesystem locks while work runs.
+"""
+
 from __future__ import annotations
 
 import time
@@ -15,25 +29,6 @@ RUNNER_WAITING_WORKER_TYPE = "runner_waiting"
 RUNNER_GATE_WORKER_TYPES = (RUNNER_ACTIVE_WORKER_TYPE, RUNNER_WAITING_WORKER_TYPE, "")
 
 
-def runtime_stack_owner_types():
-    from archivebox.machine.models import Process
-
-    return (
-        Process.TypeChoices.SERVER,
-        Process.TypeChoices.ORCHESTRATOR,
-    )
-
-
-def foreground_runner_owner_types():
-    from archivebox.machine.models import Process
-
-    return (
-        Process.TypeChoices.SERVER,
-        Process.TypeChoices.ADD,
-        Process.TypeChoices.UPDATE,
-    )
-
-
 def current_command(process_type: str, *, data_dir: str | Path, url: str | None = None):
     from archivebox.machine.models import Process
 
@@ -42,31 +37,8 @@ def current_command(process_type: str, *, data_dir: str | Path, url: str | None
     return proc
 
 
-def live_processes(*, process_type: str, data_dir: str | Path, url: str | None = None):
-    from archivebox.machine.models import Machine, Process
-
-    qs = Process.objects.filter(
-        machine=Machine.current(),
-        process_type=process_type,
-        status=Process.StatusChoices.RUNNING,
-        pwd=str(data_dir),
-    )
-    if url is not None:
-        qs = qs.filter(url=url)
-    return [proc for proc in qs.order_by("-created_at", "-modified_at").iterator(chunk_size=50) if proc.is_running]
-
-
-def newest_live_process(*, process_type: str, data_dir: str | Path, url: str | None = None):
-    processes = live_processes(process_type=process_type, data_dir=data_dir, url=url)
-    return processes[0] if processes else None
-
-
-def command_is_newest(command, *, process_type: str, data_dir: str | Path, url: str | None = None) -> bool:
-    leader = newest_live_process(process_type=process_type, data_dir=data_dir, url=url)
-    return bool(leader and leader.id == command.id)
-
-
 def runtime_stack_owner(*, data_dir: str | Path, exclude_id=None):
+    """Return the live local parent allowed to own the server runtime stack."""
     from archivebox.machine.models import Machine, Process
 
     machine = Machine.current()
@@ -74,7 +46,7 @@ def runtime_stack_owner(*, data_dir: str | Path, exclude_id=None):
         machine=machine,
         status=Process.StatusChoices.RUNNING,
         pwd=str(data_dir),
-        process_type__in=runtime_stack_owner_types(),
+        process_type__in=(Process.TypeChoices.SERVER, Process.TypeChoices.ORCHESTRATOR),
     )
     if exclude_id is not None:
         base_qs = base_qs.exclude(id=exclude_id)
@@ -103,6 +75,7 @@ def command_owns_runtime_stack(command, *, data_dir: str | Path) -> bool:
 
 
 def foreground_runner_owner(*, data_dir: str | Path, exclude_id=None):
+    """Return the newest live local parent allowed to borrow runner/sonic."""
     from archivebox.machine.models import Machine, Process
 
     machine = Machine.current()
@@ -110,7 +83,7 @@ def foreground_runner_owner(*, data_dir: str | Path, exclude_id=None):
         machine=machine,
         status=Process.StatusChoices.RUNNING,
         pwd=str(data_dir),
-        process_type__in=foreground_runner_owner_types(),
+        process_type__in=(Process.TypeChoices.SERVER, Process.TypeChoices.ADD, Process.TypeChoices.UPDATE),
     )
     if exclude_id is not None:
         qs = qs.exclude(id=exclude_id)
@@ -126,26 +99,6 @@ def command_owns_foreground_runner(command, *, data_dir: str | Path) -> bool:
     return bool(owner and owner.id == command.id)
 
 
-def runtime_stack_component_label(*, owner=None, data_dir: str | Path) -> str:
-    try:
-        from archivebox.workers.supervisord_util import active_supervisord_runtime_components
-
-        components = active_supervisord_runtime_components()
-    except Exception:
-        components = []
-
-    names = list(components)
-    if not names and owner is not None:
-        from archivebox.machine.models import Process
-
-        if owner.process_type == Process.TypeChoices.SERVER:
-            names = ["orchestrator", "server"]
-        elif owner.process_type == Process.TypeChoices.ORCHESTRATOR:
-            names = ["orchestrator"]
-
-    return ", ".join(dict.fromkeys(names)) or "runtime stack"
-
-
 def ensure_daemon_stack(*, reason: str = ""):
     from archivebox.config.common import get_config
     from archivebox.workers.supervisord_util import (
@@ -173,39 +126,26 @@ def ensure_daemon_stack(*, reason: str = ""):
 
     supervisor = get_existing_supervisord_process() or get_or_create_supervisord_process(daemonize=False)
     worker = get_worker(supervisor, sonic_worker["name"])
-    if isinstance(worker, dict) and worker.get("statename") in ("STARTING", "RUNNING"):
-        return worker
+    if not (isinstance(worker, dict) and worker.get("statename") in ("STARTING", "RUNNING")):
+        if reason:
+            rprint(f"[yellow][*] Starting daemon stack for {reason}...[/yellow]")
+        worker = start_worker(supervisor, sonic_worker)
 
-    if reason:
-        rprint(f"[yellow][*] Starting daemon stack for {reason}...[/yellow]")
-    return start_worker(supervisor, sonic_worker)
+    from archivebox.search.sonic_daemon import wait_for_sonic_daemon
+
+    wait_for_sonic_daemon(sonic_event)
+    return worker
 
 
-def healthy_orchestrator(*, data_dir: str | Path):
-    from archivebox.machine.models import Machine, Process
-    from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker
+def live_runner_processes(*, data_dir: str | Path):
+    """Return locally verifiable runners and warn about unsupported overlap.
 
-    supervisor = get_existing_supervisord_process()
-    worker = get_worker(supervisor, "worker_runner") if supervisor else None
-    if isinstance(worker, dict) and worker.get("statename") in ("STARTING", "RUNNING"):
-        return worker
-
-    for proc in Process.objects.filter(
-        machine=Machine.current(),
-        process_type=Process.TypeChoices.ORCHESTRATOR,
-        status=Process.StatusChoices.RUNNING,
-        pwd=str(data_dir),
-    ).order_by("-created_at"):
-        if proc.is_running:
-            return proc
-    return None
-
-
-def _runner_sort_key(process):
-    return (process.started_at or process.created_at, process.created_at, str(process.id))
-
-
-def live_runner_processes(*, data_dir: str | Path, exclude_id=None):
+    A Process row from another machine is observability only: its PID cannot be
+    checked or signalled here, so it neither joins the local election nor gets
+    mutated. A row for this same Machine from another PID namespace represents
+    a previous sequential container under the supported model; warn, retire the
+    unreachable row, and let the new container continue.
+    """
     from archivebox.machine.models import Machine, Process
 
     machine = Machine.current()
@@ -218,9 +158,6 @@ def live_runner_processes(*, data_dir: str | Path, exclude_id=None):
     )
     foreign_machine_exists = qs.exclude(machine=machine).exists()
     qs = qs.filter(machine=machine)
-    if exclude_id is not None:
-        qs = qs.exclude(id=exclude_id)
-
     live = []
     foreign_namespace_ids = []
     for process in qs.order_by("started_at", "created_at").iterator(chunk_size=20):
@@ -250,13 +187,15 @@ def live_runner_processes(*, data_dir: str | Path, exclude_id=None):
 
 def enter_single_runner_gate(command, *, data_dir: str | Path, graceful_timeout: float = 5.0) -> bool:
     """
-    Admit exactly one active runner for this DATA_DIR using Process rows.
+    Admit one active runner for this Machine and DATA_DIR using Process rows.
 
     The current process is a real OS process while it waits, so we keep its
     Process row RUNNING but mark worker_type=runner_waiting. Only the process
     that wins takeover is promoted to worker_type=worker_runner, which is
-    protected by a partial unique DB constraint. Older runners are terminated
-    and fully waited out before promotion, so the runner work loop never overlaps.
+    protected by a partial unique DB constraint scoped to (Machine, DATA_DIR).
+    Older locally verifiable runners are terminated and fully waited out before
+    promotion, so runner work never overlaps on one machine. Foreign machines
+    are intentionally outside this gate and only produce a warning above.
     """
     from archivebox.machine.models import Process
 
@@ -278,7 +217,7 @@ def enter_single_runner_gate(command, *, data_dir: str | Path, graceful_timeout:
             )
             runners = live_runner_processes(data_dir=data_dir)
 
-        newest = max(runners, key=_runner_sort_key)
+        newest = max(runners, key=lambda process: (process.started_at or process.created_at, process.created_at, str(process.id)))
         if newest.id != command.id:
             rprint(
                 f"[yellow][*] Newer ArchiveBox runner pid={newest.pid} is taking over; exiting this runner.[/yellow]",
@@ -310,24 +249,9 @@ def enter_single_runner_gate(command, *, data_dir: str | Path, graceful_timeout:
             time.sleep(0.1)
 
 
-def standby_until_leader_needed(command, *, process_type: str, data_dir: str | Path, url: str | None = None, interval: float = 2.0) -> None:
-    from archivebox.workers.supervisord_util import reap_foreground_supervisord_process
-
-    announced = False
-    while not command_is_newest(command, process_type=process_type, data_dir=data_dir, url=url):
-        reap_foreground_supervisord_process()
-        if not announced:
-            leader = newest_live_process(process_type=process_type, data_dir=data_dir, url=url)
-            leader_pid = leader.pid if leader else "unknown"
-            rprint(f"[yellow][*] Standing by; newer ArchiveBox process pid={leader_pid} is running the orchestrator and server.[/yellow]")
-            announced = True
-        time.sleep(interval)
-    command.modified_at = timezone.now()
-    command.save(update_fields=["modified_at"])
-
-
 def standby_until_runtime_stack_needed(command, *, data_dir: str | Path, interval: float = 2.0) -> dict[str, object]:
-    from archivebox.workers.supervisord_util import reap_foreground_supervisord_process
+    from archivebox.machine.models import Process
+    from archivebox.workers.supervisord_util import active_supervisord_runtime_components, reap_foreground_supervisord_process
 
     announced = False
     previous_owner_pid = None
@@ -336,7 +260,16 @@ def standby_until_runtime_stack_needed(command, *, data_dir: str | Path, interva
         if not announced:
             owner = runtime_stack_owner(data_dir=data_dir)
             owner_pid = owner.pid if owner else "unknown"
-            components = runtime_stack_component_label(owner=owner, data_dir=data_dir)
+            try:
+                component_names = list(active_supervisord_runtime_components())
+            except Exception:
+                component_names = []
+            if not component_names and owner is not None:
+                if owner.process_type == Process.TypeChoices.SERVER:
+                    component_names = ["orchestrator", "server"]
+                elif owner.process_type == Process.TypeChoices.ORCHESTRATOR:
+                    component_names = ["orchestrator"]
+            components = ", ".join(dict.fromkeys(component_names)) or "runtime stack"
             previous_owner_pid = owner_pid
             rprint(
                 f"[yellow][*] A newer archivebox process took over the {components} "

archivebox/core/templatetags/core_tags.py

@@ -1,8 +1,9 @@
+import logging
 import os
 from html import unescape
 from pathlib import Path
 from typing import Any
-from urllib.parse import quote
+from urllib.parse import quote, urlparse
 
 from abx_plugins.plugins.archivewebpage.replay_preview import is_replay_target as is_archivewebpage_replay_target
 from django import template
@@ -28,54 +29,38 @@ from archivebox.plugins.discovery import (
 
 register = template.Library()
 
+
+@register.simple_tag(takes_context=True)
+def plugin_ui(context, plugin: str, slot: str):
+    """Optional plugin UI must never prevent the host page from rendering."""
+    try:
+        source = get_plugin_template(plugin, slot, fallback=False)
+        if not source:
+            return ""
+        return template.engines["django"].from_string(source).render(context.flatten())
+    except Exception:
+        logging.getLogger(__name__).exception("Unable to render optional plugin UI: %s/%s", plugin, slot)
+        return ""
+
+
 _TEXT_PREVIEW_EXTS = (".json", ".jsonl", ".txt", ".csv", ".tsv", ".xml", ".yml", ".yaml", ".md", ".log")
 _IMAGE_PREVIEW_EXTS = (".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".ico", ".avif")
 _STATIC_URL_SAFE = "/@-._~!$&'()*+,;="
 
-_MEDIA_FILE_EXTS = {
-    ".mp4",
-    ".webm",
-    ".mkv",
-    ".avi",
-    ".mov",
-    ".flv",
-    ".wmv",
-    ".m4v",
-    ".mpg",
-    ".mpeg",
-    ".ts",
-    ".m2ts",
-    ".mts",
-    ".3gp",
-    ".3g2",
-    ".ogv",
-    ".mp3",
-    ".m4a",
-    ".aac",
-    ".ogg",
-    ".oga",
-    ".opus",
-    ".wav",
-    ".flac",
-    ".alac",
-    ".aiff",
-    ".wma",
-    ".mka",
-    ".ac3",
-    ".eac3",
-    ".dts",
-}
 
+def _is_web_subdomain_alias(browser_url: str, configured_base_url: str) -> bool:
+    """Hide the informational mismatch banner on the conventional public web alias.
 
-def _normalize_output_files(output_files: Any) -> dict[str, dict[str, Any]]:
-    if isinstance(output_files, dict):
-        normalized: dict[str, dict[str, Any]] = {}
-        for path, metadata in output_files.items():
-            if not path:
-                continue
-            normalized[str(path)] = dict(metadata) if isinstance(metadata, dict) else {}
-        return normalized
-    return {}
+    This is deliberately presentation-only: request routing and the restriction
+    on state-changing requests continue to use the configured BASE_URL.
+    """
+    browser = urlparse(browser_url)
+    configured = urlparse(configured_base_url)
+    return (
+        browser.scheme.lower() == configured.scheme.lower()
+        and browser.hostname == f"web.{configured.hostname}"
+        and browser.port == configured.port
+    )
 
 
 def _snapshot_id(value: Any) -> Any:
@@ -84,93 +69,6 @@ def _snapshot_id(value: Any) -> Any:
     return value.id if isinstance(value, Snapshot) else value
 
 
-def _coerce_output_file_size(value: Any) -> int | None:
-    try:
-        return max(int(value or 0), 0)
-    except (TypeError, ValueError):
-        return None
-
-
-def _count_media_files(result) -> int:
-    try:
-        output_files = _normalize_output_files(result.output_files or {})
-    except (AttributeError, TypeError, ValueError):
-        output_files = {}
-
-    if output_files:
-        return sum(1 for path in output_files if Path(path).suffix.lower() in _MEDIA_FILE_EXTS)
-
-    try:
-        plugin_dir = Path(result.snapshot_dir) / result.plugin
-    except (AttributeError, TypeError, ValueError):
-        return 0
-
-    if not plugin_dir.exists():
-        return 0
-
-    count = 0
-    scanned = 0
-    max_scan = 500
-    for file_path in plugin_dir.rglob("*"):
-        if scanned >= max_scan:
-            break
-        scanned += 1
-        if not file_path.is_file():
-            continue
-        if file_path.suffix.lower() in _MEDIA_FILE_EXTS:
-            count += 1
-    return count
-
-
-def _list_media_files(result) -> list[dict]:
-    media_files: list[dict] = []
-    try:
-        plugin_dir = Path(result.snapshot_dir) / result.plugin
-    except (AttributeError, TypeError, ValueError):
-        return media_files
-
-    output_files = _normalize_output_files(result.output_files or {})
-    candidates: list[tuple[Path, int | None]] = []
-    if output_files:
-        for path, metadata in output_files.items():
-            rel_path = Path(path)
-            if rel_path.suffix.lower() in _MEDIA_FILE_EXTS:
-                candidates.append((rel_path, _coerce_output_file_size(metadata.get("size"))))
-
-    if not candidates and plugin_dir.exists():
-        scanned = 0
-        max_scan = 2000
-        for file_path in plugin_dir.rglob("*"):
-            if scanned >= max_scan:
-                break
-            scanned += 1
-            if not file_path.is_file():
-                continue
-            if file_path.suffix.lower() in _MEDIA_FILE_EXTS:
-                try:
-                    rel_path = file_path.relative_to(plugin_dir)
-                except ValueError:
-                    continue
-                try:
-                    size = file_path.stat().st_size
-                except OSError:
-                    size = None
-                candidates.append((rel_path, size))
-
-    for rel_path, size in candidates:
-        href = str(Path(result.plugin) / rel_path)
-        media_files.append(
-            {
-                "name": rel_path.name,
-                "path": href,
-                "size": size,
-            },
-        )
-
-    media_files.sort(key=lambda item: item["name"].lower())
-    return media_files
-
-
 def _resolve_snapshot_output_file(snapshot_dir: str | Path | None, raw_output_path: str | None) -> Path | None:
     if not snapshot_dir or not raw_output_path or str(raw_output_path).strip() in (".", "/", "./"):
         return None
@@ -448,7 +346,7 @@ def system_warnings_banner(context):
     if not config.BASE_URL:
         return get_setup_wizard_context(context.get("request"), config)
     mismatch = get_base_url_mismatch_context(context.get("request"), config)
-    if mismatch:
+    if mismatch and not _is_web_subdomain_alias(mismatch["browser_url"], mismatch["configured_base_url"]):
         return mismatch
     if config.IS_LOWER_SECURITY_MODE:
         return {"mode": "unsafe"}
@@ -776,8 +674,7 @@ def plugin_card(context, result) -> str:
 
     icon_html = get_plugin_icon(plugin)
     plugin_lower = (plugin or "").lower()
-    media_file_count = _count_media_files(result) if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl") else 0
-    media_files = _list_media_files(result) if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl") else []
+    media_files = result.media_files() if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl") else []
     if context.get("STATIC_EXPORT") and media_files:
         media_files = [
             item
@@ -787,7 +684,6 @@ def plugin_card(context, result) -> str:
             and ".." not in media_path.parts
             and (Path(result.snapshot_dir) / media_path).is_file()
         ]
-        media_file_count = len(media_files)
     if media_files:
         for item in media_files:
             path = item.get("path") or ""
@@ -808,7 +704,7 @@ def plugin_card(context, result) -> str:
                     "output_path_raw": raw_output_path,
                     "plugin": plugin,
                     "plugin_icon": icon_html,
-                    "media_file_count": media_file_count,
+                    "media_file_count": len(media_files),
                     "media_files": media_files,
                 },
             )

archivebox/core/views/__init__.py

@@ -0,0 +1,39 @@
+"""HTTP views grouped by public browsing, ingestion, replay, and configuration."""
+
+from archivebox.config.common import find_config_source as find_config_source
+from archivebox.plugins.views import get_config_definition_link as get_config_definition_link
+
+from .add import AddView as AddView
+from .add import WebAddView as WebAddView
+from .config import live_config_list_view as live_config_list_view
+from .config import live_config_value_view as live_config_value_view
+from .lookup import _files_index_target as _files_index_target
+from .lookup import _find_snapshot_by_ref as _find_snapshot_by_ref
+from .public import HealthCheckView as HealthCheckView
+from .public import HomepageView as HomepageView
+from .public import PublicIndexView as PublicIndexView
+from .replay import OriginalDomainHostView as OriginalDomainHostView
+from .replay import OriginalDomainReplayView as OriginalDomainReplayView
+from .replay import SnapshotHostView as SnapshotHostView
+from .replay import SnapshotPathView as SnapshotPathView
+from .replay import SnapshotReplayView as SnapshotReplayView
+from .replay import SnapshotView as SnapshotView
+from .replay import _latest_response_match as _latest_response_match
+from .replay import _latest_responses_root as _latest_responses_root
+from .replay import _original_request_url as _original_request_url
+from .replay import _plugin_full_preview_response as _plugin_full_preview_response
+from .replay import _resolve_archiveresult_relpath as _resolve_archiveresult_relpath
+from .replay import _safe_archive_relpath as _safe_archive_relpath
+from .replay import _serve_original_domain_replay as _serve_original_domain_replay
+from .replay import _serve_responses_path as _serve_responses_path
+from .replay import _serve_snapshot_replay as _serve_snapshot_replay
+from .replay import _visible_response_snapshots_for_domain as _visible_response_snapshots_for_domain
+from .replay_auth import REPLAY_AUTH_SALT as REPLAY_AUTH_SALT
+from .replay_auth import SnapshotReplayAuthView as SnapshotReplayAuthView
+from .replay_auth import _admin_login_redirect_or_forbidden as _admin_login_redirect_or_forbidden
+from .replay_auth import _clean_replay_next as _clean_replay_next
+from .replay_auth import _has_replay_cookie as _has_replay_cookie
+from .replay_auth import _private_snapshot_auth_redirect as _private_snapshot_auth_redirect
+from .replay_auth import _replay_auth_response as _replay_auth_response
+from .replay_auth import _replay_cookie_name as _replay_cookie_name
+from .replay_auth import _replay_payload_is_valid as _replay_payload_is_valid

archivebox/core/views/add.py

@@ -0,0 +1,363 @@
+import json
+from urllib.parse import quote
+
+from django.contrib import messages
+from django.contrib.auth.mixins import UserPassesTestMixin
+from django.http import Http404, HttpRequest, HttpResponse, QueryDict
+from django.shortcuts import redirect
+from django.utils.decorators import method_decorator
+from django.utils.html import format_html
+from django.views.decorators.csrf import csrf_exempt, csrf_protect
+from django.views.generic import FormView
+
+from archivebox.config import VERSION
+from archivebox.config.common import (
+    get_config,
+    get_request_config,
+    redact_sensitive_config,
+)
+from archivebox.core.forms import AddLinkForm
+from archivebox.core.models import Snapshot
+from archivebox.core.permissions import (
+    PERMISSIONS_PUBLIC,
+    direct_snapshots_queryset,
+    filter_personas_by_permissions,
+)
+from archivebox.core.routes_util import (
+    build_admin_url,
+    build_web_url,
+    get_admin_host,
+    get_web_host,
+    host_matches,
+)
+from archivebox.crawls.models import Crawl
+from archivebox.misc.util import (
+    sanitize_html_text,
+    urldecode,
+    validate_url,
+)
+from archivebox.plugins.forms import get_plugin_config_binary_urls
+
+from .replay import SnapshotView
+
+
+@method_decorator(csrf_exempt, name="dispatch")
+class AddView(UserPassesTestMixin, FormView):
+    template_name = "add.html"
+    form_class = AddLinkForm
+
+    def get_initial(self):
+        """Prefill the AddLinkForm with the 'url' GET parameter"""
+        if self.request.method == "GET":
+            url = self.request.GET.get("url", None)
+            if url:
+                return {"url": url if "://" in url else f"https://{url}"}
+
+        return super().get_initial()
+
+    def get_form_kwargs(self):
+        kwargs = super().get_form_kwargs()
+        kwargs["request"] = self.request
+        return kwargs
+
+    def test_func(self):
+        return get_request_config(self.request).PUBLIC_ADD_VIEW or self.request.user.is_authenticated
+
+    def post(self, request: HttpRequest, *args: object, **kwargs: object):
+        if request.user.is_authenticated:
+            return csrf_protect(super().post)(request, *args, **kwargs)
+        return super().post(request, *args, **kwargs)
+
+    def _can_override_crawl_config(self) -> bool:
+        user = self.request.user
+        return bool(user.is_authenticated and user.is_active and user.is_superuser)
+
+    def _get_custom_config_overrides(self, form: AddLinkForm) -> dict:
+        custom_config = form.cleaned_data.get("config") or {}
+
+        if not isinstance(custom_config, dict):
+            return {}
+
+        if not self._can_override_crawl_config():
+            return {}
+
+        return {str(key): value for key, value in custom_config.items() if not str(key).endswith("_BINARY")}
+
+    def get_context_data(self, **kwargs):
+        context = super().get_context_data(**kwargs)
+        request_config = get_request_config(self.request, resolve_plugins=True)
+        required_search_plugin = f"search_backend_{request_config.SEARCH_BACKEND_ENGINE}".strip()
+        can_override_crawl_config = self._can_override_crawl_config()
+        public_persona_config_keys = {
+            "CRAWL_MAX_CONCURRENT_SNAPSHOTS",
+            "DELETE_AFTER",
+            "ONLY_NEW",
+            "PERMISSIONS",
+            "TIMEOUT",
+        }
+        persona_queryset = context["form"].fields["persona"].queryset
+        if not can_override_crawl_config:
+            persona_queryset = filter_personas_by_permissions(persona_queryset, {PERMISSIONS_PUBLIC})
+        persona_config_map = {}
+        for persona in persona_queryset.order_by("name"):
+            effective_config = get_config(persona=persona)
+            effective_config_redacted = redact_sensitive_config(effective_config.model_dump(mode="json"))
+            if can_override_crawl_config:
+                raw_config = redact_sensitive_config(persona.config or {})
+                effective_config_json = effective_config_redacted
+                binary_urls = get_plugin_config_binary_urls(effective_config)
+            else:
+                raw_config = {}
+                effective_config_json = {key: effective_config_redacted.get(key) for key in public_persona_config_keys}
+                binary_urls = {}
+            persona_config_map[persona.name] = {
+                "config": raw_config,
+                "effective_config": effective_config_json,
+                "binary_urls": binary_urls,
+            }
+        recent_personas = list(persona_queryset.order_by("-created_at", "name")[:5])
+        return {
+            **context,
+            "title": "Create Crawl",
+            # We can't just call request.build_absolute_uri in the template, because it would include query parameters
+            "absolute_add_path": self.request.build_absolute_uri(self.request.path),
+            "web_base_url": build_web_url("", request=self.request),
+            "VERSION": VERSION,
+            "FOOTER_INFO": request_config.FOOTER_INFO,
+            "required_search_plugin": required_search_plugin,
+            "persona_config_map_json": json.dumps(persona_config_map, sort_keys=True, default=str),
+            "recent_personas": recent_personas,
+            "can_override_crawl_config": can_override_crawl_config,
+            "stdout": "",
+        }
+
+    def _create_crawl_from_form(self, form, *, created_by_id=None) -> Crawl:
+        from archivebox.cli.archivebox_add import add
+
+        urls_input = form.cleaned_data["url"]
+        urls = urls_input
+        submitted_lines = [line.strip() for line in urls_input.splitlines() if line.strip()]
+        if len(submitted_lines) == 1:
+            try:
+                # A lone URL pasted into /add/ is the same user-facing input as
+                # `archivebox add https://...`: queue that URL directly so a
+                # narrow plugin selection like `wget` can archive it without
+                # also needing parser plugins. Multi-line or formatted text
+                # remains verbatim import content for the internal parser root.
+                urls = [validate_url(submitted_lines[0])]
+            except ValueError:
+                pass
+        print(f"[+] Adding URL: {urls_input}")
+
+        # Extract all form fields
+        tag = form.cleaned_data["tag"]
+        depth = int(form.cleaned_data["depth"])
+        max_urls = int(form.cleaned_data.get("max_urls") or 0)
+        crawl_max_size = int(form.cleaned_data.get("crawl_max_size") or 0)
+        crawl_timeout = int(form.cleaned_data.get("crawl_timeout") or 0)
+        timeout = form.cleaned_data.get("timeout")
+        snapshot_max_size = int(form.cleaned_data.get("snapshot_max_size") or 0)
+        delete_after = str(form.cleaned_data.get("delete_after") or "0").strip() or "0"
+        crawl_max_concurrent_snapshots = int(form.cleaned_data["crawl_max_concurrent_snapshots"])
+        permissions = str(form.cleaned_data.get("permissions") or "public").strip().lower()
+        can_override_crawl_config = self._can_override_crawl_config()
+        plugins = ",".join(form.cleaned_data.get("plugins", [])) if can_override_crawl_config else ""
+        schedule = form.cleaned_data.get("schedule", "").strip() if can_override_crawl_config else ""
+        persona = form.cleaned_data.get("persona")
+        start_paused = form.cleaned_data.get("start_paused", False) if can_override_crawl_config else False
+        notes = form.cleaned_data.get("notes", "")
+        url_filters = form.cleaned_data.get("url_filters") or {}
+        plugin_config = form.cleaned_data.get("plugin_config") or {}
+        if not isinstance(plugin_config, dict):
+            plugin_config = {}
+        if not can_override_crawl_config:
+            plugin_config = {}
+        custom_config = self._get_custom_config_overrides(form)
+        custom_config.pop("DEFAULT_PERSONA", None)
+        custom_config.pop("PERMISSIONS", None)
+        if persona:
+            persona.ensure_dirs()
+
+        if created_by_id is None:
+            if self.request.user.is_authenticated:
+                created_by_id = self.request.user.pk
+            else:
+                from archivebox.base_models.models import get_or_create_system_user_pk
+
+                created_by_id = get_or_create_system_user_pk()
+
+        config = {}
+        effective_config = get_config(persona=persona) if persona else get_config()
+        if delete_after != str(effective_config.DELETE_AFTER):
+            config["DELETE_AFTER"] = delete_after
+        if timeout is not None and int(timeout) != int(effective_config.TIMEOUT):
+            config["TIMEOUT"] = int(timeout)
+        if permissions:
+            config["PERMISSIONS"] = permissions
+
+        config.update(plugin_config)
+        config.update(custom_config)
+        if bool(url_filters.get("only_new")) != bool(effective_config.ONLY_NEW):
+            config["ONLY_NEW"] = bool(url_filters.get("only_new"))
+        crawl, _snapshots = add(
+            urls=urls,
+            depth=depth,
+            max_urls=max_urls,
+            crawl_max_size=crawl_max_size,
+            crawl_timeout=crawl_timeout,
+            snapshot_max_size=snapshot_max_size,
+            crawl_max_concurrent_snapshots=crawl_max_concurrent_snapshots,
+            tag=tag,
+            url_allowlist=url_filters.get("allowlist") or "",
+            url_denylist=url_filters.get("denylist") or "",
+            plugins=plugins,
+            persona=persona.name if persona else "Default",
+            bg=True,
+            created_by_id=created_by_id,
+            config=config,
+        )
+        if notes:
+            crawl.safe_update({"notes": sanitize_html_text(notes)}, refresh=False)
+        if permissions and crawl.config.get("PERMISSIONS") != permissions:
+            next_config = {**crawl.config, "PERMISSIONS": permissions}
+            crawl.safe_update({"config": next_config}, refresh=True)
+        if start_paused:
+            crawl.pause()
+
+        # 3. create a CrawlSchedule if schedule is provided
+        if schedule:
+            from archivebox.crawls.models import CrawlSchedule
+
+            crawl_schedule = CrawlSchedule.objects.create(
+                template=crawl,
+                schedule=schedule,
+                is_enabled=True,
+                config=config,
+                label=crawl.label,
+                notes=f"Auto-created from add page. {notes}".strip(),
+                created_by_id=created_by_id,
+            )
+            crawl.schedule = crawl_schedule
+            crawl.safe_update({"schedule": crawl_schedule}, refresh=False)
+
+        return crawl
+
+    def form_valid(self, form):
+        crawl = self._create_crawl_from_form(form)
+
+        urls = form.cleaned_data["url"]
+        schedule = form.cleaned_data.get("schedule", "").strip()
+        rough_url_count = len([url for url in urls.splitlines() if url.strip()])
+
+        schedule_msg = ""
+        if schedule and crawl.schedule_id:
+            schedule_msg = format_html(" and <a href='{}'>scheduled to repeat {}</a>", crawl.schedule.admin_change_url, schedule)
+
+        messages.success(
+            self.request,
+            format_html(
+                "Created crawl with {} starting URL(s){}. Snapshots will be created and archived in the background. <a href='{}'>View Crawl →</a>",
+                rough_url_count,
+                schedule_msg,
+                crawl.admin_change_url,
+            ),
+        )
+
+        # Orchestrator (managed by supervisord) will pick up the queued crawl
+        return redirect(crawl.admin_change_url)
+
+
+class WebAddView(AddView):
+    def _latest_snapshot_for_url(self, requested_url: str):
+        return (
+            direct_snapshots_queryset(
+                self.request,
+                SnapshotView.find_snapshots_for_url(requested_url),
+            )
+            .order_by("-bookmarked_at", "-created_at", "-timestamp")
+            .first()
+        )
+
+    def _normalize_add_url(self, requested_url: str) -> str:
+        if requested_url.startswith(("http://", "https://")):
+            return requested_url
+        return f"https://{requested_url}"
+
+    def dispatch(self, request, *args, **kwargs):
+        requested_url = urldecode(kwargs.get("url", "") or "")
+        if requested_url:
+            snapshot = self._latest_snapshot_for_url(requested_url)
+            if snapshot:
+                return redirect(f"/{snapshot.url_path}")
+
+        request_host = (request.get_host() or "").lower()
+        request_config = get_request_config(request)
+        web_host = get_web_host(config=request_config, request=request)
+        admin_host = get_admin_host(config=request_config, request=request)
+        is_web_host = host_matches(request_host, web_host)
+        is_admin_host = host_matches(request_host, admin_host)
+        if request.user.is_authenticated and not request_config.PUBLIC_ADD_VIEW and is_web_host and not is_admin_host:
+            return redirect(build_admin_url(request.get_full_path(), request=request))
+
+        if not self.test_func():
+            if is_web_host and not is_admin_host:
+                return redirect(build_admin_url(request.get_full_path(), request=request))
+            if is_admin_host:
+                next_url = quote(request.get_full_path(), safe="/:?=&")
+                return redirect(f"{build_admin_url('/admin/login/', request=request)}?next={next_url}")
+            return HttpResponse(
+                format_html(
+                    (
+                        "<center><br/><br/><br/>"
+                        "No Snapshots match the given url: <code>{}</code><br/><br/><br/>"
+                        'Return to the <a href="/" target="_top">Main Index</a>'
+                        "</center>"
+                    ),
+                    requested_url or "",
+                ),
+                content_type="text/html",
+                status=404,
+            )
+
+        return super().dispatch(request, *args, **kwargs)
+
+    def get(self, request: HttpRequest, *args: object, **kwargs: object):
+        requested_url = urldecode(str(kwargs.get("url") or (args[0] if args else "")))
+        if not requested_url:
+            raise Http404
+
+        snapshot = self._latest_snapshot_for_url(requested_url)
+        if snapshot:
+            return redirect(f"/{snapshot.url_path}")
+
+        add_url = self._normalize_add_url(requested_url)
+        assert self.form_class is not None
+        defaults_form = self.form_class()
+        form_data = QueryDict(mutable=True)
+        form_data.update(
+            {
+                "url": add_url,
+                "depth": defaults_form.fields["depth"].initial or "0",
+                "max_urls": defaults_form.fields["max_urls"].initial or 0,
+                "crawl_max_size": defaults_form.fields["crawl_max_size"].initial or "0",
+                "crawl_timeout": defaults_form.fields["crawl_timeout"].initial or 0,
+                "timeout": defaults_form.fields["timeout"].initial or 0,
+                "snapshot_max_size": defaults_form.fields["snapshot_max_size"].initial or "0",
+                "delete_after": defaults_form.fields["delete_after"].initial or "0",
+                "crawl_max_concurrent_snapshots": defaults_form.fields["crawl_max_concurrent_snapshots"].initial,
+                "persona": defaults_form.fields["persona"].initial or "Default",
+                "permissions": defaults_form.fields["permissions"].initial or "public",
+                "config": "{}",
+            },
+        )
+        if defaults_form.fields["start_paused"].initial:
+            form_data["start_paused"] = "on"
+
+        form = self.form_class(data=form_data)
+        if not form.is_valid():
+            return self.form_invalid(form)
+
+        crawl = self._create_crawl_from_form(form)
+        snapshot = Snapshot.from_json({"url": add_url, "tags": form.cleaned_data.get("tag", "")}, overrides={"crawl": crawl})
+        assert snapshot is not None
+        return redirect(f"/{snapshot.url_path}")

archivebox/core/views/config.py

@@ -0,0 +1,261 @@
+import os
+from typing import cast
+
+from admin_data_views.typing import ItemContext, SectionData, TableContext
+from admin_data_views.utils import ItemLink, render_with_item_view, render_with_table_view
+from django.http import HttpRequest
+from django.utils.html import format_html, format_html_join
+from django.utils.safestring import mark_safe
+
+from archivebox.config import CONSTANTS, CONSTANTS_CONFIG
+from archivebox.config.common import (
+    PLUGIN_CONFIG_SCHEMAS,
+    SENSITIVE_CONFIG_VALUE_REDACTED,
+    _plugin_config_properties,
+    find_config_default,
+    find_config_section,
+    find_config_source,
+    find_config_type,
+    get_all_configs,
+    get_config,
+    redact_sensitive_config,
+)
+from archivebox.config.configset import BaseConfigSet
+from archivebox.plugins.views import get_config_definition_link
+
+
+@render_with_table_view
+def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
+    CONFIGS = get_all_configs()
+
+    assert request.user.is_superuser, "Must be a superuser to view configuration settings."
+
+    merged_config = get_config(redact_sensitive=True)
+
+    rows = {
+        "Section": [],
+        "Key": [],
+        "Type": [],
+        "Value": [],
+        "Source": [],
+        "Default": [],
+        # "Documentation": [],
+        # "Aliases": [],
+    }
+
+    for section_id, section in reversed(list(CONFIGS.items())):
+        for key in dict(section):
+            rows["Section"].append(section_id)  # section.replace('_', ' ').title().replace(' Config', '')
+            rows["Key"].append(ItemLink(key, key=key))
+            rows["Type"].append(format_html("<code>{}</code>", find_config_type(key)))
+
+            # Use merged config value (includes machine overrides)
+            actual_value = merged_config.get(key, dict(section)[key])
+            rows["Value"].append(format_html("<code>{}</code>", actual_value))
+
+            # Show where the value comes from
+            source = find_config_source(key, merged_config)
+            source_colors = {"Machine": "purple", "Environment": "blue", "File": "green", "Plugin Default": "teal", "Default": "gray"}
+            rows["Source"].append(format_html('<code style="color: {}">{}</code>', source_colors.get(source, "gray"), source))
+
+            rows["Default"].append(
+                format_html(
+                    '<a href="https://github.com/search?q=repo%3AArchiveBox%2FArchiveBox+path%3Aconfig+{}&type=code"><code style="text-decoration: underline">{}</code></a>',
+                    key,
+                    find_config_default(key) or "See here...",
+                ),
+            )
+
+    section = "CONSTANT"
+    for key in CONSTANTS_CONFIG:
+        rows["Section"].append(section)  # section.replace('_', ' ').title().replace(' Config', '')
+        rows["Key"].append(ItemLink(key, key=key))
+        rows["Type"].append(format_html("<code>{}</code>", type(CONSTANTS_CONFIG[key]).__name__))
+        rows["Value"].append(format_html("<code>{}</code>", redact_sensitive_config(CONSTANTS_CONFIG).get(key)))
+        rows["Source"].append(mark_safe('<code style="color: gray">Constant</code>'))
+        rows["Default"].append(
+            format_html(
+                '<a href="https://github.com/search?q=repo%3AArchiveBox%2FArchiveBox+path%3Aconfig+{}&type=code"><code style="text-decoration: underline">{}</code></a>',
+                key,
+                find_config_default(key) or "See here...",
+            ),
+        )
+
+    return TableContext(
+        title="Computed Configuration Values",
+        table=rows,
+    )
+
+
+@render_with_item_view
+def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
+    from archivebox.machine.models import Machine
+
+    CONFIGS = get_all_configs()
+
+    assert request.user.is_superuser, "Must be a superuser to view configuration settings."
+
+    merged_config = get_config(redact_sensitive=True)
+
+    # Determine all sources for this config value
+    sources_info = []
+
+    # Machine config
+    machine = Machine.current()
+    machine_admin_url = machine.admin_change_url
+    if machine.config and key in machine.config:
+        sources_info.append(("Machine", redact_sensitive_config(machine.config).get(key), "purple"))
+
+    # Environment variable
+    if key in os.environ:
+        sources_info.append(("Environment", redact_sensitive_config(os.environ).get(key), "blue"))
+
+    # Config file value
+    if CONSTANTS.CONFIG_FILE.exists():
+        file_config = BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE)
+        if key in file_config:
+            sources_info.append(("File", redact_sensitive_config(file_config).get(key), "green"))
+
+    # Default value
+    default_val = find_config_default(key)
+    if key in _plugin_config_properties(PLUGIN_CONFIG_SCHEMAS):
+        sources_info.append(("Plugin Default", default_val, "gray"))
+    elif default_val:
+        sources_info.append(("Default", default_val, "gray"))
+
+    # Final computed value
+    config_source = find_config_source(key, merged_config)
+    final_value = merged_config.get(key, CONFIGS.get(key, None))
+    is_redacted = final_value == SENSITIVE_CONFIG_VALUE_REDACTED
+
+    # Build sources display
+    sources_html = format_html_join(
+        mark_safe("<br/>"),
+        '<b style="color: {}">{}:</b> <code>{}</code>',
+        ((color, source, value) for source, value, color in sources_info),
+    )
+
+    aliases = []
+
+    if key in CONSTANTS_CONFIG:
+        section_header = format_html(
+            '[CONSTANTS]   &nbsp; <b><code style="color: lightgray">{}</code></b> &nbsp; <small>(read-only, hardcoded by ArchiveBox)</small>',
+            key,
+        )
+    elif key in merged_config:
+        section_header = format_html(
+            'data / ArchiveBox.conf &nbsp; [{}]  &nbsp; <b><code style="color: lightgray">{}</code></b>',
+            find_config_section(key),
+            key,
+        )
+    else:
+        section_header = format_html(
+            '[DYNAMIC CONFIG]   &nbsp; <b><code style="color: lightgray">{}</code></b> &nbsp; <small>(read-only, calculated at runtime)</small>',
+            key,
+        )
+
+    definition_url, definition_label = get_config_definition_link(key)
+    redacted_message = (
+        mark_safe(
+            '<b style="color: red">Value is redacted for your security. (Passwords, secrets, API tokens, etc. cannot be viewed in the Web UI)</b><br/><br/>',
+        )
+        if is_redacted
+        else ""
+    )
+    default_command_value = val.strip("'") if (val := find_config_default(key)) else str(final_value).strip("'")
+    machine_config_link = (
+        format_html('<br/><a href="{}">→ Edit <code>{}</code> in Machine.config for this server</a>', machine_admin_url, key)
+        if machine_admin_url
+        else ""
+    )
+    machine_config_tip = (
+        format_html(
+            '<br/><b>Tip:</b> To override <code>{}</code> on this machine, <a href="{}">edit the Machine.config field</a> and add:<br/><code>{}</code>',
+            key,
+            machine_admin_url,
+            f'{{"{key}": "your_value_here"}}',
+        )
+        if machine_admin_url and key not in CONSTANTS_CONFIG
+        else ""
+    )
+
+    section_data = cast(
+        SectionData,
+        {
+            "name": section_header,
+            "description": None,
+            "fields": {
+                "Key": key,
+                "Type": find_config_type(key),
+                "Value": final_value,
+                "Currently read from": config_source,
+            },
+            "help_texts": {
+                "Key": format_html(
+                    """
+                <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#{}">Documentation</a>  &nbsp;
+                <span style="display: {}">
+                    Aliases: {}
+                </span>
+            """,
+                    key.lower(),
+                    "inline" if aliases else "none",
+                    ", ".join(aliases),
+                ),
+                "Type": format_html(
+                    """
+                <a href="{}" target="_blank" rel="noopener noreferrer">
+                    See full definition in <code>{}</code>...
+                </a>
+            """,
+                    definition_url,
+                    definition_label,
+                ),
+                "Value": format_html(
+                    """
+                {}
+                <br/><hr/><br/>
+                <b>Configuration Sources (highest priority first):</b><br/><br/>
+                {}
+                <br/><br/>
+                <p style="display: {}">
+                    <i>To change this value, edit <code>data/ArchiveBox.conf</code> or run:</i>
+                    <br/><br/>
+                    <code>archivebox config --set {}="{}"</code>
+                </p>
+            """,
+                    redacted_message,
+                    sources_html,
+                    "block" if key in merged_config and key not in CONSTANTS_CONFIG else "none",
+                    key,
+                    default_command_value,
+                ),
+                "Currently read from": format_html(
+                    """
+                The value shown in the "Value" field comes from the <b>{}</b> source.
+                <br/><br/>
+                Priority order (highest to lowest):
+                <ol>
+                    <li><b style="color: purple">Machine</b> - Machine-specific overrides
+                        {}
+                    </li>
+                    <li><b style="color: blue">Environment</b> - process defaults from environment variables</li>
+                    <li><b style="color: green">File</b> - data/ArchiveBox.conf</li>
+                    <li><b style="color: gray">Plugin Default</b> - Default value from plugin config.json</li>
+                    <li><b style="color: gray">Default</b> - Default value from code</li>
+                </ol>
+                {}
+            """,
+                    config_source,
+                    machine_config_link,
+                    machine_config_tip,
+                ),
+            },
+        },
+    )
+
+    return ItemContext(
+        slug=key,
+        title=key,
+        data=[section_data],
+    )

archivebox/core/views/lookup.py

@@ -0,0 +1,39 @@
+from pathlib import Path
+
+from archivebox.core.models import Snapshot
+from archivebox.core.routes_util import (
+    get_snapshot_lookup_key,
+)
+
+
+def _files_index_target(snapshot: Snapshot, archivefile: str | None) -> str:
+    target = archivefile or ""
+    if target == "index.html":
+        target = ""
+    fullpath = Path(snapshot.output_dir) / target
+    if fullpath.is_file():
+        target = str(Path(target).parent)
+        if target == ".":
+            target = ""
+    return target
+
+
+def _find_snapshot_by_ref(snapshot_ref: str) -> Snapshot | None:
+    lookup = get_snapshot_lookup_key(snapshot_ref)
+    if not lookup:
+        return None
+
+    snapshots = Snapshot.objects.select_related("crawl", "crawl__created_by")
+
+    if len(lookup) == 12 and "-" not in lookup:
+        return snapshots.filter(id__endswith=lookup).order_by("-created_at", "-downloaded_at").first()
+
+    try:
+        return snapshots.get(pk=lookup)
+    except Snapshot.DoesNotExist:
+        try:
+            return snapshots.get(id__startswith=lookup)
+        except Snapshot.DoesNotExist:
+            return None
+        except Snapshot.MultipleObjectsReturned:
+            return snapshots.filter(id__startswith=lookup).first()

archivebox/core/views/public.py

@@ -0,0 +1,292 @@
+from typing import ClassVar
+
+from django.core.paginator import InvalidPage
+from django.db.models import Case, IntegerField, Value, When
+from django.http import Http404, HttpResponse
+from django.shortcuts import redirect
+from django.views import View
+from django.views.generic.list import ListView
+
+from archivebox.config import VERSION
+from archivebox.config.common import (
+    get_request_config,
+)
+from archivebox.core.models import ArchiveResult, Snapshot, SnapshotTag
+from archivebox.core.permissions import (
+    PERMISSIONS_PRIVATE,
+    PERMISSIONS_PUBLIC,
+    PERMISSIONS_UNLISTED,
+    public_snapshots_queryset,
+)
+from archivebox.core.routes_util import (
+    build_web_url,
+)
+from archivebox.misc.paginators import AcceleratedPaginator
+from archivebox.search.config import (
+    get_search_mode,
+    get_search_mode_backend,
+    get_search_mode_base,
+    get_search_mode_options,
+)
+from archivebox.search.views import get_cached_public_search_state
+
+from .replay_auth import _admin_login_redirect_or_forbidden
+
+
+class HomepageView(View):
+    def get(self, request):
+        request_config = get_request_config(request)
+        if not request_config.BASE_URL:
+            return _admin_login_redirect_or_forbidden(request)
+
+        if request.user.is_authenticated and request_config.CONTROL_PLANE_ENABLED:
+            return redirect("/admin/core/snapshot/")
+
+        if request_config.PUBLIC_INDEX:
+            return redirect("/public")
+
+        return _admin_login_redirect_or_forbidden(request)
+
+
+class PublicIndexView(ListView):
+    template_name = "public_index.html"
+    model = Snapshot
+    ordering: ClassVar[list[str]] = ["-bookmarked_at", "-created_at"]
+    paginator_class = AcceleratedPaginator
+    public_page_scan_chunk_size = 50
+
+    def get_paginate_by(self, queryset):
+        runtime_config = self.__dict__.get("runtime_config")
+        if runtime_config is None:
+            self.runtime_config = runtime_config = get_request_config(self.request, resolve_plugins=False)
+        return runtime_config.SNAPSHOTS_PER_PAGE
+
+    def _base_public_snapshot_fields(self) -> tuple[str, ...]:
+        return (
+            "id",
+            "created_at",
+            "modified_at",
+            "url",
+            "timestamp",
+            "bookmarked_at",
+            "title",
+            "downloaded_at",
+            "status",
+            "output_size",
+            "permissions",
+        )
+
+    def _ordered_public_page_from_order_index(self, *, page_number: int, page_size: int) -> list[Snapshot] | None:
+        target_count = page_number * page_size
+        public_snapshots: list[Snapshot] = []
+        scanned = 0
+        chunk_size = max(self.public_page_scan_chunk_size, page_size)
+        ordered_snapshots = Snapshot.objects.order_by(*self.ordering).only(*self._base_public_snapshot_fields())
+
+        while len(public_snapshots) < target_count:
+            chunk = list(ordered_snapshots[scanned : scanned + chunk_size])
+            if not chunk:
+                break
+            scanned += len(chunk)
+            public_snapshots.extend(snapshot for snapshot in chunk if snapshot.permissions == PERMISSIONS_PUBLIC)
+
+        start = (page_number - 1) * page_size
+        return public_snapshots[start:target_count]
+
+    def paginate_queryset(self, queryset, page_size):
+        if self.request.GET.get("q", default="").strip():
+            return super().paginate_queryset(queryset, page_size)
+
+        public_count = self.get_exact_public_snapshot_count()
+        paginator = self.get_paginator(range(public_count), page_size)
+        page_kwarg = self.kwargs.get(self.page_kwarg)
+        page_query = self.request.GET.get(self.page_kwarg)
+        page_number = page_kwarg or page_query or 1
+
+        try:
+            page = paginator.page(page_number)
+        except InvalidPage as err:
+            raise Http404(f"Invalid page ({page_number}): {err}") from err
+
+        object_list = self._ordered_public_page_from_order_index(page_number=page.number, page_size=page_size)
+        page.object_list = object_list
+        return paginator, page, object_list, page.has_other_pages()
+
+    def get_context_data(self, **kwargs):
+        runtime_config = self.__dict__.get("runtime_config")
+        if runtime_config is None:
+            self.runtime_config = runtime_config = get_request_config(self.request, resolve_plugins=False)
+        search_mode = get_search_mode(self.request.GET.get("search_mode"), config=runtime_config)
+        search_mode_backend = get_search_mode_backend(search_mode, config=runtime_config)
+        query = self.request.GET.get("q", default="").strip()
+        public_search_state = self.__dict__.get("public_search_state")
+        public_search_pending = bool(query and (public_search_state is None or not public_search_state.get("done")))
+        context = {
+            **super().get_context_data(**kwargs),
+            "VERSION": VERSION,
+            "CONFIG": runtime_config,
+            "COMMIT_HASH": runtime_config.COMMIT_HASH,
+            "FOOTER_INFO": runtime_config.FOOTER_INFO,
+            "WEB_BASE_URL": build_web_url(request=self.request, config=runtime_config),
+            "search_mode": search_mode,
+            "search_mode_options": get_search_mode_options(config=runtime_config),
+            "public_search_stream_pending": public_search_pending,
+        }
+        context["show_search_index_hint"] = bool(
+            query
+            and not public_search_pending
+            and get_search_mode_base(search_mode, config=runtime_config) == "deep"
+            and search_mode_backend
+            and context["paginator"].count == 0,
+        )
+        snapshots = list(context.get("object_list") or ())
+        icons_by_snapshot: dict[str, set[str]] = {str(snapshot.id): set() for snapshot in snapshots}
+        tag_names_by_snapshot: dict[str, list[str]] = {str(snapshot.id): [] for snapshot in snapshots}
+        preview_paths_by_snapshot: dict[str, list[tuple[int, int, str]]] = {str(snapshot.id): [] for snapshot in snapshots}
+        favicon_paths_by_snapshot: dict[str, list[str]] = {str(snapshot.id): [] for snapshot in snapshots}
+        progress_by_snapshot: dict[str, dict[str, int]] = {
+            str(snapshot.id): {
+                "total": 0,
+                "succeeded": 0,
+                "failed": 0,
+                "running": 0,
+                "skipped": 0,
+                "noresults": 0,
+            }
+            for snapshot in snapshots
+        }
+        if icons_by_snapshot:
+            for snapshot_id, tag_name in (
+                SnapshotTag.objects.filter(snapshot_id__in=icons_by_snapshot.keys())
+                .order_by("tag__name")
+                .values_list("snapshot_id", "tag__name")
+                .iterator(chunk_size=1000)
+            ):
+                tag_names_by_snapshot[str(snapshot_id)].append(tag_name)
+
+            preview_plugin_order = {
+                "screenshot": 0,
+                "chrome_extension_screenshot": 1,
+            }
+            preview_candidates = {
+                "screenshot": ("screenshot.png",),
+                "chrome_extension_screenshot": ("screenshot-1.png", "screenshot.png"),
+            }
+
+            def result_output_path(result: ArchiveResult, filename: str) -> str | None:
+                output_files = result.output_files or {}
+                file_info = output_files.get(filename)
+                if not isinstance(file_info, dict) or int(file_info.get("size") or 0) <= 0:
+                    return None
+                if file_info.get("root_relative"):
+                    return filename
+                return f"{result.plugin}/{filename}"
+
+            for snapshot_id, plugin, status in (
+                ArchiveResult.objects.filter(
+                    snapshot_id__in=icons_by_snapshot.keys(),
+                )
+                .exclude(plugin="")
+                .values_list("snapshot_id", "plugin", "status")
+                .iterator(chunk_size=1000)
+            ):
+                snapshot_key = str(snapshot_id)
+                progress = progress_by_snapshot[snapshot_key]
+                progress["total"] += 1
+                if status == ArchiveResult.StatusChoices.SUCCEEDED:
+                    icons_by_snapshot[snapshot_key].add(plugin)
+                    progress["succeeded"] += 1
+                elif status == ArchiveResult.StatusChoices.FAILED:
+                    progress["failed"] += 1
+                elif status == ArchiveResult.StatusChoices.STARTED:
+                    progress["running"] += 1
+                elif status == ArchiveResult.StatusChoices.SKIPPED:
+                    progress["skipped"] += 1
+                elif status == ArchiveResult.StatusChoices.NORESULTS:
+                    progress["noresults"] += 1
+
+            for result in (
+                ArchiveResult.objects.filter(
+                    snapshot_id__in=icons_by_snapshot.keys(),
+                    status=ArchiveResult.StatusChoices.SUCCEEDED,
+                    plugin__in=(*preview_candidates, "favicon"),
+                )
+                .only("snapshot_id", "plugin", "output_files")
+                .iterator(chunk_size=1000)
+            ):
+                snapshot_key = str(result.snapshot_id)
+                if result.plugin in preview_candidates:
+                    plugin_rank = preview_plugin_order[result.plugin]
+                    for filename_rank, filename in enumerate(preview_candidates[result.plugin]):
+                        output_path = result_output_path(result, filename)
+                        if output_path:
+                            preview_paths_by_snapshot[snapshot_key].append((plugin_rank, filename_rank, output_path))
+                elif result.plugin == "favicon":
+                    output_path = result_output_path(result, "favicon.ico")
+                    if output_path:
+                        favicon_paths_by_snapshot[snapshot_key].append(output_path)
+
+        for snapshot in snapshots:
+            snapshot._icons_compact = True
+            snapshot._icons_archive_results = icons_by_snapshot.get(str(snapshot.id), set())
+            snapshot._icons_progress_stats = progress_by_snapshot.get(str(snapshot.id), {})
+            snapshot.num_outputs_cached = snapshot._icons_progress_stats.get("succeeded", 0)
+            snapshot._tags_str_cached = ",".join(tag_names_by_snapshot.get(str(snapshot.id), []))
+            snapshot._public_preview_paths = [
+                output_path for _plugin_rank, _filename_rank, output_path in sorted(preview_paths_by_snapshot.get(str(snapshot.id), []))
+            ]
+            snapshot._public_favicon_paths = favicon_paths_by_snapshot.get(str(snapshot.id), [])
+            snapshot._is_archived_cached = bool(snapshot.downloaded_at or snapshot.status == Snapshot.StatusChoices.SEALED)
+        context["object_list"] = snapshots
+        return context
+
+    def get_exact_public_snapshot_count(self) -> int:
+        hidden_count = Snapshot.objects.filter(permissions=PERMISSIONS_PRIVATE).count()
+        hidden_count += Snapshot.objects.filter(permissions=PERMISSIONS_UNLISTED).count()
+        return Snapshot.objects.count() - hidden_count
+
+    def get_queryset(self, **kwargs):
+        qs = public_snapshots_queryset(super().get_queryset(**kwargs)).only(*self._base_public_snapshot_fields())
+        query = self.request.GET.get("q", default="").strip()
+
+        if not query:
+            return qs
+
+        cached_state = get_cached_public_search_state(self.request)
+        self.public_search_state = cached_state
+        if cached_state is not None:
+            cached_ids = cached_state.get("ids") or []
+            if not cached_ids:
+                return qs.none()
+            search_rank = Case(
+                *(When(pk=snapshot_id, then=Value(index)) for index, snapshot_id in enumerate(cached_ids)),
+                output_field=IntegerField(),
+            )
+            return qs.filter(pk__in=cached_ids).annotate(search_rank=search_rank).order_by("search_rank", *self.ordering)
+
+        return qs.none()
+
+    def get(self, *args, **kwargs):
+        if self.request.user.is_authenticated:
+            return redirect("/admin/core/snapshot/")
+        if get_request_config(self.request).PUBLIC_INDEX:
+            response = super().get(*args, **kwargs)
+            return response
+        else:
+            return _admin_login_redirect_or_forbidden(self.request)
+
+
+class HealthCheckView(View):
+    """
+    A Django view that renders plain text "OK" for service discovery tools
+    """
+
+    def get(self, request):
+        """
+        Handle a GET request
+        """
+        response = HttpResponse("OK", content_type="text/plain", status=200)
+        response["Access-Control-Allow-Origin"] = "*"
+        response["Access-Control-Expose-Headers"] = "X-ArchiveBox-Health"
+        response["X-ArchiveBox-Health"] = "OK"
+        return response

archivebox/core/views/replay.py

@@ -0,0 +1,832 @@
+import posixpath
+from pathlib import Path
+from urllib.parse import quote, urlparse
+
+from abx_plugins.plugins.archivewebpage import replay_preview as archivewebpage_replay
+from django import template
+from django.db.models import Q
+from django.http import Http404, HttpRequest, HttpResponse
+from django.shortcuts import redirect, render
+from django.utils.html import format_html
+from django.utils.safestring import mark_safe
+from django.views import View
+
+from archivebox.config.common import (
+    get_request_config,
+)
+from archivebox.core.models import ArchiveResult, Snapshot
+from archivebox.core.permissions import (
+    can_view_snapshot,
+    direct_snapshots_queryset,
+)
+from archivebox.core.routes_util import (
+    build_snapshot_url,
+    build_web_url,
+    get_snapshot_host,
+    host_matches,
+)
+from archivebox.misc.serve_static import serve_static_with_byterange_support
+from archivebox.misc.util import (
+    base_url,
+    filter_queryset_by_uuid_substring,
+    without_fragment,
+)
+from archivebox.plugins.discovery import get_plugin_name, get_plugin_template
+from archivebox.progressmonitor.views import live_progress_view
+
+from .lookup import _files_index_target, _find_snapshot_by_ref
+from .replay_auth import _has_replay_cookie, _private_snapshot_auth_redirect, _replay_auth_response
+
+
+class SnapshotView(View):
+    # render static html index from filesystem archive/<timestamp>/index.html
+
+    @staticmethod
+    def find_snapshots_for_url(path: str, *, allow_fallback: bool = True):
+        """Return a queryset of snapshots matching a URL-ish path. URL only — never tries ID matching.
+
+        Use ``find_snapshots_for_id`` separately if you also want to match by snapshot UUID.
+        """
+
+        def _fragmentless_url_query(url: str) -> Q:
+            from archivebox.misc.db import is_postgres
+
+            canonical = without_fragment(url)
+            if not is_postgres():
+                # Use a range comparison (url >= 'canonical#' AND url < 'canonical#\U0010ffff')
+                # instead of LIKE/__startswith — SQLite's case-insensitive LIKE bypasses the
+                # url index and forces a full-table scan over ~1M rows (~250ms). The range
+                # form lets SQLite use a MULTI-INDEX OR and stays under 1ms.
+                return Q(url=canonical) | (Q(url__gte=f"{canonical}#") & Q(url__lt=f"{canonical}#\U0010ffff"))
+            # On postgres the range trick is unsafe: linguistic (ICU/libc) collations
+            # don't compare '#'-suffixed strings bytewise, so the range can miss rows.
+            # startswith compiles to LIKE 'prefix%' with wildcards escaped, which is
+            # correct under any collation and uses the url pattern-ops index.
+            return Q(url=canonical) | Q(url__startswith=f"{canonical}#")
+
+        normalized = without_fragment(path)
+        if path.startswith(("http://", "https://")):
+            # exact url match (indexed) — fastest path
+            qs = Snapshot.objects.filter(_fragmentless_url_query(path))
+            if not allow_fallback or qs.exists():
+                return qs
+            normalized = normalized.split("://", 1)[1]
+
+        # try exact match on full url (without scheme)
+        qs = Snapshot.objects.filter(
+            _fragmentless_url_query("http://" + normalized) | _fragmentless_url_query("https://" + normalized),
+        )
+        if qs.exists():
+            return qs
+
+        # fall back to match on exact base_url
+        base = base_url(normalized)
+        qs = Snapshot.objects.filter(
+            _fragmentless_url_query("http://" + base) | _fragmentless_url_query("https://" + base),
+        )
+        if qs.exists():
+            return qs
+
+        # fall back to matching base_url as prefix
+        return Snapshot.objects.filter(Q(url__startswith="http://" + base) | Q(url__startswith="https://" + base))
+
+    @staticmethod
+    def find_snapshots_for_id(slug: str):
+        """Return a queryset of snapshots matching a (possibly truncated) UUID via prefix or suffix.
+
+        Strips non-hex characters from ``slug`` (so input with or without hyphens both work).
+        Requires at least 8 hex chars — shorter inputs return an empty queryset to avoid
+        scanning the entire snapshots table on too-broad matches.
+        """
+        return filter_queryset_by_uuid_substring(Snapshot.objects.all(), slug)
+
+    @staticmethod
+    def render_live_index(request, snapshot):
+        return render(
+            template_name="core/snapshot.html",
+            request=request,
+            context=snapshot.get_html_details_context(request=request),
+        )
+
+    def get(self, request, path):
+        snapshot = None
+
+        try:
+            slug, archivefile = path.split("/", 1)
+        except (IndexError, ValueError):
+            slug, archivefile = path.split("/", 1)[0], "index.html"
+
+        # slug is a timestamp
+        if slug.replace(".", "").isdigit():
+            # missing trailing slash -> redirect to index
+            if "/" not in path:
+                return redirect(f"{path}/index.html")
+
+            try:
+                try:
+                    snapshot = Snapshot.objects.get(Q(timestamp=slug) | Q(id__startswith=slug))
+                    if not can_view_snapshot(request, snapshot):
+                        return _private_snapshot_auth_redirect(request, snapshot, archivefile or "index.html")
+                    canonical_base = snapshot.url_path
+                    if canonical_base != snapshot.legacy_archive_path:
+                        target_path = f"/{canonical_base}/{archivefile or 'index.html'}"
+                        query = request.META.get("QUERY_STRING")
+                        if query:
+                            target_path = f"{target_path}?{query}"
+                        return redirect(target_path)
+
+                    if request.GET.get("files"):
+                        target_path = _files_index_target(snapshot, archivefile)
+                        response = serve_static_with_byterange_support(
+                            request,
+                            target_path,
+                            document_root=snapshot.output_dir,
+                            show_indexes=True,
+                            is_archive_replay=True,
+                        )
+                    elif archivefile == "index.html":
+                        # if they requested snapshot index, serve live rendered template instead of static html
+                        response = self.render_live_index(request, snapshot)
+                    else:
+                        target = build_snapshot_url(str(snapshot.id), archivefile, request=request)
+                        query = request.META.get("QUERY_STRING")
+                        if query:
+                            target = f"{target}?{query}"
+                        return redirect(target)
+                    response["Link"] = f'<{snapshot.url}>; rel="canonical"'
+                    return response
+                except Snapshot.DoesNotExist:
+                    if Snapshot.objects.filter(timestamp__startswith=slug).exists():
+                        raise Snapshot.MultipleObjectsReturned
+                    else:
+                        raise
+            except Snapshot.DoesNotExist:
+                # Snapshot does not exist
+                return HttpResponse(
+                    format_html(
+                        (
+                            "<center><br/><br/><br/>"
+                            "No Snapshot directories match the given timestamp/ID: <code>{}</code><br/><br/>"
+                            'You can <a href="/add/" target="_top">add a new Snapshot</a>, or return to the <a href="/" target="_top">Main Index</a>'
+                            "</center>"
+                        ),
+                        slug,
+                        path,
+                    ),
+                    content_type="text/html",
+                    status=404,
+                )
+            except Snapshot.MultipleObjectsReturned:
+                snapshot_hrefs = mark_safe("<br/>").join(
+                    format_html(
+                        '{} <a href="/{}/index.html"><b><code>{}</code></b></a> {} <b>{}</b>',
+                        snap.bookmarked_at.strftime("%Y-%m-%d %H:%M:%S"),
+                        snap.archive_path,
+                        snap.timestamp,
+                        snap.url,
+                        snap.title_stripped[:64] or "",
+                    )
+                    for snap in direct_snapshots_queryset(request, Snapshot.objects.filter(timestamp__startswith=slug))
+                    .only("url", "timestamp", "title", "bookmarked_at")
+                    .order_by("-bookmarked_at")
+                )
+                return HttpResponse(
+                    format_html(
+                        ("Multiple Snapshots match the given timestamp/ID <code>{}</code><br/><pre>"),
+                        slug,
+                    )
+                    + snapshot_hrefs
+                    + mark_safe('</pre><br/>Choose a Snapshot to proceed or go back to the <a href="/" target="_top">Main Index</a>'),
+                    content_type="text/html",
+                    status=404,
+                )
+            except Http404:
+                assert snapshot  # (Snapshot.DoesNotExist is already handled above)
+
+                # Snapshot dir exists but file within does not e.g. 124235.324234/screenshot.png
+                return HttpResponse(
+                    format_html(
+                        """
+                        <html><head>
+                        <title>Snapshot Not Found</title>
+                        </head><body>
+                        <center><br/><br/><br/>
+                        Snapshot <a href="/{}/index.html" target="_top"><b><code>[{}]</code></b></a>: <a href="{}" target="_blank" rel="noreferrer">{}</a><br/>
+                        was queued on {}, but no files have been saved yet in:<br/><b><a href="/{}/" target="_top"><code>{}</code></a><code>/{}</code></b><br/><br/>
+                        It's possible {} during the last capture on {},<br/>or that the archiving process has not completed yet.<br/>
+                        <pre><code># run this cmd to finish/retry archiving this Snapshot</code><br/>
+                        <code style="user-select: all; color: #333">archivebox update -t timestamp {}</code></pre><br/><br/>
+                        <div class="text-align: left; width: 100%; max-width: 400px">
+                        <i><b>Next steps:</i></b><br/>
+                        - list all the <a href="/{}/" target="_top">Snapshot files <code>.*</code></a><br/>
+                        - view the <a href="/{}/index.html" target="_top">Snapshot <code>./index.html</code></a><br/>
+                        - go to the <a href="/admin/core/snapshot/{}/change/" target="_top">Snapshot admin</a> to edit<br/>
+                        - go to the <a href="/admin/core/snapshot/?id__exact={}" target="_top">Snapshot actions</a> to re-archive<br/>
+                        - or return to <a href="/" target="_top">the main index...</a></div>
+                        </center>
+                        </body></html>
+                        """,
+                        snapshot.archive_path,
+                        snapshot.timestamp,
+                        snapshot.url,
+                        snapshot.url,
+                        str(snapshot.bookmarked_at).split(".")[0],
+                        snapshot.archive_path,
+                        snapshot.timestamp,
+                        archivefile if str(archivefile) != "None" else "",
+                        f"the {archivefile} resource could not be fetched"
+                        if str(archivefile) != "None"
+                        else "the original site was not available",
+                        str(snapshot.bookmarked_at).split(".")[0],
+                        snapshot.timestamp,
+                        snapshot.archive_path,
+                        snapshot.archive_path,
+                        snapshot.pk,
+                        snapshot.id,
+                    ),
+                    content_type="text/html",
+                    status=404,
+                )
+
+        # slug is either a URL or a (possibly truncated) snapshot UUID
+        def _resolve_snapshots_for_slug(slug: str):
+            # full URLs go straight to the url-only path (fast, indexed)
+            if "://" in slug:
+                return SnapshotView.find_snapshots_for_url(slug)
+            # short uuid-shaped slugs (>=8 hex chars after stripping non-hex) try id matching first
+            id_qs = SnapshotView.find_snapshots_for_id(slug)
+            if id_qs.exists():
+                return id_qs
+            return SnapshotView.find_snapshots_for_url(slug)
+
+        snapshots = direct_snapshots_queryset(request, _resolve_snapshots_for_slug(path))
+        try:
+            if "://" in path:
+                snapshot = snapshots.order_by("-bookmarked_at").first()
+                if snapshot is None:
+                    raise Snapshot.DoesNotExist
+            else:
+                snapshot = snapshots.get()
+        except Snapshot.DoesNotExist:
+            return HttpResponse(
+                format_html(
+                    (
+                        "<center><br/><br/><br/>"
+                        "No Snapshots match the given url: <code>{}</code><br/><br/><br/>"
+                        'Return to the <a href="/" target="_top">Main Index</a>, or:<br/><br/>'
+                        '+ <i><a href="/add/?url={}" target="_top">Add a new Snapshot for <code>{}</code></a><br/><br/></i>'
+                        "</center>"
+                    ),
+                    base_url(path),
+                    path if "://" in path else f"https://{path}",
+                    path,
+                ),
+                content_type="text/html",
+                status=404,
+            )
+        except Snapshot.MultipleObjectsReturned:
+            snapshot_hrefs = mark_safe("<br/>").join(
+                format_html(
+                    '{} <code style="font-size: 0.8em">{}</code> <a href="/{}/index.html"><b><code>{}</code></b></a> {} <b>{}</b>',
+                    snap.bookmarked_at.strftime("%Y-%m-%d %H:%M:%S"),
+                    str(snap.id)[:8],
+                    snap.archive_path,
+                    snap.timestamp,
+                    snap.url,
+                    snap.title_stripped[:64] or "",
+                )
+                for snap in snapshots.only("url", "timestamp", "title", "bookmarked_at").order_by("-bookmarked_at")
+            )
+            return HttpResponse(
+                format_html(
+                    ("Multiple Snapshots match the given URL <code>{}</code><br/><pre>"),
+                    base_url(path),
+                )
+                + snapshot_hrefs
+                + mark_safe('</pre><br/>Choose a Snapshot to proceed or go back to the <a href="/" target="_top">Main Index</a>'),
+                content_type="text/html",
+                status=404,
+            )
+
+        target_path = build_snapshot_url(str(snapshot.id), "index.html", request=request)
+        query = request.META.get("QUERY_STRING")
+        if query:
+            target_path = f"{target_path}?{query}"
+        return redirect(target_path)
+
+
+class SnapshotPathView(View):
+    """Serve snapshots by the new URL scheme: /<username>/<YYYYMMDD>/<domain>/<uuid>/..."""
+
+    def get(
+        self,
+        request,
+        username: str,
+        date: str | None = None,
+        domain: str | None = None,
+        snapshot_id: str | None = None,
+        path: str = "",
+        url: str | None = None,
+    ):
+        if username == "system":
+            return redirect(request.path.replace("/system/", "/web/", 1))
+
+        if date and domain and domain == date:
+            raise Http404
+
+        requested_url = url
+        if not requested_url and domain and domain.startswith(("http://", "https://")):
+            requested_url = domain
+
+        snapshot = None
+        snapshots_qs = direct_snapshots_queryset(request, Snapshot.objects.select_related("crawl", "crawl__created_by"))
+        if snapshot_id:
+            snapshot = _find_snapshot_by_ref(snapshot_id)
+            if snapshot and not can_view_snapshot(request, snapshot):
+                return _private_snapshot_auth_redirect(request, snapshot, path or "index.html")
+        else:
+            # fuzzy lookup by date + domain/url (most recent)
+            username_lookup = "system" if username == "web" else username
+            if requested_url:
+                qs = direct_snapshots_queryset(
+                    request,
+                    SnapshotView.find_snapshots_for_url(requested_url)
+                    .select_related("crawl", "crawl__created_by")
+                    .filter(
+                        crawl__created_by__username=username_lookup,
+                    ),
+                )
+            else:
+                qs = snapshots_qs.filter(crawl__created_by__username=username_lookup)
+
+            if date:
+                try:
+                    if len(date) == 4:
+                        qs = qs.filter(bookmarked_at__year=int(date))
+                    elif len(date) == 6:
+                        qs = qs.filter(bookmarked_at__year=int(date[:4]), bookmarked_at__month=int(date[4:6]))
+                    elif len(date) == 8:
+                        qs = qs.filter(
+                            bookmarked_at__year=int(date[:4]),
+                            bookmarked_at__month=int(date[4:6]),
+                            bookmarked_at__day=int(date[6:8]),
+                        )
+                except ValueError:
+                    pass
+
+            if requested_url:
+                snapshot = qs.order_by("-bookmarked_at", "-created_at", "-timestamp").first()
+            else:
+                requested_domain = domain or ""
+                if requested_domain.startswith(("http://", "https://")):
+                    requested_domain = Snapshot.extract_domain_from_url(requested_domain)
+                else:
+                    requested_domain = Snapshot.extract_domain_from_url(f"https://{requested_domain}")
+
+                # Prefer exact domain matches
+                matches = [
+                    s for s in qs.order_by("-bookmarked_at", "-created_at") if Snapshot.extract_domain_from_url(s.url) == requested_domain
+                ]
+                snapshot = matches[0] if matches else qs.order_by("-bookmarked_at", "-created_at", "-timestamp").first()
+
+        if not snapshot:
+            return HttpResponse(
+                format_html(
+                    (
+                        "<center><br/><br/><br/>"
+                        "No Snapshots match the given id or url: <code>{}</code><br/><br/><br/>"
+                        'Return to the <a href="/" target="_top">Main Index</a>'
+                        "</center>"
+                    ),
+                    snapshot_id or requested_url or domain,
+                ),
+                content_type="text/html",
+                status=404,
+            )
+
+        canonical_base = snapshot.url_path
+        if date:
+            requested_base = f"{username}/{date}/{domain or url or ''}"
+        else:
+            requested_base = f"{username}/{domain or url or ''}"
+        if snapshot_id:
+            requested_base = f"{requested_base}/{snapshot_id}"
+        if canonical_base != requested_base:
+            target = f"/{canonical_base}/{path or 'index.html'}"
+            query = request.META.get("QUERY_STRING")
+            if query:
+                target = f"{target}?{query}"
+            return redirect(target)
+
+        archivefile = path or "index.html"
+        if archivefile != "index.html" and not request.GET.get("files"):
+            target = build_snapshot_url(str(snapshot.id), archivefile, request=request)
+            query = request.META.get("QUERY_STRING")
+            if query:
+                target = f"{target}?{query}"
+            return redirect(target)
+
+        if request.GET.get("files"):
+            target_path = _files_index_target(snapshot, archivefile)
+            return serve_static_with_byterange_support(
+                request,
+                target_path,
+                document_root=snapshot.output_dir,
+                show_indexes=True,
+                is_archive_replay=True,
+            )
+
+        if archivefile == "index.html":
+            return SnapshotView.render_live_index(request, snapshot)
+
+        return serve_static_with_byterange_support(
+            request,
+            archivefile,
+            document_root=snapshot.output_dir,
+            show_indexes=True,
+            is_archive_replay=True,
+        )
+
+
+def _safe_archive_relpath(path: str) -> str | None:
+    if not path:
+        return ""
+    cleaned = posixpath.normpath(path)
+    cleaned = cleaned.lstrip("/")
+    if cleaned.startswith("..") or "/../" in f"/{cleaned}/":
+        return None
+    return cleaned
+
+
+def _resolve_archiveresult_relpath(snapshot: Snapshot, rel_path: str) -> tuple[str, ArchiveResult | None]:
+    """Resolve plugin-relative output paths through ArchiveResult.output_files."""
+    parts = Path(rel_path).parts
+    if len(parts) < 2:
+        return rel_path, None
+
+    plugin = parts[0]
+    plugin_relpath = posixpath.join(*parts[1:])
+    results = list(
+        ArchiveResult.objects.filter(
+            snapshot=snapshot,
+            plugin=plugin,
+            status=ArchiveResult.StatusChoices.SUCCEEDED,
+        ).only("plugin", "output_files"),
+    )
+    if not results:
+        return rel_path, None
+
+    for result in results:
+        output_files = result.output_files or {}
+        for candidate in (plugin_relpath, rel_path):
+            file_info = output_files.get(candidate)
+            if not isinstance(file_info, dict):
+                continue
+            if file_info.get("root_relative"):
+                return candidate, result
+            return rel_path, result
+
+    return rel_path, results[0]
+
+
+def _plugin_full_preview_response(
+    request: HttpRequest,
+    snapshot: Snapshot,
+    rel_path: str,
+    result: ArchiveResult | None,
+) -> HttpResponse | None:
+    """Render an explicit plugin full template as a trusted preview wrapper."""
+    if not request.GET.get("preview"):
+        return None
+
+    path_parts = Path(rel_path).parts
+    plugin = get_plugin_name(result.plugin) if result else (path_parts[0] if len(path_parts) > 1 else "")
+    if not plugin:
+        return None
+
+    # ReplayWeb.page needs plugin-owned WACZ inspection and service-worker
+    # context, so it remains the one narrow preview exception below.
+    if plugin == "archivewebpage" and archivewebpage_replay.is_replay_target(rel_path):
+        return None
+
+    template_str = get_plugin_template(plugin, "full", fallback=False)
+    if not template_str:
+        return None
+
+    raw_query = request.GET.copy()
+    raw_query.pop("preview", None)
+    output_url = request.path
+    if raw_query:
+        output_url = f"{output_url}?{raw_query.urlencode()}"
+
+    rendered = (
+        template.Engine(debug=False)
+        .from_string(template_str)
+        .render(
+            template.Context(
+                {
+                    "result": result,
+                    "snapshot": snapshot,
+                    "output_path": output_url,
+                    "output_path_raw": rel_path,
+                    "plugin": plugin,
+                    "preview_base": f"{request.path.rsplit('/', 1)[0]}/",
+                },
+            ),
+        )
+    )
+    response = HttpResponse(rendered, content_type="text/html; charset=utf-8")
+    response.headers["Content-Disposition"] = f'inline; filename="{Path(rel_path).stem}.html"'
+    response.headers["X-Content-Type-Options"] = "nosniff"
+    response.headers["X-ArchiveBox-Security-Mode"] = request.archivebox_config.SERVER_SECURITY_MODE
+    response.headers["Referrer-Policy"] = "no-referrer"
+    response.headers["Content-Security-Policy"] = (
+        "default-src 'self' data: blob:; "
+        "script-src 'self' 'unsafe-inline' 'unsafe-eval' data: blob:; "
+        "style-src 'unsafe-inline' data: blob: 'self'; "
+        "connect-src 'self' data: blob:; "
+        "img-src 'self' data: blob:; "
+        "media-src 'self' data: blob:; "
+        "font-src 'self' data: blob:; "
+        "frame-src 'self' data: blob:; "
+        "worker-src 'self' blob:; "
+        "object-src 'none'; "
+        "base-uri 'none'; "
+        "form-action 'none'; "
+        "frame-ancestors 'self';"
+    )
+    return response
+
+
+def _visible_response_snapshots_for_domain(request: HttpRequest, domain: str) -> list[Snapshot]:
+    if not domain:
+        return []
+
+    requested_domain = domain.split(":", 1)[0].lower()
+    roots = (f"http://{requested_domain}", f"https://{requested_domain}")
+    domain_query = Q(url__in=roots)
+
+    from archivebox.misc.db import is_postgres
+
+    postgres = is_postgres()
+    for root in roots:
+        for separator in ("/", "?", "#"):
+            prefix = f"{root}{separator}"
+            if postgres:
+                domain_query |= Q(url__startswith=prefix)
+            else:
+                domain_query |= Q(url__gte=prefix, url__lt=f"{prefix}\U0010ffff")
+
+    candidates = (
+        Snapshot.objects.filter(domain_query)
+        .select_related("crawl", "crawl__created_by")
+        .order_by("-bookmarked_at", "-created_at", "-timestamp")
+    )
+    return [snapshot for snapshot in candidates if can_view_snapshot(request, snapshot) or _has_replay_cookie(request, snapshot)]
+
+
+def _latest_response_match(
+    snapshots: list[Snapshot],
+    domain: str,
+    rel_path: str,
+) -> tuple[Snapshot, Path, Path] | None:
+    if not domain or not rel_path:
+        return None
+    requested_domain = domain.split(":", 1)[0].lower()
+    rel_to_root = Path(rel_path)
+    for snapshot in snapshots:
+        responses_root = Path(snapshot.output_dir) / "responses" / requested_domain
+        if (responses_root / rel_to_root).exists():
+            return snapshot, responses_root, rel_to_root
+    return None
+
+
+def _latest_responses_root(snapshots: list[Snapshot], domain: str) -> tuple[Snapshot, Path] | None:
+    if not domain:
+        return None
+    requested_domain = domain.split(":", 1)[0].lower()
+    for snapshot in snapshots:
+        responses_root = Path(snapshot.output_dir) / "responses" / requested_domain
+        if responses_root.is_dir():
+            return snapshot, responses_root
+    return None
+
+
+def _original_request_url(domain: str, path: str = "", query_string: str = "") -> str:
+    normalized_domain = (domain or "").split(":", 1)[0].lower()
+    normalized_path = (path or "").lstrip("/")
+    if normalized_path in ("", "index.html"):
+        normalized_path = ""
+    target = f"https://{normalized_domain}"
+    if normalized_path:
+        target = f"{target}/{normalized_path}"
+    if query_string:
+        target = f"{target}?{query_string}"
+    return target
+
+
+def _serve_responses_path(request, responses_root: Path, rel_path: str, show_indexes: bool):
+    candidates: list[str] = []
+    rel_path = rel_path or ""
+    if rel_path.endswith("/"):
+        rel_path = f"{rel_path}index.html"
+    if "." not in Path(rel_path).name:
+        candidates.append(f"{rel_path.rstrip('/')}/index.html")
+    candidates.append(rel_path)
+
+    for candidate in candidates:
+        try:
+            return serve_static_with_byterange_support(
+                request,
+                candidate,
+                document_root=str(responses_root),
+                show_indexes=show_indexes,
+                is_archive_replay=True,
+            )
+        except Http404:
+            pass
+
+    if rel_path.endswith("index.html"):
+        rel_dir = rel_path[: -len("index.html")]
+        try:
+            return serve_static_with_byterange_support(
+                request,
+                rel_dir,
+                document_root=str(responses_root),
+                show_indexes=True,
+                is_archive_replay=True,
+            )
+        except Http404:
+            return None
+    return None
+
+
+def _serve_snapshot_replay(request: HttpRequest, snapshot: Snapshot, path: str = ""):
+    rel_path = path or ""
+    request_config = get_request_config(
+        request,
+        resolve_plugins=rel_path.startswith("replay/") or rel_path == "replay",
+    )
+    request.archivebox_config = request_config
+    request.archivebox_snapshot_url = snapshot.url
+    snapshot._runtime_config = request_config
+
+    if rel_path.startswith("replay/") or rel_path == "replay":
+        response = archivewebpage_replay.serve_replay_asset_response(rel_path, request_config, HttpResponse)
+        if response is not None:
+            return response
+
+    if rel_path == "progress.json":
+        # Host routing forwards every snap-* path to SnapshotHostView, so we forward
+        # /progress.json on through to the same view used everywhere else. The caller
+        # passes snapshot_id explicitly in the query string — we don't read it from the
+        # subdomain (this keeps the endpoint identical across all security modes).
+        return live_progress_view(request)
+
+    is_directory_request = bool(path) and path.endswith("/")
+    show_indexes = bool(request.GET.get("files")) or (request_config.USES_SUBDOMAIN_ROUTING and is_directory_request)
+    if not show_indexes and (not rel_path or rel_path == "index.html"):
+        return SnapshotView.render_live_index(request, snapshot)
+
+    if not rel_path or rel_path.endswith("/"):
+        if show_indexes:
+            rel_path = rel_path.rstrip("/")
+        else:
+            rel_path = f"{rel_path}index.html"
+    rel_path = _safe_archive_relpath(rel_path)
+    if rel_path is None:
+        raise Http404
+
+    rel_path, archive_result = _resolve_archiveresult_relpath(snapshot, rel_path)
+
+    plugin_preview = _plugin_full_preview_response(request, snapshot, rel_path, archive_result)
+    if plugin_preview is not None:
+        return plugin_preview
+
+    try:
+        return serve_static_with_byterange_support(
+            request,
+            rel_path,
+            document_root=snapshot.output_dir,
+            show_indexes=show_indexes,
+            is_archive_replay=True,
+        )
+    except Http404:
+        pass
+
+    host = urlparse(snapshot.url).hostname or snapshot.domain
+    responses_root = Path(snapshot.output_dir) / "responses" / host
+    if responses_root.exists():
+        response = _serve_responses_path(request, responses_root, rel_path, show_indexes)
+        if response is not None:
+            return response
+
+    raise Http404
+
+
+def _serve_original_domain_replay(request: HttpRequest, domain: str, path: str = ""):
+    request_config = get_request_config(request, resolve_plugins=False)
+    request.archivebox_config = request_config
+    requested_root_index = path in ("", "index.html") or path.endswith("/")
+    rel_path = path or ""
+    if not rel_path or rel_path.endswith("/"):
+        rel_path = f"{rel_path}index.html"
+    rel_path = _safe_archive_relpath(rel_path)
+    if rel_path is None:
+        raise Http404
+
+    domain = domain.split(":", 1)[0].lower()
+    snapshots = _visible_response_snapshots_for_domain(request, domain)
+    match = _latest_response_match(snapshots, domain, rel_path)
+    if not match and "." not in Path(rel_path).name:
+        index_path = f"{rel_path.rstrip('/')}/index.html"
+        match = _latest_response_match(snapshots, domain, index_path)
+    if not match and "." not in Path(rel_path).name:
+        html_path = f"{rel_path}.html"
+        match = _latest_response_match(snapshots, domain, html_path)
+
+    root_match = (match[0], match[1]) if match else _latest_responses_root(snapshots, domain)
+    responses_root = root_match[1] if root_match else None
+    if request_config.USES_SUBDOMAIN_ROUTING:
+        snapshot = root_match[0] if root_match else (snapshots[0] if requested_root_index and snapshots else None)
+        if snapshot:
+            snapshot_path = f"responses/{domain}/{match[2]}" if match else path
+            target = build_snapshot_url(str(snapshot.id), snapshot_path, request=request, config=request_config)
+            if request.META.get("QUERY_STRING"):
+                target = f"{target}?{request.META['QUERY_STRING']}"
+            return redirect(target)
+
+    show_indexes = bool(request.GET.get("files"))
+    if match:
+        _snapshot, responses_root, rel_to_root = match
+        response = _serve_responses_path(request, responses_root, str(rel_to_root), show_indexes)
+        if response is not None:
+            return response
+
+    if responses_root:
+        response = _serve_responses_path(request, responses_root, rel_path, show_indexes)
+        if response is not None:
+            return response
+
+    if requested_root_index and not show_indexes:
+        if snapshots:
+            return SnapshotView.render_live_index(request, snapshots[0])
+
+    if request_config.PUBLIC_ADD_VIEW or request.user.is_authenticated:
+        target_url = _original_request_url(domain, path, request.META.get("QUERY_STRING", ""))
+        return redirect(build_web_url(f"/web/{quote(target_url, safe=':/')}"))
+
+    raise Http404
+
+
+class SnapshotHostView(View):
+    """Serve snapshot directory contents on <snapshot-subdomain>.<listen_host>/<path>."""
+
+    def get(self, request, snapshot_id: str, path: str = ""):
+        request_config = get_request_config(request)
+        snapshot = _find_snapshot_by_ref(snapshot_id)
+
+        if not snapshot:
+            raise Http404
+        if path == "_auth":
+            return _replay_auth_response(request, snapshot)
+        if not can_view_snapshot(request, snapshot) and not _has_replay_cookie(request, snapshot):
+            return _private_snapshot_auth_redirect(request, snapshot, path)
+
+        canonical_host = get_snapshot_host(str(snapshot.id), config=request_config)
+        if not host_matches(request.get_host(), canonical_host):
+            target = build_snapshot_url(str(snapshot.id), path, request=request, config=request_config)
+            if request.META.get("QUERY_STRING"):
+                target = f"{target}?{request.META['QUERY_STRING']}"
+            return redirect(target)
+
+        return _serve_snapshot_replay(request, snapshot, path)
+
+
+class SnapshotReplayView(View):
+    """Serve snapshot directory contents on a one-domain replay path."""
+
+    def get(self, request, snapshot_id: str, path: str = ""):
+        snapshot = _find_snapshot_by_ref(snapshot_id)
+        if not snapshot:
+            raise Http404
+        if path == "_auth":
+            return _replay_auth_response(request, snapshot)
+        if not can_view_snapshot(request, snapshot) and not _has_replay_cookie(request, snapshot):
+            return _private_snapshot_auth_redirect(request, snapshot, path)
+
+        return _serve_snapshot_replay(request, snapshot, path)
+
+
+class OriginalDomainHostView(View):
+    """Serve responses from the most recent snapshot when using <domain>.<listen_host>/<path>."""
+
+    def get(self, request, domain: str, path: str = ""):
+        return _serve_original_domain_replay(request, domain, path)
+
+
+class OriginalDomainReplayView(View):
+    """Serve original-domain replay content on a one-domain replay path."""
+
+    def get(self, request, domain: str, path: str = ""):
+        return _serve_original_domain_replay(request, domain, path)

archivebox/core/views/replay_auth.py

@@ -0,0 +1,148 @@
+from urllib.parse import quote, urlparse
+
+from django.conf import settings
+from django.contrib.auth import HASH_SESSION_KEY, SESSION_KEY, get_user_model
+from django.contrib.sessions.models import Session
+from django.core import signing
+from django.http import Http404, HttpRequest, HttpResponseForbidden
+from django.shortcuts import redirect
+from django.views import View
+
+from archivebox.config import CONSTANTS
+from archivebox.config.common import (
+    get_request_config,
+)
+from archivebox.core.models import Snapshot
+from archivebox.core.permissions import (
+    is_admin_user,
+)
+from archivebox.core.routes_util import (
+    build_admin_url,
+    build_snapshot_url,
+)
+
+from .lookup import _find_snapshot_by_ref
+
+REPLAY_AUTH_SALT = "archivebox.private-snapshot-replay"
+REPLAY_COOKIE_PREFIX = f"archivebox_replay_{CONSTANTS.COLLECTION_ID}_"
+REPLAY_GRANT_MAX_AGE = 60
+
+
+def _admin_login_redirect_or_forbidden(request: HttpRequest):
+    if get_request_config(request).CONTROL_PLANE_ENABLED:
+        return redirect(f"/admin/login/?next={request.path}")
+    return HttpResponseForbidden("ArchiveBox is running with the control plane disabled in this security mode.")
+
+
+def _replay_cookie_name(snapshot: Snapshot) -> str:
+    return f"{REPLAY_COOKIE_PREFIX}{str(snapshot.id).replace('-', '')[-12:]}"
+
+
+def _clean_replay_next(path: str | None) -> str:
+    """Only allow same-snap relative replay paths; grants must never redirect off-host."""
+    path = f"/{(path or 'index.html').lstrip('/')}"
+    parsed = urlparse(path)
+    if parsed.scheme or parsed.netloc or path.startswith("//"):
+        return "/index.html"
+    return path
+
+
+def _replay_payload_is_valid(payload: dict, snapshot: Snapshot) -> bool:
+    """A replay cookie is not its own auth source; it must point at a live admin session.
+
+    Replayed pages can execute hostile JS, so admin cookies stay host-only on admin.*.
+    The snap host gets only this host-only HttpOnly cookie, and every request checks
+    that the original Django session still exists and still belongs to an active staff user.
+    Logout, session expiry, user deletion/deactivation, or password auth-hash rotation all
+    make the replay cookie inert without needing admin.* to delete a cookie on snap-*.
+    """
+    if payload.get("snapshot_id") != str(snapshot.id):
+        return False
+    try:
+        session = Session.objects.get(session_key=str(payload.get("session_key") or ""))
+        session_data = session.get_decoded()
+        user_id = str(session_data.get(SESSION_KEY) or "")
+        auth_hash = str(session_data.get(HASH_SESSION_KEY) or "")
+        user = get_user_model().objects.get(pk=user_id)
+    except (Session.DoesNotExist, get_user_model().DoesNotExist, KeyError, TypeError, ValueError):
+        return False
+    return (
+        str(payload.get("user_id")) == user_id
+        and str(payload.get("auth_hash") or "") == auth_hash
+        and user.is_active
+        and user.is_staff
+        and auth_hash == user.get_session_auth_hash()
+    )
+
+
+def _has_replay_cookie(request: HttpRequest, snapshot: Snapshot) -> bool:
+    value = request.COOKIES.get(_replay_cookie_name(snapshot))
+    if not value:
+        return False
+    try:
+        payload = signing.loads(value, salt=REPLAY_AUTH_SALT, max_age=settings.SESSION_COOKIE_AGE)
+    except signing.BadSignature:
+        return False
+    return isinstance(payload, dict) and _replay_payload_is_valid(payload, snapshot)
+
+
+def _private_snapshot_auth_redirect(request: HttpRequest, snapshot: Snapshot, path: str = "", *, preserve_query: bool = True):
+    next_path = _clean_replay_next(path or "index.html")
+    if preserve_query and request.META.get("QUERY_STRING"):
+        next_path = f"{next_path}?{request.META['QUERY_STRING']}"
+    target = build_admin_url(
+        f"/admin/core/snapshot/replay-auth/?snapshot={snapshot.id}&next={quote(next_path, safe='')}",
+        request=request,
+    )
+    return redirect(target)
+
+
+def _replay_auth_response(request: HttpRequest, snapshot: Snapshot):
+    try:
+        payload = signing.loads(str(request.GET.get("grant") or ""), salt=REPLAY_AUTH_SALT, max_age=REPLAY_GRANT_MAX_AGE)
+    except signing.BadSignature:
+        return _private_snapshot_auth_redirect(request, snapshot, "index.html", preserve_query=False)
+
+    if not isinstance(payload, dict) or not _replay_payload_is_valid(payload, snapshot):
+        return _private_snapshot_auth_redirect(request, snapshot, "index.html", preserve_query=False)
+
+    cookie_value = signing.dumps(payload, salt=REPLAY_AUTH_SALT)
+    response = redirect(_clean_replay_next(request.GET.get("next")))
+    response.set_cookie(
+        _replay_cookie_name(snapshot),
+        cookie_value,
+        max_age=settings.SESSION_COOKIE_AGE,
+        secure=request.is_secure(),
+        httponly=True,
+        samesite="Lax",
+    )
+    return response
+
+
+class SnapshotReplayAuthView(View):
+    """Admin-only handoff that lets a snap host mint its own replay cookie.
+
+    admin.* cannot set a host-only cookie for snap-* (browsers forbid that), and
+    widening the real Django session cookie to *.archivebox.localhost would let XSS
+    in replayed pages hit the admin UI. Instead admin.* proves the user is logged in
+    with a short URL grant, then snap-* validates it and sets a snap-host-only cookie.
+    """
+
+    def get(self, request: HttpRequest):
+        if not is_admin_user(request):
+            return redirect(f"{build_admin_url('/admin/login/', request=request)}?next={quote(request.get_full_path(), safe='')}")
+
+        snapshot = _find_snapshot_by_ref(str(request.GET.get("snapshot") or ""))
+        if not snapshot:
+            raise Http404
+
+        payload = {
+            "snapshot_id": str(snapshot.id),
+            "user_id": str(request.user.pk),
+            "session_key": request.session.session_key,
+            "auth_hash": request.user.get_session_auth_hash(),
+        }
+        grant = signing.dumps(payload, salt=REPLAY_AUTH_SALT)
+        next_path = _clean_replay_next(request.GET.get("next"))
+        target = build_snapshot_url(str(snapshot.id), "_auth", request=request, config=get_request_config(request))
+        return redirect(f"{target}?grant={quote(grant, safe='')}&next={quote(next_path, safe='')}")

archivebox/core/widgets.py

@@ -6,6 +6,7 @@ import hashlib
 from django import forms
 from django.db.models.manager import BaseManager
 from django.db.models.query import QuerySet
+from django.template.loader import render_to_string
 from django.utils.html import escape
 from django.utils.safestring import mark_safe
 
@@ -19,11 +20,7 @@ class TagEditorWidget(forms.Widget):
     - Uses AJAX for autocomplete and tag creation
     """
 
-    template_name = ""  # We render manually
-
-    class Media:
-        css = {"all": []}
-        js = []
+    template_name = "admin/widgets/tags.html"
 
     def __init__(self, attrs=None, snapshot_id=None):
         self.snapshot_id = snapshot_id
@@ -40,16 +37,6 @@ class TagEditorWidget(forms.Widget):
             normalized = f"t_{normalized}"
         return normalized
 
-    def _json_for_inline_script(self, value):
-        """Serialize JSON so it cannot close the surrounding inline <script> tag."""
-        return json.dumps(value).translate(
-            {
-                ord(">"): "\\u003E",
-                ord("<"): "\\u003C",
-                ord("&"): "\\u0026",
-            },
-        )
-
     def _tag_style(self, value):
         """Compute a stable pastel color style for a tag value."""
         tag = (value or "").strip().lower()
@@ -106,293 +93,10 @@ class TagEditorWidget(forms.Widget):
         widget_id_raw = attrs.get("id", name) if attrs else name
         widget_id = self._normalize_id(widget_id_raw)
 
-        # Build pills HTML
-        pills_html = ""
-        for tag in tags:
-            pills_html += f'''
-                <span class="tag-pill" data-tag="{self._escape(tag)}" style="{self._tag_style(tag)}">
-                    {self._escape(tag)}
-                    <button type="button" class="tag-remove-btn" data-tag-name="{self._escape(tag)}">&times;</button>
-                </span>
-            '''
-
-        tags_json = self._json_for_inline_script(tags)
-
-        # Build the widget HTML
-        html = f'''
-        <div id="{widget_id}_container" class="tag-editor-container" onclick="focusTagInput_{widget_id}(event)">
-            <div id="{widget_id}_pills" class="tag-pills">
-                {pills_html}
-            </div>
-            <input type="text"
-                   id="{widget_id}_input"
-                   class="tag-inline-input"
-                   list="{widget_id}_datalist"
-                   placeholder="Add tag..."
-                   autocomplete="off"
-                   onkeydown="handleTagKeydown_{widget_id}(event)"
-                   onkeypress="if(event.key==='Enter' || event.keyCode===13 || event.key===' ' || event.code==='Space' || event.key==='Spacebar'){{event.preventDefault(); event.stopPropagation();}}"
-                   oninput="fetchTagAutocomplete_{widget_id}(this.value)"
-            >
-            <datalist id="{widget_id}_datalist"></datalist>
-            <input type="hidden" name="{name}" id="{widget_id}" value="{self._escape(",".join(tags))}">
-        </div>
-
-        <script>
-        (function() {{
-            var currentTags_{widget_id} = {tags_json};
-            var autocompleteTimeout_{widget_id} = null;
-
-            window.focusTagInput_{widget_id} = function(event) {{
-                if (event.target.classList.contains('tag-remove-btn')) return;
-                document.getElementById('{widget_id}_input').focus();
-            }};
-
-            window.updateHiddenInput_{widget_id} = function() {{
-                var hiddenInput = document.getElementById('{widget_id}');
-                if (!hiddenInput) {{
-                    return;
-                }}
-                hiddenInput.value = currentTags_{widget_id}.join(',');
-                hiddenInput.dispatchEvent(new Event('input', {{ bubbles: true }}));
-                hiddenInput.dispatchEvent(new Event('change', {{ bubbles: true }}));
-            }};
-
-            function normalizeTags_{widget_id}(value) {{
-                var rawTags = Array.isArray(value) ? value : String(value || '').split(',');
-                var seen = {{}};
-                return rawTags
-                    .map(function(tag) {{ return String(tag || '').trim(); }})
-                    .filter(function(tag) {{
-                        if (!tag) return false;
-                        var normalized = tag.toLowerCase();
-                        if (seen[normalized]) return false;
-                        seen[normalized] = true;
-                        return true;
-                    }})
-                    .sort(function(a, b) {{
-                        return a.toLowerCase().localeCompare(b.toLowerCase());
-                    }});
-            }}
-
-            window.setTags_{widget_id} = function(value, options) {{
-                currentTags_{widget_id} = normalizeTags_{widget_id}(value);
-                rebuildPills_{widget_id}();
-                if (!(options && options.skipHiddenUpdate)) {{
-                    updateHiddenInput_{widget_id}();
-                }}
-            }};
-
-            window.syncTagEditorFromHidden_{widget_id} = function() {{
-                var hiddenInput = document.getElementById('{widget_id}');
-                if (!hiddenInput) {{
-                    return;
-                }}
-                setTags_{widget_id}(hiddenInput.value, {{ skipHiddenUpdate: true }});
-            }};
-
-            function computeTagStyle_{widget_id}(tagName) {{
-                var hash = 0;
-                var name = String(tagName || '').toLowerCase();
-                for (var i = 0; i < name.length; i++) {{
-                    hash = (hash * 31 + name.charCodeAt(i)) % 360;
-                }}
-                var bg = 'hsl(' + hash + ', 70%, 92%)';
-                var border = 'hsl(' + hash + ', 60%, 82%)';
-                var fg = 'hsl(' + hash + ', 35%, 28%)';
-                return {{ bg: bg, border: border, fg: fg }};
-            }}
-
-            function applyTagStyle_{widget_id}(el, tagName) {{
-                var colors = computeTagStyle_{widget_id}(tagName);
-                el.style.setProperty('--tag-bg', colors.bg);
-                el.style.setProperty('--tag-border', colors.border);
-                el.style.setProperty('--tag-fg', colors.fg);
-            }}
-
-            function getApiKey() {{
-                return (window.ARCHIVEBOX_API_KEY || '').trim();
-            }}
-
-            function buildApiUrl(path) {{
-                var apiKey = getApiKey();
-                if (!apiKey) return path;
-                var sep = path.indexOf('?') !== -1 ? '&' : '?';
-                return path + sep + 'api_key=' + encodeURIComponent(apiKey);
-            }}
-
-            function buildApiHeaders() {{
-                var headers = {{
-                    'Content-Type': 'application/json',
-                }};
-                var apiKey = getApiKey();
-                if (apiKey) headers['X-ArchiveBox-API-Key'] = apiKey;
-                var csrfToken = getCSRFToken();
-                if (csrfToken) headers['X-CSRFToken'] = csrfToken;
-                return headers;
-            }}
-
-            window.addTag_{widget_id} = function(tagName) {{
-                tagName = tagName.trim();
-                if (!tagName) return;
-
-                // Check if tag already exists (case-insensitive)
-                var exists = currentTags_{widget_id}.some(function(t) {{
-                    return t.toLowerCase() === tagName.toLowerCase();
-                }});
-                if (exists) {{
-                    document.getElementById('{widget_id}_input').value = '';
-                    return;
-                }}
-
-                // Add to current tags
-                currentTags_{widget_id}.push(tagName);
-                currentTags_{widget_id} = normalizeTags_{widget_id}(currentTags_{widget_id});
-
-                // Rebuild pills
-                rebuildPills_{widget_id}();
-                updateHiddenInput_{widget_id}();
-
-                // Clear input
-                document.getElementById('{widget_id}_input').value = '';
-
-                // Create tag via API if it doesn't exist (fire and forget)
-                fetch(buildApiUrl('/api/v1/core/tags/create/'), {{
-                    method: 'POST',
-                    headers: buildApiHeaders(),
-                    body: JSON.stringify({{ name: tagName }})
-                }}).catch(function(err) {{
-                    console.log('Tag creation note:', err);
-                }});
-            }};
-
-            window.removeTag_{widget_id} = function(tagName) {{
-                currentTags_{widget_id} = currentTags_{widget_id}.filter(function(t) {{
-                    return t.toLowerCase() !== tagName.toLowerCase();
-                }});
-                rebuildPills_{widget_id}();
-                updateHiddenInput_{widget_id}();
-            }};
-
-            window.rebuildPills_{widget_id} = function() {{
-                var container = document.getElementById('{widget_id}_pills');
-                container.innerHTML = '';
-                currentTags_{widget_id}.forEach(function(tag) {{
-                    var pill = document.createElement('span');
-                    pill.className = 'tag-pill';
-                    pill.setAttribute('data-tag', tag);
-                    applyTagStyle_{widget_id}(pill, tag);
-
-                    var tagText = document.createTextNode(tag);
-                    pill.appendChild(tagText);
-
-                    var removeBtn = document.createElement('button');
-                    removeBtn.type = 'button';
-                    removeBtn.className = 'tag-remove-btn';
-                    removeBtn.setAttribute('data-tag-name', tag);
-                    removeBtn.innerHTML = '&times;';
-                    pill.appendChild(removeBtn);
-
-                    container.appendChild(pill);
-                }});
-            }};
-
-            // Add event delegation for remove buttons
-            document.getElementById('{widget_id}_pills').addEventListener('click', function(event) {{
-                if (event.target.classList.contains('tag-remove-btn')) {{
-                    var tagName = event.target.getAttribute('data-tag-name');
-                    if (tagName) {{
-                        removeTag_{widget_id}(tagName);
-                    }}
-                }}
-            }});
-
-            document.getElementById('{widget_id}').addEventListener('change', function() {{
-                syncTagEditorFromHidden_{widget_id}();
-            }});
-
-            document.getElementById('{widget_id}').addEventListener('archivebox:sync-tags', function() {{
-                syncTagEditorFromHidden_{widget_id}();
-            }});
-
-            window.handleTagKeydown_{widget_id} = function(event) {{
-                var input = event.target;
-                var value = input.value.trim();
-                var isSpace = event.key === ' ' || event.code === 'Space' || event.key === 'Spacebar';
-                var isEnter = event.key === 'Enter' || event.keyCode === 13;
-                var isComma = event.key === ',';
-
-                if (isEnter || isSpace || isComma) {{
-                    event.preventDefault();
-                    event.stopPropagation();
-                    if (value) {{
-                        // Treat commas and whitespace as tag boundaries.
-                        value.split(/[\\s,]+/).forEach(function(tag) {{
-                            addTag_{widget_id}(tag.trim());
-                        }});
-                    }}
-                    return false;
-                }} else if (event.key === 'Backspace' && !value && currentTags_{widget_id}.length > 0) {{
-                    // Remove last tag on backspace when input is empty
-                    var lastTag = currentTags_{widget_id}.pop();
-                    rebuildPills_{widget_id}();
-                    updateHiddenInput_{widget_id}();
-                }}
-            }};
-
-            window.fetchTagAutocomplete_{widget_id} = function(query) {{
-                if (autocompleteTimeout_{widget_id}) {{
-                    clearTimeout(autocompleteTimeout_{widget_id});
-                }}
-
-                autocompleteTimeout_{widget_id} = setTimeout(function() {{
-                    if (!query || query.length < 1) {{
-                        document.getElementById('{widget_id}_datalist').innerHTML = '';
-                        return;
-                    }}
-
-                    fetch(buildApiUrl('/api/v1/core/tags/autocomplete/?q=' + encodeURIComponent(query)))
-                        .then(function(response) {{ return response.json(); }})
-                        .then(function(data) {{
-                            var datalist = document.getElementById('{widget_id}_datalist');
-                            datalist.innerHTML = '';
-                            (data.tags || []).forEach(function(tag) {{
-                                var option = document.createElement('option');
-                                option.value = tag.name;
-                                datalist.appendChild(option);
-                            }});
-                        }})
-                        .catch(function(err) {{
-                            console.log('Autocomplete error:', err);
-                        }});
-                }}, 150);
-            }};
-
-            function escapeHtml(text) {{
-                var div = document.createElement('div');
-                div.textContent = text;
-                return div.innerHTML;
-            }}
-
-            function getCSRFToken() {{
-                var cookies = document.cookie.split(';');
-                for (var i = 0; i < cookies.length; i++) {{
-                    var cookie = cookies[i].trim();
-                    if (cookie.startsWith('csrftoken=')) {{
-                        return cookie.substring('csrftoken='.length);
-                    }}
-                }}
-                // Fallback to hidden input
-                var input = document.querySelector('input[name="csrfmiddlewaretoken"]');
-                return input ? input.value : '';
-            }}
-
-            syncTagEditorFromHidden_{widget_id}();
-        }})();
-        </script>
-        '''
-
-        return mark_safe(html)
+        return render_to_string(
+            self.template_name,
+            {"name": name, "widget_id": widget_id, "tags_id": f"{widget_id}_tags", "tags": tags},
+        )
 
 
 class URLFiltersWidget(forms.Widget):
@@ -748,3 +452,58 @@ class InlineTagEditorWidget(TagEditorWidget):
         '''
 
         return mark_safe(html)
+
+
+def render_copy_block(text: str, *, multiline: bool = False, copy_text: str | None = None):
+    return render_to_string(
+        "admin/widgets/copy_block.html",
+        {
+            "text": text,
+            "copy_text": text if copy_text is None else copy_text,
+            "element": "pre" if multiline else "code",
+            "multiline": multiline,
+        },
+    )
+
+
+def render_permissions_badge(permissions, *, url: str, object_name: str):
+    from archivebox.core.permissions import PERMISSIONS_CHOICES, PERMISSIONS_META, normalize_permissions
+
+    permissions = normalize_permissions(permissions)
+    icon, label, fg, bg = PERMISSIONS_META[permissions]
+    return render_to_string(
+        "admin/widgets/permissions_badge.html",
+        {
+            "permissions": permissions,
+            "url": url,
+            "object_name": object_name,
+            "icon": icon,
+            "label": label,
+            "fg": fg,
+            "bg": bg,
+            "choices": [(value, *PERMISSIONS_META[value]) for value, _label in PERMISSIONS_CHOICES],
+        },
+    )
+
+
+def render_snapshot_progress(stats: dict, *, successful_plugins=None):
+    """Render the same hook counts and progress bar on snapshot cards and rows."""
+    from archivebox.plugins.discovery import get_plugin_icon
+
+    counts = {name: int(stats.get(name) or 0) for name in ("total", "succeeded", "failed", "skipped", "noresults", "running")}
+    total = counts["total"]
+    completed = sum(counts[name] for name in ("succeeded", "failed", "skipped", "noresults"))
+    plugins = sorted(successful_plugins or ())
+    return render_to_string(
+        "admin/widgets/snapshot_progress.html",
+        {
+            **counts,
+            "completed": completed,
+            "percent": int(completed / total * 100) if total else 0,
+            "succeeded_percent": int(counts["succeeded"] / total * 100) if total else 0,
+            "failed_percent": int((counts["succeeded"] + counts["failed"]) / total * 100) if total else 0,
+            "show_plugins": successful_plugins is not None,
+            "plugins": [(name, mark_safe(icon)) for name in plugins[:8] if str(icon := get_plugin_icon(name)).strip()],
+            "remaining_plugins": max(len(plugins) - 8, 0),
+        },
+    )

archivebox/crawls/admin.py

@@ -1,37 +1,27 @@
 __package__ = "archivebox.crawls"
 
-import json
-from copy import copy
 from typing import ClassVar
-from urllib.parse import urlencode, urlparse
 
-from django import forms
 from django.contrib import admin, messages
-from django.core.paginator import Paginator
-from django.db.models import Count, F, Q
+from django.db.models import Count, Q
 from django.http import HttpRequest, HttpResponseBadRequest, HttpResponseNotAllowed, JsonResponse
 from django.shortcuts import get_object_or_404, redirect
-from django.template.loader import render_to_string
 from django.template.response import TemplateResponse
 from django.urls import path, reverse
 from django.utils import timezone
-from django.utils.html import escape, format_html, format_html_join
+from django.utils.html import format_html, format_html_join
 from django.utils.safestring import mark_safe
 from django_object_actions import action
 
-from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin
-from archivebox.core.models import ArchiveResult, Snapshot
+from archivebox.core.widgets import render_permissions_badge
+from archivebox.base_models.admin import card_fieldset, BaseModelAdmin, ConfigEditorMixin
+from archivebox.core.models import Snapshot
 from archivebox.core.permissions import (
-    PERMISSIONS_CHOICES,
     PERMISSIONS_META,
-    PERMISSIONS_PRIVATE,
-    PERMISSIONS_PUBLIC,
-    PERMISSIONS_UNLISTED,
     PERMISSIONS_VALUES,
-    normalize_permissions,
 )
-from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget
 from archivebox.crawls.models import Crawl, CrawlSchedule
+from archivebox.crawls.forms import CrawlAdminForm
 from archivebox.misc.paginators import AcceleratedPaginator
 from archivebox.progressmonitor.views import progress_endpoint
 from archivebox.workers.models import RETRY_AT_MAX
@@ -51,554 +41,6 @@ class MaxDepthListFilter(admin.SimpleListFilter):
         return queryset
 
 
-def render_snapshots_list(snapshots_qs, request=None, crawl=None, page_size=50, prefix="snapshots"):
-    """Render a nice inline list view of snapshots with status, title, URL, and progress."""
-
-    query_param = f"{prefix}_q"
-    status_param = f"{prefix}_status"
-    page_param = f"{prefix}_page"
-    query = (request.GET.get(query_param, "") if request is not None else "").strip()
-    status_filter = (request.GET.get(status_param, "") if request is not None else "").strip()
-    valid_statuses = {choice[0] for choice in Snapshot.StatusChoices.choices}
-
-    filtered_qs = snapshots_qs
-    if query:
-        from archivebox.misc.util import filter_queryset_by_uuid_substring
-
-        id_match_pks = list(filter_queryset_by_uuid_substring(Snapshot.objects.all(), query).values_list("pk", flat=True)[:100])
-        filtered_qs = filtered_qs.filter(Q(pk__in=id_match_pks) | Q(url__icontains=query) | Q(title__icontains=query))
-    if status_filter in valid_statuses:
-        filtered_qs = filtered_qs.filter(status=status_filter)
-
-    # Keep ArchiveResult counters as scalar subqueries so the paginated
-    # Snapshot queryset does not become a join+GROUP BY over every result row.
-    snapshots_qs = filtered_qs.order_by("-created_at").annotate(
-        total_results=ArchiveResult.snapshot_count_expr(),
-        succeeded_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.SUCCEEDED),
-        failed_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.FAILED),
-        started_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.STARTED),
-        skipped_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.SKIPPED),
-        snapshot_permissions=F("permissions"),
-    )
-
-    page_number = request.GET.get(page_param, 1) if request is not None else 1
-    paginator = Paginator(snapshots_qs, page_size)
-    page_obj = paginator.get_page(page_number)
-    snapshots = page_obj.object_list
-    total_count = paginator.count
-
-    def querystring(**updates):
-        if request is None:
-            return "#"
-        params = request.GET.copy()
-        for key, value in updates.items():
-            if value in (None, ""):
-                params.pop(key, None)
-            else:
-                params[key] = str(value)
-        return f"?{params.urlencode()}" if params else "?"
-
-    preserved_inputs = ""
-    if request is not None:
-        managed_params = {query_param, status_param, page_param}
-        preserved_inputs = "".join(
-            f'<input type="hidden" name="{escape(key)}" value="{escape(value)}">'
-            for key, values in request.GET.lists()
-            if key not in managed_params
-            for value in values
-        )
-
-    status_options = "".join(
-        f'<option value="{escape(value)}"{" selected" if status_filter == value else ""}>{escape(label)}</option>'
-        for value, label in Snapshot.StatusChoices.choices
-    )
-
-    controls = f"""
-        <div class="crawl-snapshots-toolbar" style="display: flex; gap: 10px; align-items: center; justify-content: space-between; flex-wrap: wrap; padding: 10px 12px; background: #f8fafc; border-bottom: 1px solid #e2e8f0;">
-            <form method="get" style="display: flex; gap: 8px; align-items: center; flex: 1 1 540px; margin: 0;">
-                {preserved_inputs}
-                <input type="search" name="{query_param}" value="{escape(query)}" placeholder="Filter snapshots by title, URL, or ID"
-                       style="min-width: 260px; flex: 1 1 360px; padding: 7px 10px; border: 1px solid #cbd5e1; border-radius: 6px;">
-                <select name="{status_param}" style="max-width: 170px; padding: 7px 10px; border: 1px solid #cbd5e1; border-radius: 6px;">
-                    <option value="">All statuses</option>
-                    {status_options}
-                </select>
-                <input type="hidden" name="{page_param}" value="1">
-                <button type="submit" class="button" style="padding: 7px 12px;">Filter</button>
-                {f'<a href="{querystring(**{query_param: None, status_param: None, page_param: None})}" style="font-size: 12px; color: #64748b;">Clear</a>' if query or status_filter else ""}
-            </form>
-            <div style="font-size: 12px; color: #64748b; white-space: nowrap;">
-                {page_obj.start_index() if total_count else 0}-{page_obj.end_index() if total_count else 0} of {total_count}
-            </div>
-        </div>
-    """
-
-    if not snapshots:
-        return mark_safe(f"""
-            <div data-crawl-snapshots-list style="border: 1px solid #ddd; border-radius: 6px; overflow: hidden; max-width: 100%;">
-                {controls}
-                <div style="color: #666; font-style: italic; padding: 12px;">No Snapshots found.</div>
-            </div>
-        """)
-
-    # Status colors matching Django admin and progress monitor
-    status_colors = {
-        "queued": ("#6c757d", "#f8f9fa"),  # gray
-        "started": ("#856404", "#fff3cd"),  # amber
-        "paused": ("#1d4ed8", "#dbeafe"),  # blue
-        "sealed": ("#155724", "#d4edda"),  # green
-        "failed": ("#721c24", "#f8d7da"),  # red
-    }
-
-    rows = []
-    for snapshot in snapshots:
-        status = snapshot.status or "queued"
-        color, bg = status_colors.get(status, ("#6c757d", "#f8f9fa"))
-        permissions = snapshot.snapshot_permissions
-        permission_icon = {
-            PERMISSIONS_PUBLIC: "👁",
-            PERMISSIONS_UNLISTED: "🔗",
-            PERMISSIONS_PRIVATE: "🔒",
-        }[permissions]
-        permission_fg, permission_bg = {
-            PERMISSIONS_PUBLIC: ("#047857", "#d1fae5"),
-            PERMISSIONS_UNLISTED: ("#1d4ed8", "#dbeafe"),
-            PERMISSIONS_PRIVATE: ("#991b1b", "#fee2e2"),
-        }[permissions]
-
-        # Calculate progress
-        total = snapshot.total_results
-        succeeded = snapshot.succeeded_results
-        failed = snapshot.failed_results
-        running = snapshot.started_results
-        skipped = snapshot.skipped_results
-        done = succeeded + failed + skipped
-        pending = max(total - done - running, 0)
-        progress_pct = int((done / total) * 100) if total > 0 else 0
-        progress_text = f"{done}/{total}" if total > 0 else "-"
-        progress_title = f"{succeeded} succeeded, {failed} failed, {running} running, {pending} pending, {skipped} skipped"
-        progress_color = "#28a745"
-        if failed:
-            progress_color = "#dc3545"
-        elif running:
-            progress_color = "#17a2b8"
-        elif pending:
-            progress_color = "#ffc107"
-
-        # Truncate title and URL
-        snapshot_title = snapshot.title or "Untitled"
-        title = snapshot_title[:60]
-        if len(snapshot_title) > 60:
-            title += "..."
-        url_display = snapshot.url[:50]
-        if len(snapshot.url) > 50:
-            url_display += "..."
-        delete_button = ""
-        exclude_button = ""
-        if crawl is not None:
-            delete_url = reverse("admin:crawls_crawl_snapshot_delete", args=[crawl.pk, snapshot.pk])
-            exclude_url = reverse("admin:crawls_crawl_snapshot_exclude_domain", args=[crawl.pk, snapshot.pk])
-            delete_button = f'''
-                <button type="button"
-                        class="crawl-snapshots-action"
-                        data-post-url="{escape(delete_url)}"
-                        data-confirm="Delete this snapshot from the crawl?"
-                        title="Delete this snapshot from the crawl and remove its URL from the crawl queue."
-                        aria-label="Delete snapshot"
-                        style="border: 1px solid #ddd; background: #fff; color: #666; border-radius: 4px; width: 28px; height: 28px; cursor: pointer;">🗑</button>
-            '''
-            exclude_button = f'''
-                <button type="button"
-                        class="crawl-snapshots-action"
-                        data-post-url="{escape(exclude_url)}"
-                        data-confirm="Exclude this domain from the crawl? This removes matching queued URLs, deletes pending matching snapshots, and blocks future matches."
-                        title="Exclude this domain from this crawl. This removes matching URLs from the crawl queue, deletes pending matching snapshots, and blocks future matches."
-                        aria-label="Exclude domain from crawl"
-                        style="border: 1px solid #ddd; background: #fff; color: #666; border-radius: 4px; width: 28px; height: 28px; cursor: pointer;">⊘</button>
-            '''
-
-        # Format date
-        date_str = snapshot.created_at.strftime("%Y-%m-%d %H:%M") if snapshot.created_at else "-"
-
-        rows.append(f'''
-            <tr style="border-bottom: 1px solid #eee;">
-                <td style="padding: 6px 8px; white-space: nowrap;">
-                    <span style="display: inline-block; padding: 2px 8px; border-radius: 10px;
-                                 font-size: 11px; font-weight: 500; text-transform: uppercase;
-                                 color: {color}; background: {bg};">{status}</span>
-                </td>
-                <td style="padding: 6px 8px; white-space: nowrap; text-align: center;">
-                    <span title="{permissions}" style="display:inline-flex; align-items:center; justify-content:center; width:22px; height:22px; border-radius:999px; font-size:12px; color:{permission_fg}; background:{permission_bg};">{permission_icon}</span>
-                </td>
-                <td style="padding: 6px 8px; white-space: nowrap;">
-                    <a href="/{snapshot.archive_path}/" style="text-decoration: none;">
-                        <img src="/{snapshot.archive_path}/favicon.ico"
-                             style="width: 16px; height: 16px; vertical-align: middle; margin-right: 4px;"
-                             onerror="this.style.display='none'"/>
-                    </a>
-                </td>
-                <td style="padding: 6px 8px; max-width: 300px;">
-                    <a href="{snapshot.admin_change_url}" style="color: #417690; text-decoration: none; font-weight: 500;"
-                       title="{escape(snapshot_title)}">{escape(title)}</a>
-                </td>
-                <td style="padding: 6px 8px; max-width: 250px;">
-                    <a href="{escape(snapshot.url)}" target="_blank"
-                       style="color: #666; text-decoration: none; font-family: monospace; font-size: 11px;"
-                       title="{escape(snapshot.url)}">{escape(url_display)}</a>
-                </td>
-                <td style="padding: 6px 8px; white-space: nowrap; text-align: center;">
-                    <div style="display: inline-flex; align-items: center; gap: 6px;" title="{escape(progress_title)}">
-                        <div style="width: 60px; height: 6px; background: #eee; border-radius: 3px; overflow: hidden;">
-                            <div style="width: {progress_pct}%; height: 100%;
-                                        background: {progress_color};
-                                        transition: width 0.3s;"></div>
-                        </div>
-                        <a href="/admin/core/archiveresult/?snapshot__id__exact={snapshot.id}"
-                           style="font-size: 11px; color: #417690; min-width: 35px; text-decoration: none;"
-                           title="View archive results">{progress_text}</a>
-                    </div>
-                </td>
-                <td style="padding: 6px 8px; white-space: nowrap; color: #888; font-size: 11px;">
-                    {date_str}
-                </td>
-                {f'<td style="padding: 6px 8px; white-space: nowrap; text-align: right;"><div style="display: inline-flex; gap: 6px;">{exclude_button}{delete_button}</div></td>' if crawl is not None else ""}
-            </tr>
-        ''')
-
-    pagination = ""
-    if paginator.num_pages > 1:
-        pagination = f"""
-            <div style="display: flex; gap: 10px; align-items: center; justify-content: center; padding: 10px 12px; background: #f8fafc; border-top: 1px solid #e2e8f0; font-size: 12px;">
-                {"<a class='button' style='padding: 5px 10px;' href='" + querystring(**{page_param: page_obj.previous_page_number()}) + "'>Previous</a>" if page_obj.has_previous() else "<span style='color:#94a3b8;'>Previous</span>"}
-                <span style="color: #64748b;">Page {page_obj.number} of {paginator.num_pages}</span>
-                {"<a class='button' style='padding: 5px 10px;' href='" + querystring(**{page_param: page_obj.next_page_number()}) + "'>Next</a>" if page_obj.has_next() else "<span style='color:#94a3b8;'>Next</span>"}
-            </div>
-        """
-
-    return mark_safe(f"""
-        <div data-crawl-snapshots-list style="border: 1px solid #ddd; border-radius: 6px; overflow: hidden; max-width: 100%;">
-            {controls}
-            <table style="width: 100%; border-collapse: collapse; font-size: 13px;">
-                <thead>
-                    <tr style="background: #f5f5f5; border-bottom: 2px solid #ddd;">
-                        <th style="padding: 8px; text-align: left; font-weight: 600; color: #333;">Status</th>
-                        <th style="padding: 8px 4px; text-align: center; font-weight: 600; color: #333; width: 22px;">🔒</th>
-                        <th style="padding: 8px; text-align: left; font-weight: 600; color: #333; width: 24px;"></th>
-                        <th style="padding: 8px; text-align: left; font-weight: 600; color: #333;">Title</th>
-                        <th style="padding: 8px; text-align: left; font-weight: 600; color: #333;">URL</th>
-                        <th style="padding: 8px; text-align: center; font-weight: 600; color: #333;">Progress</th>
-                        <th style="padding: 8px; text-align: left; font-weight: 600; color: #333;">Created</th>
-                        {
-        '<th style="padding: 8px; text-align: right; font-weight: 600; color: #333;">Actions</th>' if crawl is not None else ""
-    }
-                    </tr>
-                </thead>
-                <tbody>
-                    {"".join(rows)}
-                </tbody>
-            </table>
-            {pagination}
-        </div>
-        {
-        '''
-        <script>
-        (function() {
-            if (window.__archiveboxCrawlSnapshotActionsBound) {
-                return;
-            }
-            window.__archiveboxCrawlSnapshotActionsBound = true;
-
-            function getCookie(name) {
-                var cookieValue = null;
-                if (!document.cookie) {
-                    return cookieValue;
-                }
-                var cookies = document.cookie.split(';');
-                for (var i = 0; i < cookies.length; i++) {
-                    var cookie = cookies[i].trim();
-                    if (cookie.substring(0, name.length + 1) === (name + '=')) {
-                        cookieValue = decodeURIComponent(cookie.substring(name.length + 1));
-                        break;
-                    }
-                }
-                return cookieValue;
-            }
-
-            document.addEventListener('click', function(event) {
-                var button = event.target.closest('.crawl-snapshots-action');
-                if (!button) {
-                    return;
-                }
-                event.preventDefault();
-
-                var confirmMessage = button.getAttribute('data-confirm');
-                if (confirmMessage && !window.confirm(confirmMessage)) {
-                    return;
-                }
-
-                button.disabled = true;
-
-                fetch(button.getAttribute('data-post-url'), {
-                    method: 'POST',
-                    credentials: 'same-origin',
-                    headers: {
-                        'X-CSRFToken': getCookie('csrftoken') || '',
-                        'X-Requested-With': 'XMLHttpRequest'
-                    }
-                }).then(function(response) {
-                    return response.json().then(function(data) {
-                        if (!response.ok) {
-                            throw new Error(data.error || 'Request failed');
-                        }
-                        return data;
-                    });
-                }).then(function() {
-                    window.location.reload();
-                }).catch(function(error) {
-                    button.disabled = false;
-                    window.alert(error.message || 'Request failed');
-                });
-            });
-        })();
-        </script>
-        '''
-        if crawl is not None
-        else ""
-    }
-    """)
-
-
-class URLFiltersField(forms.Field):
-    widget = URLFiltersWidget(source_selector="#id_urls")
-
-    def to_python(self, value):
-        if isinstance(value, dict):
-            return value
-        return {"allowlist": "", "denylist": "", "same_domain_only": False, "subpaths_only": False, "only_new": False}
-
-
-class CrawlAdminForm(forms.ModelForm):
-    """Custom form for Crawl admin to render urls field as textarea."""
-
-    tags_editor = forms.CharField(
-        label="Tags",
-        required=False,
-        widget=TagEditorWidget(),
-        help_text="Type tag names and press Enter or Space to add. Click × to remove.",
-    )
-    url_filters = URLFiltersField(
-        label="URL Filters",
-        required=False,
-        help_text="Set URL_ALLOWLIST / URL_DENYLIST for this crawl.",
-    )
-
-    class Meta:
-        model = Crawl
-        fields = "__all__"
-        widgets: ClassVar[dict[str, forms.Widget]] = {
-            "urls": forms.Textarea(
-                attrs={
-                    "rows": 8,
-                    "style": "width: 100%; font-family: monospace; font-size: 13px;",
-                    "placeholder": "https://example.com\nhttps://example2.com\n# Comments start with #",
-                },
-            ),
-            "notes": forms.Textarea(
-                attrs={
-                    "rows": 1,
-                    "style": "width: 100%; min-height: 0; resize: vertical;",
-                },
-            ),
-        }
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        config = dict(self.instance.config or {}) if self.instance and self.instance.pk else {}
-        if self.instance and self.instance.pk:
-            self.initial["tags_editor"] = self.instance.tags_str
-        effective_only_new = self.effective_only_new(self.instance if self.instance and self.instance.pk else None)
-        derived_filter_toggles = self.derive_filter_toggles(
-            self.instance.urls if self.instance and self.instance.pk else "",
-            config.get("URL_ALLOWLIST", ""),
-        )
-        self.initial["url_filters"] = {
-            "allowlist": config.get("URL_ALLOWLIST", ""),
-            "denylist": config.get("URL_DENYLIST", ""),
-            "same_domain_only": derived_filter_toggles["same_domain_only"],
-            "subpaths_only": derived_filter_toggles["subpaths_only"],
-            "only_new": effective_only_new,
-        }
-
-    @staticmethod
-    def extract_url_line(line):
-        line = str(line or "").strip()
-        if not line or line.startswith("#"):
-            return ""
-        if line.startswith("{"):
-            try:
-                return str(json.loads(line).get("url", "")).strip()
-            except (TypeError, ValueError, json.JSONDecodeError):
-                return ""
-        return line
-
-    @staticmethod
-    def regex_escape(text):
-        escaped = ""
-        for char in str(text or ""):
-            escaped += f"\\{char}" if char in r".*+?^${}()|[]\\" else char
-        return escaped
-
-    @classmethod
-    def generated_host_allowlist(cls, urls):
-        seen = set()
-        domains = []
-        for raw_line in str(urls or "").splitlines():
-            url = cls.extract_url_line(raw_line)
-            if not url:
-                continue
-            parsed = urlparse(url)
-            domain = (parsed.hostname or "").lower()
-            if not domain or domain in seen:
-                continue
-            seen.add(domain)
-            domains.append(domain)
-        if not domains:
-            return ""
-        return "^https?://(" + "|".join(cls.regex_escape(domain) for domain in domains) + ")([:/]|$)"
-
-    @staticmethod
-    def subpath_prefix(pathname):
-        path = str(pathname or "/")
-        while "//" in path:
-            path = path.replace("//", "/")
-        if not path or path == "/":
-            return "/"
-        if path.endswith("/"):
-            return path
-        last_slash = path.rfind("/")
-        last_part = path[last_slash + 1 :]
-        if "." in last_part:
-            return path[: last_slash + 1] or "/"
-        return path
-
-    @staticmethod
-    def parsed_host_and_port(parsed):
-        host = (parsed.hostname or "").lower()
-        if not host:
-            return ""
-        try:
-            port = parsed.port
-        except ValueError:
-            port = None
-        return f"{host}:{port}" if port is not None else host
-
-    @classmethod
-    def generated_subpath_allowlist(cls, urls):
-        seen = set()
-        paths = []
-        for raw_line in str(urls or "").splitlines():
-            url = cls.extract_url_line(raw_line)
-            if not url:
-                continue
-            parsed = urlparse(url)
-            domain = (parsed.hostname or "").lower()
-            if domain:
-                seen.add(domain)
-            host = cls.parsed_host_and_port(parsed)
-            path = cls.subpath_prefix(parsed.path)
-            path_key = f"{host}{path}"
-            if not host or path_key in seen:
-                continue
-            seen.add(path_key)
-            paths.append((host, path))
-        if not paths:
-            return ""
-        patterns = []
-        for host, path in paths:
-            if path == "/":
-                patterns.append(f"^https?://{cls.regex_escape(host)}([/?#]|$)")
-            elif path.endswith("/"):
-                patterns.append(f"^https?://{cls.regex_escape(host)}{cls.regex_escape(path)}")
-            else:
-                patterns.append(f"^https?://{cls.regex_escape(host)}{cls.regex_escape(path)}([/?#]|$)")
-        return "\n".join(patterns)
-
-    @classmethod
-    def derive_filter_toggles(cls, urls, allowlist):
-        normalized_allowlist = "\n".join(Crawl.split_filter_patterns(allowlist))
-        if not normalized_allowlist:
-            return {"same_domain_only": False, "subpaths_only": False}
-        if normalized_allowlist == cls.generated_subpath_allowlist(urls):
-            return {"same_domain_only": True, "subpaths_only": True}
-        if normalized_allowlist == cls.generated_host_allowlist(urls):
-            return {"same_domain_only": True, "subpaths_only": False}
-        return {"same_domain_only": False, "subpaths_only": False}
-
-    @staticmethod
-    def effective_only_new(crawl=None):
-        from archivebox.config.common import get_config
-
-        if crawl is not None:
-            return bool(get_config(crawl=crawl, resolve_plugins=False).ONLY_NEW)
-        return bool(get_config(resolve_plugins=False).ONLY_NEW)
-
-    @staticmethod
-    def inherited_only_new(crawl):
-        crawl_without_only_new = copy(crawl)
-        config = dict(crawl.config or {})
-        config.pop("ONLY_NEW", None)
-        crawl_without_only_new.config = config
-        return CrawlAdminForm.effective_only_new(crawl_without_only_new)
-
-    def clean_tags_editor(self):
-        tags_str = self.cleaned_data.get("tags_editor", "")
-        tag_names = []
-        seen = set()
-        for raw_name in tags_str.split(","):
-            name = raw_name.strip()
-            if not name:
-                continue
-            lowered = name.lower()
-            if lowered in seen:
-                continue
-            seen.add(lowered)
-            tag_names.append(name)
-        return ",".join(tag_names)
-
-    def clean_url_filters(self):
-        value = self.cleaned_data.get("url_filters") or {}
-        return {
-            "allowlist": "\n".join(Crawl.split_filter_patterns(value.get("allowlist", ""))),
-            "denylist": "\n".join(Crawl.split_filter_patterns(value.get("denylist", ""))),
-            "same_domain_only": bool(value.get("same_domain_only")),
-            "subpaths_only": bool(value.get("subpaths_only")),
-            "only_new": bool(value.get("only_new")),
-        }
-
-    def save(self, commit=True):
-        instance = super().save(commit=False)
-        instance.tags_str = self.cleaned_data.get("tags_editor", "")
-        if f"{self.add_prefix('url_filters')}_allowlist" in self.data or f"{self.add_prefix('url_filters')}_denylist" in self.data:
-            url_filters = self.cleaned_data.get("url_filters") or {}
-            instance.set_url_filters(
-                url_filters.get("allowlist", ""),
-                url_filters.get("denylist", ""),
-            )
-            config = dict(instance.config or {})
-            only_new = bool(url_filters.get("only_new"))
-            inherited_only_new = self.inherited_only_new(instance)
-            if only_new != inherited_only_new:
-                config["ONLY_NEW"] = only_new
-            else:
-                config.pop("ONLY_NEW", None)
-            instance.config = config
-        if commit:
-            instance.save()
-            instance.apply_crawl_config_filters()
-            self._save_m2m()
-        return instance
-
-
 class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
     form = CrawlAdminForm
     change_form_template = "admin/crawls/crawl/change_form.html"
@@ -646,13 +88,7 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
     readonly_fields = ("created_at", "modified_at", "stop_reason_display")
 
     fieldsets = (
-        (
-            "URLs",
-            {
-                "fields": ("urls", "url_filters"),
-                "classes": ("card", "wide"),
-            },
-        ),
+        card_fieldset("URLs", ("urls", "url_filters"), wide=True),
         (
             "Overview",
             {
@@ -674,13 +110,7 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         ),
     )
     add_fieldsets = (
-        (
-            "URLs",
-            {
-                "fields": ("urls", "url_filters"),
-                "classes": ("card", "wide"),
-            },
-        ),
+        card_fieldset("URLs", ("urls", "url_filters"), wide=True),
         (
             "Overview",
             {
@@ -1024,45 +454,11 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
 
     @admin.display(description="👁", ordering="permissions")
     def permissions_badge(self, obj):
-        permissions = normalize_permissions(obj.permissions)
-        icon, label, fg, bg = PERMISSIONS_META[permissions]
-        menu_items = format_html_join(
-            "",
-            (
-                '<button type="button" class="snapshot-permissions-menu-item{}" data-permissions="{}">'
-                '<span class="snapshot-permissions-icon" aria-hidden="true" style="color:{}; background:{};">{}</span>'
-                "<span>{}</span>"
-                "</button>"
-            ),
-            (
-                (
-                    " is-active" if choice_value == permissions else "",
-                    choice_value,
-                    choice_fg,
-                    choice_bg,
-                    choice_icon,
-                    choice_label,
-                )
-                for choice_value, choice_label in PERMISSIONS_CHOICES
-                for choice_icon, _choice_title, choice_fg, choice_bg in [PERMISSIONS_META[choice_value]]
-            ),
-        )
-        return format_html(
-            '<span class="snapshot-permissions-quick" data-current-permissions="{}" data-permissions-url="{}">'
-            '<button type="button" class="snapshot-permissions-button snapshot-permissions-{}" title="{}" aria-label="Change crawl permissions: {}" aria-expanded="false">'
-            '<span class="snapshot-permissions-icon" aria-hidden="true" style="color:{}; background:{};">{}</span>'
-            "</button>"
-            '<span class="snapshot-permissions-menu" role="menu" hidden>{}</span>'
-            "</span>",
+        permissions = obj.permissions
+        return render_permissions_badge(
             permissions,
-            reverse(f"{self.admin_site.name}:crawls_crawl_set_permissions", args=[obj.pk]),
-            permissions,
-            label,
-            label,
-            fg,
-            bg,
-            icon,
-            menu_items,
+            url=reverse(f"{self.admin_site.name}:crawls_crawl_set_permissions", args=[obj.pk]),
+            object_name="crawl",
         )
 
     @admin.display(description="Pause")
@@ -1089,36 +485,11 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
 
     @admin.display(description="Snapshots")
     def snapshots_changelist(self, obj):
-        request = self.request
-        snapshot_changelist = reverse("admin:core_snapshot_changelist")
-        scoped_params = {"crawl_id": str(obj.pk)}
-        full_url = f"{snapshot_changelist}?{urlencode(scoped_params)}"
-
-        snapshot_admin = self.admin_site._registry[Snapshot]
-        changelist_request = copy(request)
-        changelist_request.method = "GET"
-        changelist_request.path = snapshot_changelist
-        changelist_request.GET = request.GET.copy()
-        changelist_request.GET.update(
-            {
-                **scoped_params,
-                "_embedded": "crawl",
-                "per_page": "200",
-            },
+        return self.admin_site._registry[Snapshot].render_embedded_changelist(
+            self.request,
+            filters={"crawl_id": str(obj.pk)},
+            title="Snapshots in this crawl",
         )
-        changelist_request.POST = request.POST.copy()
-        changelist_request.POST.clear()
-
-        response = snapshot_admin.changelist_view(
-            changelist_request,
-            extra_context={"embedded_changelist": True},
-        )
-        context = {
-            **response.context_data,
-            "snapshot_changelist_url": full_url,
-            "crawl": obj,
-        }
-        return mark_safe(render_to_string("admin/crawls/crawl/snapshots_changelist.html", context, request=request))
 
     def delete_snapshot_view(self, request: HttpRequest, object_id: str, snapshot_id: str):
         if request.method != "POST":
@@ -1178,83 +549,25 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         first_url = next((line.strip() for line in (obj.urls or "").splitlines() if line.strip() and not line.strip().startswith("#")), "")
         return first_url[:80] + "..." if len(first_url) > 80 else first_url
 
-    @admin.display(description="URLs")
-    def urls_editor(self, obj):
-        """Editor for crawl URLs."""
-        widget_id = f"crawl_urls_{obj.pk}"
-
-        # Escape for safe HTML embedding
-        escaped_urls = (obj.urls or "").replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replace('"', "&quot;")
-
-        # Count lines for auto-expand logic
-        line_count = len((obj.urls or "").split("\n"))
-        uri_rows = min(max(3, line_count), 10)
-
-        html = f'''
-        <div id="{widget_id}_container" style="max-width: 900px;">
-            <!-- URLs input -->
-            <div style="margin-bottom: 12px;">
-                <label style="font-weight: bold; display: block; margin-bottom: 4px;">URLs (one per line):</label>
-                <textarea id="{widget_id}_urls"
-                          style="width: 100%; font-family: monospace; font-size: 13px;
-                                 padding: 8px; border: 1px solid #ccc; border-radius: 4px;
-                                 resize: vertical;"
-                          rows="{uri_rows}"
-                          placeholder="https://example.com&#10;https://example2.com&#10;# Comments start with #"
-                          readonly>{escaped_urls}</textarea>
-                <p style="color: #666; font-size: 12px; margin: 4px 0 0 0;">
-                    {line_count} URL{"s" if line_count != 1 else ""} · Note: URLs displayed here for reference only
-                </p>
-            </div>
-        </div>
-        '''
-        return mark_safe(html)
-
 
 class CrawlScheduleAdmin(BaseModelAdmin):
+    change_form_template = "admin/crawls/crawlschedule/change_form.html"
+
+    class Media:
+        css: ClassVar[dict[str, tuple[str, ...]]] = {"all": ("admin/crawls/crawl_change.css",)}
+
     list_display = ("id", "created_at", "created_by", "label", "notes", "template_str", "crawls", "num_crawls", "num_snapshots")
     sort_fields = ("id", "created_at", "created_by", "label", "notes", "template_str")
     search_fields = ("id", "created_by__username", "label", "notes", "schedule_id", "template_id", "template__urls")
 
-    readonly_fields = ("created_at", "modified_at", "crawls", "snapshots")
+    readonly_fields = ("created_at", "modified_at", "crawls")
     autocomplete_fields = ("template", "created_by")
 
     fieldsets = (
-        (
-            "Schedule Info",
-            {
-                "fields": ("label", "notes"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Configuration",
-            {
-                "fields": ("schedule", "template"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Metadata",
-            {
-                "fields": ("created_by", "created_at", "modified_at"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Crawls",
-            {
-                "fields": ("crawls",),
-                "classes": ("card", "wide"),
-            },
-        ),
-        (
-            "Snapshots",
-            {
-                "fields": ("snapshots",),
-                "classes": ("card", "wide"),
-            },
-        ),
+        card_fieldset("Schedule Info", ("label", "notes")),
+        card_fieldset("Configuration", ("schedule", "template")),
+        card_fieldset("Metadata", ("created_by", "created_at", "modified_at")),
+        card_fieldset("Crawls", ("crawls",), wide=True),
     )
 
     list_filter = ("created_by",)
@@ -1275,7 +588,15 @@ class CrawlScheduleAdmin(BaseModelAdmin):
         )
 
     def change_view(self, request, object_id, form_url="", extra_context=None):
-        self.request = request
+        schedule = self.get_object(request, object_id)
+        extra_context = {**(extra_context or {})}
+        if schedule:
+            extra_context["schedule_snapshots_changelist"] = self.admin_site._registry[Snapshot].render_embedded_changelist(
+                request,
+                filters={"crawl__schedule__id__exact": str(schedule.pk)},
+                title="Snapshots in this schedule",
+                default_search_mode="meta",
+            )
         return super().change_view(request, object_id, form_url, extra_context)
 
     def add_view(self, request, form_url="", extra_context=None):
@@ -1283,7 +604,7 @@ class CrawlScheduleAdmin(BaseModelAdmin):
 
     def get_fieldsets(self, request, obj=None):
         if obj is None:
-            return tuple(fieldset for fieldset in self.fieldsets if fieldset[0] not in {"Crawls", "Snapshots"})
+            return tuple(fieldset for fieldset in self.fieldsets if fieldset[0] != "Crawls")
         return self.fieldsets
 
     def save_model(self, request, obj, form, change):
@@ -1316,14 +637,6 @@ class CrawlScheduleAdmin(BaseModelAdmin):
             ((crawl.admin_change_url, crawl) for crawl in obj.crawl_set.all().order_by("-created_at")[:20]),
         ) or mark_safe("<i>No Crawls yet...</i>")
 
-    def snapshots(self, obj):
-        crawl_ids = obj.crawl_set.values_list("pk", flat=True)
-        return render_snapshots_list(
-            Snapshot.objects.filter(crawl_id__in=crawl_ids),
-            request=self.request,
-            prefix="schedule_snapshots",
-        )
-
 
 def register_admin(admin_site):
     admin_site.register(Crawl, CrawlAdmin)

archivebox/crawls/apps.py

@@ -5,11 +5,3 @@ class CrawlsConfig(AppConfig):
     default_auto_field = "django.db.models.BigAutoField"
     name = "archivebox.crawls"
     label = "crawls"
-
-    def ready(self):
-        """Import models to register state machines with the registry"""
-        import sys
-
-        # Skip during makemigrations to avoid premature state machine access
-        if "makemigrations" not in sys.argv:
-            from archivebox.crawls.models import CrawlMachine  # noqa: F401

archivebox/crawls/forms.py

@@ -0,0 +1,237 @@
+"""Crawl form input parsing; lifecycle operations remain on Crawl."""
+
+import json
+from copy import copy
+from typing import ClassVar
+from urllib.parse import urlparse
+
+from django import forms
+
+from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget
+from archivebox.crawls.models import Crawl
+
+
+class URLFiltersField(forms.Field):
+    widget = URLFiltersWidget(source_selector="#id_urls")
+
+    def to_python(self, value):
+        value = value if isinstance(value, dict) else {}
+        return {
+            "allowlist": "\n".join(Crawl.split_filter_patterns(value.get("allowlist", ""))),
+            "denylist": "\n".join(Crawl.split_filter_patterns(value.get("denylist", ""))),
+            "same_domain_only": bool(value.get("same_domain_only")),
+            "subpaths_only": bool(value.get("subpaths_only")),
+            "only_new": bool(value.get("only_new")),
+        }
+
+
+class CrawlAdminForm(forms.ModelForm):
+    """Custom form for Crawl admin to render urls field as textarea."""
+
+    tags_editor = forms.CharField(
+        label="Tags",
+        required=False,
+        widget=TagEditorWidget(),
+        help_text="Type tag names and press Enter or Space to add. Click × to remove.",
+    )
+    url_filters = URLFiltersField(
+        label="URL Filters",
+        required=False,
+        help_text="Set URL_ALLOWLIST / URL_DENYLIST for this crawl.",
+    )
+
+    class Meta:
+        model = Crawl
+        fields = "__all__"
+        widgets: ClassVar[dict[str, forms.Widget]] = {
+            "urls": forms.Textarea(
+                attrs={
+                    "rows": 8,
+                    "style": "width: 100%; font-family: monospace; font-size: 13px;",
+                    "placeholder": "https://example.com\nhttps://example2.com\n# Comments start with #",
+                },
+            ),
+            "notes": forms.Textarea(
+                attrs={
+                    "rows": 1,
+                    "style": "width: 100%; min-height: 0; resize: vertical;",
+                },
+            ),
+        }
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        config = dict(self.instance.config or {}) if self.instance and self.instance.pk else {}
+        if self.instance and self.instance.pk:
+            self.initial["tags_editor"] = self.instance.tags_str
+        effective_only_new = self.effective_only_new(self.instance if self.instance and self.instance.pk else None)
+        derived_filter_toggles = self.derive_filter_toggles(
+            self.instance.urls if self.instance and self.instance.pk else "",
+            config.get("URL_ALLOWLIST", ""),
+        )
+        self.initial["url_filters"] = {
+            "allowlist": config.get("URL_ALLOWLIST", ""),
+            "denylist": config.get("URL_DENYLIST", ""),
+            "same_domain_only": derived_filter_toggles["same_domain_only"],
+            "subpaths_only": derived_filter_toggles["subpaths_only"],
+            "only_new": effective_only_new,
+        }
+
+    @staticmethod
+    def extract_url_line(line):
+        line = str(line or "").strip()
+        if not line or line.startswith("#"):
+            return ""
+        if line.startswith("{"):
+            try:
+                return str(json.loads(line).get("url", "")).strip()
+            except (TypeError, ValueError, json.JSONDecodeError):
+                return ""
+        return line
+
+    @staticmethod
+    def regex_escape(text):
+        escaped = ""
+        for char in str(text or ""):
+            escaped += f"\\{char}" if char in r".*+?^${}()|[]\\" else char
+        return escaped
+
+    @classmethod
+    def generated_host_allowlist(cls, urls):
+        seen = set()
+        domains = []
+        for raw_line in str(urls or "").splitlines():
+            url = cls.extract_url_line(raw_line)
+            if not url:
+                continue
+            parsed = urlparse(url)
+            domain = (parsed.hostname or "").lower()
+            if not domain or domain in seen:
+                continue
+            seen.add(domain)
+            domains.append(domain)
+        if not domains:
+            return ""
+        return "^https?://(" + "|".join(cls.regex_escape(domain) for domain in domains) + ")([:/]|$)"
+
+    @staticmethod
+    def subpath_prefix(pathname):
+        path = str(pathname or "/")
+        while "//" in path:
+            path = path.replace("//", "/")
+        if not path or path == "/":
+            return "/"
+        if path.endswith("/"):
+            return path
+        last_slash = path.rfind("/")
+        last_part = path[last_slash + 1 :]
+        if "." in last_part:
+            return path[: last_slash + 1] or "/"
+        return path
+
+    @staticmethod
+    def parsed_host_and_port(parsed):
+        host = (parsed.hostname or "").lower()
+        if not host:
+            return ""
+        try:
+            port = parsed.port
+        except ValueError:
+            port = None
+        return f"{host}:{port}" if port is not None else host
+
+    @classmethod
+    def generated_subpath_allowlist(cls, urls):
+        seen = set()
+        paths = []
+        for raw_line in str(urls or "").splitlines():
+            url = cls.extract_url_line(raw_line)
+            if not url:
+                continue
+            parsed = urlparse(url)
+            domain = (parsed.hostname or "").lower()
+            if domain:
+                seen.add(domain)
+            host = cls.parsed_host_and_port(parsed)
+            path = cls.subpath_prefix(parsed.path)
+            path_key = f"{host}{path}"
+            if not host or path_key in seen:
+                continue
+            seen.add(path_key)
+            paths.append((host, path))
+        if not paths:
+            return ""
+        patterns = []
+        for host, path in paths:
+            if path == "/":
+                patterns.append(f"^https?://{cls.regex_escape(host)}([/?#]|$)")
+            elif path.endswith("/"):
+                patterns.append(f"^https?://{cls.regex_escape(host)}{cls.regex_escape(path)}")
+            else:
+                patterns.append(f"^https?://{cls.regex_escape(host)}{cls.regex_escape(path)}([/?#]|$)")
+        return "\n".join(patterns)
+
+    @classmethod
+    def derive_filter_toggles(cls, urls, allowlist):
+        normalized_allowlist = "\n".join(Crawl.split_filter_patterns(allowlist))
+        if not normalized_allowlist:
+            return {"same_domain_only": False, "subpaths_only": False}
+        if normalized_allowlist == cls.generated_subpath_allowlist(urls):
+            return {"same_domain_only": True, "subpaths_only": True}
+        if normalized_allowlist == cls.generated_host_allowlist(urls):
+            return {"same_domain_only": True, "subpaths_only": False}
+        return {"same_domain_only": False, "subpaths_only": False}
+
+    @staticmethod
+    def effective_only_new(crawl=None):
+        from archivebox.config.common import get_config
+
+        if crawl is not None:
+            return bool(get_config(crawl=crawl, resolve_plugins=False).ONLY_NEW)
+        return bool(get_config(resolve_plugins=False).ONLY_NEW)
+
+    @staticmethod
+    def inherited_only_new(crawl):
+        crawl_without_only_new = copy(crawl)
+        config = dict(crawl.config or {})
+        config.pop("ONLY_NEW", None)
+        crawl_without_only_new.config = config
+        return CrawlAdminForm.effective_only_new(crawl_without_only_new)
+
+    def clean_tags_editor(self):
+        tags_str = self.cleaned_data.get("tags_editor", "")
+        tag_names = []
+        seen = set()
+        for raw_name in tags_str.split(","):
+            name = raw_name.strip()
+            if not name:
+                continue
+            lowered = name.lower()
+            if lowered in seen:
+                continue
+            seen.add(lowered)
+            tag_names.append(name)
+        return ",".join(tag_names)
+
+    def save(self, commit=True):
+        instance = super().save(commit=False)
+        instance.tags_str = self.cleaned_data.get("tags_editor", "")
+        if f"{self.add_prefix('url_filters')}_allowlist" in self.data or f"{self.add_prefix('url_filters')}_denylist" in self.data:
+            url_filters = self.cleaned_data.get("url_filters") or {}
+            instance.set_url_filters(
+                url_filters.get("allowlist", ""),
+                url_filters.get("denylist", ""),
+            )
+            config = dict(instance.config or {})
+            only_new = bool(url_filters.get("only_new"))
+            inherited_only_new = self.inherited_only_new(instance)
+            if only_new != inherited_only_new:
+                config["ONLY_NEW"] = only_new
+            else:
+                config.pop("ONLY_NEW", None)
+            instance.config = config
+        if commit:
+            instance.save()
+            instance.apply_crawl_config_filters()
+            self._save_m2m()
+        return instance

archivebox/crawls/models.py

@@ -1,6 +1,6 @@
 __package__ = "archivebox.crawls"
 
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, ClassVar
 from collections.abc import Iterable, Mapping
 import uuid
 import json
@@ -19,7 +19,6 @@ from django.core.validators import MaxValueValidator, MinValueValidator
 from django.conf import settings
 from django.urls import reverse_lazy
 from django.utils import timezone
-from statemachine import State, registry
 from archivebox.config.common import rprint as print
 from archivebox.core.permissions import PERMISSIONS_VALUES, normalize_permissions
 
@@ -32,7 +31,7 @@ from archivebox.base_models.models import (
     ModelWithHealthStats,
     get_or_create_system_user_pk,
 )
-from archivebox.workers.models import RETRY_AT_MAX, ModelWithStateMachine, BaseStateMachine
+from archivebox.workers.models import DefaultStatusChoices, ModelWithQueue
 from archivebox.crawls.schedule_util import next_run_for_schedule, validate_schedule
 from archivebox.misc.util import parse_date, sanitize_html_text, validate_url, validate_url_length
 
@@ -91,6 +90,8 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes):
 
     @property
     def last_run_at(self):
+        if self.kind == "update":
+            return self.modified_at
         latest_crawl = self.crawl_set.order_by("-created_at").first()
         if latest_crawl:
             return latest_crawl.created_at
@@ -106,6 +107,22 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes):
         now = now or timezone.now()
         return self.is_enabled and self.next_run_at <= now
 
+    @property
+    def kind(self) -> str:
+        return str((self.config or {}).get("SCHEDULE_KIND") or "crawl")
+
+    def dispatch(self, queued_at=None) -> "Crawl | None":
+        """Run maintenance directly or enqueue one ordinary Crawl."""
+        queued_at = queued_at or timezone.now()
+        if self.kind == "update":
+            from archivebox.cli.archivebox_update import run_scheduled_maintenance
+
+            run_scheduled_maintenance()
+            type(self).objects.filter(pk=self.pk).update(modified_at=queued_at)
+            self.modified_at = queued_at
+            return None
+        return self.enqueue(queued_at=queued_at)
+
     def enqueue(self, queued_at=None) -> "Crawl":
         from archivebox.config.common import build_crawl_config_snapshot
 
@@ -113,10 +130,11 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes):
         template = self.template
         label = template.label or self.label
         persona = template.persona if template.persona_id else None
+        crawl_config = {key: value for key, value in (self.config or {}).items() if key != "SCHEDULE_KIND"}
 
         return Crawl.objects.create(
             urls=template.urls,
-            config=build_crawl_config_snapshot(persona=persona, overrides=self.config or {}),
+            config=build_crawl_config_snapshot(persona=persona, overrides=crawl_config),
             max_depth=template.max_depth,
             tags_str=template.tags_str,
             persona_id=template.persona_id,
@@ -129,7 +147,7 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes):
         )
 
 
-class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWithStateMachine):
+class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWithQueue):
     id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
@@ -158,16 +176,19 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
     notes = models.TextField(blank=True, null=False, default="")
     schedule = models.ForeignKey(CrawlSchedule, on_delete=models.SET_NULL, null=True, blank=True, editable=True)
 
-    status = ModelWithStateMachine.StatusField(
-        choices=ModelWithStateMachine.StatusChoices,
-        default=ModelWithStateMachine.StatusChoices.QUEUED,
+    status = ModelWithQueue.StatusField(
+        choices=ModelWithQueue.StatusChoices,
+        default=ModelWithQueue.StatusChoices.QUEUED,
     )
-    retry_at = ModelWithStateMachine.RetryAtField(default=timezone.now)
+    retry_at = ModelWithQueue.RetryAtField(default=timezone.now)
 
-    state_machine_name = "archivebox.crawls.models.CrawlMachine"
     retry_at_field_name = "retry_at"
     state_field_name = "status"
-    StatusChoices = ModelWithStateMachine.StatusChoices
+    StatusChoices: ClassVar[type[DefaultStatusChoices]] = DefaultStatusChoices
+    INITIAL_STATE = StatusChoices.QUEUED
+    ACTIVE_STATE = StatusChoices.STARTED
+    FINAL_STATES = (StatusChoices.SEALED,)
+    FINAL_OR_ACTIVE_STATES = (*FINAL_STATES, ACTIVE_STATE)
     active_state = StatusChoices.STARTED
     delete_after_final_statuses = (StatusChoices.SEALED,)
     RUNNABLE_STATES = (StatusChoices.QUEUED, StatusChoices.STARTED)
@@ -177,17 +198,12 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
 
     snapshot_set: models.Manager["Snapshot"]
 
-    if TYPE_CHECKING:
-
-        @property
-        def sm(self) -> "CrawlMachine": ...
-
     class Meta(
         ModelWithDeleteAfter.Meta,
         ModelWithOutputDir.Meta,
         ModelWithConfig.Meta,
         ModelWithHealthStats.Meta,
-        ModelWithStateMachine.Meta,
+        ModelWithQueue.Meta,
     ):
         app_label = "crawls"
         verbose_name = "Crawl"
@@ -220,7 +236,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
     def resume(self, *, when=None, save: bool = True) -> bool:
         resumed = super().resume(when=when, save=save)
         if resumed and self.pk:
-            from archivebox.core.models import ArchiveResult, Snapshot
+            from archivebox.core.models import Snapshot
 
             resume_at = when or timezone.now()
             active_snapshots = self.snapshot_set.filter(
@@ -231,7 +247,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
                 retry_at=resume_at,
                 modified_at=timezone.now(),
             )
-            ArchiveResult.resume_queryset(ArchiveResult.objects.filter(snapshot__crawl=self), when=resume_at)
         return resumed
 
     def cancel(self) -> None:
@@ -255,7 +270,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
 
         now = timezone.now()
         # Cancellation seals the Crawl first, then lets the runner seal each
-        # child Snapshot through its own state machine. Active children that
+        # child Snapshot through its own lifecycle. Active children that
         # are already due need no write; the runner will claim them as-is.
         active_children = self.snapshot_set.filter(
             status__in=Snapshot.OPEN_STATES,
@@ -272,8 +287,8 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
 
         now = timezone.now()
         # Parent pause is a scheduler command. Wake child rows only; each
-        # Snapshot runner claim performs the real pause transition and cascades
-        # its own ArchiveResults, keeping request/admin transactions tiny.
+        # Snapshot runner claim performs the real pause transition, keeping
+        # request/admin transactions tiny.
         active_children = self.snapshot_set.filter(
             status__in=Snapshot.RUNNABLE_STATES,
         )
@@ -518,18 +533,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
             return []
         return [url for _raw_line, url in self._iter_url_lines() if url]
 
-    def has_internal_input_root(self) -> bool:
-        """Return True when Crawl.urls is preserved source text, not the work queue.
-
-        The runner creates a synthetic root snapshot for raw import text so
-        parser hooks use the same Snapshot lifecycle as every other extractor.
-        In that mode the submitted text must remain in Crawl.urls forever;
-        parsed URLs live as child Snapshot rows and should not be appended back.
-        """
-        from archivebox.core.models import Snapshot
-
-        return self.snapshot_set.filter(url=Snapshot.INTERNAL_INPUT_URL, depth=0).exists()
-
     @staticmethod
     def normalize_domain(value: str) -> str:
         candidate = (value or "").strip().lower()
@@ -799,15 +802,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
             "deleted_snapshots": filter_result["deleted_snapshots"],
         }
 
-    def get_system_task(self) -> str | None:
-        urls = self.get_urls_list()
-        if len(urls) != 1:
-            return None
-        system_url = urls[0].strip().lower()
-        if system_url.startswith("archivebox://"):
-            return system_url
-        return None
-
     def resolve_persona(self):
         from archivebox.personas.models import Persona
 
@@ -971,14 +965,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
 
         if self.status == self.StatusChoices.SEALED:
             return []
-        # Internal-input crawls preserve the submitted text verbatim in
-        # Crawl.urls. The root snapshot's parser hooks are the only supported
-        # path for turning that text into child snapshots, otherwise a later
-        # runner pass could reinterpret plain URL-looking lines as direct
-        # depth-0 work and bypass format-specific metadata parsing.
-        if self.has_internal_input_root():
-            return []
-
         created_snapshots = []
         crawl_tag_names = self.current_tag_names()
         tags_by_name: dict[str, Tag] = {}
@@ -1194,7 +1180,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
                 ),
             }
             if tag_names:
-                tag_ids = [Tag.objects.get_or_create(name=tag_name)[0].pk for tag_name in tag_names]
+                tag_ids = [Tag.get_or_create_by_name(tag_name)[0].pk for tag_name in tag_names]
                 snapshot.add_tag_ids(tag_ids)
 
         existing_scope = Snapshot.objects if bool(self._config_value(config, "ONLY_NEW", True)) else self.snapshot_set
@@ -1251,16 +1237,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         if not created_snapshots:
             return []
 
-        crawl_urls = {url for _raw_line, url in self._iter_url_lines() if url}
-        new_url_lines = [snapshot.url for snapshot in created_snapshots if snapshot.url not in crawl_urls]
-        # For internal-input crawls, Crawl.urls is the immutable source text.
-        # Child snapshots are the parsed/indexed representation, so appending
-        # discovered URLs here would both duplicate state and destroy the exact
-        # import artifact users submitted through CLI/API/UI.
-        if new_url_lines and not self.has_internal_input_root():
-            self.urls = (self.urls.rstrip() + "\n" + "\n".join(new_url_lines)).lstrip("\n")
-            self.save(update_fields=["urls", "modified_at"])
-
         tag_names_by_url: dict[str, set[str]] = {}
         for snapshot in created_snapshots:
             tag_names = {
@@ -1296,170 +1272,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
 
         return created_snapshots
 
-    def install_declared_binaries(self, binary_names: set[str], machine=None) -> None:
-        """Install crawl-declared binaries through their unified state machine."""
-        from archivebox.crawls.locks import binary_lifecycle_lock
-        from archivebox.machine.models import Binary, Machine
-
-        if not binary_names:
-            return
-
-        machine = machine or Machine.current()
-        binaries = Binary.objects.filter(machine=machine, name__in=binary_names).order_by("name")
-        for binary in binaries:
-            with binary_lifecycle_lock(str(binary.id)):
-                binary.refresh_from_db()
-                if binary.status == Binary.StatusChoices.INSTALLED:
-                    continue
-                binary.update_and_requeue(retry_at=timezone.now())
-                binary.refresh_from_db()
-                binary.tick_claimed(lock_seconds=600)
-
-        unresolved_binaries = list(
-            Binary.objects.filter(
-                machine=machine,
-                name__in=binary_names,
-            )
-            .exclude(
-                status=Binary.StatusChoices.INSTALLED,
-            )
-            .order_by("name"),
-        )
-        if unresolved_binaries:
-            binary_details = ", ".join(
-                f"{binary.name} (status={binary.status}, retry_at={binary.retry_at})" for binary in unresolved_binaries
-            )
-            raise RuntimeError(
-                f"Crawl dependencies failed to install before continuing: {binary_details}",
-            )
-
-    def run(self) -> "Snapshot | None":
-        """
-        Execute this Crawl: run hooks, process JSONL, create snapshots.
-
-        Called by the state machine when entering the 'started' state.
-
-        Returns:
-            The root Snapshot for this crawl, or None for system crawls that don't create snapshots
-        """
-        import time
-        from archivebox.plugins.hooks import run_hook, discover_hooks, process_hook_records
-        from archivebox.config.common import get_config
-        from archivebox.machine.models import Machine
-
-        def get_runtime_config():
-            return get_config(crawl=self).for_crawl_runtime(
-                crawl=self,
-                persona=persona,
-                runtime_overrides=persona_runtime_overrides,
-            )
-
-        system_task = self.get_system_task()
-        if system_task == "archivebox://update":
-            from archivebox.cli.archivebox_update import process_all_db_snapshots
-
-            process_all_db_snapshots()
-            return None
-
-        machine = Machine.current()
-        declared_binary_names: set[str] = set()
-        persona_runtime_overrides: dict[str, str] = {}
-        persona = self.resolve_persona()
-        if persona:
-            base_runtime_config = get_config(crawl=self, persona=persona)
-            chrome_binary = str(base_runtime_config.get("CHROME_BINARY") or "")
-            persona_runtime_overrides = persona.prepare_runtime_for_crawl(
-                crawl=self,
-                chrome_binary=chrome_binary,
-            )
-
-        def run_crawl_hook(hook: Path) -> set[str]:
-            primary_url = next(
-                (line.strip() for line in self.urls.splitlines() if line.strip()),
-                self.urls.strip(),
-            )
-
-            hook_start = time.time()
-            plugin_name = hook.parent.name
-            output_dir = self.output_dir / plugin_name
-            output_dir.mkdir(parents=True, exist_ok=True)
-
-            process = run_hook(
-                hook,
-                output_dir=output_dir,
-                config=get_runtime_config(),
-                crawl_id=str(self.id),
-                source_url=self.urls,
-                url=primary_url,
-                snapshot_id=str(self.id),
-            )
-            hook_elapsed = time.time() - hook_start
-            if hook_elapsed > 0.5:
-                print(f"[yellow]⏱️  Hook {hook.name} took {hook_elapsed:.2f}s[/yellow]")
-
-            if process.status == process.StatusChoices.RUNNING:
-                if process.poll() is None:
-                    return set()
-
-            from archivebox.plugins.hooks import extract_records_from_process
-
-            records = []
-            # A hook can exit before its completed Process metadata is visible.
-            # Give successful hooks a brief chance to flush JSONL stdout into
-            # the Process row before downstream hooks.
-            for delay in (0.0, 0.05, 0.1, 0.25, 0.5):
-                if delay:
-                    time.sleep(delay)
-                records = extract_records_from_process(process)
-                if records:
-                    break
-            if records:
-                print(f"[cyan]📝 Processing {len(records)} records from {hook.name}[/cyan]")
-                for record in records[:3]:
-                    print(f"   Record: type={record.get('type')}, keys={list(record.keys())[:5]}")
-            if system_task:
-                records = [record for record in records if record.get("type") in ("BinaryRequest", "Binary")]
-            overrides = {"crawl": self}
-            stats = process_hook_records(records, overrides=overrides)
-            if stats:
-                print(f"[green]✓ Created: {stats}[/green]")
-
-            hook_binary_names = {
-                str(record.get("name")).strip()
-                for record in records
-                if record.get("type") in ("BinaryRequest", "Binary") and record.get("name")
-            }
-            hook_binary_names.discard("")
-            if hook_binary_names:
-                declared_binary_names.update(hook_binary_names)
-            return hook_binary_names
-
-        hooks = discover_hooks("Crawl", config=get_runtime_config())
-
-        for hook in hooks:
-            hook_binary_names = run_crawl_hook(hook)
-            if hook_binary_names:
-                self.install_declared_binaries(hook_binary_names, machine=machine)
-
-        # Safety check: don't create snapshots if any crawl-declared dependency
-        # is still unresolved after all crawl hooks have run.
-        self.install_declared_binaries(declared_binary_names, machine=machine)
-
-        # Create snapshots from all URLs in self.urls
-        if system_task:
-            leaked_snapshots = self.snapshot_set.all()
-            if leaked_snapshots.exists():
-                leaked_count = leaked_snapshots.count()
-                leaked_snapshots.delete()
-                print(f"[yellow]⚠️  Removed {leaked_count} leaked snapshot(s) created during system crawl {system_task}[/yellow]")
-            return None
-
-        self.create_snapshots_from_urls()
-
-        # Return first snapshot for this crawl (newly created or existing)
-        # This ensures the crawl doesn't seal if snapshots exist, even if they weren't just created
-        return self.snapshot_set.first()
-
     def is_finished(self) -> bool:
         """Check if crawl is finished (all snapshots sealed or no snapshots exist)."""
         from archivebox.core.models import Snapshot
@@ -1483,11 +1295,64 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
 
         return True
 
-    def cleanup(self):
-        """Clean up background hooks and run on_CrawlEnd hooks."""
-        from archivebox.plugins.hooks import run_hook, discover_hooks
+    def can_start(self) -> bool:
+        return bool(self.urls and self.get_urls_list())
 
-        # Clean up .pid files from output directory
+    def has_finished_snapshots(self) -> bool:
+        from archivebox.core.models import Snapshot
+
+        snapshots = self.snapshot_set.all()
+        return snapshots.exists() and not snapshots.exclude(status=Snapshot.StatusChoices.SEALED).exists()
+
+    def mark_started(self) -> bool:
+        now = timezone.now()
+        updated = self.safe_update(
+            {
+                "status": self.StatusChoices.STARTED,
+                "retry_at": now + timedelta(seconds=2),
+            },
+            extra_filter={"status": self.StatusChoices.QUEUED},
+        )
+        return updated
+
+    def seal(self) -> bool:
+        """Finalize a runner-owned Crawl without dispatching hooks directly."""
+        now = timezone.now()
+        updated = self.safe_update(
+            {
+                "status": self.StatusChoices.SEALED,
+                "retry_at": None,
+                "modified_at": now,
+            },
+            refresh=False,
+            extra_filter={"status__in": (*self.RUNNABLE_STATES, self.StatusChoices.SEALED)},
+        )
+        if not updated:
+            self.refresh_from_db()
+            return False
+        self.status = self.StatusChoices.SEALED
+        self.retry_at = None
+        self.modified_at = now
+        self.schedule_child_snapshots_for_sealing()
+        self.cleanup_runtime()
+        return True
+
+    def advance_lifecycle(self) -> bool:
+        """Advance one explicit lifecycle step after the runner claims this row."""
+        if self.status == self.StatusChoices.PAUSED:
+            return False
+        if self.status == self.StatusChoices.QUEUED:
+            if self.has_finished_snapshots():
+                return self.seal()
+            if not self.can_start():
+                return False
+            return self.mark_started()
+        if self.status == self.StatusChoices.STARTED and self.is_finished():
+            return self.seal()
+        return False
+
+    def cleanup_runtime(self) -> None:
+        """Remove runner-owned runtime artifacts after abx-dl cleanup hooks finish."""
         if self.output_dir.exists():
             for pid_file in self.output_dir.glob("**/*.pid"):
                 pid_file.unlink(missing_ok=True)
@@ -1495,207 +1360,3 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         persona = self.resolve_persona()
         if persona:
             persona.cleanup_runtime_for_crawl(self)
-
-        # Run on_CrawlEnd hooks
-        from archivebox.config.common import get_config
-
-        config = get_config(crawl=self)
-
-        hooks = discover_hooks("CrawlEnd", config=config)
-
-        for hook in hooks:
-            plugin_name = hook.parent.name
-            output_dir = self.output_dir / plugin_name
-            output_dir.mkdir(parents=True, exist_ok=True)
-
-            process = run_hook(
-                hook,
-                output_dir=output_dir,
-                config=config,
-                crawl_id=str(self.id),
-                source_url=self.urls,  # Pass full newline-separated URLs
-            )
-
-            # Log failures but don't block
-            if process.exit_code != 0:
-                print(f"[yellow]⚠️ CrawlEnd hook failed: {hook.name}[/yellow]")
-
-
-# =============================================================================
-# State Machines
-# =============================================================================
-
-
-class CrawlMachine(BaseStateMachine):
-    crawl: Crawl
-
-    """
-    State machine for managing Crawl lifecycle.
-
-    Hook Lifecycle:
-    ┌─────────────────────────────────────────────────────────────┐
-    │ QUEUED State                                                │
-    │  • Waiting for crawl to be ready (has URLs)                 │
-    └─────────────────────────────────────────────────────────────┘
-                            ↓ tick() when can_start()
-    ┌─────────────────────────────────────────────────────────────┐
-    │ STARTED State → enter_started()                             │
-    │  1. crawl.run()                                             │
-    │     • discover_hooks('Crawl') → finds all crawl hooks       │
-    │     • For each hook:                                        │
-    │       - run_hook(script, output_dir, ...)                   │
-    │       - Parse JSONL from hook output                        │
-    │       - process_hook_records() → creates Snapshots          │
-    │     • create_snapshots_from_urls() → from self.urls field   │
-    │                                                              │
-    │  2. Snapshots process independently with their own          │
-    │     state machines (see SnapshotMachine)                    │
-    └─────────────────────────────────────────────────────────────┘
-                            ↓ tick() when is_finished()
-    ┌─────────────────────────────────────────────────────────────┐
-    │ SEALED State → enter_sealed()                               │
-    │  • cleanup() → runs on_CrawlEnd hooks, kills background     │
-    │  • Set retry_at=None (no more processing)                   │
-    └─────────────────────────────────────────────────────────────┘
-    """
-
-    model_attr_name = "crawl"
-
-    # States
-    queued = State(value=Crawl.StatusChoices.QUEUED, initial=True)
-    started = State(value=Crawl.StatusChoices.STARTED)
-    paused = State(value=Crawl.StatusChoices.PAUSED)
-    sealed = State(value=Crawl.StatusChoices.SEALED, final=True)
-
-    # Tick Event (polled by workers)
-    tick = (
-        queued.to(sealed, cond="has_finished_snapshots")
-        | queued.to.itself(unless="can_start")
-        | queued.to(started, cond="can_start")
-        | started.to(sealed, cond="is_finished")
-        | paused.to.itself()
-    )
-
-    # Manual event (triggered by last Snapshot sealing, or by direct
-    # index-only/bg creation when every requested URL is rejected before any
-    # Snapshot rows exist).
-    seal = queued.to(sealed) | started.to(sealed) | paused.to(sealed)
-    pause_requested = queued.to(paused) | started.to(paused)
-    resume_requested = paused.to(queued)
-
-    def can_start(self) -> bool:
-        if not self.crawl.urls:
-            print(f"[red]⚠️ Crawl {self.crawl.id} cannot start: no URLs[/red]")
-            return False
-        urls_list = self.crawl.get_urls_list()
-        if not urls_list:
-            print(f"[red]⚠️ Crawl {self.crawl.id} cannot start: no valid URLs in urls field[/red]")
-            return False
-        return True
-
-    def is_finished(self) -> bool:
-        """Check if all Snapshots for this crawl are finished."""
-        return self.crawl.is_finished()
-
-    def has_finished_snapshots(self) -> bool:
-        """A queued crawl with only final Snapshot rows was interrupted before sealing."""
-        from archivebox.core.models import Snapshot
-
-        snapshots = self.crawl.snapshot_set.all()
-        return snapshots.exists() and not snapshots.exclude(status=Snapshot.StatusChoices.SEALED).exists()
-
-    @queued.enter
-    def enter_queued(self):
-        self.crawl.update_and_requeue(
-            retry_at=timezone.now(),
-            status=Crawl.StatusChoices.QUEUED,
-        )
-
-    @started.enter
-    def enter_started(self):
-        import sys
-
-        print(f"[cyan]🔄 CrawlMachine.enter_started() - creating snapshots for {self.crawl.id}[/cyan]", file=sys.stderr)
-
-        try:
-            # Run the crawl - runs hooks, processes JSONL, creates snapshots
-            first_snapshot = self.crawl.run()
-
-            if first_snapshot:
-                print(
-                    f"[cyan]🔄 Created {self.crawl.snapshot_set.count()} snapshot(s), first: {first_snapshot.url}[/cyan]",
-                    file=sys.stderr,
-                )
-                # Update status to STARTED
-                # Set retry_at to near future so tick() can poll and check is_finished()
-                self.crawl.update_and_requeue(
-                    retry_at=timezone.now() + timedelta(seconds=2),
-                    status=Crawl.StatusChoices.STARTED,
-                )
-            else:
-                # No snapshots (system crawl that only runs setup hooks)
-                print("[cyan]🔄 No snapshots created, sealing crawl immediately[/cyan]", file=sys.stderr)
-                # Seal immediately since there's no work to do
-                self.seal()
-
-        except Exception as e:
-            print(f"[red]⚠️ Crawl {self.crawl.id} failed to start: {e}[/red]")
-            import traceback
-
-            traceback.print_exc()
-            raise
-
-    @paused.enter
-    def enter_paused(self):
-        paused = self.crawl.safe_update(
-            {
-                "retry_at": RETRY_AT_MAX,
-                "status": Crawl.StatusChoices.PAUSED,
-            },
-            extra_filter={"status__in": Crawl.RUNNABLE_STATES},
-        )
-        if paused:
-            self.crawl.schedule_child_snapshots_for_pause()
-
-    @sealed.enter
-    def enter_sealed(self):
-        now = timezone.now()
-        self.crawl.status = Crawl.StatusChoices.SEALED
-        self.crawl.retry_at = None
-        # Guard: never seal a row that a concurrent writer flipped to PAUSED.
-        # Sealing is idempotent (SEALED→SEALED is a no-op rewrite), so
-        # status__in covers both the QUEUED/STARTED→SEALED transition and the
-        # rare re-entry case.
-        updated = self.crawl.safe_update(
-            {
-                "status": Crawl.StatusChoices.SEALED,
-                "retry_at": None,
-                "modified_at": now,
-            },
-            refresh=False,
-            extra_filter={
-                "status__in": [
-                    Crawl.StatusChoices.QUEUED,
-                    Crawl.StatusChoices.STARTED,
-                    Crawl.StatusChoices.SEALED,
-                ],
-            },
-        )
-        if not updated:
-            self.crawl.refresh_from_db()
-            return
-        self.crawl.modified_at = now
-
-        self.crawl.schedule_child_snapshots_for_sealing()
-        # Clean up background hooks and run on_CrawlEnd hooks after the final
-        # state is visible so cleanup projectors cannot resurrect the crawl.
-        self.crawl.cleanup()
-
-
-# =============================================================================
-# Register State Machines
-# =============================================================================
-
-# Manually register state machines with python-statemachine registry
-# (normally auto-discovered from statemachines.py, but we define them here for clarity)
-registry.register(CrawlMachine)

archivebox/machine/admin.py

@@ -1,6 +1,5 @@
 __package__ = "archivebox.machine"
 
-import json
 import shlex
 from pathlib import Path
 
@@ -14,50 +13,13 @@ from django.utils.html import format_html
 from django.utils.safestring import mark_safe
 from django_object_actions import action
 
-from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin
+from archivebox.base_models.admin import card_fieldset, BaseModelAdmin, ConfigEditorMixin
+from archivebox.core.widgets import render_copy_block
 from archivebox.machine.env_util import env_to_dotenv_text
 from archivebox.machine.models import Binary, Machine, NetworkInterface, Process
 from archivebox.misc.logging_util import printable_filesize
 
 
-def _render_copy_block(text: str, *, multiline: bool = False):
-    if multiline:
-        return format_html(
-            """
-            <div style="position: relative; width: 100%; max-width: 100%; overflow: hidden; box-sizing: border-box;">
-                <button type="button"
-                        data-command="{}"
-                        onclick="(function(btn){{var text=btn.dataset.command||''; if(navigator.clipboard&&navigator.clipboard.writeText){{navigator.clipboard.writeText(text);}} else {{var ta=document.createElement('textarea'); ta.value=text; document.body.appendChild(ta); ta.select(); document.execCommand('copy'); document.body.removeChild(ta);}}}})(this); return false;"
-                        style="position: absolute; top: 6px; right: 6px; z-index: 1; padding: 2px 8px; border: 0; border-radius: 4px; background: #e2e8f0; color: #334155; font-size: 11px; cursor: pointer;">
-                    Copy
-                </button>
-                <pre title="{}" style="display: block; width: 100%; max-width: 100%; overflow: auto; max-height: 300px; margin: 0; padding: 8px 56px 8px 8px; background: #f8fafc; border: 1px solid #e2e8f0; border-radius: 6px; font-size: 11px; line-height: 1.45; white-space: pre-wrap; word-break: break-word; box-sizing: border-box;">{}</pre>
-            </div>
-            """,
-            text,
-            text,
-            text,
-        )
-    return format_html(
-        """
-        <div style="position: relative; width: 100%; max-width: 100%; overflow: hidden; box-sizing: border-box;">
-            <button type="button"
-                    data-command="{}"
-                    onclick="(function(btn){{var text=btn.dataset.command||''; if(navigator.clipboard&&navigator.clipboard.writeText){{navigator.clipboard.writeText(text);}} else {{var ta=document.createElement('textarea'); ta.value=text; document.body.appendChild(ta); ta.select(); document.execCommand('copy'); document.body.removeChild(ta);}}}})(this); return false;"
-                    style="position: absolute; top: 6px; right: 6px; z-index: 1; padding: 2px 8px; border: 0; border-radius: 4px; background: #e2e8f0; color: #334155; font-size: 11px; cursor: pointer;">
-                Copy
-            </button>
-            <code title="{}" style="display: block; width: 100%; max-width: 100%; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; padding: 8px 56px 8px 8px; background: #f8fafc; border: 1px solid #e2e8f0; border-radius: 6px; font-size: 11px; box-sizing: border-box;">
-                {}
-            </code>
-        </div>
-        """,
-        text,
-        text,
-        text,
-    )
-
-
 def _format_process_duration_seconds(started_at, ended_at) -> str:
     if not started_at:
         return "-"
@@ -122,59 +84,27 @@ class MachineAdmin(ConfigEditorMixin, BaseModelAdmin):
     readonly_fields = ("guid", "created_at", "modified_at", "ips")
 
     fieldsets = (
-        (
-            "Identity",
-            {
-                "fields": ("hostname", "guid", "ips"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Hardware",
-            {
-                "fields": ("hw_manufacturer", "hw_product", "hw_uuid", "hw_in_docker", "hw_in_vm"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Operating System",
-            {
-                "fields": ("os_platform", "os_family", "os_arch", "os_kernel", "os_release"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Statistics",
-            {
-                "fields": ("stats", "num_uses_succeeded", "num_uses_failed"),
-                "classes": ("card",),
-            },
-        ),
-        (
+        card_fieldset("Identity", ("hostname", "guid", "ips")),
+        card_fieldset("Hardware", ("hw_manufacturer", "hw_product", "hw_uuid", "hw_in_docker", "hw_in_vm")),
+        card_fieldset("Operating System", ("os_platform", "os_family", "os_arch", "os_kernel", "os_release")),
+        card_fieldset("Statistics", ("stats", "num_uses_succeeded", "num_uses_failed")),
+        card_fieldset(
             "Configuration",
-            {
-                "fields": ("config",),
-                "classes": ("card", "wide"),
-                "description": mark_safe(
-                    '<div style="padding:8px 10px;margin-bottom:8px;background:#fff7ed;'
-                    "border:1px solid #fed7aa;border-left:4px solid #f59e0b;border-radius:4px;"
-                    'color:#7c2d12;font-size:12px;line-height:1.45;">'
-                    "<b>Heads up:</b> saving here also rewrites "
-                    "<code>data/ArchiveBox.conf</code> on disk to match — the two stores are "
-                    "kept in 1:1 sync, so any keys you remove here will be removed from the file "
-                    "too. Edits to <code>ArchiveBox.conf</code> (or <code>archivebox config --set</code>) "
-                    "propagate back into this field on the next request."
-                    "</div>",
-                ),
-            },
-        ),
-        (
-            "Timestamps",
-            {
-                "fields": ("created_at", "modified_at"),
-                "classes": ("card",),
-            },
+            ("config",),
+            wide=True,
+            description=mark_safe(
+                '<div style="padding:8px 10px;margin-bottom:8px;background:#fff7ed;'
+                "border:1px solid #fed7aa;border-left:4px solid #f59e0b;border-radius:4px;"
+                'color:#7c2d12;font-size:12px;line-height:1.45;">'
+                "<b>Heads up:</b> saving here also rewrites "
+                "<code>data/ArchiveBox.conf</code> on disk to match — the two stores are "
+                "kept in 1:1 sync, so any keys you remove here will be removed from the file "
+                "too. Edits to <code>ArchiveBox.conf</code> (or <code>archivebox config --set</code>) "
+                "propagate back into this field on the next request."
+                "</div>",
+            ),
         ),
+        card_fieldset("Timestamps", ("created_at", "modified_at")),
     )
 
     list_filter = ("hw_in_docker", "hw_in_vm", "os_arch", "os_family", "os_platform")
@@ -190,12 +120,6 @@ class MachineAdmin(ConfigEditorMixin, BaseModelAdmin):
             ", ".join(machine.networkinterface_set.values_list("ip_public", flat=True)),
         )
 
-    @admin.display(description="Health", ordering="health")
-    def health_display(self, obj):
-        h = obj.health
-        color = "green" if h >= 80 else "orange" if h >= 50 else "red"
-        return format_html('<span style="color: {};">{}</span>', color, h)
-
     @admin.display(description="ID", ordering="id")
     def id_display(self, machine):
         # Highlight the row representing the machine that ``Machine.current()``
@@ -275,41 +199,11 @@ class NetworkInterfaceAdmin(BaseModelAdmin):
     readonly_fields = ("machine", "created_at", "modified_at", "mac_address", "ip_public", "ip_local", "dns_server")
 
     fieldsets = (
-        (
-            "Machine",
-            {
-                "fields": ("machine",),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Network",
-            {
-                "fields": ("iface", "ip_public", "ip_local", "mac_address", "dns_server"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Location",
-            {
-                "fields": ("hostname", "isp", "city", "region", "country"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Usage",
-            {
-                "fields": ("num_uses_succeeded", "num_uses_failed"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Timestamps",
-            {
-                "fields": ("created_at", "modified_at"),
-                "classes": ("card",),
-            },
-        ),
+        card_fieldset("Machine", ("machine",)),
+        card_fieldset("Network", ("iface", "ip_public", "ip_local", "mac_address", "dns_server")),
+        card_fieldset("Location", ("hostname", "isp", "city", "region", "country")),
+        card_fieldset("Usage", ("num_uses_succeeded", "num_uses_failed")),
+        card_fieldset("Timestamps", ("created_at", "modified_at")),
     )
 
     list_filter = ("isp", "country", "region")
@@ -326,12 +220,6 @@ class NetworkInterfaceAdmin(BaseModelAdmin):
             iface.machine.hostname,
         )
 
-    @admin.display(description="Health", ordering="health")
-    def health_display(self, obj):
-        h = obj.health
-        color = "green" if h >= 80 else "orange" if h >= 50 else "red"
-        return format_html('<span style="color: {};">{}</span>', color, h)
-
 
 class BinaryAdmin(BaseModelAdmin):
     list_display = ("id", "created_at", "machine_info", "name", "binprovider", "version", "abspath", "sha256", "status", "health_display")
@@ -341,48 +229,12 @@ class BinaryAdmin(BaseModelAdmin):
     readonly_fields = ("created_at", "modified_at", "output_dir")
 
     fieldsets = (
-        (
-            "Binary Info",
-            {
-                "fields": ("name", "binproviders", "binprovider", "overrides"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Location",
-            {
-                "fields": ("machine", "abspath"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Version",
-            {
-                "fields": ("version", "sha256"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "State",
-            {
-                "fields": ("status", "retry_at", "output_dir"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Usage",
-            {
-                "fields": ("num_uses_succeeded", "num_uses_failed"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Timestamps",
-            {
-                "fields": ("created_at", "modified_at"),
-                "classes": ("card",),
-            },
-        ),
+        card_fieldset("Binary Info", ("name", "binproviders", "binprovider", "overrides")),
+        card_fieldset("Location", ("machine", "abspath")),
+        card_fieldset("Version", ("version", "sha256")),
+        card_fieldset("State", ("status", "retry_at", "output_dir")),
+        card_fieldset("Usage", ("num_uses_succeeded", "num_uses_failed")),
+        card_fieldset("Timestamps", ("created_at", "modified_at")),
     )
 
     list_filter = ("name", "binprovider", "status", "machine_id")
@@ -399,12 +251,6 @@ class BinaryAdmin(BaseModelAdmin):
             binary.machine.hostname,
         )
 
-    @admin.display(description="Health", ordering="health")
-    def health_display(self, obj):
-        h = obj.health
-        color = "green" if h >= 80 else "orange" if h >= 50 else "red"
-        return format_html('<span style="color: {};">{}</span>', color, h)
-
 
 class ProcessAdmin(BaseModelAdmin):
     list_display = (
@@ -463,34 +309,10 @@ class ProcessAdmin(BaseModelAdmin):
     )
 
     fieldsets = (
-        (
-            "Process Info",
-            {
-                "fields": ("machine", "archiveresult_link", "snapshot_link", "crawl_link", "status", "retry_at"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Command",
-            {
-                "fields": ("cmd_display", "pwd", "env_display", "timeout"),
-                "classes": ("card", "wide"),
-            },
-        ),
-        (
-            "Execution",
-            {
-                "fields": ("binary_link", "iface_link", "pid", "exit_code", "url"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Timing",
-            {
-                "fields": ("started_at", "ended_at", "duration_display"),
-                "classes": ("card",),
-            },
-        ),
+        card_fieldset("Process Info", ("machine", "archiveresult_link", "snapshot_link", "crawl_link", "status", "retry_at")),
+        card_fieldset("Command", ("cmd_display", "pwd", "env_display", "timeout"), wide=True),
+        card_fieldset("Execution", ("binary_link", "iface_link", "pid", "exit_code", "url")),
+        card_fieldset("Timing", ("started_at", "ended_at", "duration_display")),
         (
             "Output",
             {
@@ -498,13 +320,7 @@ class ProcessAdmin(BaseModelAdmin):
                 "classes": ("card", "wide", "collapse"),
             },
         ),
-        (
-            "Timestamps",
-            {
-                "fields": ("created_at", "modified_at"),
-                "classes": ("card",),
-            },
-        ),
+        card_fieldset("Timestamps", ("created_at", "modified_at")),
     )
 
     list_filter = ("status", "exit_code", "machine_id")
@@ -703,36 +519,9 @@ class ProcessAdmin(BaseModelAdmin):
     @admin.display(description="Output", ordering="archiveresult__output_size")
     def output_summary(self, process):
         try:
-            output_files = process.archiveresult.output_files or {}
+            file_count, total_bytes = process.archiveresult.output_file_stats()
         except Process.archiveresult.RelatedObjectDoesNotExist:
-            output_files = {}
-
-        if isinstance(output_files, str):
-            try:
-                output_files = json.loads(output_files)
-            except (json.JSONDecodeError, TypeError, ValueError):
-                output_files = {}
-
-        file_count = 0
-        total_bytes = 0
-
-        if isinstance(output_files, dict):
-            file_count = len(output_files)
-            items = output_files.values()
-        elif isinstance(output_files, (list, tuple, set)):
-            file_count = len(output_files)
-            items = output_files
-        else:
-            items = ()
-
-        for metadata in items:
-            if not isinstance(metadata, dict):
-                continue
-            size = metadata.get("size", 0)
-            try:
-                total_bytes += int(size or 0)
-            except (TypeError, ValueError):
-                continue
+            file_count, total_bytes = 0, 0
 
         file_label = "file" if file_count == 1 else "files"
         return format_html(
@@ -750,26 +539,26 @@ class ProcessAdmin(BaseModelAdmin):
             cmd = shlex.join(str(arg) for arg in process.cmd)
         else:
             cmd = str(process.cmd)
-        return _render_copy_block(cmd)
+        return render_copy_block(cmd)
 
     @admin.display(description="Environment")
     def env_display(self, process):
         env_text = env_to_dotenv_text(process.env)
         if not env_text:
             return "-"
-        return _render_copy_block(env_text, multiline=True)
+        return render_copy_block(env_text, multiline=True)
 
     @admin.display(description="Stdout")
     def stdout_display(self, process):
         if not process.stdout:
             return "-"
-        return _render_copy_block(process.stdout, multiline=True)
+        return render_copy_block(process.stdout, multiline=True)
 
     @admin.display(description="Stderr")
     def stderr_display(self, process):
         if not process.stderr:
             return "-"
-        return _render_copy_block(process.stderr, multiline=True)
+        return render_copy_block(process.stderr, multiline=True)
 
     @admin.display(description="ArchiveResult Output")
     def archiveresult_output_display(self, process):
@@ -779,7 +568,7 @@ class ProcessAdmin(BaseModelAdmin):
             return "-"
         if not output:
             return "-"
-        return _render_copy_block(output, multiline=True)
+        return render_copy_block(output, multiline=True)
 
 
 def register_admin(admin_site):

archivebox/machine/apps.py

@@ -10,14 +10,6 @@ class MachineConfig(AppConfig):
     label = "machine"  # Explicit label for migrations
     verbose_name = "Machine Info"
 
-    def ready(self):
-        """Import models to register state machines with the registry"""
-        import sys
-
-        # Skip during makemigrations to avoid premature state machine access
-        if "makemigrations" not in sys.argv:
-            from archivebox.machine import models  # noqa: F401
-
 
 def register_admin(admin_site):
     from archivebox.machine.admin import register_admin

archivebox/machine/detect.py

@@ -217,26 +217,27 @@ def get_isp_info(ip=None):
     }
 
 
+def get_local_interface(address: str) -> dict[str, str]:
+    """Describe the real host interface owning an IPv4 address."""
+    for interface, addresses in psutil.net_if_addrs().items():
+        if any(ip.address == address for ip in ip_addrs(addresses)):
+            return {
+                "iface": interface,
+                "mac_address": next((mac.address for mac in mac_addrs(addresses)), ""),
+                "ip_local": address,
+            }
+    raise ValueError(f"No host interface owns {address}")
+
+
 def get_host_network() -> dict[str, Any]:
-    default_gateway_local_ip = get_local_ip()
-    gateways = psutil.net_if_addrs()
-
-    for interface, ips in gateways.items():
-        for local_ip in ip_addrs(ips):
-            if default_gateway_local_ip == local_ip.address:
-                mac_address = next(mac_addrs(ips)).address
-                public_ip = get_public_ip()
-                return {
-                    "hostname": max([socket.gethostname(), platform.node()], key=len),
-                    "iface": interface,
-                    "mac_address": mac_address,
-                    "ip_local": local_ip.address,
-                    "ip_public": public_ip,
-                    # "is_behind_nat": local_ip.address != public_ip,
-                    **get_isp_info(public_ip),
-                }
-
-    raise Exception("Could not determine host network info")
+    interface = get_local_interface(get_local_ip())
+    public_ip = get_public_ip()
+    return {
+        "hostname": max([socket.gethostname(), platform.node()], key=len),
+        **interface,
+        "ip_public": public_ip,
+        **get_isp_info(public_ip),
+    }
 
 
 def get_os_info() -> dict[str, Any]:

archivebox/machine/models/__init__.py

@@ -0,0 +1,50 @@
+"""Machine, interface, binary, and process persistence with process-local caches."""
+
+from __future__ import annotations
+
+from .binaries import Binary as Binary
+from .binaries import BinaryManager as BinaryManager
+from .binaries import _canonical_binary_name as _canonical_binary_name
+from .binaries import _find_existing_binary_for_reference as _find_existing_binary_for_reference
+from .constants import (
+    BINARY_RECHECK_INTERVAL as BINARY_RECHECK_INTERVAL,
+)
+from .constants import (
+    MACHINE_RECHECK_INTERVAL as MACHINE_RECHECK_INTERVAL,
+)
+from .constants import (
+    NETWORK_INTERFACE_RECHECK_INTERVAL as NETWORK_INTERFACE_RECHECK_INTERVAL,
+)
+from .constants import (
+    PID_REUSE_WINDOW as PID_REUSE_WINDOW,
+)
+from .constants import (
+    PROCESS_PID_NAMESPACE_KEY as PROCESS_PID_NAMESPACE_KEY,
+)
+from .constants import (
+    PROCESS_RECHECK_INTERVAL as PROCESS_RECHECK_INTERVAL,
+)
+from .constants import (
+    PROCESS_TIMEOUT_GRACE as PROCESS_TIMEOUT_GRACE,
+)
+from .constants import (
+    PSUTIL_AVAILABLE as PSUTIL_AVAILABLE,
+)
+from .constants import (
+    START_TIME_TOLERANCE as START_TIME_TOLERANCE,
+)
+from .interfaces import NetworkInterface as NetworkInterface
+from .interfaces import NetworkInterfaceManager as NetworkInterfaceManager
+from .machines import Machine as Machine
+from .machines import MachineManager as MachineManager
+from .machines import _sanitize_machine_config as _sanitize_machine_config
+from .processes import Process as Process
+from .processes import ProcessManager as ProcessManager
+from .processes import _default_exit_code_for_unowned_process as _default_exit_code_for_unowned_process
+from .processes import _get_process_binary_env_keys as _get_process_binary_env_keys
+from .processes import get_current_pid_namespace as get_current_pid_namespace
+
+_CURRENT_MACHINE: Machine | None = None
+_CURRENT_INTERFACE: NetworkInterface | None = None
+_CURRENT_BINARIES: dict[str, Binary] = {}
+_CURRENT_PROCESS: Process | None = None

archivebox/machine/models/binaries.py

@@ -0,0 +1,425 @@
+from __future__ import annotations
+
+import sys
+import uuid
+from datetime import timedelta
+from pathlib import Path
+from typing import Any
+
+from django.db import models, transaction
+from django.utils import timezone
+from django.utils.functional import cached_property
+
+from archivebox.base_models.models import ModelWithHealthStats
+from archivebox.config import CONSTANTS
+from archivebox.config.common import rprint
+from archivebox.machine import models as state
+from archivebox.uuid_compat import CompactUUIDField, uuid7
+from archivebox.workers.models import ModelWithQueue
+
+from .constants import BINARY_RECHECK_INTERVAL
+from .machines import Machine
+
+
+def _find_existing_binary_for_reference(machine: Machine, reference: str) -> Binary | None:
+    reference = str(reference or "").strip()
+    if not reference:
+        return None
+
+    qs = Binary.objects.filter(machine=machine)
+
+    direct_match = qs.filter(abspath=reference).order_by("-modified_at").first()
+    if direct_match:
+        return direct_match
+
+    ref_name = Path(reference).name
+    if ref_name:
+        named_match = qs.filter(name=ref_name).order_by("-modified_at").first()
+        if named_match:
+            return named_match
+
+    return qs.filter(name=reference).order_by("-modified_at").first()
+
+
+def _canonical_binary_name(name: Any) -> str:
+    name = str(name or "").strip()
+    if "/" in name or "\\" in name or name.startswith("~"):
+        return Path(name).expanduser().name
+    return name
+
+
+class BinaryManager(models.Manager):
+    def get_from_db_or_cache(self, name: str, abspath: str = "", version: str = "", sha256: str = "", binprovider: str = "env") -> Binary:
+        """Get or create an Binary record from the database or cache."""
+        from archivebox.machine.models import Machine
+
+        cached = state._CURRENT_BINARIES.get(name)
+        if cached and timezone.now() < cached.modified_at + timedelta(seconds=BINARY_RECHECK_INTERVAL):
+            return cached
+        state._CURRENT_BINARIES[name], _ = self.update_or_create(
+            machine=Machine.current(),
+            name=name,
+            binprovider=binprovider,
+            version=version,
+            abspath=abspath,
+            sha256=sha256,
+        )
+        return state._CURRENT_BINARIES[name]
+
+    def get_valid_binary(self, name: str, machine: Machine | None = None) -> Binary | None:
+        """Get a valid Binary for the given name on the current machine, or None if not found."""
+        from archivebox.machine.models import Machine
+
+        machine = machine or Machine.current()
+        return (
+            self.filter(
+                machine=machine,
+                name__iexact=name,
+            )
+            .exclude(abspath="")
+            .exclude(abspath__isnull=True)
+            .order_by("-modified_at")
+            .first()
+        )
+
+
+class Binary(ModelWithHealthStats, ModelWithQueue):
+    """
+    Tracks a binary on a specific machine.
+
+    Simple queue lifecycle with 2 states:
+    - queued: Binary needs to be installed
+    - installed: Binary installed successfully (abspath, version, sha256 populated)
+
+    Installation is synchronous during queued→installed transition.
+    If installation fails, Binary stays in queued with retry_at set for later retry.
+
+    BinaryService claims queued rows with a conditional update, then run()
+    emits an abxpkg BinaryRequestEvent and persists the resolved installation.
+    The database row is the only lifecycle state; there is intentionally no
+    second in-memory state machine to reconcile after a worker interruption.
+    """
+
+    class StatusChoices(models.TextChoices):
+        QUEUED = "queued", "Queued"
+        INSTALLED = "installed", "Installed"
+
+    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    created_at = models.DateTimeField(default=timezone.now, db_index=True)
+    modified_at = models.DateTimeField(auto_now=True)
+    machine = models.ForeignKey(Machine, on_delete=models.CASCADE, null=False)
+
+    # Binary metadata
+    name = models.CharField(max_length=63, default="", null=False, blank=True, db_index=True)
+    binproviders = models.CharField(
+        max_length=127,
+        default="env",
+        null=False,
+        blank=True,
+        help_text="Comma-separated list of allowed providers: apt,brew,pip,npm,env",
+    )
+    overrides = models.JSONField(
+        default=dict,
+        blank=True,
+        help_text="Provider-specific overrides: {'apt': {'install_args': ['pkg']}, ...}",
+    )
+
+    # Installation results (populated after installation)
+    binprovider = models.CharField(
+        max_length=31,
+        default="",
+        null=False,
+        blank=True,
+        help_text="Provider that successfully installed this binary",
+    )
+    abspath = models.CharField(max_length=255, default="", null=False, blank=True)
+    version = models.CharField(max_length=32, default="", null=False, blank=True)
+    sha256 = models.CharField(max_length=64, default="", null=False, blank=True)
+
+    # Durable queue lifecycle fields
+    status = ModelWithQueue.StatusField(choices=StatusChoices.choices, default=StatusChoices.QUEUED, max_length=16)
+    retry_at = ModelWithQueue.RetryAtField(
+        default=timezone.now,
+        help_text="When to retry this binary installation",
+    )
+
+    # Health stats
+    num_uses_failed = models.PositiveIntegerField(default=0)
+    num_uses_succeeded = models.PositiveIntegerField(default=0)
+
+    machine_id: uuid.UUID
+
+    INITIAL_STATE = StatusChoices.QUEUED
+    ACTIVE_STATE = StatusChoices.QUEUED
+    FINAL_STATES = (StatusChoices.INSTALLED,)
+    FINAL_OR_ACTIVE_STATES = (*FINAL_STATES, ACTIVE_STATE)
+    active_state: str = StatusChoices.QUEUED
+    warn_on_save_outside_runner = False
+
+    objects = BinaryManager()  # pyright: ignore[reportIncompatibleVariableOverride]
+
+    class Meta(ModelWithHealthStats.Meta, ModelWithQueue.Meta):
+        app_label = "machine"
+        verbose_name = "Binary"
+        verbose_name_plural = "Binaries"
+        unique_together = (("machine", "name", "abspath", "version", "sha256"),)
+
+    def __str__(self) -> str:
+        return f"{self.name}@{self.binprovider}+{self.abspath}@{self.version}"
+
+    @property
+    def is_valid(self) -> bool:
+        """A binary is valid if it has a resolved path and is marked installed."""
+        return bool(self.abspath) and self.status == self.StatusChoices.INSTALLED
+
+    @property
+    def can_install(self) -> bool:
+        return bool(self.name and self.binproviders)
+
+    @cached_property
+    def binary_info(self) -> dict:
+        """Return info about the binary."""
+        return {
+            "name": self.name,
+            "abspath": self.abspath,
+            "version": self.version,
+            "binprovider": self.binprovider,
+            "is_valid": self.is_valid,
+        }
+
+    @property
+    def output_dir(self) -> Path:
+        """
+        Get output directory for this binary's hook logs.
+        Path: data/machines/{machine_uuid}/binaries/{binary_name}/{binary_uuid}
+        """
+        return CONSTANTS.DATA_DIR / "machines" / str(self.machine_id) / "binaries" / self.name / str(self.id)
+
+    def to_json(self) -> dict:
+        """
+        Convert Binary model instance to a JSON-serializable dict.
+        """
+        from archivebox.config import VERSION
+
+        is_installed = bool(self.abspath and self.version)
+        return {
+            "type": "Binary" if is_installed else "BinaryRequest",
+            "schema_version": VERSION,
+            "id": str(self.id),
+            "machine_id": str(self.machine_id),
+            "name": self.name,
+            "binproviders": self.binproviders,
+            "overrides": self.overrides,
+            "binprovider": self.binprovider,
+            "abspath": self.abspath,
+            "version": self.version,
+            "sha256": self.sha256,
+            "status": self.status,
+        }
+
+    @staticmethod
+    def from_json(record: dict[str, Any], overrides: dict[str, Any] | None = None):
+        """
+        Create/update Binary from JSON dict.
+
+        Handles two cases:
+        1. From binaries.json: creates queued binary with name, binproviders, overrides
+        2. From hook output: updates binary with abspath, version, sha256, binprovider
+
+        Args:
+            record: JSON dict with 'name' and either:
+                    - 'binproviders', 'overrides' (from binaries.json)
+                    - 'abspath', 'version', 'sha256', 'binprovider' (from hook output)
+            overrides: Not used
+
+        Returns:
+            Binary instance or None
+        """
+        from archivebox.machine.models import Machine
+
+        name = _canonical_binary_name(record.get("name"))
+        if not name:
+            return None
+
+        abspath, version = record.get("abspath"), record.get("version")
+        binproviders = record.get("binproviders")
+        installed = bool(abspath and version) and (bool(binproviders) or "binproviders" not in record)
+        if installed:
+            defaults = {
+                "abspath": abspath,
+                "version": version,
+                "sha256": record.get("sha256", ""),
+                "binprovider": record.get("binprovider", "env"),
+                "status": Binary.StatusChoices.INSTALLED,
+                "retry_at": None,
+            }
+            if binproviders:
+                defaults["binproviders"] = binproviders
+        elif "binproviders" in record or ("overrides" in record and not abspath):
+            binary_overrides = record.get("overrides", {})
+            defaults = {
+                "binproviders": record.get("binproviders", "env"),
+                "overrides": binary_overrides if isinstance(binary_overrides, dict) else {},
+                "status": Binary.StatusChoices.QUEUED,
+                "retry_at": timezone.now(),
+            }
+        else:
+            return None
+
+        binary, _ = Binary.objects.update_or_create(machine=Machine.current(), name=name, defaults=defaults)
+        if installed:
+            from archivebox.config.common import get_config
+
+            binary.symlink_to_lib_bin_after_commit(get_config().ABXPKG_LIB_DIR / "bin")
+        return binary
+
+    def run(self):
+        """
+        Execute binary installation through the ArchiveBox binary runner.
+        """
+        from archivebox.services.runner import run_binary
+
+        run_binary(str(self.id))
+
+    def install(self) -> bool:
+        """Run one synchronous installation attempt for a claimed Binary."""
+        if self.status == self.StatusChoices.INSTALLED:
+            return True
+        if not self.can_install:
+            return False
+
+        rprint(f"[cyan]      🔄 installing {self.name}[/cyan]", file=sys.stderr)
+        self.run()
+        self.refresh_from_db()
+        if self.status != self.StatusChoices.INSTALLED:
+            self.update_and_requeue(
+                retry_at=timezone.now() + timedelta(seconds=300),
+                status=self.StatusChoices.QUEUED,
+            )
+            self.increment_health_stats(success=False)
+            raise RuntimeError(f"Binary {self.name} installation failed")
+
+        self.update_and_requeue(retry_at=None, status=self.StatusChoices.INSTALLED)
+        self.increment_health_stats(success=True)
+        return True
+
+    def advance_lifecycle(self) -> bool:
+        """Advance the explicit binary lifecycle after its queue row is claimed."""
+        return self.install()
+
+    def install_claimed(self, *, lock_seconds: int = 600) -> bool:
+        if not self.claim_processing_lock(lock_seconds=lock_seconds):
+            return False
+        self.refresh_from_db()
+        return self.advance_lifecycle()
+
+    def cleanup(self):
+        """
+        Clean up background binary installation hooks.
+
+        Called after an installation attempt if needed (not typically used for binaries
+        since installations are foreground, but included for consistency).
+        """
+
+        # Clean up .pid files from output directory
+        output_dir = self.output_dir
+        if output_dir.exists():
+            for pid_file in output_dir.glob("**/*.pid"):
+                pid_file.unlink(missing_ok=True)
+
+    def symlink_to_lib_bin(self, lib_bin_dir: str | Path) -> Path | None:
+        """
+        Symlink this binary into a derived lib/bin directory for human-facing convenience.
+
+        After a binary is installed by any binprovider (pip, npm, brew, apt, etc),
+        we can optionally expose a flat convenience directory for shell users.
+        ArchiveBox/abx-dl runtime lookup must use the provider-specific ABXPKG_LIB_DIR
+        paths, not this indirection.
+
+        Args:
+            lib_bin_dir: Path to the derived convenience bin dir (e.g., /data/lib/bin)
+
+        Returns:
+            Path to the created symlink, or None if symlinking failed
+
+        Example:
+            >>> binary = Binary.objects.get(name='yt-dlp')
+            >>> binary.symlink_to_lib_bin('/data/lib/arm64-darwin/bin')
+            Path('/data/lib/arm64-darwin/bin/yt-dlp')
+        """
+        import sys
+        from pathlib import Path
+
+        if not self.abspath:
+            return None
+
+        binary_abspath = Path(self.abspath).resolve()
+        lib_bin_dir = Path(lib_bin_dir).resolve()
+        binary_parts = binary_abspath.parts
+        try:
+            app_index = next(index for index, part in enumerate(binary_parts) if part.endswith(".app"))
+        except StopIteration:
+            app_index = -1
+
+        # Create the derived convenience bin dir if it doesn't exist.
+        try:
+            lib_bin_dir.mkdir(parents=True, exist_ok=True)
+        except (OSError, PermissionError) as e:
+            print(f"Failed to create lib/bin convenience dir {lib_bin_dir}: {e}", file=sys.stderr)
+            return None
+
+        # Expose the canonical Binary.name in the convenience bin dir. Some providers point
+        # abspath at implementation files like cli.js or manifest.json; those
+        # are valid targets, but they are not user-facing binary names.
+        binary_name = _canonical_binary_name(self.name) or binary_abspath.name
+        symlink_path = lib_bin_dir / binary_name
+
+        if app_index != -1 and len(binary_parts) > app_index + 2 and binary_parts[app_index + 1 : app_index + 3] == ("Contents", "MacOS"):
+            if symlink_path.exists() or symlink_path.is_symlink():
+                try:
+                    symlink_path.unlink()
+                except (OSError, PermissionError) as e:
+                    print(f"Failed to remove existing file at {symlink_path}: {e}", file=sys.stderr)
+                    return None
+            return binary_abspath
+
+        # Remove existing symlink/file if it exists
+        if symlink_path.exists() or symlink_path.is_symlink():
+            try:
+                # Check if it's already pointing to the right place
+                if symlink_path.is_symlink() and symlink_path.resolve() == binary_abspath:
+                    # Already correctly symlinked, nothing to do
+                    return symlink_path
+
+                # Remove old symlink/file
+                symlink_path.unlink()
+            except (OSError, PermissionError) as e:
+                print(f"Failed to remove existing file at {symlink_path}: {e}", file=sys.stderr)
+                return None
+
+        # Create new symlink
+        try:
+            symlink_path.symlink_to(binary_abspath)
+            return symlink_path
+        except (OSError, PermissionError) as e:
+            print(f"Failed to create symlink {symlink_path} -> {binary_abspath}: {e}", file=sys.stderr)
+            return None
+
+    def symlink_to_lib_bin_after_commit(self, lib_bin_dir: str | Path) -> None:
+        """
+        Symlink after the current DB transaction commits.
+
+        Binary rows are projections of provider/hook state and are allowed to be
+        updated directly, but filesystem writes must not run while an outer
+        transaction is still open. Refetch after commit so the symlink points at
+        the committed row, not a possibly-rolled-back in-memory value.
+        """
+        binary_id = self.id
+        lib_bin_path = Path(lib_bin_dir)
+
+        def create_symlink() -> None:
+            binary = type(self).objects.filter(id=binary_id).first()
+            if binary is not None:
+                binary.symlink_to_lib_bin(lib_bin_path)
+
+        transaction.on_commit(create_symlink)

archivebox/machine/models/constants.py

@@ -0,0 +1,26 @@
+from datetime import timedelta
+from typing import TYPE_CHECKING, Any, cast
+
+_psutil: Any | None = None
+try:
+    import psutil as _psutil_import
+
+    PSUTIL_AVAILABLE = True
+except ImportError:
+    PSUTIL_AVAILABLE = False
+else:
+    _psutil = _psutil_import
+
+if TYPE_CHECKING:
+    import psutil
+else:
+    psutil = cast(Any, _psutil)
+
+MACHINE_RECHECK_INTERVAL = 7 * 24 * 60 * 60
+NETWORK_INTERFACE_RECHECK_INTERVAL = 1 * 60 * 60
+BINARY_RECHECK_INTERVAL = 1 * 30 * 60
+PROCESS_RECHECK_INTERVAL = 60  # Re-validate every 60 seconds
+PID_REUSE_WINDOW = timedelta(hours=24)  # Max age for considering a PID match valid
+PROCESS_TIMEOUT_GRACE = timedelta(seconds=30)  # Extra margin before force-cleaning timed-out RUNNING rows
+START_TIME_TOLERANCE = 5.0  # Seconds tolerance for start time matching
+PROCESS_PID_NAMESPACE_KEY = "_ARCHIVEBOX_PID_NAMESPACE"

archivebox/machine/models/interfaces.py

@@ -0,0 +1,102 @@
+from __future__ import annotations
+
+import uuid
+from datetime import timedelta
+
+from django.db import IntegrityError, models
+from django.utils import timezone
+
+from archivebox.base_models.models import ModelWithHealthStats
+from archivebox.machine import models as state
+from archivebox.uuid_compat import CompactUUIDField, uuid7
+
+from ..detect import get_host_network
+from .constants import NETWORK_INTERFACE_RECHECK_INTERVAL
+from .machines import Machine
+
+
+class NetworkInterfaceManager(models.Manager):
+    def current(self) -> NetworkInterface:
+        return NetworkInterface.current()
+
+
+class NetworkInterface(ModelWithHealthStats):
+    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    created_at = models.DateTimeField(default=timezone.now, db_index=True)
+    modified_at = models.DateTimeField(auto_now=True)
+    machine = models.ForeignKey(Machine, on_delete=models.CASCADE, default=None, null=False)
+    mac_address = models.CharField(max_length=17, default=None, null=False, editable=False)
+    ip_public = models.GenericIPAddressField(default=None, null=False, editable=False)
+    ip_local = models.GenericIPAddressField(default=None, null=False, editable=False)
+    dns_server = models.GenericIPAddressField(default=None, null=False, editable=False)
+    hostname = models.CharField(max_length=63, default="", null=False)
+    iface = models.CharField(max_length=15, default="", null=False)
+    isp = models.CharField(max_length=63, default="", null=False)
+    city = models.CharField(max_length=63, default="", null=False)
+    region = models.CharField(max_length=63, default="", null=False)
+    country = models.CharField(max_length=63, default="", null=False)
+    objects = NetworkInterfaceManager()  # pyright: ignore[reportIncompatibleVariableOverride]
+    machine_id: uuid.UUID
+
+    class Meta(ModelWithHealthStats.Meta):
+        app_label = "machine"
+        unique_together = (("machine", "ip_public", "ip_local", "dns_server"),)
+        constraints = [
+            models.UniqueConstraint(
+                fields=["machine", "ip_public", "ip_local", "dns_server"],
+                name="unique_network_interface_identity",
+            ),
+        ]
+
+    @classmethod
+    def current(cls, refresh: bool = False) -> NetworkInterface:
+        from archivebox.machine.models import Machine
+
+        machine = Machine.current(refresh=refresh)
+        if state._CURRENT_INTERFACE and state._CURRENT_INTERFACE.machine_id == machine.id:
+            if not refresh:
+                # Callers that pass refresh=False are asking for attribution to
+                # the currently known interface, not for public-IP/ISP probing.
+                # Maintenance paths create many short-lived services per run;
+                # expiring this in-memory object by age forced every crawl to
+                # hit external network APIs even though Process rows only need
+                # a stable existing FK. Active downloading paths opt into live
+                # detection with refresh=True.
+                return state._CURRENT_INTERFACE
+            if timezone.now() < state._CURRENT_INTERFACE.modified_at + timedelta(seconds=NETWORK_INTERFACE_RECHECK_INTERVAL):
+                return state._CURRENT_INTERFACE
+        state._CURRENT_INTERFACE = None
+
+        if not refresh:
+            state._CURRENT_INTERFACE = cls.objects.filter(machine=machine).order_by("-modified_at", "-created_at").first()
+            if state._CURRENT_INTERFACE is not None:
+                return state._CURRENT_INTERFACE
+
+        net_info = get_host_network()
+        lookup = dict(
+            machine=machine,
+            ip_public=net_info.pop("ip_public"),
+            ip_local=net_info.pop("ip_local"),
+            dns_server=net_info.pop("dns_server"),
+        )
+        state._CURRENT_INTERFACE = cls.objects.filter(**lookup).order_by("-modified_at", "-created_at").first()
+        if state._CURRENT_INTERFACE is None:
+            try:
+                state._CURRENT_INTERFACE = cls.objects.create(**lookup, **net_info)
+            except IntegrityError:
+                state._CURRENT_INTERFACE = cls.objects.filter(**lookup).order_by("-modified_at", "-created_at").first()
+                if state._CURRENT_INTERFACE is None:
+                    raise
+        else:
+            # Avoid update_or_create() here: command startup calls this before
+            # leadership handoff, and SQLite should not take a write lock unless
+            # the cached interface metadata is actually stale.
+            if refresh or timezone.now() >= state._CURRENT_INTERFACE.modified_at + timedelta(seconds=NETWORK_INTERFACE_RECHECK_INTERVAL):
+                updates = ["modified_at"]
+                for key, value in net_info.items():
+                    if state._CURRENT_INTERFACE.__dict__.get(key) != value:
+                        setattr(state._CURRENT_INTERFACE, key, value)
+                        updates.append(key)
+                if len(updates) > 1:
+                    state._CURRENT_INTERFACE.save(update_fields=updates)
+        return state._CURRENT_INTERFACE

archivebox/machine/models/machines.py

@@ -0,0 +1,281 @@
+from __future__ import annotations
+
+import socket
+from datetime import timedelta
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from django.db import models
+from django.utils import timezone
+
+from archivebox.base_models.models import ModelWithHealthStats, normalize_config_json_values
+from archivebox.machine import models as state
+from archivebox.uuid_compat import CompactUUIDField, uuid7
+
+from ..detect import get_host_guid, get_host_stats, get_os_info, get_vm_info
+from .constants import MACHINE_RECHECK_INTERVAL
+
+if TYPE_CHECKING:
+    from .interfaces import NetworkInterface
+
+
+def _sanitize_machine_config(config: dict[str, Any] | None, *, lib_dir: str | Path | None = None) -> dict[str, Any]:
+    """Validate ``Machine.config`` in place.
+
+    Drops stale ``*_BINARY`` overrides whose path no longer exists or whose
+    path falls outside of ``ABXPKG_LIB_DIR`` (so a binary uninstall or a lib_dir
+    move clears the override automatically). Non-``_BINARY`` keys
+    (``BASE_URL``, ``SERVER_SECURITY_MODE``, plugin tunables, etc.) are
+    pass-through — they're arbitrary config overrides and not ours to filter.
+    """
+    if not isinstance(config, dict):
+        return {}
+
+    sanitized = dict(config)
+    active_lib_dir = Path(lib_dir).expanduser().absolute() if lib_dir else None
+    for key, value in list(sanitized.items()):
+        if not str(key).endswith("_BINARY"):
+            continue
+        if not isinstance(value, str):
+            continue
+        value = value.strip()
+        if not value:
+            sanitized.pop(key, None)
+            continue
+        if "/" in value or value.startswith("~"):
+            try:
+                path = Path(value).expanduser()
+                if not path.exists():
+                    sanitized.pop(key, None)
+                    continue
+                if active_lib_dir is not None:
+                    resolved_path = path.absolute()
+                    try:
+                        resolved_path.relative_to(active_lib_dir)
+                    except ValueError:
+                        sanitized.pop(key, None)
+            except OSError:
+                sanitized.pop(key, None)
+    return sanitized
+
+
+class MachineManager(models.Manager):
+    def current(self) -> Machine:
+        return Machine.current()
+
+
+class Machine(ModelWithHealthStats):
+    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    created_at = models.DateTimeField(default=timezone.now, db_index=True)
+    modified_at = models.DateTimeField(auto_now=True)
+    guid = models.CharField(max_length=64, default=None, null=False, unique=True, editable=False)
+    hostname = models.CharField(max_length=63, default=None, null=False)
+    hw_in_docker = models.BooleanField(default=False, null=False)
+    hw_in_vm = models.BooleanField(default=False, null=False)
+    hw_manufacturer = models.CharField(max_length=63, default=None, null=False)
+    hw_product = models.CharField(max_length=63, default=None, null=False)
+    hw_uuid = models.CharField(max_length=255, default=None, null=False)
+    os_arch = models.CharField(max_length=15, default=None, null=False)
+    os_family = models.CharField(max_length=15, default=None, null=False)
+    os_platform = models.CharField(max_length=63, default=None, null=False)
+    os_release = models.CharField(max_length=63, default=None, null=False)
+    os_kernel = models.CharField(max_length=255, default=None, null=False)
+    stats = models.JSONField(default=dict, null=True, blank=True)
+    config = models.JSONField(
+        default=dict,
+        null=True,
+        blank=True,
+        help_text="Machine-specific config overrides.",
+    )
+    num_uses_failed = models.PositiveIntegerField(default=0)
+    num_uses_succeeded = models.PositiveIntegerField(default=0)
+
+    objects = MachineManager()  # pyright: ignore[reportIncompatibleVariableOverride]
+    networkinterface_set: models.Manager[NetworkInterface]
+
+    class Meta(ModelWithHealthStats.Meta):
+        app_label = "machine"
+
+    @classmethod
+    def current(cls, refresh: bool = False) -> Machine:
+        if refresh:
+            state._CURRENT_MACHINE = None
+        if state._CURRENT_MACHINE:
+            if timezone.now() < state._CURRENT_MACHINE.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL):
+                # Reconciliation can save() and clear the shared cache; keep
+                # this call's instance so we never return None after that save.
+                machine = state._CURRENT_MACHINE
+                # One-time-per-process reconciliation between ArchiveBox.conf
+                # and Machine.config. Fast-path: bool check + early-return when
+                # the sync has already run, so the cached-machine return path
+                # stays sub-microsecond.
+                try:
+                    from archivebox.config.collection import sync_machine_and_file
+
+                    sync_machine_and_file(machine)
+                except Exception:
+                    pass
+                return machine
+            else:
+                state._CURRENT_MACHINE = None
+
+        host_guid = get_host_guid()
+        # save() intentionally invalidates state._CURRENT_MACHINE so config edits
+        # made through other instances become visible in this process. Host
+        # refresh (after seven days), sanitation, and reconciliation can all
+        # save here, so the working instance must not live only in that cache.
+        # Otherwise a routine refresh leaves startup dereferencing None.
+        try:
+            machine = cls.objects.get(guid=host_guid)
+        except cls.DoesNotExist:
+            config = {}
+            try:
+                from archivebox.config.collection import _coerce_from_str_dict, _load_file_config_dict
+
+                file_config, _file_mtime = _load_file_config_dict()
+                config = _coerce_from_str_dict(file_config)
+            except Exception:
+                config = {}
+            machine = cls.objects.create(
+                guid=host_guid,
+                hostname=socket.gethostname(),
+                config=config,
+                **get_os_info(),
+                **get_vm_info(),
+                stats=get_host_stats(),
+            )
+        else:
+            if timezone.now() >= machine.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL):
+                for key, value in {
+                    "hostname": socket.gethostname(),
+                    **get_os_info(),
+                    **get_vm_info(),
+                    "stats": get_host_stats(),
+                }.items():
+                    setattr(machine, key, value)
+                machine.save(
+                    update_fields=[
+                        "hostname",
+                        "hw_in_docker",
+                        "hw_in_vm",
+                        "hw_manufacturer",
+                        "hw_product",
+                        "hw_uuid",
+                        "os_arch",
+                        "os_family",
+                        "os_platform",
+                        "os_release",
+                        "os_kernel",
+                        "stats",
+                        "modified_at",
+                    ],
+                )
+        machine = cls._sanitize_config(machine)
+        # Same one-time sync as the cached-return path. Triggers here on the
+        # very first ``Machine.current()`` call in a process before any
+        # cached return can occur.
+        try:
+            from archivebox.config.collection import sync_machine_and_file
+
+            sync_machine_and_file(machine)
+        except Exception:
+            pass
+        # Publish only after all save-capable steps; publishing earlier would
+        # let those steps invalidate the cache we are in the middle of filling.
+        state._CURRENT_MACHINE = machine
+        return machine
+
+    @classmethod
+    def _sanitize_config(cls, machine: Machine) -> Machine:
+        from archivebox.config.common import get_config
+
+        sanitized = _sanitize_machine_config(machine.config, lib_dir=get_config(include_machine=False).ABXPKG_LIB_DIR)
+        current = machine.config or {}
+        if sanitized != current:
+            machine.config = sanitized
+            machine.save(update_fields=["config", "modified_at"])
+        return machine
+
+    def to_json(self) -> dict:
+        """
+        Convert Machine model instance to a JSON-serializable dict.
+        """
+        from archivebox.config import VERSION
+        from archivebox.config.common import redact_sensitive_config
+
+        return {
+            "type": "Machine",
+            "schema_version": VERSION,
+            "id": str(self.id),
+            "guid": self.guid,
+            "hostname": self.hostname,
+            "hw_in_docker": self.hw_in_docker,
+            "hw_in_vm": self.hw_in_vm,
+            "hw_manufacturer": self.hw_manufacturer,
+            "hw_product": self.hw_product,
+            "hw_uuid": self.hw_uuid,
+            "os_arch": self.os_arch,
+            "os_family": self.os_family,
+            "os_platform": self.os_platform,
+            "os_kernel": self.os_kernel,
+            "os_release": self.os_release,
+            "stats": self.stats,
+            "config": redact_sensitive_config(self.config),
+        }
+
+    @staticmethod
+    def from_json(record: dict[str, Any], overrides: dict[str, Any] | None = None):
+        """
+        Update Machine config from JSON dict.
+
+        Args:
+            record: JSON dict with 'config': {key: value} patch
+            overrides: Not used
+
+        Returns:
+            Machine instance or None
+        """
+        config_patch = _sanitize_machine_config(record.get("config"))
+        if config_patch:
+            machine = Machine.current()
+            machine.config = _sanitize_machine_config(machine.config)
+            machine.config.update(config_patch)
+            machine.save(update_fields=["config"])
+            return machine
+        return None
+
+    def save(self, *args, **kwargs):
+        normalized_config = normalize_config_json_values(self.config)
+        if normalized_config != self.config:
+            self.config = normalized_config
+            update_fields = kwargs.get("update_fields")
+            if update_fields is not None:
+                kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "config"]))
+
+        # Drop the ``Machine.current()`` module-level cache on every save so
+        # config edits (admin form, from_json, etc.) become live in the same
+        # process without waiting out the 7-day ``MACHINE_RECHECK_INTERVAL``.
+        # The cache reads ``state._CURRENT_MACHINE.modified_at`` and that value
+        # never moves forward on the cached object even when the row is
+        # updated in the DB, so without this we'd keep serving stale
+        # ``machine.config`` (incl. ``BASE_URL``) until the worker restarts.
+        update_fields = kwargs.get("update_fields")
+        super().save(*args, **kwargs)
+        if state._CURRENT_MACHINE is not None and state._CURRENT_MACHINE.pk == self.pk:
+            state._CURRENT_MACHINE = None
+
+        # Mirror Machine.config into ArchiveBox.conf so the two stores stay
+        # 1:1. Skipped when ``update_fields`` is set and doesn't touch
+        # ``config`` (binary autodetection + ``hostname``/``stats`` refreshes
+        # save fields we don't need to disk-mirror, which keeps hot paths
+        # zero-IO). Errors during mirroring are swallowed: the DB write
+        # already succeeded and we don't want a config-file write hiccup to
+        # turn a routine save into a 500.
+        if update_fields is not None and "config" not in update_fields:
+            return
+        try:
+            from archivebox.config.collection import mirror_machine_config_to_file
+
+            mirror_machine_config_to_file(self.config)
+        except Exception:
+            pass

archivebox/misc/checks.py

@@ -255,7 +255,7 @@ def check_data_dir_permissions(config=None, **config_kwargs):
             "    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#permissions]https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#permissions[/link]",
         )
         STDERR.print(
-            "    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#puid--pgid]https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#puid--pgid[/link]",
+            "    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#puid--pgid]https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#puid--pgid[/link]",
         )
         STDERR.print(
             "    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#filesystem-doesnt-support-fsync-eg-network-mounts]https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#filesystem-doesnt-support-fsync-eg-network-mounts[/link]",
@@ -406,3 +406,86 @@ def check_lib_dir(lib_dir: Path | None = None, throw=False, quiet=False, must_ex
                 f"ABXPKG_LIB_DIR={lib_dir} is invalid, ArchiveBox is unable to use it and dependencies will fail to install.",
             ) from e
     return False
+
+
+MIN_SERVER_WORKER_AVAILABLE_MEMORY_BYTES = 256 * 1024 * 1024
+
+
+MIN_DEPENDENCY_INSTALL_AVAILABLE_MEMORY_BYTES = MIN_SERVER_WORKER_AVAILABLE_MEMORY_BYTES
+
+
+MIN_CRAWL_AVAILABLE_MEMORY_BYTES = 512 * 1024 * 1024
+
+
+def _read_cgroup_limit(path: Path) -> int | None:
+    try:
+        raw_value = path.read_text().strip()
+        return None if raw_value == "max" else int(raw_value)
+    except (OSError, ValueError):
+        return None
+
+
+def effective_available_memory_bytes() -> int:
+    import psutil
+
+    available = psutil.virtual_memory().available + psutil.swap_memory().free
+
+    cgroup_root = Path("/sys/fs/cgroup")
+    memory_max = _read_cgroup_limit(cgroup_root / "memory.max")
+    memory_current = _read_cgroup_limit(cgroup_root / "memory.current")
+    swap_max = _read_cgroup_limit(cgroup_root / "memory.swap.max")
+    swap_current = _read_cgroup_limit(cgroup_root / "memory.swap.current")
+    if memory_max is not None and memory_current is not None:
+        cgroup_available = max(memory_max - memory_current, 0)
+        if swap_max is not None and swap_current is not None:
+            cgroup_available += max(swap_max - swap_current, 0)
+        available = min(available, cgroup_available)
+
+    return available
+
+
+def _require_available_memory(operation: str, idle_message: str, available_bytes: int, required_bytes: int) -> None:
+    from archivebox.misc.logging import STDERR
+
+    if available_bytes >= required_bytes:
+        return
+
+    available_mib = available_bytes // (1024 * 1024)
+    required_mib = required_bytes // (1024 * 1024)
+    STDERR.print(f"[red][X] Not enough available memory to {operation} safely.[/red]")
+    STDERR.print(
+        f"    Available RAM + swap: {available_mib} MiB; at least {required_mib} MiB must be free before this operation.",
+    )
+    STDERR.print(idle_message)
+    STDERR.print("    Use a host/container with at least 1 GB RAM or configure swap, then run the same command again.")
+    raise SystemExit(1)
+
+
+def require_server_worker_memory(available_bytes: int | None = None) -> None:
+    available_bytes = effective_available_memory_bytes() if available_bytes is None else available_bytes
+    _require_available_memory(
+        "start ArchiveBox",
+        "    No server, runner, or Sonic workers were started.",
+        available_bytes,
+        MIN_SERVER_WORKER_AVAILABLE_MEMORY_BYTES,
+    )
+
+
+def require_dependency_install_memory(available_bytes: int | None = None) -> None:
+    available_bytes = effective_available_memory_bytes() if available_bytes is None else available_bytes
+    _require_available_memory(
+        "install ArchiveBox dependencies",
+        "    No plugin dependency installers were started.",
+        available_bytes,
+        MIN_DEPENDENCY_INSTALL_AVAILABLE_MEMORY_BYTES,
+    )
+
+
+def require_crawl_memory(available_bytes: int | None = None) -> None:
+    available_bytes = effective_available_memory_bytes() if available_bytes is None else available_bytes
+    _require_available_memory(
+        "archive a crawl",
+        "    No crawl, runner, or Sonic workers were started.",
+        available_bytes,
+        MIN_CRAWL_AVAILABLE_MEMORY_BYTES,
+    )

archivebox/misc/db.py

@@ -350,7 +350,9 @@ def sqlite_lock_holders(db_path: Path = CONSTANTS.DATABASE_FILE) -> list[str]:
     for proc in psutil.process_iter(["pid", "ppid", "name", "cmdline", "status"]):
         try:
             open_files = proc.open_files()
-        except (psutil.AccessDenied, psutil.NoSuchProcess, psutil.ZombieProcess):
+        except (psutil.AccessDenied, psutil.NoSuchProcess, psutil.ZombieProcess, RuntimeError):
+            # macOS libproc can report an uninspectable process as RuntimeError.
+            # Optional holder diagnostics must not abort the database operation.
             continue
         for open_file in open_files:
             try:
@@ -380,29 +382,40 @@ def log_sqlite_lock_holders(console: Any, *, db_path: Path = CONSTANTS.DATABASE_
 def sqlite_lock_error(error: BaseException) -> bool:
     from django.db import OperationalError as DjangoOperationalError
 
-    return isinstance(error, (SQLiteOperationalError, DjangoOperationalError)) and "database is locked" in str(error).lower()
+    message = str(error).lower()
+    return isinstance(error, (SQLiteOperationalError, DjangoOperationalError)) and (
+        "database is locked" in message or "database table is locked" in message
+    )
 
 
 def retry_sqlite_locks(action: Callable[[], Any], *, label: str, stderr: TextIO | None = None) -> Any:
+    from django.conf import settings
     from django.db import OperationalError, connections
     from rich.console import Console
 
     console = Console(file=stderr or None, stderr=stderr is None)
+    started_at = time.monotonic()
+    retry_timeout = settings.CONFIG.SQLITE_LOCK_RETRY_TIMEOUT
+    retry_interval = settings.CONFIG.SQLITE_LOCK_RETRY_INTERVAL
     while True:
         try:
             return action()
-        except OperationalError as err:
-            if "database is locked" not in str(err).lower():
-                raise
-        except SQLiteOperationalError as err:
+        except (OperationalError, SQLiteOperationalError) as err:
             if not sqlite_lock_error(err):
                 raise
+            if retry_timeout:
+                remaining = retry_timeout - (time.monotonic() - started_at)
+                if remaining <= 0:
+                    raise
+                sleep_for = min(retry_interval, remaining)
+            else:
+                sleep_for = retry_interval
 
         connections.close_all()
-        console.print(f"[yellow][*] SQLite database is locked while {label}; retrying in 5s...[/yellow]")
+        console.print(f"[yellow][*] SQLite database is locked while {label}; retrying in {sleep_for:g}s...[/yellow]")
         log_sqlite_lock_holders(console)
         with console.status("[yellow]Waiting for SQLite database lock to clear...[/yellow]", spinner="dots"):
-            time.sleep(5.0)
+            time.sleep(sleep_for)
 
 
 @contextmanager
@@ -651,3 +664,24 @@ def apply_migrations(
         out1 = retry_sqlite_locks(migrate, label="applying migrations")
 
         return [line.strip() for line in out1.readlines() if line.strip()]
+
+
+def uuid_ref_query(field_name: str, ref: str):
+    from uuid import UUID
+    from django.db.models import Q
+
+    raw_ref = str(ref or "").strip()
+    query = Q(**{f"{field_name}__startswith": raw_ref})
+    if raw_ref:
+        query |= Q(**{f"{field_name}__icontains": raw_ref})
+    try:
+        parsed_uuid = UUID(raw_ref)
+    except (TypeError, ValueError):
+        normalized_ref = raw_ref.replace("-", "")
+        if normalized_ref and normalized_ref != raw_ref:
+            query |= Q(**{f"{field_name}__startswith": normalized_ref})
+            query |= Q(**{f"{field_name}__icontains": normalized_ref})
+    else:
+        query |= Q(**{field_name: parsed_uuid})
+        query |= Q(**{f"{field_name}__startswith": parsed_uuid.hex})
+    return query

archivebox/misc/hashing.py

@@ -1,260 +0,0 @@
-# Bootable hashing helpers (file content + mtime/size cache, mime detection).
-# MUST NOT import archivebox.config, archivebox.core, or Django — stdlib only.
-
-import hashlib
-import mimetypes
-from functools import lru_cache
-from pathlib import Path
-from collections.abc import Callable
-from datetime import datetime
-
-
-@lru_cache(maxsize=1024)
-def _cached_file_hash(filepath: str, size: int, mtime: float) -> str:
-    """Internal function to calculate file hash with cache key based on path, size and mtime."""
-    sha256_hash = hashlib.sha256()
-
-    with open(filepath, "rb") as f:
-        for chunk in iter(lambda: f.read(4096), b""):
-            sha256_hash.update(chunk)
-
-    return sha256_hash.hexdigest()
-
-
-@lru_cache(maxsize=10)
-def hash_file(file_path: Path, pwd: Path | None = None) -> str:
-    """Calculate SHA256 hash of a file with caching based on path, size and mtime."""
-    pwd = Path(pwd) if pwd else None
-    file_path = Path(file_path)
-    if not file_path.is_absolute():
-        file_path = pwd / file_path if pwd else file_path.absolute()
-
-    abs_path = file_path.resolve()
-    stat_info = abs_path.stat()
-
-    return _cached_file_hash(
-        str(abs_path),
-        stat_info.st_size,
-        stat_info.st_mtime,
-    )
-
-
-@lru_cache(maxsize=10)
-def get_dir_hashes(dir_path: Path, pwd: Path | None = None, filter_func: Callable | None = None, max_depth: int = -1) -> dict[str, str]:
-    """Calculate SHA256 hashes for all files and directories recursively."""
-    pwd = Path(pwd) if pwd else None
-    dir_path = Path(dir_path)
-    if not dir_path.is_absolute():
-        dir_path = pwd / dir_path if pwd else dir_path.absolute()
-
-    if not dir_path.is_dir():
-        raise ValueError(f"Not a directory: {dir_path}")
-    if max_depth < -1:
-        raise ValueError(f"max_depth must be >= -1, got {max_depth}")
-
-    # Get all files recursively
-    all_files = get_dir_entries(
-        dir_path,
-        pwd=pwd,
-        recursive=True,
-        include_files=True,
-        include_dirs=False,
-        filter_func=filter_func,
-    )
-
-    hashes: dict[str, str] = {}
-    hashable_summary = []
-
-    # Calculate hashes for all files
-    for subfile in all_files:
-        subfile_path = dir_path / subfile
-        sha256_hash = hash_file(subfile_path)
-        hashes[subfile] = sha256_hash
-        hashable_summary.append(f"{sha256_hash}  ./{subfile}")
-
-    # Calculate hashes for all directories
-    subdirs = get_dir_entries(
-        dir_path,
-        pwd=pwd,
-        recursive=True,
-        include_files=False,
-        include_dirs=True,
-        include_hidden=False,
-        filter_func=filter_func,
-        max_depth=max_depth,
-    )
-
-    for subdir in subdirs:
-        subdir_path = dir_path / subdir
-        subdir_hashes = get_dir_hashes(
-            subdir_path,
-            filter_func=filter_func,
-            max_depth=0,
-        )
-        hashes[subdir] = subdir_hashes["."]
-
-    # Filter results by max_depth
-    if max_depth >= 0:
-        hashes = {path: value for path, value in hashes.items() if len(Path(path).parts) <= max_depth + 1}
-
-    # Calculate root directory hash
-    hashable_summary.sort()
-    root_sha256 = hashlib.sha256("\n".join(hashable_summary).encode()).hexdigest()
-    hashes["."] = root_sha256
-
-    return hashes
-
-
-@lru_cache(maxsize=128)
-def get_dir_entries(
-    dir_path: Path,
-    pwd: Path | None = None,
-    recursive: bool = True,
-    include_files: bool = True,
-    include_dirs: bool = True,
-    include_hidden: bool = False,
-    filter_func: Callable | None = None,
-    max_depth: int = -1,
-) -> tuple[str, ...]:
-    """Get filtered list of directory entries."""
-    pwd = Path(pwd) if pwd else None
-    dir_path = Path(dir_path)
-    if not dir_path.is_absolute():
-        dir_path = pwd / dir_path if pwd else dir_path.absolute()
-
-    results = []
-
-    def process_path(path: Path, depth: int):
-        if not include_hidden and path.name.startswith("."):
-            return False
-        if max_depth >= 0 and depth > max_depth:
-            return False
-        if filter_func:
-            info = {
-                "abspath": str(path.absolute()),
-                "relpath": str(path.relative_to(dir_path)),
-            }
-            if not filter_func(info):
-                return False
-        return True
-
-    for path in dir_path.rglob("*") if recursive else dir_path.glob("*"):
-        current_depth = len(path.relative_to(dir_path).parts)
-
-        if path.is_file() and include_files and process_path(path, current_depth):
-            results.append(str(path.relative_to(dir_path)))
-        elif path.is_dir() and include_dirs and process_path(path, current_depth):
-            results.append(str(path.relative_to(dir_path)))
-
-        if not recursive:
-            break
-
-    return tuple(sorted(results))  # Make immutable for caching
-
-
-@lru_cache(maxsize=1024)
-def get_dir_sizes(dir_path: Path, pwd: Path | None = None, **kwargs) -> dict[str, int]:
-    """Calculate sizes for all files and directories recursively."""
-    sizes: dict[str, int] = {}
-    hashes = get_dir_hashes(dir_path, pwd=pwd, **kwargs)
-    dir_path = Path(dir_path)
-
-    for path_key in hashes:
-        full_path = dir_path / path_key
-        if full_path.is_file():
-            sizes[path_key] = full_path.stat().st_size
-        else:
-            total = 0
-            for file_path in full_path.rglob("*"):
-                if file_path.is_file() and not file_path.name.startswith("."):
-                    total += file_path.stat().st_size
-            sizes[path_key + "/"] = total
-
-    return sizes
-
-
-@lru_cache(maxsize=10)
-def get_dir_info(dir_path: Path, pwd: Path | None = None, filter_func: Callable | None = None, max_depth: int = -1) -> dict:
-    """Get detailed information about directory contents including hashes and sizes."""
-    pwd = Path(pwd) if pwd else None
-    dir_path = Path(dir_path)
-    if not dir_path.is_absolute():
-        dir_path = pwd / dir_path if pwd else dir_path.absolute()
-
-    hashes = get_dir_hashes(dir_path, pwd=pwd, filter_func=filter_func, max_depth=max_depth)
-    sizes = get_dir_sizes(str(dir_path), pwd=pwd, filter_func=filter_func, max_depth=max_depth)
-
-    num_total_subpaths = sum(1 for name in hashes if name != ".")
-    details = {}
-
-    for filename, sha256_hash in sorted(hashes.items()):
-        abs_path = (dir_path / filename).resolve()
-        stat_info = abs_path.stat()
-        num_subpaths = sum(1 for p in hashes if p.startswith(filename + "/"))
-        is_dir = abs_path.is_dir()
-        if is_dir:
-            mime_type = "inode/directory"
-            basename = abs_path.name
-            extension = ""
-            num_bytes = sizes[filename + "/"]
-            if filename == ".":
-                num_subpaths = num_total_subpaths
-            else:
-                filename += "/"
-                num_subpaths = num_subpaths
-        else:  # is_file
-            num_subpaths = None
-            mime_type = mimetypes.guess_type(str(abs_path))[0]
-            extension = abs_path.suffix
-            basename = abs_path.name.rsplit(extension, 1)[0]
-            num_bytes = sizes[filename]
-
-        details[filename] = {
-            "basename": basename,
-            "mime_type": mime_type,
-            "extension": extension,
-            "num_subpaths": num_subpaths,
-            "num_bytes": num_bytes,
-            "hash_sha256": sha256_hash,
-            "created_at": datetime.fromtimestamp(stat_info.st_ctime).isoformat(),
-            "modified_at": datetime.fromtimestamp(stat_info.st_mtime).isoformat(),
-        }
-
-        if filter_func and not filter_func(details[filename]):
-            del details[filename]
-
-    return details
-
-
-if __name__ == "__main__":
-    import json
-
-    dir_info = get_dir_info(Path("."), max_depth=6)
-    with open(".hashes.json", "w") as f:
-        json.dump(dir_info, f, indent=4)
-    print("Wrote .hashes.json")
-
-# Example output:
-# {
-#     ".": {
-#         "basename": "misc",
-#         "mime_type": "inode/directory",
-#         "extension": "",
-#         "num_subpaths": 25,
-#         "num_bytes": 214677,
-#         "hash_sha256": "addfacf88b2ff6b564846415fb7b21dcb7e63ee4e911bc0aec255ee354958530",
-#         "created_at": "2024-12-04T00:08:38.537449",
-#         "modified_at": "2024-12-04T00:08:38.537449"
-#     },
-#     "__init__.py": {
-#         "basename": "__init__",
-#         "mime_type": "text/x-python",
-#         "extension": ".py",
-#         "num_subpaths": null,
-#         "num_bytes": 32,
-#         "hash_sha256": "b0e5e7ff17db3b60535cf664282787767c336e3e203a43e21b6326c6fe457551",
-#         "created_at": "2024-10-08T00:51:41.001359",
-#         "modified_at": "2024-10-08T00:51:41.001359"
-#     },
-#     ...
-# }

archivebox/misc/jsonl.py

@@ -32,7 +32,6 @@ import sys
 import json
 from typing import Any, TextIO
 from collections.abc import Iterable, Iterator
-from pathlib import Path
 
 
 # Type constants for JSONL records
@@ -120,19 +119,6 @@ def read_stdin(stream: TextIO | None = None) -> Iterator[dict[str, Any]]:
             yield record
 
 
-def read_file(path: Path) -> Iterator[dict[str, Any]]:
-    """
-    Read JSONL or plain URLs from a file.
-
-    Yields parsed records as dicts.
-    """
-    with open(path) as f:
-        for line in f:
-            record = parse_line(line)
-            if record:
-                yield record
-
-
 def read_args_or_stdin(args: Iterable[str], stream: TextIO | None = None) -> Iterator[dict[str, Any]]:
     """
     Read from CLI arguments if provided, otherwise from stdin.

archivebox/misc/logging_util.py

@@ -5,133 +5,21 @@ __package__ = "archivebox"
 # references Django ORM types. For pre-bootstrap logging primitives use
 # misc/logging.py, which has no archivebox or Django dependencies.
 
-import re
 import os
 import sys
 import time
 
-from math import log
-from multiprocessing import Process
 from pathlib import Path
 
-from datetime import datetime, timezone
-from typing import Any, Optional, TYPE_CHECKING, cast
+from typing import Any, TYPE_CHECKING, cast
 
 if TYPE_CHECKING:
-    from archivebox.core.models import Snapshot
+    pass
 
 from rich import print
 
 from archivebox.config import CONSTANTS
-from archivebox.config.common import get_config
 from archivebox.misc.util import enforce_types
-from archivebox.misc.logging import ANSI
-
-
-class TimedProgress:
-    """Show a progress bar and measure elapsed time until .end() is called"""
-
-    def __init__(self, seconds, prefix="", config=None, **config_kwargs):
-
-        config = config or get_config(**config_kwargs)
-        self.SHOW_PROGRESS = config.SHOW_PROGRESS
-        self.ANSI = config.ANSI
-        self.TERM_WIDTH = config.TERM_WIDTH
-
-        if self.SHOW_PROGRESS:
-            self.p = Process(target=progress_bar, args=(seconds, prefix, self.ANSI))
-            self.p.start()
-
-        self.stats = {"start_ts": datetime.now(timezone.utc), "end_ts": None}
-
-    def end(self):
-        """immediately end progress, clear the progressbar line, and save end_ts"""
-
-        end_ts = datetime.now(timezone.utc)
-        self.stats["end_ts"] = end_ts
-
-        if self.SHOW_PROGRESS:
-            # terminate if we havent already terminated
-            try:
-                # kill the progress bar subprocess
-                try:
-                    self.p.close()  # must be closed *before* its terminnated
-                except (KeyboardInterrupt, SystemExit):
-                    print()
-                    raise
-                except BaseException:  # lgtm [py/catch-base-exception]
-                    pass
-                self.p.terminate()
-                time.sleep(0.1)
-                # sometimes the timer doesn't terminate properly, then blocks at the join until
-                # the full time has elapsed. sending a kill tries to avoid that.
-                try:
-                    self.p.kill()
-                except Exception:
-                    pass
-
-                # clear whole terminal line
-                try:
-                    sys.stdout.write("\r{}{}\r".format((" " * self.TERM_WIDTH), self.ANSI["reset"]))
-                except (OSError, BrokenPipeError):
-                    # ignore when the parent proc has stopped listening to our stdout
-                    pass
-            except ValueError:
-                pass
-
-
-@enforce_types
-def progress_bar(seconds: int, prefix: str = "", ANSI: dict[str, str] = ANSI, config=None, **config_kwargs) -> None:
-    """show timer in the form of progress bar, with percentage and seconds remaining"""
-    output_buf = sys.stdout or sys.__stdout__ or sys.stderr or sys.__stderr__
-    chunk = "█" if output_buf and output_buf.encoding.upper() == "UTF-8" else "#"
-    config = config or get_config(**config_kwargs)
-    last_width = config.TERM_WIDTH
-    chunks = last_width - len(prefix) - 20  # number of progress chunks to show (aka max bar width)
-    try:
-        for s in range(seconds * chunks):
-            max_width = config.TERM_WIDTH
-            if max_width < last_width:
-                # when the terminal size is shrunk, we have to write a newline
-                # otherwise the progress bar will keep wrapping incorrectly
-                sys.stdout.write("\r\n")
-                sys.stdout.flush()
-            chunks = max_width - len(prefix) - 20
-            pct_complete = s / chunks / seconds * 100
-            log_pct = (log(pct_complete or 1, 10) / 2) * 100  # everyone likes faster progress bars ;)
-            bar_width = round(log_pct / (100 / chunks))
-            last_width = max_width
-
-            # ████████████████████           0.9% (1/60sec)
-            sys.stdout.write(
-                "\r{}{}{}{} {}% ({}/{}sec)".format(
-                    prefix,
-                    ANSI["green" if pct_complete < 80 else "lightyellow"],
-                    (chunk * bar_width).ljust(chunks),
-                    ANSI["reset"],
-                    round(pct_complete, 1),
-                    round(s / chunks),
-                    seconds,
-                ),
-            )
-            sys.stdout.flush()
-            time.sleep(1 / chunks)
-
-        # ██████████████████████████████████ 100.0% (60/60sec)
-        sys.stdout.write(
-            "\r{}{}{}{} {}% ({}/{}sec)".format(
-                prefix,
-                ANSI["red"],
-                chunk * chunks,
-                ANSI["reset"],
-                100.0,
-                seconds,
-                seconds,
-            ),
-        )
-        sys.stdout.flush()
-    except (KeyboardInterrupt, BrokenPipeError):
-        print()
 
 
 def log_list_started(filter_patterns: list[str] | None, filter_type: str):
@@ -236,152 +124,6 @@ def format_duration(seconds: float) -> str:
         return f"{hours}hr {minutes}min" if minutes else f"{hours}hr"
 
 
-@enforce_types
-def truncate_url(url: str, max_length: int = 60) -> str:
-    """Truncate URL to max_length, keeping domain and adding ellipsis."""
-    if len(url) <= max_length:
-        return url
-    # Try to keep the domain and beginning of path
-    if "://" in url:
-        protocol, rest = url.split("://", 1)
-        if "/" in rest:
-            domain, path = rest.split("/", 1)
-            available = max_length - len(protocol) - len(domain) - 6  # for "://", "/", "..."
-            if available > 10:
-                return f"{protocol}://{domain}/{path[:available]}..."
-    # Fallback: just truncate
-    return url[: max_length - 3] + "..."
-
-
-@enforce_types
-def log_worker_event(
-    worker_type: str,
-    event: str,
-    indent_level: int = 0,
-    pid: int | None = None,
-    worker_id: str | None = None,
-    url: str | None = None,
-    plugin: str | None = None,
-    metadata: dict[str, Any] | None = None,
-    error: Exception | None = None,
-) -> None:
-    """
-    Log a worker event with structured metadata and indentation.
-
-    Args:
-        worker_type: Type of worker (Orchestrator, CrawlWorker, SnapshotWorker)
-        event: Event name (Starting, Completed, Failed, etc.)
-        indent_level: Indentation level (0=Orchestrator, 1=CrawlWorker, 2=SnapshotWorker)
-        pid: Process ID
-        worker_id: Worker ID (UUID for workers)
-        url: URL being processed (for SnapshotWorker)
-        plugin: Plugin name (for hook processes)
-        metadata: Dict of metadata to show in curly braces
-        error: Exception if event is an error
-    """
-    indent = "    " * indent_level
-
-    from rich.markup import escape
-
-    # Build worker identifier (without URL/plugin)
-    worker_parts = [worker_type]
-    # Don't add pid/worker_id for DB operations (they happen in whatever process is running)
-    if pid and worker_type != "DB":
-        worker_parts.append(f"pid={pid}")
-    if worker_id and worker_type in ("CrawlWorker", "Orchestrator") and worker_type != "DB":
-        worker_parts.append(f"id={worker_id}")
-
-    # Build worker label parts for brackets (shown inside brackets)
-    worker_label_base = worker_parts[0]
-    worker_bracket_content = ", ".join(worker_parts[1:]) if len(worker_parts) > 1 else None
-
-    # Build URL/plugin display (shown AFTER the label, outside brackets)
-    url_extractor_parts = []
-    if url:
-        url_extractor_parts.append(f"url: {escape(url)}")
-    if plugin:
-        url_extractor_parts.append(f"extractor: {escape(plugin)}")
-
-    url_extractor_str = " | ".join(url_extractor_parts) if url_extractor_parts else ""
-
-    # Build metadata string
-    metadata_str = ""
-    if metadata:
-        # Format metadata nicely
-        meta_parts = []
-        for k, v in metadata.items():
-            if isinstance(v, float):
-                # Format floats nicely (durations, sizes)
-                if "duration" in k.lower():
-                    meta_parts.append(f"{k}: {format_duration(v)}")
-                elif "size" in k.lower():
-                    meta_parts.append(f"{k}: {printable_filesize(int(v))}")
-                else:
-                    meta_parts.append(f"{k}: {v:.2f}")
-            elif isinstance(v, int):
-                # Format integers - check if it's a size
-                if "size" in k.lower() or "bytes" in k.lower():
-                    meta_parts.append(f"{k}: {printable_filesize(v)}")
-                else:
-                    meta_parts.append(f"{k}: {v}")
-            elif isinstance(v, (list, tuple)):
-                meta_parts.append(f"{k}: {len(v)}")
-            else:
-                meta_parts.append(f"{k}: {v}")
-        metadata_str = " | ".join(meta_parts)
-
-    # Determine color based on event
-    color = "white"
-    if event in ("Starting...", "Started", "STARTED", "Started in background"):
-        color = "green"
-    elif event.startswith("Created"):
-        color = "cyan"  # DB creation events
-    elif event in ("Completed", "COMPLETED", "All work complete"):
-        color = "blue"
-    elif event in ("Failed", "ERROR", "Failed to spawn worker"):
-        color = "red"
-    elif event in ("Shutting down", "SHUTDOWN"):
-        color = "grey53"
-
-    # Build final message
-    error_str = f" {type(error).__name__}: {error}" if error else ""
-    from archivebox.misc.logging import CONSOLE, STDERR
-    from rich.text import Text
-
-    # Create a Rich Text object for proper formatting
-    # Text.append() treats content as literal (no markup parsing)
-    text = Text()
-    text.append(indent)
-    text.append(worker_label_base, style=color)
-
-    # Add bracketed content if present (using Text.append to avoid markup issues)
-    if worker_bracket_content:
-        text.append("[", style=color)
-        text.append(worker_bracket_content, style=color)
-        text.append("]", style=color)
-
-    text.append(f" {event}{error_str}", style=color)
-
-    # Add URL/plugin info first (more important)
-    if url_extractor_str:
-        text.append(f" | {url_extractor_str}")
-
-    # Then add other metadata
-    if metadata_str:
-        text.append(f" | {metadata_str}")
-
-    # Stdout is reserved for JSONL records whenever commands are piped together.
-    # Route worker/DB progress to stderr in non-TTY contexts so pipelines like
-    # `archivebox snapshot list | archivebox run` keep stdout machine-readable.
-    output_console = CONSOLE if sys.stdout.isatty() else STDERR
-    output_console.print(text, soft_wrap=True)
-
-
-@enforce_types
-def printable_folders(folders: dict[str, Optional["Snapshot"]], with_headers: bool = False) -> str:
-    return "\n".join(f'{folder} {snapshot and snapshot.url} "{snapshot and snapshot.title}"' for folder, snapshot in folders.items())
-
-
 @enforce_types
 def printable_config(config: dict, prefix: str = "") -> str:
     return f"\n{prefix}".join(f"{key}={val}" for key, val in config.items() if not (isinstance(val, dict) or callable(val)))
@@ -432,32 +174,93 @@ def printable_folder_status(name: str, folder: dict) -> str:
     )
 
 
-@enforce_types
-def printable_dependency_version(name: str, dependency: dict) -> str:
-    color, symbol, note, version = "red", "X", "invalid", "?"
+def _warn_background_cleanup(context: str, err: BaseException) -> None:
+    from archivebox.misc.logging import STDERR
 
-    if dependency["enabled"]:
-        if dependency["is_valid"]:
-            color, symbol, note = "green", "√", "valid"
+    STDERR.print(f"[yellow][!] {context}: {err!s}[/yellow]")
 
-            parsed_version_num = re.search(r"[\d\.]+", dependency["version"])
-            if parsed_version_num:
-                version = f"v{parsed_version_num[0]}"
-    else:
-        color, symbol, note, version = "lightyellow", "-", "disabled", "-"
 
-    path = pretty_path(dependency["path"])
+def tail_multiple_worker_logs(log_files: list[str], follow=True, proc=None, keep_running=None):
+    """Tail multiple log files simultaneously, interleaving their output.
 
-    return " ".join(
-        (
-            ANSI[color],
-            symbol,
-            ANSI["reset"],
-            name.ljust(21),
-            version.ljust(14),
-            ANSI[color],
-            note.ljust(8),
-            ANSI["reset"],
-            path.ljust(76),
-        ),
-    )
+    Args:
+        log_files: List of log file paths to tail
+        follow: Whether to keep following (True) or just read existing content (False)
+        proc: Optional subprocess.Popen object - stop tailing when this process exits
+    """
+    import re
+    from archivebox.config.common import rprint as print
+
+    # Convert relative paths to absolute paths
+    log_paths = []
+    for log_file in log_files:
+        log_path = Path(log_file)
+        if not log_path.is_absolute():
+            log_path = CONSTANTS.DATA_DIR / log_path
+
+        # Create log file if it doesn't exist
+        if not log_path.exists():
+            log_path.parent.mkdir(parents=True, exist_ok=True)
+            log_path.touch()
+
+        log_paths.append(log_path)
+
+    # Open all log files
+    file_handles = []
+    for log_path in log_paths:
+        try:
+            f = log_path.open()
+            # Seek to end - only show NEW logs from now on, not old logs
+            f.seek(0, 2)  # Go to end
+
+            file_handles.append((log_path, f))
+            print(f"    [tailing {log_path.name}]")
+        except OSError as e:
+            sys.stderr.write(f"Warning: Could not open {log_path}: {e}\n")
+
+    if not file_handles:
+        sys.stderr.write("No log files could be opened\n")
+        return
+
+    print()
+
+    try:
+        while follow:
+            if keep_running is not None and not keep_running():
+                print("\n[newer ArchiveBox process is now running the orchestrator and server]")
+                return "transferred"
+
+            # Check if the monitored process has exited
+            if proc is not None and proc.poll() is not None:
+                print(f"\n[server process exited with code {proc.returncode}]")
+                return "exited"
+
+            had_output = False
+            # Read ALL available lines from all files (not just one per iteration)
+            for log_path, f in file_handles:
+                while True:
+                    line = f.readline()
+                    if not line:
+                        break  # No more lines available in this file
+                    had_output = True
+                    # Strip ANSI codes if present (supervisord does this but just in case)
+                    line_clean = re.sub(r"\x1b\[[0-9;]*m", "", line.rstrip())
+                    if line_clean:
+                        print(line_clean)
+
+            # Small sleep to avoid busy-waiting (only when no output)
+            if not had_output:
+                time.sleep(0.05)
+
+    except (KeyboardInterrupt, BrokenPipeError, OSError):
+        return "interrupted"  # Let the caller handle the cleanup message
+    except SystemExit:
+        return "interrupted"
+    finally:
+        # Close all file handles
+        for _, f in file_handles:
+            try:
+                f.close()
+            except OSError as err:
+                _warn_background_cleanup("Could not close worker log file", err)
+    return "stopped"

archivebox/misc/replay_preview.py

@@ -0,0 +1,319 @@
+"""Transform captured text, Markdown, and images into replay documents.
+
+HTTP caching, range requests, and replay security headers belong to serve_static;
+these helpers only render content and resolve references to captured responses.
+"""
+
+import html
+import posixpath
+import re
+from pathlib import Path
+from urllib.parse import quote, urljoin
+
+from markdown_it import MarkdownIt
+
+IMG_SRC_ATTR_RE = re.compile(r'(<img\b[^>]*?\s(?:src|data-src)=["\'])([^"\']+)(["\'])', re.IGNORECASE)
+
+
+TRANSFORMED_HTML_PREVIEW_STYLE = """<style id="archivebox-static-html-preview-style">
+html {
+    width: 100%;
+    min-width: 100%;
+    background: #fff;
+}
+body {
+    box-sizing: border-box;
+    width: min(100%, 72rem);
+    max-width: none;
+    min-height: 100vh;
+    margin: 0 auto;
+    padding: clamp(1rem, 3vw, 2rem);
+    background: #fff;
+    color: #111827;
+    font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
+    line-height: 1.55;
+}
+body > * {
+    max-width: 100%;
+}
+img:not([width]):not([height]) {
+    max-width: min(100%, 12rem);
+    max-height: 12rem;
+    width: auto;
+    height: auto;
+    object-fit: contain;
+}
+a > img:not([width]):not([height]) {
+    max-width: min(100%, 2.5rem);
+    max-height: 2.5rem;
+}
+</style>"""
+
+
+MARKDOWN_INLINE_LINK_RE = re.compile(r"\[([^\]]+)\]\(([^)\s]+(?:\([^)]*\)[^)\s]*)*)\)")
+
+
+HTML_BODY_RE = re.compile(r"<body[^>]*>(.*)</body>", flags=re.IGNORECASE | re.DOTALL)
+
+
+def _extract_markdown_candidate(text: str) -> str:
+    candidate = text
+    body_match = HTML_BODY_RE.search(candidate)
+    if body_match:
+        candidate = body_match.group(1)
+    candidate = re.sub(r"^\s*<p[^>]*>", "", candidate, flags=re.IGNORECASE)
+    candidate = re.sub(r"</p>\s*$", "", candidate, flags=re.IGNORECASE)
+    return candidate.strip()
+
+
+def _looks_like_markdown(text: str) -> bool:
+    lower = text.lower()
+    if "<html" in lower and "<head" in lower and "</body>" in lower:
+        return False
+    md_markers = 0
+    md_markers += len(re.findall(r"^\s{0,3}#{1,6}\s+\S", text, flags=re.MULTILINE))
+    md_markers += len(re.findall(r"^\s*[-*+]\s+\S", text, flags=re.MULTILINE))
+    md_markers += len(re.findall(r"^\s*\d+\.\s+\S", text, flags=re.MULTILINE))
+    md_markers += text.count("[TOC]")
+    md_markers += len(MARKDOWN_INLINE_LINK_RE.findall(text))
+    md_markers += text.count("\n---") + text.count("\n***")
+    return md_markers >= 6
+
+
+def _render_text_preview_document(text: str, title: str) -> str:
+    escaped_title = html.escape(title)
+    escaped_text = html.escape(text)
+    return f"""<!doctype html>
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <title>{escaped_title}</title>
+    <style>
+        :root {{
+            color-scheme: dark;
+        }}
+        html, body {{
+            margin: 0;
+            padding: 0;
+            background: #111;
+            color: #f3f3f3;
+            font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", monospace;
+        }}
+        .archivebox-text-preview-header {{
+            position: sticky;
+            top: 0;
+            z-index: 1;
+            padding: 10px 14px;
+            font-size: 12px;
+            line-height: 1.4;
+            color: #bbb;
+            background: rgba(17, 17, 17, 0.96);
+            border-bottom: 1px solid rgba(255, 255, 255, 0.08);
+            backdrop-filter: blur(8px);
+        }}
+        .archivebox-text-preview {{
+            margin: 0;
+            padding: 14px;
+            white-space: pre-wrap;
+            word-break: break-word;
+            tab-size: 2;
+            line-height: 1.45;
+            font-size: 13px;
+        }}
+    </style>
+</head>
+<body>
+    <div class="archivebox-text-preview-header">{escaped_title}</div>
+    <pre class="archivebox-text-preview">{escaped_text}</pre>
+</body>
+</html>"""
+
+
+def _render_image_preview_document(image_url: str, title: str) -> str:
+    escaped_title = html.escape(title)
+    escaped_url = html.escape(image_url, quote=True)
+    return f"""<!doctype html>
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <title>{escaped_title}</title>
+    <style>
+        :root {{
+            color-scheme: dark;
+        }}
+        html, body {{
+            margin: 0;
+            padding: 0;
+            width: 100%;
+            min-height: 100%;
+            background: #fff;
+        }}
+        body {{
+            overflow: auto;
+        }}
+        .archivebox-image-preview {{
+            width: 100%;
+            min-width: 100%;
+            min-height: 100vh;
+            display: flex;
+            flex-direction: column;
+            align-items: center;
+            justify-content: flex-start;
+            box-sizing: border-box;
+        }}
+        .archivebox-image-preview img {{
+            display: block;
+            width: auto;
+            max-width: 100%;
+            height: auto;
+            margin: 0 auto;
+        }}
+    </style>
+</head>
+<body>
+    <div class="archivebox-image-preview">
+        <img src="{escaped_url}" alt="{escaped_title}">
+    </div>
+</body>
+</html>"""
+
+
+def _encoded_responses_image_url(image_url: str, page_url: str | None) -> str | None:
+    raw_url = str(image_url or "").strip()
+    if not raw_url or raw_url.startswith(("#", "data:", "blob:", "about:", "javascript:")):
+        return None
+
+    absolute_url = urljoin(page_url or "", raw_url)
+    if not absolute_url.startswith(("http://", "https://")):
+        return None
+
+    return quote(absolute_url, safe="").replace("%", "_")
+
+
+def _index_responses_paths_for_html_images(
+    snapshot_root: Path,
+    html_rel_path: str,
+    encoded_urls: set[str],
+) -> dict[str, str]:
+    responses_root = snapshot_root / "responses"
+    if not encoded_urls or not responses_root.is_dir():
+        return {}
+
+    best_matches: dict[str, str] = {}
+    image_matches: set[str] = set()
+    image_suffixes = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".ico"}
+    for candidate in responses_root.rglob("*"):
+        if not candidate.is_file():
+            continue
+        try:
+            rel_path = candidate.relative_to(snapshot_root)
+        except ValueError:
+            continue
+        relative_match = posixpath.relpath(rel_path.as_posix(), start=posixpath.dirname(html_rel_path) or ".")
+        candidate_name = candidate.name
+        is_image = candidate.suffix.lower() in image_suffixes
+        for encoded_url in encoded_urls:
+            if encoded_url in image_matches or f"__GET__{encoded_url}" not in candidate_name:
+                continue
+            best_matches[encoded_url] = relative_match
+            if is_image:
+                # Preserve the old rglob behavior: the first image match wins,
+                # otherwise the last matching response file is used.
+                image_matches.add(encoded_url)
+    return best_matches
+
+
+def _rewrite_html_image_sources_to_responses(
+    html_text: str,
+    snapshot_root: Path,
+    html_rel_path: str,
+    page_url: str | None,
+) -> tuple[str, int]:
+    encoded_urls_by_src: dict[str, str | None] = {}
+    for match in IMG_SRC_ATTR_RE.finditer(html_text):
+        image_url = html.unescape(match.group(2))
+        if image_url not in encoded_urls_by_src:
+            encoded_urls_by_src[image_url] = _encoded_responses_image_url(image_url, page_url)
+
+    response_paths = _index_responses_paths_for_html_images(
+        snapshot_root,
+        html_rel_path,
+        {encoded_url for encoded_url in encoded_urls_by_src.values() if encoded_url},
+    )
+    rewrites = 0
+
+    def replace_src(match: re.Match[str]) -> str:
+        nonlocal rewrites
+        image_url = html.unescape(match.group(2))
+        encoded_url = encoded_urls_by_src.get(image_url)
+        local_path = response_paths.get(encoded_url or "")
+        if not local_path:
+            return match.group(0)
+        rewrites += 1
+        return f"{match.group(1)}{html.escape(local_path, quote=True)}{match.group(3)}"
+
+    return IMG_SRC_ATTR_RE.sub(replace_src, html_text), rewrites
+
+
+def _rewrite_html_image_sources_for_request(
+    request,
+    html_text: str,
+    document_root: Path | None,
+    html_rel_path: str,
+) -> tuple[str, int]:
+    if not document_root:
+        return html_text, 0
+    return _rewrite_html_image_sources_to_responses(
+        html_text,
+        document_root,
+        html_rel_path,
+        request.__dict__.get("archivebox_snapshot_url"),
+    )
+
+
+def _apply_transformed_html_preview_style(html_text: str) -> str:
+    if "archivebox-static-html-preview-style" in html_text:
+        return html_text
+    if re.search(r"</head\s*>", html_text, flags=re.IGNORECASE):
+        return re.sub(r"</head\s*>", f"{TRANSFORMED_HTML_PREVIEW_STYLE}\\g<0>", html_text, count=1, flags=re.IGNORECASE)
+    return f"{TRANSFORMED_HTML_PREVIEW_STYLE}\n{html_text}"
+
+
+def _render_markdown(markdown_text: str) -> str:
+    """Render archived Markdown with stable heading links and optional [TOC]."""
+    parser = MarkdownIt("commonmark").enable("table")
+    tokens = parser.parse(markdown_text)
+    headings = []
+    for index, token in enumerate(tokens):
+        if token.type == "heading_open":
+            title = tokens[index + 1].content
+            slug = re.sub(r"[^A-Za-z0-9]+", "-", title).strip("-") or "section"
+            token.attrSet("id", slug)
+            headings.append(f'<li class="toc-level-{token.tag[1:]}"><a href="#{slug}">{html.escape(title)}</a></li>')
+    toc = '<nav class="toc"><ul>' + "".join(headings) + "</ul></nav>"
+    # Replace only a standalone TOC paragraph, leaving code and inline mentions intact.
+    for index in range(len(tokens) - 2):
+        opening, content, closing = tokens[index : index + 3]
+        if opening.type == "paragraph_open" and content.content == "[TOC]" and closing.type == "paragraph_close":
+            opening.hidden = closing.hidden = True
+            content.type = "html_block"
+            content.content = toc
+    return parser.renderer.render(tokens, parser.options, {})
+
+
+def _render_markdown_document(markdown_text: str) -> str:
+    body = _render_markdown(markdown_text)
+    wrapped = (
+        '<!doctype html><html><head><meta charset="utf-8">'
+        '<meta name="viewport" content="width=device-width,initial-scale=1">'
+        "<style>body{max-width:900px;margin:24px auto;padding:0 16px;"
+        "font-family:system-ui,-apple-system,Segoe UI,Roboto,Helvetica,Arial,sans-serif;"
+        "line-height:1.55;} img{max-width:100%;} pre{background:#f6f6f6;padding:12px;overflow:auto;}"
+        ".toc ul{list-style:none;padding-left:0;} .toc li{margin:4px 0;}</style>"
+        "</head><body>"
+        f"{body}"
+        "</body></html>"
+    )
+    return wrapped

archivebox/misc/serve_static.py

@@ -1,21 +1,18 @@
 import asyncio
 import html
-import importlib
+import io
 import json
 import mimetypes
 import os
 import posixpath
-import queue
 import re
 import stat
 import sys
-import threading
-import time
 import zipfile
-from collections.abc import Callable
 from datetime import UTC, datetime
 from pathlib import Path
-from urllib.parse import quote, urlencode, urljoin
+from functools import partial
+from urllib.parse import urlencode
 
 from abx_plugins.plugins.archivewebpage import replay_preview as archivewebpage_replay
 from django.contrib.staticfiles import finders
@@ -28,23 +25,10 @@ from django.utils.translation import gettext as _
 from django.views import static
 
 from archivebox.config.common import get_config
+from archivebox.misc import replay_preview
 from archivebox.misc.logging_util import printable_filesize
 
 _HASHES_CACHE: dict[Path, tuple[float, dict[str, str]]] = {}
-IMG_SRC_ATTR_RE = re.compile(r'(<img\b[^>]*?\s(?:src|data-src)=["\'])([^"\']+)(["\'])', re.IGNORECASE)
-TRANSFORMED_HTML_PREVIEW_STYLE = """<style id="archivebox-static-html-preview-style">
-img:not([width]):not([height]) {
-    max-width: min(100%, 12rem);
-    max-height: 12rem;
-    width: auto;
-    height: auto;
-    object-fit: contain;
-}
-a > img:not([width]):not([height]) {
-    max-width: min(100%, 2.5rem);
-    max-height: 2.5rem;
-}
-</style>"""
 
 
 def _load_hash_map(snapshot_dir: Path) -> dict[str, str] | None:
@@ -122,33 +106,25 @@ def _safe_zip_stem(name: str) -> str:
     return safe_name or "archivebox"
 
 
-class _StreamingQueueWriter:
-    """Expose a write-only file-like object so zipfile can stream into a queue."""
+class _ZipBuffer(io.RawIOBase):
+    """A non-seekable ZIP destination drained after each source block."""
 
-    def __init__(self, output_queue: queue.Queue[bytes | BaseException | object]) -> None:
-        self.output_queue = output_queue
+    def __init__(self):
+        self.pending = bytearray()
         self.position = 0
 
-    def write(self, data: bytes) -> int:
-        if data:
-            self.output_queue.put(data)
-            self.position += len(data)
+    def write(self, data):
+        self.pending.extend(data)
+        self.position += len(data)
         return len(data)
 
-    def tell(self) -> int:
+    def tell(self):
         return self.position
 
-    def flush(self) -> None:
-        return None
-
-    def close(self) -> None:
-        return None
-
-    def writable(self) -> bool:
-        return True
-
-    def seekable(self) -> bool:
-        return False
+    def drain(self):
+        if self.pending:
+            yield bytes(self.pending)
+            self.pending.clear()
 
 
 def _iter_visible_files(root: Path):
@@ -160,6 +136,42 @@ def _iter_visible_files(root: Path):
             yield Path(current_root) / filename
 
 
+def _iter_directory_zip(fullpath: Path, root_name: str):
+    # Reading a block and yielding its compressed bytes in the same iterator
+    # provides backpressure and closes all files when a download disconnects.
+    with _ZipBuffer() as buffer:
+        with zipfile.ZipFile(buffer, mode="w", compression=zipfile.ZIP_DEFLATED, compresslevel=6) as archive:
+            for entry in _iter_visible_files(fullpath):
+                info = zipfile.ZipInfo.from_file(entry, (Path(root_name) / entry.relative_to(fullpath)).as_posix())
+                info.compress_type = archive.compression
+                info.compress_level = archive.compresslevel
+                with entry.open("rb") as source, archive.open(info, "w") as destination:
+                    while chunk := source.read(64 * 1024):
+                        destination.write(chunk)
+                        yield from buffer.drain()
+                yield from buffer.drain()
+        yield from buffer.drain()
+
+
+async def _stream_async(stream):
+    # Django ASGI buffers synchronous iterators. Advance off the event loop,
+    # waiting for an in-flight read before closing its iterator on cancellation.
+    iterator = iter(stream)
+    try:
+        while True:
+            read = asyncio.create_task(asyncio.to_thread(next, iterator, None))
+            try:
+                chunk = await asyncio.shield(read)
+            except asyncio.CancelledError:
+                await read
+                raise
+            if chunk is None:
+                break
+            yield chunk
+    finally:
+        iterator.close()
+
+
 def _build_directory_zip_response(
     fullpath: Path,
     path: str,
@@ -169,74 +181,11 @@ def _build_directory_zip_response(
     config=None,
 ) -> StreamingHttpResponse:
     root_name = _safe_zip_stem(fullpath.name or Path(path).name or "archivebox")
-    sentinel = object()
-    output_queue: queue.Queue[bytes | BaseException | object] = queue.Queue(maxsize=8)
-    initial_chunk_target = 64 * 1024
-    initial_chunk_wait = 0.05
+    stream = _iter_directory_zip(fullpath, root_name)
 
-    def build_zip() -> None:
-        # zipfile wants a write-only file object. Feed those bytes straight into
-        # a queue so the response can stream them out as soon as they are ready.
-        writer = _StreamingQueueWriter(output_queue)
-        try:
-            with zipfile.ZipFile(writer, mode="w", compression=zipfile.ZIP_DEFLATED, compresslevel=6) as zip_file:
-                for entry in _iter_visible_files(fullpath):
-                    rel_parts = entry.relative_to(fullpath).parts
-                    arcname = Path(root_name, *rel_parts).as_posix()
-                    zip_file.write(entry, arcname)
-        except (OSError, RuntimeError, TypeError, ValueError, zipfile.BadZipFile) as err:
-            output_queue.put(err)
-        finally:
-            output_queue.put(sentinel)
-
-    threading.Thread(target=build_zip, name=f"zip-stream-{root_name}", daemon=True).start()
-
-    def iter_zip_chunks():
-        # Emit a meaningful first chunk quickly so browsers show the download
-        # immediately instead of waiting on dozens of tiny ZIP header writes.
-        first_chunk = bytearray()
-        initial_deadline = time.monotonic() + initial_chunk_wait
-
-        while True:
-            timeout = max(initial_deadline - time.monotonic(), 0) if len(first_chunk) < initial_chunk_target else None
-            try:
-                chunk = output_queue.get(timeout=timeout) if timeout is not None else output_queue.get()
-            except queue.Empty:
-                if first_chunk:
-                    yield bytes(first_chunk)
-                    first_chunk.clear()
-                    continue
-                chunk = output_queue.get()
-
-            if chunk is sentinel:
-                if first_chunk:
-                    yield bytes(first_chunk)
-                break
-            if isinstance(chunk, BaseException):
-                raise chunk
-            if len(first_chunk) < initial_chunk_target:
-                first_chunk.extend(chunk)
-                if len(first_chunk) >= initial_chunk_target or time.monotonic() >= initial_deadline:
-                    yield bytes(first_chunk)
-                    first_chunk.clear()
-                continue
-            yield chunk
-
-    async def stream_zip_async():
-        # Django ASGI buffers sync StreamingHttpResponse iterators by consuming
-        # them into a list. Drive the same sync iterator from a worker thread so
-        # Daphne can send each chunk as it arrives instead of buffering the ZIP.
-        iterator = iter(iter_zip_chunks())
-        while True:
-            chunk = await asyncio.to_thread(next, iterator, None)
-            if chunk is None:
-                break
-            yield chunk
-
-    response = StreamingHttpResponse(
-        stream_zip_async() if use_async_stream else iter_zip_chunks(),
-        content_type="application/zip",
-    )
+    response = StreamingHttpResponse(stream, content_type="application/zip")
+    if use_async_stream:
+        response.streaming_content = _stream_async(stream)
     response.headers["Content-Disposition"] = f'attachment; filename="{root_name}.zip"'
     response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=60, stale-while-revalidate=300"
     response.headers["Last-Modified"] = http_date(fullpath.stat().st_mtime)
@@ -250,18 +199,6 @@ def _build_directory_zip_response(
     )
 
 
-async def _stream_ranged_file_async(ranged_file: "RangedFileReader"):
-    iterator = iter(ranged_file)
-    try:
-        while True:
-            chunk = await asyncio.to_thread(next, iterator, None)
-            if chunk is None:
-                break
-            yield chunk
-    finally:
-        ranged_file.close()
-
-
 def _render_directory_index(request, path: str, fullpath: Path) -> HttpResponse:
     try:
         template = loader.select_template(
@@ -336,17 +273,6 @@ mimetypes.add_type("image/svg+xml", ".svg")
 mimetypes.add_type("multipart/related", ".mhtml")
 mimetypes.add_type("multipart/related", ".mht")
 
-try:
-    _markdown = importlib.import_module("markdown").markdown
-except ImportError:
-    _markdown: Callable[..., str] | None = None
-
-MARKDOWN_INLINE_LINK_RE = re.compile(r"\[([^\]]+)\]\(([^)\s]+(?:\([^)]*\)[^)\s]*)*)\)")
-MARKDOWN_INLINE_IMAGE_RE = re.compile(r"!\[([^\]]*)\]\(([^)]+)\)")
-MARKDOWN_BOLD_RE = re.compile(r"\*\*([^*]+)\*\*")
-MARKDOWN_ITALIC_RE = re.compile(r"(?<!\*)\*([^*]+)\*(?!\*)")
-HTML_TAG_RE = re.compile(r"<[A-Za-z][^>]*>")
-HTML_BODY_RE = re.compile(r"<body[^>]*>(.*)</body>", flags=re.IGNORECASE | re.DOTALL)
 RISKY_REPLAY_MIMETYPES = {
     "text/html",
     "application/xhtml+xml",
@@ -360,231 +286,6 @@ RISKY_REPLAY_MARKERS = (
 )
 
 
-def _extract_markdown_candidate(text: str) -> str:
-    candidate = text
-    body_match = HTML_BODY_RE.search(candidate)
-    if body_match:
-        candidate = body_match.group(1)
-    candidate = re.sub(r"^\s*<p[^>]*>", "", candidate, flags=re.IGNORECASE)
-    candidate = re.sub(r"</p>\s*$", "", candidate, flags=re.IGNORECASE)
-    return candidate.strip()
-
-
-def _looks_like_markdown(text: str) -> bool:
-    lower = text.lower()
-    if "<html" in lower and "<head" in lower and "</body>" in lower:
-        return False
-    md_markers = 0
-    md_markers += len(re.findall(r"^\s{0,3}#{1,6}\s+\S", text, flags=re.MULTILINE))
-    md_markers += len(re.findall(r"^\s*[-*+]\s+\S", text, flags=re.MULTILINE))
-    md_markers += len(re.findall(r"^\s*\d+\.\s+\S", text, flags=re.MULTILINE))
-    md_markers += text.count("[TOC]")
-    md_markers += len(MARKDOWN_INLINE_LINK_RE.findall(text))
-    md_markers += text.count("\n---") + text.count("\n***")
-    return md_markers >= 6
-
-
-def _render_text_preview_document(text: str, title: str) -> str:
-    escaped_title = html.escape(title)
-    escaped_text = html.escape(text)
-    return f"""<!doctype html>
-<html lang="en">
-<head>
-    <meta charset="utf-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1">
-    <title>{escaped_title}</title>
-    <style>
-        :root {{
-            color-scheme: dark;
-        }}
-        html, body {{
-            margin: 0;
-            padding: 0;
-            background: #111;
-            color: #f3f3f3;
-            font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", monospace;
-        }}
-        .archivebox-text-preview-header {{
-            position: sticky;
-            top: 0;
-            z-index: 1;
-            padding: 10px 14px;
-            font-size: 12px;
-            line-height: 1.4;
-            color: #bbb;
-            background: rgba(17, 17, 17, 0.96);
-            border-bottom: 1px solid rgba(255, 255, 255, 0.08);
-            backdrop-filter: blur(8px);
-        }}
-        .archivebox-text-preview {{
-            margin: 0;
-            padding: 14px;
-            white-space: pre-wrap;
-            word-break: break-word;
-            tab-size: 2;
-            line-height: 1.45;
-            font-size: 13px;
-        }}
-    </style>
-</head>
-<body>
-    <div class="archivebox-text-preview-header">{escaped_title}</div>
-    <pre class="archivebox-text-preview">{escaped_text}</pre>
-</body>
-</html>"""
-
-
-def _render_image_preview_document(image_url: str, title: str) -> str:
-    escaped_title = html.escape(title)
-    escaped_url = html.escape(image_url, quote=True)
-    return f"""<!doctype html>
-<html lang="en">
-<head>
-    <meta charset="utf-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1">
-    <title>{escaped_title}</title>
-    <style>
-        :root {{
-            color-scheme: dark;
-        }}
-        html, body {{
-            margin: 0;
-            padding: 0;
-            width: 100%;
-            min-height: 100%;
-            background: #fff;
-        }}
-        body {{
-            overflow: auto;
-        }}
-        .archivebox-image-preview {{
-            width: 100%;
-            min-width: 100%;
-            min-height: 100vh;
-            display: flex;
-            flex-direction: column;
-            align-items: center;
-            justify-content: flex-start;
-            box-sizing: border-box;
-        }}
-        .archivebox-image-preview img {{
-            display: block;
-            width: auto;
-            max-width: 100%;
-            height: auto;
-            margin: 0 auto;
-        }}
-    </style>
-</head>
-<body>
-    <div class="archivebox-image-preview">
-        <img src="{escaped_url}" alt="{escaped_title}">
-    </div>
-</body>
-</html>"""
-
-
-def _encoded_responses_image_url(image_url: str, page_url: str | None) -> str | None:
-    raw_url = str(image_url or "").strip()
-    if not raw_url or raw_url.startswith(("#", "data:", "blob:", "about:", "javascript:")):
-        return None
-
-    absolute_url = urljoin(page_url or "", raw_url)
-    if not absolute_url.startswith(("http://", "https://")):
-        return None
-
-    return quote(absolute_url, safe="").replace("%", "_")
-
-
-def _index_responses_paths_for_html_images(
-    snapshot_root: Path,
-    html_rel_path: str,
-    encoded_urls: set[str],
-) -> dict[str, str]:
-    responses_root = snapshot_root / "responses"
-    if not encoded_urls or not responses_root.is_dir():
-        return {}
-
-    best_matches: dict[str, str] = {}
-    image_matches: set[str] = set()
-    image_suffixes = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".ico"}
-    for candidate in responses_root.rglob("*"):
-        if not candidate.is_file():
-            continue
-        try:
-            rel_path = candidate.relative_to(snapshot_root)
-        except ValueError:
-            continue
-        relative_match = posixpath.relpath(rel_path.as_posix(), start=posixpath.dirname(html_rel_path) or ".")
-        candidate_name = candidate.name
-        is_image = candidate.suffix.lower() in image_suffixes
-        for encoded_url in encoded_urls:
-            if encoded_url in image_matches or f"__GET__{encoded_url}" not in candidate_name:
-                continue
-            best_matches[encoded_url] = relative_match
-            if is_image:
-                # Preserve the old rglob behavior: the first image match wins,
-                # otherwise the last matching response file is used.
-                image_matches.add(encoded_url)
-    return best_matches
-
-
-def _rewrite_html_image_sources_to_responses(
-    html_text: str,
-    snapshot_root: Path,
-    html_rel_path: str,
-    page_url: str | None,
-) -> tuple[str, int]:
-    encoded_urls_by_src: dict[str, str | None] = {}
-    for match in IMG_SRC_ATTR_RE.finditer(html_text):
-        image_url = html.unescape(match.group(2))
-        if image_url not in encoded_urls_by_src:
-            encoded_urls_by_src[image_url] = _encoded_responses_image_url(image_url, page_url)
-
-    response_paths = _index_responses_paths_for_html_images(
-        snapshot_root,
-        html_rel_path,
-        {encoded_url for encoded_url in encoded_urls_by_src.values() if encoded_url},
-    )
-    rewrites = 0
-
-    def replace_src(match: re.Match[str]) -> str:
-        nonlocal rewrites
-        image_url = html.unescape(match.group(2))
-        encoded_url = encoded_urls_by_src.get(image_url)
-        local_path = response_paths.get(encoded_url or "")
-        if not local_path:
-            return match.group(0)
-        rewrites += 1
-        return f"{match.group(1)}{html.escape(local_path, quote=True)}{match.group(3)}"
-
-    return IMG_SRC_ATTR_RE.sub(replace_src, html_text), rewrites
-
-
-def _rewrite_html_image_sources_for_request(
-    request,
-    html_text: str,
-    document_root: Path | None,
-    html_rel_path: str,
-) -> tuple[str, int]:
-    if not document_root:
-        return html_text, 0
-    return _rewrite_html_image_sources_to_responses(
-        html_text,
-        document_root,
-        html_rel_path,
-        request.__dict__.get("archivebox_snapshot_url"),
-    )
-
-
-def _apply_transformed_html_preview_style(html_text: str) -> str:
-    if "archivebox-static-html-preview-style" in html_text:
-        return html_text
-    if re.search(r"</head\s*>", html_text, flags=re.IGNORECASE):
-        return re.sub(r"</head\s*>", f"{TRANSFORMED_HTML_PREVIEW_STYLE}\\g<0>", html_text, count=1, flags=re.IGNORECASE)
-    return f"{TRANSFORMED_HTML_PREVIEW_STYLE}\n{html_text}"
-
-
 def _set_transformed_response_headers(response, fullpath: Path, statobj: os.stat_result, encoding: str | None, config) -> None:
     response.headers["Last-Modified"] = http_date(statobj.st_mtime)
     response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=60, stale-while-revalidate=300"
@@ -593,177 +294,6 @@ def _set_transformed_response_headers(response, fullpath: Path, statobj: os.stat
         response.headers["Content-Encoding"] = encoding
 
 
-def _render_markdown_fallback(text: str) -> str:
-    if _markdown is not None and not HTML_TAG_RE.search(text):
-        try:
-            return _markdown(
-                text,
-                extensions=["extra", "toc", "sane_lists"],
-                output_format="html",
-            )
-        except (ImportError, RuntimeError, ValueError):
-            pass
-
-    lines = text.splitlines()
-    headings = []
-
-    def slugify(value: str) -> str:
-        slug = re.sub(r"[^A-Za-z0-9]+", "-", value).strip("-")
-        return slug or "section"
-
-    for raw_line in lines:
-        heading_match = re.match(r"^\s{0,3}(#{1,6})\s+(.*)$", raw_line)
-        if heading_match:
-            level = len(heading_match.group(1))
-            content = heading_match.group(2).strip()
-            headings.append((level, content, slugify(content)))
-
-    html_lines = []
-    in_code = False
-    in_ul = False
-    in_ol = False
-    in_blockquote = False
-
-    def render_inline(markup: str) -> str:
-        content = MARKDOWN_INLINE_IMAGE_RE.sub(r'<img alt="\1" src="\2">', markup)
-        content = MARKDOWN_INLINE_LINK_RE.sub(r'<a href="\2">\1</a>', content)
-        content = MARKDOWN_BOLD_RE.sub(r"<strong>\1</strong>", content)
-        content = MARKDOWN_ITALIC_RE.sub(r"<em>\1</em>", content)
-        return content
-
-    def close_lists():
-        nonlocal in_ul, in_ol
-        if in_ul:
-            html_lines.append("</ul>")
-            in_ul = False
-        if in_ol:
-            html_lines.append("</ol>")
-            in_ol = False
-
-    for raw_line in lines:
-        line = raw_line.rstrip("\n")
-        stripped = line.strip()
-
-        if stripped.startswith("```"):
-            if in_code:
-                html_lines.append("</code></pre>")
-                in_code = False
-            else:
-                close_lists()
-                if in_blockquote:
-                    html_lines.append("</blockquote>")
-                    in_blockquote = False
-                html_lines.append("<pre><code>")
-                in_code = True
-            continue
-
-        if in_code:
-            html_lines.append(html.escape(line))
-            continue
-
-        if not stripped:
-            close_lists()
-            if in_blockquote:
-                html_lines.append("</blockquote>")
-                in_blockquote = False
-            html_lines.append("<br/>")
-            continue
-
-        heading_match = re.match(r"^\s*((?:<[^>]+>\s*)*)(#{1,6})\s+(.*)$", line)
-        if heading_match:
-            close_lists()
-            if in_blockquote:
-                html_lines.append("</blockquote>")
-                in_blockquote = False
-            leading_tags = heading_match.group(1).strip()
-            level = len(heading_match.group(2))
-            content = heading_match.group(3).strip()
-            if leading_tags:
-                html_lines.append(leading_tags)
-            html_lines.append(f'<h{level} id="{slugify(content)}">{render_inline(content)}</h{level}>')
-            continue
-
-        if stripped in ("---", "***"):
-            close_lists()
-            html_lines.append("<hr/>")
-            continue
-
-        if stripped.startswith("> "):
-            if not in_blockquote:
-                close_lists()
-                html_lines.append("<blockquote>")
-                in_blockquote = True
-            content = stripped[2:]
-            html_lines.append(render_inline(content))
-            continue
-        else:
-            if in_blockquote:
-                html_lines.append("</blockquote>")
-                in_blockquote = False
-
-        ul_match = re.match(r"^\s*[-*+]\s+(.*)$", line)
-        if ul_match:
-            if in_ol:
-                html_lines.append("</ol>")
-                in_ol = False
-            if not in_ul:
-                html_lines.append("<ul>")
-                in_ul = True
-            html_lines.append(f"<li>{render_inline(ul_match.group(1))}</li>")
-            continue
-
-        ol_match = re.match(r"^\s*\d+\.\s+(.*)$", line)
-        if ol_match:
-            if in_ul:
-                html_lines.append("</ul>")
-                in_ul = False
-            if not in_ol:
-                html_lines.append("<ol>")
-                in_ol = True
-            html_lines.append(f"<li>{render_inline(ol_match.group(1))}</li>")
-            continue
-
-        close_lists()
-
-        # Inline conversions (leave raw HTML intact)
-        if stripped == "[TOC]":
-            toc_items = []
-            for level, title, slug in headings:
-                toc_items.append(
-                    f'<li class="toc-level-{level}"><a href="#{slug}">{title}</a></li>',
-                )
-            html_lines.append(
-                '<nav class="toc"><ul>' + "".join(toc_items) + "</ul></nav>",
-            )
-            continue
-
-        html_lines.append(f"<p>{render_inline(line)}</p>")
-
-    close_lists()
-    if in_blockquote:
-        html_lines.append("</blockquote>")
-    if in_code:
-        html_lines.append("</code></pre>")
-
-    return "\n".join(html_lines)
-
-
-def _render_markdown_document(markdown_text: str) -> str:
-    body = _render_markdown_fallback(markdown_text)
-    wrapped = (
-        '<!doctype html><html><head><meta charset="utf-8">'
-        '<meta name="viewport" content="width=device-width,initial-scale=1">'
-        "<style>body{max-width:900px;margin:24px auto;padding:0 16px;"
-        "font-family:system-ui,-apple-system,Segoe UI,Roboto,Helvetica,Arial,sans-serif;"
-        "line-height:1.55;} img{max-width:100%;} pre{background:#f6f6f6;padding:12px;overflow:auto;}"
-        ".toc ul{list-style:none;padding-left:0;} .toc li{margin:4px 0;}</style>"
-        "</head><body>"
-        f"{body}"
-        "</body></html>"
-    )
-    return wrapped
-
-
 def _content_type_base(content_type: str) -> str:
     return (content_type or "").split(";", 1)[0].strip().lower()
 
@@ -839,6 +369,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
     if config is None:
         config = get_config(resolve_plugins=False)
     fullpath, path = _resolve_archive_path(document_root, path)
+    replay_response = partial(_apply_archive_replay_headers, fullpath=fullpath, is_archive_replay=is_archive_replay, config=config)
     if os.access(fullpath, os.R_OK) and fullpath.is_dir():
         if request.GET.get("download") == "zip" and show_indexes:
             return _build_directory_zip_response(
@@ -852,13 +383,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
             response = _render_directory_index(request, path, fullpath)
             response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=60, stale-while-revalidate=300"
             response.headers["Last-Modified"] = http_date(fullpath.stat().st_mtime)
-            return _apply_archive_replay_headers(
-                response,
-                fullpath=fullpath,
-                content_type="text/html",
-                is_archive_replay=is_archive_replay,
-                config=config,
-            )
+            return replay_response(response, content_type="text/html")
         raise Http404(_("Directory indexes are not allowed here."))
     if not os.access(fullpath, os.R_OK):
         raise Http404(_("“%(path)s” does not exist") % {"path": fullpath})
@@ -881,15 +406,12 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
                 not_modified.headers["ETag"] = etag
                 not_modified.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=31536000, immutable"
                 not_modified.headers["Last-Modified"] = http_date(statobj.st_mtime)
-                return _apply_archive_replay_headers(
-                    not_modified,
-                    fullpath=fullpath,
-                    content_type="",
-                    is_archive_replay=is_archive_replay,
-                    config=config,
-                )
+                return replay_response(not_modified, content_type="")
 
     content_type, encoding = mimetypes.guess_type(str(fullpath))
+    preserve_plain_text = fullpath.suffix.lower() in {".log", ".sh"}
+    if preserve_plain_text:
+        content_type = "text/plain"
     content_type = content_type or "application/octet-stream"
     # Add charset for text-like types (best guess), but don't override the type.
     is_text_like = content_type.startswith("text/") or content_type in {
@@ -917,13 +439,12 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
         request.META.get("HTTP_IF_MODIFIED_SINCE"),
         statobj.st_mtime,
     ):
-        return _apply_archive_replay_headers(
-            HttpResponseNotModified(),
-            fullpath=fullpath,
-            content_type=content_type,
-            is_archive_replay=is_archive_replay,
-            config=config,
-        )
+        return replay_response(HttpResponseNotModified(), content_type=content_type)
+
+    def transformed_response(body: str, response_type: str):
+        response = HttpResponse(body, content_type=response_type)
+        _set_transformed_response_headers(response, fullpath, statobj, encoding, config)
+        return replay_response(response, content_type=response_type)
 
     # Wrap text-like outputs in HTML when explicitly requested for iframe previewing.
     if preview_as_text_html:
@@ -931,16 +452,8 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
             max_preview_size = 10 * 1024 * 1024
             if statobj.st_size <= max_preview_size:
                 decoded = fullpath.read_text(encoding="utf-8", errors="replace")
-                wrapped = _render_text_preview_document(decoded, fullpath.name)
-                response = HttpResponse(wrapped, content_type="text/html; charset=utf-8")
-                _set_transformed_response_headers(response, fullpath, statobj, encoding, config)
-                return _apply_archive_replay_headers(
-                    response,
-                    fullpath=fullpath,
-                    content_type="text/html; charset=utf-8",
-                    is_archive_replay=is_archive_replay,
-                    config=config,
-                )
+                wrapped = replay_preview._render_text_preview_document(decoded, fullpath.name)
+                return transformed_response(wrapped, "text/html; charset=utf-8")
         except (OSError, UnicodeDecodeError, ValueError):
             preview_as_text_html = False
 
@@ -951,16 +464,8 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
             raw_image_url = request.path
             if preview_query:
                 raw_image_url = f"{raw_image_url}?{urlencode(list(preview_query.lists()), doseq=True)}"
-            wrapped = _render_image_preview_document(raw_image_url, fullpath.name)
-            response = HttpResponse(wrapped, content_type="text/html; charset=utf-8")
-            _set_transformed_response_headers(response, fullpath, statobj, encoding, config)
-            return _apply_archive_replay_headers(
-                response,
-                fullpath=fullpath,
-                content_type="text/html; charset=utf-8",
-                is_archive_replay=is_archive_replay,
-                config=config,
-            )
+            wrapped = replay_preview._render_image_preview_document(raw_image_url, fullpath.name)
+            return transformed_response(wrapped, "text/html; charset=utf-8")
         except (OSError, ValueError):
             preview_as_image_html = False
 
@@ -988,19 +493,13 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
             response = HttpResponse(body, content_type=preview_content_type)
             for key, value in headers.items():
                 response.headers[key] = value
-            return _apply_archive_replay_headers(
-                response,
-                fullpath=fullpath,
-                content_type=preview_content_type,
-                is_archive_replay=is_archive_replay,
-                config=config,
-            )
+            return replay_response(response, content_type=preview_content_type)
         except (OSError, RuntimeError, ValueError):
             preview_as_archivewebpage_html = False
 
     # Heuristic fix: some archived HTML outputs are stored with HTML-escaped markup
     # or markdown sources. If so, render sensibly.
-    if content_type.startswith(("text/plain", "text/html")):
+    if not preserve_plain_text and content_type.startswith(("text/plain", "text/html")):
         try:
             max_unescape_size = 10 * 1024 * 1024  # 10MB cap to avoid heavy memory use
             if statobj.st_size <= max_unescape_size:
@@ -1012,52 +511,37 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
                     decoded = html.unescape(decoded)
                 rewritten_html, rewritten_count = ("", 0)
                 if content_type.startswith("text/html") and document_root:
-                    rewritten_html, rewritten_count = _rewrite_html_image_sources_for_request(request, decoded, document_root, rel_path)
-                markdown_candidate = _extract_markdown_candidate(decoded)
-                if _looks_like_markdown(markdown_candidate):
-                    wrapped = _render_markdown_document(markdown_candidate)
-                    wrapped, _rewrite_count = _rewrite_html_image_sources_for_request(request, wrapped, document_root, rel_path)
-                    wrapped = _apply_transformed_html_preview_style(wrapped)
-                    response = HttpResponse(wrapped, content_type="text/html; charset=utf-8")
-                    _set_transformed_response_headers(response, fullpath, statobj, encoding, config)
-                    return _apply_archive_replay_headers(
-                        response,
-                        fullpath=fullpath,
-                        content_type="text/html; charset=utf-8",
-                        is_archive_replay=is_archive_replay,
-                        config=config,
+                    rewritten_html, rewritten_count = replay_preview._rewrite_html_image_sources_for_request(
+                        request,
+                        decoded,
+                        document_root,
+                        rel_path,
                     )
+                markdown_candidate = replay_preview._extract_markdown_candidate(decoded)
+                if replay_preview._looks_like_markdown(markdown_candidate):
+                    wrapped = replay_preview._render_markdown_document(markdown_candidate)
+                    wrapped, _rewrite_count = replay_preview._rewrite_html_image_sources_for_request(
+                        request,
+                        wrapped,
+                        document_root,
+                        rel_path,
+                    )
+                    wrapped = replay_preview._apply_transformed_html_preview_style(wrapped)
+                    return transformed_response(wrapped, "text/html; charset=utf-8")
                 if rewritten_count:
-                    rewritten_html = _apply_transformed_html_preview_style(rewritten_html)
-                    response = HttpResponse(rewritten_html, content_type=content_type)
-                    _set_transformed_response_headers(response, fullpath, statobj, encoding, config)
-                    return _apply_archive_replay_headers(
-                        response,
-                        fullpath=fullpath,
-                        content_type=content_type,
-                        is_archive_replay=is_archive_replay,
-                        config=config,
-                    )
+                    rewritten_html = replay_preview._apply_transformed_html_preview_style(rewritten_html)
+                    return transformed_response(rewritten_html, content_type)
                 if escaped_count and escaped_count > tag_count * 2:
-                    decoded = _apply_transformed_html_preview_style(decoded)
-                    response = HttpResponse(decoded, content_type=content_type)
-                    _set_transformed_response_headers(response, fullpath, statobj, encoding, config)
-                    return _apply_archive_replay_headers(
-                        response,
-                        fullpath=fullpath,
-                        content_type=content_type,
-                        is_archive_replay=is_archive_replay,
-                        config=config,
-                    )
+                    decoded = replay_preview._apply_transformed_html_preview_style(decoded)
+                    return transformed_response(decoded, content_type)
         except (OSError, UnicodeDecodeError, ValueError):
             pass
 
     # setup response object
     ranged_file = RangedFileReader(fullpath.open("rb"))
-    response = StreamingHttpResponse(
-        _stream_ranged_file_async(ranged_file) if _is_asgi_request(request) else ranged_file,
-        content_type=content_type,
-    )
+    response = StreamingHttpResponse(ranged_file, content_type=content_type)
+    if _is_asgi_request(request):
+        response.streaming_content = _stream_async(ranged_file)
     response.headers["Last-Modified"] = http_date(statobj.st_mtime)
     if etag:
         response.headers["ETag"] = etag
@@ -1095,13 +579,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
                 response.status_code = 206
     if encoding:
         response.headers["Content-Encoding"] = encoding
-    return _apply_archive_replay_headers(
-        response,
-        fullpath=fullpath,
-        content_type=content_type,
-        is_archive_replay=is_archive_replay,
-        config=config,
-    )
+    return replay_response(response, content_type=content_type)
 
 
 def serve_static(request, path, **kwargs):

archivebox/misc/toml_util.py

@@ -2,94 +2,11 @@ from typing import Any, cast
 from collections.abc import Callable
 
 import json
-import ast
-import inspect
 import toml
 import re
-import configparser
 
 from pathlib import Path, PosixPath
 
-from pydantic.json_schema import GenerateJsonSchema
-from pydantic_core import to_jsonable_python
-
-JSONValue = str | bool | int | None | list["JSONValue"]
-
-TOML_HEADER = "# Converted from INI to TOML format: https://toml.io/en/\n\n"
-
-
-def load_ini_value(val: str) -> JSONValue:
-    """Convert lax INI values into strict TOML-compliant (JSON) values"""
-    if val.lower() in ("true", "yes", "1"):
-        return True
-    if val.lower() in ("false", "no", "0"):
-        return False
-    if val.isdigit():
-        return int(val)
-
-    try:
-        return ast.literal_eval(val)
-    except Exception:
-        pass
-
-    try:
-        return json.loads(val)
-    except Exception:
-        pass
-
-    return val
-
-
-def convert(ini_str: str) -> str:
-    """Convert a string of INI config into its TOML equivalent (warning: strips comments)"""
-
-    config = configparser.ConfigParser()
-    setattr(config, "optionxform", str)  # capitalize key names
-    config.read_string(ini_str)
-
-    # Initialize an empty dictionary to store the TOML representation
-    toml_dict = {}
-
-    # Iterate over each section in the INI configuration
-    for section in config.sections():
-        toml_dict[section] = {}
-
-        # Iterate over each key-value pair in the section
-        for key, value in config.items(section):
-            parsed_value = load_ini_value(value)
-
-            # Convert the parsed value to its TOML-compatible JSON representation
-            toml_dict[section.upper()][key.upper()] = json.dumps(parsed_value)
-
-    # Build the TOML string
-    toml_str = TOML_HEADER
-    for section, items in toml_dict.items():
-        toml_str += f"[{section}]\n"
-        for key, value in items.items():
-            toml_str += f"{key} = {value}\n"
-        toml_str += "\n"
-
-    return toml_str.strip()
-
-
-class JSONSchemaWithLambdas(GenerateJsonSchema):
-    """
-    Encode lambda functions in default values properly.
-    Usage:
-    >>> json.dumps(value, encoder=JSONSchemaWithLambdas())
-    """
-
-    def encode_default(self, dft: Any) -> Any:
-        config = self._config
-        if isinstance(dft, Callable):
-            return "{{lambda " + inspect.getsource(dft).split("=lambda ")[-1].strip()[:-1] + "}}"
-        return to_jsonable_python(
-            dft,
-            timedelta_mode=config.ser_json_timedelta,
-            bytes_mode=config.ser_json_bytes,
-            serialize_unknown=True,
-        )
-
 
 def better_toml_dump_str(val: Any) -> str:
     try:

archivebox/misc/util.py

@@ -192,7 +192,10 @@ def validate_url_length(url: str) -> str:
 
 
 def validate_url(url: str) -> str:
-    url = validate_url_length((url or "").strip())
+    url = (url or "").strip()
+    if "\n" in url or "\r" in url:
+        raise ValueError("URL must be a single line.")
+    url = validate_url_length(url)
     parsed = urlparse(url)
     if parsed.scheme.lower() not in ("http", "https") or not parsed.hostname:
         raise ValueError("URL must start with http:// or https:// and include a hostname.")

archivebox/opencode/apps.py

@@ -1,6 +0,0 @@
-from django.apps import AppConfig
-
-
-class OpencodeConfig(AppConfig):
-    name = "archivebox.opencode"
-    label = "opencode_plugin"

archivebox/opencode/templates/opencode/agent.html

@@ -1,211 +0,0 @@
-{% extends "admin/base.html" %}
-{% load i18n %}
-
-{% block title %}Agent{% endblock %}
-
-{% block breadcrumbs %}
-    <div class="breadcrumbs">
-        <a href="{% url 'admin:index' %}">{% trans 'Home' %}</a>
-        &rsaquo; Agent
-    </div>
-{% endblock %}
-
-{% block extrastyle %}
-    {{ block.super }}
-    <style>
-        body.opencode-agent #content {
-            padding: 0;
-        }
-        body.opencode-agent #content-main {
-            height: calc(100vh - 150px);
-            min-height: 560px;
-        }
-        .opencode-agent-error {
-            margin: 24px;
-            color: #0f172a;
-            font: 14px -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
-        }
-        .opencode-agent-error code {
-            color: #0369a1;
-        }
-        .opencode-agent-frame {
-            width: 100%;
-            height: 100%;
-            border: 0;
-            display: block;
-        }
-        .opencode-agent-shell {
-            position: relative;
-            height: 100%;
-        }
-        .opencode-agent-welcome {
-            position: absolute;
-            inset: 0;
-            z-index: 20;
-            display: flex;
-            align-items: center;
-            justify-content: center;
-            padding: 24px;
-            background: rgba(0, 0, 0, 0.48);
-            color: #111827;
-        }
-        .opencode-agent-welcome[hidden] {
-            display: none;
-        }
-        .opencode-agent-welcome-panel {
-            width: min(720px, 100%);
-            max-height: min(720px, calc(100vh - 190px));
-            overflow: auto;
-            border-radius: 8px;
-            border: 1px solid #d1d5db;
-            background: Canvas;
-            color: CanvasText;
-            box-shadow: 0 20px 60px rgba(0, 0, 0, 0.32);
-            padding: 24px;
-            font: 14px/1.5 -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
-        }
-        .opencode-agent-welcome-panel h1 {
-            margin: 0 0 10px;
-            font-size: 22px;
-            line-height: 1.25;
-        }
-        .opencode-agent-welcome-panel p {
-            margin: 0 0 14px;
-        }
-        .opencode-agent-welcome-panel ul {
-            margin: 0 0 16px 20px;
-            padding: 0;
-        }
-        .opencode-agent-welcome-panel li {
-            margin: 0 0 8px;
-        }
-        .opencode-agent-welcome-warning {
-            margin: 16px 0;
-            padding: 12px 14px;
-            border: 1px solid #f59e0b;
-            border-radius: 6px;
-            background: color-mix(in srgb, #f59e0b 12%, Canvas);
-        }
-        .opencode-agent-welcome-actions {
-            display: flex;
-            justify-content: flex-end;
-            gap: 8px;
-        }
-        .opencode-agent-welcome button {
-            cursor: pointer;
-            border: 1px solid #2563eb;
-            border-radius: 6px;
-            background: #2563eb;
-            color: #fff;
-            padding: 8px 14px;
-            font-weight: 600;
-        }
-        @media (max-width: 700px) {
-            .opencode-agent-welcome {
-                padding: 12px;
-            }
-            .opencode-agent-welcome-panel {
-                max-height: calc(100vh - 170px);
-                padding: 18px;
-            }
-        }
-    </style>
-{% endblock %}
-
-{% block bodyclass %}{{ block.super }} opencode-agent{% endblock %}
-
-{% block content %}
-    <div id="content-main">
-        {% if error %}
-            <div class="opencode-agent-error">
-                {{ error }}
-                {% if command %}<br><br><code>{{ command }}</code>{% endif %}
-            </div>
-        {% else %}
-            <script>
-                (() => {
-                    const workdir = "{{ workdir|escapejs }}";
-                    const recentSessionId = "{{ recent_session_id|escapejs }}";
-                    const merge = (left, right) => {
-                        if (!left || typeof left !== "object" || Array.isArray(left)) return right;
-                        const out = {...left};
-                        for (const [key, value] of Object.entries(right)) {
-                            out[key] = value && typeof value === "object" && !Array.isArray(value) ? merge(out[key], value) : value;
-                        }
-                        return out;
-                    };
-                    const seed = (key, value) => {
-                        try {
-                            localStorage.setItem(key, JSON.stringify(merge(JSON.parse(localStorage.getItem(key) || "{}"), value)));
-                        } catch {
-                            localStorage.setItem(key, JSON.stringify(value));
-                        }
-                    };
-                    const setJSON = (key, value) => localStorage.setItem(key, JSON.stringify(value));
-                    const page = {
-                        activeProject: workdir,
-                        activeWorkspace: workdir,
-                        gettingStartedDismissed: true,
-                        workspaceExpanded: {[workdir]: true},
-                    };
-                    if (recentSessionId) {
-                        page.lastProjectSession = {[workdir]: {directory: workdir, id: recentSessionId, at: Date.now()}};
-                    }
-                    seed("opencode.global.dat:layout.page", page);
-                    seed("opencode.global.dat:layout", {
-                        sidebar: {opened: true, width: 280},
-                        fileTree: {opened: true, width: 280, tab: "all"},
-                    });
-                    const serverKey = `${location.origin}/admin/agent/opencode`;
-                    setJSON("opencode.global.dat:server", {
-                        list: [],
-                        projects: {
-                            [serverKey]: [{worktree: workdir, expanded: true}],
-                        },
-                        lastProject: {
-                            [serverKey]: workdir,
-                        },
-                    });
-                    localStorage.setItem("opencode.settings.dat:defaultServerUrl", serverKey);
-                })();
-            </script>
-            <div class="opencode-agent-shell">
-                <iframe src="{{ proxy_url }}" title="OpenCode Agent" class="opencode-agent-frame"></iframe>
-                <div class="opencode-agent-welcome" id="opencode-agent-welcome" hidden>
-                    <div class="opencode-agent-welcome-panel" role="dialog" aria-modal="true" aria-labelledby="opencode-agent-welcome-title">
-                        <h1 id="opencode-agent-welcome-title">ArchiveBox AI Agent</h1>
-                        <p>This agent can work directly with your ArchiveBox collection. It can inspect the crawl database, create and monitor crawls, run maintenance operations, answer research questions, query archived data, and help with complex collection workflows.</p>
-                        <p>Example prompts:</p>
-                        <ul>
-                            <li>Archive this list of URLs with depth 0, then report which ones failed and why.</li>
-                            <li>Find snapshots from the last month that failed PDF or screenshot extraction and retry them.</li>
-                            <li>Create a constrained crawl for this site, avoid login/privacy/sitemap URLs, and watch the logs as it runs.</li>
-                            <li>Summarize what my collection contains about this topic and link to the best saved pages.</li>
-                        </ul>
-                        <div class="opencode-agent-welcome-warning">
-                            The agent has unrestricted access to this collection and can make destructive edits. Review requests carefully. Permission settings can be changed from the OpenCode gear icon in the lower left.
-                        </div>
-                        <div class="opencode-agent-welcome-actions">
-                            <button type="button" id="opencode-agent-welcome-dismiss">Start using Agent</button>
-                        </div>
-                    </div>
-                </div>
-            </div>
-            <script>
-                (() => {
-                    const key = "archivebox.opencode.agentWelcomeDismissed.v1";
-                    const welcome = document.getElementById("opencode-agent-welcome");
-                    const dismiss = document.getElementById("opencode-agent-welcome-dismiss");
-                    if (!welcome || !dismiss) return;
-                    if (localStorage.getItem(key) !== "1") {
-                        welcome.hidden = false;
-                    }
-                    dismiss.addEventListener("click", () => {
-                        localStorage.setItem(key, "1");
-                        welcome.hidden = true;
-                    });
-                })();
-            </script>
-        {% endif %}
-    </div>
-{% endblock %}

archivebox/opencode/views.py

@@ -1,716 +1,140 @@
-from __future__ import annotations
+"""Django adapter for the optional plugin; never import its runtime at startup."""
 
-import atexit
-import base64
-import os
-import re
-import shutil
-import subprocess
-import threading
-import time
-from pathlib import Path
-from typing import Any
-from urllib.parse import urljoin, urlsplit
+import logging
+from io import BytesIO
 
-import httpx
-import requests
-from abx_plugins.plugins import opencode as opencode_plugin
-from archivebox.config import CONSTANTS
-from archivebox.config.common import get_config
-from archivebox.core.routes_util import build_admin_url, get_api_base_url, get_base_url
-from django.http import (
-    Http404,
-    HttpRequest,
-    HttpResponse,
-    HttpResponseForbidden,
-    StreamingHttpResponse,
-)
-from django.shortcuts import redirect, render
+from asgiref.sync import sync_to_async
+
+from django.contrib.auth.middleware import AuthenticationMiddleware
+from django.contrib.auth.views import redirect_to_login
+from django.contrib.sessions.middleware import SessionMiddleware
+from django.core.exceptions import PermissionDenied
+from django.core.handlers.asgi import ASGIRequest
+from django.db import close_old_connections
+from django.http import Http404, HttpResponse, HttpResponseForbidden, StreamingHttpResponse
+from django.template import engines
+from django.urls import Resolver404, resolve
 from django.views.decorators.csrf import csrf_exempt
 
+from archivebox.config import CONSTANTS
+from archivebox.config.common import get_request_config
+from archivebox.core.middleware import ReverseProxyAuthMiddleware
+from archivebox.core.routes_util import build_admin_url, get_admin_host, get_api_base_url, get_base_url, host_matches
+from archivebox.plugins.discovery import get_plugin_template
 
-_PROCESS: subprocess.Popen | None = None
-_PROCESS_LOCK = threading.Lock()
-_PROXY_PREFIX = "/admin/agent/opencode"
-_PROXY_PREFIX_REGEX = _PROXY_PREFIX.replace("/", r"\/")
-_PROXY_PREFIX_NO_SLASH_REGEX = _PROXY_PREFIX.lstrip("/").replace("/", r"\/")
-_CONFIG_PATH = Path(opencode_plugin.__file__).with_name("config.json")
-
-_TEXT_CONTENT_TYPES = (
-    "text/",
-    "application/javascript",
-    "application/json",
-    "application/x-javascript",
-)
-_HOP_BY_HOP_HEADERS = {
-    "connection",
-    "keep-alive",
-    "proxy-authenticate",
-    "proxy-authorization",
-    "te",
-    "trailers",
-    "transfer-encoding",
-    "upgrade",
-}
-_ARCHIVEBOX_SKILL = """---
-name: archivebox
-description: Use ArchiveBox's CLI and local REST API from an ArchiveBox collection.
----
-
-You are running inside an ArchiveBox collection directory.
-
-- ArchiveBox collection directory: {archivebox_data_dir}
-- ArchiveBox BASE_URL: {archivebox_base_url}
-- ArchiveBox Admin URL: {archivebox_admin_url}
-- ArchiveBox REST API URL: {archivebox_api_url}
-- Prefer the `archivebox` CLI for authenticated changes, e.g. `archivebox add`, `archivebox schedule`, `archivebox update`, and `archivebox shell`.
-- Run ArchiveBox CLI commands from the ArchiveBox collection directory above.
-- Get command help with `archivebox list --help`, `archivebox add --help`, `archivebox schedule --help`, etc. Do not use `archivebox help <command>`.
-- Use `--depth=0` by default. Only use recursive crawling when the user explicitly asks for it; use `--depth=1` when you need pages one hop out.
-- Before any recursive crawl, constrain scope with ArchiveBox config such as `CRAWL_MAX_URLS`, `CRAWL_MAX_SIZE`, `SNAPSHOT_MAX_*`, `URL_ALLOWLIST`, `URL_DENYLIST`, and related limits.
-- Respect the configured `archivebox config --get ONLY_NEW` behavior unless the user explicitly says otherwise. Remind users that expected crawl URLs can be skipped when the collection already contains snapshots with the same URL.
-- Always audit newly discovered crawl URLs before letting a crawl run broadly. Treat junk URLs such as privacy policies, legal pages, tag archives, sitemap files, feeds, login/logout URLs, and other low-value boilerplate as unwanted unless the user explicitly asked to archive them.
-- Always watch crawl output and logs as the crawl progresses, and correct errors early instead of waiting until the crawl finishes.
-- If a crawl contains bad URLs, pause it, edit the crawl's `urls` field to remove them, delete any unneeded snapshots already created under that crawl, then resume the crawl.
-- Use `archivebox shell -c '...'` or `archivebox shell <<'PY' ... PY` for Django ORM work. Shell Plus prints an import banner first; keep stderr visible while debugging.
-- Use full ArchiveBox module paths in shell code: `from archivebox.crawls.models import Crawl, CrawlSchedule` and `from archivebox.core.models import Snapshot, ArchiveResult`.
-- If a model/field/relation is unclear, inspect `_meta.fields` before guessing, e.g. `archivebox shell -c "from archivebox.crawls.models import Crawl; print([f.name for f in Crawl._meta.fields])"`.
-- Use `archivebox config --get BASE_URL` only to verify the configured base URL; prefer the seeded URLs above for API/admin requests.
-- Use `$ARCHIVEBOX_API_URL` for REST API inspection when helpful. Do not assume admin session cookies authenticate API subdomain requests; prefer CLI/shell for authenticated mutations unless the admin provides or asks you to create an API token.
-- Discover REST endpoints from `${{ARCHIVEBOX_API_URL}}v1/openapi.json`; crawl endpoints live under `/api/v1/crawls/`, snapshots under `/api/v1/core/`.
-- Do not bypass ArchiveBox auth, expose API keys, or modify config unless the admin explicitly asks.
-- After creating crawls or snapshots, report the crawl/snapshot IDs and the exact command or API request used.
-"""
+_LOGGER = logging.getLogger(__name__)
 
 
-def _stop_owned_process(process: subprocess.Popen | None = None) -> None:
-    global _PROCESS
-    owned_process = process or _PROCESS
-    if owned_process is None:
-        return
-    if owned_process.poll() is None:
-        owned_process.terminate()
-        try:
-            owned_process.wait(timeout=5)
-        except subprocess.TimeoutExpired:
-            owned_process.kill()
-            owned_process.wait()
-    if _PROCESS is owned_process:
-        _PROCESS = None
+def _runtime_settings(request, config):
+    from abx_plugins.plugins.opencode import runtime
 
-
-atexit.register(_stop_owned_process)
-
-
-def _machine_config() -> dict[str, Any]:
-    resolved = get_config()
-    return dict(resolved.model_dump(mode="json"))
-
-
-def _archivebox_data_dir_default() -> Path:
-    return Path(CONSTANTS.DATA_DIR)
-
-
-def _archivebox_route_urls(request: HttpRequest, route_config) -> tuple[str, str, str]:
-    base_url = get_base_url(
-        request=request,
-        config=route_config,
-    ).rstrip("/")
-    admin_url = build_admin_url(
-        "/admin/",
-        request=request,
-        config=route_config,
-    ).rstrip("/")
-    api_url = f"{get_api_base_url(request=request, config=route_config).rstrip('/')}/api/"
-    return base_url, admin_url, api_url
-
-
-def _config_value(config: dict, key: str, default):
-    value = config.get(key, default)
-    if value in (None, ""):
-        return default
-    return value
-
-
-def _opencode_enabled(config: dict) -> bool:
-    value = config.get("OPENCODE_ENABLED", False)
-    if isinstance(value, bool):
-        return value
-    return str(value).strip().lower() in {"1", "true", "yes", "on"}
-
-
-def _require_enabled(config: dict) -> None:
-    if not _opencode_enabled(config):
-        raise Http404
-
-
-def _require_superuser(request: HttpRequest):
-    user = getattr(request, "user", None)
-    if user is None:
-        return redirect(f"/admin/login/?next={request.get_full_path()}")
-    if (
-        bool(getattr(user, "is_authenticated", False))
-        and bool(getattr(user, "is_active", False))
-        and bool(getattr(user, "is_superuser", False))
-    ):
-        return None
-    if bool(getattr(user, "is_authenticated", False)):
-        return HttpResponseForbidden(
-            b"ArchiveBox agent access requires a superuser account.",
-        )
-    return redirect(f"/admin/login/?next={request.get_full_path()}")
-
-
-def _origin_allowed(request: HttpRequest, path: str | None = None) -> bool:
-    if request.method in {"GET", "HEAD", "OPTIONS", "TRACE"}:
-        return True
-
-    expected_host = request.get_host()
-    pty_connect = bool(
-        path and path.startswith("pty/") and path.endswith("/connect-token"),
-    )
-    if pty_connect:
-        return True
-
-    origin = _request_header(request, "Origin")
-    if origin:
-        return _same_host(origin, expected_host)
-
-    referer = _request_header(request, "Referer")
-    if referer:
-        return _same_host(referer, expected_host)
-
-    fetch_site = _request_header(request, "Sec-Fetch-Site")
-    if fetch_site:
-        return fetch_site in {"same-origin", "same-site", "none"}
-
-    return False
-
-
-def _same_host(value: str, expected_host: str) -> bool:
-    parsed = urlsplit(value)
-    return parsed.scheme in {"http", "https"} and parsed.netloc == expected_host
-
-
-def _settings(config: dict) -> dict:
-    host = str(_config_value(config, "OPENCODE_HOST", "127.0.0.1"))
-    port = int(_config_value(config, "OPENCODE_PORT", 4096))
-    default_data_dir = _archivebox_data_dir_default()
-    workdir = Path(
-        str(_config_value(config, "OPENCODE_WORKDIR", default_data_dir)),
-    ).expanduser()
-    opencode_dir = Path(
-        str(_config_value(config, "OPENCODE_STATE_DIR", workdir / "opencode")),
-    ).expanduser()
-    binary = str(_config_value(config, "OPENCODE_BINARY", "opencode"))
-    timeout = int(_config_value(config, "OPENCODE_TIMEOUT", 30))
-    return {
-        "host": host,
-        "port": port,
-        "origin": f"http://{host}:{port}",
-        "workdir": workdir,
-        "opencode_dir": opencode_dir,
-        "config_home": opencode_dir / "config",
-        "data_home": opencode_dir / "data",
-        "state_home": opencode_dir / "state",
-        "cache_home": opencode_dir / "cache",
-        "home": opencode_dir / "home",
-        "binary": binary,
-        "config": config,
-        "timeout": timeout,
-    }
-
-
-def _resolve_binary(binary: str, config: dict) -> tuple[Any, Any, dict[str, str]]:
-    try:
-        from abxpkg import BinProvider
-        from abx_plugins.plugins.base.utils import load_required_binary_from_config
-
-        binary_environ = os.environ.copy()
-        lib_dir = config.get("ABXPKG_LIB_DIR")
-        if lib_dir:
-            binary_environ["ABXPKG_LIB_DIR"] = str(lib_dir)
-        loaded_dependencies = [
-            load_required_binary_from_config(
-                required_binary,
-                _CONFIG_PATH,
-                global_config=config,
-                environ=binary_environ,
-                install=False,
-            )
-            for required_binary in (
-                str(config.get("NODE_BINARY") or "node"),
-                str(config.get("GIT_BINARY") or "git"),
-                binary,
-            )
-        ]
-    except Exception as err:
-        raise RuntimeError(
-            f"OpenCode dependency is not installed from required_binaries: {err}",
-        ) from err
-
-    if any(not loaded.loaded_abspath for loaded in loaded_dependencies):
-        raise RuntimeError(
-            "OpenCode dependency is not installed from required_binaries.",
-        )
-
-    providers = [loaded.loaded_binprovider for loaded in loaded_dependencies if loaded.loaded_binprovider is not None]
-    binary_env = BinProvider.build_exec_env(
-        providers=providers,
-        base_env=binary_environ,
-    )
-    return (
-        loaded_dependencies[-1],
-        loaded_dependencies[1],
-        binary_env,
-    )
-
-
-def _project_route(workdir: Path, session_id: str = "") -> str:
-    encoded = base64.b64encode(str(workdir.resolve()).encode()).decode()
-    encoded = encoded.replace("+", "-").replace("/", "_").rstrip("=")
-    route = f"{_PROXY_PREFIX}/{encoded}/session"
-    return f"{route}/{session_id}" if session_id else route
-
-
-def _ensure_project_files(settings: dict) -> None:
-    workdir = settings["workdir"].resolve()
-    workdir.mkdir(parents=True, exist_ok=True)
-    git_marker = workdir / ".git" / "not-a-git"
-    if git_marker.exists():
-        # Current OpenCode hangs on the legacy fake marker, so remove only
-        # that invalid shape before initializing the real worktree.
-        shutil.rmtree(git_marker.parent)
-
-    editable_skill_path = settings["opencode_dir"] / "SKILL.md"
-    editable_skill_path.parent.mkdir(parents=True, exist_ok=True)
-    if not editable_skill_path.exists():
-        editable_skill_path.write_text(
-            _ARCHIVEBOX_SKILL.format(
-                archivebox_data_dir=workdir,
-                archivebox_base_url=settings.get("archivebox_base_url", ""),
-                archivebox_admin_url=settings.get("archivebox_admin_url", ""),
-                archivebox_api_url=settings.get("archivebox_api_url", ""),
-            ),
-        )
-
-    opencode_skill_path = settings["config_home"] / "opencode" / "skills" / "archivebox" / "SKILL.md"
-    opencode_skill_path.parent.mkdir(parents=True, exist_ok=True)
-    if opencode_skill_path.resolve() != editable_skill_path.resolve():
-        if opencode_skill_path.exists() or opencode_skill_path.is_symlink():
-            opencode_skill_path.unlink()
-        opencode_skill_path.symlink_to(editable_skill_path)
-
-
-def _ensure_default_session(settings: dict) -> str:
-    workdir = settings["workdir"].resolve()
-    params = {"directory": str(workdir)}
-    timeout = settings["timeout"]
-    project = requests.post(
-        f"{settings['origin']}/project/git/init",
-        params=params,
-        timeout=timeout,
-    )
-    project.raise_for_status()
-    project_data = project.json()
-    if Path(str(project_data.get("worktree") or "/")).resolve() != workdir:
-        raise RuntimeError(
-            f"OpenCode initialized the wrong project worktree: {project_data.get('worktree')!r}",
-        )
-
-    sessions = requests.get(
-        f"{settings['origin']}/session",
-        params={**params, "roots": "true", "limit": 55},
-        timeout=timeout,
-    )
-    sessions.raise_for_status()
-    session_data = sessions.json()
-    if not isinstance(session_data, list):
-        raise RuntimeError("OpenCode returned an invalid project session list.")
-    for session_data_item in session_data:
-        if not isinstance(session_data_item, dict):
-            continue
-        session_id = str(session_data_item.get("id") or "")
-        session_directory = session_data_item.get("directory")
-        if session_id and session_directory and Path(str(session_directory)).resolve() == workdir:
-            return session_id
-
-    session = requests.post(
-        f"{settings['origin']}/session",
-        params=params,
-        json={},
-        timeout=timeout,
-    )
-    session.raise_for_status()
-    session_data = session.json()
-    session_id = str(session_data.get("id") or "")
-    session_directory = session_data.get("directory")
-    if not session_id or not session_directory or Path(str(session_directory)).resolve() != workdir:
-        raise RuntimeError(
-            "OpenCode did not create a session for the requested worktree.",
-        )
-    return session_id
-
-
-def _recent_session_id(settings: dict) -> str:
-    workdir = str(settings["workdir"].resolve())
-    try:
-        response = requests.get(
-            f"{settings['origin']}/session",
-            params={"directory": workdir, "roots": "true", "limit": 1},
-            timeout=settings["timeout"],
-        )
-        response.raise_for_status()
-        sessions = response.json()
-        if sessions:
-            return str(sessions[0].get("id") or "")
-    except requests.RequestException:
-        pass
-    return ""
-
-
-def _health(settings: dict) -> bool:
-    try:
-        response = requests.get(
-            f"{settings['origin']}/global/health",
-            timeout=2,
-        )
-        return response.status_code == 200
-    except requests.RequestException:
-        return False
-
-
-def _ensure_opencode(settings: dict) -> tuple[bool, str]:
-    global _PROCESS
-    started_process: subprocess.Popen | None = None
-    workdir = settings["workdir"].resolve()
-    try:
-        binary, git_binary, binary_env = _resolve_binary(
-            settings["binary"],
-            settings["config"],
-        )
-    except RuntimeError as err:
-        return False, str(err)
-
-    env = {
-        **os.environ,
-        **binary_env,
-        "ARCHIVEBOX_BASE_URL": str(settings.get("archivebox_base_url", "")),
-        "ARCHIVEBOX_ADMIN_URL": str(settings.get("archivebox_admin_url", "")),
-        "ARCHIVEBOX_API_URL": str(settings.get("archivebox_api_url", "")),
-        "BROWSER": "false",
-        "GIT_CEILING_DIRECTORIES": str(workdir),
-        "HOME": str(settings["home"]),
-        "OPENCODE_DISABLE_PROJECT_CONFIG": "true",
-        "XDG_CONFIG_HOME": str(settings["config_home"]),
-        "XDG_DATA_HOME": str(settings["data_home"]),
-        "XDG_STATE_HOME": str(settings["state_home"]),
-        "XDG_CACHE_HOME": str(settings["cache_home"]),
-    }
-
-    with _PROCESS_LOCK:
-        settings["workdir"].mkdir(parents=True, exist_ok=True)
-        settings["config_home"].mkdir(parents=True, exist_ok=True)
-        settings["data_home"].mkdir(parents=True, exist_ok=True)
-        settings["state_home"].mkdir(parents=True, exist_ok=True)
-        settings["cache_home"].mkdir(parents=True, exist_ok=True)
-        settings["home"].mkdir(parents=True, exist_ok=True)
-        _ensure_project_files(settings)
-
-        if not (workdir / ".git").exists():
-            try:
-                git_init = git_binary.exec(
-                    cmd=("init", "--quiet"),
-                    cwd=workdir,
-                    env=env,
-                    timeout=settings["timeout"],
-                )
-            except (AssertionError, OSError, subprocess.SubprocessError) as err:
-                return False, f"OpenCode project initialization failed: {err}"
-            if git_init.returncode != 0:
-                output = (git_init.stderr or git_init.stdout or "").strip()
-                return (
-                    False,
-                    f"OpenCode project initialization failed: {output or f'git exited with {git_init.returncode}'}",
-                )
-
-        if _health(settings):
-            try:
-                _ensure_default_session(settings)
-            except (requests.RequestException, RuntimeError, ValueError) as err:
-                return False, f"OpenCode project initialization failed: {err}"
-            return True, ""
-
-        binary_abspath = binary.loaded_abspath
-        if binary.loaded_binprovider is not None:
-            binary_abspath = binary.loaded_binprovider._exec_bin_abspath(
-                Path(binary.loaded_abspath),
-            )
-        cmd = [
-            str(binary_abspath),
-            "serve",
-            "--hostname",
-            settings["host"],
-            "--port",
-            str(settings["port"]),
-        ]
-        try:
-            _PROCESS = subprocess.Popen(
-                cmd,
-                cwd=workdir,
-                env=env,
-                stdin=subprocess.DEVNULL,
-                stdout=subprocess.DEVNULL,
-                stderr=subprocess.DEVNULL,
-                start_new_session=True,
-            )
-            started_process = _PROCESS
-        except FileNotFoundError:
-            return False, f"OpenCode binary not found: {settings['binary']}"
-
-        deadline = time.monotonic() + settings["timeout"]
-        while time.monotonic() < deadline:
-            if _health(settings):
-                try:
-                    _ensure_default_session(settings)
-                except (requests.RequestException, RuntimeError, ValueError) as err:
-                    _stop_owned_process(started_process)
-                    return False, f"OpenCode project initialization failed: {err}"
-                return True, ""
-            if started_process and started_process.poll() is not None:
-                if _PROCESS is started_process:
-                    _PROCESS = None
-                return False, "OpenCode exited before the web server became ready."
-            time.sleep(0.25)
-
-        _stop_owned_process(started_process)
-        return False, "Timed out waiting for OpenCode to start."
-
-
-def agent_view(request: HttpRequest):
-    config = _machine_config()
-    _require_enabled(config)
-    auth_response = _require_superuser(request)
-    if auth_response:
-        return auth_response
-
-    settings = _settings(config)
+    settings = runtime._settings(config, CONSTANTS.DATA_DIR)
     route_config = request.__dict__.get("archivebox_config")
-    base_url, admin_url, api_url = _archivebox_route_urls(request, route_config)
-    settings["archivebox_base_url"] = base_url
-    settings["archivebox_admin_url"] = admin_url
-    settings["archivebox_api_url"] = api_url
-    ok, error = _ensure_opencode(settings)
-    from archivebox.core.admin_site import archivebox_admin
-
-    recent_session_id = _recent_session_id(settings) if ok else ""
-    context = {
-        **archivebox_admin.each_context(request),
-        "title": "Agent",
-        "error": "" if ok else error,
-        "command": f"{settings['binary']} serve --hostname {settings['host']} --port {settings['port']}" if error else "",
-        # OpenCode 1.17+ keeps durable sessions on the explicit
-        # /<dirBase64>/session/<sessionId> route. We still seed localStorage
-        # below because the sidebar state uses it, but the iframe itself must
-        # open the durable session URL so a fresh browser does not land on the
-        # transient new-session route and appear to have lost prior sessions.
-        "proxy_url": _project_route(settings["workdir"], recent_session_id),
-        "workdir": str(settings["workdir"].resolve()),
-        "recent_session_id": recent_session_id,
-    }
-    return render(
-        request,
-        "opencode/agent.html",
-        context,
-        status=200 if ok else 502,
+    settings.update(
+        archivebox_base_url=get_base_url(request=request, config=route_config).rstrip("/"),
+        archivebox_admin_url=build_admin_url("/admin/", request=request, config=route_config).rstrip("/"),
+        archivebox_api_url=f"{get_api_base_url(request=request, config=route_config).rstrip('/')}/api/",
     )
+    return runtime, settings
 
 
-def _proxy_url(settings: dict, path: str | None) -> str:
-    rel = "/" if not path else f"/{path}"
-    return urljoin(settings["origin"], rel)
+def _dispatch(request, path=None):
+    try:
+        # Middleware already merged server defaults and live Machine overrides.
+        # Re-resolving every extractor's config here stalls the UI request burst.
+        config = get_request_config(request).model_dump(mode="json")
+        if not config.get("OPENCODE_ENABLED", False):
+            raise Http404
+        if not request.user.is_authenticated:
+            return redirect_to_login(request.get_full_path(), login_url="/admin/login/")
+        if not request.user.is_active or not request.user.is_superuser:
+            return HttpResponseForbidden("Agent access requires a superuser account.")
+
+        runtime, settings = _runtime_settings(request, config)
+        if path is None:
+            from archivebox.core.admin_site import archivebox_admin
+
+            context = {**archivebox_admin.each_context(request), **runtime.agent_context(settings)}
+            source = get_plugin_template("opencode", "agent", fallback=False)
+            if source is None:
+                raise RuntimeError("Agent template unavailable")
+            return HttpResponse(engines["django"].from_string(source).render(context, request))
+
+        if not runtime._origin_allowed(request.method, request.get_host(), request.headers):
+            return HttpResponseForbidden("Cross-origin agent requests are blocked.")
+        status, headers, body = runtime.proxy(
+            settings,
+            request.method,
+            path,
+            tuple((key, value) for key, values in request.GET.lists() for value in values),
+            request.headers,
+            request.body,
+        )
+        response_type = HttpResponse if isinstance(body, bytes) else StreamingHttpResponse
+        response = response_type(body, status=status, headers=headers)
+        response.xframe_options_exempt = True
+        response.headers["X-Frame-Options"] = "SAMEORIGIN"
+        response.headers["Content-Security-Policy"] = "frame-ancestors 'self'"
+        return response
+    except Http404:
+        raise
+    except Exception:
+        # Optional-service boundary, including imports and template rendering.
+        _LOGGER.exception("Optional AI service failed")
+        return HttpResponse("AI service unavailable. See server logs.", status=503, content_type="text/plain")
 
 
-def _request_header(request: HttpRequest, name: str) -> str | None:
-    meta = getattr(request, "META", {})
-    if name == "Content-Type":
-        value = meta.get("CONTENT_TYPE")
-    elif name == "Content-Length":
-        value = meta.get("CONTENT_LENGTH")
-    else:
-        value = meta.get(f"HTTP_{name.upper().replace('-', '_')}")
-    return str(value) if value else None
-
-
-def _request_headers(request: HttpRequest, settings: dict) -> dict[str, str]:
-    forwarded = {}
-    for key in ("Accept", "Accept-Language", "Content-Type", "Range", "User-Agent"):
-        value = _request_header(request, key)
-        if value:
-            forwarded[key] = value
-    return forwarded
-
-
-def _request_params(request: HttpRequest) -> tuple[tuple[str, str], ...]:
-    if hasattr(request.GET, "lists"):
-        return tuple((key, str(value)) for key, values in request.GET.lists() for value in values)
-    return tuple((key, str(value)) for key, value in dict(request.GET).items())
-
-
-async def _event_chunks(request: HttpRequest, settings: dict, path: str | None):
-    timeout = httpx.Timeout(settings["timeout"], read=None)
-    url = _proxy_url(settings, path)
-    method = request.method or "GET"
-    async with httpx.AsyncClient(timeout=timeout, follow_redirects=False) as client:
-        async with client.stream(
-            method,
-            url,
-            params=_request_params(request),
-            headers=_request_headers(request, settings),
-        ) as upstream:
-            async for chunk in upstream.aiter_raw(chunk_size=512):
-                yield chunk
-
-
-def _rewrite_text(body: bytes, settings: dict) -> bytes:
-    text = body.decode("utf-8", errors="replace")
-    text = text.replace(settings["origin"], _PROXY_PREFIX)
-    text = text.replace("location.origin", f'location.origin+"{_PROXY_PREFIX}"')
-    text = text.replace(
-        "k(k5,{get component(){return t.router??Az},",
-        f'k(k5,{{base:"{_PROXY_PREFIX}",get component(){{return t.router??Az}},',
-    )
-    text = text.replace('"/assets/', f'"{_PROXY_PREFIX}/assets/')
-    text = text.replace("'/assets/", f"'{_PROXY_PREFIX}/assets/")
-    proxy_path = rf'(\1.replace(/^{_PROXY_PREFIX_REGEX}(?=\/|$)/,"")||"/")'
-    text = re.sub(r"\b(window\.location\.pathname)\b", proxy_path, text)
-    text = re.sub(r"(?<![.\w])(location\.pathname)\b", proxy_path, text)
-    text = text.replace(
-        'window.history.replaceState(nz(o),"",r):window.history.pushState(o,"",r)',
-        (
-            f'window.history.replaceState(nz(o),"",r.startsWith("{_PROXY_PREFIX}")?r:r.startsWith("/")?"{_PROXY_PREFIX}"+r:r):'
-            f'window.history.pushState(o,"",r.startsWith("{_PROXY_PREFIX}")?r:r.startsWith("/")?"{_PROXY_PREFIX}"+r:r)'
-        ),
-    )
-    text = text.replace(
-        'const BL="modulepreload",UL=function(t){return"/"+t}',
-        f'const BL="modulepreload",UL=function(t){{return"{_PROXY_PREFIX}/"+t}}',
-    )
-    text = re.sub(
-        rf"""(?P<prefix>\b(?:href|src|action)=["'])/(?!{_PROXY_PREFIX_NO_SLASH_REGEX}(?:/|$))""",
-        rf"\g<prefix>{_PROXY_PREFIX}/",
-        text,
-    )
-    text = re.sub(
-        rf"""(?P<prefix>\b(?:fetch|EventSource)\(["'])/(?!{_PROXY_PREFIX_NO_SLASH_REGEX}(?:/|$))""",
-        rf"\g<prefix>{_PROXY_PREFIX}/",
-        text,
-    )
-    text = re.sub(
-        rf"""(?P<prefix>\burl\(["']?)/(?!{_PROXY_PREFIX_NO_SLASH_REGEX}(?:/|$))""",
-        rf"\g<prefix>{_PROXY_PREFIX}/",
-        text,
-    )
-    return text.encode("utf-8")
-
-
-def _response_headers(upstream: requests.Response, settings: dict) -> dict[str, str]:
-    headers = {}
-    for key, value in upstream.headers.items():
-        lower = key.lower()
-        if lower in _HOP_BY_HOP_HEADERS or lower in {
-            "content-length",
-            "content-encoding",
-            "x-frame-options",
-        }:
-            continue
-        if lower == "location":
-            if value.startswith(settings["origin"]):
-                value = value.replace(settings["origin"], _PROXY_PREFIX, 1)
-            elif value.startswith("/"):
-                value = f"{_PROXY_PREFIX}{value}"
-        headers[key] = value
-    return headers
+def agent_view(request):
+    return _dispatch(request)
 
 
 @csrf_exempt
-def opencode_proxy_view(request: HttpRequest, path: str | None = None):
-    config = _machine_config()
-    _require_enabled(config)
-    auth_response = _require_superuser(request)
-    if auth_response:
-        return auth_response
-    if not _origin_allowed(request, path):
-        return HttpResponseForbidden(
-            b"Cross-origin OpenCode agent requests are blocked.",
-        )
+def opencode_proxy_view(request, path=None):
+    return _dispatch(request, path=path or "")
 
-    settings = _settings(config)
-    route_config = request.__dict__.get("archivebox_config")
-    base_url, admin_url, api_url = _archivebox_route_urls(request, route_config)
-    settings["archivebox_base_url"] = base_url
-    settings["archivebox_admin_url"] = admin_url
-    settings["archivebox_api_url"] = api_url
-    ok, error = _ensure_opencode(settings)
-    if not ok:
-        return HttpResponse(
-            error.encode(),
-            status=502,
-            content_type="text/plain; charset=utf-8",
-        )
-
-    if request.method == "GET" and (path or "").endswith("/event"):
-        response = StreamingHttpResponse(
-            _event_chunks(request, settings, path),
-            content_type="text/event-stream",
-        )
-        response.headers["Cache-Control"] = "no-store"
-        response.headers["X-Accel-Buffering"] = "no"
-        return response
 
+def _websocket_context(scope):
+    close_old_connections()
     try:
-        method = request.method or "GET"
-        upstream = requests.request(
-            method,
-            _proxy_url(settings, path),
-            params=_request_params(request),
-            data=request.body if method not in {"GET", "HEAD"} else None,
-            headers=_request_headers(request, settings),
-            stream=True,
-            timeout=(settings["timeout"], None),
-            allow_redirects=False,
-        )
-    except requests.RequestException as err:
-        return HttpResponse(
-            str(err).encode(),
-            status=502,
-            content_type="text/plain; charset=utf-8",
+        match = resolve(scope["path"])
+        if match.url_name != "opencode-proxy":
+            raise PermissionDenied
+        request = ASGIRequest(
+            {**scope, "type": "http", "method": "GET", "scheme": "https" if scope.get("scheme") == "wss" else "http"},
+            BytesIO(),
         )
+        route_config = get_request_config(request, resolve_plugins=False)
+        request.archivebox_config = route_config
+        if not route_config.CONTROL_PLANE_ENABLED:
+            raise PermissionDenied
+        if (
+            route_config.USES_SUBDOMAIN_ROUTING
+            and route_config.BASE_URL
+            and not host_matches(request.get_host(), get_admin_host(config=route_config, request=request))
+        ):
+            raise PermissionDenied
+        SessionMiddleware(_dispatch).process_request(request)
+        AuthenticationMiddleware(_dispatch).process_request(request)
+        ReverseProxyAuthMiddleware(_dispatch).process_request(request)
+        config = route_config.model_dump(mode="json")
+        if not config.get("OPENCODE_ENABLED") or not request.user.is_active or not request.user.is_superuser:
+            raise PermissionDenied
+        runtime, settings = _runtime_settings(request, config)
+        if not request.headers.get("Origin") or not runtime._origin_allowed("POST", request.get_host(), request.headers):
+            raise PermissionDenied
+        return runtime, settings, match.kwargs.get("path", "")
+    finally:
+        close_old_connections()
 
-    content_type = upstream.headers.get("Content-Type", "")
-    is_event_stream = content_type.startswith("text/event-stream")
-    is_text = not is_event_stream and any(content_type.startswith(prefix) for prefix in _TEXT_CONTENT_TYPES)
-    headers = _response_headers(upstream, settings)
-    if is_text:
-        body = _rewrite_text(upstream.content, settings)
-        response = HttpResponse(
-            body,
-            status=upstream.status_code,
-            content_type=content_type or "text/plain; charset=utf-8",
-        )
-    elif is_event_stream:
-        response = StreamingHttpResponse(
-            upstream.iter_lines(chunk_size=1),
-            status=upstream.status_code,
-            content_type=content_type or "text/event-stream",
-        )
-    else:
-        response = StreamingHttpResponse(
-            upstream.iter_content(chunk_size=64 * 1024),
-            status=upstream.status_code,
-            content_type=content_type or "application/octet-stream",
-        )
-    for key, value in headers.items():
-        response.headers[key] = value
-    response.headers["Cache-Control"] = "no-store"
-    return response
+
+async def websocket_view(scope, receive, send):
+    if (await receive())["type"] != "websocket.connect":
+        return
+    try:
+        runtime, settings, path = await sync_to_async(_websocket_context)(scope)
+        await runtime.websocket_proxy(settings, path, scope.get("query_string", b""), scope.get("subprotocols", []), receive, send)
+    except (PermissionDenied, Resolver404, Http404):
+        await send({"type": "websocket.close", "code": 1008})
+    except Exception:
+        _LOGGER.exception("Optional AI WebSocket failed")
+        await send({"type": "websocket.close", "code": 1011})

archivebox/personas/admin.py

@@ -5,7 +5,7 @@ import shutil
 from django.contrib import admin, messages
 from django.utils.html import format_html, format_html_join
 
-from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin
+from archivebox.base_models.admin import card_fieldset, BaseModelAdmin, ConfigEditorMixin
 from archivebox.personas.forms import PersonaAdminForm
 from archivebox.personas.importers import discover_local_browser_profiles
 from archivebox.personas.models import Persona
@@ -30,28 +30,20 @@ class PersonaAdmin(ConfigEditorMixin, BaseModelAdmin):
                 "classes": ("card", "persona-card-primary"),
             },
         ),
-        (
+        card_fieldset(
             "Browser Import",
-            {
-                "fields": (
-                    "import_mode",
-                    "import_discovered_profile",
-                    "import_source",
-                    "import_profile_name",
-                    "import_copy_profile",
-                    "import_extract_cookies",
-                    "import_capture_storage",
-                ),
-                "classes": ("card", "wide"),
-            },
-        ),
-        (
-            "Advanced",
-            {
-                "fields": ("config",),
-                "classes": ("card", "wide"),
-            },
+            (
+                "import_mode",
+                "import_discovered_profile",
+                "import_source",
+                "import_profile_name",
+                "import_copy_profile",
+                "import_extract_cookies",
+                "import_capture_storage",
+            ),
+            wide=True,
         ),
+        card_fieldset("Advanced", ("config",), wide=True),
     )
 
     change_fieldsets = (
@@ -65,13 +57,7 @@ class PersonaAdmin(ConfigEditorMixin, BaseModelAdmin):
         ),
         add_fieldsets[1],
         add_fieldsets[2],
-        (
-            "Artifacts",
-            {
-                "fields": ("persona_paths", "import_artifact_status"),
-                "classes": ("card", "wide"),
-            },
-        ),
+        card_fieldset("Artifacts", ("persona_paths", "import_artifact_status"), wide=True),
     )
 
     @admin.display(description="Chrome Profile")