Bump release version to 0.9.35rc437

* Stop initializing Git repositories for OpenCode sessions

* Isolate optional AI failures behind a lazy plugin adapter

* Keep explicit plugin enable flags boolean after config resolution

* Use Django login redirect encoding at the optional agent boundary

* Cover cold-start failures and real browser storage isolation

* Verify persisted dismissal and native storage denial variants

* Keep headless browser interaction independent of display timing

* Install optional OpenCode clients through the plugin extra

* Authenticate and route optional OpenCode WebSockets with real browser coverage

* Apply control-plane policy and proxy authentication to agent sockets

* Require immediate delivery of the first real OpenCode event

* Exercise browser same-origin headers in event regression

* Exercise OpenCode terminal tickets in proxy and browser regressions

* Preserve primary terminal failure while verifying cleanup

.github/workflows/deploy-publicsite.yml

@@ -27,7 +27,6 @@ concurrency:
 
 jobs:
   deploy:
-    if: github.event_name != 'push' || github.event.head_commit.author.email != 'release-bot@archivebox.io'
     environment:
       name: github-pages
       url: ${{ steps.deployment.outputs.page_url }}
@@ -51,6 +50,22 @@ jobs:
         env:
           BASE_URL: http://archivebox.localhost:8000
 
+      - name: Verify gallery was generated from this exact revision
+        run: |
+          uv run --no-cache --no-sync python - <<'PY'
+          import json
+          import os
+          import tomllib
+          from pathlib import Path
+
+          provenance = json.loads(Path("publicsite/screenshots/build.json").read_text())
+          version = tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"]
+          assert provenance["revision"] == os.environ["GITHUB_SHA"], provenance
+          assert provenance["version"] == version, provenance
+          assert provenance["capture_count"] >= 3 * 35, provenance
+          assert len(provenance["files"]) == provenance["capture_count"], provenance
+          PY
+
       - name: Setup Pages
         uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5
 

.github/workflows/docker.yml

@@ -33,12 +33,10 @@ jobs:
         include:
           - platform: linux/amd64
             runner: ubuntu-24.04
-            cache_scope: docker-amd64
             artifact_name: digest-linux-amd64
             local_tag: archivebox-ci:amd64
           - platform: linux/arm64
             runner: ubuntu-24.04-arm
-            cache_scope: docker-arm64
             artifact_name: digest-linux-arm64
             local_tag: archivebox-ci:arm64
     steps:
@@ -208,17 +206,17 @@ jobs:
             ARCHIVEBOX_COMMIT_HASH=${{ github.sha }}
           pull: true
           platforms: ${{ matrix.platform }}
-          cache-from: |
-            type=gha,scope=${{ matrix.cache_scope }}
-            type=registry,ref=${{ env.DOCKERHUB_IMAGE }}:dev
-          # The abx-dl base and prior ArchiveBox release already contain the
-          # compressed blobs we want. Inline metadata lets BuildKit reuse them
-          # directly; force-compression would counter-intuitively recompress
-          # the whole ~740 MiB image even when every filesystem layer is cached.
-          # The existing compressed-size gate remains the regression guard.
+          # Keep the released image as the one cache authority. A second GHA
+          # cache can hold gzip variants of identical layers, and BuildKit may
+          # prefer those over :dev's zstd blobs. That counter-intuitively makes
+          # a cache hit larger than the released image. :dev advances only
+          # after both architectures pass the release checks.
+          cache-from: type=registry,ref=${{ env.DOCKERHUB_IMAGE }}:dev
+          # Inline metadata reuses the released compressed blobs directly;
+          # force-compression would recompress the whole ~740 MiB image. The
+          # compressed-size gate below remains the regression guard.
           cache-to: |
             ${{ inputs.push_digests && 'type=inline' || '' }}
-            ${{ inputs.full_tests && format('type=gha,mode=max,scope={0}', matrix.cache_scope) || '' }}
           outputs: |
             ${{ inputs.full_tests && format('type=oci,dest={0}/archivebox-image.tar,compression=zstd,compression-level=3', runner.temp) || '' }}
             ${{ inputs.push_digests && format('type=image,"name={0},{1}",push-by-digest=true,name-canonical=true,push=true,compression=zstd,compression-level=3', env.DOCKERHUB_IMAGE, env.GHCR_IMAGE) || '' }}
@@ -237,8 +235,9 @@ jobs:
             ARCHIVEBOX_COMMIT_HASH=${{ github.sha }}
           pull: true
           platforms: ${{ matrix.platform }}
-          cache-from: type=gha,scope=${{ matrix.cache_scope }}
-          cache-to: type=gha,mode=max,scope=${{ matrix.cache_scope }}
+          # Test the same layer lineage users receive. Reusing :dev avoids a
+          # second cache whose compression can drift from the released image.
+          cache-from: type=registry,ref=${{ env.DOCKERHUB_IMAGE }}:dev
           outputs: |
             type=docker,name=${{ matrix.local_tag }}
             type=oci,dest=${{ runner.temp }}/archivebox-image.tar

.github/workflows/docs.yml

@@ -99,9 +99,10 @@ jobs:
           if [[ '${{ matrix.environment }}' != macos ]]; then
             dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docs_binaries")
           fi
-          if [[ '${{ matrix.environment }}' == ubuntu ]]; then
+          if [[ '${{ matrix.name }}' == ubuntu-AGENTS || '${{ matrix.name }}' == ubuntu-skills-archivebox-SKILL || '${{ matrix.name }}' == ubuntu-docs-Configuration || '${{ matrix.name }}' == ubuntu-docs-Setting-up-Authentication ]]; then
             dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:required_binaries")
-          elif [[ '${{ matrix.environment }}' == docker ]]; then
+          fi
+          if [[ '${{ matrix.environment }}' == docker ]]; then
             dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries")
           elif [[ '${{ matrix.environment }}' == macos ]]; then
             dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-macos-brew.json:required_binaries")

.github/workflows/duplicate-issue-detection.yml

@@ -1,96 +0,0 @@
-name: Duplicate Issue Detection
-
-on:
-  issues:
-    types: [opened]
-
-jobs:
-  check-duplicates:
-    runs-on: ubuntu-24.04
-    permissions:
-      contents: read
-      issues: write
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 1
-
-      - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: "3.13"
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-
-      - name: Resolve OpenCode and workflow tools through abxpkg
-        env:
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-        run: |
-          set -Eeuo pipefail
-          uv sync --no-cache --locked --no-sources --no-install-project
-          OPENCODE_CONFIG="$(uv run --no-cache --no-sync --no-sources python -c 'from abx_plugins import get_plugins_dir; print(get_plugins_dir() / "opencode" / "config.json")')"
-          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
-          export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
-          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
-          echo "$ABXPKG_LIB_DIR/env/bin" >> "$GITHUB_PATH"
-          opencode_env="$(
-            uv run --no-cache --no-sync --no-sources abxpkg env \
-              --install \
-              --json \
-              --lib="$ABXPKG_LIB_DIR" \
-              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \
-              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:github_binaries" \
-              --deps-from="$OPENCODE_CONFIG:required_binaries"
-          )"
-          jq_binary="$ABXPKG_LIB_DIR/env/bin/jq"
-          "$jq_binary" -r 'to_entries[] | "\(.key)=\(.value)"' \
-            <<< "$opencode_env" >> "$GITHUB_ENV"
-
-          opencode_binary="$ABXPKG_LIB_DIR/env/bin/opencode"
-          echo "OPENCODE_BINARY=$opencode_binary" >> "$GITHUB_ENV"
-          test -L "$opencode_binary"
-          test -x "$opencode_binary"
-
-      - name: Check for duplicate issues
-        env:
-          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          OPENCODE_PERMISSION: |
-            {
-              "bash": {
-                "gh issue*": "allow",
-                "gh pr*": "allow",
-                "*": "deny"
-              },
-              "webfetch": "allow"
-            }
-        run: |
-          "$OPENCODE_BINARY" run -m anthropic/claude-haiku-4-5 "A new issue has been created:
-
-          Issue number:
-          ${{ github.event.issue.number }}
-
-          Lookup this issue and search through existing issues and PRs (excluding #${{ github.event.issue.number }}) in this repository to find any potential duplicates of this new issue.
-          Consider:
-          1. Similar titles or descriptions
-          2. Same error messages or symptoms
-          3. Related functionality or components
-          4. Similar feature requests
-
-          If you find any potential duplicates, please comment on the new issue with:
-          - A brief explanation of why it might be a duplicate
-          - Links to the potentially duplicate issues or PRs
-          - A suggestion to check those issues first
-
-          Use this format for the comment:
-          'This issue might be a duplicate of existing issues. Please check:
-          - #[issue_number]: [brief description of similarity]
-
-          Feel free to ignore if none of these address your specific case.'
-
-          If no clear duplicates are found, do not comment."

AGENTS.md

@@ -15,6 +15,13 @@ ArchiveBox is the full self-hosted web archiving app. Keep this repo on the `dev
 - Trace root causes from observed behavior. Do not paper over failures with retries, wider timeouts, broad fallbacks, or looser assertions.
 - Read `README.md` for the full setup, CLI, Docker, API, and release surface.
 
+## Concurrency Contract
+
+- A collection has one orchestrator at a time. Local PID/process checks may warn about obvious same-machine duplicates, but must not claim to enforce ownership across machines or shared filesystems.
+- SQLite remains supported with concurrent short writes from CLI and server processes. Keep the SQLite database on a local filesystem, never NFS/SMB, and use short autocommit/CAS updates instead of long transactions or database locks.
+- Network calls, hook execution, filesystem migrations, and other long work belong in the orchestrator. Never hold a database transaction or lock across that work.
+- PostgreSQL and shared data directories are the path to future multi-machine scheduling. Coordinate that work through database-backed per-crawl/per-snapshot claims; do not introduce file leases or timer-based orchestrator election.
+
 ## Development Setup
 
 ```bash

Dockerfile

@@ -10,10 +10,10 @@
 #       --build-context abx-plugins=../abx-plugins \
 #       -t archivebox/abx-dl:dev
 #   docker buildx build . -f Dockerfile \
-#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:1.12.188 \
+#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:1.12.269 \
 #       -t archivebox:multistage
 
-ARG ABX_DL_IMAGE=archivebox/abx-dl:1.12.188
+ARG ABX_DL_IMAGE=archivebox/abx-dl:1.12.269
 
 FROM archivebox/sonic:1.4.9 AS sonic
 FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base
@@ -156,19 +156,20 @@ PY
     --no-install-project \
     --no-install-workspace \
     --no-sources
-ABXPKG_NO_CACHE=True abxpkg env --install --binproviders=env,apt --lib="$ABXPKG_LIB_DIR" --overrides='{"apt":{"install_args":["binutils"]}}' strip >/dev/null
+builder_abxpkg_lib_dir=/tmp/archivebox-builder-abxpkg
+ABXPKG_NO_CACHE=True abxpkg env --install --binproviders=env,apt --lib="$builder_abxpkg_lib_dir" --overrides='{"apt":{"install_args":["binutils"]}}' strip >/dev/null
 /usr/bin/find /venv/lib/python3.*/site-packages -type f -name '*.so' -print0 > /tmp/archivebox-native-libraries
 while IFS= read -r -d '' native_library; do
     magic=''
     if IFS= read -r -N 4 magic < "$native_library" && [[ "$magic" == $'\x7fELF' ]]; then
-        "$ABXPKG_LIB_DIR/env/bin/strip" --strip-unneeded "$native_library" || exit $?
+        "$builder_abxpkg_lib_dir/env/bin/strip" --strip-unneeded "$native_library" || exit $?
     fi
 done < /tmp/archivebox-native-libraries
 rm -f /tmp/archivebox-native-libraries
 rm -f /venv/bin/uv /venv/bin/uvx
-abxpkg run --binproviders=env --lib="$ABXPKG_LIB_DIR" apt-get purge -y binutils build-essential gcc libldap2-dev libsasl2-dev libssl-dev
-abxpkg run --binproviders=env --lib="$ABXPKG_LIB_DIR" apt-get autoremove -y
-/usr/bin/find "$ABXPKG_LIB_DIR/env/bin" -maxdepth 1 -type l -name strip -delete
+abxpkg run --binproviders=env --lib="$builder_abxpkg_lib_dir" apt-get purge -y binutils build-essential gcc libldap2-dev libsasl2-dev libssl-dev
+abxpkg run --binproviders=env --lib="$builder_abxpkg_lib_dir" apt-get autoremove -y
+rm -rf "$builder_abxpkg_lib_dir"
 rm -rf /venv/lib/python3.*/site-packages/pip* \
     /venv/lib/python3.*/site-packages/wheel* \
     /venv/bin/pip /venv/bin/pip3 /venv/bin/pip3.* /venv/bin/wheel

README.md

@@ -80,8 +80,6 @@ docker compose up -d --wait                                                # ini
 <br/>
 # Option B: Or use it as a plain Docker container:
 mkdir -p ~/archivebox/data && cd ~/archivebox/data
-docker run --rm -it -v "$PWD:/data" archivebox/archivebox:dev init
-docker run --rm -it -v "$PWD:/data" archivebox/archivebox:dev install
 docker run -d --name archivebox -v "$PWD:/data" -p 8000:8000 archivebox/archivebox:dev
 # open http://admin.archivebox.localhost:8000 to finish setup
 # docker run -it -v $PWD:/data archivebox/archivebox:dev add 'https://example.com'
@@ -98,7 +96,7 @@ archivebox install
 # archivebox server 0.0.0.0:8000
 <br/>
 <br/>
-# Option D: Or use the optional auto setup script to install it
+# Option D: Or use the uv install shortcut for Option C
 curl -fsSL 'https://get.archivebox.io' | bash
 </code></pre>
 <br/>
@@ -187,17 +185,16 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <br/>
 <ol>
 <li>Install <a href="https://docs.docker.com/get-docker/">Docker</a> on your system (if not already installed).</li>
-<li>Create a new empty directory and initialize your collection (can be anywhere).
+<li>Create a new empty directory and start the server, which initializes the collection automatically (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
-docker run --rm -v $PWD:/data -it archivebox/archivebox:dev init
-docker run --rm -v $PWD:/data -it archivebox/archivebox:dev install
+docker run -d --name archivebox -v $PWD:/data -p 8000:8000 archivebox/archivebox:dev
 </code></pre>
 </li>
-<li>Optional: Start the server, then open <code>/admin/</code> on the hostname or IP used to reach ArchiveBox (local example: <a href="http://admin.archivebox.localhost:8000/admin/">http://admin.archivebox.localhost:8000/admin/</a>) to create the first admin. If <code>BASE_URL</code> is not configured yet, continue through the web setup wizard.
-<pre lang="bash"><code style="white-space: pre-line">docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox:dev
+<li>Open <code>/admin/</code> on the hostname or IP used to reach ArchiveBox (local example: <a href="http://admin.archivebox.localhost:8000/admin/">http://admin.archivebox.localhost:8000/admin/</a>) to create the first admin. If <code>BASE_URL</code> is not configured yet, continue through the web setup wizard.
+<pre lang="bash"><code style="white-space: pre-line">
 # completely optional, CLI can always be used without running a server
-# docker run -v $PWD:/data -it archivebox/archivebox:dev [subcommand] [--help]
-docker run -v $PWD:/data -it archivebox/archivebox:dev help
+# docker exec archivebox archivebox [subcommand] [--help]
+docker exec archivebox archivebox help
 </code></pre>
 <i>For more info, see <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-a-docker--docker-compose-setup-%EF%B8%8F">Install: Docker Compose</a> in the Wiki. ➡️</i>
 </li>
@@ -208,18 +205,17 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 </details>
 
 <details>
-<summary><b><img src="https://user-images.githubusercontent.com/511499/117456282-08665e80-af16-11eb-91a1-8102eff54091.png" alt="curl sh automatic setup script" height="28px" align="top"/> <code>bash</code> auto-setup script</b>  (macOS/Linux)</summary>
+<summary><b><img src="https://user-images.githubusercontent.com/511499/117456282-08665e80-af16-11eb-91a1-8102eff54091.png" alt="curl sh uv install shortcut" height="28px" align="top"/> <code>bash</code> uv install shortcut</b>  (macOS/Linux/BSD)</summary>
 <br/>
 <ol>
-<li>Install <a href="https://docs.docker.com/get-docker/">Docker</a> on your system (optional, highly recommended but not required).</li>
-<li>Run the automatic setup script.
+<li>Run the shortcut for the <code>uv</code> install method. It installs <code>uv</code> first when needed.
 <pre lang="bash"><code style="white-space: pre-line">curl -fsSL 'https://get.archivebox.io' | bash</code></pre>
 <i>For more info, see <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-b-automatic-setup-script">Install: Bare Metal</a> in the Wiki. ➡️</i>
 </li>
 </ol>
 
 See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.<br/>
-See <a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/bin/setup.sh"><code>setup.sh</code></a> for the source code of the auto-install script.<br/>
+See <a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/bin/setup.sh"><code>setup.sh</code></a> for the source code of the <code>uv</code> install shortcut.<br/>
 See <a href="https://docs.sweeting.me/s/against-curl-sh">"Against curl | sh as an install method"</a> blog post for my thoughts on the shortcomings of this install method.
 <br/><br/>
 </details>
@@ -340,7 +336,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 
 <ul>
 <li>Arch: <a href="https://aur.archlinux.org/packages/archivebox/"><code>yay -S archivebox</code></a> (contributed by <a href="https://github.com/imlonghao"><code>@imlonghao</code></a>, maintained by <a href="https://github.com/jasongodev"><code>@jasongodev</code></a>)</li>
-<li>FreeBSD: <a href="https://github.com/ArchiveBox/ArchiveBox#%EF%B8%8F-easy-setup"><code>curl -fsSL 'https://get.archivebox.io' | bash</code></a> (uses <code>pkg</code> + <code>uv</code> under-the-hood)</li>
+<li>FreeBSD: <a href="https://github.com/ArchiveBox/ArchiveBox#%EF%B8%8F-easy-setup"><code>curl -fsSL 'https://get.archivebox.io' | bash</code></a> (uses <code>uv</code>)</li>
 <li>Nix: <a href="https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/misc/archivebox/default.nix"><code>nix-env --install archivebox</code></a> (contributed by <a href="https://github.com/siraben"><code>@siraben</code></a>)</li>
 <li>Guix: <a href="https://packages.guix.gnu.org/packages/archivebox/"><code>guix install archivebox</code></a> (contributed by <a href="https://github.com/rakino"><code>@rakino</code></a>)</li>
 <li>More: <a href="https://github.com/ArchiveBox/ArchiveBox/issues/new"><i>contribute another distribution...!</i></a></li>
@@ -498,12 +494,10 @@ archivebox help              # get list of archivebox subcommands that can be ru
 <pre lang="bash"><code style="white-space: pre-line">
 # make sure you have `docker-compose.yml` from the Quickstart instructions first
 <br/>
-# docker compose run --rm archivebox [subcommand] [--help]
-docker compose run --rm archivebox init
-docker compose run --rm archivebox install
-docker compose run --rm archivebox version
-docker compose run --rm archivebox help
-docker compose run --rm archivebox add 'https://example.com'
+# docker compose exec archivebox archivebox [subcommand] [--help]
+docker compose exec archivebox archivebox version
+docker compose exec archivebox archivebox help
+docker compose exec archivebox archivebox add 'https://example.com'
 # to start webserver: docker compose up
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage">Usage: Docker Compose CLI</a> wiki. ➡️</i>
@@ -515,15 +509,12 @@ docker compose run --rm archivebox add 'https://example.com'
 <summary><img src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" height="22px" align="top"/> <b>CLI Usage Examples: Docker</b></summary>
 <br/>
 <pre lang="bash"><code style="white-space: pre-line">
-# make sure you create and cd into in a new empty directory first  
+# make sure the `archivebox` server container from the Quickstart is running first
 <br/>
-# docker run -it -v $PWD:/data archivebox/archivebox:dev [subcommand] [--help]
-docker run -v $PWD:/data -it archivebox/archivebox:dev init
-docker run -v $PWD:/data -it archivebox/archivebox:dev install
-docker run -v $PWD:/data -it archivebox/archivebox:dev version
-docker run -v $PWD:/data -it archivebox/archivebox:dev help
-docker run -v $PWD:/data -it archivebox/archivebox:dev add 'https://example.com'
-# to start webserver: docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox:dev
+# docker exec archivebox archivebox [subcommand] [--help]
+docker exec archivebox archivebox version
+docker exec archivebox archivebox help
+docker exec archivebox archivebox add 'https://example.com'
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage-1">Usage: Docker CLI</a> wiki. ➡️</i>
 </details>
@@ -683,7 +674,7 @@ It uses all available methods out-of-the-box, but you can disable extractors and
 <summary><i>Expand to see the full list of ways it saves each page...</i></summary>
 
 
-<code>data/archive/{Snapshot.id}/</code><br/>
+<code>data/archive/users/{username}/snapshots/{YYYYMMDD}/{domain}/{Snapshot.id}/</code><br/>
 <ul>
 <li><strong>Index:</strong> <code>index.html</code> &amp; <code>index.json</code> HTML and JSON index files containing metadata and details</li>
 <li><strong>Title</strong>, <strong>Favicon</strong>, <strong>Headers</strong> Response headers, site favicon, and parsed site title</li>
@@ -850,7 +841,7 @@ The on-disk layout is optimized to be easy to browse by hand and durable long-te
                                 ...
 </code></pre>
 
-Each snapshot subfolder includes static metadata and plain extractor output files. ArchiveBox also maintains a backwards-compatible <code>data/archive/TIMESTAMP</code> symlink for each snapshot.
+Each snapshot subfolder includes static metadata and plain extractor output files. Current releases do not create top-level <code>data/archive/TIMESTAMP</code> projections; legacy timestamp directories are migrated into the user-scoped tree by <code>archivebox update --migrate-only</code>.
 
 <h4>Learn More</h4>
 <ul>
@@ -1346,14 +1337,11 @@ archivebox server 0.0.0.0:8000
 # inside the container will reload and pick up your changes
 ./bin/build_docker.sh dev
 
-docker run -it -v $PWD/data:/data archivebox/archivebox:dev init
-docker run -it -v $PWD/data:/data archivebox/archivebox:dev install
-
 # Run the development server w/ autoreloading (but no bg workers)
-docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server --debug --reload 0.0.0.0:8000
+docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server --init --debug --reload 0.0.0.0:8000
 
 # Run the production server (with bg workers but no autoreloading)
-docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server
+docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server --init
 
 # (remove the --reload flag and add the --nothreading flag when profiling with the django debug toolbar)
 # When using --reload, make sure any files you create can be read by the user in the Docker container, eg with 'chmod a+rX'.
@@ -1414,7 +1402,7 @@ services:
 
 # or with plain Docker:
 docker build -t archivebox:dev https://github.com/ArchiveBox/ArchiveBox.git#dev
-docker run -it -v $PWD:/data archivebox:dev init
+docker run -it -v $PWD:/data -p 8000:8000 archivebox:dev
 
 # or with uv:
 uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'

archivebox/api/v1_cli.py

@@ -133,6 +133,15 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
             validate_url(args.urls[0])
         except ValueError:
             submitted_urls = args.urls[0]
+    else:
+        submitted_urls = []
+        for url in args.urls:
+            try:
+                submitted_urls.append(validate_url(url))
+            except ValueError:
+                continue
+        if not submitted_urls:
+            raise HttpError(400, "No valid URLs were submitted")
     crawl, snapshots = add(
         urls=submitted_urls,
         snapshot_ids=args.snapshot_ids,
@@ -156,7 +165,7 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
         "crawl_id": str(crawl.id),
         "num_snapshots": len(snapshot_ids),
         "snapshot_ids": snapshot_ids,
-        "queued_urls": args.urls,
+        "queued_urls": args.urls if isinstance(submitted_urls, str) else submitted_urls,
     }
     stdout = request.__dict__.get("stdout")
     stderr = request.__dict__.get("stderr")

archivebox/api/v1_core.py

@@ -11,7 +11,7 @@ from uuid import UUID
 from typing import Union, Any, Annotated
 from datetime import datetime, time
 
-from django.db import transaction
+from django.db import IntegrityError
 from django.db.models import Model, Q
 from django.http import HttpRequest, HttpResponse
 from django.http.multipartparser import MultiPartParser, MultiPartParserError
@@ -342,21 +342,10 @@ def _parse_archiveresult_upload_int(value: str, field_name: str, *, default: int
 
 
 def _summarize_archiveresult_output_files(output_files: dict[str, dict[str, Any]]) -> tuple[int, str]:
-    mime_sizes: dict[str, int] = defaultdict(int)
-    total_size = 0
-    for metadata in output_files.values():
-        if not isinstance(metadata, dict):
-            continue
-        try:
-            size = max(int(metadata.get("size") or 0), 0)
-        except (TypeError, ValueError):
-            size = 0
-        mime_type = str(metadata.get("mimetype") or "").strip()
-        total_size += size
-        if mime_type and size:
-            mime_sizes[mime_type] += size
-    output_mimetypes = ",".join(mime for mime, _size in sorted(mime_sizes.items(), key=lambda item: item[1], reverse=True))
-    return total_size, output_mimetypes
+    from abx_dl.output_files import OutputManifest
+
+    manifest = OutputManifest.from_value(output_files)
+    return manifest.total_size, ",".join(manifest.mimetypes)
 
 
 def _get_snapshot_by_ref(snapshot_id: str):
@@ -533,51 +522,48 @@ def create_archiveresult(
         "plugin": plugin_name,
         "hook_name": hook,
     }
-    existing_result = ArchiveResult.objects.filter(**result_lookup).first()
-    existing_output_files = dict(existing_result.output_files or {}) if existing_result else {}
-    output_files = _write_archiveresult_files(
+    uploaded_output_files = _write_archiveresult_files(
         request,
         snapshot,
         plugin_name,
-        existing_output_files=existing_output_files,
         allow_empty=True,
     )
-    now = timezone.now()
-
-    with transaction.atomic():
-        Snapshot.objects.select_for_update().get(pk=snapshot.pk)
-        existing_result = ArchiveResult.objects.filter(**result_lookup).first()
-        if existing_result:
-            output_files = {
-                **dict(existing_result.output_files or {}),
-                **output_files,
-            }
-            result = existing_result
-        else:
-            existing_result = None
-            result = ArchiveResult(
-                snapshot=snapshot,
-                plugin=plugin_name,
-                hook_name=hook,
-            )
-
-        if (
-            existing_result
-            and normalized_status == ArchiveResult.StatusChoices.STARTED
-            and existing_result.status != ArchiveResult.StatusChoices.STARTED
-        ):
-            normalized_status = existing_result.status
+    result = ArchiveResult.objects.filter(**result_lookup).first()
+    for _attempt in range(3):
+        output_files = {
+            **(result.output_file_map() if result else {}),
+            **uploaded_output_files,
+        }
         output_size, output_mimetypes = _summarize_archiveresult_output_files(output_files)
         output_file_paths = list(output_files.keys())
-        result.status = normalized_status
-        result.output_str = output_str or (output_file_paths[0] if output_file_paths else "")
-        result.output_json = parsed_output_json
-        result.output_files = output_files
-        result.output_size = output_size
-        result.output_mimetypes = output_mimetypes
-        result.start_ts = result.start_ts or now
-        result.end_ts = now
-        result.save()
+        result_status = normalized_status
+        if result and result_status == ArchiveResult.StatusChoices.STARTED and result.status != ArchiveResult.StatusChoices.STARTED:
+            result_status = result.status
+        now = timezone.now()
+        values = {
+            "status": result_status,
+            "output_str": output_str or (output_file_paths[0] if output_file_paths else ""),
+            "output_json": parsed_output_json,
+            "output_files": output_files,
+            "output_size": output_size,
+            "output_mimetypes": output_mimetypes,
+            "start_ts": result.start_ts or now if result else now,
+            "end_ts": now,
+        }
+        if result:
+            if result.safe_update(values):
+                break
+            continue
+        result, created = ArchiveResult.get_or_create_by_hook(
+            snapshot,
+            plugin_name,
+            hook,
+            defaults=values,
+        )
+        if created:
+            break
+    else:
+        raise HttpError(409, "ArchiveResult changed while upload metadata was being updated")
 
     if result.status != ArchiveResult.StatusChoices.STARTED:
         _queue_archiveresult_snapshot_maintenance(snapshot)
@@ -591,44 +577,46 @@ def patch_archiveresult(
 ):
     """Append or replace files on an existing ArchiveResult."""
     result = ArchiveResult.objects.select_related("snapshot__crawl__created_by").get(_uuid_ref_query("id", archiveresult_id))
-    output_files = _write_archiveresult_files(
+    uploaded_output_files = _write_archiveresult_files(
         request,
         result.snapshot,
         result.plugin,
-        existing_output_files=result.output_file_map(),
     )
-    latest_result = ArchiveResult.objects.only("output_files", "status").get(pk=result.pk)
-    output_files = {
-        **latest_result.output_file_map(),
-        **output_files,
-    }
-    output_size, output_mimetypes = _summarize_archiveresult_output_files(output_files)
-
-    update_fields = ["output_files", "output_size", "output_mimetypes", "end_ts", "modified_at"]
-    result.output_files = output_files
-    result.output_size = output_size
-    result.output_mimetypes = output_mimetypes
-    result.end_ts = timezone.now()
     output_str = _get_archiveresult_upload_form_value(request, "output_str")
     status = _get_archiveresult_upload_form_value(request, "status")
     output_json = _get_archiveresult_upload_form_value(request, "output_json")
-    if output_str:
-        result.output_str = output_str
-        update_fields.append("output_str")
-    if status:
-        normalized_status = ArchiveResult.normalize_status(status)
-        if normalized_status == ArchiveResult.StatusChoices.STARTED and latest_result.status != ArchiveResult.StatusChoices.STARTED:
-            normalized_status = latest_result.status
-        result.status = normalized_status
-        update_fields.append("status")
-    elif latest_result.status == ArchiveResult.StatusChoices.QUEUED and ArchiveResult.output_files_upload_complete(output_files):
-        result.status = ArchiveResult.StatusChoices.SUCCEEDED
-        update_fields.append("status")
-    if output_json:
-        result.output_json = _parse_archiveresult_output_json(output_json)
-        update_fields.append("output_json")
+    parsed_output_json = _parse_archiveresult_output_json(output_json) if output_json else None
+
+    if not ArchiveResult.output_files_upload_complete(uploaded_output_files):
+        result.output_files = {**result.output_file_map(), **uploaded_output_files}
+        result.output_size, result.output_mimetypes = _summarize_archiveresult_output_files(result.output_files)
+        return result
+
+    for _attempt in range(3):
+        output_files = {**result.output_file_map(), **uploaded_output_files}
+        output_size, output_mimetypes = _summarize_archiveresult_output_files(output_files)
+        values: dict[str, Any] = {
+            "output_files": output_files,
+            "output_size": output_size,
+            "output_mimetypes": output_mimetypes,
+            "end_ts": timezone.now(),
+        }
+        if output_str:
+            values["output_str"] = output_str
+        if status:
+            normalized_status = ArchiveResult.normalize_status(status)
+            if normalized_status == ArchiveResult.StatusChoices.STARTED and result.status != ArchiveResult.StatusChoices.STARTED:
+                normalized_status = result.status
+            values["status"] = normalized_status
+        elif result.status == ArchiveResult.StatusChoices.QUEUED:
+            values["status"] = ArchiveResult.StatusChoices.SUCCEEDED
+        if output_json:
+            values["output_json"] = parsed_output_json
+        if result.safe_update(values):
+            break
+    else:
+        raise HttpError(409, "ArchiveResult changed while upload metadata was being updated")
 
-    result.save(update_fields=update_fields)
     if result.status != ArchiveResult.StatusChoices.STARTED:
         _queue_archiveresult_snapshot_maintenance(result.snapshot)
 
@@ -945,52 +933,47 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
             created_by=request.user if isinstance(request.user, User) else None,
         )
 
-    # Browser clients sync metadata and upload capture artifacts immediately
-    # after queueing a URL. The runner holds the crawl lifecycle lock for the
-    # whole crawl, so idempotently posting an already-created snapshot must not
-    # wait behind that potentially long-running archive job.
-    existing_snapshot = Snapshot.objects.filter(url=data.url, crawl=crawl).first()
-    if existing_snapshot is not None and (status is None or existing_snapshot.status == status):
-        if data.title is not None and existing_snapshot.title != data.title:
-            Snapshot.objects.filter(pk=existing_snapshot.pk).update(title=data.title, modified_at=timezone.now())
-            existing_snapshot.title = data.title
-        if tags:
-            existing_snapshot.save_tags(tags)
-        setattr(request, "with_archiveresults", False)
-        return existing_snapshot
+    # Browser uploads must not wait behind the runner's crawl-wide lifecycle
+    # lock. The unique insert recovery and CAS update below are the request-side
+    # coordination boundary for this idempotent metadata sync.
+    snapshot = Snapshot.objects.filter(url=data.url, crawl=crawl).first()
+    if snapshot is None:
+        try:
+            snapshot = Snapshot.objects.create(
+                url=data.url,
+                crawl=crawl,
+                depth=data.depth,
+                title=data.title,
+                timestamp=str(timezone.now().timestamp()),
+                status=status or Snapshot.StatusChoices.QUEUED,
+                retry_at=timezone.now(),
+            )
+        except IntegrityError:
+            snapshot = Snapshot.objects.filter(url=data.url, crawl=crawl).first()
+            if snapshot is None:
+                raise
 
-    with crawl_lifecycle_lock(str(crawl.id)):
-        snapshot_defaults = {
-            "depth": data.depth,
-            "title": data.title,
-            "timestamp": str(timezone.now().timestamp()),
-            "status": status or Snapshot.StatusChoices.QUEUED,
-            "retry_at": timezone.now(),
-        }
-        snapshot, _ = Snapshot.objects.get_or_create(
-            url=data.url,
-            crawl=crawl,
-            defaults=snapshot_defaults,
+    for _attempt in range(3):
+        updates: dict[str, Any] = {}
+        if data.title is not None and snapshot.title != data.title:
+            updates["title"] = data.title
+        if status is not None and snapshot.status != status:
+            updates["status"] = status
+        if not updates or snapshot.safe_update(updates, extra_filter={"modified_at": snapshot.modified_at}):
+            break
+    else:
+        raise HttpError(409, "Snapshot changed while metadata was being updated")
+
+    if tags:
+        snapshot.save_tags(
+            tags,
+            created_by=request.user if isinstance(request.user, User) else None,
         )
 
-        update_fields: list[str] = []
-        if data.title is not None and snapshot.title != data.title:
-            snapshot.title = data.title
-            update_fields.append("title")
-        if status is not None and snapshot.status != status:
-            snapshot.status = status
-            update_fields.append("status")
-        if update_fields:
-            update_fields.append("modified_at")
-            snapshot.save(update_fields=update_fields)
-
-        if tags:
-            snapshot.save_tags(tags)
-
-        try:
-            snapshot.ensure_crawl_symlink()
-        except Exception:
-            pass
+    try:
+        snapshot.ensure_crawl_symlink()
+    except Exception:
+        pass
 
     setattr(request, "with_archiveresults", False)
     return snapshot
@@ -1038,7 +1021,10 @@ def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateS
             update_fields.append("retry_at")
 
         if tags is not None:
-            snapshot.save_tags(normalize_tag_list(tags))
+            snapshot.save_tags(
+                normalize_tag_list(tags),
+                created_by=request.user if isinstance(request.user, User) else None,
+            )
 
         if payload.get("status") == Snapshot.StatusChoices.SEALED:
             snapshot.cancel()
@@ -1425,8 +1411,7 @@ def tags_add_to_snapshot(request: HttpRequest, data: TagSnapshotRequestSchema):
     else:
         raise HttpError(400, "Either tag_name or tag_id is required")
 
-    # Add the tag to the snapshot
-    snapshot.tags.add(tag.pk)
+    snapshot.add_tag_ids([tag.pk])
 
     return {
         "success": True,
@@ -1454,8 +1439,7 @@ def tags_remove_from_snapshot(request: HttpRequest, data: TagSnapshotRequestSche
     else:
         raise HttpError(400, "Either tag_name or tag_id is required")
 
-    # Remove the tag from the snapshot
-    snapshot.tags.remove(tag.pk)
+    snapshot.remove_tag_ids([tag.pk])
 
     return {
         "success": True,

archivebox/api/v1_crawls.py

@@ -1,6 +1,5 @@
 __package__ = "archivebox.api"
 
-import json
 from pathlib import Path
 from uuid import UUID
 from datetime import datetime
@@ -23,7 +22,7 @@ from archivebox.core.permissions import (
 )
 from archivebox.config.common import get_config
 from archivebox.crawls.models import Crawl
-from archivebox.misc.util import filter_queryset_by_uuid_substring
+from archivebox.misc.util import filter_queryset_by_uuid_substring, validate_url
 
 from .auth import API_AUTH_METHODS, authenticated_user_from_request
 
@@ -113,7 +112,10 @@ def get_crawls(request: HttpRequest):
 
 @router.post("/crawls", response=CrawlSchema, url_name="create_crawl")
 def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
-    urls = [url.strip() for url in data.urls if url and url.strip()]
+    try:
+        urls = [validate_url(url) for url in data.urls if url and url.strip()]
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
     if not urls:
         raise HttpError(400, "At least one URL is required")
     if data.max_depth not in (0, 1, 2, 3, 4):
@@ -121,9 +123,10 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
 
     tags = normalize_tag_list(data.tags, data.tags_str)
     config = dict(data.config or {})
+    config.setdefault("PARSER", "url_list")
     config.setdefault("PERMISSIONS", str(get_config().PERMISSIONS))
     crawl = Crawl.objects.create(
-        urls="\n".join(json.dumps({"type": "CrawlSeed", "url": url, "depth": 0}, separators=(",", ":")) for url in urls),
+        urls="\n".join(urls),
         max_depth=data.max_depth,
         tags_str=",".join(tags),
         label=data.label,

archivebox/cli/archivebox_add.py

@@ -6,7 +6,6 @@ __command__ = "archivebox add"
 
 import sys
 import os
-import json
 from pathlib import Path
 
 from typing import Any, TYPE_CHECKING
@@ -71,17 +70,7 @@ def add(
     created_by_id: int | None = None,
     config: dict[str, Any] | None = None,
 ) -> tuple[Crawl, QuerySet[Snapshot]]:
-    """Add a new URL or list of URLs to your archive.
-
-    The flow is:
-    1. Save URLs to sources file
-    2. Create Crawl with URLs and max_depth
-    3. Crawl runner creates Snapshots from Crawl URLs (depth=0)
-    4. Crawl runner runs parser extractors on root snapshots
-    5. Parser extractors output to urls.jsonl
-    6. URLs are added to Crawl.urls and child Snapshots are created
-    7. Repeat until max_depth is reached
-    """
+    """Add a URL list or imported URL document to a new Crawl."""
 
     from rich import print
 
@@ -125,12 +114,12 @@ def add(
     created_by_id = created_by_id or get_or_create_system_user_pk()
     started_at = timezone.now()
 
-    use_internal_input_root = isinstance(urls, str)
-    source_text = (
-        urls
-        if use_internal_input_root
-        else "\n".join(json.dumps({"type": "CrawlSeed", "url": str(url), "depth": 0}, separators=(",", ":")) for url in urls)
-    )
+    if isinstance(urls, str):
+        source_text = urls
+    else:
+        from archivebox.misc.util import validate_url
+
+        source_text = "\n".join(validate_url(str(url)) for url in urls)
 
     # 2. Create a new Crawl with inline URLs
     # Foreground add must claim runner ownership before publishing runnable
@@ -187,12 +176,7 @@ def add(
     try:
         crawl = Crawl.objects.create(
             urls=source_text,
-            # Stdin/import text gets an extra hop because the synthetic
-            # archivebox://internal root lives at depth 0 and parser-discovered
-            # URLs land at depth 1; direct URL args become the depth=0 input
-            # snapshots themselves so --depth=N matches the deepest hop the user
-            # asked for.
-            max_depth=depth + 1 if use_internal_input_root else depth,
+            max_depth=depth,
             tags_str=tag,
             persona_id=persona_obj.id,
             label=f"{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]",
@@ -213,9 +197,8 @@ def add(
     print(f"[green]\\[+] Created Crawl {crawl.id} with max_depth={depth}[/green]")
     print(f"    [dim]First URL: {first_url}[/dim]")
 
-    # 3. The CrawlMachine will create Snapshots from all URLs when started
-    #    Parser extractors run on snapshots and discover more URLs
-    #    Discovered URLs become child Snapshots (depth+1)
+    # The runner parses Crawl.urls after claiming the Crawl, then persists the
+    # resulting depth-0 Snapshot facts. Recursive discoveries start at depth 1.
 
     if index_only:
         print("[yellow]\\[*] Index-only mode - URLs queued, runner not started[/yellow]")
@@ -421,13 +404,13 @@ def main(**kwargs):
         if only_new is not None:
             kwargs["config"] = {"ONLY_NEW": bool(only_new)}
         if extract:
-            from abx_dl.models import discover_plugins, plugins_matching_output
+            from archivebox.plugins.discovery import get_plugin_catalog
 
-            all_plugins = discover_plugins()
+            catalog = get_plugin_catalog()
             tokens = [token.strip() for token in extract.split(",") if token.strip()]
-            plugin_names = {name.lower(): name for name in all_plugins}
+            plugin_names = {name.lower(): name for name in catalog}
             selected = [plugin_names[token.lower()] for token in tokens if token.lower() in plugin_names]
-            selected += plugins_matching_output(all_plugins, tokens)
+            selected += catalog.matching_output(tokens)
             if not selected:
                 raise click.UsageError(f"No plugins found matching extract types: {extract}")
             existing = [token.strip() for token in (kwargs.get("plugins") or "").split(",") if token.strip()]

archivebox/cli/archivebox_archiveresult.py

@@ -6,13 +6,13 @@ archivebox archiveresult <action> [args...] [--filters]
 Manage ArchiveResult records (plugin extraction results).
 
 Actions:
-    create  - Create ArchiveResults for Snapshots (queue extractions)
+    create  - Emit plugin extraction request records for Snapshots
     list    - List ArchiveResults as JSONL (with optional filters)
     update  - Update ArchiveResults from stdin JSONL
     delete  - Delete ArchiveResults from stdin JSONL
 
 Examples:
-    # Create ArchiveResults for snapshots (queue for extraction)
+    # Emit extraction requests; `archivebox run` schedules their parent snapshots
     archivebox snapshot list --status=queued | archivebox archiveresult create
     archivebox archiveresult create --plugin=screenshot --snapshot-id=<uuid>
 
@@ -20,9 +20,6 @@ Examples:
     archivebox archiveresult list --status=failed
     archivebox archiveresult list --plugin=screenshot --status=succeeded
 
-    # Update (reset failed extractions to queued)
-    archivebox archiveresult list --status=failed | archivebox archiveresult update --status=queued
-
     # Delete
     archivebox archiveresult list --plugin=singlefile | archivebox archiveresult delete --yes
 
@@ -157,7 +154,7 @@ def create_archiveresults(
                 write_record(build_archiveresult_request(snapshot.id, plugin_name, hook_name=hook_name, status=status))
             created_count += 1
 
-    rprint(f"[green]Created {created_count} archive result request records[/green]", file=sys.stderr)
+    rprint(f"[green]Created {created_count} extraction request records[/green]", file=sys.stderr)
     return 0
 
 
@@ -341,7 +338,7 @@ def main():
 @click.option("--plugin", "-p", help="Plugin name (e.g., screenshot, singlefile)")
 @click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
 def create_cmd(snapshot_id: str | None, plugin: str | None, status: str):
-    """Create ArchiveResults for Snapshots from stdin JSONL."""
+    """Emit Snapshot plugin extraction requests as JSONL."""
     sys.exit(create_archiveresults(snapshot_id=snapshot_id, plugin=plugin, status=status))
 
 

archivebox/cli/archivebox_extract.py

@@ -1,100 +1,135 @@
 #!/usr/bin/env python3
 
-"""
-archivebox extract [snapshot_ids...] [--plugins=NAMES]
-
-Run plugins on Snapshots. Accepts snapshot IDs as arguments, from stdin, or via JSONL.
-
-Input formats:
-    - Snapshot UUIDs (one per line)
-    - JSONL: {"type": "Snapshot", "id": "...", "url": "..."}
-    - JSONL: {"type": "ArchiveResult", "snapshot_id": "...", "plugin": "..."}
-
-Output (JSONL):
-    {"type": "ArchiveResult", "id": "...", "snapshot_id": "...", "plugin": "...", "status": "..."}
-
-Examples:
-    # Extract specific snapshot
-    archivebox extract 01234567-89ab-cdef-0123-456789abcdef
-
-    # Pipe from snapshot command
-    archivebox snapshot https://example.com | archivebox extract
-
-    # Run specific plugins only
-    archivebox extract --plugins=screenshot,singlefile 01234567-89ab-cdef-0123-456789abcdef
-
-    # Chain commands
-    archivebox crawl https://example.com | archivebox snapshot | archivebox extract
-"""
+"""Run abx-dl snapshot hooks for existing ArchiveBox snapshots."""
 
 __package__ = "archivebox.cli"
 __command__ = "archivebox extract"
 
 import sys
 from collections import defaultdict
-from itertools import product
+from contextlib import redirect_stdout
 
 import rich_click as click
 
 
-def process_archiveresult_by_id(archiveresult_id: str) -> int:
-    """
-    Re-run extraction for a single ArchiveResult by ID.
+def _resolve_requests(records: list[dict], plugins: str) -> tuple[dict[str, set[str]], set[str]]:
+    """Resolve CLI records to snapshot-level execution requests.
 
-    ArchiveResults are projected status rows, not queued work items. Re-running
-    a single result means resetting that row and queueing its parent snapshot
-    through the shared crawl runner with the corresponding plugin selected.
+    ArchiveResult input is accepted as a convenient reference to its parent
+    Snapshot and plugin. It is never reset or converted into queued work.
     """
-    from rich import print as rprint
-    from archivebox.core.models import ArchiveResult
     from archivebox.api.v1_core import _uuid_ref_query
+    from archivebox.core.models import ArchiveResult, Snapshot
+    from archivebox.misc.jsonl import TYPE_ARCHIVERESULT
+
+    explicit_plugins = {name.strip() for name in plugins.split(",") if name.strip()}
+    requested: dict[str, set[str]] = defaultdict(set)
+    missing: set[str] = set()
+
+    for record in records:
+        record_type = record.get("type")
+        record_id = str(record.get("id") or "")
+
+        if record_type == TYPE_ARCHIVERESULT and record.get("snapshot_id"):
+            requested[str(record["snapshot_id"])].update(explicit_plugins or {str(record.get("plugin") or "")})
+            requested[str(record["snapshot_id"])].discard("")
+            continue
+
+        if record_type == TYPE_ARCHIVERESULT:
+            result = ArchiveResult.objects.filter(_uuid_ref_query("id", record_id)).only("snapshot_id", "plugin").first()
+            if result is not None:
+                requested[str(result.snapshot_id)].update(explicit_plugins or {result.plugin})
+                continue
+
+        snapshot_id = str(record.get("snapshot_id") or record_id or "")
+        snapshot = Snapshot.objects.filter(id=snapshot_id).only("id").first() if snapshot_id else None
+        if snapshot is None and record.get("url"):
+            snapshot = Snapshot.objects.filter(url=record["url"]).order_by("-created_at").only("id").first()
+        # Bare UUID CLI arguments are parsed as Snapshot records because the
+        # input layer cannot know which model owns them. Preserve the public
+        # convenience of passing an ArchiveResult ID by trying that reference
+        # only after the Snapshot lookup misses.
+        if snapshot is None and record_id and not record.get("url"):
+            result = ArchiveResult.objects.filter(_uuid_ref_query("id", record_id)).only("snapshot_id", "plugin").first()
+            if result is not None:
+                requested[str(result.snapshot_id)].update(explicit_plugins or {result.plugin})
+                continue
+        if snapshot is None:
+            missing.add(snapshot_id or str(record.get("url") or ""))
+            continue
+        requested[str(snapshot.id)].update(explicit_plugins)
+
+    return requested, missing
+
+
+def _run_snapshot_requests(requested: dict[str, set[str]], *, wait: bool, show_progress: bool) -> int:
+    from django.utils import timezone
+    from rich import print as rprint
+
+    from archivebox.core.models import Snapshot
     from archivebox.services.runner import run_crawl
 
-    try:
-        archiveresult = ArchiveResult.objects.get(_uuid_ref_query("id", archiveresult_id))
-    except ArchiveResult.DoesNotExist:
-        rprint(f"[red]ArchiveResult {archiveresult_id} not found[/red]", file=sys.stderr)
+    snapshots = {str(snapshot.id): snapshot for snapshot in Snapshot.objects.filter(id__in=requested).select_related("crawl")}
+    if not snapshots:
+        rprint("[red]No snapshots to process[/red]", file=sys.stderr)
         return 1
 
-    rprint(f"[blue]Extracting {archiveresult.plugin} for {archiveresult.snapshot.url}[/blue]", file=sys.stderr)
+    from archivebox.config.common import get_config
+    from archivebox.plugins.discovery import get_enabled_plugins
 
-    try:
-        archiveresult.reset_for_retry()
-        snapshot = archiveresult.snapshot
-        snapshot.queue_for_extraction()
-        crawl = snapshot.crawl
-        if not crawl.claim_processing_lock(lock_seconds=10):
-            rprint(
-                f"[yellow]Crawl {crawl.id} is already owned by another runner[/yellow]",
-                file=sys.stderr,
-            )
-            return 1
+    for snapshot_id, plugin_names in requested.items():
+        snapshot = snapshots.get(snapshot_id)
+        if snapshot is not None and not plugin_names:
+            plugin_names.update(get_enabled_plugins(config=get_config(crawl=snapshot.crawl, snapshot=snapshot)))
 
+    if wait and any("search_backend_sonic" in plugin_names for plugin_names in requested.values()):
+        from archivebox.core.takeover_util import ensure_daemon_stack
+
+        with redirect_stdout(sys.stderr):
+            ensure_daemon_stack(reason="Sonic snapshot indexing")
+
+    # Explicit extraction resumes open/paused snapshots at the snapshot level.
+    # Sealed snapshots stay sealed during targeted maintenance backfills.
+    if wait:
+        for snapshot in snapshots.values():
+            if snapshot.status != Snapshot.StatusChoices.SEALED:
+                snapshot.update_and_requeue(
+                    status=Snapshot.StatusChoices.QUEUED,
+                    retry_at=timezone.now(),
+                )
+
+    if not wait:
+        for snapshot_id, plugin_names in requested.items():
+            snapshot = snapshots.get(snapshot_id)
+            if snapshot is None:
+                continue
+            if plugin_names:
+                snapshot.schedule_plugin_run(plugin_names, when=timezone.now())
+            elif snapshot.status == Snapshot.StatusChoices.SEALED:
+                snapshot.update_and_requeue(retry_at=timezone.now())
+            else:
+                snapshot.update_and_requeue(
+                    status=Snapshot.StatusChoices.QUEUED,
+                    retry_at=timezone.now(),
+                )
+        if show_progress:
+            rprint(f"[blue]Queued {len(snapshots)} snapshots for extraction[/blue]", file=sys.stderr)
+        return 0
+
+    grouped: dict[tuple[str, tuple[str, ...]], list[str]] = defaultdict(list)
+    for snapshot_id, plugin_names in requested.items():
+        snapshot = snapshots.get(snapshot_id)
+        if snapshot is not None:
+            grouped[(str(snapshot.crawl_id), tuple(sorted(plugin_names)))].append(snapshot_id)
+
+    for (crawl_id, plugin_names), snapshot_ids in grouped.items():
         run_crawl(
-            str(snapshot.crawl_id),
-            snapshot_ids=[str(snapshot.id)],
-            selected_plugins=[archiveresult.plugin],
-            selected_plugins_are_explicit=False,
+            crawl_id,
+            snapshot_ids=sorted(snapshot_ids),
+            selected_plugins=list(plugin_names) or None,
+            show_progress=show_progress,
         )
-        archiveresult.refresh_from_db()
-
-        if archiveresult.status == ArchiveResult.StatusChoices.SUCCEEDED:
-            print(f"[green]Extraction succeeded: {archiveresult.output_str}[/green]")
-            return 0
-        elif archiveresult.status == ArchiveResult.StatusChoices.NORESULTS:
-            print(f"[dim]Extraction completed with no results: {archiveresult.output_str}[/dim]")
-            return 0
-        elif archiveresult.status == ArchiveResult.StatusChoices.FAILED:
-            print(f"[red]Extraction failed: {archiveresult.output_str}[/red]", file=sys.stderr)
-            return 1
-        else:
-            # Still in progress or backoff - not a failure
-            print(f"[yellow]Extraction status: {archiveresult.status}[/yellow]")
-            return 0
-
-    except Exception as e:
-        print(f"[red]Extraction error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
-        return 1
+    return 0
 
 
 def run_plugins(
@@ -104,384 +139,64 @@ def run_plugins(
     wait: bool = True,
     emit_results: bool = True,
     show_progress: bool = True,
-    preserve_queued: bool = False,
 ) -> int:
-    """
-    Run plugins on Snapshots from input.
-
-    Reads Snapshot IDs or JSONL from args/stdin, runs plugins, outputs JSONL.
-
-    Exit codes:
-        0: Success
-        1: Failure
-    """
+    """Execute selected plugins through the snapshot-level runner."""
     from rich import print as rprint
-    from django.utils import timezone
 
-    from archivebox.misc.jsonl import (
-        read_args_or_stdin,
-        write_record,
-        TYPE_SNAPSHOT,
-        TYPE_ARCHIVERESULT,
-    )
     from archivebox.core.models import Snapshot
-    from archivebox.core.models import ArchiveResult
-    from archivebox.services.runner import run_crawl
-    from abx_dl.models import discover_plugins
+    from archivebox.misc.jsonl import read_args_or_stdin, write_record
 
-    is_tty = sys.stdout.isatty()
-
-    # Parse comma-separated plugins list once (reused in creation and filtering)
-    plugins_list = [p.strip() for p in plugins.split(",") if p.strip()] if plugins else []
-
-    # Parse stdin/args exactly once per CLI invocation.
-    # `main()` may already have consumed stdin to distinguish Snapshot input from
-    # ArchiveResult IDs; if so, it must pass the parsed records through here
-    # instead of asking this helper to reread an already-drained pipe.
     if records is None:
         records = list(read_args_or_stdin(args))
-
     if not records:
         rprint("[yellow]No snapshots provided. Pass snapshot IDs as arguments or via stdin.[/yellow]", file=sys.stderr)
         return 1
 
-    # Gather snapshot IDs and optional plugin constraints to process
-    snapshot_ids = set()
-    requested_plugins_by_snapshot: dict[str, set[str]] = defaultdict(set)
-    requested_hooks_by_snapshot: dict[str, set[tuple[str, str]]] = defaultdict(set)
-    plugin_level_requests_by_snapshot: dict[str, set[str]] = defaultdict(set)
-    for record in records:
-        record_type = record.get("type")
-
-        if record_type == TYPE_SNAPSHOT:
-            snapshot_id = record.get("id")
-            if snapshot_id:
-                snapshot_ids.add(str(snapshot_id))
-            elif record.get("url"):
-                # Look up by URL (get most recent if multiple exist)
-                snap = Snapshot.objects.filter(url=record["url"]).order_by("-created_at").first()
-                if snap:
-                    snapshot_ids.add(str(snap.id))
-                else:
-                    rprint(f"[yellow]Snapshot not found for URL: {record['url']}[/yellow]", file=sys.stderr)
-
-        elif record_type == TYPE_ARCHIVERESULT:
-            snapshot_id = record.get("snapshot_id")
-            if snapshot_id:
-                snapshot_ids.add(str(snapshot_id))
-                plugin_name = record.get("plugin")
-                if plugin_name and not plugins_list:
-                    snapshot_key = str(snapshot_id)
-                    plugin_key = str(plugin_name)
-                    requested_plugins_by_snapshot[snapshot_key].add(plugin_key)
-                    hook_name = str(record.get("hook_name") or "")
-                    if hook_name:
-                        requested_hooks_by_snapshot[snapshot_key].add((plugin_key, hook_name))
-                    else:
-                        plugin_level_requests_by_snapshot[snapshot_key].add(plugin_key)
-
-        elif "id" in record:
-            # Assume it's a snapshot ID
-            snapshot_ids.add(str(record["id"]))
-
-    if not snapshot_ids:
-        rprint("[red]No valid snapshot IDs found in input[/red]", file=sys.stderr)
+    requested, missing = _resolve_requests(records, plugins)
+    for value in sorted(missing):
+        rprint(f"[yellow]Snapshot or ArchiveResult not found: {value}[/yellow]", file=sys.stderr)
+    if not requested:
         return 1
 
-    existing_snapshots = list(Snapshot.objects.filter(id__in=snapshot_ids).values_list("id", "crawl_id"))
-    existing_snapshot_ids = {str(snapshot_id) for snapshot_id, _crawl_id in existing_snapshots}
-    existing_crawl_ids = {str(crawl_id) for _snapshot_id, crawl_id in existing_snapshots}
-    missing_snapshot_ids = sorted(str(snapshot_id) for snapshot_id in snapshot_ids - existing_snapshot_ids)
-    for snapshot_id in missing_snapshot_ids:
-        rprint(f"[yellow]Snapshot {snapshot_id} not found[/yellow]", file=sys.stderr)
-
-    # Queue only the target plugin rows. Bulk updates keep large reindex runs
-    # from doing one SELECT+UPDATE per snapshot/plugin before hooks even start.
-    requested_pairs: set[tuple[str, str]] = set()
-    if plugins_list:
-        requested_pairs.update((snapshot_id, plugin_name) for snapshot_id, plugin_name in product(existing_snapshot_ids, plugins_list))
-    else:
-        requested_pairs.update(
-            (snapshot_id, plugin_name)
-            for snapshot_id, plugin_names in requested_plugins_by_snapshot.items()
-            if snapshot_id in existing_snapshot_ids
-            for plugin_name in plugin_names
-        )
-    plugins_by_name = discover_plugins(runtime="archivebox")
-    requested_rows: set[tuple[str, str, str]] = set()
-    for snapshot_id, plugin_name in requested_pairs:
-        exact_hook_names = {
-            hook_name
-            for requested_plugin, hook_name in requested_hooks_by_snapshot.get(snapshot_id, set())
-            if requested_plugin == plugin_name
-        }
-        if exact_hook_names and plugin_name not in plugin_level_requests_by_snapshot.get(snapshot_id, set()):
-            requested_rows.update((snapshot_id, plugin_name, hook_name) for hook_name in exact_hook_names)
-            continue
-        plugin = plugins_by_name.get(plugin_name)
-        hooks = plugin.filter_hooks("Snapshot") if plugin is not None else []
-        if hooks:
-            requested_rows.update((snapshot_id, plugin_name, hook.name) for hook in hooks)
-        else:
-            requested_rows.add((snapshot_id, plugin_name, ""))
-
-    queued_rows: set[tuple[str, str, str]] = set()
-    if preserve_queued and requested_rows:
-        queued_rows = {
-            (str(snapshot_id), plugin_name, hook_name)
-            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
-                snapshot_id__in=existing_snapshot_ids,
-                plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in requested_rows},
-                status=ArchiveResult.StatusChoices.QUEUED,
-            ).values_list("snapshot_id", "plugin", "hook_name")
-        }
-    rows_to_queue = requested_rows - queued_rows
-
-    reset_fields = {
-        "status": ArchiveResult.StatusChoices.QUEUED,
-        "output_str": "",
-        "output_json": None,
-        "output_files": {},
-        "output_size": 0,
-        "output_mimetypes": "",
-        "start_ts": None,
-        "end_ts": None,
-        "modified_at": timezone.now(),
-    }
-    if rows_to_queue and plugins_list:
-        rows_to_reset_by_hook: dict[tuple[str, str], set[str]] = defaultdict(set)
-        for snapshot_id, plugin_name, hook_name in rows_to_queue:
-            rows_to_reset_by_hook[(plugin_name, hook_name)].add(snapshot_id)
-        for (plugin_name, hook_name), plugin_snapshot_ids in rows_to_reset_by_hook.items():
-            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name, hook_name=hook_name).update(
-                **reset_fields,
-            )
-    elif rows_to_queue and requested_plugins_by_snapshot:
-        snapshot_ids_by_hook: dict[tuple[str, str], set[str]] = defaultdict(set)
-        for snapshot_id, plugin_name, hook_name in rows_to_queue:
-            snapshot_ids_by_hook[(plugin_name, hook_name)].add(snapshot_id)
-        for (plugin_name, hook_name), plugin_snapshot_ids in snapshot_ids_by_hook.items():
-            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name, hook_name=hook_name).update(
-                **reset_fields,
-            )
-    existing_rows = (
-        {
-            (str(snapshot_id), plugin_name, hook_name)
-            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
-                snapshot_id__in=existing_snapshot_ids,
-                plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in rows_to_queue},
-            ).values_list("snapshot_id", "plugin", "hook_name")
-        }
-        if rows_to_queue
-        else set()
-    )
-    missing_rows = rows_to_queue - existing_rows
-    if missing_rows:
-        ArchiveResult.objects.bulk_create(
-            [
-                ArchiveResult(
-                    snapshot_id=snapshot_id,
-                    plugin=plugin_name,
-                    hook_name=hook_name,
-                    status=ArchiveResult.StatusChoices.QUEUED,
-                )
-                for snapshot_id, plugin_name, hook_name in sorted(missing_rows)
-            ],
-            batch_size=500,
-        )
-
-    processed_count = len(existing_snapshot_ids)
-    queue_at = timezone.now()
-    if existing_snapshot_ids:
-        if requested_rows:
-            # Search indexing on a sealed Snapshot is the only targeted hook
-            # allowed to bypass the normal lifecycle. Every other requested
-            # plugin requeues its Snapshot through the unified state machine.
-            affected_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in rows_to_queue}
-            if preserve_queued and queued_rows:
-                queued_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in queued_rows}
-                affected_snapshot_ids.update(
-                    str(snapshot_id)
-                    for snapshot_id in Snapshot.objects.filter(id__in=queued_snapshot_ids)
-                    .filter(retry_at__gt=queue_at)
-                    .values_list("id", flat=True)
-                )
-                affected_snapshot_ids.update(
-                    str(snapshot_id)
-                    for snapshot_id in Snapshot.objects.filter(id__in=queued_snapshot_ids, retry_at__isnull=True).values_list(
-                        "id",
-                        flat=True,
-                    )
-                )
-            requested_plugins_by_id: dict[str, set[str]] = defaultdict(set)
-            for snapshot_id, plugin_name, _hook_name in requested_rows:
-                requested_plugins_by_id[snapshot_id].add(plugin_name)
-            for snapshot in Snapshot.objects.filter(id__in=affected_snapshot_ids).only("id", "status", "modified_at"):
-                # Guard the read-time status so we never bump retry_at on a
-                # row that's been re-queued / started by a concurrent runner.
-                plugin_names = requested_plugins_by_id.get(str(snapshot.id), set())
-                sealed_search_backfill = (
-                    snapshot.status == Snapshot.StatusChoices.SEALED
-                    and plugin_names
-                    and all(plugin_name.startswith("search_backend_") for plugin_name in plugin_names)
-                )
-                if sealed_search_backfill:
-                    snapshot.safe_update(
-                        {"retry_at": queue_at, "modified_at": queue_at},
-                        refresh=False,
-                        extra_filter={"status": snapshot.status},
-                    )
-                else:
-                    snapshot.update_and_requeue(
-                        status=Snapshot.StatusChoices.QUEUED,
-                        retry_at=queue_at,
-                        current_step=0,
-                    )
-        else:
-            # No plugin rows were requested, so this is a full snapshot retry.
-            for snapshot in Snapshot.objects.filter(id__in=existing_snapshot_ids).only("id", "status", "retry_at", "modified_at"):
-                snapshot.safe_update(
-                    {
-                        "status": Snapshot.StatusChoices.QUEUED,
-                        "retry_at": queue_at,
-                        "current_step": 0,
-                        "modified_at": queue_at,
-                    },
-                    refresh=False,
-                    extra_filter={"status": snapshot.status},
-                )
-    if existing_crawl_ids and not requested_rows:
-        from archivebox.crawls.models import Crawl
-
-        for crawl in Crawl.objects.filter(id__in=existing_crawl_ids).only("id", "status", "retry_at", "modified_at"):
-            update_fields = {
-                "retry_at": queue_at,
-                "modified_at": queue_at,
-            }
-            if crawl.status != Crawl.StatusChoices.STARTED:
-                update_fields["status"] = Crawl.StatusChoices.QUEUED
-            crawl.safe_update(
-                update_fields,
-                refresh=False,
-                extra_filter={"status": crawl.status},
-            )
-
-    if processed_count == 0:
-        rprint("[red]No snapshots to process[/red]", file=sys.stderr)
-        return 1
-
-    if show_progress:
-        rprint(f"[blue]Queued {processed_count} snapshots for extraction[/blue]", file=sys.stderr)
-
-    # Run orchestrator if --wait (default)
-    if wait:
-        if show_progress:
-            rprint("[blue]Running plugins...[/blue]", file=sys.stderr)
-        snapshot_ids_by_crawl: dict[str, set[str]] = defaultdict(set)
-        for snapshot_id, crawl_id in existing_snapshots:
-            snapshot_ids_by_crawl[str(crawl_id)].add(str(snapshot_id))
-
-        for crawl_id, crawl_snapshot_ids in snapshot_ids_by_crawl.items():
-            from archivebox.crawls.models import Crawl
-
-            crawl = Crawl.objects.get(id=crawl_id)
-            if not crawl.claim_processing_lock(lock_seconds=10):
-                rprint(
-                    f"[yellow]Crawl {crawl_id} is already owned by another runner[/yellow]",
-                    file=sys.stderr,
-                )
-                return 1
-            selected_plugins = (
-                plugins_list
-                or sorted(
-                    {plugin for snapshot_id, plugin, _hook_name in requested_rows if snapshot_id in crawl_snapshot_ids},
-                )
-                or None
-            )
-            run_crawl(
-                crawl_id,
-                snapshot_ids=sorted(crawl_snapshot_ids),
-                selected_plugins=selected_plugins,
-                show_progress=show_progress,
-                selected_plugins_are_explicit=bool(plugins_list),
-            )
-
-    if not emit_results:
-        return 0
-
-    # Output results as JSONL (when piped) or human-readable (when TTY)
-    for snapshot_id in snapshot_ids:
-        try:
-            snapshot = Snapshot.objects.get(id=snapshot_id)
-            results = snapshot.archiveresult_set.all()
-            if plugins_list:
-                results = results.filter(plugin__in=plugins_list)
-
-            for result in results:
-                if is_tty:
-                    status_color = {
-                        "succeeded": "green",
-                        "failed": "red",
-                        "skipped": "yellow",
-                    }.get(result.status, "dim")
-                    rprint(
-                        f"  [{status_color}]{result.status}[/{status_color}] {result.plugin} → {result.output_str or ''}",
-                        file=sys.stderr,
-                    )
-                else:
-                    write_record(result.to_json())
-        except Snapshot.DoesNotExist:
-            continue
+    exit_code = _run_snapshot_requests(requested, wait=wait, show_progress=show_progress)
+    if exit_code or not emit_results:
+        return exit_code
 
+    is_tty = sys.stdout.isatty()
+    for snapshot in Snapshot.objects.filter(id__in=requested):
+        results = snapshot.archiveresult_set.all()
+        requested_plugins = requested[str(snapshot.id)]
+        if requested_plugins:
+            results = results.filter(plugin__in=requested_plugins)
+        for result in results:
+            if is_tty:
+                color = {"succeeded": "green", "failed": "red", "skipped": "yellow"}.get(result.status, "dim")
+                rprint(f"  [{color}]{result.status}[/{color}] {result.plugin} → {result.output_str or ''}", file=sys.stderr)
+            else:
+                write_record(result.to_json())
     return 0
 
 
-def is_archiveresult_id(value: str) -> bool:
-    """Check if value resolves to an ArchiveResult ID."""
-    from archivebox.core.models import ArchiveResult
-    from archivebox.api.v1_core import _uuid_ref_query
-
-    return ArchiveResult.objects.filter(_uuid_ref_query("id", value)).exists()
+def process_archiveresult_by_id(archiveresult_id: str) -> int:
+    """Re-run the parent Snapshot plugin referenced by an ArchiveResult."""
+    return run_plugins((), records=[{"id": archiveresult_id}], wait=True)
 
 
 @click.command()
-@click.option("--plugins", "--plugin", "-p", default="", help="Comma-separated list of plugins to run (e.g., screenshot,singlefile)")
+@click.option("--plugins", "--plugin", "-p", default="", help="Comma-separated list of plugins to run")
 @click.option("--wait/--no-wait", default=True, help="Wait for plugins to complete (default: wait)")
 @click.argument("args", nargs=-1)
 def main(plugins: str, wait: bool, args: tuple):
-    """Run plugins on Snapshots, or process existing ArchiveResults by ID"""
+    """Run plugins on Snapshots; ArchiveResult IDs select their parent plugin."""
     from archivebox.misc.jsonl import read_args_or_stdin
 
-    # Read all input
     records = list(read_args_or_stdin(args))
-
     if not records:
         from rich import print as rprint
 
         rprint("[yellow]No Snapshot IDs or ArchiveResult IDs provided. Pass as arguments or via stdin.[/yellow]", file=sys.stderr)
         sys.exit(1)
-
-    # Check if input looks like existing ArchiveResult IDs to process
-    all_are_archiveresult_ids = all(is_archiveresult_id(r.get("id") or r.get("url", "")) for r in records)
-
-    if all_are_archiveresult_ids:
-        # Process existing ArchiveResults by ID
-        from rich import print as rprint
-
-        exit_code = 0
-        for record in records:
-            archiveresult_id = record.get("id") or record.get("url")
-            if not isinstance(archiveresult_id, str):
-                rprint(f"[red]Invalid ArchiveResult input: {record}[/red]", file=sys.stderr)
-                exit_code = 1
-                continue
-            result = process_archiveresult_by_id(archiveresult_id)
-            if result != 0:
-                exit_code = result
-        sys.exit(exit_code)
-    else:
-        # Default behavior: run plugins on Snapshots from input
-        sys.exit(run_plugins(args, records=records, plugins=plugins, wait=wait))
+    sys.exit(run_plugins(args, records=records, plugins=plugins, wait=wait))
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_install.py

@@ -17,12 +17,12 @@ def _resolve_install_targets(
     requested_names: tuple[str, ...],
 ) -> tuple[list[str], list[str]]:
     """Resolve plugin names and declared binary aliases, leaving unknown tokens raw."""
-    from abx_dl.models import discover_plugins, filter_plugins
+    from archivebox.plugins.discovery import get_plugin_catalog
 
-    plugins = discover_plugins(runtime="archivebox")
-    plugin_names_by_lower = {plugin_name.lower(): plugin_name for plugin_name in plugins}
+    catalog = get_plugin_catalog()
+    plugin_names_by_lower = {plugin_name.lower(): plugin_name for plugin_name in catalog}
     plugin_names_by_binary_alias: dict[str, set[str]] = {}
-    for plugin_name, plugin in plugins.items():
+    for plugin_name, plugin in catalog.items():
         for required_binary in plugin.config.required_binaries:
             aliases = {required_binary.name.lower()}
             template_match = re.fullmatch(r"\{([A-Za-z_][A-Za-z0-9_]*)\}", required_binary.name)
@@ -47,14 +47,14 @@ def _resolve_install_targets(
         else:
             raw_binary_names.append(name)
 
-    selected_plugins = filter_plugins(plugins, requested_plugins, include_providers=True) if requested_plugins else {}
+    selected_plugins = catalog.select(requested_plugins) if requested_plugins else catalog.select([])
     selected_plugin_names = {name.lower() for name in selected_plugins}
     raw_binary_names = [name for name in raw_binary_names if name.lower() not in selected_plugin_names]
     return sorted(selected_plugins), sorted(set(raw_binary_names))
 
 
 def _install_raw_binary_names(binary_names: list[str], binproviders: str) -> None:
-    """Install user-requested standalone binaries through the Binary state machine."""
+    """Install user-requested standalone binaries through the Binary lifecycle."""
     from django.utils import timezone
 
     from archivebox.machine.models import Binary, Machine, _canonical_binary_name
@@ -179,7 +179,7 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo
         run_install(plugin_names=install_plugin_names or None)
 
     if raw_binary_names:
-        print(f"[+] Running direct binary installer via ArchiveBox binary state machine: {', '.join(raw_binary_names)}")
+        print(f"[+] Running direct binary installer via ArchiveBox binary lifecycle: {', '.join(raw_binary_names)}")
         print()
         _install_raw_binary_names(raw_binary_names, binproviders)
 

archivebox/cli/archivebox_pluginmap.py

@@ -48,7 +48,7 @@ def pluginmap(
     Show the current abx-dl event phases and their associated plugin hooks.
 
     This command reflects the new bus-driven runtime, not the legacy ArchiveBox
-    state-machine executor. Event names are normalized to hook prefixes by
+    event runtime. Event names are normalized to hook prefixes by
     stripping a trailing `Event`, then ArchiveBox checks whether any matching
     `on_{EventFamily}__*` scripts actually exist.
     """

archivebox/cli/archivebox_run.py

@@ -54,6 +54,11 @@ RUNNER_DAEMON_ENV = "ARCHIVEBOX_RUNNER_DAEMON"
 
 
 def _exit_daemon_runner_on_signal(sig: signal.Signals) -> None:
+    # A supervised `archivebox run --daemon` is intentionally a disposable
+    # child. If it receives SIGINT/SIGTERM directly, exit with the conventional
+    # signal status so supervisord treats it as an unexpected worker death and
+    # restarts only the runner. The parent `archivebox server` owns supervisord
+    # shutdown and must not be pulled down by a killed daemon worker.
     os._exit(128 + int(sig))
 
 
@@ -67,8 +72,8 @@ def process_stdin_records() -> int:
 
     Outputs JSONL of all processed records (for chaining).
 
-    Handles any record type: Crawl, Snapshot, ArchiveResult.
-    Auto-cascades: Crawl → Snapshots → ArchiveResults.
+    Handles Crawl and Snapshot work records. ArchiveResult records are accepted
+    as references to their parent Snapshot and plugin.
 
     Returns exit code (0 = success, 1 = error).
     """
@@ -85,6 +90,7 @@ def process_stdin_records() -> int:
     )
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.core.models import Snapshot, ArchiveResult
+    from archivebox.api.v1_core import _uuid_ref_query
     from archivebox.crawls.models import Crawl
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
     from archivebox.machine.models import Binary
@@ -151,20 +157,15 @@ def process_stdin_records() -> int:
                     queued_count += 1
 
             elif record_type == TYPE_ARCHIVERESULT:
-                archiveresult = ArchiveResult.from_json(record)
-                if archiveresult:
-                    if archiveresult.status in [
-                        ArchiveResult.StatusChoices.FAILED,
-                        ArchiveResult.StatusChoices.SKIPPED,
-                        ArchiveResult.StatusChoices.NORESULTS,
-                        ArchiveResult.StatusChoices.BACKOFF,
-                    ]:
-                        archiveresult.reset_for_retry()
-                    snapshot = archiveresult.snapshot
-                    plugin_name = archiveresult.plugin
-                else:
-                    snapshot = None
-                    plugin_name = None
+                snapshot_id = str(record.get("snapshot_id") or "")
+                plugin_name = str(record.get("plugin") or "")
+                archiveresult = None
+                if not snapshot_id and record_id:
+                    archiveresult = ArchiveResult.objects.filter(_uuid_ref_query("id", str(record_id))).select_related("snapshot").first()
+                    if archiveresult:
+                        snapshot_id = str(archiveresult.snapshot_id)
+                        plugin_name = plugin_name or archiveresult.plugin
+                snapshot = Snapshot.objects.filter(id=snapshot_id).first() if snapshot_id else None
 
                 if snapshot:
                     snapshot.queue_for_extraction()
@@ -172,7 +173,7 @@ def process_stdin_records() -> int:
                     snapshot_ids_by_crawl[crawl_id].add(str(snapshot.id))
                     if plugin_name:
                         plugin_names_by_crawl[crawl_id].add(str(plugin_name))
-                    output_records.append(record if not archiveresult else archiveresult.to_json())
+                    output_records.append(archiveresult.to_json() if archiveresult else record)
                     queued_count += 1
 
             elif record_type in {TYPE_BINARYREQUEST, TYPE_BINARY}:
@@ -230,7 +231,6 @@ def process_stdin_records() -> int:
                     crawl_id,
                     snapshot_ids=None if crawl_id in full_crawl_ids else sorted(snapshot_ids_by_crawl[crawl_id]),
                     selected_plugins=None if crawl_id in run_all_plugins_for_crawl else sorted(plugin_names_by_crawl[crawl_id]),
-                    selected_plugins_are_explicit=False,
                 )
     return 0
 
@@ -239,7 +239,6 @@ def run_runner(
     daemon: bool = False,
     crawl_id: str | None = None,
     maintenance_only: bool = False,
-    maintenance_batch_size: int | None = None,
 ) -> int:
     """
     Run the background runner loop.
@@ -292,6 +291,7 @@ def run_runner(
     interactive_interrupts = current.root.process_type == Process.TypeChoices.ADD
     if daemon:
         os.environ[RUNNER_DAEMON_ENV] = "1"
+
     try:
         with (
             foreground_shutdown_signals(
@@ -305,7 +305,6 @@ def run_runner(
                 crawl_id=crawl_id,
                 maintenance_only=maintenance_only,
                 interactive_interrupts=interactive_interrupts,
-                **({"maintenance_batch_size": maintenance_batch_size} if maintenance_batch_size else {}),
             )
         return 0
     except KeyboardInterrupt:
@@ -330,13 +329,6 @@ def run_runner(
 @click.option("--snapshot-id", help="Run one snapshot through its crawl")
 @click.option("--binary-id", help="Run one queued binary install directly on the bus")
 @click.option("--maintenance-only", is_flag=True, help="Only process sealed Snapshot maintenance and search-index backfills")
-@click.option(
-    "--maintenance-batch-size",
-    type=int,
-    default=None,
-    hidden=True,
-    help="Limit queued maintenance snapshots claimed per scheduler tick",
-)
 @click.option("--no-stdin", is_flag=True, hidden=True, help="Run the scheduler even when stdin is not a TTY")
 def main(
     daemon: bool,
@@ -344,7 +336,6 @@ def main(
     snapshot_id: str,
     binary_id: str,
     maintenance_only: bool,
-    maintenance_batch_size: int | None,
     no_stdin: bool,
 ):
     """
@@ -369,7 +360,7 @@ def main(
                 ),
                 foreground_parent_watchdog(enabled=False),
             ):
-                sys.exit(run_runner(daemon=True, maintenance_only=maintenance_only, maintenance_batch_size=maintenance_batch_size))
+                sys.exit(run_runner(daemon=True, maintenance_only=maintenance_only))
         except KeyboardInterrupt:
             sys.exit(0)
 
@@ -398,17 +389,16 @@ def main(
                     daemon=False,
                     crawl_id=crawl_id,
                     maintenance_only=maintenance_only,
-                    maintenance_batch_size=maintenance_batch_size,
                 ),
             )
 
         if maintenance_only:
-            sys.exit(run_runner(daemon=daemon, maintenance_only=True, maintenance_batch_size=maintenance_batch_size))
+            sys.exit(run_runner(daemon=daemon, maintenance_only=True))
 
         if not no_stdin and not sys.stdin.isatty():
             sys.exit(process_stdin_records())
         else:
-            sys.exit(run_runner(daemon=daemon, maintenance_only=maintenance_only, maintenance_batch_size=maintenance_batch_size))
+            sys.exit(run_runner(daemon=daemon, maintenance_only=maintenance_only))
 
 
 def run_snapshot_worker(snapshot_id: str) -> int:

archivebox/cli/archivebox_schedule.py

@@ -37,6 +37,7 @@ def schedule(
         "created_schedule_ids": [],
         "disabled_count": 0,
         "run_all_enqueued": 0,
+        "run_all_maintained": 0,
         "active_schedule_ids": [],
     }
 
@@ -57,12 +58,12 @@ def schedule(
 
         created_by_id = get_or_create_system_user_pk()
         is_update_schedule = not import_path
-        template_urls = import_path or "archivebox://update"
+        template_urls = import_path or ""
         template_label = (f"Scheduled import: {template_urls}" if import_path else "Scheduled ArchiveBox update")[:64]
         template_notes = (
             f"Created by archivebox schedule for {template_urls}"
             if import_path
-            else "Created by archivebox schedule to queue recurring archivebox://update maintenance crawls."
+            else "Created by archivebox schedule to run recurring ArchiveBox maintenance."
         )
 
         template = Crawl.objects.create(
@@ -76,7 +77,6 @@ def schedule(
             retry_at=None,
             config={
                 "DEPTH": 0 if is_update_schedule else depth,
-                "SCHEDULE_KIND": "update" if is_update_schedule else "crawl",
                 # Caller-supplied overrides (e.g. {"ONLY_NEW": False}) win over the
                 # template defaults. Anything left unset falls through to the
                 # standard config stack at crawl-resolution time.
@@ -86,6 +86,7 @@ def schedule(
         crawl_schedule = CrawlSchedule.objects.create(
             template=template,
             schedule=schedule_str,
+            config={**template.config, "SCHEDULE_KIND": "update" if is_update_schedule else "crawl"},
             is_enabled=True,
             label=template_label,
             notes=template_notes,
@@ -119,12 +120,19 @@ def schedule(
 
     if run_all:
         enqueued = 0
+        maintained = 0
         now = timezone.now()
         for scheduled_crawl in schedules:
-            scheduled_crawl.enqueue(queued_at=now)
-            enqueued += 1
+            if scheduled_crawl.dispatch(queued_at=now) is None:
+                maintained += 1
+            else:
+                enqueued += 1
         result["run_all_enqueued"] = enqueued
+        result["run_all_maintained"] = maintained
         print(f"[green]\\[*] Enqueued {enqueued} scheduled crawl(s) immediately.[/green]")
+        if maintained:
+            run_pending_crawls(maintenance_only=True)
+            print(f"[green]\\[*] Ran {maintained} scheduled maintenance update(s).[/green]")
         if enqueued:
             print(
                 "[yellow]\\[*] Start `archivebox server`, `archivebox run --daemon`, or `archivebox schedule --foreground` to process the queued crawls.[/yellow]",
@@ -171,7 +179,7 @@ def schedule(
 @click.option("--clear", is_flag=True, help="Disable all currently enabled schedules")
 @click.option("--show", is_flag=True, help="Print all currently enabled schedules")
 @click.option("--foreground", "-f", is_flag=True, help="Run the global crawl runner in the foreground (no crontab required)")
-@click.option("--run-all", is_flag=True, help="Enqueue all enabled schedules immediately and process them once")
+@click.option("--run-all", is_flag=True, help="Dispatch all enabled schedules immediately")
 @click.argument("import_path", required=False)
 @docstring(schedule.__doc__)
 def main(**kwargs):

archivebox/cli/archivebox_update.py

@@ -77,107 +77,71 @@ def reindex_snapshots(
 ) -> dict[str, Any]:
     from archivebox.cli.archivebox_extract import run_plugins
     from archivebox.core.models import ArchiveResult, Snapshot
-    from abx_dl.models import discover_plugins
+    from django.db.models import Exists, OuterRef
 
     # Search backfill is the one maintenance hook allowed to execute without
     # reopening a Snapshot. Restrict that exception to already-sealed rows;
-    # every open lifecycle state remains owned by the normal state machine.
+    # every open lifecycle state remains owned by the normal runner lifecycle.
     snapshots = snapshots.filter(status=Snapshot.StatusChoices.SEALED)
 
-    stats: dict[str, Any] = {"processed": 0, "requested": 0, "queued": 0, "skipped_queued": 0, "reindexed": 0, "snapshot_ids": []}
-    records: list[dict[str, str]] = []
-    plugins_by_name = discover_plugins(runtime="archivebox")
-    required_hooks_by_plugin = {
-        plugin_name: frozenset(hook.name for hook in plugins_by_name[plugin_name].filter_hooks("Snapshot"))
-        for plugin_name in search_plugins
-        if plugin_name in plugins_by_name
-    }
+    stats: dict[str, Any] = {"processed": 0, "requested": 0, "queued": 0, "reindexed": 0, "snapshot_ids": []}
+    print(f"[*] Reindexing missing search indexes with: {', '.join(search_plugins)}")
 
-    total = snapshots.count()
-    print(f"[*] Reindexing {total} snapshots with search plugins: {', '.join(search_plugins)}")
-
-    def run_batch() -> None:
-        if not records:
-            return
-        if wait_for_turn:
-            wait_for_turn()
-        batch_records = list(records)
-        snapshot_ids = {record["snapshot_id"] for record in batch_records}
-        plugin_names = {record["plugin"] for record in batch_records}
-        queued_rows = {
-            (str(snapshot_id), plugin_name, hook_name)
-            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
-                snapshot_id__in=snapshot_ids,
-                plugin__in=plugin_names,
-                status=ArchiveResult.StatusChoices.QUEUED,
-            ).values_list("snapshot_id", "plugin", "hook_name")
-        }
-        records_to_queue = []
-        for record in batch_records:
-            snapshot_id = record["snapshot_id"]
-            plugin_name = record["plugin"]
-            required_hooks = required_hooks_by_plugin.get(plugin_name, frozenset())
-            if required_hooks and all((snapshot_id, plugin_name, hook_name) in queued_rows for hook_name in required_hooks):
-                stats["skipped_queued"] += 1
-                continue
-            records_to_queue.append(record)
-        if not records_to_queue:
-            print(
-                f"    [{stats['processed']}/{total}] Already queued {len(batch_records)} index jobs",
-            )
-            records.clear()
-            return
-        # `archivebox update --index-only` intentionally breaks the usual
-        # "runner discovers work" rule by inserting synthetic queued
-        # ArchiveResult rows for search backends. run_plugins() keeps this as
-        # statement-sized UPDATE/bulk_create work, then bumps Snapshot.retry_at
-        # so the orchestrator owns actual hook execution. Paused snapshots stay
-        # PAUSED; run_due_snapshot restores retry_at=MAX after targeted rows
-        # finish.
-        exit_code = run_plugins(
-            args=(),
-            records=records_to_queue,
-            wait=False,
-            emit_results=False,
-            show_progress=False,
-            preserve_queued=True,
+    completed_statuses = [ArchiveResult.StatusChoices.SUCCEEDED, ArchiveResult.StatusChoices.NORESULTS]
+    for plugin_name in search_plugins:
+        completed_result = ArchiveResult.objects.filter(
+            snapshot_id=OuterRef("pk"),
+            plugin=plugin_name,
+            status__in=completed_statuses,
         )
-        if exit_code != 0:
-            raise SystemExit(exit_code)
-        stats["queued"] += len(records_to_queue)
-        print(
-            f"    [{stats['processed']}/{total}] Queued {len(records_to_queue)} index jobs for orchestrator",
-        )
-        records.clear()
-
-    for snapshot in snapshots.select_related("crawl").paged_iterator(chunk_size=batch_size):
-        try:
-            stats["processed"] += 1
-
-            if _get_snapshot_crawl(snapshot) is None:
-                continue
-
+        candidates = snapshots.annotate(has_completed_index=Exists(completed_result)).filter(has_completed_index=False).order_by("id")
+        after_id = None
+        while True:
+            if wait_for_turn:
+                wait_for_turn()
+            page = candidates.filter(id__gt=after_id) if after_id is not None else candidates
+            batch = list(page.select_related(None).only("id", "timestamp")[:batch_size])
+            if not batch:
+                break
+            after_id = batch[-1].id
+            records = [{"type": "Snapshot", "id": str(snapshot.id)} for snapshot in batch]
+            stats["processed"] += len(batch)
+            stats["requested"] += len(batch)
             if collect_ids:
-                stats["snapshot_ids"].append(str(snapshot.id))
-            for plugin_name in search_plugins:
-                records.append(
-                    {
-                        "type": "ArchiveResult",
-                        "snapshot_id": str(snapshot.id),
-                        "plugin": plugin_name,
-                    },
-                )
-                stats["requested"] += 1
-            if len(records) >= batch_size:
-                run_batch()
-        except KeyboardInterrupt as err:
-            err.archivebox_resume = snapshot.timestamp
-            raise
-
-    run_batch()
+                stats["snapshot_ids"].extend(str(snapshot.id) for snapshot in batch)
+            exit_code = run_plugins(
+                args=(),
+                records=records,
+                plugins=plugin_name,
+                wait=True,
+                emit_results=False,
+                show_progress=False,
+            )
+            if exit_code != 0:
+                raise SystemExit(exit_code)
+            stats["reindexed"] += len(batch)
+            print(f"    [{plugin_name}] indexed {stats['reindexed']} missing snapshots")
     return stats
 
 
+def run_scheduled_maintenance(*, batch_size: int = 500) -> dict[str, Any]:
+    """Queue stale filesystem rows and backfill missing search facts."""
+    from archivebox.core.models import Snapshot
+
+    filesystem_stats = process_all_db_snapshots(batch_size=batch_size)
+    search_plugins = _get_search_indexing_plugins()
+    search_stats = (
+        reindex_snapshots(
+            Snapshot.objects.filter(fs_version=Snapshot._fs_current_version()),
+            search_plugins=search_plugins,
+            batch_size=batch_size,
+        )
+        if search_plugins
+        else {"processed": 0, "requested": 0, "queued": 0, "reindexed": 0, "snapshot_ids": []}
+    )
+    return {"filesystem": filesystem_stats, "search": search_stats}
+
+
 @enforce_types
 def update(
     filter_patterns: Iterable[str] = (),
@@ -203,9 +167,9 @@ def update(
     Update snapshots: migrate old dirs, reconcile DB, and re-queue for archiving.
 
     Three-phase operation (without filters):
-    - Phase 1: Drain old archive/ dirs by moving to new fs location (0.8.x → 0.9.x)
-    - Phase 2: O(n) scan over entire DB from most recent to least recent
-    - No orphan scans needed (trust 1:1 mapping between DB and filesystem after phase 1)
+    - Phase 1: Drain legacy archive/ directories into the current layout
+    - Phase 2: Select only stale fs_version rows through the indexed column
+    - Phase 3: Run queued snapshot-level filesystem maintenance until idle
 
     With filters: Only phase 2 (DB query), no filesystem operations.
     Without filters: All phases (full update).
@@ -232,7 +196,6 @@ def update(
     from archivebox.core.takeover_util import (
         command_owns_foreground_runner,
         current_command,
-        foreground_runner_owner,
         standby_until_foreground_runner_needed,
     )
     from archivebox.workers.supervisord_util import run_runner_worker, stop_own_supervisord_process
@@ -291,11 +254,6 @@ def update(
             while True:
                 do_migrate = migrate_only or not index_only
                 do_index = index_only or not migrate_only
-                do_run_until_idle = do_migrate or do_index
-                ran_post_migrate_runner = False
-                full_update_empty = False
-                maintenance_work_queued = False
-                runner_work_queued = False
 
                 if do_migrate:
                     if (
@@ -327,13 +285,11 @@ def update(
                             after=after,
                             resume=resume,
                             batch_size=batch_size,
-                            queue_for_archiving=do_run_until_idle,
+                            queue_for_archiving=True,
                             wait_for_turn=wait_for_turn,
                         )
                         print_stats(stats)
                         touched_snapshot_ids.update(stats.get("snapshot_ids", []))
-                        maintenance_work_queued = stats.get("queued", 0) > 0
-                        runner_work_queued = runner_work_queued or maintenance_work_queued
                     else:
                         stats_combined = {"phase1": {}, "phase2": {}}
 
@@ -343,162 +299,54 @@ def update(
                             batch_size=batch_size,
                         )
 
-                        print("[*] Phase 2: Processing all database snapshots (most recent first)...")
+                        print("[*] Phase 2: Selecting database snapshots with stale filesystem versions...")
                         stats_combined["phase2"] = process_all_db_snapshots(
                             batch_size=batch_size,
                             resume=resume,
                             wait_for_turn=wait_for_turn,
                         )
                         print_combined_stats(stats_combined)
-                        full_update_empty = (
-                            stats_combined["phase1"].get("processed", 0) == 0 and stats_combined["phase2"].get("snapshots", 0) == 0
-                        )
-                        maintenance_work_queued = any(
-                            (
-                                stats_combined["phase1"].get("queued", 0),
-                                stats_combined["phase2"].get("queued", 0),
-                                stats_combined["phase2"].get("crawls_sealed", 0),
-                            ),
-                        )
-                        runner_work_queued = runner_work_queued or maintenance_work_queued
-
-                    if do_run_until_idle:
-                        # Filesystem migration is maintenance on existing
-                        # Snapshot rows: Snapshot.save() moves archive/<ts> to
-                        # the current output_dir and preserves the lifecycle
-                        # status. Drain those retry_at ticks before queuing
-                        # search backfill below. Otherwise the sealed search
-                        # runner branch correctly sees queued ArchiveResult
-                        # rows first, runs the targeted plugins, and may leave
-                        # the fs_version maintenance tick hidden behind that
-                        # plugin work until another update pass.
-                        if full_update_empty:
-                            print("[*] No snapshots or legacy archive directories found; skipping filesystem maintenance runner.")
-                        elif not maintenance_work_queued:
-                            print("[*] No filesystem maintenance work queued; skipping filesystem maintenance runner.")
-                        else:
-                            print("[*] Phase 3: Running filesystem maintenance until idle...")
-                        if full_update_empty:
-                            pass
-                        elif not maintenance_work_queued:
-                            pass
-                        elif is_filtered_update:
-                            if not touched_snapshot_ids:
-                                print("[*] No matching snapshots queued work for the runner.")
-                            for snapshot_id in sorted(touched_snapshot_ids):
-                                run_scoped_runner("--snapshot-id", snapshot_id)
-                        else:
-                            run_scoped_runner("--maintenance-only", "--maintenance-batch-size", str(batch_size))
-                        ran_post_migrate_runner = True
-
-                if do_index:
-                    if full_update_empty:
-                        print("[*] No snapshots found; skipping search indexing backfill.")
-                    else:
-                        search_plugins = _get_search_indexing_plugins()
-                        if not search_plugins:
-                            print("[*] No search indexing plugins are available, nothing to backfill.")
-                        else:
-                            snapshots = _build_filtered_snapshots_queryset(
-                                filter_patterns=filter_patterns,
-                                filter_type=filter_type,
-                                status=status,
-                                url__icontains=url__icontains,
-                                url__istartswith=url__istartswith,
-                                tag=tag,
-                                crawl_id=crawl_id,
-                                limit=limit,
-                                sort=sort,
-                                search=search,
-                                before=before,
-                                after=after,
-                                resume=resume,
-                            )
-                            from django.db.models import Exists, OuterRef, Q
-                            from django.utils import timezone
-                            from archivebox.core.models import ArchiveResult, Snapshot
-
-                            scoped_snapshot_ids = snapshots.order_by().values("id") if is_filtered_update else None
-                            queued_index_results = ArchiveResult.objects.filter(
-                                status=ArchiveResult.StatusChoices.QUEUED,
-                                plugin__in=search_plugins,
-                            )
-                            if scoped_snapshot_ids is not None:
-                                queued_index_results = queued_index_results.filter(snapshot_id__in=scoped_snapshot_ids)
-
-                            if queued_index_results.exists():
-                                runner_work_queued = True
-                                now = timezone.now()
-                                queued_result_for_snapshot = queued_index_results.filter(snapshot_id=OuterRef("pk"))
-                                snapshots_to_wake = (
-                                    Snapshot.objects.filter(status=Snapshot.StatusChoices.SEALED)
-                                    .annotate(
-                                        has_queued_index_result=Exists(queued_result_for_snapshot),
-                                    )
-                                    .filter(
-                                        has_queued_index_result=True,
-                                    )
-                                    .filter(
-                                        Q(retry_at__isnull=True) | Q(retry_at__gt=now),
-                                    )
-                                )
-                                if scoped_snapshot_ids is not None:
-                                    snapshots_to_wake = snapshots_to_wake.filter(id__in=scoped_snapshot_ids)
-                                woken_count = snapshots_to_wake.update(
-                                    retry_at=now,
-                                    modified_at=now,
-                                )
-                                print(
-                                    "[*] Existing queued search index jobs found; "
-                                    f"skipping backfill scan and waking {woken_count} snapshot(s) for the runner.",
-                                )
-                            else:
-                                collect_index_ids = (
-                                    is_filtered_update
-                                    or foreground_runner_owner(
-                                        data_dir=CONSTANTS.DATA_DIR,
-                                        exclude_id=command.id,
-                                    )
-                                    is not None
-                                )
-                                stats = reindex_snapshots(
-                                    snapshots,
-                                    search_plugins=search_plugins,
-                                    batch_size=batch_size,
-                                    collect_ids=collect_index_ids,
-                                    wait_for_turn=wait_for_turn,
-                                )
-                                print_index_stats(stats)
-                                touched_snapshot_ids.update(stats.get("snapshot_ids", []))
-                                runner_work_queued = runner_work_queued or stats["queued"] > 0
-
-                if do_run_until_idle and (do_index or not ran_post_migrate_runner):
-                    # Search/index backfill intentionally queues targeted
-                    # ArchiveResult rows without reopening sealed snapshots.
-                    # This second runner pass drains those plugin
-                    # rows after filesystem maintenance has had its own turn.
-                    # For a normal unfiltered `archivebox update`, keep the
-                    # historical final pass broad enough to resume genuinely
-                    # queued/interrupted crawl work after maintenance is done.
-                    if full_update_empty:
-                        print("[*] No snapshots found; skipping queued/interrupted crawl runner.")
-                    elif not runner_work_queued:
-                        print("[*] No queued/interrupted crawl work found; skipping queued/interrupted crawl runner.")
-                    else:
-                        print("[*] Phase 3: Running queued/interrupted crawl work until idle...")
-                    if full_update_empty:
-                        pass
-                    elif not runner_work_queued:
-                        pass
-                    elif touched_snapshot_ids and is_filtered_update:
-                        if not touched_snapshot_ids:
-                            print("[*] No matching snapshots queued work for the runner.")
+                    # The due selectors are indexed and cheap when empty, so
+                    # always drain them instead of preceding the runner with
+                    # whole-table counts merely to decide whether to call it.
+                    print("[*] Phase 3: Running filesystem maintenance until idle...")
+                    if is_filtered_update:
                         for snapshot_id in sorted(touched_snapshot_ids):
                             run_scoped_runner("--snapshot-id", snapshot_id)
+                    elif migrate_only:
+                        run_scoped_runner("--maintenance-only")
                     else:
-                        run_scoped_runner(
-                            *(["--maintenance-only", "--maintenance-batch-size", str(batch_size)] if index_only or migrate_only else []),
+                        run_scoped_runner()
+
+                if do_index:
+                    search_plugins = _get_search_indexing_plugins()
+                    if not search_plugins:
+                        print("[*] No search indexing plugins are available, nothing to backfill.")
+                    else:
+                        snapshots = _build_filtered_snapshots_queryset(
+                            filter_patterns=filter_patterns,
+                            filter_type=filter_type,
+                            status=status,
+                            url__icontains=url__icontains,
+                            url__istartswith=url__istartswith,
+                            tag=tag,
+                            crawl_id=crawl_id,
+                            limit=limit,
+                            sort=sort,
+                            search=search,
+                            before=before,
+                            after=after,
+                            resume=resume,
                         )
+                        stats = reindex_snapshots(
+                            snapshots,
+                            search_plugins=search_plugins,
+                            batch_size=batch_size,
+                            collect_ids=is_filtered_update,
+                            wait_for_turn=wait_for_turn,
+                        )
+                        print_index_stats(stats)
+                        touched_snapshot_ids.update(stats.get("snapshot_ids", []))
 
                 if not continuous:
                     break
@@ -561,21 +409,20 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500
     """
     Drain old archive/ directories (0.8.x → 0.9.x migration).
 
-    Only processes real directories (skips symlinks - those are already migrated).
+    Removes obsolete timestamp symlinks and processes real legacy directories.
     For each old dir found in archive/:
       1. Load or create DB snapshot
       2. Trigger fs migration on save() to move to data/archive/users/{user}/...
-      3. Leave symlink in archive/ pointing to new location
+      3. Remove the old timestamp path after the verified migration commits
 
-    After this drains, archive/ should only contain symlinks and we can trust
-    1:1 mapping between DB and filesystem.
+    After this drains, current snapshot data exists only under archive/users/.
     """
     from archivebox.core.models import Snapshot
     from archivebox.config import CONSTANTS
     from archivebox.crawls.models import Crawl
     from django.utils import timezone
 
-    stats = {"processed": 0, "migrated": 0, "queued": 0, "skipped": 0, "invalid": 0}
+    stats = {"processed": 0, "migrated": 0, "queued": 0, "skipped": 0, "invalid": 0, "removed_symlinks": 0}
     crawl_url_lines: dict[str, list[str]] = {}
     crawl_url_sets: dict[str, set[str]] = {}
     dirty_crawl_ids: set[str] = set()
@@ -606,8 +453,23 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500
             if changed:
                 Crawl.objects.filter(pk=crawl.pk).update(urls="\n".join(lines), modified_at=timezone.now())
 
-    # Scan for real directories only (skip symlinks - they're already migrated)
+    # Compatibility timestamp projections are harmful in portable exports and
+    # duplicate the obsolete 0.7.x-looking namespace without containing data.
     all_entries = list(os.scandir(archive_dir))
+    for entry in all_entries:
+        entry_path = Path(entry.path)
+        if entry.is_symlink() and Snapshot.is_legacy_archive_dir(entry_path):
+            try:
+                target_path = entry_path.resolve(strict=True)
+                points_into_current_layout = target_path.is_relative_to(CONSTANTS.USERS_DIR.resolve(strict=True))
+            except OSError:
+                points_into_current_layout = False
+
+            if points_into_current_layout:
+                entry_path.unlink(missing_ok=True)
+                stats["removed_symlinks"] += 1
+
+    # Scan real legacy directories only; these still contain data to migrate.
     entries = [
         (e.stat().st_mtime, e.path)
         for e in all_entries
@@ -723,18 +585,8 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500
 
 
 def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, wait_for_turn=None) -> dict[str, int]:
-    """
-    O(n) scan over entire DB from most recent to least recent.
-
-    For each snapshot:
-      1. Reconcile index.json with DB (merge titles, tags, archive results)
-      2. Mark migrated snapshots sealed unless explicitly re-queued elsewhere
-
-    No orphan detection needed - we trust 1:1 mapping between DB and filesystem
-    after Phase 1 has drained all old archive/ directories.
-    """
+    """Queue only snapshots whose indexed filesystem version is stale."""
     from archivebox.core.models import Snapshot
-    from archivebox.crawls.models import Crawl
     from django.db.models import Q
     from django.utils import timezone
 
@@ -747,128 +599,35 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
         "sealed": 0,
         "crawls_sealed": 0,
     }
-    current_fs_version = Snapshot._fs_current_version()
-
-    queryset = Snapshot.objects.all()
+    queryset = Snapshot.objects.filter(fs_version__in=Snapshot._FS_VERSION_MIGRATION_PATHS)
     if resume:
         queryset = queryset.filter(timestamp__lte=resume)
-    total = queryset.count()
-    stats["snapshots"] = total
-    print(f"[*] Processing {total} snapshots from database (most recent first)...")
-
-    def update_in_batches(rows, *, label: str, **updates) -> int:
-        updated = 0
-        checked = 0
-        while True:
-            if wait_for_turn:
-                wait_for_turn()
-            batch = list(rows.only("id", "modified_at").order_by("-timestamp")[:batch_size])
-            if not batch:
-                if updated:
-                    print(f"    [{label}] complete: {updated} rows updated")
-                return updated
-            checked += len(batch)
-            print(f"    [{label}] updating next {len(batch)} rows (seen {checked})...")
-            for snapshot in batch:
-                # This maintenance scan intentionally bypasses save(); it is
-                # only normalizing scheduler fields, and Snapshot.save() may
-                # do filesystem migration work that belongs in the runner.
-                # Guard each single-row UPDATE with modified_at so stale scan
-                # pages cannot overwrite newer runner/admin writes.
-                updated += int(
-                    snapshot.safe_update(
-                        updates,
-                        refresh=False,
-                        extra_filter={"modified_at": snapshot.modified_at},
-                    ),
-                )
-            print(f"    [{label}] updated {updated} rows so far")
-
-    now = timezone.now()
-    updated_rows = update_in_batches(
-        queryset.exclude(
-            status__in=[
-                Snapshot.StatusChoices.QUEUED,
-                Snapshot.StatusChoices.STARTED,
-                Snapshot.StatusChoices.PAUSED,
-                Snapshot.StatusChoices.SEALED,
-            ],
-        ),
-        label="snapshot status normalization",
-        status=Snapshot.StatusChoices.SEALED,
-        retry_at=None,
-        modified_at=now,
-    )
-    stats["sealed"] += updated_rows
-    stats["updated_db"] += updated_rows
-
-    fs_version_rows = queryset.exclude(fs_version=current_fs_version).filter(Q(retry_at__isnull=True) | Q(retry_at__gt=now))
-    stale_batch = []
-
-    def queue_stale_fs_batch() -> None:
-        if not stale_batch:
-            return
+    initial_now = timezone.now()
+    rows_to_wake = queryset.filter(Q(retry_at__isnull=True) | Q(retry_at__gt=initial_now))
+    after_id = None
+    while True:
         if wait_for_turn:
             wait_for_turn()
+        page = rows_to_wake.filter(id__gt=after_id) if after_id is not None else rows_to_wake
+        batch = list(page.only("id", "fs_version", "modified_at").order_by("id")[:batch_size])
+        if not batch:
+            break
+        after_id = batch[-1].id
         now = timezone.now()
-        # Do not bump fs_version here. The orchestrator calls Snapshot.save(),
-        # which performs the idempotent filesystem migration and commits the new
-        # fs_version in the same serialized worker path as normal crawls.
         updated = 0
-        for snapshot in stale_batch:
-            # Each row gets its own short autocommit UPDATE because this scan
-            # can touch millions of snapshots while a server is also alive.
-            # The modified_at predicate is the CAS guard: if the runner or
-            # admin changed the snapshot after paged_iterator read it, skip it
-            # and let the newer state decide whether migration is still due.
+        for snapshot in batch:
             updated += int(
                 snapshot.safe_update(
-                    {
-                        "retry_at": now,
-                        "modified_at": now,
-                    },
+                    {"retry_at": now, "modified_at": now},
                     refresh=False,
                     extra_filter={"fs_version": snapshot.fs_version},
                 ),
             )
-        stats["processed"] += len(stale_batch)
+        stats["processed"] += len(batch)
         stats["updated_db"] += updated
         stats["queued"] += updated
-        print(f"    [{stats['processed']}/{total}] Queued {updated} filesystem migrations for orchestrator...")
-        stale_batch.clear()
-
-    for snapshot in (
-        fs_version_rows.only("id", "crawl_id", "timestamp", "fs_version", "modified_at")
-        .order_by("-timestamp")
-        .paged_iterator(chunk_size=batch_size)
-    ):
-        try:
-            stale_batch.append(snapshot)
-            if len(stale_batch) >= batch_size:
-                queue_stale_fs_batch()
-        except KeyboardInterrupt as err:
-            err.archivebox_resume = snapshot.timestamp
-            raise
-    queue_stale_fs_batch()
-
-    now = timezone.now()
-    # Crawls with no open child snapshots are already finished. Seal them here
-    # instead of waking the foreground runner; otherwise migration/update can
-    # accidentally re-enter full crawl execution for historical rows.
-    stats["crawls_sealed"] = (
-        Crawl.objects.filter(
-            status__in=Crawl.RUNNABLE_STATES,
-        )
-        .exclude(
-            snapshot_set__status__in=Snapshot.OPEN_STATES,
-        )
-        .update(
-            status=Crawl.StatusChoices.SEALED,
-            retry_at=None,
-            modified_at=now,
-        )
-    )
-    stats["updated_db"] += stats["crawls_sealed"]
+        print(f"    Queued {stats['queued']} stale filesystem snapshots so far...")
+    stats["snapshots"] = stats["processed"]
     return stats
 
 
@@ -928,8 +687,6 @@ def process_filtered_snapshots(
             stats["snapshot_ids"].append(str(snapshot.id))
             update_values = {}
             updated = 0
-            if not isinstance(snapshot.current_step, int):
-                update_values["current_step"] = 0
             if queue_for_archiving:
                 update_values.update(
                     {
@@ -1015,10 +772,9 @@ def print_index_stats(stats: dict[str, Any]) -> None:
 
     print(f"""
 [green]Search Reindex Complete[/green]
-  Scanned rows:      {stats["processed"]}
-  Requested jobs:    {stats.get("requested", stats["queued"])}
-  Queued index jobs: {stats["queued"]}
-  Already queued:    {stats.get("skipped_queued", 0)}
+  Missing index runs: {stats["processed"]}
+  Requested runs:    {stats.get("requested", 0)}
+  Indexed snapshots: {stats.get("reindexed", 0)}
 """)
 
 

archivebox/cli/archivebox_version.py

@@ -265,15 +265,14 @@ def version(
     seen_failures: set[str] = set()
     seen_rows: set[tuple[str, str, str, str]] = set()
 
-    from archivebox.plugins.discovery import get_enabled_plugins
+    from archivebox.plugins.discovery import get_enabled_plugins, get_plugin_catalog
     from abx_dl.config import get_required_binary_requests
     from abx_dl.dependencies import resolve_binary_requests
-    from abx_dl.models import discover_plugins, filter_plugins
     from abx_dl.orchestrator import create_bus
     from abxpkg.binary_service import BinaryEvent, BinaryService
 
-    plugins = discover_plugins(runtime="archivebox")
-    enabled_plugins = filter_plugins(plugins, get_enabled_plugins(config=config), include_providers=True)
+    plugins = get_plugin_catalog()
+    enabled_plugins = plugins.select(get_enabled_plugins(config=config))
     enabled_plugin_names = set(enabled_plugins)
     runtime_config = normalize_runtime_config(config.for_crawl(), json_safe=False)
     derived_config: dict[str, object] = {}

archivebox/config/collection.py

@@ -160,7 +160,7 @@ def _coerce_from_str_dict(file_config: dict[str, str]) -> dict[str, Any]:
     so they round-trip through INI's string-only storage. When reading the
     file back into ``Machine.config`` (a JSONField that holds native types)
     those strings have to be decoded — otherwise downstream consumers like
-    ``_emit_machine_config`` → ``MachineEvent`` → abx-dl see a JSON string
+    ``MachineEvent`` → abx-dl see a JSON string
     where they expect a dict and raise ``TypeError``.
     Declared fields go through pydantic-settings' own ``field_is_complex`` /
     ``prepare_field_value`` so they're decoded per annotation. Undeclared

archivebox/config/common.py

@@ -16,9 +16,9 @@ from pathlib import Path
 from typing import Any, ClassVar, cast
 from urllib.parse import quote, urlparse
 
-from abx_plugins.plugins.base.utils import BASE_CONFIG_PATH, build_config_model, resolve_plugin_configs
+from abx_plugins.plugins.base.utils import build_config_model
 from django.db import DatabaseError
-from pydantic import BaseModel, Field, PrivateAttr, create_model, field_validator, model_validator
+from pydantic import BaseModel, Field, PrivateAttr, TypeAdapter, create_model, field_validator, model_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
 from rich.console import Console
 
@@ -406,7 +406,11 @@ class DatabaseConfig(BaseConfigSet):
     DATABASE_USER: str = Field(default="archivebox", alias="ARCHIVEBOX_DATABASE_USER")
     DATABASE_PASSWORD: str = Field(default="", alias="ARCHIVEBOX_DATABASE_PASSWORD")
     SQLITE_JOURNAL_MODE: str = Field(
-        default="WAL",
+        # Docker collections commonly live on a host bind mount. WAL's -shm
+        # locking is only safe when every SQLite process is on the same host;
+        # Docker Desktop/OrbStack place the container and host in different
+        # locking domains and a host-side reader can corrupt the live DB.
+        default="DELETE" if IN_DOCKER else "WAL",
         alias="ARCHIVEBOX_SQLITE_JOURNAL_MODE",
         pattern=r"(?i)^(DELETE|TRUNCATE|PERSIST|MEMORY|WAL|OFF)$",
     )
@@ -419,6 +423,15 @@ class DatabaseConfig(BaseConfigSet):
     SQLITE_LOCK_RETRY_TIMEOUT: float = Field(default=60.0, alias="ARCHIVEBOX_SQLITE_LOCK_RETRY_TIMEOUT", ge=0)
     SQLITE_LOCK_RETRY_INTERVAL: float = Field(default=5.0, alias="ARCHIVEBOX_SQLITE_LOCK_RETRY_INTERVAL", gt=0)
 
+    @model_validator(mode="after")
+    def reject_docker_sqlite_wal(self):
+        if IN_DOCKER and self.DATABASE_ENGINE.lower() == "sqlite" and self.SQLITE_JOURNAL_MODE.upper() == "WAL":
+            raise ValueError(
+                "SQLITE_JOURNAL_MODE=WAL is unsafe for Docker collections because host bind mounts cross SQLite "
+                "locking domains; use DELETE (the Docker default) or PostgreSQL",
+            )
+        return self
+
 
 class ArchivingConfig(BaseConfigSet):
     toml_section_header: str = "ARCHIVING_CONFIG"
@@ -428,6 +441,7 @@ class ArchivingConfig(BaseConfigSet):
         default="",
         description="Comma-separated plugin selection for this run. Empty means use enabled plugin defaults.",
     )
+    PARSER: str = Field(default="auto", description="Parser used to read the submitted Crawl input document.")
 
     ONLY_NEW: bool = Field(default=True)
     INDEX_ONLY: bool = Field(default=False)
@@ -588,13 +602,9 @@ def _explicit_plugin_enabled_keys(config: Mapping[str, object]) -> set[str]:
 
 
 def _discover_plugin_config_schemas() -> PluginSchemaDocuments:
-    from archivebox.plugins.discovery import discover_plugin_configs
+    from archivebox.plugins.discovery import get_plugin_config_resolver
 
-    schemas: PluginSchemaDocuments = {}
-    if BASE_CONFIG_PATH.exists():
-        schemas["base"] = json.loads(BASE_CONFIG_PATH.read_text())
-    schemas.update(discover_plugin_configs())
-    return schemas
+    return get_plugin_config_resolver().schemas
 
 
 def _plugin_config_properties(plugin_schemas: PluginSchemaDocuments) -> dict[str, dict[str, Any]]:
@@ -843,13 +853,11 @@ class ArchiveBoxBaseConfig(
             disabled_plugins = [plugin_name for plugin_name, enabled_key in enabled_config_keys.items() if not getattr(self, enabled_key)]
         selected_plugin_roots = plugin_names
         if selected_plugin_roots:
-            from abx_dl.models import discover_plugins, filter_plugins
+            from archivebox.plugins.discovery import get_plugin_catalog
 
             selected_plugins = set(
-                filter_plugins(
-                    discover_plugins(runtime="archivebox"),
+                get_plugin_catalog().select(
                     sorted(selected_plugin_roots),
-                    include_providers=True,
                     disabled_names=disabled_plugins,
                 ),
             )
@@ -1182,7 +1190,6 @@ def get_config(
 
     explicit_plugin_enabled_keys: set[str] = set()
     if resolve_plugins:
-        plugin_schemas = {plugin_name: schema for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items() if isinstance(schema, dict)}
         plugin_global_config = {key: str(value) if isinstance(value, Path) else value for key, value in config_data.items()}
         crawl_selected_plugins = crawl_config_base and bool(_normalize_plugins_config_value(dict(crawl.config or {}).get("PLUGINS")))
         # A frozen crawl-level PLUGINS selector is the exact extractor set for
@@ -1206,8 +1213,9 @@ def get_config(
                 **_plugin_user_config(_plugin_input_config(file_config)),
                 **plugin_user_config,
             }
-        plugin_sections = resolve_plugin_configs(
-            plugin_schemas,
+        from archivebox.plugins.discovery import get_plugin_config_resolver
+
+        plugin_sections = get_plugin_config_resolver().resolve(
             global_config=plugin_global_config,
             user_config=plugin_user_config,
             environ={},
@@ -1245,7 +1253,7 @@ def get_config(
             config = config.model_copy(update={"SERVER_SECURITY_MODE": "safe-subdomains-fullreplay"})
     for key in explicit_plugin_enabled_keys:
         if key in config_data:
-            setattr(config, key, config_data[key])
+            setattr(config, key, TypeAdapter(bool).validate_python(config_data[key]))
     if config.PLUGINS:
         config._derive_plugin_enabled_config(respect_current_enabled=True)
     if redact_sensitive:

archivebox/core/admin_archiveresults.py

@@ -10,9 +10,12 @@ from pathlib import Path
 from urllib.parse import quote
 
 from django.contrib import admin
-from django.core.exceptions import ValidationError
+from django.contrib.admin.actions import delete_selected
+from django.contrib.admin.helpers import ACTION_CHECKBOX_NAME
+from django.core.exceptions import PermissionDenied, SuspiciousOperation, ValidationError
 from django.db.models import Count, Min, Prefetch, Q, Subquery, TextField, Window
 from django.db.models.functions import Cast
+from django.shortcuts import redirect
 from django.urls import resolve, reverse
 from django.utils import timezone
 from django.utils.html import format_html
@@ -198,8 +201,7 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None, can_del
         if can_delete:
             delete_button = f'''
                 <button type="button" data-archive-result-ids="{result.id}" data-delete-url="{delete_url}"
-                        style="padding: 4px 8px; border: 0; background: #f1f5f9; border-radius: 4px; color: #475569; font-size: 11px; cursor: pointer;"
-                        title="Delete this output">❌</button>
+                        title="Delete this output">×</button>
             '''
 
         rows.append(f'''
@@ -254,13 +256,11 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None, can_del
                 <td style="padding: 10px 12px; white-space: nowrap; font-family: ui-monospace, monospace; font-size: 11px; color: #64748b;">
                     {version}
                 </td>
-                <td style="padding: 10px 8px; white-space: nowrap;">
-                    <div style="display: flex; gap: 4px;">
+                <td class="archive-results-actions-cell">
+                    <div class="archive-results-actions">
                         <a href="{output_link_attr}" target="_blank"
-                           style="padding: 4px 8px; background: #f1f5f9; border-radius: 4px; color: #475569; text-decoration: none; font-size: 11px;"
                            title="View output">📄</a>
                         <a href="{reverse("admin:core_archiveresult_change", args=[result.id])}"
-                           style="padding: 4px 8px; background: #f1f5f9; border-radius: 4px; color: #475569; text-decoration: none; font-size: 11px;"
                            title="Edit">✏️</a>
                         {delete_button}
                     </div>
@@ -331,7 +331,7 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None, can_del
                         <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Process</th>
                         <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Machine</th>
                         <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Version</th>
-                        <th style="padding: 10px 8px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Actions</th>
+                        <th class="archive-results-actions-cell" style="text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Actions</th>
                     </tr>
                 </thead>
                 <tbody>
@@ -344,7 +344,7 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None, can_del
 
 
 class ArchiveResultInline(admin.TabularInline):
-    name = "Archive Results Log"
+    name = "Archive Results"
     model = ArchiveResult
     parent_model = Snapshot
     extra = 0
@@ -546,10 +546,44 @@ class ArchiveResultAdmin(BaseModelAdmin):
 
     def changelist_view(self, request, extra_context=None):
         self.request = request
+        selected = request.GET.getlist(ACTION_CHECKBOX_NAME)
+        if request.method == "GET" and request.GET.get("action") == "delete_selected" and selected:
+            if not request.user.is_superuser:
+                raise PermissionDenied
+            if len(selected) > 100:
+                raise SuspiciousOperation("Too many ArchiveResults selected for deletion")
+            try:
+                queryset = self.get_queryset(request).filter(pk__in=selected)
+                if not queryset.exists():
+                    snapshot = Snapshot.objects.only("id").filter(pk=request.GET.get("snapshot")).first()
+                    return redirect(build_snapshot_url(str(snapshot.id), "index.html", request=request) if snapshot else request.path)
+            except (ValidationError, ValueError):
+                return redirect(request.path)
+            return delete_selected(self, request, queryset)
+        handoff_snapshot = (
+            request.GET.get("snapshot") if request.method == "POST" and request.GET.get("action") == "delete_selected" else None
+        )
+        if handoff_snapshot:
+            request.GET = request.GET.copy()
+            request.GET.clear()
+            request.META["QUERY_STRING"] = ""
+            try:
+                handoff_snapshot = Snapshot.objects.only("id").filter(pk=handoff_snapshot).first()
+            except (ValidationError, ValueError):
+                handoff_snapshot = None
         saved_list_per_page = self.list_per_page
         self.list_per_page = request.archivebox_config.SNAPSHOTS_PER_PAGE
         try:
-            return super().changelist_view(request, extra_context)
+            response = super().changelist_view(request, extra_context)
+            if (
+                handoff_snapshot
+                and response.status_code in (301, 302)
+                and not ArchiveResult.objects.filter(
+                    pk__in=request.POST.getlist(ACTION_CHECKBOX_NAME),
+                ).exists()
+            ):
+                return redirect(build_snapshot_url(str(handoff_snapshot.id), "index.html", request=request))
+            return response
         finally:
             self.list_per_page = saved_list_per_page
 

archivebox/core/admin_snapshots.py

@@ -346,19 +346,8 @@ class SnapshotAdminForm(forms.ModelForm):
 
             # Parse and save tags from tags_editor
             tags_str = self.cleaned_data.get("tags_editor", "")
-            if tags_str:
-                tag_names = [name.strip() for name in tags_str.split(",") if name.strip()]
-                tags = []
-                for name in tag_names:
-                    tag, _ = Tag.objects.get_or_create(
-                        name__iexact=name,
-                        defaults={"name": name},
-                    )
-                    tag = Tag.objects.filter(name__iexact=name).first() or tag
-                    tags.append(tag)
-                instance.tags.set(tags)
-            else:
-                instance.tags.clear()
+            tag_names = [name.strip() for name in tags_str.split(",") if name.strip()]
+            instance.save_tags(tag_names)
 
         return instance
 
@@ -367,6 +356,16 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     form = SnapshotAdminForm
     raw_id_fields = ("crawl", "parent_snapshot")
     list_select_related = ()
+
+    def save_related(self, request, form, formsets, change):
+        super().save_related(request, form, formsets, change)
+        tags_str = form.cleaned_data.get("tags_editor", "")
+        tag_names = [name.strip() for name in tags_str.split(",") if name.strip()]
+        form.instance.save_tags(
+            tag_names,
+            created_by=request.user if request.user.is_authenticated else None,
+        )
+
     list_display = (
         "permissions_badge",
         "created_at",

archivebox/core/admin_users.py

@@ -96,7 +96,7 @@ class CustomUserAdmin(UserAdmin):
             + f'<br/><a href="/admin/core/snapshot/?created_by__id__exact={obj.pk}">{total_count} total records...<a>',
         )
 
-    @admin.display(description="Archive Result Logs")
+    @admin.display(description="Archive Results")
     def archiveresult_set(self, obj):
         total_count = obj.archiveresult_set.count()
         return mark_safe(

archivebox/core/apps.py

@@ -10,7 +10,6 @@ class CoreConfig(AppConfig):
 
     def ready(self):
         """Register the archivebox.core.admin_site as the main django admin site"""
-        import sys
         from django.utils.autoreload import DJANGO_AUTORELOAD_ENV
 
         from archivebox.core.admin_site import register_admin_site
@@ -28,11 +27,6 @@ class CoreConfig(AppConfig):
 
         pre_save.connect(truncate_overlong_charfields, dispatch_uid="archivebox_truncate_overlong_charfields")
 
-        # Import models to register state machines with the registry
-        # Skip during makemigrations to avoid premature state machine access
-        if "makemigrations" not in sys.argv:
-            from archivebox.core import models  # noqa: F401
-
         def _should_prepare_runtime() -> bool:
             if os.environ.get("ARCHIVEBOX_RUNSERVER") == "1":
                 if os.environ.get("ARCHIVEBOX_AUTORELOAD") == "1":

archivebox/core/asgi.py

@@ -13,64 +13,12 @@ from archivebox.config.django import setup_django
 
 setup_django(check_db=True)
 
-
-def _patch_thread_sensitive_context_shutdown() -> None:
-    """Stop ``ThreadSensitiveContext.__aexit__`` from blocking the daphne loop.
-
-    Django 6.0's ASGIHandler wraps every request in ``async with
-    ThreadSensitiveContext():`` (django/core/handlers/asgi.py:169). On exit
-    asgiref calls ``executor.shutdown()`` with the default ``wait=True``
-    (asgiref/sync.py:148), which is a *synchronous* ``Thread.join()`` inside
-    an async function — so it blocks the daphne event loop until the
-    executor's worker thread exits.
-
-    That's normally fine because the request handler has already awaited
-    every ``sync_to_async`` it submitted, so the worker is idle and dies as
-    soon as the shutdown sentinel reaches it. The blocking turns into a
-    problem when a client disconnects mid-request:
-
-    * ``SyncToAsync.__call__`` shields the executor work with
-      ``await asyncio.shield(exec_coro)`` (asgiref/sync.py:506) so that the
-      sync DB call doesn't get torn down halfway through.
-    * On cancellation it calls ``exec_coro.cancel()`` (line 522) which only
-      flips the asyncio ``Future`` to cancelled — the underlying thread
-      keeps running the SQL query.
-    * Control unwinds to ``__aexit__`` while the orphaned thread is still
-      mid-query. ``shutdown(wait=True)`` then blocks the event loop until
-      that orphan finishes.
-
-    Under heavy SQLite contention (the load-test scenario that surfaced
-    this on cabbage) those orphan threads can take 30 seconds each waiting
-    for write locks, and the daphne loop is single-threaded — so every
-    such orphan stalls every other in-flight request, healthchecks time
-    out, and the container goes ``unhealthy``.
-
-    Switching to ``shutdown(wait=False)`` queues the sentinel and returns
-    immediately; the worker thread still exits cleanly once its current
-    task finishes, and asgiref's ``WeakKeyDictionary`` releases the
-    executor as soon as the request's context is GC'd. No per-request
-    teardown guarantee is lost — there was no caller relying on it.
-    """
-    from asgiref import sync as _asgiref_sync
-
-    original_aexit = _asgiref_sync.ThreadSensitiveContext.__aexit__
-
-    async def __aexit__(self, exc, value, tb):  # type: ignore[no-redef]
-        if not self.token:
-            return
-        executor = _asgiref_sync.SyncToAsync.context_to_thread_executor.pop(self, None)
-        if executor is not None:
-            executor.shutdown(wait=False)
-        _asgiref_sync.SyncToAsync.thread_sensitive_context.reset(self.token)
-
-    # Idempotent: only patch once even if asgi.py is reloaded.
-    if getattr(original_aexit, "_archivebox_patched", False):
-        return
-    __aexit__._archivebox_patched = True  # type: ignore[attr-defined]
-    _asgiref_sync.ThreadSensitiveContext.__aexit__ = __aexit__
+django_application = get_asgi_application()
 
 
-_patch_thread_sensitive_context_shutdown()
+async def application(scope, receive, send):
+    if scope["type"] == "websocket":
+        from archivebox.opencode.views import websocket_view
 
-# Standard Django ASGI application (no websockets/channels needed)
-application = get_asgi_application()
+        return await websocket_view(scope, receive, send)
+    return await django_application(scope, receive, send)

archivebox/core/forms.py

@@ -12,9 +12,9 @@ from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget
 from archivebox.crawls.schedule_util import validate_schedule
 from archivebox.misc.util import URL_REGEX, find_all_urls, parse_filesize_to_bytes
 from archivebox.personas.models import Persona
-from archivebox.plugins.discovery import get_plugins
+from archivebox.plugins.discovery import get_plugin_catalog
 from archivebox.plugins.forms import (
-    PLUGIN_GROUP_DEFINITIONS,
+    PLUGIN_GROUPS,
     TIMEOUT_INPUT_PATTERN,
     PluginConfigFormMixin,
     get_choice_field,
@@ -333,10 +333,9 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
         if self.can_override_crawl_config:
             self.build_plugin_groups(get_config(persona=selected_persona) if selected_persona else get_config())
         else:
-            all_plugins = get_plugins()
-            for field_name, *_rest, plugin_names in PLUGIN_GROUP_DEFINITIONS:
-                get_choice_field(self, field_name).choices = [(p, p) for p in all_plugins if p in plugin_names]
-            get_choice_field(self, "other_plugins").choices = [(p, p) for p in all_plugins]
+            grouped_plugins = get_plugin_catalog().groups()
+            for category, field_name, _title in PLUGIN_GROUPS:
+                get_choice_field(self, field_name).choices = [(plugin.name, plugin.name) for plugin in grouped_plugins.get(category, [])]
             self.plugin_groups = []
 
     def clean(self):

archivebox/core/middleware.py

@@ -39,8 +39,8 @@ ADMIN_LOGIN_HINT_COOKIE = "archivebox_admin_logged_in"
 def _admin_login_hint_cookie_domain(config) -> str | None:
     """Resolve the parent domain to scope the cross-subdomain login hint.
 
-    NOTE: this cookie carries only the single bit "user is logged in on
-    admin somewhere"; it MUST NOT be confused with the session cookie,
+    NOTE: this cookie carries only the single bit "a superuser is logged in
+    on admin somewhere"; it MUST NOT be confused with the session cookie,
     which stays admin-host-scoped (see core/settings.py
     SESSION_COOKIE_DOMAIN comment — admin/web is a security boundary).
 
@@ -88,6 +88,11 @@ def AdminCookieIsolationMiddleware(get_response):
     def middleware(request):
         response = get_response(request)
 
+        if request.path == "/admin" or request.path.startswith("/admin/"):
+            if not getattr(response, "xframe_options_exempt", False):
+                response.headers["X-Frame-Options"] = "DENY"
+                response.headers["Content-Security-Policy"] = "frame-ancestors 'none'"
+
         config = request.__dict__.get("archivebox_config")
         if config is None or config.SERVER_SECURITY_MODE == "auto":
             from archivebox.config.common import get_request_config
@@ -197,6 +202,21 @@ def ServerSecurityModeMiddleware(get_response):
 
             config = get_request_config(request, resolve_plugins=False)
 
+        if config.USES_SUBDOMAIN_ROUTING and config.BASE_URL and request.method.upper() not in allowed_methods:
+            request_host, _request_port = split_host_port((request.get_host() or "").lower())
+            control_hosts = {
+                split_host_port(host)[0]
+                for host in (
+                    get_base_host(config=config),
+                    get_admin_host(config=config),
+                    get_api_host(config=config),
+                    get_web_host(config=config),
+                )
+                if host
+            }
+            if request_host not in control_hosts:
+                return HttpResponseForbidden("ArchiveBox is running with the control plane disabled on this host.")
+
         if config.CONTROL_PLANE_ENABLED:
             return get_response(request)
 
@@ -319,7 +339,12 @@ def HostRoutingMiddleware(get_response):
                 return redirect(target)
             response = get_response(request)
             hint_cookie_domain = _admin_login_hint_cookie_domain(config)
-            if request.user.is_authenticated and not request.path.startswith("/admin/logout"):
+            if (
+                request.user.is_authenticated
+                and request.user.is_active
+                and request.user.is_superuser
+                and not request.path.startswith("/admin/logout")
+            ):
                 response.set_cookie(
                     ADMIN_LOGIN_HINT_COOKIE,
                     "1",

archivebox/core/migrations/0053_alter_archiveresult_options.py

@@ -0,0 +1,16 @@
+# Generated by Django 6.1 on 2026-08-30 18:20
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0051_postgres_url_pattern_ops_index"),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name="archiveresult",
+            options={"verbose_name": "Archive Result", "verbose_name_plural": "Archive Results"},
+        ),
+    ]

archivebox/core/migrations/0054_snapshot_fs_version_index.py

@@ -0,0 +1,26 @@
+# Generated by Django 6.1 on 2026-09-02
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0053_alter_archiveresult_options"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="snapshot",
+            name="current_step",
+        ),
+        migrations.AlterField(
+            model_name="snapshot",
+            name="fs_version",
+            field=models.CharField(
+                db_index=True,
+                default="0.9.0",
+                help_text='Filesystem version of this snapshot (e.g., "0.7.0", "0.8.0", "0.9.4").',
+                max_length=10,
+            ),
+        ),
+    ]

archivebox/core/recovery_util.py

@@ -18,7 +18,7 @@ def _canonical_hook_name(hook_name: str) -> str:
 def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str | None = None) -> dict[str, int]:
     from archivebox.crawls.models import Crawl
     from archivebox.core.models import ArchiveResult, Snapshot
-    from archivebox.services.archive_result_service import _collect_output_metadata
+    from abx_dl.output_files import OutputManifest
     from archivebox.machine.models import Process
     from django.core.exceptions import ValidationError
     from django.db.models import Exists, OuterRef, Q, Subquery, Value
@@ -36,9 +36,7 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
         "crawls_queued_without_retry_at": 0,
         "snapshots_queued_without_retry_at": 0,
         "snapshots_sealed_with_extension_uploads_only": 0,
-        "archiveresults_backoff": 0,
-        "snapshots_queued_plugin_rows_waiting_on_stale_lease": 0,
-        "archiveresults_started_without_running_process": 0,
+        "archiveresults_interrupted_without_running_process": 0,
         "archiveresults_missing_for_orphaned_hook_processes": 0,
         "snapshots_started_without_running_results": 0,
         "crawls_started_with_due_snapshots": 0,
@@ -46,10 +44,10 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
         "crawls_started_without_active_snapshots": 0,
     }
 
-    running_archiveresults = ArchiveResult.objects.filter(
-        snapshot_id=OuterRef("pk"),
-        status=ArchiveResult.StatusChoices.STARTED,
-        process__status=Process.StatusChoices.RUNNING,
+    running_hook_processes = Process.objects.filter(
+        archiveresult__snapshot_id=OuterRef("pk"),
+        process_type=Process.TypeChoices.HOOK,
+        status=Process.StatusChoices.RUNNING,
     )
     active_child_snapshots = Snapshot.objects.filter(
         crawl_id=OuterRef("pk"),
@@ -109,49 +107,18 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
         retry_at=now,
         modified_at=now,
     )
-    backoff_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.BACKOFF, **result_filter)
     orphaned_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.STARTED, **result_filter).exclude(
         process__status=Process.StatusChoices.RUNNING,
     )
-    # ArchiveResult has no retry_at scheduler. Wake only the parent Snapshots
-    # for result rows we are about to repair, then requeue those exact rows.
-    # Using subqueries keeps million-row recovery in SQLite instead of building
-    # Python ID lists or scanning all sealed snapshots.
-    Snapshot.objects.filter(
-        id__in=backoff_results.values("snapshot_id"),
-        status__in=[Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED],
-        retry_at__isnull=True,
-    ).update(retry_at=now, modified_at=now)
+    # ArchiveResult rows are projections, never work items. Close interrupted
+    # projections as failed and wake their parent Snapshot so abx-dl can replay
+    # the snapshot-level sequence. Indexed subqueries keep this bounded.
     Snapshot.objects.filter(
         id__in=orphaned_results.values("snapshot_id"),
-        status__in=[Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED],
-        retry_at__isnull=True,
+        status=Snapshot.StatusChoices.STARTED,
     ).update(retry_at=now, modified_at=now)
-    cleaned["archiveresults_backoff"] = backoff_results.update(status=ArchiveResult.StatusChoices.QUEUED, modified_at=now)
-    # Targeted plugin rows on final/paused Snapshots are scheduled through the
-    # parent Snapshot.retry_at. retry_at=NULL is the normal idle marker for a
-    # sealed Snapshot and must not be interpreted as queued work just because
-    # old/synthetic ArchiveResult rows exist. If takeover kills the runner
-    # after it leases the Snapshot but before queued ArchiveResult rows finish,
-    # the rows remain QUEUED while retry_at sits in the future. Recovery runs
-    # only after this runner has won the single-runner gate, so it can safely
-    # unlock those stale plugin leases for immediate processing instead of
-    # waiting out the previous owner's full lock timeout.
-    queued_plugin_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.QUEUED, **result_filter)
-    cleaned["snapshots_queued_plugin_rows_waiting_on_stale_lease"] = (
-        Snapshot.objects.filter(
-            id__in=queued_plugin_results.values("snapshot_id"),
-            status__in=[Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED],
-        )
-        .filter(retry_at__gt=now)
-        .update(retry_at=now, modified_at=now)
-    )
-    # Impossible state repair: STARTED ArchiveResults without a live Process
-    # have no owner left to emit completion. Requeue only the result row; the
-    # snapshot/crawl schedulers will pick up normal retry processing.
-    cleaned["archiveresults_started_without_running_process"] = orphaned_results.update(
-        status=ArchiveResult.StatusChoices.QUEUED,
-        process=None,
+    cleaned["archiveresults_interrupted_without_running_process"] = orphaned_results.update(
+        status=ArchiveResult.StatusChoices.FAILED,
         modified_at=now,
     )
     orphaned_hook_processes = Process.objects.filter(
@@ -201,21 +168,24 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
             continue
         if snapshot is None:
             continue
-        result, created = ArchiveResult.objects.get_or_create(
-            snapshot=snapshot,
-            plugin=plugin_dir.name,
-            hook_name=hook_name,
+        result, created = ArchiveResult.get_or_create_by_hook(
+            snapshot,
+            plugin_dir.name,
+            hook_name,
             defaults={
-                "status": ArchiveResult.StatusChoices.QUEUED,
+                "status": ArchiveResult.StatusChoices.FAILED,
             },
         )
         process_is_newer = bool(process.started_at and (result.start_ts is None or process.started_at >= result.start_ts))
-        if result.status == ArchiveResult.StatusChoices.QUEUED or process_is_newer:
+        if created or process_is_newer:
             requeue_snapshot = False
             # A runner can die after the hook Process exits but before the
             # ProcessCompletedEvent projector links/finalizes ArchiveResult.
             # Reconstruct only that exact hook row from the durable Process row.
-            output_files, output_size, output_mimetypes = _collect_output_metadata(plugin_dir)
+            manifest = OutputManifest.scan(plugin_dir, containment_root=snapshot.output_dir)
+            output_files = manifest.as_mapping()
+            output_size = manifest.total_size
+            output_mimetypes = ",".join(manifest.mimetypes)
             emitted_records = [
                 record
                 for record in Process.parse_records_from_text(process.stdout or "")
@@ -228,16 +198,14 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
             result.start_ts = process.started_at
             result.end_ts = process.ended_at
             if _is_signal_interrupted_exit(process.exit_code):
-                # The owning runner died or was asked to stop while the hook was
-                # still active. Keep the work item queued so takeover retries the
-                # same hook; treating an unknown signal exit as success would
-                # silently skip unfinished side effects.
+                # The Process is a durable fact; record the interruption as a
+                # failure and wake the parent Snapshot for replay.
                 result.output_files = {}
                 result.output_size = 0
                 result.output_mimetypes = ""
                 result.output_str = ""
                 result.output_json = None
-                result.status = ArchiveResult.StatusChoices.QUEUED
+                result.status = ArchiveResult.StatusChoices.FAILED
                 requeue_snapshot = True
             else:
                 result.output_files = output_files
@@ -277,7 +245,6 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
                 Snapshot.objects.filter(id=snapshot.id).update(retry_at=now, modified_at=now)
         if created:
             cleaned["archiveresults_missing_for_orphaned_hook_processes"] += 1
-            Snapshot.objects.filter(id=snapshot.id).update(retry_at=now, modified_at=now)
     started_snapshots = Snapshot.objects.filter(status=Snapshot.StatusChoices.STARTED).filter(
         Q(retry_at__isnull=True) | Q(retry_at__gt=now),
         **snapshot_filter,
@@ -291,8 +258,8 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
     # We only unlock scheduling; normal Snapshot runner code owns the next
     # transition and side effects.
     cleaned["snapshots_started_without_running_results"] = (
-        started_snapshots.annotate(has_running_results=Exists(running_archiveresults))
-        .filter(has_running_results=False)
+        started_snapshots.annotate(has_running_process=Exists(running_hook_processes))
+        .filter(has_running_process=False)
         .update(
             retry_at=now,
             modified_at=now,
@@ -301,7 +268,7 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
 
     # Broken lock repair: STARTED + retry_at=NULL is an orphaned ownership
     # lease. Recovery only unlocks scheduling; the runner owns any subsequent
-    # state-machine transition, including sealing rows whose children/results
+    # lifecycle transition, including sealing rows whose children/results
     # are already final.
     recoverable_started_crawls = Crawl.objects.filter(status=Crawl.StatusChoices.STARTED).filter(
         Q(retry_at__isnull=True) | Q(retry_at__gt=now),
@@ -349,13 +316,9 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
             "Finishing {count} browser-extension Snapshot(s) that received uploaded files before server extractors started "
             "(uploaded and server-created results will remain together on the same Snapshot)."
         ),
-        "archiveresults_backoff": (
-            "Retrying {count} extractor result(s) that were waiting to retry "
-            "(ArchiveBox may have been interrupted before it was able to try them again; affected outputs will be retried)."
-        ),
-        "archiveresults_started_without_running_process": (
-            "Retrying {count} extractor result(s) that were interrupted before finishing "
-            "(ArchiveBox may have been interrupted before it was able to save their final status; partial files will be overwritten with fresh results upon retry)."
+        "archiveresults_interrupted_without_running_process": (
+            "Closing {count} interrupted extractor result projection(s) "
+            "(their parent Snapshots are resumed through the normal snapshot-level runner)."
         ),
         "snapshots_started_without_running_results": (
             "Resuming {count} Snapshot(s) that were interrupted before finishing "

archivebox/core/settings.py

@@ -76,7 +76,6 @@ INSTALLED_APPS = [
     "archivebox.crawls",  # handles Crawl and CrawlSchedule models and management (depends on core)
     "archivebox.progressmonitor",  # live progress endpoint and admin monitor template
     "archivebox.api",  # Django-Ninja-based Rest API interfaces, config, APIToken model, etc.
-    "archivebox.opencode",
     # 3rd-party apps from PyPI that need to be loaded last
     "admin_data_views",  # handles rendering some convenient automatic read-only views of data in Django admin
     "django_extensions",  # provides Django Debug Toolbar (and other non-debug helpers)

archivebox/core/tag_util.py

@@ -152,15 +152,8 @@ def get_or_create_tag(name: str, created_by: User | None = None) -> tuple[Tag, b
     if not normalized_name:
         raise ValueError("Tag name is required")
 
-    existing = Tag.objects.filter(name__iexact=normalized_name).first()
-    if existing:
-        return existing, False
-
-    tag = Tag.objects.create(
-        name=normalized_name,
-        created_by=created_by,
-    )
-    return tag, True
+    defaults = {"created_by": created_by} if created_by is not None else None
+    return Tag.get_or_create_by_name(normalized_name, defaults=defaults)
 
 
 def rename_tag(tag: Tag, name: str) -> Tag:

archivebox/core/takeover_util.py

@@ -1,3 +1,17 @@
+"""Coordinate local foreground processes without pretending to provide a distributed lock.
+
+ArchiveBox wants one orchestrator per collection, but Process rows can only prove
+liveness for PIDs visible on the current machine/PID namespace. We therefore
+enforce one active runner per ``(Machine, DATA_DIR)`` locally, retire stale rows
+from sequential containers on that machine, and only warn about rows owned by a
+different machine. Foreign-machine rows must never block progress or be killed:
+multi-machine coordination belongs in the Crawl/Snapshot CAS claim layer, not in
+process takeover.
+
+These helpers only hand local supervisord/runner ownership between CLI parents.
+They must not hold database transactions or filesystem locks while work runs.
+"""
+
 from __future__ import annotations
 
 import time
@@ -15,25 +29,6 @@ RUNNER_WAITING_WORKER_TYPE = "runner_waiting"
 RUNNER_GATE_WORKER_TYPES = (RUNNER_ACTIVE_WORKER_TYPE, RUNNER_WAITING_WORKER_TYPE, "")
 
 
-def runtime_stack_owner_types():
-    from archivebox.machine.models import Process
-
-    return (
-        Process.TypeChoices.SERVER,
-        Process.TypeChoices.ORCHESTRATOR,
-    )
-
-
-def foreground_runner_owner_types():
-    from archivebox.machine.models import Process
-
-    return (
-        Process.TypeChoices.SERVER,
-        Process.TypeChoices.ADD,
-        Process.TypeChoices.UPDATE,
-    )
-
-
 def current_command(process_type: str, *, data_dir: str | Path, url: str | None = None):
     from archivebox.machine.models import Process
 
@@ -42,31 +37,8 @@ def current_command(process_type: str, *, data_dir: str | Path, url: str | None
     return proc
 
 
-def live_processes(*, process_type: str, data_dir: str | Path, url: str | None = None):
-    from archivebox.machine.models import Machine, Process
-
-    qs = Process.objects.filter(
-        machine=Machine.current(),
-        process_type=process_type,
-        status=Process.StatusChoices.RUNNING,
-        pwd=str(data_dir),
-    )
-    if url is not None:
-        qs = qs.filter(url=url)
-    return [proc for proc in qs.order_by("-created_at", "-modified_at").iterator(chunk_size=50) if proc.is_running]
-
-
-def newest_live_process(*, process_type: str, data_dir: str | Path, url: str | None = None):
-    processes = live_processes(process_type=process_type, data_dir=data_dir, url=url)
-    return processes[0] if processes else None
-
-
-def command_is_newest(command, *, process_type: str, data_dir: str | Path, url: str | None = None) -> bool:
-    leader = newest_live_process(process_type=process_type, data_dir=data_dir, url=url)
-    return bool(leader and leader.id == command.id)
-
-
 def runtime_stack_owner(*, data_dir: str | Path, exclude_id=None):
+    """Return the live local parent allowed to own the server runtime stack."""
     from archivebox.machine.models import Machine, Process
 
     machine = Machine.current()
@@ -74,7 +46,7 @@ def runtime_stack_owner(*, data_dir: str | Path, exclude_id=None):
         machine=machine,
         status=Process.StatusChoices.RUNNING,
         pwd=str(data_dir),
-        process_type__in=runtime_stack_owner_types(),
+        process_type__in=(Process.TypeChoices.SERVER, Process.TypeChoices.ORCHESTRATOR),
     )
     if exclude_id is not None:
         base_qs = base_qs.exclude(id=exclude_id)
@@ -103,6 +75,7 @@ def command_owns_runtime_stack(command, *, data_dir: str | Path) -> bool:
 
 
 def foreground_runner_owner(*, data_dir: str | Path, exclude_id=None):
+    """Return the newest live local parent allowed to borrow runner/sonic."""
     from archivebox.machine.models import Machine, Process
 
     machine = Machine.current()
@@ -110,7 +83,7 @@ def foreground_runner_owner(*, data_dir: str | Path, exclude_id=None):
         machine=machine,
         status=Process.StatusChoices.RUNNING,
         pwd=str(data_dir),
-        process_type__in=foreground_runner_owner_types(),
+        process_type__in=(Process.TypeChoices.SERVER, Process.TypeChoices.ADD, Process.TypeChoices.UPDATE),
     )
     if exclude_id is not None:
         qs = qs.exclude(id=exclude_id)
@@ -126,26 +99,6 @@ def command_owns_foreground_runner(command, *, data_dir: str | Path) -> bool:
     return bool(owner and owner.id == command.id)
 
 
-def runtime_stack_component_label(*, owner=None, data_dir: str | Path) -> str:
-    try:
-        from archivebox.workers.supervisord_util import active_supervisord_runtime_components
-
-        components = active_supervisord_runtime_components()
-    except Exception:
-        components = []
-
-    names = list(components)
-    if not names and owner is not None:
-        from archivebox.machine.models import Process
-
-        if owner.process_type == Process.TypeChoices.SERVER:
-            names = ["orchestrator", "server"]
-        elif owner.process_type == Process.TypeChoices.ORCHESTRATOR:
-            names = ["orchestrator"]
-
-    return ", ".join(dict.fromkeys(names)) or "runtime stack"
-
-
 def ensure_daemon_stack(*, reason: str = ""):
     from archivebox.config.common import get_config
     from archivebox.workers.supervisord_util import (
@@ -181,56 +134,65 @@ def ensure_daemon_stack(*, reason: str = ""):
     return start_worker(supervisor, sonic_worker)
 
 
-def healthy_orchestrator(*, data_dir: str | Path):
-    from archivebox.machine.models import Machine, Process
-    from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker
+def live_runner_processes(*, data_dir: str | Path):
+    """Return locally verifiable runners and warn about unsupported overlap.
 
-    supervisor = get_existing_supervisord_process()
-    worker = get_worker(supervisor, "worker_runner") if supervisor else None
-    if isinstance(worker, dict) and worker.get("statename") in ("STARTING", "RUNNING"):
-        return worker
-
-    for proc in Process.objects.filter(
-        machine=Machine.current(),
-        process_type=Process.TypeChoices.ORCHESTRATOR,
-        status=Process.StatusChoices.RUNNING,
-        pwd=str(data_dir),
-    ).order_by("-created_at"):
-        if proc.is_running:
-            return proc
-    return None
-
-
-def _runner_sort_key(process):
-    return (process.started_at or process.created_at, process.created_at, str(process.id))
-
-
-def live_runner_processes(*, data_dir: str | Path, exclude_id=None):
+    A Process row from another machine is observability only: its PID cannot be
+    checked or signalled here, so it neither joins the local election nor gets
+    mutated. A row for this same Machine from another PID namespace represents
+    a previous sequential container under the supported model; warn, retire the
+    unreachable row, and let the new container continue.
+    """
     from archivebox.machine.models import Machine, Process
 
     machine = Machine.current()
     Process.cleanup_stale_running(machine=machine)
     qs = Process.objects.filter(
-        machine=machine,
         status=Process.StatusChoices.RUNNING,
         process_type=Process.TypeChoices.ORCHESTRATOR,
         worker_type__in=RUNNER_GATE_WORKER_TYPES,
         pwd=str(data_dir),
     )
-    if exclude_id is not None:
-        qs = qs.exclude(id=exclude_id)
-    return [process for process in qs.order_by("started_at", "created_at").iterator(chunk_size=20) if process.is_running]
+    foreign_machine_exists = qs.exclude(machine=machine).exists()
+    qs = qs.filter(machine=machine)
+    live = []
+    foreign_namespace_ids = []
+    for process in qs.order_by("started_at", "created_at").iterator(chunk_size=20):
+        if not process.shares_pid_namespace:
+            foreign_namespace_ids.append(process.id)
+            continue
+        if process.is_running:
+            live.append(process)
+    if foreign_machine_exists or foreign_namespace_ids:
+        rprint(
+            "[bold yellow]WARNING: Multiple orchestrators sharing a single collection is not officially supported! "
+            "Corruption may occur if you run two ArchiveBox workers on the same collection at once.[/bold yellow]",
+            file=sys.stderr,
+            soft_wrap=True,
+        )
+    if foreign_namespace_ids:
+        now = timezone.now()
+        Process.objects.filter(id__in=foreign_namespace_ids, status=Process.StatusChoices.RUNNING).update(
+            status=Process.StatusChoices.EXITED,
+            exit_code=0,
+            ended_at=now,
+            retry_at=None,
+            modified_at=now,
+        )
+    return live
 
 
 def enter_single_runner_gate(command, *, data_dir: str | Path, graceful_timeout: float = 5.0) -> bool:
     """
-    Admit exactly one active runner for this DATA_DIR using Process rows.
+    Admit one active runner for this Machine and DATA_DIR using Process rows.
 
     The current process is a real OS process while it waits, so we keep its
     Process row RUNNING but mark worker_type=runner_waiting. Only the process
     that wins takeover is promoted to worker_type=worker_runner, which is
-    protected by a partial unique DB constraint. Older runners are terminated
-    and fully waited out before promotion, so the runner work loop never overlaps.
+    protected by a partial unique DB constraint scoped to (Machine, DATA_DIR).
+    Older locally verifiable runners are terminated and fully waited out before
+    promotion, so runner work never overlaps on one machine. Foreign machines
+    are intentionally outside this gate and only produce a warning above.
     """
     from archivebox.machine.models import Process
 
@@ -252,7 +214,7 @@ def enter_single_runner_gate(command, *, data_dir: str | Path, graceful_timeout:
             )
             runners = live_runner_processes(data_dir=data_dir)
 
-        newest = max(runners, key=_runner_sort_key)
+        newest = max(runners, key=lambda process: (process.started_at or process.created_at, process.created_at, str(process.id)))
         if newest.id != command.id:
             rprint(
                 f"[yellow][*] Newer ArchiveBox runner pid={newest.pid} is taking over; exiting this runner.[/yellow]",
@@ -263,14 +225,8 @@ def enter_single_runner_gate(command, *, data_dir: str | Path, graceful_timeout:
         older_runners = [process for process in runners if process.id != command.id]
         if older_runners:
             for process in older_runners:
-                if process.shares_pid_namespace:
-                    rprint(f"[yellow][*] Stopping older ArchiveBox runner process (pid={process.pid})...[/yellow]", file=sys.stderr)
-                    process.kill_tree(graceful_timeout=graceful_timeout)
-                else:
-                    rprint(
-                        "[yellow][*] Waiting for older ArchiveBox runner in another PID namespace to stop...[/yellow]",
-                        file=sys.stderr,
-                    )
+                rprint(f"[yellow][*] Stopping older ArchiveBox runner process (pid={process.pid})...[/yellow]", file=sys.stderr)
+                process.kill_tree(graceful_timeout=graceful_timeout)
             time.sleep(0.1)
             continue
 
@@ -290,24 +246,9 @@ def enter_single_runner_gate(command, *, data_dir: str | Path, graceful_timeout:
             time.sleep(0.1)
 
 
-def standby_until_leader_needed(command, *, process_type: str, data_dir: str | Path, url: str | None = None, interval: float = 2.0) -> None:
-    from archivebox.workers.supervisord_util import reap_foreground_supervisord_process
-
-    announced = False
-    while not command_is_newest(command, process_type=process_type, data_dir=data_dir, url=url):
-        reap_foreground_supervisord_process()
-        if not announced:
-            leader = newest_live_process(process_type=process_type, data_dir=data_dir, url=url)
-            leader_pid = leader.pid if leader else "unknown"
-            rprint(f"[yellow][*] Standing by; newer ArchiveBox process pid={leader_pid} is running the orchestrator and server.[/yellow]")
-            announced = True
-        time.sleep(interval)
-    command.modified_at = timezone.now()
-    command.save(update_fields=["modified_at"])
-
-
 def standby_until_runtime_stack_needed(command, *, data_dir: str | Path, interval: float = 2.0) -> dict[str, object]:
-    from archivebox.workers.supervisord_util import reap_foreground_supervisord_process
+    from archivebox.machine.models import Process
+    from archivebox.workers.supervisord_util import active_supervisord_runtime_components, reap_foreground_supervisord_process
 
     announced = False
     previous_owner_pid = None
@@ -316,7 +257,16 @@ def standby_until_runtime_stack_needed(command, *, data_dir: str | Path, interva
         if not announced:
             owner = runtime_stack_owner(data_dir=data_dir)
             owner_pid = owner.pid if owner else "unknown"
-            components = runtime_stack_component_label(owner=owner, data_dir=data_dir)
+            try:
+                component_names = list(active_supervisord_runtime_components())
+            except Exception:
+                component_names = []
+            if not component_names and owner is not None:
+                if owner.process_type == Process.TypeChoices.SERVER:
+                    component_names = ["orchestrator", "server"]
+                elif owner.process_type == Process.TypeChoices.ORCHESTRATOR:
+                    component_names = ["orchestrator"]
+            components = ", ".join(dict.fromkeys(component_names)) or "runtime stack"
             previous_owner_pid = owner_pid
             rprint(
                 f"[yellow][*] A newer archivebox process took over the {components} "

archivebox/core/templatetags/core_tags.py

@@ -1,7 +1,9 @@
+import logging
 import os
 from html import unescape
 from pathlib import Path
 from typing import Any
+from urllib.parse import quote
 
 from abx_plugins.plugins.archivewebpage.replay_preview import is_replay_target as is_archivewebpage_replay_target
 from django import template
@@ -11,6 +13,7 @@ from django.utils.html import escape
 from django.utils.safestring import mark_safe
 from django.utils.text import Truncator
 
+from archivebox.config import CONSTANTS
 from archivebox.core.routes_util import (
     build_snapshot_url,
     get_admin_base_url,
@@ -26,10 +29,25 @@ from archivebox.plugins.discovery import (
 
 register = template.Library()
 
+
+@register.simple_tag(takes_context=True)
+def plugin_ui(context, plugin: str, slot: str):
+    """Optional plugin UI must never prevent the host page from rendering."""
+    try:
+        source = get_plugin_template(plugin, slot, fallback=False)
+        if not source:
+            return ""
+        return template.engines["django"].from_string(source).render(context.flatten())
+    except Exception:
+        logging.getLogger(__name__).exception("Unable to render optional plugin UI: %s/%s", plugin, slot)
+        return ""
+
+
 _TEXT_PREVIEW_EXTS = (".json", ".jsonl", ".txt", ".csv", ".tsv", ".xml", ".yml", ".yaml", ".md", ".log")
 _IMAGE_PREVIEW_EXTS = (".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".ico", ".avif")
+_STATIC_URL_SAFE = "/@-._~!$&'()*+,;="
 
-_MEDIA_FILE_EXTS = {
+_VIDEO_FILE_EXTS = {
     ".mp4",
     ".webm",
     ".mkv",
@@ -46,6 +64,9 @@ _MEDIA_FILE_EXTS = {
     ".3gp",
     ".3g2",
     ".ogv",
+}
+
+_AUDIO_FILE_EXTS = {
     ".mp3",
     ".m4a",
     ".aac",
@@ -63,16 +84,15 @@ _MEDIA_FILE_EXTS = {
     ".dts",
 }
 
+_MEDIA_FILE_EXTS = _VIDEO_FILE_EXTS | _AUDIO_FILE_EXTS
+_BROWSER_VIDEO_FILE_EXTS = {".mp4", ".webm", ".m4v", ".ogv"}
+_BROWSER_AUDIO_FILE_EXTS = {".mp3", ".m4a", ".aac", ".ogg", ".oga", ".opus", ".wav", ".flac"}
+
 
 def _normalize_output_files(output_files: Any) -> dict[str, dict[str, Any]]:
-    if isinstance(output_files, dict):
-        normalized: dict[str, dict[str, Any]] = {}
-        for path, metadata in output_files.items():
-            if not path:
-                continue
-            normalized[str(path)] = dict(metadata) if isinstance(metadata, dict) else {}
-        return normalized
-    return {}
+    from abx_dl.output_files import OutputManifest
+
+    return OutputManifest.from_value(output_files).as_mapping()
 
 
 def _snapshot_id(value: Any) -> Any:
@@ -156,15 +176,27 @@ def _list_media_files(result) -> list[dict]:
 
     for rel_path, size in candidates:
         href = str(Path(result.plugin) / rel_path)
+        suffix = rel_path.suffix.lower()
+        media_type = "video" if suffix in _VIDEO_FILE_EXTS else "audio"
         media_files.append(
             {
                 "name": rel_path.name,
                 "path": href,
                 "size": size,
+                "media_type": media_type,
+                "is_video": media_type == "video",
+                "is_audio": media_type == "audio",
+                "is_browser_playable": suffix in _BROWSER_VIDEO_FILE_EXTS or suffix in _BROWSER_AUDIO_FILE_EXTS,
             },
         )
 
-    media_files.sort(key=lambda item: item["name"].lower())
+    media_files.sort(
+        key=lambda item: (
+            0 if item["is_video"] and item["is_browser_playable"] else 1 if item["is_audio"] and item["is_browser_playable"] else 2,
+            -int(item.get("size") or 0),
+            item["name"].lower(),
+        ),
+    )
     return media_files
 
 
@@ -202,16 +234,70 @@ def _is_root_snapshot_output_path(raw_output_path: str | None) -> bool:
     return normalized in ("", ".", "./", "/", "index.html", "index.json")
 
 
-def _build_snapshot_files_url(snapshot_id: str, request=None, config=None) -> str:
-    return build_snapshot_url(str(snapshot_id), "/?files=1", request=request, config=config)
+def _static_snapshot_base_url(context, snapshot) -> str:
+    start_dir = Path(context["STATIC_EXPORT_DIR"])
+    relative_path = os.path.relpath(Path(snapshot.output_dir), start=start_dir).replace(os.sep, "/")
+    if relative_path == ".":
+        return "."
+    return f"./{quote(relative_path, safe=_STATIC_URL_SAFE)}"
 
 
-def _build_snapshot_preview_url(snapshot_id: str, path: str = "", request=None, config=None, plugin: str = "") -> str:
+def _snapshot_base_url_for_context(context, snapshot) -> str:
+    if context.get("STATIC_EXPORT"):
+        return _static_snapshot_base_url(context, snapshot)
+    return get_snapshot_base_url(
+        str(_snapshot_id(snapshot)),
+        request=context.get("request"),
+        config=context.get("CONFIG"),
+    )
+
+
+def _snapshot_url_for_context(context, snapshot, path: str = "") -> str:
+    if not context.get("STATIC_EXPORT"):
+        return build_snapshot_url(
+            str(_snapshot_id(snapshot)),
+            path,
+            request=context.get("request"),
+            config=context.get("CONFIG"),
+        )
+
+    base_url = _static_snapshot_base_url(context, snapshot)
+    raw_path = str(path or "")
+    if not raw_path:
+        return base_url
+    path_part, separator, query = raw_path.lstrip("/").partition("?")
+    quoted_path = quote(path_part, safe=_STATIC_URL_SAFE)
+    suffix = f"?{query}" if separator else ""
+    return f"{base_url.rstrip('/')}/{quoted_path}{suffix}"
+
+
+def _build_snapshot_files_url(snapshot_id: str, request=None, config=None, base_url: str | None = None) -> str:
+    return (
+        f"{base_url.rstrip('/')}/index.jsonl"
+        if base_url
+        else build_snapshot_url(str(snapshot_id), "/?files=1", request=request, config=config)
+    )
+
+
+def _build_snapshot_preview_url(
+    snapshot_id: str,
+    path: str = "",
+    request=None,
+    config=None,
+    plugin: str = "",
+    base_url: str | None = None,
+) -> str:
     if path == "about:blank":
         return path
     if _is_root_snapshot_output_path(path):
-        return _build_snapshot_files_url(snapshot_id, request=request, config=config)
-    url = build_snapshot_url(str(snapshot_id), path, request=request, config=config)
+        return _build_snapshot_files_url(snapshot_id, request=request, config=config, base_url=base_url)
+    if base_url:
+        path_part, separator, query = str(path).lstrip("/").partition("?")
+        url = f"{base_url.rstrip('/')}/{quote(path_part, safe=_STATIC_URL_SAFE)}"
+        if separator:
+            url = f"{url}?{query}"
+    else:
+        url = build_snapshot_url(str(snapshot_id), path, request=request, config=config)
     path_parts = Path(path).parts
     plugin = get_plugin_name(plugin) if plugin else (path_parts[0] if len(path_parts) > 1 else "")
     has_plugin_preview = bool(plugin and get_plugin_template(plugin, "full", fallback=False))
@@ -441,23 +527,35 @@ def web_base_url(context) -> str:
     return get_web_base_url(request=context.get("request"), config=context.get("CONFIG"))
 
 
+@register.simple_tag(takes_context=True)
+def static_export_root_url(context) -> str:
+    if not context.get("STATIC_EXPORT"):
+        return ""
+    relative_path = os.path.relpath(CONSTANTS.DATA_DIR, start=Path(context["STATIC_EXPORT_DIR"]))
+    relative_path = relative_path.replace(os.sep, "/")
+    return "./" if relative_path == "." else f"{quote(relative_path, safe=_STATIC_URL_SAFE)}/"
+
+
 @register.simple_tag(takes_context=True)
 def snapshot_base_url(context, snapshot) -> str:
-    snapshot_id = _snapshot_id(snapshot)
-    return get_snapshot_base_url(str(snapshot_id), request=context.get("request"), config=context.get("CONFIG"))
+    return _snapshot_base_url_for_context(context, snapshot)
 
 
 @register.simple_tag(takes_context=True)
 def snapshot_url(context, snapshot, path: str = "") -> str:
-    snapshot_id = _snapshot_id(snapshot)
-    return build_snapshot_url(str(snapshot_id), path, request=context.get("request"), config=context.get("CONFIG"))
+    return _snapshot_url_for_context(context, snapshot, path)
 
 
 @register.simple_tag(takes_context=True)
 def snapshot_archiveresult_url(context, snapshot, plugin: str, filename: str) -> str:
     snapshot_id = str(_snapshot_id(snapshot))
     url_cache = snapshot.__dict__.setdefault("_snapshot_archiveresult_url_cache", {})
-    cache_key = (plugin, filename)
+    cache_key = (
+        plugin,
+        filename,
+        bool(context.get("STATIC_EXPORT")),
+        str(context.get("STATIC_EXPORT_DIR") or ""),
+    )
     if cache_key in url_cache:
         return url_cache[cache_key]
 
@@ -493,7 +591,7 @@ def snapshot_archiveresult_url(context, snapshot, plugin: str, filename: str) ->
         if not isinstance(file_info, dict) or int(file_info.get("size") or 0) <= 0:
             continue
         output_path = filename if file_info.get("root_relative") else f"{plugin}/{filename}"
-        url_cache[cache_key] = build_snapshot_url(snapshot_id, output_path, request=context.get("request"), config=context.get("CONFIG"))
+        url_cache[cache_key] = _snapshot_url_for_context(context, snapshot, output_path)
         return url_cache[cache_key]
 
     url_cache[cache_key] = ""
@@ -502,10 +600,7 @@ def snapshot_archiveresult_url(context, snapshot, plugin: str, filename: str) ->
 
 @register.simple_tag(takes_context=True)
 def snapshot_index_row(context, link) -> str:
-    snapshot_id = str(_snapshot_id(link))
-    request = context.get("request")
-    config = context.get("CONFIG")
-    snapshot_base = get_snapshot_base_url(snapshot_id, request=request, config=config)
+    snapshot_base = _snapshot_base_url_for_context(context, link)
 
     status = getattr(link, "status", None) or "unknown"
     bookmarked_at = getattr(link, "bookmarked_at", None)
@@ -534,22 +629,28 @@ def snapshot_index_row(context, link) -> str:
         tag_cell = '<span class="empty-value">...</span>'
 
     num_outputs = int(getattr(link, "num_outputs", 0) or 0)
-    icons = link.icons() if callable(getattr(link, "icons", None)) else getattr(link, "icons", "")
+    if context.get("STATIC_EXPORT") and callable(getattr(link, "icons", None)):
+        icons = link.icons(path=quote(link.static_archive_path, safe=_STATIC_URL_SAFE), prefix="./", quote_paths=True)
+    else:
+        icons = link.icons() if callable(getattr(link, "icons", None)) else getattr(link, "icons", "")
     icons_cell = str(icons) if icons else '<span class="empty-value">...</span>'
     archive_size = int(getattr(link, "archive_size", 0) or 0)
     size_cell = file_size(archive_size) if archive_size else '<span class="empty-value">...</span>'
     output_plural = "" if num_outputs == 1 else "s"
+    files_url = _snapshot_url_for_context(context, link, "index.jsonl") if context.get("STATIC_EXPORT") else f"{snapshot_base}/?files=1"
 
     if is_pending:
-        preview_html = (
-            '<span class="snapshot-preview snapshot-preview-spinner" aria-label="Archiving in progress">'
-            f'<img src="{escape(static("spinner.gif"))}" alt="" decoding="async" loading="lazy">'
-            "</span>"
-        )
+        preview_html = '<span class="snapshot-preview snapshot-preview-spinner" aria-label="Archiving in progress"></span>'
+        if not context.get("STATIC_EXPORT"):
+            preview_html = (
+                '<span class="snapshot-preview snapshot-preview-spinner" aria-label="Archiving in progress">'
+                f'<img src="{escape(static("spinner.gif"))}" alt="" decoding="async" loading="lazy">'
+                "</span>"
+            )
     elif "_public_preview_paths" in link.__dict__:
         preview_paths = list(getattr(link, "_public_preview_paths", []) or [])
         if preview_paths:
-            preview_urls = [build_snapshot_url(snapshot_id, path, request=request, config=config) for path in preview_paths]
+            preview_urls = [_snapshot_url_for_context(context, link, path) for path in preview_paths]
             preview_html = (
                 f'<img src="{escape(preview_urls[0])}" '
                 f'data-fallbacks="{escape(",".join(preview_urls[1:]))}" '
@@ -579,7 +680,7 @@ def snapshot_index_row(context, link) -> str:
     if "_public_favicon_paths" in link.__dict__:
         favicon_paths = list(getattr(link, "_public_favicon_paths", []) or [])
         if favicon_paths:
-            favicon_urls = [build_snapshot_url(snapshot_id, path, request=request, config=config) for path in favicon_paths]
+            favicon_urls = [_snapshot_url_for_context(context, link, path) for path in favicon_paths]
             favicon_html = (
                 f'<img src="{escape(favicon_urls[0])}" '
                 f'data-fallbacks="{escape(",".join(favicon_urls[1:]))}" '
@@ -640,7 +741,7 @@ def snapshot_index_row(context, link) -> str:
         </span>
     </td>
     <td class="snapshot-size-cell">
-        <a href="{escape(snapshot_base)}/?files=1" title="View archived files">
+        <a href="{escape(files_url)}" title="View archived file manifest">
             {size_cell}
         </a>
         <small>{num_outputs} output{output_plural}</small>
@@ -660,6 +761,7 @@ def snapshot_preview_url(context, snapshot, path: str = "", result=None) -> str:
         request=context.get("request"),
         config=context.get("CONFIG"),
         plugin=plugin,
+        base_url=_snapshot_base_url_for_context(context, snapshot) if context.get("STATIC_EXPORT") else None,
     )
 
 
@@ -699,24 +801,26 @@ def plugin_card(context, result) -> str:
 
     # Use embed_path() for the display path
     raw_output_path = result.embed_path() or ""
-    output_url = build_snapshot_url(
-        str(result.snapshot_id),
-        raw_output_path or "",
-        request=context.get("request"),
-        config=context.get("CONFIG"),
-    )
+    output_url = _snapshot_url_for_context(context, result.snapshot, raw_output_path or "")
 
     icon_html = get_plugin_icon(plugin)
     plugin_lower = (plugin or "").lower()
     media_file_count = _count_media_files(result) if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl") else 0
     media_files = _list_media_files(result) if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl") else []
+    if context.get("STATIC_EXPORT") and media_files:
+        media_files = [
+            item
+            for item in media_files
+            if (media_path := Path(str(item.get("path") or "")))
+            and not media_path.is_absolute()
+            and ".." not in media_path.parts
+            and (Path(result.snapshot_dir) / media_path).is_file()
+        ]
+        media_file_count = len(media_files)
     if media_files:
-        snapshot_id = str(result.snapshot_id)
-        request = context.get("request")
-        config = context.get("CONFIG")
         for item in media_files:
             path = item.get("path") or ""
-            item["url"] = build_snapshot_url(snapshot_id, path, request=request, config=config) if path else ""
+            item["url"] = _snapshot_url_for_context(context, result.snapshot, path) if path else ""
 
     output_lower = (raw_output_path or "").lower()
     force_text_preview = output_lower.endswith(_TEXT_PREVIEW_EXTS)
@@ -794,12 +898,7 @@ def plugin_full(context, result) -> str:
         raw_output_path = result.embed_path() or ""
     if _is_root_snapshot_output_path(raw_output_path):
         return ""
-    output_url = build_snapshot_url(
-        str(result.snapshot_id),
-        raw_output_path,
-        request=context.get("request"),
-        config=context.get("CONFIG"),
-    )
+    output_url = _snapshot_url_for_context(context, result.snapshot, raw_output_path)
 
     try:
         tpl = template.Template(template_str)

archivebox/core/views.py

@@ -21,7 +21,6 @@ from django.core.paginator import InvalidPage
 from django.db.models import Case, IntegerField, Q, Value, When
 from django.http import Http404, HttpRequest, HttpResponse, HttpResponseForbidden, QueryDict
 from django.shortcuts import redirect, render
-from django.utils import timezone
 from django.utils.decorators import method_decorator
 from django.utils.html import format_html, format_html_join
 from django.utils.safestring import mark_safe
@@ -54,7 +53,6 @@ from archivebox.core.permissions import (
     can_view_snapshot,
     direct_snapshots_queryset,
     filter_personas_by_permissions,
-    get_snapshot_permissions,
     is_admin_user,
     public_snapshots_queryset,
 )
@@ -69,15 +67,12 @@ from archivebox.core.routes_util import (
     host_matches,
 )
 from archivebox.crawls.models import Crawl
-from archivebox.misc.logging_util import printable_filesize
 from archivebox.misc.paginators import AcceleratedPaginator
 from archivebox.misc.serve_static import serve_static_with_byterange_support
 from archivebox.misc.util import (
     base_url,
     filter_queryset_by_uuid_substring,
-    htmldecode,
     sanitize_html_text,
-    ts_to_date_str,
     urldecode,
     validate_url,
     without_fragment,
@@ -85,7 +80,7 @@ from archivebox.misc.util import (
 from archivebox.plugins.discovery import get_plugin_name, get_plugin_template
 from archivebox.plugins.forms import get_plugin_config_binary_urls
 from archivebox.plugins.views import get_config_definition_link
-from archivebox.progressmonitor.views import live_progress_view, progress_endpoint
+from archivebox.progressmonitor.views import live_progress_view
 from archivebox.search.config import (
     get_search_mode,
     get_search_mode_backend,
@@ -332,162 +327,11 @@ class SnapshotView(View):
 
     @staticmethod
     def render_live_index(request, snapshot):
-        TITLE_LOADING_MSG = "Not yet archived..."
-        from archivebox.core.widgets import TagEditorWidget
-
-        # Reuse the middleware-attached config; never re-bootstrap from env + plugin
-        # schemas just to render a snapshot page (that pays ~30ms for no reason).
-        runtime_config = get_request_config(request)
-        snapshot._runtime_config = runtime_config
-        snapshot_permissions = get_snapshot_permissions(snapshot)
-        archive_results = list(snapshot.archiveresult_set.all().order_by("start_ts"))
-        tags = list(snapshot.tags.all())
-        snapshot.__dict__["_admin_archiveresults"] = archive_results
-        snapshot.__dict__["_tags_str_cached"] = ",".join(sorted(tag.name for tag in tags))
-        snapshot.__dict__["num_outputs_cached"] = sum(result.status == ArchiveResult.StatusChoices.SUCCEEDED for result in archive_results)
-        snapshot.__dict__["num_failures_cached"] = sum(result.status == ArchiveResult.StatusChoices.FAILED for result in archive_results)
-        hidden_card_plugins = {"archivedotorg", "favicon", "title"}
-        outputs = [
-            out
-            for out in snapshot.discover_outputs(include_filesystem_fallback=True, archive_results=archive_results)
-            if (out.get("size") or 0) > 0 and out.get("name") not in hidden_card_plugins
-        ]
-        archiveresults = {}
-        result_ids_by_name = {}
-        for output in outputs:
-            if output.get("result"):
-                result_ids_by_name.setdefault(output["name"], []).append(str(output["result"].id))
-            current = archiveresults.get(output["name"])
-            if current is None or (output.get("size") or 0) > (current.get("size") or 0):
-                archiveresults[output["name"]] = output
-        for name, output in archiveresults.items():
-            output["result_ids"] = ",".join(result_ids_by_name.get(name, ()))
-        hash_index = snapshot.hashes_index
-        loose_items, failed_items = snapshot.get_detail_page_auxiliary_items(
-            outputs,
-            hidden_card_plugins=hidden_card_plugins,
-            archive_results=archive_results,
+        return render(
+            template_name="core/snapshot.html",
+            request=request,
+            context=snapshot.get_html_details_context(request=request),
         )
-        preview_priority = [
-            "singlefile",
-            "screenshot",
-            "wget",
-            "dom",
-            "pdf",
-            "readability",
-        ]
-        preferred_types = tuple(preview_priority)
-        output_order = {result_type: index for index, result_type in enumerate(archiveresults.keys())}
-
-        best_result = {"path": "about:blank", "result": None}
-        for result_type in preferred_types:
-            if result_type in archiveresults:
-                best_result = archiveresults[result_type]
-                break
-
-        related_snapshots_qs = SnapshotView.find_snapshots_for_url(
-            snapshot.url,
-            allow_fallback=False,
-        ).only("id", "url", "bookmarked_at", "created_at", "downloaded_at", "output_size")
-        related_snapshots = list(
-            related_snapshots_qs.exclude(id=snapshot.id).order_by("-bookmarked_at", "-created_at", "-timestamp")[:25],
-        )
-        related_years_map: dict[int, list[Snapshot]] = {}
-        for snap in [snapshot, *related_snapshots]:
-            snap_dt = snap.bookmarked_at or snap.created_at or snap.downloaded_at
-            if not snap_dt:
-                continue
-            related_years_map.setdefault(snap_dt.year, []).append(snap)
-        related_years = []
-        for year, snaps in related_years_map.items():
-            snaps_sorted = sorted(
-                snaps,
-                key=lambda s: s.bookmarked_at or s.created_at or s.downloaded_at or timezone.now(),
-                reverse=True,
-            )
-            related_years.append(
-                {
-                    "year": year,
-                    "latest": snaps_sorted[0],
-                    "snapshots": snaps_sorted,
-                },
-            )
-        related_years.sort(key=lambda item: item["year"], reverse=True)
-
-        warc_path = next(
-            (rel_path for rel_path in hash_index if rel_path.startswith("warc/") and ".warc" in Path(rel_path).name),
-            "warc/",
-        )
-
-        ordered_outputs = sorted(
-            archiveresults.values(),
-            key=lambda r: (
-                preferred_types.index(r["name"]) if r["name"] in preferred_types else len(preferred_types),
-                output_order.get(r["name"], len(output_order)),
-            ),
-        )
-        if best_result["path"] == "about:blank" and ordered_outputs:
-            best_result = ordered_outputs[0]
-        non_compact_outputs = [out for out in ordered_outputs if not out.get("is_compact") and not out.get("is_metadata")]
-        compact_outputs = [out for out in ordered_outputs if out.get("is_compact") or out.get("is_metadata")]
-        tag_widget = TagEditorWidget()
-        output_size = sum(int(out.get("size") or 0) for out in ordered_outputs)
-        archive_dates = [result.start_ts for result in archive_results if result.start_ts]
-        has_outputs = bool(ordered_outputs)
-        is_archived = has_outputs or snapshot.status == Snapshot.StatusChoices.SEALED
-        snapshot_status = str(snapshot.status or "").lower()
-        status_label_by_state = {
-            "queued": ("queued", "info"),
-            "started": ("running", "warning"),
-            "paused": ("paused", "default"),
-            "sealed": ("archived", "success"),
-        }
-        if has_outputs and not is_archived:
-            status_label, status_color = ("partial", "warning")
-        elif has_outputs:
-            status_label, status_color = ("archived", "success")
-        else:
-            status_label, status_color = status_label_by_state.get(snapshot_status, ("not yet archived", "danger"))
-
-        context = {
-            "id": str(snapshot.id),
-            "snapshot_id": str(snapshot.id),
-            "progress_endpoint": progress_endpoint("snapshot", snapshot.id),
-            "progress_auto_expand": snapshot_status in {"queued", "started", "paused"},
-            "url": snapshot.url,
-            "archive_path": snapshot.archive_path_from_db,
-            "title": htmldecode(snapshot.resolved_title or (snapshot.base_url if is_archived else TITLE_LOADING_MSG)),
-            "extension": snapshot.extension or "html",
-            "tags": snapshot.tags_str() or "untagged",
-            "size": printable_filesize(output_size) if output_size else "—",
-            "status": status_label,
-            "status_color": status_color,
-            "snapshot_state": snapshot_status,
-            "has_outputs": has_outputs,
-            "snapshot_permissions": snapshot_permissions,
-            "snapshot_permissions_icon": {
-                "public": "👥",
-                "unlisted": "🔗",
-                "private": "🔒",
-            }.get(snapshot_permissions, "👥"),
-            "bookmarked_date": snapshot.bookmarked_date,
-            "downloaded_datestr": snapshot.downloaded_datestr,
-            "num_outputs": snapshot.num_outputs,
-            "num_failures": snapshot.num_failures,
-            "oldest_archive_date": ts_to_date_str(min(archive_dates) if archive_dates else None),
-            "warc_path": warc_path,
-            "archiveresults": [*non_compact_outputs, *compact_outputs],
-            "best_result": best_result,
-            "snapshot": snapshot,  # Pass the snapshot object for template tags
-            "CONFIG": runtime_config,
-            "related_snapshots": related_snapshots,
-            "related_years": related_years,
-            "loose_items": loose_items,
-            "failed_items": failed_items,
-            "can_delete_outputs": bool(request.user.is_authenticated and request.user.is_active and request.user.is_superuser),
-            "title_tags": [{"name": tag.name, "style": tag_widget._tag_style(tag.name)} for tag in sorted(tags, key=lambda tag: tag.name)],
-        }
-        return render(template_name="core/snapshot.html", request=request, context=context)
 
     def get(self, request, path):
         snapshot = None

archivebox/crawls/apps.py

@@ -5,11 +5,3 @@ class CrawlsConfig(AppConfig):
     default_auto_field = "django.db.models.BigAutoField"
     name = "archivebox.crawls"
     label = "crawls"
-
-    def ready(self):
-        """Import models to register state machines with the registry"""
-        import sys
-
-        # Skip during makemigrations to avoid premature state machine access
-        if "makemigrations" not in sys.argv:
-            from archivebox.crawls.models import CrawlMachine  # noqa: F401

archivebox/crawls/models.py

@@ -19,7 +19,6 @@ from django.core.validators import MaxValueValidator, MinValueValidator
 from django.conf import settings
 from django.urls import reverse_lazy
 from django.utils import timezone
-from statemachine import State, registry
 from archivebox.config.common import rprint as print
 from archivebox.core.permissions import PERMISSIONS_VALUES, normalize_permissions
 
@@ -32,7 +31,7 @@ from archivebox.base_models.models import (
     ModelWithHealthStats,
     get_or_create_system_user_pk,
 )
-from archivebox.workers.models import RETRY_AT_MAX, ModelWithStateMachine, BaseStateMachine
+from archivebox.workers.models import ModelWithQueue
 from archivebox.crawls.schedule_util import next_run_for_schedule, validate_schedule
 from archivebox.misc.util import parse_date, sanitize_html_text, validate_url, validate_url_length
 
@@ -91,6 +90,8 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes):
 
     @property
     def last_run_at(self):
+        if self.kind == "update":
+            return self.modified_at
         latest_crawl = self.crawl_set.order_by("-created_at").first()
         if latest_crawl:
             return latest_crawl.created_at
@@ -106,6 +107,22 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes):
         now = now or timezone.now()
         return self.is_enabled and self.next_run_at <= now
 
+    @property
+    def kind(self) -> str:
+        return str((self.config or {}).get("SCHEDULE_KIND") or "crawl")
+
+    def dispatch(self, queued_at=None) -> "Crawl | None":
+        """Run maintenance directly or enqueue one ordinary Crawl."""
+        queued_at = queued_at or timezone.now()
+        if self.kind == "update":
+            from archivebox.cli.archivebox_update import run_scheduled_maintenance
+
+            run_scheduled_maintenance()
+            type(self).objects.filter(pk=self.pk).update(modified_at=queued_at)
+            self.modified_at = queued_at
+            return None
+        return self.enqueue(queued_at=queued_at)
+
     def enqueue(self, queued_at=None) -> "Crawl":
         from archivebox.config.common import build_crawl_config_snapshot
 
@@ -113,10 +130,11 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes):
         template = self.template
         label = template.label or self.label
         persona = template.persona if template.persona_id else None
+        crawl_config = {key: value for key, value in (self.config or {}).items() if key != "SCHEDULE_KIND"}
 
         return Crawl.objects.create(
             urls=template.urls,
-            config=build_crawl_config_snapshot(persona=persona, overrides=self.config or {}),
+            config=build_crawl_config_snapshot(persona=persona, overrides=crawl_config),
             max_depth=template.max_depth,
             tags_str=template.tags_str,
             persona_id=template.persona_id,
@@ -129,7 +147,7 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes):
         )
 
 
-class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWithStateMachine):
+class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWithQueue):
     id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
@@ -158,16 +176,19 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
     notes = models.TextField(blank=True, null=False, default="")
     schedule = models.ForeignKey(CrawlSchedule, on_delete=models.SET_NULL, null=True, blank=True, editable=True)
 
-    status = ModelWithStateMachine.StatusField(
-        choices=ModelWithStateMachine.StatusChoices,
-        default=ModelWithStateMachine.StatusChoices.QUEUED,
+    status = ModelWithQueue.StatusField(
+        choices=ModelWithQueue.StatusChoices,
+        default=ModelWithQueue.StatusChoices.QUEUED,
     )
-    retry_at = ModelWithStateMachine.RetryAtField(default=timezone.now)
+    retry_at = ModelWithQueue.RetryAtField(default=timezone.now)
 
-    state_machine_name = "archivebox.crawls.models.CrawlMachine"
     retry_at_field_name = "retry_at"
     state_field_name = "status"
-    StatusChoices = ModelWithStateMachine.StatusChoices
+    StatusChoices = ModelWithQueue.StatusChoices
+    INITIAL_STATE = StatusChoices.QUEUED
+    ACTIVE_STATE = StatusChoices.STARTED
+    FINAL_STATES = (StatusChoices.SEALED,)
+    FINAL_OR_ACTIVE_STATES = (*FINAL_STATES, ACTIVE_STATE)
     active_state = StatusChoices.STARTED
     delete_after_final_statuses = (StatusChoices.SEALED,)
     RUNNABLE_STATES = (StatusChoices.QUEUED, StatusChoices.STARTED)
@@ -177,17 +198,12 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
 
     snapshot_set: models.Manager["Snapshot"]
 
-    if TYPE_CHECKING:
-
-        @property
-        def sm(self) -> "CrawlMachine": ...
-
     class Meta(
         ModelWithDeleteAfter.Meta,
         ModelWithOutputDir.Meta,
         ModelWithConfig.Meta,
         ModelWithHealthStats.Meta,
-        ModelWithStateMachine.Meta,
+        ModelWithQueue.Meta,
     ):
         app_label = "crawls"
         verbose_name = "Crawl"
@@ -220,7 +236,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
     def resume(self, *, when=None, save: bool = True) -> bool:
         resumed = super().resume(when=when, save=save)
         if resumed and self.pk:
-            from archivebox.core.models import ArchiveResult, Snapshot
+            from archivebox.core.models import Snapshot
 
             resume_at = when or timezone.now()
             active_snapshots = self.snapshot_set.filter(
@@ -231,7 +247,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
                 retry_at=resume_at,
                 modified_at=timezone.now(),
             )
-            ArchiveResult.resume_queryset(ArchiveResult.objects.filter(snapshot__crawl=self), when=resume_at)
         return resumed
 
     def cancel(self) -> None:
@@ -255,7 +270,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
 
         now = timezone.now()
         # Cancellation seals the Crawl first, then lets the runner seal each
-        # child Snapshot through its own state machine. Active children that
+        # child Snapshot through its own lifecycle. Active children that
         # are already due need no write; the runner will claim them as-is.
         active_children = self.snapshot_set.filter(
             status__in=Snapshot.OPEN_STATES,
@@ -272,8 +287,8 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
 
         now = timezone.now()
         # Parent pause is a scheduler command. Wake child rows only; each
-        # Snapshot runner claim performs the real pause transition and cascades
-        # its own ArchiveResults, keeping request/admin transactions tiny.
+        # Snapshot runner claim performs the real pause transition, keeping
+        # request/admin transactions tiny.
         active_children = self.snapshot_set.filter(
             status__in=Snapshot.RUNNABLE_STATES,
         )
@@ -518,18 +533,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
             return []
         return [url for _raw_line, url in self._iter_url_lines() if url]
 
-    def has_internal_input_root(self) -> bool:
-        """Return True when Crawl.urls is preserved source text, not the work queue.
-
-        The runner creates a synthetic root snapshot for raw import text so
-        parser hooks use the same Snapshot lifecycle as every other extractor.
-        In that mode the submitted text must remain in Crawl.urls forever;
-        parsed URLs live as child Snapshot rows and should not be appended back.
-        """
-        from archivebox.core.models import Snapshot
-
-        return self.snapshot_set.filter(url=Snapshot.INTERNAL_INPUT_URL, depth=0).exists()
-
     @staticmethod
     def normalize_domain(value: str) -> str:
         candidate = (value or "").strip().lower()
@@ -799,15 +802,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
             "deleted_snapshots": filter_result["deleted_snapshots"],
         }
 
-    def get_system_task(self) -> str | None:
-        urls = self.get_urls_list()
-        if len(urls) != 1:
-            return None
-        system_url = urls[0].strip().lower()
-        if system_url.startswith("archivebox://"):
-            return system_url
-        return None
-
     def resolve_persona(self):
         from archivebox.personas.models import Persona
 
@@ -971,14 +965,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
 
         if self.status == self.StatusChoices.SEALED:
             return []
-        # Internal-input crawls preserve the submitted text verbatim in
-        # Crawl.urls. The root snapshot's parser hooks are the only supported
-        # path for turning that text into child snapshots, otherwise a later
-        # runner pass could reinterpret plain URL-looking lines as direct
-        # depth-0 work and bypass format-specific metadata parsing.
-        if self.has_internal_input_root():
-            return []
-
         created_snapshots = []
         crawl_tag_names = self.current_tag_names()
         tags_by_name: dict[str, Tag] = {}
@@ -1194,7 +1180,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
                 ),
             }
             if tag_names:
-                tag_ids = [Tag.objects.get_or_create(name=tag_name)[0].pk for tag_name in tag_names]
+                tag_ids = [Tag.get_or_create_by_name(tag_name)[0].pk for tag_name in tag_names]
                 snapshot.add_tag_ids(tag_ids)
 
         existing_scope = Snapshot.objects if bool(self._config_value(config, "ONLY_NEW", True)) else self.snapshot_set
@@ -1251,16 +1237,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         if not created_snapshots:
             return []
 
-        crawl_urls = {url for _raw_line, url in self._iter_url_lines() if url}
-        new_url_lines = [snapshot.url for snapshot in created_snapshots if snapshot.url not in crawl_urls]
-        # For internal-input crawls, Crawl.urls is the immutable source text.
-        # Child snapshots are the parsed/indexed representation, so appending
-        # discovered URLs here would both duplicate state and destroy the exact
-        # import artifact users submitted through CLI/API/UI.
-        if new_url_lines and not self.has_internal_input_root():
-            self.urls = (self.urls.rstrip() + "\n" + "\n".join(new_url_lines)).lstrip("\n")
-            self.save(update_fields=["urls", "modified_at"])
-
         tag_names_by_url: dict[str, set[str]] = {}
         for snapshot in created_snapshots:
             tag_names = {
@@ -1296,170 +1272,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
 
         return created_snapshots
 
-    def install_declared_binaries(self, binary_names: set[str], machine=None) -> None:
-        """Install crawl-declared binaries through their unified state machine."""
-        from archivebox.crawls.locks import binary_lifecycle_lock
-        from archivebox.machine.models import Binary, Machine
-
-        if not binary_names:
-            return
-
-        machine = machine or Machine.current()
-        binaries = Binary.objects.filter(machine=machine, name__in=binary_names).order_by("name")
-        for binary in binaries:
-            with binary_lifecycle_lock(str(binary.id)):
-                binary.refresh_from_db()
-                if binary.status == Binary.StatusChoices.INSTALLED:
-                    continue
-                binary.update_and_requeue(retry_at=timezone.now())
-                binary.refresh_from_db()
-                binary.tick_claimed(lock_seconds=600)
-
-        unresolved_binaries = list(
-            Binary.objects.filter(
-                machine=machine,
-                name__in=binary_names,
-            )
-            .exclude(
-                status=Binary.StatusChoices.INSTALLED,
-            )
-            .order_by("name"),
-        )
-        if unresolved_binaries:
-            binary_details = ", ".join(
-                f"{binary.name} (status={binary.status}, retry_at={binary.retry_at})" for binary in unresolved_binaries
-            )
-            raise RuntimeError(
-                f"Crawl dependencies failed to install before continuing: {binary_details}",
-            )
-
-    def run(self) -> "Snapshot | None":
-        """
-        Execute this Crawl: run hooks, process JSONL, create snapshots.
-
-        Called by the state machine when entering the 'started' state.
-
-        Returns:
-            The root Snapshot for this crawl, or None for system crawls that don't create snapshots
-        """
-        import time
-        from archivebox.plugins.hooks import run_hook, discover_hooks, process_hook_records
-        from archivebox.config.common import get_config
-        from archivebox.machine.models import Machine
-
-        def get_runtime_config():
-            return get_config(crawl=self).for_crawl_runtime(
-                crawl=self,
-                persona=persona,
-                runtime_overrides=persona_runtime_overrides,
-            )
-
-        system_task = self.get_system_task()
-        if system_task == "archivebox://update":
-            from archivebox.cli.archivebox_update import process_all_db_snapshots
-
-            process_all_db_snapshots()
-            return None
-
-        machine = Machine.current()
-        declared_binary_names: set[str] = set()
-        persona_runtime_overrides: dict[str, str] = {}
-        persona = self.resolve_persona()
-        if persona:
-            base_runtime_config = get_config(crawl=self, persona=persona)
-            chrome_binary = str(base_runtime_config.get("CHROME_BINARY") or "")
-            persona_runtime_overrides = persona.prepare_runtime_for_crawl(
-                crawl=self,
-                chrome_binary=chrome_binary,
-            )
-
-        def run_crawl_hook(hook: Path) -> set[str]:
-            primary_url = next(
-                (line.strip() for line in self.urls.splitlines() if line.strip()),
-                self.urls.strip(),
-            )
-
-            hook_start = time.time()
-            plugin_name = hook.parent.name
-            output_dir = self.output_dir / plugin_name
-            output_dir.mkdir(parents=True, exist_ok=True)
-
-            process = run_hook(
-                hook,
-                output_dir=output_dir,
-                config=get_runtime_config(),
-                crawl_id=str(self.id),
-                source_url=self.urls,
-                url=primary_url,
-                snapshot_id=str(self.id),
-            )
-            hook_elapsed = time.time() - hook_start
-            if hook_elapsed > 0.5:
-                print(f"[yellow]⏱️  Hook {hook.name} took {hook_elapsed:.2f}s[/yellow]")
-
-            if process.status == process.StatusChoices.RUNNING:
-                if process.poll() is None:
-                    return set()
-
-            from archivebox.plugins.hooks import extract_records_from_process
-
-            records = []
-            # A hook can exit before its completed Process metadata is visible.
-            # Give successful hooks a brief chance to flush JSONL stdout into
-            # the Process row before downstream hooks.
-            for delay in (0.0, 0.05, 0.1, 0.25, 0.5):
-                if delay:
-                    time.sleep(delay)
-                records = extract_records_from_process(process)
-                if records:
-                    break
-            if records:
-                print(f"[cyan]📝 Processing {len(records)} records from {hook.name}[/cyan]")
-                for record in records[:3]:
-                    print(f"   Record: type={record.get('type')}, keys={list(record.keys())[:5]}")
-            if system_task:
-                records = [record for record in records if record.get("type") in ("BinaryRequest", "Binary")]
-            overrides = {"crawl": self}
-            stats = process_hook_records(records, overrides=overrides)
-            if stats:
-                print(f"[green]✓ Created: {stats}[/green]")
-
-            hook_binary_names = {
-                str(record.get("name")).strip()
-                for record in records
-                if record.get("type") in ("BinaryRequest", "Binary") and record.get("name")
-            }
-            hook_binary_names.discard("")
-            if hook_binary_names:
-                declared_binary_names.update(hook_binary_names)
-            return hook_binary_names
-
-        hooks = discover_hooks("Crawl", config=get_runtime_config())
-
-        for hook in hooks:
-            hook_binary_names = run_crawl_hook(hook)
-            if hook_binary_names:
-                self.install_declared_binaries(hook_binary_names, machine=machine)
-
-        # Safety check: don't create snapshots if any crawl-declared dependency
-        # is still unresolved after all crawl hooks have run.
-        self.install_declared_binaries(declared_binary_names, machine=machine)
-
-        # Create snapshots from all URLs in self.urls
-        if system_task:
-            leaked_snapshots = self.snapshot_set.all()
-            if leaked_snapshots.exists():
-                leaked_count = leaked_snapshots.count()
-                leaked_snapshots.delete()
-                print(f"[yellow]⚠️  Removed {leaked_count} leaked snapshot(s) created during system crawl {system_task}[/yellow]")
-            return None
-
-        self.create_snapshots_from_urls()
-
-        # Return first snapshot for this crawl (newly created or existing)
-        # This ensures the crawl doesn't seal if snapshots exist, even if they weren't just created
-        return self.snapshot_set.first()
-
     def is_finished(self) -> bool:
         """Check if crawl is finished (all snapshots sealed or no snapshots exist)."""
         from archivebox.core.models import Snapshot
@@ -1483,11 +1295,64 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
 
         return True
 
-    def cleanup(self):
-        """Clean up background hooks and run on_CrawlEnd hooks."""
-        from archivebox.plugins.hooks import run_hook, discover_hooks
+    def can_start(self) -> bool:
+        return bool(self.urls and self.get_urls_list())
 
-        # Clean up .pid files from output directory
+    def has_finished_snapshots(self) -> bool:
+        from archivebox.core.models import Snapshot
+
+        snapshots = self.snapshot_set.all()
+        return snapshots.exists() and not snapshots.exclude(status=Snapshot.StatusChoices.SEALED).exists()
+
+    def mark_started(self) -> bool:
+        now = timezone.now()
+        updated = self.safe_update(
+            {
+                "status": self.StatusChoices.STARTED,
+                "retry_at": now + timedelta(seconds=2),
+            },
+            extra_filter={"status": self.StatusChoices.QUEUED},
+        )
+        return updated
+
+    def seal(self) -> bool:
+        """Finalize a runner-owned Crawl without dispatching hooks directly."""
+        now = timezone.now()
+        updated = self.safe_update(
+            {
+                "status": self.StatusChoices.SEALED,
+                "retry_at": None,
+                "modified_at": now,
+            },
+            refresh=False,
+            extra_filter={"status__in": (*self.RUNNABLE_STATES, self.StatusChoices.SEALED)},
+        )
+        if not updated:
+            self.refresh_from_db()
+            return False
+        self.status = self.StatusChoices.SEALED
+        self.retry_at = None
+        self.modified_at = now
+        self.schedule_child_snapshots_for_sealing()
+        self.cleanup_runtime()
+        return True
+
+    def advance_lifecycle(self) -> bool:
+        """Advance one explicit lifecycle step after the runner claims this row."""
+        if self.status == self.StatusChoices.PAUSED:
+            return False
+        if self.status == self.StatusChoices.QUEUED:
+            if self.has_finished_snapshots():
+                return self.seal()
+            if not self.can_start():
+                return False
+            return self.mark_started()
+        if self.status == self.StatusChoices.STARTED and self.is_finished():
+            return self.seal()
+        return False
+
+    def cleanup_runtime(self) -> None:
+        """Remove runner-owned runtime artifacts after abx-dl cleanup hooks finish."""
         if self.output_dir.exists():
             for pid_file in self.output_dir.glob("**/*.pid"):
                 pid_file.unlink(missing_ok=True)
@@ -1495,207 +1360,3 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         persona = self.resolve_persona()
         if persona:
             persona.cleanup_runtime_for_crawl(self)
-
-        # Run on_CrawlEnd hooks
-        from archivebox.config.common import get_config
-
-        config = get_config(crawl=self)
-
-        hooks = discover_hooks("CrawlEnd", config=config)
-
-        for hook in hooks:
-            plugin_name = hook.parent.name
-            output_dir = self.output_dir / plugin_name
-            output_dir.mkdir(parents=True, exist_ok=True)
-
-            process = run_hook(
-                hook,
-                output_dir=output_dir,
-                config=config,
-                crawl_id=str(self.id),
-                source_url=self.urls,  # Pass full newline-separated URLs
-            )
-
-            # Log failures but don't block
-            if process.exit_code != 0:
-                print(f"[yellow]⚠️ CrawlEnd hook failed: {hook.name}[/yellow]")
-
-
-# =============================================================================
-# State Machines
-# =============================================================================
-
-
-class CrawlMachine(BaseStateMachine):
-    crawl: Crawl
-
-    """
-    State machine for managing Crawl lifecycle.
-
-    Hook Lifecycle:
-    ┌─────────────────────────────────────────────────────────────┐
-    │ QUEUED State                                                │
-    │  • Waiting for crawl to be ready (has URLs)                 │
-    └─────────────────────────────────────────────────────────────┘
-                            ↓ tick() when can_start()
-    ┌─────────────────────────────────────────────────────────────┐
-    │ STARTED State → enter_started()                             │
-    │  1. crawl.run()                                             │
-    │     • discover_hooks('Crawl') → finds all crawl hooks       │
-    │     • For each hook:                                        │
-    │       - run_hook(script, output_dir, ...)                   │
-    │       - Parse JSONL from hook output                        │
-    │       - process_hook_records() → creates Snapshots          │
-    │     • create_snapshots_from_urls() → from self.urls field   │
-    │                                                              │
-    │  2. Snapshots process independently with their own          │
-    │     state machines (see SnapshotMachine)                    │
-    └─────────────────────────────────────────────────────────────┘
-                            ↓ tick() when is_finished()
-    ┌─────────────────────────────────────────────────────────────┐
-    │ SEALED State → enter_sealed()                               │
-    │  • cleanup() → runs on_CrawlEnd hooks, kills background     │
-    │  • Set retry_at=None (no more processing)                   │
-    └─────────────────────────────────────────────────────────────┘
-    """
-
-    model_attr_name = "crawl"
-
-    # States
-    queued = State(value=Crawl.StatusChoices.QUEUED, initial=True)
-    started = State(value=Crawl.StatusChoices.STARTED)
-    paused = State(value=Crawl.StatusChoices.PAUSED)
-    sealed = State(value=Crawl.StatusChoices.SEALED, final=True)
-
-    # Tick Event (polled by workers)
-    tick = (
-        queued.to(sealed, cond="has_finished_snapshots")
-        | queued.to.itself(unless="can_start")
-        | queued.to(started, cond="can_start")
-        | started.to(sealed, cond="is_finished")
-        | paused.to.itself()
-    )
-
-    # Manual event (triggered by last Snapshot sealing, or by direct
-    # index-only/bg creation when every requested URL is rejected before any
-    # Snapshot rows exist).
-    seal = queued.to(sealed) | started.to(sealed) | paused.to(sealed)
-    pause_requested = queued.to(paused) | started.to(paused)
-    resume_requested = paused.to(queued)
-
-    def can_start(self) -> bool:
-        if not self.crawl.urls:
-            print(f"[red]⚠️ Crawl {self.crawl.id} cannot start: no URLs[/red]")
-            return False
-        urls_list = self.crawl.get_urls_list()
-        if not urls_list:
-            print(f"[red]⚠️ Crawl {self.crawl.id} cannot start: no valid URLs in urls field[/red]")
-            return False
-        return True
-
-    def is_finished(self) -> bool:
-        """Check if all Snapshots for this crawl are finished."""
-        return self.crawl.is_finished()
-
-    def has_finished_snapshots(self) -> bool:
-        """A queued crawl with only final Snapshot rows was interrupted before sealing."""
-        from archivebox.core.models import Snapshot
-
-        snapshots = self.crawl.snapshot_set.all()
-        return snapshots.exists() and not snapshots.exclude(status=Snapshot.StatusChoices.SEALED).exists()
-
-    @queued.enter
-    def enter_queued(self):
-        self.crawl.update_and_requeue(
-            retry_at=timezone.now(),
-            status=Crawl.StatusChoices.QUEUED,
-        )
-
-    @started.enter
-    def enter_started(self):
-        import sys
-
-        print(f"[cyan]🔄 CrawlMachine.enter_started() - creating snapshots for {self.crawl.id}[/cyan]", file=sys.stderr)
-
-        try:
-            # Run the crawl - runs hooks, processes JSONL, creates snapshots
-            first_snapshot = self.crawl.run()
-
-            if first_snapshot:
-                print(
-                    f"[cyan]🔄 Created {self.crawl.snapshot_set.count()} snapshot(s), first: {first_snapshot.url}[/cyan]",
-                    file=sys.stderr,
-                )
-                # Update status to STARTED
-                # Set retry_at to near future so tick() can poll and check is_finished()
-                self.crawl.update_and_requeue(
-                    retry_at=timezone.now() + timedelta(seconds=2),
-                    status=Crawl.StatusChoices.STARTED,
-                )
-            else:
-                # No snapshots (system crawl that only runs setup hooks)
-                print("[cyan]🔄 No snapshots created, sealing crawl immediately[/cyan]", file=sys.stderr)
-                # Seal immediately since there's no work to do
-                self.seal()
-
-        except Exception as e:
-            print(f"[red]⚠️ Crawl {self.crawl.id} failed to start: {e}[/red]")
-            import traceback
-
-            traceback.print_exc()
-            raise
-
-    @paused.enter
-    def enter_paused(self):
-        paused = self.crawl.safe_update(
-            {
-                "retry_at": RETRY_AT_MAX,
-                "status": Crawl.StatusChoices.PAUSED,
-            },
-            extra_filter={"status__in": Crawl.RUNNABLE_STATES},
-        )
-        if paused:
-            self.crawl.schedule_child_snapshots_for_pause()
-
-    @sealed.enter
-    def enter_sealed(self):
-        now = timezone.now()
-        self.crawl.status = Crawl.StatusChoices.SEALED
-        self.crawl.retry_at = None
-        # Guard: never seal a row that a concurrent writer flipped to PAUSED.
-        # Sealing is idempotent (SEALED→SEALED is a no-op rewrite), so
-        # status__in covers both the QUEUED/STARTED→SEALED transition and the
-        # rare re-entry case.
-        updated = self.crawl.safe_update(
-            {
-                "status": Crawl.StatusChoices.SEALED,
-                "retry_at": None,
-                "modified_at": now,
-            },
-            refresh=False,
-            extra_filter={
-                "status__in": [
-                    Crawl.StatusChoices.QUEUED,
-                    Crawl.StatusChoices.STARTED,
-                    Crawl.StatusChoices.SEALED,
-                ],
-            },
-        )
-        if not updated:
-            self.crawl.refresh_from_db()
-            return
-        self.crawl.modified_at = now
-
-        self.crawl.schedule_child_snapshots_for_sealing()
-        # Clean up background hooks and run on_CrawlEnd hooks after the final
-        # state is visible so cleanup projectors cannot resurrect the crawl.
-        self.crawl.cleanup()
-
-
-# =============================================================================
-# Register State Machines
-# =============================================================================
-
-# Manually register state machines with python-statemachine registry
-# (normally auto-discovered from statemachines.py, but we define them here for clarity)
-registry.register(CrawlMachine)

archivebox/machine/apps.py

@@ -10,14 +10,6 @@ class MachineConfig(AppConfig):
     label = "machine"  # Explicit label for migrations
     verbose_name = "Machine Info"
 
-    def ready(self):
-        """Import models to register state machines with the registry"""
-        import sys
-
-        # Skip during makemigrations to avoid premature state machine access
-        if "makemigrations" not in sys.argv:
-            from archivebox.machine import models  # noqa: F401
-
 
 def register_admin(admin_site):
     from archivebox.machine.admin import register_admin

archivebox/machine/models.py

@@ -12,8 +12,6 @@ from archivebox.uuid_compat import CompactUUIDField, uuid7
 from datetime import timedelta, datetime
 from typing import TYPE_CHECKING, Any, cast
 
-from statemachine import State, registry
-
 from django.db import IntegrityError, transaction
 from django.db import models
 from django.db.models import Q, QuerySet
@@ -23,7 +21,7 @@ from django.utils.functional import cached_property
 from archivebox.config import CONSTANTS
 from archivebox.config.common import rprint
 from archivebox.base_models.models import ModelWithDeleteAfter, ModelWithHealthStats, normalize_config_json_values
-from archivebox.workers.models import BaseStateMachine, ModelWithStateMachine
+from archivebox.workers.models import ModelWithQueue
 from .detect import get_host_guid, get_os_info, get_vm_info, get_host_network, get_host_stats
 
 _psutil: Any | None = None
@@ -515,19 +513,21 @@ class BinaryManager(models.Manager):
         )
 
 
-class Binary(ModelWithHealthStats, ModelWithStateMachine):
+class Binary(ModelWithHealthStats, ModelWithQueue):
     """
     Tracks a binary on a specific machine.
 
-    Simple state machine with 2 states:
+    Simple queue lifecycle with 2 states:
     - queued: Binary needs to be installed
     - installed: Binary installed successfully (abspath, version, sha256 populated)
 
     Installation is synchronous during queued→installed transition.
     If installation fails, Binary stays in queued with retry_at set for later retry.
 
-    State machine calls run(), which emits an abxpkg BinaryRequestEvent through
-    the ArchiveBox runner and installs the binary using the specified providers.
+    BinaryService claims queued rows with a conditional update, then run()
+    emits an abxpkg BinaryRequestEvent and persists the resolved installation.
+    The database row is the only lifecycle state; there is intentionally no
+    second in-memory state machine to reconcile after a worker interruption.
     """
 
     class StatusChoices(models.TextChoices):
@@ -566,9 +566,9 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
     version = models.CharField(max_length=32, default="", null=False, blank=True)
     sha256 = models.CharField(max_length=64, default="", null=False, blank=True)
 
-    # State machine fields
-    status = ModelWithStateMachine.StatusField(choices=StatusChoices.choices, default=StatusChoices.QUEUED, max_length=16)
-    retry_at = ModelWithStateMachine.RetryAtField(
+    # Durable queue lifecycle fields
+    status = ModelWithQueue.StatusField(choices=StatusChoices.choices, default=StatusChoices.QUEUED, max_length=16)
+    retry_at = ModelWithQueue.RetryAtField(
         default=timezone.now,
         help_text="When to retry this binary installation",
     )
@@ -579,18 +579,16 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
 
     machine_id: uuid.UUID
 
-    state_machine_name: str | None = "archivebox.machine.models.BinaryMachine"
+    INITIAL_STATE = StatusChoices.QUEUED
+    ACTIVE_STATE = StatusChoices.QUEUED
+    FINAL_STATES = (StatusChoices.INSTALLED,)
+    FINAL_OR_ACTIVE_STATES = (*FINAL_STATES, ACTIVE_STATE)
     active_state: str = StatusChoices.QUEUED
     warn_on_save_outside_runner = False
 
     objects = BinaryManager()  # pyright: ignore[reportIncompatibleVariableOverride]
 
-    if TYPE_CHECKING:
-
-        @property
-        def sm(self) -> BinaryMachine: ...
-
-    class Meta(ModelWithHealthStats.Meta, ModelWithStateMachine.Meta):
+    class Meta(ModelWithHealthStats.Meta, ModelWithQueue.Meta):
         app_label = "machine"
         verbose_name = "Binary"
         verbose_name_plural = "Binaries"
@@ -604,6 +602,10 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
         """A binary is valid if it has a resolved path and is marked installed."""
         return bool(self.abspath) and self.status == self.StatusChoices.INSTALLED
 
+    @property
+    def can_install(self) -> bool:
+        return bool(self.name and self.binproviders)
+
     @cached_property
     def binary_info(self) -> dict:
         """Return info about the binary."""
@@ -748,11 +750,43 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
 
         run_binary(str(self.id))
 
+    def install(self) -> bool:
+        """Run one synchronous installation attempt for a claimed Binary."""
+        if self.status == self.StatusChoices.INSTALLED:
+            return True
+        if not self.can_install:
+            return False
+
+        rprint(f"[cyan]      🔄 installing {self.name}[/cyan]", file=sys.stderr)
+        self.run()
+        self.refresh_from_db()
+        if self.status != self.StatusChoices.INSTALLED:
+            self.update_and_requeue(
+                retry_at=timezone.now() + timedelta(seconds=300),
+                status=self.StatusChoices.QUEUED,
+            )
+            self.increment_health_stats(success=False)
+            raise RuntimeError(f"Binary {self.name} installation failed")
+
+        self.update_and_requeue(retry_at=None, status=self.StatusChoices.INSTALLED)
+        self.increment_health_stats(success=True)
+        return True
+
+    def advance_lifecycle(self) -> bool:
+        """Advance the explicit binary lifecycle after its queue row is claimed."""
+        return self.install()
+
+    def install_claimed(self, *, lock_seconds: int = 600) -> bool:
+        if not self.claim_processing_lock(lock_seconds=lock_seconds):
+            return False
+        self.refresh_from_db()
+        return self.advance_lifecycle()
+
     def cleanup(self):
         """
         Clean up background binary installation hooks.
 
-        Called by state machine if needed (not typically used for binaries
+        Called after an installation attempt if needed (not typically used for binaries
         since installations are foreground, but included for consistency).
         """
 
@@ -953,12 +987,14 @@ class Process(ModelWithDeleteAfter, models.Model):
     One Process can optionally be associated with an ArchiveResult (via OneToOne),
     but Process can also exist standalone for internal operations.
 
-    Follows the unified state machine pattern:
+    Follows the unified process lifecycle:
     - queued: Process ready to launch
     - running: Process actively executing
     - exited: Process completed (check exit_code for success/failure)
 
-    State machine calls launch() to spawn the process and monitors its lifecycle.
+    Direct Process methods and the abx-dl event projector own these transitions.
+    Keeping the DB row as the only lifecycle state makes interrupted subprocess
+    recovery observable without reconciling a second in-memory state machine.
     """
 
     class StatusChoices(models.TextChoices):
@@ -1117,7 +1153,7 @@ class Process(ModelWithDeleteAfter, models.Model):
     # Reverse relation to ArchiveResult (OneToOne from AR side)
     # archiveresult: OneToOneField defined on ArchiveResult model
 
-    # State machine fields
+    # Durable process lifecycle fields
     status = models.CharField(
         max_length=16,
         choices=StatusChoices.choices,
@@ -1138,7 +1174,6 @@ class Process(ModelWithDeleteAfter, models.Model):
     children: models.Manager[Process]
     archiveresult: ArchiveResult
 
-    state_machine_name: str = "archivebox.machine.models.ProcessMachine"
     delete_after_final_statuses = (StatusChoices.EXITED,)
 
     objects = ProcessManager()  # pyright: ignore[reportIncompatibleVariableOverride]
@@ -1156,6 +1191,11 @@ class Process(ModelWithDeleteAfter, models.Model):
             models.Index(fields=["machine", "status", "process_type"], name="mach_proc_running_idx"),
         ]
         constraints = [
+            # This is deliberately machine-scoped. It prevents two locally
+            # verifiable runners from working the same collection, while not
+            # claiming to coordinate independent hosts that share a database.
+            # Cross-machine work ownership belongs to short Crawl/Snapshot CAS
+            # claims so PostgreSQL deployments can support that model later.
             models.UniqueConstraint(
                 fields=["machine", "pwd"],
                 condition=Q(status="running", process_type="orchestrator", worker_type="worker_runner"),
@@ -1303,8 +1343,8 @@ class Process(ModelWithDeleteAfter, models.Model):
         """
         Compare-and-swap update for short Process scheduler writes.
 
-        Process is not a ModelWithStateMachine subclass yet, but its
-        state-machine methods still need the same modified_at CAS behavior as
+        Process is not a ModelWithQueue subclass, but its scheduler methods
+        still need the same modified_at CAS behavior as
         Crawl/Snapshot/Binary without falling back to save().
         """
         values = dict(update_fields)
@@ -1362,7 +1402,7 @@ class Process(ModelWithDeleteAfter, models.Model):
         self.save(update_fields=updates)
 
     def heartbeat(self) -> None:
-        """Touch modified_at so standby/leader selection can see this parent is alive."""
+        """Keep a long-lived watcher visible in recent-process monitoring."""
         self.save(update_fields=["modified_at"])
 
     def mark_exited(self, *, exit_code: int = 0) -> None:
@@ -2228,8 +2268,8 @@ class Process(ModelWithDeleteAfter, models.Model):
         """
         Gracefully terminate process: SIGTERM → wait → SIGKILL.
 
-        This consolidates the scattered SIGTERM/SIGKILL logic from:
-        - crawls/models.py Crawl.cleanup()
+        This consolidates SIGTERM/SIGKILL logic used by:
+        - workers/management/commands/runner_watch.py
         - workers/pid_utils.py stop_worker()
         - supervisord_util.py stop_existing_supervisord_process()
 
@@ -2292,8 +2332,8 @@ class Process(ModelWithDeleteAfter, models.Model):
         Uses parallel polling approach - sends SIGTERM to all processes at once,
         then polls all simultaneously with individual deadline tracking.
 
-        This consolidates the scattered child-killing logic from:
-        - crawls/models.py Crawl.cleanup() os.killpg()
+        This consolidates child-killing logic used by:
+        - core/takeover_util.py
         - supervisord_util.py stop_existing_supervisord_process()
 
         Args:
@@ -2593,206 +2633,3 @@ class Process(ModelWithDeleteAfter, models.Model):
         if cleaned:
             rprint(f"[yellow]🧹 Cleaned up {cleaned} orphaned worker/hook process record(s)[/yellow]")
         return cleaned
-
-
-# =============================================================================
-# Binary State Machine
-# =============================================================================
-
-
-class BinaryMachine(BaseStateMachine):
-    """
-    State machine for managing Binary installation lifecycle.
-
-    Simple 2-state machine:
-    ┌─────────────────────────────────────────────────────────────┐
-    │ QUEUED State                                                │
-    │  • Binary needs to be installed                             │
-    └─────────────────────────────────────────────────────────────┘
-                            ↓ tick() when can_install()
-                            ↓ Synchronous installation during transition
-    ┌─────────────────────────────────────────────────────────────┐
-    │ INSTALLED State                                             │
-    │  • Binary installed (abspath, version, sha256 set)          │
-    │  • Health stats incremented                                 │
-    └─────────────────────────────────────────────────────────────┘
-
-    If installation fails, Binary stays in QUEUED with retry_at bumped.
-    """
-
-    model_attr_name = "binary"
-    binary: Binary
-
-    # States
-    queued = State(value=Binary.StatusChoices.QUEUED, initial=True)
-    installed = State(value=Binary.StatusChoices.INSTALLED, final=True)
-
-    # Tick Event - install happens during transition
-    tick = queued.to.itself(unless="can_install") | queued.to(installed, cond="can_install", on="on_install")
-
-    def can_install(self) -> bool:
-        """Check if binary installation can start."""
-        return bool(self.binary.name and self.binary.binproviders)
-
-    @queued.enter
-    def enter_queued(self):
-        """Binary is queued for installation."""
-        self.binary.update_and_requeue(
-            retry_at=timezone.now(),
-            status=Binary.StatusChoices.QUEUED,
-        )
-
-    def on_install(self):
-        """Called during queued→installed transition. Runs installation synchronously."""
-        import sys
-
-        rprint(f"[cyan]      🔄 BinaryMachine.on_install() - installing {self.binary.name}[/cyan]", file=sys.stderr)
-
-        # Run installation hooks (synchronous, updates abspath/version/sha256 and sets status)
-        self.binary.run()
-
-        # Check if installation succeeded by looking at updated status
-        # Note: Binary.run() updates self.binary.status internally but doesn't refresh our reference
-        self.binary.refresh_from_db()
-
-        if self.binary.status != Binary.StatusChoices.INSTALLED:
-            # Installation failed - abort transition, stay in queued
-            rprint(f"[red]      ❌ BinaryMachine - {self.binary.name} installation failed, retrying later[/red]", file=sys.stderr)
-
-            # Bump retry_at to try again later
-            self.binary.update_and_requeue(
-                retry_at=timezone.now() + timedelta(seconds=300),  # Retry in 5 minutes
-                status=Binary.StatusChoices.QUEUED,  # Ensure we stay queued
-            )
-
-            # Increment health stats for failure
-            self.binary.increment_health_stats(success=False)
-
-            # Abort the transition - this will raise an exception and keep us in queued
-            raise Exception(f"Binary {self.binary.name} installation failed")
-
-        rprint(f"[cyan]      ✅ BinaryMachine - {self.binary.name} installed successfully[/cyan]", file=sys.stderr)
-
-    @installed.enter
-    def enter_installed(self):
-        """Binary installed successfully."""
-        self.binary.update_and_requeue(
-            retry_at=None,
-            status=Binary.StatusChoices.INSTALLED,
-        )
-
-        # Increment health stats
-        self.binary.increment_health_stats(success=True)
-
-
-# =============================================================================
-# Process State Machine
-# =============================================================================
-
-
-class ProcessMachine(BaseStateMachine):
-    """
-    State machine for managing Process (OS subprocess) lifecycle.
-
-    Process Lifecycle:
-    ┌─────────────────────────────────────────────────────────────┐
-    │ QUEUED State                                                │
-    │  • Process ready to launch, waiting for resources           │
-    └─────────────────────────────────────────────────────────────┘
-                            ↓ tick() when can_start()
-    ┌─────────────────────────────────────────────────────────────┐
-    │ RUNNING State → enter_running()                             │
-    │  1. process.launch()                                        │
-    │     • Spawn subprocess with cmd, pwd, env, timeout          │
-    │     • Set pid, started_at                                   │
-    │     • Process runs in background or foreground              │
-    │  2. Monitor process completion                              │
-    │     • Check exit code when process completes                │
-    └─────────────────────────────────────────────────────────────┘
-                            ↓ tick() checks is_exited()
-    ┌─────────────────────────────────────────────────────────────┐
-    │ EXITED State                                                │
-    │  • Process completed (exit_code set)                        │
-    │  • Health stats incremented                                 │
-    │  • stdout/stderr captured                                   │
-    └─────────────────────────────────────────────────────────────┘
-
-    Note: This is a simpler state machine than ArchiveResult.
-    Process is just about execution lifecycle. ArchiveResult handles
-    the archival-specific logic (status, output parsing, etc.).
-    """
-
-    model_attr_name = "process"
-    process: Process
-
-    # States
-    queued = State(value=Process.StatusChoices.QUEUED, initial=True)
-    running = State(value=Process.StatusChoices.RUNNING)
-    exited = State(value=Process.StatusChoices.EXITED, final=True)
-
-    # Tick Event - transitions based on conditions
-    tick = (
-        queued.to.itself(unless="can_start")
-        | queued.to(running, cond="can_start")
-        | running.to.itself(unless="is_exited")
-        | running.to(exited, cond="is_exited")
-    )
-
-    # Additional events (for explicit control)
-    launch = queued.to(running)
-    kill = running.to(exited)
-
-    def can_start(self) -> bool:
-        """Check if process can start (has cmd and machine)."""
-        return bool(self.process.cmd and self.process.machine)
-
-    def is_exited(self) -> bool:
-        """Check if process has exited (exit_code is set)."""
-        return self.process.exit_code is not None
-
-    @queued.enter
-    def enter_queued(self):
-        """Process is queued for execution."""
-        self.process.update_and_requeue(
-            retry_at=timezone.now(),
-            status=Process.StatusChoices.QUEUED,
-        )
-
-    @running.enter
-    def enter_running(self):
-        """Start process execution."""
-        # Lock the process while it runs
-        self.process.update_and_requeue(
-            retry_at=timezone.now() + timedelta(seconds=self.process.timeout),
-            status=Process.StatusChoices.RUNNING,
-            started_at=timezone.now(),
-        )
-
-        # Launch the subprocess
-        # NOTE: This is a placeholder - actual launch logic would
-        # be implemented based on how hooks currently spawn processes
-        # For now, Process is a data model that tracks execution metadata
-        # The actual subprocess spawning is still handled by run_hook()
-
-        # Mark as immediately exited for now (until we refactor run_hook)
-        # In the future, this would actually spawn the subprocess
-        self.process.exit_code = 0  # Placeholder
-        self.process.save()
-
-    @exited.enter
-    def enter_exited(self):
-        """Process has exited."""
-        self.process.update_and_requeue(
-            retry_at=None,
-            status=Process.StatusChoices.EXITED,
-            ended_at=timezone.now(),
-        )
-
-
-# =============================================================================
-# State Machine Registration
-# =============================================================================
-
-# Manually register state machines with python-statemachine registry
-registry.register(BinaryMachine)
-registry.register(ProcessMachine)

archivebox/misc/checks.py

@@ -48,6 +48,11 @@ def _exit_on_migration_interrupt():
             os.write(sys.stderr.fileno(), _migration_interrupt_message().encode())
         except Exception:
             pass
+        # Django's migration executor can catch or delay normal exceptions while
+        # unwinding transactions. Use the real process exit path after printing
+        # the recovery command so Ctrl+C/SIGTERM during auto-migrations does not
+        # leave `archivebox server` apparently hung after the user asked it to
+        # stop. Migrations are atomic, so this does not record partial progress.
         os._exit(130)
 
     try:

archivebox/misc/db.py

@@ -380,29 +380,40 @@ def log_sqlite_lock_holders(console: Any, *, db_path: Path = CONSTANTS.DATABASE_
 def sqlite_lock_error(error: BaseException) -> bool:
     from django.db import OperationalError as DjangoOperationalError
 
-    return isinstance(error, (SQLiteOperationalError, DjangoOperationalError)) and "database is locked" in str(error).lower()
+    message = str(error).lower()
+    return isinstance(error, (SQLiteOperationalError, DjangoOperationalError)) and (
+        "database is locked" in message or "database table is locked" in message
+    )
 
 
 def retry_sqlite_locks(action: Callable[[], Any], *, label: str, stderr: TextIO | None = None) -> Any:
+    from django.conf import settings
     from django.db import OperationalError, connections
     from rich.console import Console
 
     console = Console(file=stderr or None, stderr=stderr is None)
+    started_at = time.monotonic()
+    retry_timeout = settings.CONFIG.SQLITE_LOCK_RETRY_TIMEOUT
+    retry_interval = settings.CONFIG.SQLITE_LOCK_RETRY_INTERVAL
     while True:
         try:
             return action()
-        except OperationalError as err:
-            if "database is locked" not in str(err).lower():
-                raise
-        except SQLiteOperationalError as err:
+        except (OperationalError, SQLiteOperationalError) as err:
             if not sqlite_lock_error(err):
                 raise
+            if retry_timeout:
+                remaining = retry_timeout - (time.monotonic() - started_at)
+                if remaining <= 0:
+                    raise
+                sleep_for = min(retry_interval, remaining)
+            else:
+                sleep_for = retry_interval
 
         connections.close_all()
-        console.print(f"[yellow][*] SQLite database is locked while {label}; retrying in 5s...[/yellow]")
+        console.print(f"[yellow][*] SQLite database is locked while {label}; retrying in {sleep_for:g}s...[/yellow]")
         log_sqlite_lock_holders(console)
         with console.status("[yellow]Waiting for SQLite database lock to clear...[/yellow]", spinner="dots"):
-            time.sleep(5.0)
+            time.sleep(sleep_for)
 
 
 @contextmanager

archivebox/misc/serve_static.py

@@ -33,6 +33,26 @@ from archivebox.misc.logging_util import printable_filesize
 _HASHES_CACHE: dict[Path, tuple[float, dict[str, str]]] = {}
 IMG_SRC_ATTR_RE = re.compile(r'(<img\b[^>]*?\s(?:src|data-src)=["\'])([^"\']+)(["\'])', re.IGNORECASE)
 TRANSFORMED_HTML_PREVIEW_STYLE = """<style id="archivebox-static-html-preview-style">
+html {
+    width: 100%;
+    min-width: 100%;
+    background: #fff;
+}
+body {
+    box-sizing: border-box;
+    width: min(100%, 72rem);
+    max-width: none;
+    min-height: 100vh;
+    margin: 0 auto;
+    padding: clamp(1rem, 3vw, 2rem);
+    background: #fff;
+    color: #111827;
+    font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
+    line-height: 1.55;
+}
+body > * {
+    max-width: 100%;
+}
 img:not([width]):not([height]) {
     max-width: min(100%, 12rem);
     max-height: 12rem;

archivebox/misc/util.py

@@ -192,7 +192,10 @@ def validate_url_length(url: str) -> str:
 
 
 def validate_url(url: str) -> str:
-    url = validate_url_length((url or "").strip())
+    url = (url or "").strip()
+    if "\n" in url or "\r" in url:
+        raise ValueError("URL must be a single line.")
+    url = validate_url_length(url)
     parsed = urlparse(url)
     if parsed.scheme.lower() not in ("http", "https") or not parsed.hostname:
         raise ValueError("URL must start with http:// or https:// and include a hostname.")

archivebox/opencode/apps.py

@@ -1,6 +0,0 @@
-from django.apps import AppConfig
-
-
-class OpencodeConfig(AppConfig):
-    name = "archivebox.opencode"
-    label = "opencode_plugin"

archivebox/opencode/templates/opencode/agent.html

@@ -1,211 +0,0 @@
-{% extends "admin/base.html" %}
-{% load i18n %}
-
-{% block title %}Agent{% endblock %}
-
-{% block breadcrumbs %}
-    <div class="breadcrumbs">
-        <a href="{% url 'admin:index' %}">{% trans 'Home' %}</a>
-        &rsaquo; Agent
-    </div>
-{% endblock %}
-
-{% block extrastyle %}
-    {{ block.super }}
-    <style>
-        body.opencode-agent #content {
-            padding: 0;
-        }
-        body.opencode-agent #content-main {
-            height: calc(100vh - 150px);
-            min-height: 560px;
-        }
-        .opencode-agent-error {
-            margin: 24px;
-            color: #0f172a;
-            font: 14px -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
-        }
-        .opencode-agent-error code {
-            color: #0369a1;
-        }
-        .opencode-agent-frame {
-            width: 100%;
-            height: 100%;
-            border: 0;
-            display: block;
-        }
-        .opencode-agent-shell {
-            position: relative;
-            height: 100%;
-        }
-        .opencode-agent-welcome {
-            position: absolute;
-            inset: 0;
-            z-index: 20;
-            display: flex;
-            align-items: center;
-            justify-content: center;
-            padding: 24px;
-            background: rgba(0, 0, 0, 0.48);
-            color: #111827;
-        }
-        .opencode-agent-welcome[hidden] {
-            display: none;
-        }
-        .opencode-agent-welcome-panel {
-            width: min(720px, 100%);
-            max-height: min(720px, calc(100vh - 190px));
-            overflow: auto;
-            border-radius: 8px;
-            border: 1px solid #d1d5db;
-            background: Canvas;
-            color: CanvasText;
-            box-shadow: 0 20px 60px rgba(0, 0, 0, 0.32);
-            padding: 24px;
-            font: 14px/1.5 -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
-        }
-        .opencode-agent-welcome-panel h1 {
-            margin: 0 0 10px;
-            font-size: 22px;
-            line-height: 1.25;
-        }
-        .opencode-agent-welcome-panel p {
-            margin: 0 0 14px;
-        }
-        .opencode-agent-welcome-panel ul {
-            margin: 0 0 16px 20px;
-            padding: 0;
-        }
-        .opencode-agent-welcome-panel li {
-            margin: 0 0 8px;
-        }
-        .opencode-agent-welcome-warning {
-            margin: 16px 0;
-            padding: 12px 14px;
-            border: 1px solid #f59e0b;
-            border-radius: 6px;
-            background: color-mix(in srgb, #f59e0b 12%, Canvas);
-        }
-        .opencode-agent-welcome-actions {
-            display: flex;
-            justify-content: flex-end;
-            gap: 8px;
-        }
-        .opencode-agent-welcome button {
-            cursor: pointer;
-            border: 1px solid #2563eb;
-            border-radius: 6px;
-            background: #2563eb;
-            color: #fff;
-            padding: 8px 14px;
-            font-weight: 600;
-        }
-        @media (max-width: 700px) {
-            .opencode-agent-welcome {
-                padding: 12px;
-            }
-            .opencode-agent-welcome-panel {
-                max-height: calc(100vh - 170px);
-                padding: 18px;
-            }
-        }
-    </style>
-{% endblock %}
-
-{% block bodyclass %}{{ block.super }} opencode-agent{% endblock %}
-
-{% block content %}
-    <div id="content-main">
-        {% if error %}
-            <div class="opencode-agent-error">
-                {{ error }}
-                {% if command %}<br><br><code>{{ command }}</code>{% endif %}
-            </div>
-        {% else %}
-            <script>
-                (() => {
-                    const workdir = "{{ workdir|escapejs }}";
-                    const recentSessionId = "{{ recent_session_id|escapejs }}";
-                    const merge = (left, right) => {
-                        if (!left || typeof left !== "object" || Array.isArray(left)) return right;
-                        const out = {...left};
-                        for (const [key, value] of Object.entries(right)) {
-                            out[key] = value && typeof value === "object" && !Array.isArray(value) ? merge(out[key], value) : value;
-                        }
-                        return out;
-                    };
-                    const seed = (key, value) => {
-                        try {
-                            localStorage.setItem(key, JSON.stringify(merge(JSON.parse(localStorage.getItem(key) || "{}"), value)));
-                        } catch {
-                            localStorage.setItem(key, JSON.stringify(value));
-                        }
-                    };
-                    const setJSON = (key, value) => localStorage.setItem(key, JSON.stringify(value));
-                    const page = {
-                        activeProject: workdir,
-                        activeWorkspace: workdir,
-                        gettingStartedDismissed: true,
-                        workspaceExpanded: {[workdir]: true},
-                    };
-                    if (recentSessionId) {
-                        page.lastProjectSession = {[workdir]: {directory: workdir, id: recentSessionId, at: Date.now()}};
-                    }
-                    seed("opencode.global.dat:layout.page", page);
-                    seed("opencode.global.dat:layout", {
-                        sidebar: {opened: true, width: 280},
-                        fileTree: {opened: true, width: 280, tab: "all"},
-                    });
-                    const serverKey = `${location.origin}/admin/agent/opencode`;
-                    setJSON("opencode.global.dat:server", {
-                        list: [],
-                        projects: {
-                            [serverKey]: [{worktree: workdir, expanded: true}],
-                        },
-                        lastProject: {
-                            [serverKey]: workdir,
-                        },
-                    });
-                    localStorage.setItem("opencode.settings.dat:defaultServerUrl", serverKey);
-                })();
-            </script>
-            <div class="opencode-agent-shell">
-                <iframe src="{{ proxy_url }}" title="OpenCode Agent" class="opencode-agent-frame"></iframe>
-                <div class="opencode-agent-welcome" id="opencode-agent-welcome" hidden>
-                    <div class="opencode-agent-welcome-panel" role="dialog" aria-modal="true" aria-labelledby="opencode-agent-welcome-title">
-                        <h1 id="opencode-agent-welcome-title">ArchiveBox AI Agent</h1>
-                        <p>This agent can work directly with your ArchiveBox collection. It can inspect the crawl database, create and monitor crawls, run maintenance operations, answer research questions, query archived data, and help with complex collection workflows.</p>
-                        <p>Example prompts:</p>
-                        <ul>
-                            <li>Archive this list of URLs with depth 0, then report which ones failed and why.</li>
-                            <li>Find snapshots from the last month that failed PDF or screenshot extraction and retry them.</li>
-                            <li>Create a constrained crawl for this site, avoid login/privacy/sitemap URLs, and watch the logs as it runs.</li>
-                            <li>Summarize what my collection contains about this topic and link to the best saved pages.</li>
-                        </ul>
-                        <div class="opencode-agent-welcome-warning">
-                            The agent has unrestricted access to this collection and can make destructive edits. Review requests carefully. Permission settings can be changed from the OpenCode gear icon in the lower left.
-                        </div>
-                        <div class="opencode-agent-welcome-actions">
-                            <button type="button" id="opencode-agent-welcome-dismiss">Start using Agent</button>
-                        </div>
-                    </div>
-                </div>
-            </div>
-            <script>
-                (() => {
-                    const key = "archivebox.opencode.agentWelcomeDismissed.v1";
-                    const welcome = document.getElementById("opencode-agent-welcome");
-                    const dismiss = document.getElementById("opencode-agent-welcome-dismiss");
-                    if (!welcome || !dismiss) return;
-                    if (localStorage.getItem(key) !== "1") {
-                        welcome.hidden = false;
-                    }
-                    dismiss.addEventListener("click", () => {
-                        localStorage.setItem(key, "1");
-                        welcome.hidden = true;
-                    });
-                })();
-            </script>
-        {% endif %}
-    </div>
-{% endblock %}

archivebox/opencode/views.py

@@ -1,716 +1,140 @@
-from __future__ import annotations
+"""Django adapter for the optional plugin; never import its runtime at startup."""
 
-import atexit
-import base64
-import os
-import re
-import shutil
-import subprocess
-import threading
-import time
-from pathlib import Path
-from typing import Any
-from urllib.parse import urljoin, urlsplit
+import logging
+from io import BytesIO
 
-import httpx
-import requests
-from abx_plugins.plugins import opencode as opencode_plugin
-from archivebox.config import CONSTANTS
-from archivebox.config.common import get_config
-from archivebox.core.routes_util import build_admin_url, get_api_base_url, get_base_url
-from django.http import (
-    Http404,
-    HttpRequest,
-    HttpResponse,
-    HttpResponseForbidden,
-    StreamingHttpResponse,
-)
-from django.shortcuts import redirect, render
+from asgiref.sync import sync_to_async
+
+from django.contrib.auth.middleware import AuthenticationMiddleware
+from django.contrib.auth.views import redirect_to_login
+from django.contrib.sessions.middleware import SessionMiddleware
+from django.core.exceptions import PermissionDenied
+from django.core.handlers.asgi import ASGIRequest
+from django.db import close_old_connections
+from django.http import Http404, HttpResponse, HttpResponseForbidden, StreamingHttpResponse
+from django.template import engines
+from django.urls import Resolver404, resolve
 from django.views.decorators.csrf import csrf_exempt
 
+from archivebox.config import CONSTANTS
+from archivebox.config.common import get_request_config
+from archivebox.core.middleware import ReverseProxyAuthMiddleware
+from archivebox.core.routes_util import build_admin_url, get_admin_host, get_api_base_url, get_base_url, host_matches
+from archivebox.plugins.discovery import get_plugin_template
 
-_PROCESS: subprocess.Popen | None = None
-_PROCESS_LOCK = threading.Lock()
-_PROXY_PREFIX = "/admin/agent/opencode"
-_PROXY_PREFIX_REGEX = _PROXY_PREFIX.replace("/", r"\/")
-_PROXY_PREFIX_NO_SLASH_REGEX = _PROXY_PREFIX.lstrip("/").replace("/", r"\/")
-_CONFIG_PATH = Path(opencode_plugin.__file__).with_name("config.json")
-
-_TEXT_CONTENT_TYPES = (
-    "text/",
-    "application/javascript",
-    "application/json",
-    "application/x-javascript",
-)
-_HOP_BY_HOP_HEADERS = {
-    "connection",
-    "keep-alive",
-    "proxy-authenticate",
-    "proxy-authorization",
-    "te",
-    "trailers",
-    "transfer-encoding",
-    "upgrade",
-}
-_ARCHIVEBOX_SKILL = """---
-name: archivebox
-description: Use ArchiveBox's CLI and local REST API from an ArchiveBox collection.
----
-
-You are running inside an ArchiveBox collection directory.
-
-- ArchiveBox collection directory: {archivebox_data_dir}
-- ArchiveBox BASE_URL: {archivebox_base_url}
-- ArchiveBox Admin URL: {archivebox_admin_url}
-- ArchiveBox REST API URL: {archivebox_api_url}
-- Prefer the `archivebox` CLI for authenticated changes, e.g. `archivebox add`, `archivebox schedule`, `archivebox update`, and `archivebox shell`.
-- Run ArchiveBox CLI commands from the ArchiveBox collection directory above.
-- Get command help with `archivebox list --help`, `archivebox add --help`, `archivebox schedule --help`, etc. Do not use `archivebox help <command>`.
-- Use `--depth=0` by default. Only use recursive crawling when the user explicitly asks for it; use `--depth=1` when you need pages one hop out.
-- Before any recursive crawl, constrain scope with ArchiveBox config such as `CRAWL_MAX_URLS`, `CRAWL_MAX_SIZE`, `SNAPSHOT_MAX_*`, `URL_ALLOWLIST`, `URL_DENYLIST`, and related limits.
-- Respect the configured `archivebox config --get ONLY_NEW` behavior unless the user explicitly says otherwise. Remind users that expected crawl URLs can be skipped when the collection already contains snapshots with the same URL.
-- Always audit newly discovered crawl URLs before letting a crawl run broadly. Treat junk URLs such as privacy policies, legal pages, tag archives, sitemap files, feeds, login/logout URLs, and other low-value boilerplate as unwanted unless the user explicitly asked to archive them.
-- Always watch crawl output and logs as the crawl progresses, and correct errors early instead of waiting until the crawl finishes.
-- If a crawl contains bad URLs, pause it, edit the crawl's `urls` field to remove them, delete any unneeded snapshots already created under that crawl, then resume the crawl.
-- Use `archivebox shell -c '...'` or `archivebox shell <<'PY' ... PY` for Django ORM work. Shell Plus prints an import banner first; keep stderr visible while debugging.
-- Use full ArchiveBox module paths in shell code: `from archivebox.crawls.models import Crawl, CrawlSchedule` and `from archivebox.core.models import Snapshot, ArchiveResult`.
-- If a model/field/relation is unclear, inspect `_meta.fields` before guessing, e.g. `archivebox shell -c "from archivebox.crawls.models import Crawl; print([f.name for f in Crawl._meta.fields])"`.
-- Use `archivebox config --get BASE_URL` only to verify the configured base URL; prefer the seeded URLs above for API/admin requests.
-- Use `$ARCHIVEBOX_API_URL` for REST API inspection when helpful. Do not assume admin session cookies authenticate API subdomain requests; prefer CLI/shell for authenticated mutations unless the admin provides or asks you to create an API token.
-- Discover REST endpoints from `${{ARCHIVEBOX_API_URL}}v1/openapi.json`; crawl endpoints live under `/api/v1/crawls/`, snapshots under `/api/v1/core/`.
-- Do not bypass ArchiveBox auth, expose API keys, or modify config unless the admin explicitly asks.
-- After creating crawls or snapshots, report the crawl/snapshot IDs and the exact command or API request used.
-"""
+_LOGGER = logging.getLogger(__name__)
 
 
-def _stop_owned_process(process: subprocess.Popen | None = None) -> None:
-    global _PROCESS
-    owned_process = process or _PROCESS
-    if owned_process is None:
-        return
-    if owned_process.poll() is None:
-        owned_process.terminate()
-        try:
-            owned_process.wait(timeout=5)
-        except subprocess.TimeoutExpired:
-            owned_process.kill()
-            owned_process.wait()
-    if _PROCESS is owned_process:
-        _PROCESS = None
+def _runtime_settings(request, config):
+    from abx_plugins.plugins.opencode import runtime
 
-
-atexit.register(_stop_owned_process)
-
-
-def _machine_config() -> dict[str, Any]:
-    resolved = get_config()
-    return dict(resolved.model_dump(mode="json"))
-
-
-def _archivebox_data_dir_default() -> Path:
-    return Path(CONSTANTS.DATA_DIR)
-
-
-def _archivebox_route_urls(request: HttpRequest, route_config) -> tuple[str, str, str]:
-    base_url = get_base_url(
-        request=request,
-        config=route_config,
-    ).rstrip("/")
-    admin_url = build_admin_url(
-        "/admin/",
-        request=request,
-        config=route_config,
-    ).rstrip("/")
-    api_url = f"{get_api_base_url(request=request, config=route_config).rstrip('/')}/api/"
-    return base_url, admin_url, api_url
-
-
-def _config_value(config: dict, key: str, default):
-    value = config.get(key, default)
-    if value in (None, ""):
-        return default
-    return value
-
-
-def _opencode_enabled(config: dict) -> bool:
-    value = config.get("OPENCODE_ENABLED", False)
-    if isinstance(value, bool):
-        return value
-    return str(value).strip().lower() in {"1", "true", "yes", "on"}
-
-
-def _require_enabled(config: dict) -> None:
-    if not _opencode_enabled(config):
-        raise Http404
-
-
-def _require_superuser(request: HttpRequest):
-    user = getattr(request, "user", None)
-    if user is None:
-        return redirect(f"/admin/login/?next={request.get_full_path()}")
-    if (
-        bool(getattr(user, "is_authenticated", False))
-        and bool(getattr(user, "is_active", False))
-        and bool(getattr(user, "is_superuser", False))
-    ):
-        return None
-    if bool(getattr(user, "is_authenticated", False)):
-        return HttpResponseForbidden(
-            b"ArchiveBox agent access requires a superuser account.",
-        )
-    return redirect(f"/admin/login/?next={request.get_full_path()}")
-
-
-def _origin_allowed(request: HttpRequest, path: str | None = None) -> bool:
-    if request.method in {"GET", "HEAD", "OPTIONS", "TRACE"}:
-        return True
-
-    expected_host = request.get_host()
-    pty_connect = bool(
-        path and path.startswith("pty/") and path.endswith("/connect-token"),
-    )
-    if pty_connect:
-        return True
-
-    origin = _request_header(request, "Origin")
-    if origin:
-        return _same_host(origin, expected_host)
-
-    referer = _request_header(request, "Referer")
-    if referer:
-        return _same_host(referer, expected_host)
-
-    fetch_site = _request_header(request, "Sec-Fetch-Site")
-    if fetch_site:
-        return fetch_site in {"same-origin", "same-site", "none"}
-
-    return False
-
-
-def _same_host(value: str, expected_host: str) -> bool:
-    parsed = urlsplit(value)
-    return parsed.scheme in {"http", "https"} and parsed.netloc == expected_host
-
-
-def _settings(config: dict) -> dict:
-    host = str(_config_value(config, "OPENCODE_HOST", "127.0.0.1"))
-    port = int(_config_value(config, "OPENCODE_PORT", 4096))
-    default_data_dir = _archivebox_data_dir_default()
-    workdir = Path(
-        str(_config_value(config, "OPENCODE_WORKDIR", default_data_dir)),
-    ).expanduser()
-    opencode_dir = Path(
-        str(_config_value(config, "OPENCODE_STATE_DIR", workdir / "opencode")),
-    ).expanduser()
-    binary = str(_config_value(config, "OPENCODE_BINARY", "opencode"))
-    timeout = int(_config_value(config, "OPENCODE_TIMEOUT", 30))
-    return {
-        "host": host,
-        "port": port,
-        "origin": f"http://{host}:{port}",
-        "workdir": workdir,
-        "opencode_dir": opencode_dir,
-        "config_home": opencode_dir / "config",
-        "data_home": opencode_dir / "data",
-        "state_home": opencode_dir / "state",
-        "cache_home": opencode_dir / "cache",
-        "home": opencode_dir / "home",
-        "binary": binary,
-        "config": config,
-        "timeout": timeout,
-    }
-
-
-def _resolve_binary(binary: str, config: dict) -> tuple[Any, Any, dict[str, str]]:
-    try:
-        from abxpkg import BinProvider
-        from abx_plugins.plugins.base.utils import load_required_binary_from_config
-
-        binary_environ = os.environ.copy()
-        lib_dir = config.get("ABXPKG_LIB_DIR")
-        if lib_dir:
-            binary_environ["ABXPKG_LIB_DIR"] = str(lib_dir)
-        loaded_dependencies = [
-            load_required_binary_from_config(
-                required_binary,
-                _CONFIG_PATH,
-                global_config=config,
-                environ=binary_environ,
-                install=False,
-            )
-            for required_binary in (
-                str(config.get("NODE_BINARY") or "node"),
-                str(config.get("GIT_BINARY") or "git"),
-                binary,
-            )
-        ]
-    except Exception as err:
-        raise RuntimeError(
-            f"OpenCode dependency is not installed from required_binaries: {err}",
-        ) from err
-
-    if any(not loaded.loaded_abspath for loaded in loaded_dependencies):
-        raise RuntimeError(
-            "OpenCode dependency is not installed from required_binaries.",
-        )
-
-    providers = [loaded.loaded_binprovider for loaded in loaded_dependencies if loaded.loaded_binprovider is not None]
-    binary_env = BinProvider.build_exec_env(
-        providers=providers,
-        base_env=binary_environ,
-    )
-    return (
-        loaded_dependencies[-1],
-        loaded_dependencies[1],
-        binary_env,
-    )
-
-
-def _project_route(workdir: Path, session_id: str = "") -> str:
-    encoded = base64.b64encode(str(workdir.resolve()).encode()).decode()
-    encoded = encoded.replace("+", "-").replace("/", "_").rstrip("=")
-    route = f"{_PROXY_PREFIX}/{encoded}/session"
-    return f"{route}/{session_id}" if session_id else route
-
-
-def _ensure_project_files(settings: dict) -> None:
-    workdir = settings["workdir"].resolve()
-    workdir.mkdir(parents=True, exist_ok=True)
-    git_marker = workdir / ".git" / "not-a-git"
-    if git_marker.exists():
-        # Current OpenCode hangs on the legacy fake marker, so remove only
-        # that invalid shape before initializing the real worktree.
-        shutil.rmtree(git_marker.parent)
-
-    editable_skill_path = settings["opencode_dir"] / "SKILL.md"
-    editable_skill_path.parent.mkdir(parents=True, exist_ok=True)
-    if not editable_skill_path.exists():
-        editable_skill_path.write_text(
-            _ARCHIVEBOX_SKILL.format(
-                archivebox_data_dir=workdir,
-                archivebox_base_url=settings.get("archivebox_base_url", ""),
-                archivebox_admin_url=settings.get("archivebox_admin_url", ""),
-                archivebox_api_url=settings.get("archivebox_api_url", ""),
-            ),
-        )
-
-    opencode_skill_path = settings["config_home"] / "opencode" / "skills" / "archivebox" / "SKILL.md"
-    opencode_skill_path.parent.mkdir(parents=True, exist_ok=True)
-    if opencode_skill_path.resolve() != editable_skill_path.resolve():
-        if opencode_skill_path.exists() or opencode_skill_path.is_symlink():
-            opencode_skill_path.unlink()
-        opencode_skill_path.symlink_to(editable_skill_path)
-
-
-def _ensure_default_session(settings: dict) -> str:
-    workdir = settings["workdir"].resolve()
-    params = {"directory": str(workdir)}
-    timeout = settings["timeout"]
-    project = requests.post(
-        f"{settings['origin']}/project/git/init",
-        params=params,
-        timeout=timeout,
-    )
-    project.raise_for_status()
-    project_data = project.json()
-    if Path(str(project_data.get("worktree") or "/")).resolve() != workdir:
-        raise RuntimeError(
-            f"OpenCode initialized the wrong project worktree: {project_data.get('worktree')!r}",
-        )
-
-    sessions = requests.get(
-        f"{settings['origin']}/session",
-        params={**params, "roots": "true", "limit": 55},
-        timeout=timeout,
-    )
-    sessions.raise_for_status()
-    session_data = sessions.json()
-    if not isinstance(session_data, list):
-        raise RuntimeError("OpenCode returned an invalid project session list.")
-    for session_data_item in session_data:
-        if not isinstance(session_data_item, dict):
-            continue
-        session_id = str(session_data_item.get("id") or "")
-        session_directory = session_data_item.get("directory")
-        if session_id and session_directory and Path(str(session_directory)).resolve() == workdir:
-            return session_id
-
-    session = requests.post(
-        f"{settings['origin']}/session",
-        params=params,
-        json={},
-        timeout=timeout,
-    )
-    session.raise_for_status()
-    session_data = session.json()
-    session_id = str(session_data.get("id") or "")
-    session_directory = session_data.get("directory")
-    if not session_id or not session_directory or Path(str(session_directory)).resolve() != workdir:
-        raise RuntimeError(
-            "OpenCode did not create a session for the requested worktree.",
-        )
-    return session_id
-
-
-def _recent_session_id(settings: dict) -> str:
-    workdir = str(settings["workdir"].resolve())
-    try:
-        response = requests.get(
-            f"{settings['origin']}/session",
-            params={"directory": workdir, "roots": "true", "limit": 1},
-            timeout=settings["timeout"],
-        )
-        response.raise_for_status()
-        sessions = response.json()
-        if sessions:
-            return str(sessions[0].get("id") or "")
-    except requests.RequestException:
-        pass
-    return ""
-
-
-def _health(settings: dict) -> bool:
-    try:
-        response = requests.get(
-            f"{settings['origin']}/global/health",
-            timeout=2,
-        )
-        return response.status_code == 200
-    except requests.RequestException:
-        return False
-
-
-def _ensure_opencode(settings: dict) -> tuple[bool, str]:
-    global _PROCESS
-    started_process: subprocess.Popen | None = None
-    workdir = settings["workdir"].resolve()
-    try:
-        binary, git_binary, binary_env = _resolve_binary(
-            settings["binary"],
-            settings["config"],
-        )
-    except RuntimeError as err:
-        return False, str(err)
-
-    env = {
-        **os.environ,
-        **binary_env,
-        "ARCHIVEBOX_BASE_URL": str(settings.get("archivebox_base_url", "")),
-        "ARCHIVEBOX_ADMIN_URL": str(settings.get("archivebox_admin_url", "")),
-        "ARCHIVEBOX_API_URL": str(settings.get("archivebox_api_url", "")),
-        "BROWSER": "false",
-        "GIT_CEILING_DIRECTORIES": str(workdir),
-        "HOME": str(settings["home"]),
-        "OPENCODE_DISABLE_PROJECT_CONFIG": "true",
-        "XDG_CONFIG_HOME": str(settings["config_home"]),
-        "XDG_DATA_HOME": str(settings["data_home"]),
-        "XDG_STATE_HOME": str(settings["state_home"]),
-        "XDG_CACHE_HOME": str(settings["cache_home"]),
-    }
-
-    with _PROCESS_LOCK:
-        settings["workdir"].mkdir(parents=True, exist_ok=True)
-        settings["config_home"].mkdir(parents=True, exist_ok=True)
-        settings["data_home"].mkdir(parents=True, exist_ok=True)
-        settings["state_home"].mkdir(parents=True, exist_ok=True)
-        settings["cache_home"].mkdir(parents=True, exist_ok=True)
-        settings["home"].mkdir(parents=True, exist_ok=True)
-        _ensure_project_files(settings)
-
-        if not (workdir / ".git").exists():
-            try:
-                git_init = git_binary.exec(
-                    cmd=("init", "--quiet"),
-                    cwd=workdir,
-                    env=env,
-                    timeout=settings["timeout"],
-                )
-            except (AssertionError, OSError, subprocess.SubprocessError) as err:
-                return False, f"OpenCode project initialization failed: {err}"
-            if git_init.returncode != 0:
-                output = (git_init.stderr or git_init.stdout or "").strip()
-                return (
-                    False,
-                    f"OpenCode project initialization failed: {output or f'git exited with {git_init.returncode}'}",
-                )
-
-        if _health(settings):
-            try:
-                _ensure_default_session(settings)
-            except (requests.RequestException, RuntimeError, ValueError) as err:
-                return False, f"OpenCode project initialization failed: {err}"
-            return True, ""
-
-        binary_abspath = binary.loaded_abspath
-        if binary.loaded_binprovider is not None:
-            binary_abspath = binary.loaded_binprovider._exec_bin_abspath(
-                Path(binary.loaded_abspath),
-            )
-        cmd = [
-            str(binary_abspath),
-            "serve",
-            "--hostname",
-            settings["host"],
-            "--port",
-            str(settings["port"]),
-        ]
-        try:
-            _PROCESS = subprocess.Popen(
-                cmd,
-                cwd=workdir,
-                env=env,
-                stdin=subprocess.DEVNULL,
-                stdout=subprocess.DEVNULL,
-                stderr=subprocess.DEVNULL,
-                start_new_session=True,
-            )
-            started_process = _PROCESS
-        except FileNotFoundError:
-            return False, f"OpenCode binary not found: {settings['binary']}"
-
-        deadline = time.monotonic() + settings["timeout"]
-        while time.monotonic() < deadline:
-            if _health(settings):
-                try:
-                    _ensure_default_session(settings)
-                except (requests.RequestException, RuntimeError, ValueError) as err:
-                    _stop_owned_process(started_process)
-                    return False, f"OpenCode project initialization failed: {err}"
-                return True, ""
-            if started_process and started_process.poll() is not None:
-                if _PROCESS is started_process:
-                    _PROCESS = None
-                return False, "OpenCode exited before the web server became ready."
-            time.sleep(0.25)
-
-        _stop_owned_process(started_process)
-        return False, "Timed out waiting for OpenCode to start."
-
-
-def agent_view(request: HttpRequest):
-    config = _machine_config()
-    _require_enabled(config)
-    auth_response = _require_superuser(request)
-    if auth_response:
-        return auth_response
-
-    settings = _settings(config)
+    settings = runtime._settings(config, CONSTANTS.DATA_DIR)
     route_config = request.__dict__.get("archivebox_config")
-    base_url, admin_url, api_url = _archivebox_route_urls(request, route_config)
-    settings["archivebox_base_url"] = base_url
-    settings["archivebox_admin_url"] = admin_url
-    settings["archivebox_api_url"] = api_url
-    ok, error = _ensure_opencode(settings)
-    from archivebox.core.admin_site import archivebox_admin
-
-    recent_session_id = _recent_session_id(settings) if ok else ""
-    context = {
-        **archivebox_admin.each_context(request),
-        "title": "Agent",
-        "error": "" if ok else error,
-        "command": f"{settings['binary']} serve --hostname {settings['host']} --port {settings['port']}" if error else "",
-        # OpenCode 1.17+ keeps durable sessions on the explicit
-        # /<dirBase64>/session/<sessionId> route. We still seed localStorage
-        # below because the sidebar state uses it, but the iframe itself must
-        # open the durable session URL so a fresh browser does not land on the
-        # transient new-session route and appear to have lost prior sessions.
-        "proxy_url": _project_route(settings["workdir"], recent_session_id),
-        "workdir": str(settings["workdir"].resolve()),
-        "recent_session_id": recent_session_id,
-    }
-    return render(
-        request,
-        "opencode/agent.html",
-        context,
-        status=200 if ok else 502,
+    settings.update(
+        archivebox_base_url=get_base_url(request=request, config=route_config).rstrip("/"),
+        archivebox_admin_url=build_admin_url("/admin/", request=request, config=route_config).rstrip("/"),
+        archivebox_api_url=f"{get_api_base_url(request=request, config=route_config).rstrip('/')}/api/",
     )
+    return runtime, settings
 
 
-def _proxy_url(settings: dict, path: str | None) -> str:
-    rel = "/" if not path else f"/{path}"
-    return urljoin(settings["origin"], rel)
+def _dispatch(request, path=None):
+    try:
+        # Middleware already merged server defaults and live Machine overrides.
+        # Re-resolving every extractor's config here stalls the UI request burst.
+        config = get_request_config(request).model_dump(mode="json")
+        if not config.get("OPENCODE_ENABLED", False):
+            raise Http404
+        if not request.user.is_authenticated:
+            return redirect_to_login(request.get_full_path(), login_url="/admin/login/")
+        if not request.user.is_active or not request.user.is_superuser:
+            return HttpResponseForbidden("Agent access requires a superuser account.")
+
+        runtime, settings = _runtime_settings(request, config)
+        if path is None:
+            from archivebox.core.admin_site import archivebox_admin
+
+            context = {**archivebox_admin.each_context(request), **runtime.agent_context(settings)}
+            source = get_plugin_template("opencode", "agent", fallback=False)
+            if source is None:
+                raise RuntimeError("Agent template unavailable")
+            return HttpResponse(engines["django"].from_string(source).render(context, request))
+
+        if not runtime._origin_allowed(request.method, request.get_host(), request.headers):
+            return HttpResponseForbidden("Cross-origin agent requests are blocked.")
+        status, headers, body = runtime.proxy(
+            settings,
+            request.method,
+            path,
+            tuple((key, value) for key, values in request.GET.lists() for value in values),
+            request.headers,
+            request.body,
+        )
+        response_type = HttpResponse if isinstance(body, bytes) else StreamingHttpResponse
+        response = response_type(body, status=status, headers=headers)
+        response.xframe_options_exempt = True
+        response.headers["X-Frame-Options"] = "SAMEORIGIN"
+        response.headers["Content-Security-Policy"] = "frame-ancestors 'self'"
+        return response
+    except Http404:
+        raise
+    except Exception:
+        # Optional-service boundary, including imports and template rendering.
+        _LOGGER.exception("Optional AI service failed")
+        return HttpResponse("AI service unavailable. See server logs.", status=503, content_type="text/plain")
 
 
-def _request_header(request: HttpRequest, name: str) -> str | None:
-    meta = getattr(request, "META", {})
-    if name == "Content-Type":
-        value = meta.get("CONTENT_TYPE")
-    elif name == "Content-Length":
-        value = meta.get("CONTENT_LENGTH")
-    else:
-        value = meta.get(f"HTTP_{name.upper().replace('-', '_')}")
-    return str(value) if value else None
-
-
-def _request_headers(request: HttpRequest, settings: dict) -> dict[str, str]:
-    forwarded = {}
-    for key in ("Accept", "Accept-Language", "Content-Type", "Range", "User-Agent"):
-        value = _request_header(request, key)
-        if value:
-            forwarded[key] = value
-    return forwarded
-
-
-def _request_params(request: HttpRequest) -> tuple[tuple[str, str], ...]:
-    if hasattr(request.GET, "lists"):
-        return tuple((key, str(value)) for key, values in request.GET.lists() for value in values)
-    return tuple((key, str(value)) for key, value in dict(request.GET).items())
-
-
-async def _event_chunks(request: HttpRequest, settings: dict, path: str | None):
-    timeout = httpx.Timeout(settings["timeout"], read=None)
-    url = _proxy_url(settings, path)
-    method = request.method or "GET"
-    async with httpx.AsyncClient(timeout=timeout, follow_redirects=False) as client:
-        async with client.stream(
-            method,
-            url,
-            params=_request_params(request),
-            headers=_request_headers(request, settings),
-        ) as upstream:
-            async for chunk in upstream.aiter_raw(chunk_size=512):
-                yield chunk
-
-
-def _rewrite_text(body: bytes, settings: dict) -> bytes:
-    text = body.decode("utf-8", errors="replace")
-    text = text.replace(settings["origin"], _PROXY_PREFIX)
-    text = text.replace("location.origin", f'location.origin+"{_PROXY_PREFIX}"')
-    text = text.replace(
-        "k(k5,{get component(){return t.router??Az},",
-        f'k(k5,{{base:"{_PROXY_PREFIX}",get component(){{return t.router??Az}},',
-    )
-    text = text.replace('"/assets/', f'"{_PROXY_PREFIX}/assets/')
-    text = text.replace("'/assets/", f"'{_PROXY_PREFIX}/assets/")
-    proxy_path = rf'(\1.replace(/^{_PROXY_PREFIX_REGEX}(?=\/|$)/,"")||"/")'
-    text = re.sub(r"\b(window\.location\.pathname)\b", proxy_path, text)
-    text = re.sub(r"(?<![.\w])(location\.pathname)\b", proxy_path, text)
-    text = text.replace(
-        'window.history.replaceState(nz(o),"",r):window.history.pushState(o,"",r)',
-        (
-            f'window.history.replaceState(nz(o),"",r.startsWith("{_PROXY_PREFIX}")?r:r.startsWith("/")?"{_PROXY_PREFIX}"+r:r):'
-            f'window.history.pushState(o,"",r.startsWith("{_PROXY_PREFIX}")?r:r.startsWith("/")?"{_PROXY_PREFIX}"+r:r)'
-        ),
-    )
-    text = text.replace(
-        'const BL="modulepreload",UL=function(t){return"/"+t}',
-        f'const BL="modulepreload",UL=function(t){{return"{_PROXY_PREFIX}/"+t}}',
-    )
-    text = re.sub(
-        rf"""(?P<prefix>\b(?:href|src|action)=["'])/(?!{_PROXY_PREFIX_NO_SLASH_REGEX}(?:/|$))""",
-        rf"\g<prefix>{_PROXY_PREFIX}/",
-        text,
-    )
-    text = re.sub(
-        rf"""(?P<prefix>\b(?:fetch|EventSource)\(["'])/(?!{_PROXY_PREFIX_NO_SLASH_REGEX}(?:/|$))""",
-        rf"\g<prefix>{_PROXY_PREFIX}/",
-        text,
-    )
-    text = re.sub(
-        rf"""(?P<prefix>\burl\(["']?)/(?!{_PROXY_PREFIX_NO_SLASH_REGEX}(?:/|$))""",
-        rf"\g<prefix>{_PROXY_PREFIX}/",
-        text,
-    )
-    return text.encode("utf-8")
-
-
-def _response_headers(upstream: requests.Response, settings: dict) -> dict[str, str]:
-    headers = {}
-    for key, value in upstream.headers.items():
-        lower = key.lower()
-        if lower in _HOP_BY_HOP_HEADERS or lower in {
-            "content-length",
-            "content-encoding",
-            "x-frame-options",
-        }:
-            continue
-        if lower == "location":
-            if value.startswith(settings["origin"]):
-                value = value.replace(settings["origin"], _PROXY_PREFIX, 1)
-            elif value.startswith("/"):
-                value = f"{_PROXY_PREFIX}{value}"
-        headers[key] = value
-    return headers
+def agent_view(request):
+    return _dispatch(request)
 
 
 @csrf_exempt
-def opencode_proxy_view(request: HttpRequest, path: str | None = None):
-    config = _machine_config()
-    _require_enabled(config)
-    auth_response = _require_superuser(request)
-    if auth_response:
-        return auth_response
-    if not _origin_allowed(request, path):
-        return HttpResponseForbidden(
-            b"Cross-origin OpenCode agent requests are blocked.",
-        )
+def opencode_proxy_view(request, path=None):
+    return _dispatch(request, path=path or "")
 
-    settings = _settings(config)
-    route_config = request.__dict__.get("archivebox_config")
-    base_url, admin_url, api_url = _archivebox_route_urls(request, route_config)
-    settings["archivebox_base_url"] = base_url
-    settings["archivebox_admin_url"] = admin_url
-    settings["archivebox_api_url"] = api_url
-    ok, error = _ensure_opencode(settings)
-    if not ok:
-        return HttpResponse(
-            error.encode(),
-            status=502,
-            content_type="text/plain; charset=utf-8",
-        )
-
-    if request.method == "GET" and (path or "").endswith("/event"):
-        response = StreamingHttpResponse(
-            _event_chunks(request, settings, path),
-            content_type="text/event-stream",
-        )
-        response.headers["Cache-Control"] = "no-store"
-        response.headers["X-Accel-Buffering"] = "no"
-        return response
 
+def _websocket_context(scope):
+    close_old_connections()
     try:
-        method = request.method or "GET"
-        upstream = requests.request(
-            method,
-            _proxy_url(settings, path),
-            params=_request_params(request),
-            data=request.body if method not in {"GET", "HEAD"} else None,
-            headers=_request_headers(request, settings),
-            stream=True,
-            timeout=(settings["timeout"], None),
-            allow_redirects=False,
-        )
-    except requests.RequestException as err:
-        return HttpResponse(
-            str(err).encode(),
-            status=502,
-            content_type="text/plain; charset=utf-8",
+        match = resolve(scope["path"])
+        if match.url_name != "opencode-proxy":
+            raise PermissionDenied
+        request = ASGIRequest(
+            {**scope, "type": "http", "method": "GET", "scheme": "https" if scope.get("scheme") == "wss" else "http"},
+            BytesIO(),
         )
+        route_config = get_request_config(request, resolve_plugins=False)
+        request.archivebox_config = route_config
+        if not route_config.CONTROL_PLANE_ENABLED:
+            raise PermissionDenied
+        if (
+            route_config.USES_SUBDOMAIN_ROUTING
+            and route_config.BASE_URL
+            and not host_matches(request.get_host(), get_admin_host(config=route_config, request=request))
+        ):
+            raise PermissionDenied
+        SessionMiddleware(_dispatch).process_request(request)
+        AuthenticationMiddleware(_dispatch).process_request(request)
+        ReverseProxyAuthMiddleware(_dispatch).process_request(request)
+        config = route_config.model_dump(mode="json")
+        if not config.get("OPENCODE_ENABLED") or not request.user.is_active or not request.user.is_superuser:
+            raise PermissionDenied
+        runtime, settings = _runtime_settings(request, config)
+        if not request.headers.get("Origin") or not runtime._origin_allowed("POST", request.get_host(), request.headers):
+            raise PermissionDenied
+        return runtime, settings, match.kwargs.get("path", "")
+    finally:
+        close_old_connections()
 
-    content_type = upstream.headers.get("Content-Type", "")
-    is_event_stream = content_type.startswith("text/event-stream")
-    is_text = not is_event_stream and any(content_type.startswith(prefix) for prefix in _TEXT_CONTENT_TYPES)
-    headers = _response_headers(upstream, settings)
-    if is_text:
-        body = _rewrite_text(upstream.content, settings)
-        response = HttpResponse(
-            body,
-            status=upstream.status_code,
-            content_type=content_type or "text/plain; charset=utf-8",
-        )
-    elif is_event_stream:
-        response = StreamingHttpResponse(
-            upstream.iter_lines(chunk_size=1),
-            status=upstream.status_code,
-            content_type=content_type or "text/event-stream",
-        )
-    else:
-        response = StreamingHttpResponse(
-            upstream.iter_content(chunk_size=64 * 1024),
-            status=upstream.status_code,
-            content_type=content_type or "application/octet-stream",
-        )
-    for key, value in headers.items():
-        response.headers[key] = value
-    response.headers["Cache-Control"] = "no-store"
-    return response
+
+async def websocket_view(scope, receive, send):
+    if (await receive())["type"] != "websocket.connect":
+        return
+    try:
+        runtime, settings, path = await sync_to_async(_websocket_context)(scope)
+        await runtime.websocket_proxy(settings, path, scope.get("query_string", b""), scope.get("subprotocols", []), receive, send)
+    except (PermissionDenied, Resolver404, Http404):
+        await send({"type": "websocket.close", "code": 1008})
+    except Exception:
+        _LOGGER.exception("Optional AI WebSocket failed")
+        await send({"type": "websocket.close", "code": 1011})

archivebox/plugins/discovery.py

@@ -1,12 +1,12 @@
 __package__ = "archivebox.plugins"
 
-import json
 from collections.abc import Iterable
 from functools import lru_cache
 from pathlib import Path
 from typing import Any, Protocol, TypedDict
 
 from abx_plugins import get_plugins_dir
+from abx_dl.catalog import PluginCatalog, PluginConfigResolver
 from django.utils.safestring import mark_safe
 
 from archivebox.config.constants import CONSTANTS
@@ -29,18 +29,18 @@ USER_PLUGINS_DIR = CONSTANTS.USER_PLUGINS_DIR
 
 
 def iter_plugin_dirs() -> list[Path]:
-    """Iterate over all built-in and user plugin directories."""
-    plugin_dirs: list[Path] = []
+    """Return the exact plugin directories exposed by the shared catalog."""
+    return [plugin.path for plugin in get_plugin_catalog().values()]
 
-    for base_dir in (BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR):
-        if not base_dir.exists():
-            continue
 
-        for plugin_dir in base_dir.iterdir():
-            if plugin_dir.is_dir() and not plugin_dir.name.startswith("_"):
-                plugin_dirs.append(plugin_dir)
+@lru_cache(maxsize=1)
+def get_plugin_catalog() -> PluginCatalog:
+    return PluginCatalog.discover(extra_plugin_dirs=[USER_PLUGINS_DIR], runtime="archivebox")
 
-    return plugin_dirs
+
+@lru_cache(maxsize=1)
+def get_plugin_config_resolver() -> PluginConfigResolver:
+    return PluginConfigResolver(get_plugin_catalog())
 
 
 @lru_cache(maxsize=1)
@@ -52,25 +52,11 @@ def get_plugins() -> list[str]:
     or a standardized templates/icon.html asset. This includes non-extractor
     plugins such as binary providers and shared base plugins.
     """
-    plugins = []
-
-    for plugin_dir in iter_plugin_dirs():
-        has_hooks = any(plugin_dir.glob("on_*__*.*"))
-        has_config = (plugin_dir / "config.json").exists()
-        has_icon = (plugin_dir / "templates" / "icon.html").exists()
-        if has_hooks or has_config or has_icon:
-            plugins.append(plugin_dir.name)
-
-    return sorted(set(plugins))
+    return sorted(get_plugin_catalog())
 
 
 def get_plugin_models():
-    from abx_dl.models import discover_plugins
-
-    plugins = {}
-    for base_dir in (BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR):
-        plugins.update(discover_plugins(plugins_dir=base_dir, runtime="archivebox"))
-    return plugins
+    return get_plugin_catalog().plugins
 
 
 def get_plugin_name(plugin: str) -> str:
@@ -99,91 +85,17 @@ def get_enabled_plugins(config: ConfigLookup | None = None, **config_kwargs: Any
 
         config = get_config(**config_kwargs)
 
-    enabled = []
-    disabled = []
-    for plugin in get_plugins():
-        plugin_config = get_plugin_special_config(plugin, config)
-        if plugin_config["enabled"]:
-            enabled.append(plugin)
-        else:
-            disabled.append(plugin)
-
-    from abx_dl.models import filter_plugins
-
-    return list(filter_plugins(get_plugin_models(), enabled, include_providers=True, disabled_names=disabled))
+    return get_plugin_config_resolver().enabled_plugin_names_from_flat(dict(config.items()))
 
 
-def discover_plugins_that_provide_interface(
-    module_name: str,
-    required_attrs: list[str],
-    plugin_prefix: str | None = None,
-) -> dict[str, Any]:
-    """
-    Discover plugins that provide a specific Python module with required interface.
-
-    This enables dynamic plugin discovery for features like search backends,
-    storage backends, etc. without hardcoding imports.
-    """
-    import importlib.util
-
-    backends = {}
-
-    for base_dir in (BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR):
-        if not base_dir.exists():
-            continue
-
-        for plugin_dir in base_dir.iterdir():
-            if not plugin_dir.is_dir():
-                continue
-
-            plugin_name = plugin_dir.name
-            if plugin_prefix and not plugin_name.startswith(plugin_prefix):
-                continue
-
-            module_path = plugin_dir / f"{module_name}.py"
-            if not module_path.exists():
-                continue
-
-            try:
-                spec = importlib.util.spec_from_file_location(
-                    f"archivebox.dynamic_plugins.{plugin_name}.{module_name}",
-                    module_path,
-                )
-                if spec is None or spec.loader is None:
-                    continue
-
-                module = importlib.util.module_from_spec(spec)
-                spec.loader.exec_module(module)
-
-                if not all(attr in vars(module) for attr in required_attrs):
-                    continue
-
-                if plugin_prefix:
-                    backend_name = plugin_name[len(plugin_prefix) :]
-                else:
-                    backend_name = plugin_name
-
-                backends[backend_name] = module
-
-            except Exception:
-                continue
-
-    return backends
-
-
-def get_search_backends() -> dict[str, Any]:
-    """
-    Discover all available search backend plugins.
-
-    Search backends must provide a search.py module with:
-        - search(query: str) -> List[str]  (returns snapshot IDs)
-        - flush(snapshot_ids: Iterable[str]) -> None
-    """
-    return discover_plugins_that_provide_interface(
-        module_name="search",
-        required_attrs=["search", "flush"],
-        plugin_prefix="search_backend_",
-    )
+def get_search_backends():
+    """Return plugins that declare both standalone search commands."""
+    catalog = get_plugin_catalog()
+    return {
+        plugin.name.removeprefix("search_backend_"): plugin
+        for plugin in catalog.values()
+        if catalog.command(plugin.name, "search") is not None and catalog.command(plugin.name, "flush") is not None
+    }
 
 
 @lru_cache(maxsize=1)
@@ -196,33 +108,7 @@ def discover_plugin_configs() -> dict[str, dict[str, Any]]:
     schemas are plugin package metadata, not live user config; runtime values
     still come from env/db config at each callsite.
     """
-    configs = {}
-
-    for plugin_dir in iter_plugin_dirs():
-        config_path = plugin_dir / "config.json"
-        if not config_path.exists():
-            continue
-
-        try:
-            with open(config_path) as f:
-                schema = json.load(f)
-
-            if not isinstance(schema, dict):
-                continue
-            if schema.get("type") != "object":
-                continue
-            if "properties" not in schema:
-                continue
-
-            configs[plugin_dir.name] = schema
-
-        except (json.JSONDecodeError, OSError) as e:
-            import sys
-
-            print(f"Warning: Failed to load config.json from {plugin_dir.name}: {e}", file=sys.stderr)
-            continue
-
-    return configs
+    return get_plugin_config_resolver().schemas
 
 
 def get_plugin_special_config(plugin_name: str, config: ConfigLookup, _visited: set[str] | None = None) -> PluginSpecialConfig:
@@ -234,26 +120,7 @@ def get_plugin_special_config(plugin_name: str, config: ConfigLookup, _visited:
         - {PLUGIN}_TIMEOUT: Plugin-specific timeout (fallback to TIMEOUT, default 300)
         - {PLUGIN}_BINARY: Primary binary path (default to plugin_name)
     """
-    plugin_upper = plugin_name.upper()
-
-    enabled_key = f"{plugin_upper}_ENABLED"
-    enabled = config.get(enabled_key)
-    if enabled is None:
-        enabled = True
-    elif isinstance(enabled, str):
-        enabled = enabled.lower() not in ("false", "0", "no", "")
-
-    timeout_key = f"{plugin_upper}_TIMEOUT"
-    timeout = config.get(timeout_key) or config.get("TIMEOUT", 300)
-
-    binary_key = f"{plugin_upper}_BINARY"
-    binary = config.get(binary_key, plugin_name)
-
-    return {
-        "enabled": bool(enabled),
-        "timeout": int(timeout),
-        "binary": str(binary),
-    }
+    return get_plugin_config_resolver().runtime_settings(plugin_name, dict(config.items()))
 
 
 DEFAULT_TEMPLATES = {
@@ -295,11 +162,11 @@ def get_plugin_template(plugin: str, template_name: str, fallback: bool = True)
     if base_name in ("yt-dlp", "youtube-dl"):
         base_name = "ytdlp"
 
-    for plugin_dir in iter_plugin_dirs():
-        if plugin_dir.name == base_name or plugin_dir.name.endswith(f"_{base_name}"):
-            template_path = plugin_dir / "templates" / f"{template_name}.html"
-            if template_path.exists():
-                return template_path.read_text()
+    catalog = get_plugin_catalog()
+    if base_name in catalog:
+        template_path = catalog.template_path(base_name, template_name)
+        if template_path is not None:
+            return template_path.read_text()
 
     if fallback:
         return DEFAULT_TEMPLATES.get(template_name, "")

archivebox/plugins/forms.py

@@ -11,133 +11,19 @@ from django.utils.html import format_html
 
 from archivebox.config import CONSTANTS_CONFIG
 from archivebox.config.common import ArchiveBoxConfig, get_config
-from archivebox.plugins.discovery import discover_plugin_configs, get_plugin_icon, get_plugins
+from archivebox.plugins.discovery import discover_plugin_configs, get_plugin_catalog, get_plugin_icon, get_plugins
 
 
 PLUGIN_CONFIG_FIELD_PREFIX = "plugin_config__"
-PLUGIN_GROUP_DEFINITIONS = (
-    (
-        "main_plugins",
-        "Main",
-        "",
-        "",
-        "",
-        (
-            "dom",
-            "screenshot",
-            "pdf",
-            "singlefile",
-            "wget",
-            "archivedotorg",
-            "chrome_mhtml",
-            "archivewebpage",
-        ),
-    ),
-    (
-        "page_setup_plugins",
-        "Page Setup",
-        "",
-        "",
-        "",
-        (
-            "chrome",
-            "infiniscroll",
-            "modalcloser",
-            "ublock",
-            "istilldontcareaboutcookies",
-            "twocaptcha",
-            "claudechrome",
-        ),
-    ),
-    (
-        "media_plugins",
-        "Media",
-        "",
-        "",
-        "",
-        (
-            "staticfile",
-            "responses",
-            "chrome_screencast",
-            "ytdlp",
-            "gallerydl",
-            "git",
-        ),
-    ),
-    (
-        "text_plugins",
-        "Text",
-        "",
-        "",
-        "",
-        (
-            "readability",
-            "htmltotext",
-            "defuddle",
-            "forumdl",
-            "mercury",
-            "trafilatura",
-            "liteparse",
-            "opendataloader",
-            "papersdl",
-        ),
-    ),
-    (
-        "metadata_plugins",
-        "Metadata",
-        "",
-        "",
-        "",
-        (
-            "title",
-            "favicon",
-            "headers",
-            "redirects",
-            "accessibility",
-            "consolelog",
-            "sslcerts",
-            "dns",
-            "seo",
-            "hashes",
-        ),
-    ),
-    (
-        "postprocessing_plugins",
-        "Postprocessing",
-        "",
-        "",
-        "",
-        (
-            "parse_dom_outlinks",
-            "parse_html_urls",
-            "parse_jsonl_urls",
-            "parse_netscape_urls",
-            "parse_rss_urls",
-            "parse_txt_urls",
-            "claudecode",
-            "claudecodecleanup",
-            "claudecodeextract",
-        ),
-    ),
+PLUGIN_GROUPS = (
+    ("main", "main_plugins", "Main"),
+    ("page_setup", "page_setup_plugins", "Page Setup"),
+    ("media", "media_plugins", "Media"),
+    ("text", "text_plugins", "Text"),
+    ("metadata", "metadata_plugins", "Metadata"),
+    ("postprocessing", "postprocessing_plugins", "Postprocessing"),
+    ("other", "other_plugins", "Other"),
 )
-HIDDEN_PLUGIN_CONFIG_UI_PLUGINS = {
-    "apt",
-    "base",
-    "bash",
-    "brew",
-    "cargo",
-    "chromewebstore",
-    "env",
-    "media",
-    "npm",
-    "opencode",
-    "pip",
-    "puppeteer",
-    "search_backend_ripgrep",
-    "search_backend_sonic",
-    "search_backend_sqlite",
-    "ssl",
-}
 TIMEOUT_INPUT_PATTERN = r"(0|[1-9][0-9]*|[0-9]+(?:\.[0-9]+)?\s*(?:s|sec|secs|second|seconds|m|min|mins|minute|minutes|h|hr|hrs|hour|hours))"
 
 
@@ -255,26 +141,21 @@ class PluginConfigFormMixin:
     allow_crawl_execution_config_fields = True
 
     def build_plugin_groups(self, runtime_config: Mapping[str, Any] | None = None) -> None:
-        all_plugins = get_plugins()
+        catalog = get_plugin_catalog()
+        all_plugins = set(catalog)
         plugin_configs = discover_plugin_configs()
         runtime_config = runtime_config or get_config()
         self.plugin_config_binary_urls = get_plugin_config_binary_urls(runtime_config)
-        grouped_plugins = set().union(*(group[-1] for group in PLUGIN_GROUP_DEFINITIONS))
-        other_plugins = tuple(sorted(set(all_plugins) - grouped_plugins - HIDDEN_PLUGIN_CONFIG_UI_PLUGINS))
+        grouped_plugins = catalog.groups()
 
-        for field_name, *_rest, plugin_names in PLUGIN_GROUP_DEFINITIONS:
+        group_specs = []
+        for category, field_name, title in PLUGIN_GROUPS:
+            plugin_names = tuple(plugin.name for plugin in grouped_plugins.get(category, []))
+            group_specs.append((field_name, title, "", "", "", plugin_names))
             if field_name in self.fields:
                 get_choice_field(self, field_name).choices = [
                     (p, get_plugin_choice_label(p, plugin_configs)) for p in plugin_names if p in all_plugins
                 ]
-
-        if "other_plugins" in self.fields:
-            get_choice_field(self, "other_plugins").choices = [(p, get_plugin_choice_label(p, plugin_configs)) for p in other_plugins]
-
-        group_specs = (
-            *PLUGIN_GROUP_DEFINITIONS,
-            ("other_plugins", "Other", "", "", "", other_plugins),
-        )
         binary_url_lookup = _build_required_binary_url_lookup(plugin_configs, runtime_config)
         self.plugin_groups = [
             {

archivebox/plugins/hooks.py

@@ -1,146 +1,32 @@
+"""ArchiveBox adapters around the framework-free abx-dl plugin runtime.
+
+Discovery and execution are owned by abx-dl. ArchiveBox keeps only the small
+Django projection adapter and its application-specific URL-output reader.
 """
-Hook discovery and execution helpers for ArchiveBox plugins.
 
-ArchiveBox no longer drives plugin execution itself during normal crawls.
-`abx-dl` owns the live runtime and emits typed bus events; ArchiveBox mainly:
-
-- discovers hook files for inspection / docs / legacy direct execution helpers
-- executes individual hook scripts when explicitly requested
-- parses hook stdout JSONL records into ArchiveBox models when needed
-
-Hook-backed event families are discovered from filenames like:
-    on_CrawlSetup__*
-    on_Snapshot__*
-
-Internal bus event names are normalized to the corresponding
-`on_{EventFamily}__*` prefix by a simple string transform. If no scripts exist
-for that prefix, discovery returns `[]`.
-
-Directory structure:
-    abx_plugins/plugins/<plugin_name>/on_<Event>__<hook_name>.<ext>     (built-in package)
-    data/custom_plugins/<plugin_name>/on_<Event>__<hook_name>.<ext>     (user)
-
-Hook contract:
-    Input:  --url=<url> (and other --key=value args)
-    Output: JSONL records to stdout, files to $PWD
-    Exit:   0 = success, non-zero = failure
-
-Execution order:
-    - Hooks are named with two-digit prefixes (00-99) and sorted lexicographically by filename
-    - Foreground hooks run sequentially in that order
-    - Background hooks (.bg suffix) run concurrently and do not block foreground progress
-    - After all foreground hooks complete, background hooks receive SIGTERM and must finalize
-
-Hook naming convention:
-    on_{EventFamily}__{run_order}_{description}[.bg].{ext}
-
-API:
-    discover_hooks(event)     -> List[Path]     Find hook scripts for a hook-backed event family
-    run_hook(script, ...)     -> Process        Execute a hook script directly
-    is_background_hook(name)  -> bool           Check if hook is background (.bg suffix)
-"""
+from __future__ import annotations
 
 __package__ = "archivebox.plugins"
 
-import json
-import os
-from collections.abc import Mapping
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Optional, Protocol, TypeGuard, runtime_checkable
+from typing import Any
+
+from abx_dl.models import parse_hook_filename
 
-from archivebox.config.constants import CONSTANTS
-from archivebox.config.version import VERSION
 from archivebox.misc.util import fix_url_from_markdown, sanitize_extracted_url
-from archivebox.plugins.discovery import (
-    BUILTIN_PLUGINS_DIR,
-    USER_PLUGINS_DIR,
-    ConfigLookup,
-    get_enabled_plugins,
-    get_plugin_special_config,
-)
-
-if TYPE_CHECKING:
-    from archivebox.machine.models import Process
-
-
-@runtime_checkable
-class ConfigDump(Protocol):
-    def as_dict(self) -> dict[str, Any]: ...
-
-
-def _has_config_dump(config: object) -> TypeGuard[ConfigDump]:
-    return isinstance(config, ConfigDump)
-
-
-def _config_to_overrides(config: ConfigLookup | Mapping[str, Any] | None) -> dict[str, Any]:
-    if config is None:
-        return {}
-    if _has_config_dump(config):
-        return dict(config.as_dict())
-    return dict(config.items())
-
-
-# =============================================================================
-# Hook Step Extraction
-# =============================================================================
+from archivebox.plugins.discovery import ConfigLookup, get_enabled_plugins, get_plugin_catalog
 
 
 def is_background_hook(hook_name: str) -> bool:
-    """
-    Check if a hook is a background hook (doesn't block foreground progression).
-
-    Background hooks have '.bg.' in their filename before the extension.
-
-    Args:
-        hook_name: Hook filename (e.g., 'on_Snapshot__10_chrome_tab.daemon.bg.js')
-
-    Returns:
-        True if background hook, False if foreground.
-
-    Examples:
-        is_background_hook('on_Snapshot__10_chrome_tab.daemon.bg.js') -> True
-        is_background_hook('on_Snapshot__50_wget.py') -> False
-        is_background_hook('on_Snapshot__63_media.finite.bg.py') -> True
-    """
-    return ".bg." in hook_name or "__background" in hook_name
+    parsed = parse_hook_filename(Path(hook_name).name)
+    return bool(parsed and parsed[2])
 
 
 def normalize_hook_event_name(event_name: str) -> str | None:
-    """
-    Normalize a hook event family or event class name to its on_* prefix.
-
-    Examples:
-        CrawlSetupEvent -> CrawlSetup
-        SnapshotEvent -> Snapshot
-        BinaryEvent -> Binary
-        CrawlCleanupEvent -> CrawlCleanup
-    """
     normalized = str(event_name or "").strip()
     if not normalized:
         return None
-
-    if normalized.endswith("Event"):
-        return normalized[:-5] or None
-    return normalized
-
-
-def _model_output_dir_from_child_path(path: Path, marker: str) -> Path | None:
-    """
-    Infer the model output dir from a model dir or one of its plugin subdirs.
-
-    Current ArchiveBox snapshot/crawl dirs are:
-        .../{snapshots,crawls}/YYYYMMDD/domain/uuid[/plugin]
-    """
-    parts = path.resolve().parts
-    try:
-        marker_index = parts.index(marker)
-    except ValueError:
-        return None
-
-    model_end_index = marker_index + 4
-    if len(parts) < model_end_index:
-        return None
-    return Path(*parts[:model_end_index])
+    return normalized.removesuffix("Event") or None
 
 
 def discover_hooks(
@@ -149,481 +35,40 @@ def discover_hooks(
     config: ConfigLookup | None = None,
     **config_kwargs: Any,
 ) -> list[Path]:
-    """
-    Find all hook scripts for an event family.
-
-    Searches both built-in and user plugin directories.
-    Filters out hooks from disabled plugins by default (respects USE_/SAVE_ flags).
-    Returns scripts sorted alphabetically by filename for deterministic execution order.
-
-    Hook naming convention uses numeric prefixes to control order:
-        on_Snapshot__10_title.py        # runs first
-        on_Snapshot__15_singlefile.py   # runs second
-        on_Snapshot__26_readability.py  # runs later (depends on singlefile)
-
-    Args:
-        event_name: Hook event family or event class name.
-            Examples: 'CrawlSetupEvent', 'Snapshot'.
-            Event names are normalized by stripping a trailing `Event`.
-            If no matching `on_{EventFamily}__*` scripts exist, returns [].
-        filter_disabled: If True, skip hooks from disabled plugins (default: True)
-        config: Optional pre-merged config dict from get_config().
-        **config_kwargs: Scope/override args forwarded to get_config() when config is not supplied.
-
-    Returns:
-        Sorted list of hook script paths from enabled plugins only.
-
-    Examples:
-        # With proper config context (recommended):
-        from archivebox.config.common import get_config
-        config = get_config(crawl=my_crawl, snapshot=my_snapshot)
-        discover_hooks('Snapshot', config=config)
-        # Returns: [Path('.../on_Snapshot__10_title.py'), ...] (wget excluded if SAVE_WGET=False)
-
-        # Without config (uses global defaults):
-        discover_hooks('Snapshot')
-        # Returns: [Path('.../on_Snapshot__10_title.py'), ...]
-
-        # Show all plugins regardless of enabled status:
-        discover_hooks('Snapshot', filter_disabled=False)
-        # Returns: [Path('.../on_Snapshot__10_title.py'), ..., Path('.../on_Snapshot__50_wget.py')]
-    """
-    hook_event_name = normalize_hook_event_name(event_name)
-    if not hook_event_name:
+    """Return the exact catalog hooks used by abx-dl, in execution order."""
+    normalized = normalize_hook_event_name(event_name)
+    if not normalized or normalized == "BinaryRequest":
         return []
-    if hook_event_name == "BinaryRequest":
-        return []
-
-    hooks = []
-
-    for base_dir in (BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR):
-        if not base_dir.exists():
-            continue
-
-        # Search for hook scripts in all subdirectories
-        for ext in ("sh", "py", "js"):
-            pattern = f"*/on_{hook_event_name}__*.{ext}"
-            hooks.extend(base_dir.glob(pattern))
-
-            # Also check for hooks directly in the plugins directory
-            pattern_direct = f"on_{hook_event_name}__*.{ext}"
-            hooks.extend(base_dir.glob(pattern_direct))
-
+    names = None
     if filter_disabled:
-        # Get merged config if not provided (lazy import to avoid circular dependency)
         if config is None:
             from archivebox.config.common import get_config
 
             config = get_config(**config_kwargs)
-
-        enabled_plugins = set(get_enabled_plugins(config=config))
-        enabled_hooks = []
-
-        for hook in hooks:
-            # Get plugin name from parent directory
-            # e.g., abx_plugins/plugins/wget/on_Snapshot__50_wget.py -> 'wget'
-            plugin_name = hook.parent.name
-
-            # Check if this is a plugin directory (not the root plugins dir)
-            if hook.parent.resolve() in (BUILTIN_PLUGINS_DIR.resolve(), USER_PLUGINS_DIR.resolve()):
-                # Hook is in root plugins directory, not a plugin subdir
-                # Include it by default (no filtering for non-plugin hooks)
-                enabled_hooks.append(hook)
-                continue
-
-            if plugin_name in enabled_plugins:
-                enabled_hooks.append(hook)
-
-        hooks = enabled_hooks
-
-    # Sort by filename (not full path) to ensure numeric prefix ordering works
-    # e.g., on_Snapshot__10_title.py sorts before on_Snapshot__26_readability.py
-    return sorted(set(hooks), key=lambda p: p.name)
-
-
-def run_hook(
-    script: Path,
-    output_dir: Path,
-    config: ConfigLookup | Mapping[str, Any] | None = None,
-    timeout: int | None = None,
-    parent: Optional["Process"] = None,
-    **kwargs: Any,
-) -> "Process":
-    """
-    Execute a hook script with the given arguments using Process model.
-
-    This is the low-level hook executor that creates a Process record and
-    uses Process.launch() for subprocess management.
-
-    Config is passed to hooks via environment variables. Crawl/snapshot callers
-    should pass the runtime config produced by for_crawl_runtime().
-
-    Args:
-        script: Path to the hook script (.sh, .py, or .js)
-        output_dir: Working directory for the script (where output files go)
-        config: Optional runtime config dict from for_crawl_runtime().
-                If omitted, pass scope/override args using kwargs prefixed with config_.
-        timeout: Maximum execution time in seconds
-                 If None, auto-detects from PLUGINNAME_TIMEOUT config (fallback to TIMEOUT, default 300)
-        parent: Optional parent Process (for tracking worker->hook hierarchy)
-        **kwargs: Arguments passed to the script as --key=value
-
-    Returns:
-        Process model instance (use process.exit_code, process.stdout, process.get_records())
-
-    Example:
-        from archivebox.config.common import get_config
-        config = get_config(crawl=my_crawl, snapshot=my_snapshot).for_crawl_runtime(crawl=my_crawl, snapshot=my_snapshot)
-        process = run_hook(hook_path, output_dir, config=config, url=url, snapshot_id=id)
-        if process.status == 'exited':
-            records = process.get_records()  # Get parsed JSONL output
-    """
-    from archivebox.machine.models import Process, Machine, NetworkInterface
-    from archivebox.config.common import (
-        ArchiveBoxConfig,
-        _archivebox_config_input_names,
-        get_config,
-        normalize_runtime_config,
-        _plugin_enabled_config_keys,
-    )
-
-    config_scope = {key.removeprefix("config_"): kwargs.pop(key) for key in list(kwargs) if key.startswith("config_")}
-    config_overrides = _config_to_overrides(config)
-    explicit_override_keys = set(config_overrides)
-    resolved_config = get_config(overrides=config_overrides, **config_scope)
-    hook_config = normalize_runtime_config(
-        resolved_config.for_crawl_runtime(runtime_overrides=config_overrides),
-        json_safe=False,
-    )
-    hook_config.update(normalize_runtime_config(config_overrides, json_safe=False))
-    plugin_enabled_keys = set(_plugin_enabled_config_keys().values())
-    if plugin_enabled_keys.intersection(hook_config):
-        for enabled_key in plugin_enabled_keys:
-            hook_config.setdefault(enabled_key, False)
-
-    # Auto-detect timeout from plugin config if not explicitly provided
-    if timeout is None:
-        plugin_name = script.parent.name
-        plugin_config = get_plugin_special_config(plugin_name, resolved_config)
-        timeout = plugin_config["timeout"]
-    if timeout:
-        timeout = min(int(timeout), int(CONSTANTS.MAX_HOOK_RUNTIME_SECONDS))
-
-    # Get current machine
-    machine = Machine.current()
-    iface = NetworkInterface.current(refresh=True)
-    machine = iface.machine
-
-    # Auto-detect parent process if not explicitly provided
-    # This enables automatic hierarchy tracking: Worker -> Hook
-    if parent is None:
-        try:
-            parent = Process.current()
-        except Exception:
-            # If Process.current() fails (e.g., not in a worker context), leave parent as None
-            pass
-
-    if not script.is_file():
-        raise FileNotFoundError(f"Hook script not found: {script}")
-
-    # Hooks are opaque executables. Their shipped abxpkg shebang owns runtime
-    # and dependency resolution just as it does under abx-dl.
-    cmd = [str(script)]
-
-    # Build CLI arguments from kwargs
-    for key, value in kwargs.items():
-        # Skip keys that start with underscore (internal parameters)
-        if key.startswith("_"):
-            continue
-
-        arg_key = f"--{key.replace('_', '-')}"
-        if isinstance(value, bool):
-            if value:
-                cmd.append(arg_key)
-        elif value is not None and value != "":
-            # JSON-encode complex values, use str for simple ones
-            # Skip empty strings to avoid --key= which breaks argument parsers
-            if isinstance(value, (dict, list)):
-                cmd.append(f"{arg_key}={json.dumps(value)}")
-            else:
-                # Ensure value is converted to string and strip whitespace
-                str_value = str(value).strip()
-                if str_value:  # Only add if non-empty after stripping
-                    cmd.append(f"{arg_key}={str_value}")
-
-    # Set up environment with base paths
-    env = os.environ.copy()
-    archivebox_config_input_names = _archivebox_config_input_names()
-    for key in archivebox_config_input_names:
-        env.pop(key, None)
-    env.pop("PLUGINS", None)
-    env["DATA_DIR"] = str(CONSTANTS.DATA_DIR)
-    env["LIBRARY_VERSION"] = VERSION
-    env.setdefault("MACHINE_ID", os.environ.get("MACHINE_ID", CONSTANTS.MACHINE_ID))
-    snap_dir = hook_config.get("SNAP_DIR") or _model_output_dir_from_child_path(output_dir, CONSTANTS.SNAPSHOTS_DIR_NAME)
-    crawl_dir = hook_config.get("CRAWL_DIR") or _model_output_dir_from_child_path(output_dir, CONSTANTS.CRAWLS_DIR_NAME)
-    if snap_dir:
-        env["SNAP_DIR"] = str(snap_dir)
-    if crawl_dir:
-        env["CRAWL_DIR"] = str(crawl_dir)
-
-    # Export the runtime library root; abx-dl/abxpkg own executable lookup env.
-    lib_dir = hook_config.get("ABXPKG_LIB_DIR")
-    if lib_dir:
-        env["ABXPKG_LIB_DIR"] = str(lib_dir)
-
-    # Set Node.js module resolution paths.
-    # NODE_PATH may be a path list, but NODE_MODULES_DIR is a single canonical directory.
-    node_modules_dir = hook_config.get("NODE_MODULES_DIR")
-    if lib_dir and "ABXPKG_LIB_DIR" in explicit_override_keys and "NODE_MODULES_DIR" not in explicit_override_keys:
-        node_modules_dir = Path(lib_dir) / "pnpm" / "packages" / "chrome" / "node_modules"
-    elif not node_modules_dir and lib_dir:
-        node_modules_dir = Path(lib_dir) / "pnpm" / "packages" / "chrome" / "node_modules"
-
-    node_path_parts = [part for part in str(hook_config.get("NODE_PATH") or "").split(os.pathsep) if part]
-    if node_modules_dir:
-        node_modules_dir = Path(node_modules_dir)
-        node_modules_dir.mkdir(parents=True, exist_ok=True)
-        node_modules_dir_str = str(node_modules_dir)
-        env["NODE_MODULES_DIR"] = node_modules_dir_str
-        env["NODE_MODULE_DIR"] = node_modules_dir_str
-        if node_modules_dir_str not in node_path_parts:
-            node_path_parts.append(node_modules_dir_str)
-    if node_path_parts:
-        env["NODE_PATH"] = os.pathsep.join(node_path_parts)
-
-    # Export all config values to environment (already merged by get_config())
-    # Skip keys we've already handled specially above (PATH, ABXPKG_LIB_DIR, NODE_PATH, etc.)
-    SKIP_KEYS = {
-        "PATH",
-        "ABXPKG_LIB_DIR",
-        "NODE_PATH",
-        "NODE_MODULES_DIR",
-        "NODE_MODULE_DIR",
-        "DATA_DIR",
-        "MACHINE_ID",
-        "SNAP_DIR",
-        "CRAWL_DIR",
-    }
-    canonical_config_keys = set(ArchiveBoxConfig.model_fields)
-    for key, value in hook_config.items():
-        if key in SKIP_KEYS:
-            continue  # Already handled specially above, don't overwrite
-        if key in archivebox_config_input_names and key not in canonical_config_keys:
-            continue
-        if value is None:
-            continue
-        elif isinstance(value, bool):
-            env[key] = "true" if value else "false"
-        elif isinstance(value, (list, dict)):
-            env[key] = json.dumps(value)
-        else:
-            env[key] = str(value)
-
-    # Create output directory if needed
-    output_dir.mkdir(parents=True, exist_ok=True)
-
-    # Detect if this is a background hook.
-    # Background hooks use the .bg. filename marker.
-    # Old convention: __background in stem (for backwards compatibility)
-    is_background = ".bg." in script.name or "__background" in script.stem
-
-    try:
-        # Create Process record
-        process = Process.objects.create(
-            machine=machine,
-            iface=iface,
-            parent=parent,
-            process_type=Process.TypeChoices.HOOK,
-            pwd=str(output_dir),
-            cmd=cmd,
-            timeout=timeout,
-        )
-
-        # Copy the env dict we already built (includes os.environ + all customizations)
-        process.env = env.copy()
-        process.hydrate_binary_from_context(plugin_name=script.parent.name, hook_path=str(script))
-
-        # Save env before launching
-        process.save()
-
-        # Launch subprocess using Process.launch()
-        process.launch(background=is_background)
-
-        # Return Process object (caller can use process.exit_code, process.stdout, process.get_records())
-        return process
-
-    except Exception as e:
-        # Create a failed Process record for exceptions
-        process = Process.objects.create(
-            machine=machine,
-            iface=iface,
-            process_type=Process.TypeChoices.HOOK,
-            pwd=str(output_dir),
-            cmd=cmd,
-            timeout=timeout,
-            status=Process.StatusChoices.EXITED,
-            exit_code=1,
-            stderr=f"Failed to run hook: {type(e).__name__}: {e}",
-        )
-        return process
-
-
-def extract_records_from_process(process: "Process") -> list[dict[str, Any]]:
-    """
-    Extract JSONL records from a Process's stdout.
-
-    Adds plugin metadata to each record.
-
-    Args:
-        process: Process model instance with stdout captured
-
-    Returns:
-        List of parsed JSONL records with plugin metadata
-    """
-    records = process.get_records()
-    if not records:
-        return []
-
-    # Extract plugin metadata from process.pwd and the shipped hook path in cmd.
-    # Python hooks execute directly through their shebang, while JS and shell
-    # hooks have an interpreter in cmd[0].
-    plugin_name = Path(process.pwd).name if process.pwd else "unknown"
-    plugin_hook = next((str(arg) for arg in process.cmd if Path(str(arg)).name.startswith("on_")), "")
-    hook_name = Path(plugin_hook).name if plugin_hook else "unknown"
-
-    for record in records:
-        # Add plugin metadata to record
-        record.setdefault("plugin", plugin_name)
-        record.setdefault("hook_name", hook_name)
-        record.setdefault("plugin_hook", plugin_hook)
-
-    return records
+        names = get_enabled_plugins(config=config)
+    return [hook.path for _plugin, hook in get_plugin_catalog().hooks(normalized, names=names)]
 
 
 def collect_urls_from_plugins(snapshot_dir: Path) -> list[dict[str, Any]]:
-    """
-    Collect all urls.jsonl entries from parser plugin output subdirectories.
-
-    Each parser plugin outputs urls.jsonl to its own subdir:
-        snapshot_dir/parse_rss_urls/urls.jsonl
-        snapshot_dir/parse_html_urls/urls.jsonl
-        etc.
-
-    This is not special handling - urls.jsonl is just a normal output file.
-    This utility collects them all for the crawl system.
-    """
-    urls = []
-
-    # Look in each immediate subdirectory for urls.jsonl
+    """Read the durable urls.jsonl interface emitted by parser plugins."""
+    urls: list[dict[str, Any]] = []
     if not snapshot_dir.exists():
         return urls
 
+    from archivebox.machine.models import Process
+
     for subdir in snapshot_dir.iterdir():
-        if not subdir.is_dir():
-            continue
-
         urls_file = subdir / "urls.jsonl"
-        if not urls_file.exists():
+        if not subdir.is_dir() or not urls_file.is_file():
             continue
-
         try:
-            from archivebox.machine.models import Process
-
-            text = urls_file.read_text()
-            for entry in Process.parse_records_from_text(text):
-                if entry.get("url"):
-                    entry["url"] = sanitize_extracted_url(fix_url_from_markdown(str(entry["url"]).strip()))
-                    if not entry["url"]:
-                        continue
-                    # Track which parser plugin found this URL
+            for entry in Process.parse_records_from_text(urls_file.read_text()):
+                if not entry.get("url"):
+                    continue
+                entry["url"] = sanitize_extracted_url(fix_url_from_markdown(str(entry["url"]).strip()))
+                if entry["url"]:
                     entry["plugin"] = subdir.name
                     urls.append(entry)
-        except Exception:
-            pass
-
+        except (OSError, UnicodeError):
+            continue
     return urls
-
-
-# =============================================================================
-# Hook Result Processing Helpers
-# =============================================================================
-
-
-def process_hook_records(records: list[dict[str, Any]], overrides: dict[str, Any] | None = None) -> dict[str, int]:
-    """
-    Process JSONL records emitted by hook stdout.
-
-    This handles hook-emitted record types such as Snapshot, Tag, and Binary.
-    It does not process internal bus lifecycle events, since those
-    are not emitted as JSONL records by hook subprocesses.
-
-    Args:
-        records: List of JSONL record dicts from result['records']
-        overrides: Dict with 'snapshot', 'crawl', 'dependency', 'created_by_id', etc.
-
-    Returns:
-        Dict with counts by record type
-    """
-    stats = {}
-    overrides = overrides or {}
-
-    for record in records:
-        record_type = record.get("type")
-        if not record_type:
-            continue
-
-        # Skip ArchiveResult records (they update the calling ArchiveResult, not create new ones)
-        if record_type == "ArchiveResult":
-            continue
-
-        try:
-            # Dispatch to appropriate model's from_json() method
-            if record_type == "Snapshot":
-                from archivebox.core.models import Snapshot
-
-                if record.get("url"):
-                    record = {
-                        **record,
-                        "url": sanitize_extracted_url(fix_url_from_markdown(str(record["url"]).strip())),
-                    }
-                    if not record["url"]:
-                        continue
-
-                # Check if discovered snapshot exceeds crawl max_depth
-                snapshot_depth = record.get("depth", 0)
-                crawl = overrides.get("crawl")
-                if crawl and snapshot_depth > crawl.max_depth:
-                    # Skip - this URL was discovered but exceeds max crawl depth
-                    continue
-
-                obj = Snapshot.from_json(record.copy(), overrides)
-                if obj:
-                    stats["Snapshot"] = stats.get("Snapshot", 0) + 1
-
-            elif record_type == "Tag":
-                from archivebox.core.models import Tag
-
-                obj = Tag.from_json(record.copy(), overrides)
-                if obj:
-                    stats["Tag"] = stats.get("Tag", 0) + 1
-
-            elif record_type == "Binary":
-                from archivebox.machine.models import Binary
-
-                obj = Binary.from_json(record.copy(), overrides)
-                if obj:
-                    stats[record_type] = stats.get(record_type, 0) + 1
-
-            else:
-                import sys
-
-                print(f"Warning: Unknown record type '{record_type}' from hook output", file=sys.stderr)
-
-        except Exception as e:
-            import sys
-
-            print(f"Warning: Failed to create {record_type}: {e}", file=sys.stderr)
-            continue
-
-    return stats

archivebox/progressmonitor/views.py

@@ -30,12 +30,16 @@ def progress_endpoint(scope: Literal["crawl", "snapshot"] | None = None, object_
 @lru_cache(maxsize=1)
 def _live_progress_plugin_names() -> tuple[frozenset[str], frozenset[str]]:
     plugin_configs = discover_plugin_configs()
+    indexing_plugin_names = frozenset(
+        plugin_name
+        for plugin_name, plugin_config in plugin_configs.items()
+        if {"search", "flush"}.issubset(plugin_config.get("commands", {}))
+    )
     download_plugin_names = frozenset(
         plugin_name
         for plugin_name, plugin_config in plugin_configs.items()
-        if plugin_config.get("output_mimetypes") and not plugin_name.startswith("search_backend_")
+        if plugin_config.get("output_mimetypes") and plugin_name not in indexing_plugin_names
     )
-    indexing_plugin_names = frozenset(plugin_name for plugin_name in plugin_configs if plugin_name.startswith("search_backend_"))
     return download_plugin_names, indexing_plugin_names
 
 

archivebox/search/backends.py

@@ -1,7 +1,7 @@
 __package__ = "archivebox.search"
 
+import json
 import os
-from contextlib import contextmanager
 from typing import Any
 
 from archivebox.config.common import get_config
@@ -10,29 +10,21 @@ from archivebox.config.common import get_config
 _search_backends_cache: dict | None = None
 
 
-@contextmanager
-def search_backend_env(config: dict[str, Any] | None = None, **config_kwargs: Any):
-    """Temporarily expose resolved search config through os.environ for backend code."""
+def search_backend_command_env(config: dict[str, Any] | None = None, **config_kwargs: Any) -> dict[str, str]:
+    """Serialize resolved application config for a standalone plugin command."""
     config = config or get_config(**config_kwargs)
-    updates = {}
+    env = os.environ.copy()
     for key, value in config.items():
         key = str(key)
-        if not (key.startswith("SEARCH_BACKEND_") or key.endswith("_BINARY")):
-            continue
         if value is None:
             continue
-        if isinstance(value, (str, int, float, bool, os.PathLike)):
-            updates[key] = str(value)
-    previous = {key: os.environ.get(key) for key in updates}
-    os.environ.update(updates)
-    try:
-        yield
-    finally:
-        for key, value in previous.items():
-            if value is None:
-                os.environ.pop(key, None)
-            else:
-                os.environ[key] = value
+        if isinstance(value, bool):
+            env[key] = "true" if value else "false"
+        elif isinstance(value, (dict, list, tuple)):
+            env[key] = json.dumps(value)
+        elif isinstance(value, (str, int, float, os.PathLike)):
+            env[key] = str(value)
+    return env
 
 
 def normalize_search_backend_name(backend_name: str | None) -> str:
@@ -41,7 +33,7 @@ def normalize_search_backend_name(backend_name: str | None) -> str:
 
 
 def get_available_backends() -> dict:
-    """Discover search backend plugin modules and cache them in memory."""
+    """Discover search-capable plugins and cache their catalog entries."""
     global _search_backends_cache
 
     if _search_backends_cache is None:
@@ -53,7 +45,7 @@ def get_available_backends() -> dict:
 
 
 def get_backend(config: dict[str, Any] | None = None, **config_kwargs: Any) -> Any:
-    """Resolve the configured search backend module."""
+    """Resolve the configured search-capable plugin."""
     config = config or get_config(**config_kwargs)
     backend_name = normalize_search_backend_name(config.SEARCH_BACKEND_ENGINE)
     backends = get_available_backends()

archivebox/search/query.py

@@ -8,7 +8,7 @@ from django.db.models import Case, IntegerField, Q, QuerySet, Value, When
 from archivebox.config.common import get_config
 from archivebox.misc.logging import stderr
 from archivebox.misc.util import enforce_types
-from archivebox.search.backends import get_available_backends, get_backend, normalize_search_backend_name, search_backend_env
+from archivebox.search.backends import get_available_backends, get_backend, normalize_search_backend_name, search_backend_command_env
 from archivebox.search.config import get_search_mode, get_search_mode_backend, get_search_mode_base
 
 
@@ -278,18 +278,27 @@ def iter_query_search_ids(
                     # --csv/--json. Keep daemon diagnostics on stderr.
                     with redirect_stdout(sys.stderr):
                         ensure_daemon_stack(reason="search query")
-                with search_backend_env(config=config):
-                    if backend_name == "ripgrep":
-                        ids = backend.iter_search(query, search_mode=search_mode_base)
-                    else:
-                        ids = backend.search(query)
-                    for snapshot_id in ids:
-                        if snapshot_id in seen:
-                            continue
-                        seen.add(snapshot_id)
-                        yield snapshot_id
-                        if max_results and len(seen) >= max_results:
-                            return
+                from abx_dl.execution import iter_plugin_command
+                from archivebox.config.constants import CONSTANTS
+                from archivebox.plugins.discovery import get_plugin_catalog
+
+                command = get_plugin_catalog().command(backend.name, "search")
+                if command is None:
+                    raise RuntimeError(f'Plugin "{backend.name}" does not expose a search command')
+                ids = iter_plugin_command(
+                    command,
+                    arguments={"query": query, "search_mode": search_mode_base},
+                    env=search_backend_command_env(config=config),
+                    cwd=CONSTANTS.DATA_DIR,
+                    timeout=max(1, int(config.get("TIMEOUT", 60))) * 4,
+                )
+                for snapshot_id in ids:
+                    if snapshot_id in seen:
+                        continue
+                    seen.add(snapshot_id)
+                    yield snapshot_id
+                    if max_results and len(seen) >= max_results:
+                        return
                 successful_backends += 1
             except Exception as err:
                 errors.append(err)
@@ -318,8 +327,22 @@ def flush_search_index(snapshots: QuerySet, config: dict[str, Any] | None = None
     snapshot_pks = [str(pk) for pk in snapshots.values_list("pk", flat=True)]
 
     try:
-        with search_backend_env(config=config):
-            backend.flush(snapshot_pks)
+        from abx_dl.execution import iter_plugin_command
+        from archivebox.config.constants import CONSTANTS
+        from archivebox.plugins.discovery import get_plugin_catalog
+
+        command = get_plugin_catalog().command(backend.name, "flush")
+        if command is None:
+            raise RuntimeError(f'Plugin "{backend.name}" does not expose a flush command')
+        list(
+            iter_plugin_command(
+                command,
+                stdin=snapshot_pks,
+                env=search_backend_command_env(config=config),
+                cwd=CONSTANTS.DATA_DIR,
+                timeout=max(1, int(config.get("TIMEOUT", 60))) * 4,
+            ),
+        )
     except Exception as err:
         stderr()
         stderr(

archivebox/services/archive_result_service.py

@@ -2,24 +2,19 @@ from __future__ import annotations
 
 import asyncio
 import inspect
-import json
 import os
-import signal
 import sys
 import time
-from collections import defaultdict
-from collections.abc import Iterable
 from contextlib import contextmanager
 from functools import wraps
 from pathlib import Path
-from typing import Any, Protocol, runtime_checkable
+from typing import Any
 
 from asgiref.sync import sync_to_async
-from django.db import IntegrityError
 from django.utils import timezone
 
-from abx_dl.events import PROCESS_EXIT_SKIPPED, ArchiveResultEvent, ProcessCompletedEvent, ProcessStartedEvent, SnapshotEvent
-from abx_dl.output_files import guess_mimetype
+from abx_dl.events import ArchiveResultEvent, ProcessStartedEvent
+from abx_dl.output_files import OutputManifest
 from abx_dl.services.base import BaseService
 
 from .process_service import parse_event_datetime
@@ -71,129 +66,15 @@ def _perf_span(label: str):
         print(f"PERF_TRACE label={label} ms={elapsed_ms:.3f}", file=sys.stderr, flush=True)
 
 
-@runtime_checkable
-class ModelDumpable(Protocol):
-    def model_dump(self) -> dict[str, Any]: ...
-
-
-def _collect_output_metadata(plugin_dir: Path) -> tuple[dict[str, dict], int, str]:
-    exclude_names = {"stdout.log", "stderr.log", "process.pid", "hook.pid", "listener.pid"}
-    output_files: dict[str, dict] = {}
-    mime_sizes: dict[str, int] = defaultdict(int)
-    total_size = 0
-
-    if not plugin_dir.exists():
-        return output_files, total_size, ""
-
-    for file_path in plugin_dir.rglob("*"):
-        if not file_path.is_file():
-            continue
-        if ".hooks" in file_path.parts:
-            continue
-        if file_path.name in exclude_names:
-            continue
-        try:
-            stat = file_path.stat()
-        except OSError:
-            continue
-        mime_type = guess_mimetype(file_path) or "application/octet-stream"
-        relative_path = str(file_path.relative_to(plugin_dir))
-        output_files[relative_path] = {
-            "extension": file_path.suffix.lower().lstrip("."),
-            "mimetype": mime_type,
-            "size": stat.st_size,
-        }
-        mime_sizes[mime_type] += stat.st_size
-        total_size += stat.st_size
-
-    output_mimetypes = ",".join(mime for mime, _size in sorted(mime_sizes.items(), key=lambda item: item[1], reverse=True))
-    return output_files, total_size, output_mimetypes
-
-
-def _coerce_output_file_size(value: Any) -> int:
-    try:
-        return max(int(value or 0), 0)
-    except (TypeError, ValueError):
-        return 0
-
-
-def _normalize_output_files(raw_output_files: Any) -> dict[str, dict]:
-    def _enrich_metadata(path: str, metadata: dict[str, Any]) -> dict[str, Any]:
-        normalized = dict(metadata)
-        if "extension" not in normalized:
-            normalized["extension"] = Path(path).suffix.lower().lstrip(".")
-        if "mimetype" not in normalized:
-            guessed = guess_mimetype(path)
-            if guessed:
-                normalized["mimetype"] = guessed
-        return normalized
-
-    if raw_output_files is None:
-        return {}
-
-    if isinstance(raw_output_files, str):
-        try:
-            raw_output_files = json.loads(raw_output_files)
-        except json.JSONDecodeError:
-            return {}
-
-    if isinstance(raw_output_files, dict):
-        normalized: dict[str, dict] = {}
-        for path, metadata in raw_output_files.items():
-            if not path:
-                continue
-            metadata_dict = dict(metadata) if isinstance(metadata, dict) else {}
-            metadata_dict.pop("path", None)
-            normalized[str(path)] = _enrich_metadata(str(path), metadata_dict)
-        return normalized
-
-    if not isinstance(raw_output_files, Iterable):
-        return {}
-
-    normalized: dict[str, dict] = {}
-    for item in raw_output_files:
-        if isinstance(item, str):
-            normalized[item] = _enrich_metadata(item, {})
-            continue
-        if isinstance(item, ModelDumpable):
-            item = item.model_dump()
-        if not isinstance(item, dict):
-            continue
-        path = str(item.get("path") or "").strip()
-        if not path:
-            continue
-        normalized[path] = _enrich_metadata(path, {key: value for key, value in item.items() if key != "path" and value not in (None, "")})
-
-    return normalized
-
-
-def _has_structured_output_metadata(output_files: dict[str, dict]) -> bool:
-    return any(any(key in metadata for key in ("extension", "mimetype", "size")) for metadata in output_files.values())
-
-
-def _summarize_output_files(output_files: dict[str, dict]) -> tuple[int, str]:
-    mime_sizes: dict[str, int] = defaultdict(int)
-    total_size = 0
-
-    for metadata in output_files.values():
-        if not isinstance(metadata, dict):
-            continue
-        size = _coerce_output_file_size(metadata.get("size"))
-        mimetype = str(metadata.get("mimetype") or "").strip()
-        total_size += size
-        if mimetype and size:
-            mime_sizes[mimetype] += size
-
-    output_mimetypes = ",".join(mime for mime, _size in sorted(mime_sizes.items(), key=lambda item: item[1], reverse=True))
-    return total_size, output_mimetypes
+def _manifest_metadata(manifest: OutputManifest) -> tuple[dict[str, dict], int, str]:
+    return manifest.as_mapping(), manifest.total_size, ",".join(manifest.mimetypes)
 
 
 def _resolve_output_metadata(raw_output_files: Any, plugin_dir: Path) -> tuple[dict[str, dict], int, str]:
-    normalized_output_files = _normalize_output_files(raw_output_files)
-    if normalized_output_files and _has_structured_output_metadata(normalized_output_files):
-        output_size, output_mimetypes = _summarize_output_files(normalized_output_files)
-        return normalized_output_files, output_size, output_mimetypes
-    return _collect_output_metadata(plugin_dir)
+    manifest = OutputManifest.from_value(raw_output_files)
+    if manifest.files and any(output_file.size for output_file in manifest.files):
+        return _manifest_metadata(manifest)
+    return _manifest_metadata(OutputManifest.scan(plugin_dir, containment_root=plugin_dir.parent))
 
 
 def _normalize_status(status: str) -> str:
@@ -238,37 +119,6 @@ def _should_update_snapshot_title(current_title: str, next_title: str, *, snapsh
     return len(next_title) > len(current)
 
 
-def _status_for_process_without_archive_result(event: ProcessCompletedEvent) -> str:
-    if event.exit_code == PROCESS_EXIT_SKIPPED:
-        return "skipped"
-    if event.exit_code in {128 + signal.SIGHUP, 128 + signal.SIGINT, 128 + signal.SIGTERM}:
-        # This fallback only runs when a snapshot hook exited before emitting a
-        # structured ArchiveResult. A polite shutdown signal means the runner
-        # was interrupted during ownership transfer, not that the extractor
-        # produced a durable negative result. Keep the hook queued so the next
-        # runner can retry the exact work item instead of sealing in a transient
-        # process-lifecycle failure.
-        return "queued"
-    if event.exit_code != 0:
-        return "failed"
-    return "noresults"
-
-
-def _iter_archiveresult_records(stdout: str) -> list[dict]:
-    records: list[dict] = []
-    for raw_line in stdout.splitlines():
-        line = raw_line.strip()
-        if not line.startswith("{"):
-            continue
-        try:
-            record = json.loads(line)
-        except json.JSONDecodeError:
-            continue
-        if record.get("type") == "ArchiveResult":
-            records.append(record)
-    return records
-
-
 @_perf_trace("archivebox.ArchiveResultService._save_archiveresult_event_sync")
 def _save_archiveresult_event_to_db(
     event: ArchiveResultEvent,
@@ -333,28 +183,13 @@ def _save_archiveresult_event_to_db(
         if event.error:
             defaults["notes"] = event.error
 
-    with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_lookup"):
-        result = ArchiveResult.objects.filter(
-            snapshot=snapshot,
-            plugin=event.plugin,
-            hook_name=event.hook_name,
-        ).first()
-    if result is None:
-        try:
-            with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_create"):
-                result = ArchiveResult.objects.create(
-                    snapshot=snapshot,
-                    plugin=event.plugin,
-                    hook_name=event.hook_name,
-                    **defaults,
-                )
-        except IntegrityError:
-            with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_get_after_integrity"):
-                result = ArchiveResult.objects.get(
-                    snapshot=snapshot,
-                    plugin=event.plugin,
-                    hook_name=event.hook_name,
-                )
+    with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_get_or_create"):
+        result, _created = ArchiveResult.get_or_create_by_hook(
+            snapshot,
+            event.plugin,
+            event.hook_name,
+            defaults=defaults,
+        )
 
     with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.diff_fields"):
         update_fields = []
@@ -366,13 +201,6 @@ def _save_archiveresult_event_to_db(
         with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_update"):
             result.save(update_fields=[*update_fields, "modified_at"])
 
-    if result.status == ArchiveResult.StatusChoices.QUEUED:
-        # ArchiveResult has no retry_at column. If a shutdown/takeover projects
-        # a killed hook back to QUEUED, wake the parent Snapshot/Crawl so the
-        # next runner retries that exact hook instead of waiting on a stale
-        # active-state lease.
-        snapshot.update_and_requeue(retry_at=timezone.now())
-
     if result.status in (ArchiveResult.StatusChoices.SUCCEEDED, ArchiveResult.StatusChoices.NORESULTS):
         with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.title_update"):
             title_output_str = result.output_str if result.status == ArchiveResult.StatusChoices.SUCCEEDED else ""
@@ -396,36 +224,45 @@ def _save_archiveresult_event_to_db(
 
 
 def mark_archiveresult_started(event: ProcessStartedEvent, *, snapshot_id: str, process_id: str) -> None:
-    """Advance an existing queued hook row after its OS process is persisted."""
-    from archivebox.core.models import ArchiveResult
+    """Project a running abx-dl hook after its OS process is persisted."""
+    from archivebox.core.models import ArchiveResult, Snapshot
 
     started_at = parse_event_datetime(event.start_ts)
     if started_at is None:
         raise ValueError("ProcessStartedEvent.start_ts is required")
-    ArchiveResult.objects.filter(
-        snapshot_id=snapshot_id,
-        plugin=event.plugin_name,
-        hook_name=event.hook_name,
-        status=ArchiveResult.StatusChoices.QUEUED,
-    ).update(
-        status=ArchiveResult.StatusChoices.STARTED,
-        start_ts=started_at,
-        end_ts=None,
-        process_id=process_id,
-        modified_at=timezone.now(),
+    snapshot = Snapshot.objects.filter(id=snapshot_id).first()
+    if snapshot is None:
+        return
+    result, _created = ArchiveResult.get_or_create_by_hook(
+        snapshot,
+        event.plugin_name,
+        event.hook_name,
+        defaults={
+            "status": ArchiveResult.StatusChoices.STARTED,
+            "start_ts": started_at,
+            "end_ts": None,
+            "process_id": process_id,
+        },
     )
+    if result.start_ts is not None and started_at <= result.start_ts:
+        return
+    result.status = ArchiveResult.StatusChoices.STARTED
+    result.start_ts = started_at
+    result.end_ts = None
+    result.process_id = process_id
+    result.save(update_fields=["status", "start_ts", "end_ts", "process_id", "modified_at"])
 
 
 class ArchiveResultService(BaseService):
-    LISTENS_TO = [ArchiveResultEvent, ProcessCompletedEvent]
+    """Project abx-dl ArchiveResult facts into Django models."""
+
+    LISTENS_TO = [ArchiveResultEvent]
     EMITS = []
 
     def __init__(self, bus):
-        self._completed_process_event_ids: set[str] = set()
         self._save_locks: dict[tuple[str, str, str], asyncio.Lock] = {}
         super().__init__(bus)
         self.bus.on(ArchiveResultEvent, self.on_ArchiveResultEvent__save_to_db)
-        self.bus.on(ProcessCompletedEvent, self.on_ProcessCompletedEvent__save_to_db)
 
     @_perf_trace("archivebox.ArchiveResultService.on_ArchiveResultEvent__save_to_db")
     async def on_ArchiveResultEvent__save_to_db(self, event: ArchiveResultEvent) -> None:
@@ -441,64 +278,3 @@ class ArchiveResultService(BaseService):
         lock = self._save_locks.setdefault(key, asyncio.Lock())
         async with lock:
             await sync_to_async(_save_archiveresult_event_to_db, thread_sensitive=True)(event, process_started)
-
-    @_perf_trace("archivebox.ArchiveResultService.on_ProcessCompletedEvent__save_to_db")
-    async def on_ProcessCompletedEvent__save_to_db(self, event: ProcessCompletedEvent) -> None:
-        if event.event_id in self._completed_process_event_ids:
-            return
-        self._completed_process_event_ids.add(event.event_id)
-
-        if not event.hook_name.startswith("on_Snapshot"):
-            return
-        with _perf_span("archivebox.ArchiveResultService.on_ProcessCompletedEvent.find_snapshot_event"):
-            snapshot_event = await self.bus.find(
-                SnapshotEvent,
-                past=True,
-                future=False,
-                where=lambda candidate: self.bus.event_is_child_of(event, candidate),
-            )
-        if snapshot_event is None:
-            return
-
-        with _perf_span("archivebox.ArchiveResultService.on_ProcessCompletedEvent.parse_stdout_records"):
-            records = _iter_archiveresult_records(event.stdout)
-        if records:
-            if len(records) > 1:
-                raise RuntimeError(
-                    f"Hook {event.plugin_name}:{event.hook_name} emitted {len(records)} ArchiveResult records; expected exactly one",
-                )
-            for record in records:
-                record_status = _normalize_status(record.get("status") or "")
-                record_failed = record_status == "failed" or (not record_status and event.exit_code not in (0, PROCESS_EXIT_SKIPPED))
-                with _perf_span("archivebox.ArchiveResultService.on_ProcessCompletedEvent.emit_archive_result_record"):
-                    await event.emit(
-                        ArchiveResultEvent(
-                            snapshot_id=record.get("snapshot_id") or snapshot_event.snapshot_id,
-                            plugin=record.get("plugin") or event.plugin_name,
-                            hook_name=record.get("hook_name") or event.hook_name,
-                            status=record_status,
-                            output_str=record.get("output_str") or "",
-                            output_json=record.get("output_json") if isinstance(record.get("output_json"), dict) else None,
-                            output_files=event.output_files,
-                            start_ts=event.start_ts,
-                            end_ts=event.end_ts,
-                            error=record.get("error") or (event.stderr if record_failed else ""),
-                        ),
-                    ).now()
-            return
-
-        process_failed = _status_for_process_without_archive_result(event) == "failed"
-        with _perf_span("archivebox.ArchiveResultService.on_ProcessCompletedEvent.emit_archive_result_fallback"):
-            await event.emit(
-                ArchiveResultEvent(
-                    snapshot_id=snapshot_event.snapshot_id,
-                    plugin=event.plugin_name,
-                    hook_name=event.hook_name,
-                    status=_status_for_process_without_archive_result(event),
-                    output_str=event.stderr if process_failed else "",
-                    output_files=event.output_files,
-                    start_ts=event.start_ts,
-                    end_ts=event.end_ts,
-                    error=event.stderr if process_failed else "",
-                ),
-            ).now()

archivebox/services/process_service.py

@@ -43,10 +43,10 @@ def normalize_process_env(env: dict) -> dict:
         if is_sensitive_config_key(key) or (key in config_input_names and key not in allowed_config_keys):
             normalized.pop(key, None)
     if selected_plugins:
-        from abx_dl.models import discover_plugins, filter_plugins
         from archivebox.config.common import _plugin_enabled_config_keys
+        from archivebox.plugins.discovery import get_plugin_catalog
 
-        selected_plugins = set(filter_plugins(discover_plugins(runtime="archivebox"), sorted(selected_plugins), include_providers=True))
+        selected_plugins = set(get_plugin_catalog().select(sorted(selected_plugins)))
         for plugin_name, enabled_key in _plugin_enabled_config_keys().items():
             normalized.setdefault(enabled_key, "True" if plugin_name in selected_plugins else "False")
     return normalized
@@ -100,6 +100,7 @@ class ProcessService(BaseService):
             process = await Process.objects.acreate(
                 machine=iface.machine,
                 iface=iface,
+                parent_id=None,
                 process_type=process_type,
                 worker_type=worker_type,
                 pwd=event.output_dir,
@@ -133,6 +134,7 @@ class ProcessService(BaseService):
             hook_path=event.hook_path,
         )
         await Process.objects.filter(id=process.id).aupdate(
+            parent_id=process.parent_id,
             pwd=process.pwd,
             cmd=process.cmd,
             env=process.env,
@@ -219,6 +221,7 @@ class ProcessService(BaseService):
             await Process.objects.acreate(
                 machine=iface.machine,
                 iface=iface,
+                parent_id=None,
                 process_type=process_type,
                 worker_type=worker_type,
                 pwd=event.output_dir,
@@ -239,6 +242,7 @@ class ProcessService(BaseService):
         updates = {
             "machine_id": iface.machine_id,
             "iface_id": iface.id,
+            "parent_id": process.parent_id,
             "pwd": event.output_dir,
             "env": process_env,
             "pid": event.pid or process.pid,

archivebox/services/snapshot_service.py

@@ -1,12 +1,10 @@
 from __future__ import annotations
 
-import sys
+import asyncio
 from pathlib import Path
 
 from asgiref.sync import sync_to_async
 from django.utils import timezone
-from django.core.exceptions import ValidationError
-from rich import print as rprint
 from abx_dl.events import SnapshotCompletedEvent, SnapshotEvent
 from abx_dl.limits import CrawlLimitState
 from abx_dl.services.base import BaseService
@@ -44,6 +42,9 @@ def project_discovered_snapshots(snapshot_id: str) -> list:
 def finalize_completed_snapshot(
     snapshot_id: str,
     *,
+    owned_retry_at,
+    was_sealed: bool,
+    consumed_retry_plugins: list[str] | None = None,
     output_dir=None,
     crawl_limit_stop_reason: str | None = None,
 ) -> None:
@@ -73,13 +74,40 @@ def finalize_completed_snapshot(
             modified_at=timezone.now(),
         )
 
-    if snapshot.status == Snapshot.StatusChoices.QUEUED:
-        snapshot.sm.tick()
-        snapshot.refresh_from_db()
-    if snapshot.status == Snapshot.StatusChoices.STARTED and snapshot.is_finished_processing():
-        snapshot.sm.seal()
+    # SnapshotCompletedEvent is abx-dl's authoritative signal that the complete
+    # snapshot hook sequence (including cleanup) finished. ArchiveResult rows
+    # are projections of that work, never prerequisites used to decide whether
+    # the Snapshot may seal.
+    if not was_sealed and snapshot.status == Snapshot.StatusChoices.STARTED and snapshot.retry_at == owned_retry_at:
+        snapshot.seal()
         snapshot.refresh_from_db()
 
+    consumed_plugins = sorted({str(name).strip() for name in (consumed_retry_plugins or []) if str(name).strip()})
+    if consumed_plugins and snapshot.status == Snapshot.StatusChoices.SEALED:
+        for _attempt in range(3):
+            current = Snapshot.objects.only("status", "retry_at", "config").get(pk=snapshot.pk)
+            pending_plugins = sorted(
+                {str(name).strip() for name in (current.config or {}).get("RETRY_PLUGINS", []) if str(name).strip()},
+            )
+            if pending_plugins != consumed_plugins:
+                break
+            owned_filter = {"retry_at": owned_retry_at} if was_sealed else {"retry_at__isnull": True}
+            updates: dict[str, object] = {
+                "config": {key: value for key, value in current.config.items() if key != "RETRY_PLUGINS"},
+                "modified_at": timezone.now(),
+            }
+            if was_sealed:
+                updates["retry_at"] = None
+            updated = Snapshot.objects.filter(
+                pk=current.pk,
+                status=Snapshot.StatusChoices.SEALED,
+                config=current.config,
+                **owned_filter,
+            ).update(**updates)
+            if updated:
+                snapshot.refresh_from_db()
+                break
+
     snapshot.write_index_jsonl(output_dir=output_dir)
 
 
@@ -100,6 +128,8 @@ class SnapshotService(BaseService):
 
     def __init__(self, bus, *, crawl_id: str):
         self.crawl_id = crawl_id
+        self._run_ownership: dict[str, tuple[str, object, bool, list[str]]] = {}
+        self._ownership_lock = asyncio.Lock()
         super().__init__(bus)
         self.bus.on(SnapshotEvent, self.on_SnapshotEvent)
         self.bus.on(SnapshotCompletedEvent, self.on_SnapshotCompletedEvent)
@@ -112,33 +142,49 @@ class SnapshotService(BaseService):
         if snapshot is not None:
             if snapshot.is_paused:
                 return
+            was_sealed = snapshot.status == Snapshot.StatusChoices.SEALED
             if snapshot.status == Snapshot.StatusChoices.QUEUED:
-                if not await snapshot.archiveresult_set.aexists():
-                    from archivebox.services.runner import snapshot_hooks_for_pending_archiveresults
-
-                    hooks = await sync_to_async(snapshot_hooks_for_pending_archiveresults, thread_sensitive=True)(snapshot)
-                    await sync_to_async(snapshot.create_pending_archiveresults, thread_sensitive=True)(hooks=hooks)
-                try:
-                    await sync_to_async(snapshot.sm.tick, thread_sensitive=True)()
-                except ValidationError as err:
-                    if "ArchiveBox cannot archive its own admin, web, api, or snapshot URLs." not in str(err):
-                        raise
-                    await Snapshot.objects.filter(id=snapshot.id).aupdate(
-                        status=Snapshot.StatusChoices.SEALED,
-                        retry_at=None,
-                        modified_at=timezone.now(),
-                    )
-                    rprint(
-                        f"[red][X] Refusing to archive ArchiveBox internal URL for security: {snapshot.url}[/red]",
-                        file=sys.stderr,
-                    )
-                    return
+                await sync_to_async(snapshot.advance_lifecycle, thread_sensitive=True)()
                 await sync_to_async(snapshot.refresh_from_db, thread_sensitive=True)()
-            elif snapshot.status != Snapshot.StatusChoices.STARTED:
+            elif snapshot.status not in (Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.SEALED):
                 return
-            if snapshot.status != Snapshot.StatusChoices.STARTED:
-                return
-            await sync_to_async(snapshot.ensure_crawl_symlink, thread_sensitive=True)()
+            if snapshot.status == Snapshot.StatusChoices.STARTED:
+                await sync_to_async(snapshot.ensure_crawl_symlink, thread_sensitive=True)()
+            retry_plugins = [str(name).strip() for name in (snapshot.config or {}).get("RETRY_PLUGINS", []) if str(name).strip()]
+            async with self._ownership_lock:
+                self._run_ownership[str(event.snapshot_id)] = (str(event.event_id), snapshot.retry_at, was_sealed, retry_plugins)
+
+    async def renew_lease(self, snapshot_id: str, lease_until) -> bool | None:
+        from archivebox.core.models import Snapshot
+
+        async with self._ownership_lock:
+            ownership = self._run_ownership.get(str(snapshot_id))
+            if ownership is None or ownership[2]:
+                return None
+            event_id, owned_retry_at, was_sealed, retry_plugins = ownership
+            updated = await Snapshot.objects.filter(
+                id=snapshot_id,
+                status=Snapshot.StatusChoices.STARTED,
+                retry_at=owned_retry_at,
+            ).aupdate(
+                retry_at=lease_until,
+                modified_at=timezone.now(),
+            )
+            if updated:
+                self._run_ownership[str(snapshot_id)] = (event_id, lease_until, was_sealed, retry_plugins)
+            return bool(updated)
 
     async def on_SnapshotCompletedEvent(self, event: SnapshotCompletedEvent) -> None:
-        await sync_to_async(finalize_completed_snapshot, thread_sensitive=True)(event.snapshot_id)
+        snapshot_id = str(event.snapshot_id)
+        async with self._ownership_lock:
+            ownership = self._run_ownership.get(snapshot_id)
+            if ownership is None or ownership[0] != str(event.event_parent_id):
+                return
+            self._run_ownership.pop(snapshot_id, None)
+        _, owned_retry_at, was_sealed, retry_plugins = ownership
+        await sync_to_async(finalize_completed_snapshot, thread_sensitive=True)(
+            event.snapshot_id,
+            owned_retry_at=owned_retry_at,
+            was_sealed=was_sealed,
+            consumed_retry_plugins=retry_plugins,
+        )

archivebox/services/tag_service.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from asgiref.sync import sync_to_async
+
 from abx_dl.events import TagEvent
 from abx_dl.services.base import BaseService
 
@@ -13,10 +15,10 @@ class TagService(BaseService):
         self.bus.on(TagEvent, self.on_TagEvent__save_to_db)
 
     async def on_TagEvent__save_to_db(self, event: TagEvent) -> None:
-        from archivebox.core.models import Snapshot, SnapshotTag, Tag
+        from archivebox.core.models import Snapshot, Tag
 
         snapshot = await Snapshot.objects.filter(id=event.snapshot_id).afirst()
         if snapshot is None:
             return
-        tag, _ = await Tag.objects.aget_or_create(name=event.name)
-        await SnapshotTag.objects.aget_or_create(snapshot=snapshot, tag=tag)
+        tag, _ = await sync_to_async(Tag.get_or_create_by_name, thread_sensitive=True)(event.name)
+        await sync_to_async(snapshot.add_tag_ids, thread_sensitive=True)([tag.pk])

archivebox/templates/core/add.html

@@ -1,6 +1,6 @@
 {% extends "core/base.html" %}
 
-{% load static %}
+{% load static core_tags %}
 {% load i18n %}
 
 {% block breadcrumbs %}
@@ -48,9 +48,7 @@
                 <div class="crawl-explanation">
                     <p class="crawl-tip" style="user-select: none;">
                         <span style="float: right">
-                            {% if user.is_authenticated and user.is_superuser and request.archivebox_config.OPENCODE_ENABLED %}
-                                <a href="/admin/agent">💬 Crawl with AI</a> &nbsp; | &nbsp;
-                            {% endif %}
+                            {% plugin_ui "opencode" "add" %}
                             <a href="https://github.com/ArchiveBox/archivebox-browser-extension"><img src="{% static 'chrome_extension_icon.png' %}" alt="Chrome Extension" style="height: 33px; margin-top: -5px;">&nbsp; Get the extension</a>
                         </span>
                         💡 <strong>Tip:</strong> Instantly save a single URL by visiting:

archivebox/templates/core/navigation.html

@@ -1,4 +1,4 @@
-{% load i18n static %}
+{% load i18n static core_tags %}
 
 <div id="user-tools">
     <a href="{% url 'add' %}" class="navbar-item navbar-add-link">Add ➕</a>
@@ -11,10 +11,7 @@
     <span class="navbar-separator" aria-hidden="true">|</span>
     <a href="/admin/core/tag/" class="navbar-item navbar-tags">Tags</a>
     <span class="navbar-separator navbar-separator-wide" aria-hidden="true"></span>
-    {% if user.is_authenticated and user.is_superuser and request.archivebox_config.OPENCODE_ENABLED %}
-        <a href="/admin/agent" class="navbar-item navbar-ai">💬 AI</a>
-        <span class="navbar-separator" aria-hidden="true">|</span>
-    {% endif %}
+    {% plugin_ui "opencode" "navigation" %}
     <a href="{% url 'Docs' %}" class="navbar-item navbar-docs" target="_blank" rel="noopener noreferrer">Docs</a>
     <span class="navbar-separator" aria-hidden="true">|</span>
     <a href="/api/v1/docs" class="navbar-item navbar-api">API</a>

archivebox/templates/core/snapshot.html

@@ -319,9 +319,9 @@
                 user-select: all;
             }
             .header-toggle {
-                line-height: 12px;
-                font-size: 70px;
-                margin-top: -13px;
+                line-height: 1;
+                font-size: 24px;
+                margin-top: 0;
                 margin-left: 4px;
             }
             @container snapshot-header (max-width: 900px) {
@@ -1056,8 +1056,9 @@
             .thumb-grid {
                 display: grid;
                 grid-template-columns: repeat(auto-fit, minmax(clamp(180px, 14vw, 250px), 1fr));
+                grid-auto-flow: row dense;
+                grid-auto-rows: 42px;
                 gap: 6px;
-                align-content: start;
                 width: 100%;
                 max-width: 100%;
                 margin-left: 0;
@@ -1077,17 +1078,23 @@
                 display: inline-flex;
                 flex-direction: column;
                 align-items: stretch;
+                grid-row: span 3;
                 break-inside: avoid;
-                margin-bottom: 6px;
+                margin-bottom: 0;
                 vertical-align: top;
                 content-visibility: auto;
                 contain-intrinsic-size: 138px;
             }
+            .thumb-card:not([data-plugin-name]) {
+                order: 1;
+            }
             .thumb-card:has([data-compact]) {
-                height: 46px;
-                min-height: 46px;
-                max-height: 46px;
-                contain-intrinsic-size: 46px;
+                height: 42px;
+                min-height: 42px;
+                max-height: 42px;
+                grid-row: span 1;
+                order: 2;
+                contain-intrinsic-size: 42px;
             }
             .thumb-card .thumb-body {
                 display: grid;
@@ -1247,7 +1254,7 @@
             }
             .thumb-card:has([data-compact]) .thumbnail-wrapper,
             .thumb-card:has([data-compact]) .thumbnail-wrapper.compact {
-                height: 24px;
+                height: 20px;
                 flex: 0 0 auto;
             }
             .thumb-card:has([data-compact]) .thumb-body {
@@ -1468,17 +1475,19 @@
     </head>
     <body>
         <div id="main-frame-wrapper" class="full-page-wrapper" data-has-outputs="{{ has_outputs|yesno:'1,0' }}" data-snapshot-state="{{ snapshot_state }}">
+            {% if not STATIC_EXPORT %}
             {% if snapshot_state == 'queued' or snapshot_state == 'started' or snapshot_state == 'paused' %}
             <div id="snapshot-progress-wrapper">
                 {% include "progressmonitor/progress_monitor.html" with progress_endpoint=progress_endpoint progress_scope="snapshot" %}
             </div>
             {% endif %}
+            {% endif %}
             {% if has_outputs %}
             <iframe id="main-frame"
                     sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms"
                     class="full-page-iframe"
                     src="about:blank"
-                    data-default-src="{% if best_result.path %}{% snapshot_preview_url snapshot best_result.path %}{% else %}{% snapshot_base_url snapshot %}/?files=1{% endif %}"
+                    data-default-src="{% if best_result.path %}{% snapshot_preview_url snapshot best_result.path %}{% elif STATIC_EXPORT %}{% snapshot_url snapshot 'index.jsonl' %}{% else %}{% snapshot_base_url snapshot %}/?files=1{% endif %}"
                     name="preview"
                     loading="eager"
                     fetchpriority="high"></iframe>
@@ -1506,7 +1515,9 @@
                     <div class="snapshot-empty-actions">
                         <button type="button" id="snapshot-empty-refresh" class="snapshot-empty-btn">🔄 Refresh page</button>
                         <a href="{{ url }}" target="_blank" rel="noopener noreferrer" class="snapshot-empty-btn">🌐 Open original URL</a>
+                        {% if not STATIC_EXPORT %}
                         <a href="{% admin_base_url %}/admin/core/snapshot/{{ snapshot_id }}/change/" class="snapshot-empty-btn">✏️ Edit in admin</a>
+                        {% endif %}
                     </div>
                 </div>
             </div>
@@ -1528,7 +1539,7 @@
                 const frame = document.getElementById('main-frame')
                 if (!frame) return
                 const snapshotBaseUrlEarly = "{% snapshot_base_url snapshot %}"
-                const snapshotFilesUrlEarly = `${snapshotBaseUrlEarly}/?files=1`
+                const snapshotFilesUrlEarly = "{% if STATIC_EXPORT %}{% snapshot_url snapshot 'index.jsonl' %}{% else %}{% snapshot_base_url snapshot %}/?files=1{% endif %}"
                 const defaultSrc = frame.dataset.defaultSrc || snapshotFilesUrlEarly
                 const rawHash = window.location.hash ? window.location.hash.slice(1) : ''
 
@@ -1560,8 +1571,9 @@
                 <div class="header-nav">
                     <div class="header-col header-left" style="line-height: 58px; vertical-align: middle">
                         {% web_base_url as web_base %}
-                        <a href="{% if web_base %}{{ web_base }}/public/{% else %}/{% endif %}" class="header-archivebox" title="Go to Public Index...">
-                            <img src="{% if web_base %}{{ web_base }}/static/archive.png{% else %}{% static 'archive.png' %}{% endif %}" alt="Archive Icon">
+                        {% static_export_root_url as static_root %}
+                        <a href="{% if STATIC_EXPORT %}{{ static_root }}index.html{% elif web_base %}{{ web_base }}/public/{% else %}/{% endif %}" class="header-archivebox" title="Go to Public Index...">
+                            <img src="{% if STATIC_EXPORT %}data:image/svg+xml;utf8,&lt;svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 64 64'&gt;&lt;rect x='8' y='5' width='48' height='54' rx='7' fill='%23fff' opacity='.95'/&gt;&lt;path d='M18 16h28M18 26h28M18 36h20M18 46h24' stroke='%23aa1e55' stroke-width='5' stroke-linecap='round'/&gt;&lt;/svg&gt;{% elif web_base %}{{ web_base }}/static/archive.png{% else %}{% static 'archive.png' %}{% endif %}" alt="Archive Icon">
                             ArchiveBox
                         </a>
                     </div>
@@ -1607,34 +1619,44 @@
                             </div>
                             <div class="badge badge-info">
                                 <span class="badge-label">Size</span>
+                                {% if STATIC_EXPORT %}
+                                <span class="badge-value">{{size}}</span>
+                                {% else %}
                                 <a class="badge-value" href="{% admin_base_url %}/admin/core/snapshot/{{snapshot_id|default:id}}/change/" title="Click to edit this Snapshot in the Admin UI">
                                     {{size}}
                                 </a>
+                                {% endif %}
                             </div>
+                            {% if not STATIC_EXPORT %}
                             <div class="badge badge-default">
                                 <span class="badge-label">Manage</span>
                                 <a class="badge-value" href="{% admin_base_url %}/admin/core/snapshot/{{snapshot_id|default:id}}/change/" title="Click to edit this Snapshot in the Admin UI">
                                     <span class="badge-desktop-icon">✏️</span><span class="badge-mobile-text">Edit</span>
                                 </a>
                             </div>
+                            {% endif %}
                             <div class="badge badge-{{status_color}}">
                                 <span class="badge-label">Status</span>
+                                {% if STATIC_EXPORT %}
+                                <span class="badge-value">{{status|upper}}</span>
+                                {% else %}
                                 <a class="badge-value" href="{% admin_base_url %}/admin/core/snapshot/?q={{snapshot_id|default:id}}" title="Click to see options to pull, re-snapshot, or delete this Snapshot">
                                     {{status|upper}}
                                 </a>
+                                {% endif %}
                             </div>
                         </div>
                         <div class="header-mobile-badges">
-                            <a class="header-mobile-badge" href="{% snapshot_base_url snapshot %}/?files=1" title="Browse all files for this snapshot">
+                            <a class="header-mobile-badge" href="{% if STATIC_EXPORT %}{% snapshot_url snapshot 'index.jsonl' %}{% else %}{% snapshot_base_url snapshot %}/?files=1{% endif %}" title="Browse all files for this snapshot">
                                 <span>{{num_outputs}} Files</span>
                                 <span class="header-mobile-badge-separator">|</span>
                                 <span>{{size}}</span>
                             </a>
-                            <a class="header-mobile-badge" href="{% admin_base_url %}/admin/core/snapshot/{{snapshot_id|default:id}}/change/" title="Edit this snapshot in the Admin UI">
+                            {% if not STATIC_EXPORT %}<a class="header-mobile-badge" href="{% admin_base_url %}/admin/core/snapshot/{{snapshot_id|default:id}}/change/" title="Edit this snapshot in the Admin UI">
                                 <span>{{status|capfirst}}</span>
                                 <span class="header-mobile-badge-separator">|</span>
                                 <span>Edit</span>
-                            </a>
+                            </a>{% endif %}
                         </div>
                         {% if related_years %}
                         <div class="header-year-badges">
@@ -1680,14 +1702,14 @@
                                 </div>
                             </details>
                         {% else %}
-                            <a class="header-date" href="{% web_base_url %}/{{archive_path}}/index.html" title="Date Added: {{bookmarked_date}}  |  First Archived: {{oldest_archive_date|default:downloaded_datestr}}  |  Last Checked: {{downloaded_datestr}}   (UTC)">
+                            <a class="header-date" href="{% snapshot_base_url snapshot %}/index.html" title="Date Added: {{bookmarked_date}}  |  First Archived: {{oldest_archive_date|default:downloaded_datestr}}  |  Last Checked: {{downloaded_datestr}}   (UTC)">
                                 <span class="header-date-label">Captures</span>
                                 {{oldest_archive_date|default:downloaded_datestr|default:bookmarked_date|slice:":10"}}
                             </a>
                         {% endif %}
                         <br/>
                         <div class="external-links">
-                            <a href="{% snapshot_base_url snapshot %}/?files=1" title="Browse the full SNAP_DIR for this snapshot" target="_blank">📁 See all files...</a>
+                            <a href="{% if STATIC_EXPORT %}{% snapshot_url snapshot 'index.jsonl' %}{% else %}{% snapshot_base_url snapshot %}/?files=1{% endif %}" title="Browse the full SNAP_DIR for this snapshot" target="_blank">📁 {% if STATIC_EXPORT %}Snapshot manifest{% else %}See all files...{% endif %}</a>
                             <span class="external-links-separator">|</span>
                             <a href="https://web.archive.org/web/{{url}}" title="Search for a copy of the URL saved in Archive.org" target="_blank" rel="noreferrer">🏛️ Archive.org</a>
                             <!--<a href="https://archive.md/{{url}}" title="Search for a copy of the URL saved in Archive.today" target="_blank" rel="noreferrer">Archive.today</a>  &nbsp;|&nbsp; -->
@@ -1707,12 +1729,12 @@
                         <div class="thumb-card{% if forloop.first %} selected-card{% endif %}" data-plugin-name="{{result.name|plugin_name}}"{% if preview_url %} data-preview-url="{{preview_url}}"{% endif %}{% if display_path %} data-output-path="{{display_path}}"{% endif %}>
                                 <div class="thumb-body">
                                     <div class="thumb-actions">
-                                        <a href="{% snapshot_url snapshot result.name %}/?files=1" data-no-preview="1" title="Open output folder" target="_blank" rel="noopener">📁</a>
+                                        {% if not STATIC_EXPORT %}<a href="{% snapshot_url snapshot result.name %}/?files=1" data-no-preview="1" title="Open output folder" target="_blank" rel="noopener">📁</a>{% endif %}
                                         {% if display_path %}
                                             <a href="{{display_url}}" data-no-preview="1" title="Download output file" download>⬇️</a>
                                         {% endif %}
                                         {% if can_delete_outputs and result.result %}
-                                            <button type="button" data-no-preview="1" data-archive-result-ids="{{result.result_ids}}" data-delete-url="{% admin_base_url %}/admin/core/archiveresult/" title="Delete this output">❌</button>
+                                            <button type="button" data-no-preview="1" data-archive-result-ids="{{result.result_ids}}" data-delete-url="{% admin_base_url %}/admin/core/archiveresult/" data-delete-handoff="1" data-delete-snapshot-id="{{snapshot.id}}" title="Delete this output">×</button>
                                         {% endif %}
                                     </div>
                                     {% if display_path %}
@@ -1761,13 +1783,13 @@
                     <div class="thumb-card">
                             <div class="thumb-body">
                                 <div class="thumb-actions">
-                                    <a href="{% snapshot_base_url snapshot %}/?files=1" data-no-preview="1" title="Browse all snapshot files" target="_blank" rel="noopener">📁</a>
+                                    <a href="{% if STATIC_EXPORT %}{% snapshot_url snapshot 'index.jsonl' %}{% else %}{% snapshot_base_url snapshot %}/?files=1{% endif %}" data-no-preview="1" title="{% if STATIC_EXPORT %}Open snapshot manifest{% else %}Browse all snapshot files{% endif %}" target="_blank" rel="noopener">📁</a>
                                 </div>
                                 <h4>📦 Other files</h4>
                                 <div class="loose-items">
                                     {% for item in loose_items %}
                                         {% if item.is_dir %}
-                                            <a href="{% snapshot_url snapshot item.path %}/?files=1" data-no-preview="1" target="_blank" rel="noopener">📁 {{item.name}}</a>
+                                            <a href="{% if STATIC_EXPORT %}{% snapshot_url snapshot 'index.jsonl' %}{% else %}{% snapshot_url snapshot item.path %}/?files=1{% endif %}" data-no-preview="1" target="_blank" rel="noopener">📁 {{item.name}}</a>
                                         {% else %}
                                             <a href="{% snapshot_url snapshot item.path %}" data-no-preview="1" target="_blank" rel="noopener">📄 {{item.name}}</a>
                                         {% endif %}
@@ -1780,13 +1802,13 @@
                     <div class="thumb-card">
                             <div class="thumb-body">
                                 <div class="thumb-actions">
-                                    <a href="{% snapshot_base_url snapshot %}/?files=1" data-no-preview="1" title="Browse all snapshot files" target="_blank" rel="noopener">📁</a>
+                                    <a href="{% if STATIC_EXPORT %}{% snapshot_url snapshot 'index.jsonl' %}{% else %}{% snapshot_base_url snapshot %}/?files=1{% endif %}" data-no-preview="1" title="{% if STATIC_EXPORT %}Open snapshot manifest{% else %}Browse all snapshot files{% endif %}" target="_blank" rel="noopener">📁</a>
                                 </div>
                                 <h4>⚠️ Failed</h4>
                                 <div class="loose-items failed-items">
                                     {% for item in failed_items %}
                                         {% if item.is_dir %}
-                                            <a href="{% snapshot_url snapshot item.path %}/?files=1" data-no-preview="1" target="_blank" rel="noopener">📁 {{item.name}}</a>
+                                            <a href="{% if STATIC_EXPORT %}{% snapshot_url snapshot 'index.jsonl' %}{% else %}{% snapshot_url snapshot item.path %}/?files=1{% endif %}" data-no-preview="1" target="_blank" rel="noopener">📁 {{item.name}}</a>
                                         {% else %}
                                             <a href="{% snapshot_url snapshot item.path %}" data-no-preview="1" target="_blank" rel="noopener">📄 {{item.name}}</a>
                                         {% endif %}
@@ -1799,13 +1821,11 @@
             </div>
         </header>
 
-        {% if can_delete_outputs %}<input type="hidden" id="delete-output-csrf" value="{{csrf_token}}">{% endif %}
-        <script src="{% static 'jquery.min.js' %}" type="text/javascript"></script>
         {% if can_delete_outputs %}{% include "includes/output_delete_controls.html" %}{% endif %}
 
         <script>
             const snapshotBaseUrl = "{% snapshot_base_url snapshot %}";
-            const snapshotFilesUrl = `${snapshotBaseUrl}/?files=1`;
+            const snapshotFilesUrl = "{% if STATIC_EXPORT %}{% snapshot_url snapshot 'index.jsonl' %}{% else %}{% snapshot_base_url snapshot %}/?files=1{% endif %}";
 
             function tryCenterImageFrame(frame) {
                 try {
@@ -1925,9 +1945,7 @@
             }
 
             // un-sandbox iframes showing pdfs (required to display pdf viewer)
-            jQuery('iframe').map(function() {
-                attachPreviewFrameHandlers(this)
-            })
+            document.querySelectorAll('iframe').forEach((frame) => attachPreviewFrameHandlers(frame))
 
             function getPreviewHashValueFromHref(href, normalize=false) {
                 if (href == './') {
@@ -1987,7 +2005,7 @@
                     const basePath = (base.pathname || '').replace(/\/+$/, '')
                     const targetPath = (target.pathname || '').replace(/\/+$/, '')
                     return (
-                        target.search !== '?files=1'
+                        target.search !== "{% if STATIC_EXPORT %}{% else %}?files=1{% endif %}"
                         && (
                             targetPath === basePath
                             || targetPath === `${basePath}/index.html`
@@ -2051,8 +2069,8 @@
                     return false
                 }
 
-                jQuery('.selected-card').removeClass('selected-card')
-                jQuery(card).closest('.thumb-card').addClass('selected-card')
+                document.querySelectorAll('.selected-card').forEach((selected) => selected.classList.remove('selected-card'))
+                card.closest('.thumb-card').classList.add('selected-card')
 
                 const nextSrc = isSnapshotRootPreview(target) ? snapshotFilesUrl : target
                 const existingFrame = document.getElementById('main-frame')
@@ -2151,8 +2169,8 @@
 
             function hideSnapshotHeader() {
                 console.log('Collapsing Snapshot header...')
-                jQuery('.header-toggle').text('▸')
-                jQuery('.header-bottom').hide()
+                document.querySelectorAll('.header-toggle').forEach((toggle) => { toggle.textContent = '▸' })
+                document.querySelectorAll('.header-bottom').forEach((header) => { header.hidden = true })
                 try {
                     localStorage.setItem("archivebox-snapshot-header-visible", "false")
                 } catch (e) {
@@ -2161,8 +2179,8 @@
             }
             function showSnapshotHeader() {
                 console.log('Expanding Snapshot header...')
-                jQuery('.header-toggle').text('▾')
-                jQuery('.header-bottom').show()
+                document.querySelectorAll('.header-toggle').forEach((toggle) => { toggle.textContent = '▾' })
+                document.querySelectorAll('.header-bottom').forEach((header) => { header.hidden = false })
                 try {
                     localStorage.setItem("archivebox-snapshot-header-visible", "true")
                 } catch (e) {
@@ -2183,7 +2201,7 @@
             }
             function handleSnapshotHeaderToggle(event) {
                 event.preventDefault()
-                if (jQuery('.header-toggle').text().includes('▾')) {
+                if ([...document.querySelectorAll('.header-toggle')].some((toggle) => toggle.textContent.includes('▾'))) {
                     hideSnapshotHeader()
                 } else {
                     showSnapshotHeader()
@@ -2192,7 +2210,7 @@
             }
 
             // Hide or show the header once when its title row or collapse icon is clicked.
-            jQuery('.header-toggle-trigger').on('click', handleSnapshotHeaderToggle)
+            document.querySelectorAll('.header-toggle-trigger').forEach((trigger) => trigger.addEventListener('click', handleSnapshotHeaderToggle))
 
             // check URL for hash e.g. #git and load relevant preview
             selectInitialPreview()

archivebox/templates/core/static_index.html

@@ -1,258 +1,129 @@
-{% load static core_tags %}
-
+{% load core_tags %}
 <!DOCTYPE html>
 <html lang="en">
-    <head>
-        <title>Archived Sites</title>
-        <meta charset="utf-8" name="viewport" content="width=device-width, initial-scale=1">
-        <style>
-            :root {
-                --bg-main: #efefef;
-                --accent-1: #aa1e55;
-                --accent-2: #ffebeb;
-                --accent-3: #efefef;
-
-                --text-1: #1c1c1c;
-                --text-2: #eaeaea;
-                --text-main: #1a1a1a;
-                --font-main: "Gill Sans", Helvetica, sans-serif;
+<head>
+    <title>Archived Sites</title>
+    <meta charset="utf-8" name="viewport" content="width=device-width, initial-scale=1">
+    <style>
+        * { box-sizing: border-box; }
+        body { margin: 0; background: #f8fafc; color: #0f172a; font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; }
+        a { color: inherit; }
+        .static-header { display: flex; align-items: center; justify-content: space-between; gap: 16px; padding: 14px 20px; background: #aa1e55; color: #fff; }
+        .static-brand { display: flex; align-items: center; gap: 10px; font-size: 20px; font-weight: 700; text-decoration: none; }
+        .static-brand-mark { display: grid; place-items: center; width: 30px; height: 30px; border-radius: 7px; background: #fff; color: #aa1e55; font-size: 19px; }
+        .static-meta { color: rgba(255,255,255,.82); font-size: 12px; }
+        main { width: min(1500px, 100%); margin: 0 auto; padding: 18px; }
+        .toolbar { display: flex; align-items: center; justify-content: space-between; gap: 12px; margin-bottom: 14px; padding: 10px 12px; border: 1px solid #e2e8f0; border-radius: 10px; background: #fff; box-shadow: 0 1px 2px rgba(15,23,42,.04); }
+        #static-search { width: min(560px, 100%); height: 36px; padding: 7px 11px; border: 1px solid #cbd5e1; border-radius: 7px; font: inherit; font-size: 13px; }
+        .count { color: #64748b; font-size: 12px; white-space: nowrap; }
+        .table-wrap { overflow-x: auto; border: 1px solid #e2e8f0; border-radius: 10px; background: #fff; box-shadow: 0 1px 2px rgba(15,23,42,.04); }
+        table { width: 100%; min-width: 980px; table-layout: fixed; border-collapse: collapse; }
+        th { padding: 9px 10px; border-bottom: 1px solid #e2e8f0; background: #f8fafc; color: #475569; font-size: 11px; letter-spacing: .04em; text-align: left; text-transform: uppercase; }
+        td { padding: 8px 10px; border-bottom: 1px solid #edf2f7; vertical-align: middle; }
+        tbody tr:hover { background: #f8fafc; }
+        tbody tr:last-child td { border-bottom: 0; }
+        .snapshot-time { width: 102px; white-space: nowrap; }
+        .snapshot-time a { display: inline-flex; flex-direction: column; color: #334155; line-height: 1.2; text-decoration: none; }
+        .snapshot-time small { color: #94a3b8; font-size: 11px; }
+        .snapshot-preview-cell { width: 116px; }
+        .snapshot-preview-cell a { display: inline-flex; width: 102px; height: 102px; align-items: center; justify-content: center; }
+        .snapshot-preview { display: block; width: 100px; height: 100px; border: 1px solid #e2e8f0; border-radius: 8px; background: #f8fafc; object-fit: cover; object-position: top; }
+        .snapshot-preview-spinner { display: grid; place-items: center; }
+        .snapshot-preview-spinner::after { width: 20px; height: 20px; border: 2px solid #cbd5e1; border-top-color: #aa1e55; border-radius: 50%; content: ""; animation: spin .8s linear infinite; }
+        .snapshot-title-line { display: flex; align-items: center; gap: 8px; min-width: 0; margin-bottom: 2px; }
+        .snapshot-favicon-link { display: inline-flex; width: 22px; min-width: 22px; height: 22px; align-items: center; justify-content: center; }
+        .link-favicon { display: block; width: 18px; height: 18px; border-radius: 4px; object-fit: contain; }
+        .snapshot-title { overflow: hidden; color: #0f172a; font-size: 14px; font-weight: 650; line-height: 1.25; text-overflow: ellipsis; white-space: nowrap; text-decoration: none; }
+        .snapshot-url { display: block; margin-left: 30px; overflow: hidden; color: #64748b; font-size: 12px; text-overflow: ellipsis; white-space: nowrap; text-decoration: none; }
+        .snapshot-mobile-saved { display: none; }
+        .snapshot-tags-cell { width: 180px; }
+        .snapshot-tags { display: flex; flex-wrap: wrap; gap: 4px; max-height: 46px; overflow: hidden; }
+        .snapshot-tag { max-width: 150px; padding: 2px 7px; overflow: hidden; border: 1px solid #bfdbfe; border-radius: 999px; background: #eff6ff; color: #1d4ed8; font-size: 11px; font-weight: 600; text-overflow: ellipsis; white-space: nowrap; }
+        .snapshot-status-cell { width: 96px; }
+        .snapshot-status { display: inline-flex; min-width: 68px; justify-content: center; padding: 3px 8px; border-radius: 999px; background: #f1f5f9; color: #475569; font-size: 11px; font-weight: 700; text-transform: uppercase; }
+        .status-sealed .snapshot-status, .status-succeeded .snapshot-status { background: #d1fae5; color: #047857; }
+        .status-started .snapshot-status { background: #dbeafe; color: #1d4ed8; }
+        .status-queued .snapshot-status, .status-backoff .snapshot-status { background: #fef3c7; color: #b45309; }
+        .status-failed .snapshot-status { background: #fee2e2; color: #b91c1c; }
+        .snapshot-files-cell { width: 220px; overflow: hidden; }
+        .snapshot-files-cell .files-icons { max-width: 200px; }
+        .snapshot-size-cell { width: 102px; white-space: nowrap; }
+        .snapshot-size-cell a { display: block; color: #334155; font-size: 12px; font-weight: 700; text-decoration: none; }
+        .snapshot-size-cell small, .empty-value { color: #94a3b8; font-size: 10px; }
+        footer { padding: 24px 16px; color: #64748b; font-size: 12px; text-align: center; }
+        @keyframes spin { to { transform: rotate(360deg); } }
+        @media (max-width: 700px) {
+            .static-header, .toolbar { align-items: stretch; flex-direction: column; }
+            main { padding: 10px; }
+            table { min-width: 0; }
+            th:nth-child(n+4), td:nth-child(n+4), th.snapshot-time, td.snapshot-time { display: none; }
+            .snapshot-preview-cell { width: 88px; }
+            .snapshot-preview-cell a { width: 76px; height: 76px; }
+            .snapshot-preview { width: 74px; height: 74px; }
+            .snapshot-mobile-saved { display: block; margin: 4px 0 0 30px; color: #64748b; font-size: 11px; }
+        }
+    </style>
+    <script>
+        function nextPublicSnapshotPreview(img) {
+            const fallbacks = (img.dataset.fallbacks || '').split(',').filter(Boolean)
+            if (fallbacks.length) {
+                img.src = fallbacks.shift()
+                img.dataset.fallbacks = fallbacks.join(',')
+                return
             }
-            /* Dark Mode (WIP) */
-            /*
-            @media (prefers-color-scheme: dark) {
-                :root {
-                    --accent-2: hsl(160, 100%, 96%);
-
-                    --text-1: #eaeaea;
-                    --text-2: #1a1a1a;
-                    --bg-main: #101010;
-                }
-
-                #table-bookmarks_wrapper,
-                #table-bookmarks_wrapper img,
-                tbody td:nth-child(3),
-                tbody td:nth-child(3) span,
-                footer {
-                    filter: invert(100%);
-                }
-            }*/
-
-            html, body {
-                width: 100%;
-                height: 100%;
-                font-size: 18px;
-                font-weight: 200;
-                text-align: center;
-                margin: 0px;
-                padding: 0px;
-                font-family: var(--font-main);
-            }
-
-            .header-top small {
-                font-weight: 200;
-                color: var(--accent-3);
-            }
-            
-            .header-top {
-                width: 100%;
-                height: auto;
-                min-height: 40px;
-                margin: 0px;
-                text-align: center;
-                color: white;
-                font-size: calc(11px + 0.84vw);
-                font-weight: 200;
-                padding: 4px 4px;
-                border-bottom: 3px solid var(--accent-1);
-                background-color: var(--accent-1);
-            }
-            input[type=search] {
-                width: 22vw;
-                border-radius: 4px;
-                border: 1px solid #aeaeae;
-                padding: 3px 5px;
-            }
-            .nav > div {
-                min-height: 30px;
-            }
-            .header-top a {
-                text-decoration: none;
-                color: rgba(0,0,0,0.6);
-            }
-            .header-top a:hover {
-                text-decoration: none;
-                color: rgba(0,0,0,0.9);
-            }
-            .header-top .col-lg-4 {
-                text-align: center;
-                padding-top: 4px;
-                padding-bottom: 4px;
-            }
-            .header-archivebox img {
-                display: inline-block;
-                margin-right: 3px;
-                height: 30px;
-                margin-left: 12px;
-                margin-top: -4px;
-                margin-bottom: 2px;
-            }
-            .header-archivebox img:hover {
-                opacity: 0.5;
-            }
-
-            #table-bookmarks_length, #table-bookmarks_filter {
-                padding-top: 12px;
-                opacity: 0.8;
-                padding-left: 24px;
-                padding-right: 22px;
-                margin-bottom: -16px;
-            }
-            table {
-                padding: 6px;
-                width: 100%;
-            }
-            table thead th {
-                font-weight: 400;
-            }
-            table tr {
-                height: 35px;
-            }
-            tbody tr:nth-child(odd) {
-               background-color: var(--accent-2) !important;
-            }
-            table tr td {
-                white-space: nowrap;
-                overflow: hidden;
-                /*padding-bottom: 0.4em;*/
-                /*padding-top: 0.4em;*/
-                padding-left: 2px;
-                text-align: center;
-            }
-            table tr td a {
-                text-decoration: none;
-            }
-            table tr td img, table tr td object {
-                display: inline-block;
-                margin: auto;
-                height: 24px;
-                width: 24px;
-                padding: 0px;
-                padding-right: 5px;
-                vertical-align: middle;
-                margin-left: 4px;
-            }
-            #table-bookmarks {
-                width: 100%; 
-                overflow-y: scroll;
-                table-layout: fixed;
-            }
-            .dataTables_wrapper {
-                background-color: #fafafa;
-            }
-            table tr a span[data-archived~=False] {
-                opacity: 0.4;
-            }
-            .files-spinner {
-                height: 15px;
-                width: auto;
-                opacity: 0.5;
-                vertical-align: -2px;
-            }
-            .in-progress {
-                display: none;
-            }
-            tr td a.favicon img {
-                padding-left: 6px;
-                padding-right: 12px;
-                vertical-align: -4px;
-            }
-            tr td a.title {
-                font-size: 1.4em;
-                text-decoration:none;
-                color:black;
-            }
-            tr td a.title small {
-                background-color: var(--accent-3);
-                border-radius: 4px;
-                float:right
-            }
-            input[type=search]::-webkit-search-cancel-button {
-                -webkit-appearance: searchfield-cancel-button;
-            }
-            .title-col {
-                text-align: left;
-            }
-            .title-col a {
-                color: black;
-            }
-        </style>
-        <link rel="stylesheet" href="{% static 'bootstrap.min.css' %}">
-        <link rel="stylesheet" href="{% static 'jquery.dataTables.min.css' %}"/>
-        <script src="{% static 'jquery.min.js' %}"></script>
-        <script src="{% static 'jquery.dataTables.min.js' %}"></script>
-        <script>
-            document.addEventListener('error', function(e) {
-              e.target.style.opacity = 0;
-            }, true)
-            jQuery(document).ready(function() {
-                jQuery('#table-bookmarks').DataTable({
-                    stateSave: true, // save state (filtered input, number of entries shown, etc) in localStorage
-                    dom: '<lf<t>ip>', // how to show the table and its helpers (filter, etc) in the DOM
-                    order: [[0, 'desc']],
-                    iDisplayLength: 100,
-                });
-            });
-        </script>
-    </head>
-    <body>
-        <header>
-            <div class="header-top container-fluid">
-                <div class="row nav">
-                    <div class="col-sm-2">
-                        <a href="/" class="header-archivebox">
-                            <img src="{% static 'archive.png' %}" alt="Logo"/>
-                            ArchiveBox: Index
-                        </a>
-                    </div>
-                    <div class="col-sm-10" style="text-align: right">
-                        <a href="/add/">Add Links</a> &nbsp; | &nbsp; 
-                        <a href="/admin/core/snapshot/">Admin</a> &nbsp; | &nbsp; 
-                        <a href="https://github.com/ArchiveBox/ArchiveBox/wiki">Docs</a>
-                    </div>
-                </div>
-            </div>
-        </header>
-        
-        <table id="table-bookmarks">
-            <thead>
-                <tr>
-                    <th class="snapshot-time">Saved</th>
-                    <th class="snapshot-preview-cell">Preview</th>
-                    <th>Snapshot ({{num_links}})</th>
-                    <th class="snapshot-tags-cell">Tags</th>
-                    <th class="snapshot-status-cell">Status</th>
-                    <th class="snapshot-files-cell">Files</th>
-                    <th class="snapshot-size-cell">Size</th>
-                </tr>
-            </thead>
-            <tbody>
-                {% for link in links %}
-                   {% snapshot_index_row link %}
-                {% endfor %}
-            </tbody>
-        </table>
-        <footer>
-            <br/>
-            <center>
-                <small>
-                    Archive created using <a href="https://github.com/ArchiveBox/ArchiveBox" title="Github">ArchiveBox</a>
-                    version <a href="https://github.com/ArchiveBox/ArchiveBox/releases/tag/v{{version}}" title="View source code and release info">v{{version}}</a> &nbsp; | &nbsp; 
-                    Download index as <a href="index.json" title="JSON summary of archived links.">JSON</a>
-                    <br/><br/>
-                    {{FOOTER_INFO}}
-                </small>
-            </center>
-            <br/>
-        </footer>
-    </body>
+            img.style.opacity = 0
+        }
+    </script>
+</head>
+<body>
+    <header class="static-header">
+        <a class="static-brand" href="./index.html"><span class="static-brand-mark">▤</span> ArchiveBox</a>
+        <span class="static-meta">Portable static archive · updated {{ time_updated }}</span>
+    </header>
+    <main>
+        <div class="toolbar">
+            <input id="static-search" type="search" placeholder="Filter title, URL, tag, or status…" autocomplete="off">
+            <span class="count"><span id="visible-count">{{ num_links }}</span> of {{ num_links }} snapshots</span>
+        </div>
+        <div class="table-wrap">
+            <table id="table-bookmarks">
+                <thead>
+                    <tr>
+                        <th class="snapshot-time">Saved</th>
+                        <th class="snapshot-preview-cell">Preview</th>
+                        <th>Snapshot ({{ num_links }})</th>
+                        <th class="snapshot-tags-cell">Tags</th>
+                        <th class="snapshot-status-cell">Status</th>
+                        <th class="snapshot-files-cell">Files</th>
+                        <th class="snapshot-size-cell">Size</th>
+                    </tr>
+                </thead>
+                <tbody>
+                    {% for link in links %}{% snapshot_index_row link %}{% endfor %}
+                </tbody>
+            </table>
+        </div>
+    </main>
+    <footer>
+        Generated by <a href="https://github.com/ArchiveBox/ArchiveBox">ArchiveBox v{{ version }}</a>.
+        <a href="./index.jsonl">Download JSONL metadata</a>.<br><br>{{ FOOTER_INFO }}
+    </footer>
+    <script>
+        (() => {
+            const search = document.getElementById('static-search')
+            const rows = [...document.querySelectorAll('#table-bookmarks tbody tr')]
+            const count = document.getElementById('visible-count')
+            search.addEventListener('input', () => {
+                const query = search.value.trim().toLowerCase()
+                let visible = 0
+                rows.forEach((row) => {
+                    const show = !query || row.textContent.toLowerCase().includes(query)
+                    row.hidden = !show
+                    if (show) visible += 1
+                })
+                count.textContent = visible
+            })
+        })()
+    </script>
+</body>
 </html>

archivebox/templates/includes/output_delete_controls.html

@@ -1,3 +1,8 @@
+<style>
+    [data-archive-result-ids] { color: #64748b; }
+    [data-archive-result-ids]:hover,
+    [data-archive-result-ids].delete-pending { color: #b91c1c; }
+</style>
 <script>
     (() => {
         for (const sortButton of document.querySelectorAll('[data-output-size-sort]')) {
@@ -23,8 +28,8 @@
         function renderDeleteCountdown(label='') {
             for (const button of deleteOutputButtons) {
                 const queued = button.dataset.archiveResultIds.split(',').every((id) => queuedOutputIds.has(id))
-                button.textContent = queued ? label : '❌'
-                button.style.color = queued ? '#b91c1c' : ''
+                button.textContent = queued ? label : '×'
+                button.classList.toggle('delete-pending', queued)
             }
         }
 
@@ -50,10 +55,20 @@
         }
 
         function deleteQueuedOutputs() {
-            const csrf = document.querySelector('#delete-output-csrf, input[name="csrfmiddlewaretoken"]')?.value
-            const deleteUrl = deleteOutputButtons.find((button) => button.dataset.deleteUrl)?.dataset.deleteUrl
-            if (!csrf || !deleteUrl) return window.alert('Delete failed: missing admin request data')
+            const button = deleteOutputButtons.find((candidate) => candidate.dataset.deleteUrl)
+            let deleteUrl = button?.dataset.deleteUrl
+            if (!deleteUrl) return window.alert('Delete failed: missing admin request data')
             deleteOutputButtons.forEach((button) => button.disabled = true)
+            if (button.dataset.deleteHandoff) {
+                deleteUrl = new URL(deleteUrl, window.location.href)
+                deleteUrl.searchParams.set('action', 'delete_selected')
+                deleteUrl.searchParams.set('snapshot', button.dataset.deleteSnapshotId)
+                queuedOutputIds.forEach((id) => deleteUrl.searchParams.append('_selected_action', id))
+                window.location.assign(deleteUrl)
+                return
+            }
+            const csrf = document.querySelector('input[name="csrfmiddlewaretoken"]')?.value
+            if (!csrf) return window.alert('Delete failed: missing admin request data')
             const body = new URLSearchParams({action: 'delete_selected', post: 'yes', csrfmiddlewaretoken: csrf})
             queuedOutputIds.forEach((id) => body.append('_selected_action', id))
             fetch(deleteUrl, {

archivebox/templates/static/admin.css

@@ -1461,6 +1461,38 @@ body.model-crawl.change-list #result_list td.field-short_id a {
     white-space: normal;
 }
 
+.archive-results-table .archive-results-actions-cell {
+    width: 100px;
+    min-width: 100px;
+    padding: 10px 8px;
+    white-space: nowrap;
+}
+
+.archive-results-table .archive-results-actions {
+    display: flex;
+    align-items: center;
+    gap: 4px;
+}
+
+.archive-results-table .archive-results-actions a,
+.archive-results-table .archive-results-actions button {
+    display: inline-flex;
+    align-items: center;
+    justify-content: center;
+    margin: 0;
+    width: 28px;
+    height: 28px;
+    padding: 0;
+    border: 0;
+    border-radius: 4px;
+    background: #f1f5f9;
+    color: #475569;
+    font-size: 14px;
+    line-height: 1;
+    text-decoration: none;
+    cursor: pointer;
+}
+
 
 #result_list tbody td.field-cmd_str pre,
 #result_list tbody td.field-output_str pre {

archivebox/tests/conftest.py

@@ -85,7 +85,7 @@ def _set_test_source_pythonpath(env: dict[str, str]) -> None:
         for entry in (env.get("PYTHONPATH") or "").split(os.pathsep)
         if entry and Path(entry).expanduser().is_absolute() and Path(entry).expanduser().exists()
     ]
-    entries = [entry for entry in [*source_pythonpath.split(os.pathsep), *existing_entries] if entry]
+    entries = [entry for entry in [*existing_entries, *source_pythonpath.split(os.pathsep)] if entry]
     if entries:
         env["PYTHONPATH"] = os.pathsep.join(dict.fromkeys(entries))
     else:
@@ -1660,6 +1660,74 @@ def resolve_abxpkg_binary_env(
     return {str(key): str(value) for key, value in payload.items()}
 
 
+def run_test_hook(
+    script: Path,
+    output_dir: Path,
+    config: dict[str, Any] | None = None,
+    timeout: int = 60,
+    **arguments: Any,
+):
+    """Execute a shipped finite hook through abx-dl and ArchiveBox's real DB projector."""
+    import asyncio
+
+    from abx_dl.execution import execute_hook
+    from abx_dl.orchestrator import create_bus
+    from archivebox.machine.models import Process
+    from archivebox.plugins.discovery import get_plugin_catalog
+    from archivebox.services.process_service import ProcessService, parse_event_datetime
+
+    resolved_script = script.resolve()
+    hook = next(
+        (hook for plugin in get_plugin_catalog().values() for hook in plugin.hooks if hook.path.resolve() == resolved_script),
+        None,
+    )
+    assert hook is not None, f"shipped hook is not in the plugin catalog: {script}"
+    assert not hook.is_background, f"run_test_hook only supports finite hooks: {hook.full_name}"
+
+    env = os.environ.copy()
+    for key, value in (config or {}).items():
+        if value is None:
+            continue
+        if isinstance(value, bool):
+            env[key] = "true" if value else "false"
+        elif isinstance(value, (dict, list, tuple)):
+            env[key] = json.dumps(value)
+        else:
+            env[key] = str(value)
+    if env.get("NODE_MODULES_DIR"):
+        env.setdefault("NODE_MODULE_DIR", env["NODE_MODULES_DIR"])
+
+    bus = create_bus(name=f"test_hook_{hook.plugin_name}", total_timeout=float(timeout) + 30.0)
+    ProcessService(bus)
+
+    async def execute_and_close():
+        try:
+            return await execute_hook(
+                hook,
+                output_dir=output_dir,
+                env=env,
+                arguments=arguments,
+                timeout=timeout,
+                bus=bus,
+                process_type=Process.TypeChoices.HOOK,
+            )
+        finally:
+            await bus.wait_until_idle()
+            await bus.destroy(clear=False)
+
+    completed = asyncio.run(execute_and_close())
+    process = (
+        Process.objects.filter(
+            pid=completed.pid or None,
+            started_at=parse_event_datetime(completed.start_ts),
+        )
+        .order_by("-modified_at")
+        .first()
+    )
+    assert process is not None, f"hook completed without an ArchiveBox Process projection: {hook.full_name}"
+    return process
+
+
 def resolve_abxpkg_chrome_env(lib_dir: Path, env: dict[str, str] | None = None) -> dict[str, str]:
     from abx_plugins import get_plugins_dir
 
@@ -1683,7 +1751,7 @@ def install_real_binary(
     binproviders: str = "env",
     overrides: dict[str, dict[str, Any]] | None = None,
 ):
-    """Install and persist a real binary through the normal Binary state machine."""
+    """Install and persist a real binary through the normal Binary lifecycle."""
     from archivebox.machine.models import Binary, Machine
 
     binary = Binary.objects.create(
@@ -1693,7 +1761,7 @@ def install_real_binary(
         overrides=overrides or {},
         status=Binary.StatusChoices.QUEUED,
     )
-    assert binary.tick_claimed(lock_seconds=600)
+    assert binary.install_claimed(lock_seconds=600)
     binary.refresh_from_db()
     assert binary.status == Binary.StatusChoices.INSTALLED
     assert binary.retry_at is None

archivebox/tests/migrations_helpers.py

@@ -13,6 +13,7 @@ import hashlib
 import sqlite3
 from pathlib import Path
 from datetime import datetime, timezone
+from urllib.parse import urlparse
 
 from archivebox.tests.conftest import cli_env, run_archivebox_cmd
 from archivebox.uuid_compat import uuid7
@@ -1272,6 +1273,24 @@ def filesystem_manifest(root: Path) -> dict[str, tuple[str, str | int]]:
     return manifest
 
 
+def current_snapshot_dir(data_dir: Path, db_path: Path, timestamp: str) -> Path:
+    """Resolve a migrated snapshot's canonical archive/users/... directory."""
+    with sqlite3.connect(db_path) as connection:
+        username, bookmarked_at, snapshot_id, url = connection.execute(
+            """
+            SELECT u.username, s.bookmarked_at, s.id, s.url
+            FROM core_snapshot s
+            JOIN crawls_crawl c ON c.id = s.crawl_id
+            JOIN auth_user u ON u.id = c.created_by_id
+            WHERE s.timestamp = ?
+            """,
+            (timestamp,),
+        ).fetchone()
+    date_bucket = datetime.fromisoformat(bookmarked_at).strftime("%Y%m%d")
+    domain = urlparse(url).hostname or "unknown"
+    return data_dir / "archive" / "users" / username / "snapshots" / date_bucket / domain / snapshot_id
+
+
 def verify_snapshot_count(db_path: Path, expected: int) -> tuple[bool, str]:
     """Verify the number of snapshots in the database."""
     conn = sqlite3.connect(str(db_path))

archivebox/tests/test_api_v1_cli_add.py

@@ -264,10 +264,31 @@ def test_basic_success_case_request(client, tmp_path, api_headers):
     assert response.status_code == 200, response.content
     assert response.json()["success"] is True
     crawl = Crawl.objects.get()
-    assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": submitted_url, "depth": 0}
+    assert crawl.urls == submitted_url
     assert Snapshot.objects.count() == 0
 
 
+def test_api_cli_add_filters_invalid_items_from_multi_url_batch(client, tmp_path, api_headers):
+    init_archive(tmp_path)
+    submitted_url = "https://example.com/api-cli-add-valid-batch-item"
+
+    response = api_client_request(
+        client,
+        "post",
+        "/api/v1/cli/add",
+        payload={
+            "urls": [submitted_url, "not a URL", "https://example.org\nhttps://example.net"],
+            "plugins": "__archivebox_test_no_plugins__",
+            "index_only": True,
+        },
+        headers=api_headers,
+    )
+
+    assert response.status_code == 200, response.content
+    assert response.json()["result"]["queued_urls"] == [submitted_url]
+    assert Crawl.objects.get().urls == submitted_url
+
+
 @pytest.mark.timeout(180)
 def test_api_cli_add_concurrent_first_time_default_persona_creation(tmp_path):
     """Concurrent live API add requests should share one first-created Default persona."""
@@ -322,10 +343,7 @@ def test_api_cli_add_concurrent_first_time_default_persona_creation(tmp_path):
         crawls = list(Crawl.objects.order_by("urls").values_list("urls", flat=True))
         assert Snapshot.objects.count() == 0
 
-    expected_crawl_sources = sorted(
-        json.dumps({"type": "CrawlSeed", "url": url, "depth": 0}, separators=(",", ":")) for url in submitted_urls
-    )
-    assert crawls == expected_crawl_sources
+    assert crawls == sorted(submitted_urls)
 
 
 @pytest.mark.timeout(360)
@@ -366,15 +384,10 @@ def test_api_cli_add_import_text_formats_preserve_metadata_and_crawl_inner_urls(
         stop_archivebox_process(api_server)
         api_server = None
         run_queued_crawls(tmp_path, env=env, timeout=240)
-        with use_archivebox_db(tmp_path):
-            root_counts = {
-                str(crawl.id): crawl.snapshot_set.filter(url=Snapshot.INTERNAL_INPUT_URL).count() for crawl in Crawl.objects.all()
-            }
-        assert root_counts and all(count == 1 for count in root_counts.values()), root_counts
         with use_archivebox_db(tmp_path):
             for crawl in Crawl.objects.all():
-                root_snapshot = crawl.snapshot_set.get(url=Snapshot.INTERNAL_INPUT_URL)
-                root_input = (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8")
+                assert not crawl.snapshot_set.filter(url__startswith="archivebox://").exists()
+                root_input = (crawl.output_dir / "input" / "staticfile" / "stdin.txt").read_text(encoding="utf-8")
                 assert root_input == crawl.urls
         api_server = start_api_server_without_runner(tmp_path, env, port)
         assert_expected_import_snapshots(tmp_path, expected_urls)

archivebox/tests/test_api_v1_cli_schedule.py

@@ -34,6 +34,8 @@ def test_schedule_api_creates_schedule_via_view_request(api_admin_user):
 
     assert response["success"] is True
     assert response["result_format"] == "json"
+    assert response["result"]["run_all_enqueued"] == 0
+    assert response["result"]["run_all_maintained"] == 0
     assert CrawlSchedule.objects.count() == 1
     assert len(response["result"]["created_schedule_ids"]) == 1
 

archivebox/tests/test_api_v1_core_archiveresults.py

@@ -3,6 +3,7 @@ from datetime import timedelta
 import pytest
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.db import connection
+from django.test.client import BOUNDARY, MULTIPART_CONTENT, encode_multipart
 from django.test.utils import CaptureQueriesContext
 from django.utils import timezone
 
@@ -47,6 +48,122 @@ def test_archiveresult_upload_upserts_by_snapshot_plugin_and_hook(client, api_ad
     server_result = results.get(hook_name="on_Snapshot__50_screenshot")
     assert set(extension_result.output_files) == {"browser.png", "browser-2.png"}
     assert set(server_result.output_files) == {"server.png"}
+    snapshot.refresh_from_db()
+    assert snapshot.output_size == len(b"browser") + len(b"browser-2") + len(b"server")
+
+
+def test_archiveresult_create_does_not_open_a_database_transaction(client, api_admin_user, api_headers):
+    crawl = Crawl.objects.create(urls="https://example.com", created_by=api_admin_user)
+    snapshot = Snapshot.objects.create(url="https://example.com/autocommit-result", crawl=crawl)
+
+    with CaptureQueriesContext(connection) as queries:
+        response = client.post(
+            "/api/v1/core/archiveresults",
+            {
+                "snapshot_id": str(snapshot.id),
+                "plugin": "chrome_extension_screenshot",
+                "files": SimpleUploadedFile("screenshot.png", b"screenshot", content_type="image/png"),
+                "output_paths": "screenshot.png",
+            },
+            **api_headers,
+        )
+
+    assert response.status_code == 200, response.content
+    if connection.vendor == "sqlite":
+        transaction_queries = [query["sql"] for query in queries if query["sql"].strip().upper() in {"BEGIN", "COMMIT"}]
+        assert transaction_queries == []
+
+
+def test_intermediate_archiveresult_chunks_only_write_to_disk(client, api_admin_user, api_headers):
+    crawl = Crawl.objects.create(urls="https://example.com", created_by=api_admin_user)
+    snapshot = Snapshot.objects.create(url="https://example.com/chunked-result", crawl=crawl)
+    create_response = client.post(
+        "/api/v1/core/archiveresults",
+        {
+            "snapshot_id": str(snapshot.id),
+            "plugin": "chrome_extension_mhtml",
+            "status": ArchiveResult.StatusChoices.STARTED,
+        },
+        **api_headers,
+    )
+    assert create_response.status_code == 200, create_response.content
+    result = ArchiveResult.objects.get(pk=create_response.json()["id"])
+    original_modified_at = result.modified_at
+
+    with CaptureQueriesContext(connection) as queries:
+        chunk_response = client.patch(
+            f"/api/v1/core/archiveresult/{result.id}",
+            encode_multipart(
+                BOUNDARY,
+                {
+                    "files": SimpleUploadedFile("snapshot.mhtml.part-000000", b"first", content_type="multipart/related"),
+                    "chunk_output_path": "snapshot.mhtml",
+                    "chunk_index": "0",
+                    "chunk_count": "2",
+                    "chunk_offset": "0",
+                    "chunk_total_size": "11",
+                    "mime_type": "multipart/related",
+                    "status": ArchiveResult.StatusChoices.STARTED,
+                },
+            ),
+            content_type=MULTIPART_CONTENT,
+            **api_headers,
+        )
+
+    assert chunk_response.status_code == 200, chunk_response.content
+    result.refresh_from_db()
+    assert result.output_files == {}
+    assert result.output_size == 0
+    assert result.modified_at == original_modified_at
+    writes = [query["sql"] for query in queries if query["sql"].lstrip().upper().startswith(("INSERT", "UPDATE", "DELETE"))]
+    assert writes == []
+
+    final_response = client.patch(
+        f"/api/v1/core/archiveresult/{result.id}",
+        encode_multipart(
+            BOUNDARY,
+            {
+                "files": SimpleUploadedFile("snapshot.mhtml.part-000001", b"-final", content_type="multipart/related"),
+                "chunk_output_path": "snapshot.mhtml",
+                "chunk_index": "1",
+                "chunk_count": "2",
+                "chunk_offset": "5",
+                "chunk_total_size": "11",
+                "mime_type": "multipart/related",
+                "status": ArchiveResult.StatusChoices.SUCCEEDED,
+            },
+        ),
+        content_type=MULTIPART_CONTENT,
+        **api_headers,
+    )
+    assert final_response.status_code == 200, final_response.content
+    result.refresh_from_db()
+    assert result.status == ArchiveResult.StatusChoices.SUCCEEDED
+    assert result.output_size == 11
+    assert result.output_files["snapshot.mhtml"]["upload"]["complete"] is True
+    snapshot.refresh_from_db()
+    assert snapshot.output_size == 11
+
+
+def test_archiveresult_safe_update_rejects_stale_writers(api_admin_user):
+    crawl = Crawl.objects.create(urls="https://example.com", created_by=api_admin_user)
+    snapshot = Snapshot.objects.create(url="https://example.com/cas-result", crawl=crawl)
+    result = ArchiveResult.objects.create(
+        snapshot=snapshot,
+        plugin="chrome_extension_dom",
+        hook_name=Snapshot.BROWSER_EXTENSION_UPLOAD_HOOK_NAME,
+        status=ArchiveResult.StatusChoices.STARTED,
+        output_size=1,
+    )
+    stale_result = ArchiveResult.objects.get(pk=result.pk)
+
+    assert result.safe_update({"output_size": 2}) is True
+    assert stale_result.safe_update({"output_size": 3}) is False
+
+    stale_result.refresh_from_db()
+    snapshot.refresh_from_db()
+    assert stale_result.output_size == 2
+    assert snapshot.output_size == 2
 
 
 def test_archiveresult_upload_api_queues_snapshot_maintenance_without_finalizing(client, api_admin_user, api_headers):

archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py

@@ -102,7 +102,7 @@ def test_snapshot_pause_wins_over_concurrent_runner_lease(api_admin_user):
     assert snapshot.retry_at == RETRY_AT_MAX
 
 
-def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_finished_rows(
+def test_snapshot_pause_resume_api_leaves_archiveresult_facts_unchanged(
     request,
     tmp_path,
     client,
@@ -159,10 +159,10 @@ def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_
             url=blocking_http_server.url,
             status=Snapshot.StatusChoices.QUEUED,
             retry_at=now,
+            config={"PLUGINS": "wget"},
         )
         Crawl.objects.filter(pk=snapshot.crawl_id).update(status=Crawl.StatusChoices.STARTED, retry_at=now)
         snapshot.refresh_from_db()
-        [started_result] = snapshot.create_pending_archiveresults(hooks=[("wget", _snapshot_hook_name("wget"))])
         errors = []
 
         def run_snapshot():
@@ -185,10 +185,13 @@ def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_
         request.addfinalizer(finish_runner)
         blocking_http_server.request_started.wait()
         assert errors == []
-        started_result.refresh_from_db()
+        started_result = ArchiveResult.objects.get(snapshot=snapshot, plugin="wget")
         assert started_result.status == ArchiveResult.StatusChoices.STARTED
-        [queued_result] = snapshot.create_pending_archiveresults(
-            hooks=[("parse_txt_urls", "on_Snapshot__71_parse_txt_urls")],
+        queued_result = ArchiveResult.objects.create(
+            snapshot=snapshot,
+            plugin="parse_txt_urls",
+            hook_name="on_Snapshot__71_parse_txt_urls",
+            status=ArchiveResult.StatusChoices.QUEUED,
         )
 
         invalid_response = api_client_request(
@@ -222,8 +225,8 @@ def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_
             row.plugin: (row.status, row.retry_at) for row in ArchiveResult.objects.filter(id__in=[queued_result.id, started_result.id])
         }
         assert active_rows == {
-            "parse_txt_urls": (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX),
-            "wget": (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX),
+            "parse_txt_urls": (ArchiveResult.StatusChoices.QUEUED, None),
+            "wget": (ArchiveResult.StatusChoices.STARTED, None),
         }
 
         finished_rows = {
@@ -262,12 +265,7 @@ def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_
         resumed_rows = {
             row.plugin: (row.status, row.retry_at) for row in ArchiveResult.objects.filter(id__in=[queued_result.id, started_result.id])
         }
-        assert resumed_rows["parse_txt_urls"][0] == ArchiveResult.StatusChoices.QUEUED
-        assert resumed_rows["parse_txt_urls"][1] is not None
-        assert resumed_rows["parse_txt_urls"][1] != RETRY_AT_MAX
-        assert resumed_rows["wget"][0] == ArchiveResult.StatusChoices.QUEUED
-        assert resumed_rows["wget"][1] is not None
-        assert resumed_rows["wget"][1] != RETRY_AT_MAX
+        assert resumed_rows == active_rows
 
         assert ArchiveResult.objects.get(id=succeeded_result.id).status == ArchiveResult.StatusChoices.SUCCEEDED
         assert ArchiveResult.objects.get(id=failed_result.id).status == ArchiveResult.StatusChoices.FAILED
@@ -317,8 +315,11 @@ def test_targeted_extract_retries_one_failed_archiveresult_through_normal_snapsh
         assert "wget failed (exit=4)" in wget_result.output_str
         Snapshot.objects.filter(pk=snapshot.pk).update(url=recursive_test_site["root_url"])
         snapshot.refresh_from_db()
-        [unrelated_result] = snapshot.create_pending_archiveresults(
-            hooks=[("parse_txt_urls", "on_Snapshot__71_parse_txt_urls")],
+        unrelated_result = ArchiveResult.objects.create(
+            snapshot=snapshot,
+            plugin="parse_txt_urls",
+            hook_name="on_Snapshot__71_parse_txt_urls",
+            status=ArchiveResult.StatusChoices.QUEUED,
         )
         snapshot.output_dir.mkdir(parents=True, exist_ok=True)
         (snapshot.output_dir / "source.txt").write_text("finished row must survive targeted retry", encoding="utf-8")
@@ -343,7 +344,7 @@ def test_targeted_extract_retries_one_failed_archiveresult_through_normal_snapsh
         assert snapshot.status == Snapshot.StatusChoices.PAUSED
         assert snapshot.retry_at == RETRY_AT_MAX
         assert ArchiveResult.objects.get(id=wget_result.id).status == ArchiveResult.StatusChoices.FAILED
-        assert ArchiveResult.objects.get(id=unrelated_result.id).status == ArchiveResult.StatusChoices.PAUSED
+        assert ArchiveResult.objects.get(id=unrelated_result.id).status == ArchiveResult.StatusChoices.QUEUED
         finished_row = ArchiveResult.objects.get(id=finished_result.id)
         finished_output_path = Path(snapshot.output_dir) / finished_row.plugin / next(iter(finished_row.output_files))
         assert finished_output_path.is_file()
@@ -365,10 +366,9 @@ def test_targeted_extract_retries_one_failed_archiveresult_through_normal_snapsh
 
     with use_archivebox_db(tmp_path):
         snapshot = Snapshot.objects.get(id=snapshot_id)
-        assert snapshot.status == Snapshot.StatusChoices.STARTED
-        assert snapshot.retry_at is not None
-        assert snapshot.retry_at != RETRY_AT_MAX
-        assert snapshot.crawl.status == snapshot.crawl.StatusChoices.STARTED
+        assert snapshot.status == Snapshot.StatusChoices.SEALED
+        assert snapshot.retry_at is None
+        assert snapshot.crawl.status == snapshot.crawl.StatusChoices.SEALED
 
         retried_wget = ArchiveResult.objects.get(id=wget_result.id)
         assert retried_wget.status == ArchiveResult.StatusChoices.SUCCEEDED
@@ -376,8 +376,8 @@ def test_targeted_extract_retries_one_failed_archiveresult_through_normal_snapsh
         assert retried_wget.output_files
 
         unrelated = ArchiveResult.objects.get(id=unrelated_result.id)
-        assert unrelated.status == ArchiveResult.StatusChoices.PAUSED
-        assert unrelated.retry_at == RETRY_AT_MAX
+        assert unrelated.status == ArchiveResult.StatusChoices.QUEUED
+        assert unrelated.retry_at is None
 
         finished = ArchiveResult.objects.get(id=finished_result.id)
         assert finished.status == ArchiveResult.StatusChoices.SUCCEEDED

archivebox/tests/test_api_v1_core_snapshots.py

@@ -2,6 +2,8 @@ import threading
 import time
 
 import pytest
+from django.db import connection
+from django.test.utils import CaptureQueriesContext
 
 from archivebox.core.models import Snapshot
 from archivebox.crawls.locks import crawl_lifecycle_lock
@@ -85,3 +87,66 @@ def test_existing_snapshot_metadata_sync_does_not_wait_for_active_crawl(client,
     assert response.status_code == 200, response.content
     assert response.json()["id"] == str(snapshot.id)
     assert elapsed < 1
+
+
+def test_new_snapshot_creation_does_not_open_a_database_transaction(client, api_admin_user, api_headers):
+    url = "https://example.com/browser-extension-new-snapshot"
+    crawl = Crawl.objects.create(urls=url, created_by=api_admin_user)
+
+    with CaptureQueriesContext(connection) as queries:
+        response = client.post(
+            "/api/v1/core/snapshots",
+            data={
+                "url": url,
+                "crawl_id": str(crawl.id),
+                "depth": 0,
+                "status": Snapshot.StatusChoices.QUEUED,
+                "tags": ["browser-extension-upload"],
+            },
+            content_type="application/json",
+            **api_headers,
+        )
+
+    assert response.status_code == 200, response.content
+    assert Snapshot.objects.filter(url=url, crawl=crawl).count() == 1
+    assert Snapshot.objects.get(url=url, crawl=crawl).tags.filter(name="browser-extension-upload").exists()
+    if connection.vendor == "sqlite":
+        transaction_queries = [query["sql"] for query in queries if query["sql"].strip().upper() in {"BEGIN", "COMMIT"}]
+        assert transaction_queries == []
+
+
+def test_new_snapshot_creation_does_not_wait_for_active_crawl(client, api_admin_user, api_headers):
+    url = "https://example.com/browser-extension-new-snapshot-active-crawl"
+    crawl = Crawl.objects.create(urls=url, created_by=api_admin_user)
+    lock_acquired = threading.Event()
+    release_lock = threading.Event()
+
+    def hold_active_crawl_lock():
+        with crawl_lifecycle_lock(str(crawl.id)):
+            lock_acquired.set()
+            release_lock.wait(timeout=3)
+
+    holder = threading.Thread(target=hold_active_crawl_lock)
+    holder.start()
+    assert lock_acquired.wait(timeout=1)
+
+    started_at = time.monotonic()
+    response = client.post(
+        "/api/v1/core/snapshots",
+        data={
+            "url": url,
+            "crawl_id": str(crawl.id),
+            "depth": 0,
+            "status": Snapshot.StatusChoices.QUEUED,
+            "tags": ["browser-extension-upload"],
+        },
+        content_type="application/json",
+        **api_headers,
+    )
+    elapsed = time.monotonic() - started_at
+    release_lock.set()
+    holder.join(timeout=3)
+
+    assert response.status_code == 200, response.content
+    assert Snapshot.objects.filter(url=url, crawl=crawl).count() == 1
+    assert elapsed < 1

archivebox/tests/test_api_v1_core_tags_add_to_snapshot.py

@@ -1,4 +1,6 @@
 import pytest
+from django.db import IntegrityError, connection
+from django.test.utils import CaptureQueriesContext
 
 from archivebox.core.models import Snapshot, Tag
 from archivebox.crawls.models import Crawl
@@ -13,13 +15,39 @@ def test_basic_success_case_request(client, api_admin_user, api_headers):
     snapshot = Snapshot.objects.create(url="https://example.com/tag-add", crawl=crawl)
     tag = Tag.objects.create(name="api-basic-add-tag", created_by=api_admin_user)
 
-    response = api_client_request(
-        client,
-        "post",
-        "/api/v1/core/tags/add-to-snapshot/",
-        payload={"snapshot_id": str(snapshot.id), "tag_id": tag.id},
-        headers=api_headers,
-    )
+    with CaptureQueriesContext(connection) as queries:
+        response = api_client_request(
+            client,
+            "post",
+            "/api/v1/core/tags/add-to-snapshot/",
+            payload={"snapshot_id": str(snapshot.id), "tag_id": tag.id},
+            headers=api_headers,
+        )
 
     assert response.status_code == 200, response.content
     assert response.json()["success"] is True
+    assert snapshot.tags.filter(pk=tag.pk).exists()
+    if connection.vendor == "sqlite":
+        transaction_queries = [query["sql"] for query in queries if query["sql"].strip().upper() in {"BEGIN", "COMMIT"}]
+        assert transaction_queries == []
+
+
+def test_add_tag_ids_reraises_non_duplicate_integrity_errors(snapshot):
+    with pytest.raises(IntegrityError):
+        snapshot.add_tag_ids([2**31 - 1])
+
+
+def test_add_tag_ids_treats_existing_snapshot_tag_as_idempotent(snapshot, admin_user):
+    tag = Tag.objects.create(name="already-attached", created_by=admin_user)
+
+    snapshot.add_tag_ids([tag.pk])
+    snapshot.add_tag_ids([tag.pk])
+
+    assert snapshot.tags.filter(pk=tag.pk).count() == 1
+
+
+def test_save_tags_without_creator_uses_tag_model_default(snapshot):
+    snapshot.save_tags(["model-default-creator"])
+
+    tag = Tag.objects.get(name="model-default-creator")
+    assert tag.created_by_id is not None

archivebox/tests/test_api_v1_core_tags_remove_from_snapshot.py

@@ -1,4 +1,6 @@
 import pytest
+from django.db import connection
+from django.test.utils import CaptureQueriesContext
 
 from archivebox.core.models import Snapshot, Tag
 from archivebox.crawls.models import Crawl
@@ -14,13 +16,18 @@ def test_basic_success_case_request(client, api_admin_user, api_headers):
     tag = Tag.objects.create(name="api-basic-remove-tag", created_by=api_admin_user)
     snapshot.tags.add(tag)
 
-    response = api_client_request(
-        client,
-        "post",
-        "/api/v1/core/tags/remove-from-snapshot/",
-        payload={"snapshot_id": str(snapshot.id), "tag_id": tag.id},
-        headers=api_headers,
-    )
+    with CaptureQueriesContext(connection) as queries:
+        response = api_client_request(
+            client,
+            "post",
+            "/api/v1/core/tags/remove-from-snapshot/",
+            payload={"snapshot_id": str(snapshot.id), "tag_id": tag.id},
+            headers=api_headers,
+        )
 
     assert response.status_code == 200, response.content
     assert response.json()["success"] is True
+    assert not snapshot.tags.filter(pk=tag.pk).exists()
+    if connection.vendor == "sqlite":
+        transaction_queries = [query["sql"] for query in queries if query["sql"].strip().upper() in {"BEGIN", "COMMIT"}]
+        assert transaction_queries == []

archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py

@@ -12,7 +12,7 @@ from django.utils import timezone
 from archivebox.core.models import ArchiveResult, Snapshot
 from archivebox.crawls.models import Crawl
 from archivebox.tests.test_orm_helpers import use_archivebox_db
-from archivebox.tests.test_archive_result_service import _run_shipped_snapshot_hook, _snapshot_hook_name
+from archivebox.tests.test_archive_result_service import _run_shipped_snapshot_hook
 from archivebox.workers.models import RETRY_AT_MAX
 
 from .conftest import (
@@ -114,6 +114,44 @@ def test_basic_success_case_request(client, tmp_path, api_admin_user, api_header
     assert response.status_code == 200, response.content
 
 
+def test_create_crawl_rejects_multiline_url_item(client, api_headers):
+    response = client.post(
+        "/api/v1/crawls/crawls",
+        data=json.dumps({"urls": ["https://example.com/one\nhttps://example.net/two"]}),
+        content_type="application/json",
+        **api_headers,
+    )
+
+    assert response.status_code == 400, response.content
+    assert not Crawl.objects.exists()
+
+
+def test_create_crawl_marks_structured_urls_as_url_list(client, api_headers):
+    response = client.post(
+        "/api/v1/crawls/crawls",
+        data=json.dumps({"urls": ["https://example.com/one"], "config": {"PLUGINS": "title"}}),
+        content_type="application/json",
+        **api_headers,
+    )
+
+    assert response.status_code == 200, response.content
+    crawl = Crawl.objects.get()
+    assert crawl.urls == "https://example.com/one"
+    assert crawl.config["PARSER"] == "url_list"
+
+
+def test_create_crawl_preserves_explicit_parser(client, api_headers):
+    response = client.post(
+        "/api/v1/crawls/crawls",
+        data=json.dumps({"urls": ["https://example.com/one"], "config": {"PARSER": "txt"}}),
+        content_type="application/json",
+        **api_headers,
+    )
+
+    assert response.status_code == 200, response.content
+    assert Crawl.objects.get().config["PARSER"] == "txt"
+
+
 def test_crawl_pause_wins_over_concurrent_runner_lease(api_admin_user):
     crawl = Crawl.objects.create(urls="https://example.com/crawl-pause-race", created_by=api_admin_user)
     claimed_until = timezone.now() + timedelta(seconds=60)
@@ -126,7 +164,7 @@ def test_crawl_pause_wins_over_concurrent_runner_lease(api_admin_user):
     assert crawl.retry_at == RETRY_AT_MAX
 
 
-def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snapshot_results_alone(
+def test_crawl_pause_resume_api_leaves_archiveresult_facts_unchanged(
     request,
     tmp_path,
     client,
@@ -195,7 +233,7 @@ def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snap
             hook_name="on_Snapshot__93_hashes.py",
             lib_dir=lib_dir,
         )
-        sealed_snapshot.sm.seal()
+        sealed_snapshot.seal()
         sealed_snapshot.refresh_from_db()
         assert sealed_snapshot.status == Snapshot.StatusChoices.SEALED
         assert sealed_snapshot.retry_at is None
@@ -210,9 +248,12 @@ def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snap
         )
         now = timezone.now()
         Crawl.objects.filter(pk=crawl_id).update(status=Crawl.StatusChoices.STARTED, retry_at=now)
-        Snapshot.objects.filter(pk=active_snapshot.pk).update(status=Snapshot.StatusChoices.QUEUED, retry_at=now)
+        Snapshot.objects.filter(pk=active_snapshot.pk).update(
+            status=Snapshot.StatusChoices.QUEUED,
+            retry_at=now,
+            config={"PLUGINS": "wget"},
+        )
         active_snapshot.refresh_from_db()
-        [active_started] = active_snapshot.create_pending_archiveresults(hooks=[("wget", _snapshot_hook_name("wget"))])
         errors = []
 
         def run_snapshot():
@@ -235,10 +276,13 @@ def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snap
         request.addfinalizer(finish_runner)
         blocking_http_server.request_started.wait()
         assert errors == []
-        active_started.refresh_from_db()
+        active_started = ArchiveResult.objects.get(snapshot=active_snapshot, plugin="wget")
         assert active_started.status == ArchiveResult.StatusChoices.STARTED
-        [active_queued] = active_snapshot.create_pending_archiveresults(
-            hooks=[("parse_txt_urls", "on_Snapshot__71_parse_txt_urls")],
+        active_queued = ArchiveResult.objects.create(
+            snapshot=active_snapshot,
+            plugin="parse_txt_urls",
+            hook_name="on_Snapshot__71_parse_txt_urls",
+            status=ArchiveResult.StatusChoices.QUEUED,
         )
         pause_response = api_client_request(
             client,
@@ -260,11 +304,11 @@ def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snap
         assert sealed_snapshot.status == Snapshot.StatusChoices.SEALED
         assert sealed_snapshot.retry_at is None
 
-        paused_rows = {
+        unchanged_rows = {
             row.plugin: (row.status, row.retry_at) for row in ArchiveResult.objects.filter(id__in=[active_queued.id, active_started.id])
         }
-        assert paused_rows["parse_txt_urls"] == (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX)
-        assert paused_rows["wget"] == (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX)
+        assert unchanged_rows["parse_txt_urls"] == (ArchiveResult.StatusChoices.QUEUED, None)
+        assert unchanged_rows["wget"] == (ArchiveResult.StatusChoices.STARTED, None)
 
         active_done_row = ArchiveResult.objects.get(id=active_done.id)
         sealed_done_row = ArchiveResult.objects.get(id=sealed_done.id)
@@ -302,12 +346,7 @@ def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snap
         resumed_rows = {
             row.plugin: (row.status, row.retry_at) for row in ArchiveResult.objects.filter(id__in=[active_queued.id, active_started.id])
         }
-        assert resumed_rows["parse_txt_urls"][0] == ArchiveResult.StatusChoices.QUEUED
-        assert resumed_rows["parse_txt_urls"][1] is not None
-        assert resumed_rows["parse_txt_urls"][1] != RETRY_AT_MAX
-        assert resumed_rows["wget"][0] == ArchiveResult.StatusChoices.QUEUED
-        assert resumed_rows["wget"][1] is not None
-        assert resumed_rows["wget"][1] != RETRY_AT_MAX
+        assert resumed_rows == unchanged_rows
         assert ArchiveResult.objects.get(id=active_done.id).status == ArchiveResult.StatusChoices.SUCCEEDED
         assert ArchiveResult.objects.get(id=sealed_done.id).status == ArchiveResult.StatusChoices.SUCCEEDED
         assert active_done_path.is_file()

archivebox/tests/test_archive_result_service.py

@@ -19,9 +19,9 @@ pytestmark = pytest.mark.django_db(transaction=True)
 
 
 def _snapshot_hook_name(plugin_name: str) -> str:
-    from abx_dl.models import discover_plugins
+    from archivebox.plugins.discovery import get_plugin_catalog
 
-    plugin = discover_plugins().get(plugin_name)
+    plugin = get_plugin_catalog().get(plugin_name)
     assert plugin is not None, f"missing test plugin {plugin_name}"
     hooks = plugin.filter_hooks("Snapshot")
     assert hooks, f"missing Snapshot hooks for {plugin_name}"
@@ -47,15 +47,16 @@ def _run_shipped_snapshot_hook(
     """Run one shipped hook through the production process/result bus services."""
     import asyncio
 
-    from abx_dl.models import discover_plugins
     from abx_dl.services.process_service import ProcessService as HookProcessService
+    from abx_dl.services.archive_result_service import ArchiveResultService as HookArchiveResultService
     from abx_plugins.plugins.base.utils import get_hydrated_required_binaries
     from archivebox.core.models import ArchiveResult
     from archivebox.machine.models import Process
+    from archivebox.plugins.discovery import get_plugin_catalog
     from archivebox.services.archive_result_service import ArchiveResultService
     from archivebox.services.process_service import ProcessService as PersistedProcessService
 
-    discovered_plugin = discover_plugins().get(plugin)
+    discovered_plugin = get_plugin_catalog().get(plugin)
     assert discovered_plugin is not None, f"missing test plugin {plugin}"
     matching_hooks = [hook for hook in discovered_plugin.filter_hooks("Snapshot") if hook.name == hook_name or hook.path.name == hook_name]
     assert len(matching_hooks) == 1, f"missing or ambiguous Snapshot hook {plugin}:{hook_name}"
@@ -76,6 +77,7 @@ def _run_shipped_snapshot_hook(
     output_dir.mkdir(parents=True, exist_ok=True)
     bus = create_bus(name=f"test_real_{plugin}_{snapshot.id}")
     HookProcessService(bus, emit_jsonl=False, interactive_tty=False)
+    HookArchiveResultService(bus, emit_jsonl=False)
     PersistedProcessService(bus)
     ArchiveResultService(bus)
 
@@ -251,6 +253,32 @@ def test_archiveresult_duplicate_hook_rows_are_rejected():
         )
 
 
+def test_archiveresult_event_create_uses_one_result_lookup():
+    from abx_dl.events import ArchiveResultEvent
+    from django.db import connection
+    from django.test.utils import CaptureQueriesContext
+
+    from archivebox.services.archive_result_service import _save_archiveresult_event_to_db
+
+    snapshot = _create_snapshot()
+    event = ArchiveResultEvent(
+        snapshot_id=str(snapshot.id),
+        plugin="review-query-count",
+        hook_name="on_Snapshot__99_review.py",
+        status="failed",
+    )
+
+    with CaptureQueriesContext(connection) as queries:
+        _save_archiveresult_event_to_db(event, None)
+
+    result_lookups = [
+        query["sql"]
+        for query in queries
+        if query["sql"].lstrip().upper().startswith("SELECT") and 'FROM "core_archiveresult"' in query["sql"]
+    ]
+    assert len(result_lookups) == 1
+
+
 def test_process_completed_projects_failed_archiveresult_from_shipped_hook(tmp_path, hermetic_lib_dir):
     from archivebox.core.models import ArchiveResult
 
@@ -265,7 +293,6 @@ def test_process_completed_projects_failed_archiveresult_from_shipped_hook(tmp_p
     assert result.status == ArchiveResult.StatusChoices.FAILED
     assert result.process_id == process.id
     assert "Chrome session" in result.output_str
-    assert result.output_str in result.notes
     _cleanup_machine_process_rows()
 
 
@@ -438,15 +465,13 @@ def test_retry_failed_archiveresults_requeues_snapshot_in_queued_state():
     assert reset_count == 1
     assert snapshot.status == Snapshot.StatusChoices.QUEUED
     assert snapshot.retry_at is not None
-    assert snapshot.current_step == 0
-    assert result.status == ArchiveResult.StatusChoices.QUEUED
-    assert result.output_str == ""
-    assert result.output_json is None
-    assert result.output_files == {}
-    assert result.output_size == 0
-    assert result.output_mimetypes == ""
-    assert result.start_ts is None
-    assert result.end_ts is None
+    assert snapshot.config["RETRY_PLUGINS"] == ["chrome"]
+    assert "PLUGINS" not in snapshot.config
+    assert result.status == ArchiveResult.StatusChoices.FAILED
+    assert result.output_str == "timed out"
+    assert result.output_files == {"stderr.log": {}}
+    assert result.output_size == 123
+    assert result.output_mimetypes == "text/plain"
     assert ArchiveResult.objects.get(snapshot=snapshot, plugin="ublock").status == ArchiveResult.StatusChoices.SKIPPED
     assert ArchiveResult.objects.get(snapshot=snapshot, plugin="forumdl").status == ArchiveResult.StatusChoices.NORESULTS
     snapshot.refresh_from_db()
@@ -517,14 +542,17 @@ def test_collect_output_metadata_preserves_file_metadata():
 
 
 def test_collect_output_metadata_detects_warc_gz_mimetype(tmp_path):
-    from archivebox.services.archive_result_service import _collect_output_metadata
+    from abx_dl.output_files import OutputManifest
 
     plugin_dir = tmp_path / "wget"
     warc_file = plugin_dir / "warc" / "capture.warc.gz"
     warc_file.parent.mkdir(parents=True, exist_ok=True)
     warc_file.write_bytes(b"warc-bytes")
 
-    output_files, output_size, output_mimetypes = _collect_output_metadata(plugin_dir)
+    manifest = OutputManifest.scan(plugin_dir)
+    output_files = manifest.as_mapping()
+    output_size = manifest.total_size
+    output_mimetypes = ",".join(manifest.mimetypes)
 
     assert output_files["warc/capture.warc.gz"] == {
         "extension": "gz",

archivebox/tests/test_binary_service.py

@@ -22,13 +22,13 @@ def _runtime_env(data_dir: Path, *, lib_dir: Path | None = None, **extra: str) -
     }
 
 
-def _run_real_binary_state_machine(data_dir: Path, *, name: str, binproviders: str, env: dict[str, str]):
-    """Run a real Binary model through its abxpkg-backed state machine."""
+def _run_real_binary_lifecycle(data_dir: Path, *, name: str, binproviders: str, env: dict[str, str]):
+    """Run a real Binary model through its abxpkg-backed lifecycle."""
     script = (
         "from archivebox.machine.models import Binary, Machine; "
         f"binary = Binary.objects.create(machine=Machine.current(), name={name!r}, binproviders={binproviders!r}, status=Binary.StatusChoices.QUEUED); "
-        "assert binary.tick_claimed(lock_seconds=600); "
-        "print('BINARY_STATE_MACHINE_E2E_DONE')"
+        "assert binary.install_claimed(lock_seconds=600); "
+        "print('BINARY_LIFECYCLE_E2E_DONE')"
     )
     return run_archivebox_cmd(
         ["shell", "-c", script],
@@ -59,7 +59,7 @@ def test_binary_request_preserves_native_overrides_in_db():
         status=Binary.StatusChoices.QUEUED,
         retry_at=timezone.now(),
     )
-    assert binary.tick_claimed(lock_seconds=600)
+    assert binary.install_claimed(lock_seconds=600)
     binary.refresh_from_db()
     assert binary.status == Binary.StatusChoices.INSTALLED
     assert Path(binary.abspath).resolve() == Path(sys.executable).resolve()
@@ -98,11 +98,11 @@ def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized
     host_binary = shutil.which(name)
     assert host_binary is not None
     runtime_env = _runtime_env(initialized_archive)
-    _cmd_result = _run_real_binary_state_machine(initialized_archive, name=name, binproviders="env", env=runtime_env)
+    _cmd_result = _run_real_binary_lifecycle(initialized_archive, name=name, binproviders="env", env=runtime_env)
     stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
     assert returncode == 0, stderr
-    assert "BINARY_STATE_MACHINE_E2E_DONE" in stdout
+    assert "BINARY_LIFECYCLE_E2E_DONE" in stdout
 
     with use_archivebox_db(initialized_archive):
         binary = Binary.objects.get(name=name)
@@ -203,7 +203,7 @@ def test_missing_binary_request_stays_queued_then_recovers_when_provider_can_res
     provider_bin_dir = initialized_archive / "lib" / "pip" / "venv" / "bin"
     runtime_env = _runtime_env(initialized_archive)
 
-    _cmd_result = _run_real_binary_state_machine(initialized_archive, name=name, binproviders="env", env=runtime_env)
+    _cmd_result = _run_real_binary_lifecycle(initialized_archive, name=name, binproviders="env", env=runtime_env)
     stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
     assert returncode != 0, stdout + stderr

archivebox/tests/test_cli_add.py

@@ -225,7 +225,7 @@ def test_add_single_url_records_url_in_crawl(initialized_archive):
         crawl = Crawl.objects.get()
         snapshots = list(Snapshot.objects.all())
 
-    assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": "https://example.com", "depth": 0}
+    assert crawl.urls == "https://example.com"
     assert crawl.get_urls_list() == ["https://example.com"]
     assert snapshots == []
 
@@ -295,13 +295,10 @@ def test_add_stdin_import_formats_preserve_metadata_and_crawl_inner_urls(initial
         assert crawl.urls == source_text
 
     run_queued_crawls(initialized_archive, env=env, timeout=240)
-    with use_archivebox_db(initialized_archive):
-        root_counts = {str(crawl.id): crawl.snapshot_set.filter(url=Snapshot.INTERNAL_INPUT_URL).count() for crawl in Crawl.objects.all()}
-    assert root_counts and all(count == 1 for count in root_counts.values()), root_counts
     with use_archivebox_db(initialized_archive):
         for crawl in Crawl.objects.all():
-            root_snapshot = crawl.snapshot_set.get(url=Snapshot.INTERNAL_INPUT_URL)
-            root_input = (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8")
+            assert not crawl.snapshot_set.filter(url__startswith="archivebox://").exists()
+            root_input = (crawl.output_dir / "input" / "staticfile" / "stdin.txt").read_text(encoding="utf-8")
             assert root_input == crawl.urls
     assert_expected_import_snapshots(initialized_archive, expected_urls)
 
@@ -387,7 +384,7 @@ def test_run_rejects_file_url_injected_directly_into_crawl_urls_with_db_update(i
             status=Crawl.StatusChoices.QUEUED,
             retry_at=timezone.now(),
         )
-        bad_jsonl = json.dumps({"type": "CrawlSeed", "url": file_url, "depth": 0, "tags": "sql-file-url"})
+        bad_jsonl = json.dumps({"type": "Snapshot", "url": file_url, "depth": 0, "tags": "sql-file-url"})
         Crawl.objects.filter(pk=crawl.pk).update(urls=bad_jsonl)
 
     result = run_archivebox_cmd(
@@ -480,7 +477,7 @@ def test_add_bg_queues_direct_url_snapshot(initialized_archive):
 
     assert crawl.status == Crawl.StatusChoices.QUEUED
     assert crawl.retry_at is not None
-    assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": "https://example.com", "depth": 0}
+    assert crawl.urls == "https://example.com"
     assert snapshots == []
 
 
@@ -622,7 +619,7 @@ def test_add_index_only_rejected_urls_leave_empty_crawl_for_runner_to_seal(initi
 
     assert crawl.status == Crawl.StatusChoices.QUEUED
     assert crawl.retry_at is None
-    assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": "https://example.com", "depth": 0}
+    assert crawl.urls == "https://example.com"
     assert snapshot_urls == set()
 
     run_queued_crawls(initialized_archive, env)
@@ -633,7 +630,7 @@ def test_add_index_only_rejected_urls_leave_empty_crawl_for_runner_to_seal(initi
 
     assert crawl.status == Crawl.StatusChoices.SEALED
     assert crawl.retry_at is None
-    assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": "https://example.com", "depth": 0}
+    assert crawl.urls == "https://example.com"
     assert snapshot_urls == set()
 
 
@@ -658,9 +655,7 @@ def test_add_index_only_rejects_archivebox_internal_urls(initialized_archive):
         crawl = Crawl.objects.get()
         snapshot_urls = set(Snapshot.objects.values_list("url", flat=True))
 
-    assert [json.loads(line) for line in crawl.urls.splitlines()] == [
-        {"type": "CrawlSeed", "url": url, "depth": 0} for url in internal_urls
-    ]
+    assert crawl.urls.splitlines() == internal_urls
     assert crawl.status == Crawl.StatusChoices.QUEUED
     assert crawl.retry_at is None
     assert snapshot_urls == set()
@@ -713,10 +708,7 @@ def test_add_multiple_urls_single_command(initialized_archive):
         crawl = Crawl.objects.get()
         snapshots = list(Snapshot.objects.order_by("url").values_list("url", "depth"))
 
-    assert [json.loads(line) for line in crawl.urls.splitlines()] == [
-        {"type": "CrawlSeed", "url": "https://example.com", "depth": 0},
-        {"type": "CrawlSeed", "url": "https://example.org", "depth": 0},
-    ]
+    assert crawl.urls.splitlines() == ["https://example.com", "https://example.org"]
     assert snapshots == []
 
 
@@ -994,7 +986,7 @@ def test_add_index_only_queues_crawl_without_starting_runner(initialized_archive
 
     assert crawl.status == Crawl.StatusChoices.QUEUED
     assert crawl.retry_at is None
-    assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": "https://example.com", "depth": 0}
+    assert crawl.urls == "https://example.com"
     assert snapshots == []
 
 
@@ -1012,7 +1004,7 @@ def test_add_index_only_creates_direct_url_snapshot(initialized_archive):
         crawl = Crawl.objects.get()
         snapshot = Snapshot.objects.get()
 
-    assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": "https://example.com", "depth": 0}
+    assert crawl.urls == "https://example.com"
     assert snapshot.url == "https://example.com"
     assert snapshot.depth == 0
 

archivebox/tests/test_cli_archiveresult.py

@@ -61,7 +61,10 @@ class TestArchiveResultCreate:
 
         ar = next(r for r in records if r["type"] == "ArchiveResult")
         assert ar["plugin"] == "title"
-        assert ar["hook_name"].startswith("on_Snapshot__")
+        # Hook identity is known at scheduling time and is the durable retry
+        # key. Emitting it here prevents multiple hooks in one plugin from
+        # collapsing into the same ArchiveResult row.
+        assert ar["hook_name"] == "on_Snapshot__54_title"
         assert "id" not in ar
 
     def test_create_with_specific_plugin(self, initialized_archive):
@@ -88,9 +91,9 @@ class TestArchiveResultCreate:
         assert code == 0
         records = parse_jsonl_output(stdout2)
         ar_records = [r for r in records if r.get("type") == "ArchiveResult"]
-        assert len(ar_records) >= 1
+        assert len(ar_records) == 1
         assert all(record["plugin"] == "screenshot" for record in ar_records)
-        assert all(record["hook_name"].startswith("on_Snapshot__") for record in ar_records)
+        assert [record["hook_name"] for record in ar_records] == ["on_Snapshot__51_screenshot"]
 
     def test_create_pass_through_crawl(self, initialized_archive):
         """Pass-through Crawl records unchanged."""

archivebox/tests/test_cli_extract_input.py

@@ -22,12 +22,16 @@ def create_extract_snapshot(initialized_archive, env, url="https://example.com")
     )
 
 
-def test_extract_archiveresult_record_queues_only_exact_hook(initialized_archive):
+def test_extract_archiveresult_record_queues_parent_snapshot_plugin(initialized_archive):
     env = cli_env(PLUGINS="archivewebpage")
     create_extract_snapshot(initialized_archive, env)
 
     with use_archivebox_db(initialized_archive):
-        snapshot_id = Snapshot.objects.values_list("id", flat=True).first()
+        snapshot = Snapshot.objects.get()
+        snapshot.pause()
+        snapshot.refresh_from_db()
+        assert snapshot.status == Snapshot.StatusChoices.PAUSED
+        snapshot_id = snapshot.id
 
     hook_name = "on_Snapshot__65_archivewebpage_stop"
     request = json.dumps(
@@ -49,10 +53,69 @@ def test_extract_archiveresult_record_queues_only_exact_hook(initialized_archive
 
     assert result.returncode == 0, result.stderr or result.stdout
     with use_archivebox_db(initialized_archive):
-        rows = list(
-            ArchiveResult.objects.filter(snapshot_id=snapshot_id, plugin="archivewebpage").values_list("hook_name", "status"),
-        )
-    assert rows == [(hook_name, ArchiveResult.StatusChoices.QUEUED)]
+        snapshot = Snapshot.objects.get(id=snapshot_id)
+        rows = list(ArchiveResult.objects.filter(snapshot_id=snapshot_id, plugin="archivewebpage"))
+    assert rows == []
+    assert snapshot.status == Snapshot.StatusChoices.QUEUED
+    assert snapshot.retry_at is not None
+    assert snapshot.config["RETRY_PLUGINS"] == ["archivewebpage"]
+
+
+def test_extract_no_wait_keeps_sealed_snapshot_sealed(initialized_archive):
+    env = cli_env(PLUGINS="title")
+    create_extract_snapshot(initialized_archive, env)
+
+    with use_archivebox_db(initialized_archive):
+        snapshot = Snapshot.objects.get()
+        snapshot.status = Snapshot.StatusChoices.SEALED
+        snapshot.retry_at = None
+        snapshot.save(update_fields=["status", "retry_at", "modified_at"])
+        snapshot_id = snapshot.id
+        crawl_id = snapshot.crawl_id
+        snapshot.crawl.status = snapshot.crawl.StatusChoices.SEALED
+        snapshot.crawl.retry_at = None
+        snapshot.crawl.save(update_fields=["status", "retry_at", "modified_at"])
+
+    request = json.dumps(
+        {
+            "type": "ArchiveResult",
+            "snapshot_id": str(snapshot_id),
+            "plugin": "title",
+            "hook_name": "on_Snapshot__60_title",
+        },
+    )
+    result = run_archivebox_cmd(
+        ["extract", "--no-wait"],
+        cwd=initialized_archive,
+        input=f"{request}\n",
+        env=env,
+        timeout=30,
+    )
+
+    assert result.returncode == 0, result.stderr or result.stdout
+    with use_archivebox_db(initialized_archive):
+        snapshot.refresh_from_db()
+        assert snapshot.status == Snapshot.StatusChoices.SEALED
+        assert snapshot.retry_at is not None
+        assert snapshot.config["RETRY_PLUGINS"] == ["title"]
+        assert not ArchiveResult.objects.filter(snapshot=snapshot).exists()
+
+    run_result = run_archivebox_cmd(
+        ["run", f"--crawl-id={crawl_id}"],
+        cwd=initialized_archive,
+        env=env,
+        timeout=90,
+    )
+
+    assert run_result.returncode == 0, run_result.stderr or run_result.stdout
+    with use_archivebox_db(initialized_archive):
+        snapshot.refresh_from_db()
+        result = ArchiveResult.objects.get(snapshot=snapshot, plugin="title")
+        assert snapshot.status == Snapshot.StatusChoices.SEALED
+        assert snapshot.retry_at is None
+        assert "RETRY_PLUGINS" not in snapshot.config
+        assert result.status == ArchiveResult.StatusChoices.SUCCEEDED
+        assert result.output_str == "Example Domain"
 
 
 def test_extract_runs_on_snapshot_id(initialized_archive):

archivebox/tests/test_cli_list.py

@@ -4,6 +4,7 @@ Verify list emits snapshot JSONL and applies the documented filters.
 """
 
 import json
+from pathlib import Path
 
 import pytest
 from django.contrib.auth import get_user_model
@@ -19,6 +20,147 @@ from archivebox.tests.test_orm_helpers import use_archivebox_db
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
+def test_static_export_creates_detail_page_for_unarchived_snapshot(snapshot):
+    from archivebox.config import CONSTANTS
+
+    snapshot_dir = Path(snapshot.output_dir)
+    assert snapshot_dir.is_dir()
+    assert not any(snapshot_dir.iterdir())
+    snapshot_dir.rmdir()
+    assert not snapshot_dir.exists()
+
+    html = Snapshot.objects.filter(pk=snapshot.pk).to_html(with_headers=True)
+
+    static_path = snapshot_dir.relative_to(CONSTANTS.DATA_DIR).as_posix()
+    detail_path = snapshot_dir / "index.html"
+    assert f"./{static_path}/index.html" in html
+    # The portable export command emits JSONL records; index.json is a legacy
+    # filename that is not created and leaves a broken footer link offline.
+    assert 'href="./index.jsonl"' in html
+    assert 'href="./index.json"' not in html
+    root_manifest = CONSTANTS.DATA_DIR / "index.jsonl"
+    assert root_manifest.exists()
+    manifest_records = [json.loads(line) for line in root_manifest.read_text().splitlines() if line.strip()]
+    assert [record["id"] for record in manifest_records] == [str(snapshot.id)]
+    # JSON and JSONL are alternate containers for one static-export schema;
+    # consumers must not see TYPE/tags/archive paths change by file format.
+    assert manifest_records[0]["TYPE"] == "core.models.Snapshot"
+    assert "type" not in manifest_records[0]
+    assert isinstance(manifest_records[0]["tags"], list)
+    assert manifest_records[0]["archive_path"] == static_path
+    assert manifest_records[0]["archive_url"] == f"./{static_path}/index.html"
+    assert detail_path.exists()
+    detail_html = detail_path.read_text()
+    assert f"/snapshot/{snapshot.id.hex}" not in detail_html
+    assert "/admin/" not in detail_html
+
+
+def test_static_exports_use_filesystem_paths_not_live_django_routes(snapshot):
+    from archivebox.config import CONSTANTS
+    from archivebox.core.models import ArchiveResult
+
+    snapshot_dir = Path(snapshot.output_dir)
+    screenshot_dir = snapshot_dir / "screenshot"
+    screenshot_dir.mkdir(parents=True, exist_ok=True)
+    screenshot_file = screenshot_dir / "screenshot.png"
+    screenshot_file.write_bytes(b"real screenshot")
+    ArchiveResult.objects.create(
+        snapshot=snapshot,
+        plugin="screenshot",
+        hook_name="on_Snapshot__50_screenshot.py",
+        status=ArchiveResult.StatusChoices.SUCCEEDED,
+        output_str="screenshot.png",
+        output_files={"screenshot.png": {"size": screenshot_file.stat().st_size}},
+        output_size=screenshot_file.stat().st_size,
+    )
+    ArchiveResult.objects.create(
+        snapshot=snapshot,
+        plugin="chrome_screencast",
+        hook_name="on_Snapshot__02_chrome_screencast.py",
+        status=ArchiveResult.StatusChoices.SUCCEEDED,
+        output_str="2 screencast frames (0 kept)",
+        output_files={"hook.stderr.log": {"size": 12}},
+        output_size=12,
+    )
+    wget_dir = snapshot_dir / "wget"
+    wget_dir.mkdir()
+    wget_file = wget_dir / "index%3A.html"
+    wget_file.write_text("archived page")
+    ArchiveResult.objects.create(
+        snapshot=snapshot,
+        plugin="wget",
+        hook_name="on_Snapshot__35_wget.py",
+        status=ArchiveResult.StatusChoices.SUCCEEDED,
+        output_str="index%3A.html",
+        output_files={"index%3A.html": {"size": wget_file.stat().st_size}},
+        output_size=wget_file.stat().st_size,
+    )
+    ArchiveResult.objects.create(
+        snapshot=snapshot,
+        plugin="staticfile",
+        hook_name="on_Snapshot__26_staticfile.py",
+        status=ArchiveResult.StatusChoices.SUCCEEDED,
+        output_str="prenav.json",
+        output_files={"prenav.json": {"size": 66}},
+        output_size=66,
+    )
+    ytdlp_dir = snapshot_dir / "ytdlp"
+    ytdlp_dir.mkdir()
+    media_file = ytdlp_dir / "saved.m4a"
+    media_file.write_bytes(b"audio")
+    ArchiveResult.objects.create(
+        snapshot=snapshot,
+        plugin="ytdlp",
+        hook_name="on_Snapshot__60_ytdlp.py",
+        status=ArchiveResult.StatusChoices.SUCCEEDED,
+        output_str="saved.m4a",
+        output_files={
+            "saved.m4a": {"size": media_file.stat().st_size},
+            "deleted.temp.m4a": {"size": 123},
+        },
+        output_size=media_file.stat().st_size,
+    )
+    hashes_dir = snapshot_dir / "hashes"
+    hashes_dir.mkdir()
+    (hashes_dir / "hashes.json").write_text(
+        json.dumps(
+            {
+                "screenshot/screenshot.png": {"size": screenshot_file.stat().st_size},
+                "wget/index%3A.html": {"size": wget_file.stat().st_size},
+                "staticfile/prenav.json": {"size": 66},
+                "ytdlp/saved.m4a": {"size": media_file.stat().st_size},
+                "ytdlp/deleted.temp.m4a": {"size": 123},
+            },
+        ),
+    )
+    static_path = snapshot_dir.relative_to(CONSTANTS.DATA_DIR).as_posix()
+    queryset = Snapshot.objects.filter(pk=snapshot.pk).prefetch_related("tags")
+
+    html = queryset.to_html(with_headers=True)
+    [record] = json.loads(queryset.to_json(with_headers=False))
+    detail_html = snapshot_dir / "index.html"
+
+    assert f"./{static_path}/index.html" in html
+    assert f"./{static_path}/screenshot/screenshot.png" in html
+    assert f"./{static_path}/wget/index%253A.html" in html
+    assert f"./{static_path}/index.jsonl" in html
+    assert f"/snapshot/{snapshot.id.hex}" not in html
+    assert "/web/" not in html
+    assert "/static/" not in html
+    assert "/None" not in html
+    assert "staticfile/prenav.json" not in html
+    assert record["archive_path"] == static_path
+    assert record["archive_url"] == f"./{static_path}/index.html"
+    assert detail_html.exists()
+    rendered_detail = detail_html.read_text()
+    assert "core/snapshot.html" not in rendered_detail
+    assert "screenshot/screenshot.png" in rendered_detail
+    assert "staticfile/prenav.json" not in rendered_detail
+    assert "ytdlp/saved.m4a" in rendered_detail
+    assert "ytdlp/deleted.temp.m4a" not in rendered_detail
+    assert f"/snapshot/{snapshot.id.hex}" not in rendered_detail
+
+
 def test_streaming_json_matches_snapshot_serializer(initialized_archive):
     from archivebox.crawls.models import Crawl
 
@@ -90,7 +232,6 @@ def test_list_limit_zero_streams_one_million_snapshots_without_materializing(ini
                     fs_version,
                     crawl_id,
                     config,
-                    current_step,
                     depth,
                     notes,
                     num_uses_failed,
@@ -114,7 +255,6 @@ def test_list_limit_zero_streams_one_million_snapshots_without_materializing(ini
                     %s,
                     '{}',
                     0,
-                    0,
                     '',
                     0,
                     0,

archivebox/tests/test_cli_mcp.py

@@ -1,11 +1,18 @@
 #!/usr/bin/env python3
-"""
-Tests for archivebox mcp command.
-"""
+"""Tests for the ArchiveBox MCP server and CLI entry point."""
 
+import json
+import os
+
+import pytest
+
+from archivebox.mcp.server import MCPServer
 from archivebox.tests.conftest import run_archivebox_cmd
 
 
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
 def test_mcp_help_runs_successfully(tmp_path):
     """The mcp command should be registered and expose help."""
 
@@ -13,3 +20,150 @@ def test_mcp_help_runs_successfully(tmp_path):
 
     assert result.returncode == 0
     assert "mcp" in result.stdout.lower()
+
+
+def test_mcp_stdio_handles_handshake_notification_and_ping(initialized_archive):
+    requests = [
+        {
+            "jsonrpc": "2.0",
+            "id": 1,
+            "method": "initialize",
+            "params": {
+                "protocolVersion": "2025-11-25",
+                "capabilities": {},
+                "clientInfo": {"name": "archivebox-test", "version": "1"},
+            },
+        },
+        {"jsonrpc": "2.0", "method": "notifications/initialized", "params": {}},
+        {"jsonrpc": "2.0", "id": 2, "method": "ping", "params": {}},
+    ]
+    result = run_archivebox_cmd(
+        ["mcp"],
+        cwd=initialized_archive,
+        input="".join(json.dumps(request) + "\n" for request in requests),
+        default_cli_env=True,
+    )
+    responses = [json.loads(line) for line in result.stdout.splitlines()]
+
+    assert result.returncode == 0
+    assert [response["id"] for response in responses] == [1, 2]
+    assert responses[0]["result"]["protocolVersion"] == "2025-11-25"
+    assert responses[1]["result"] == {}
+
+
+def test_mcp_exposes_six_focused_tools():
+    tools = MCPServer().handle_tools_list({})["tools"]
+    tools_by_name = {tool["name"]: tool for tool in tools}
+
+    assert set(tools_by_name) == {"add", "search", "crawl", "snapshot", "archiveresult", "shell"}
+    assert len(tools) == 6
+    assert all("outputSchema" in tool for tool in tools)
+
+    crawl_schema = tools_by_name["crawl"]["inputSchema"]
+    assert crawl_schema["properties"]["action"]["enum"] == ["create", "delete", "list", "update"]
+    assert crawl_schema["properties"]["urls"]["type"] == "array"
+    assert crawl_schema["properties"]["records"]["type"] == "array"
+    assert tools_by_name["search"]["annotations"]["readOnlyHint"] is True
+    assert tools_by_name["shell"]["inputSchema"]["required"] == ["code"]
+    assert tools_by_name["shell"]["annotations"]["destructiveHint"] is True
+
+
+def test_mcp_crawl_create_returns_structured_json(initialized_archive):
+    os.chdir(initialized_archive)
+    result = MCPServer().handle_tools_call(
+        {
+            "name": "crawl",
+            "arguments": {
+                "action": "create",
+                "urls": ["https://mcp-test.example.com/"],
+                "depth": 1,
+                "tag": "mcp-test",
+            },
+        },
+    )
+
+    assert result["isError"] is False, result
+    assert result["structuredContent"]["success"] is True
+    assert result["structuredContent"]["error"] is None
+    assert result["structuredContent"]["command"] == "archivebox crawl create"
+    assert result["structuredContent"]["exitCode"] == 0
+    assert result["structuredContent"]["records"][0]["urls"] == "https://mcp-test.example.com/"
+    assert result["structuredContent"]["records"][0]["max_depth"] == 1
+    assert json.loads(result["content"][0]["text"]) == result["structuredContent"]
+
+
+def test_mcp_snapshot_update_accepts_records_without_a_jsonl_pipeline(initialized_archive):
+    os.chdir(initialized_archive)
+    server = MCPServer()
+    created = server.handle_tools_call(
+        {
+            "name": "snapshot",
+            "arguments": {
+                "action": "create",
+                "urls": ["https://mcp-update.example.com/"],
+            },
+        },
+    )
+    assert created["isError"] is False, created
+    snapshot = created["structuredContent"]["records"][0]
+
+    updated = server.handle_tools_call(
+        {
+            "name": "snapshot",
+            "arguments": {
+                "action": "update",
+                "records": [{"id": snapshot["id"]}],
+                "tag": "updated-through-mcp",
+            },
+        },
+    )
+
+    assert updated["isError"] is False
+    assert updated["structuredContent"]["records"][0]["id"] == snapshot["id"]
+    assert "updated-through-mcp" in updated["structuredContent"]["records"][0]["tags"]
+
+
+def test_mcp_cli_errors_are_structured_for_agents(initialized_archive):
+    os.chdir(initialized_archive)
+    result = MCPServer().handle_tools_call(
+        {
+            "name": "crawl",
+            "arguments": {"action": "create"},
+        },
+    )
+
+    assert result["isError"] is True
+    assert result["structuredContent"]["success"] is False
+    assert result["structuredContent"]["exitCode"] == 1
+    assert "No URLs provided" in result["structuredContent"]["error"]
+    assert json.loads(result["content"][0]["text"]) == result["structuredContent"]
+
+
+def test_mcp_invalid_action_is_a_protocol_error():
+    response = MCPServer().handle_request(
+        {
+            "jsonrpc": "2.0",
+            "id": 1,
+            "method": "tools/call",
+            "params": {"name": "crawl", "arguments": {"action": "bogus"}},
+        },
+    )
+
+    assert response["error"]["code"] == -32602
+    assert "Choose one of: create, delete, list, update" in response["error"]["message"]
+
+
+def test_mcp_shell_runs_python_through_archivebox_shell(initialized_archive):
+    os.chdir(initialized_archive)
+    result = MCPServer().handle_tools_call(
+        {
+            "name": "shell",
+            "arguments": {
+                "code": "from archivebox.core.models import Snapshot; print(f'shell_ok={Snapshot.objects.count() >= 0}')",
+            },
+        },
+    )
+
+    assert result["isError"] is False, result
+    assert result["structuredContent"]["command"] == "archivebox shell"
+    assert result["structuredContent"]["stdout"] == "shell_ok=True\n"

archivebox/tests/test_cli_piping.py

@@ -38,7 +38,7 @@ PIPE_TEST_ENV = {
 
 def run_real_txt_parser(tmp_path, text):
     """Run the shipped text parser and return its real snapshot output directory."""
-    from archivebox.plugins.hooks import run_hook
+    from archivebox.tests.conftest import run_test_hook
 
     snap_dir = tmp_path / "parser-snapshot"
     staticfile_dir = snap_dir / "staticfile"
@@ -47,7 +47,7 @@ def run_real_txt_parser(tmp_path, text):
     output_dir.mkdir(parents=True)
     (staticfile_dir / "input.txt").write_text(text, encoding="utf-8")
     hook_path = Path(str(files("abx_plugins.plugins.parse_txt_urls").joinpath("on_Snapshot__71_parse_txt_urls.py")))
-    process = run_hook(
+    process = run_test_hook(
         hook_path,
         output_dir,
         config={"ABXPKG_LIB_DIR": str(tmp_path / "lib"), "SNAP_DIR": str(snap_dir)},

archivebox/tests/test_cli_run.py

@@ -240,7 +240,7 @@ class TestRunWithArchiveResult:
     """Tests for `archivebox run` with ArchiveResult input."""
 
     @pytest.mark.django_db(transaction=True)
-    def test_run_creates_and_runs_exact_no_id_archiveresult_request(self, initialized_archive):
+    def test_run_treats_no_id_archiveresult_as_parent_snapshot_plugin_request(self, initialized_archive):
         import json
 
         from archivebox.core.models import ArchiveResult
@@ -284,16 +284,13 @@ class TestRunWithArchiveResult:
                     "output_str",
                 ),
             )
-        assert rows == [
-            (
-                missing_hook,
-                ArchiveResult.StatusChoices.FAILED,
-                "Queued hook is no longer available in the installed plugin",
-            ),
-        ]
+        assert len(rows) == 1
+        assert rows[0][0] != missing_hook
+        assert rows[0][0].startswith("on_Snapshot__")
+        assert rows[0][1] in ArchiveResult.FINAL_STATES
 
     def test_run_requeues_failed_archiveresult(self, initialized_archive):
-        """Run re-queues a failed ArchiveResult."""
+        """Run uses a failed ArchiveResult as a parent Snapshot/plugin reference."""
         url = create_test_url()
 
         # Create snapshot and archive result
@@ -737,6 +734,59 @@ class TestRunDaemonMode:
                 cleanup_process_group(proc.pid)
                 proc.wait(timeout=15)
 
+    def test_run_daemon_retires_runner_from_previous_pid_namespace(self, initialized_archive, db):
+        from django.utils import timezone
+
+        from archivebox.core.takeover_util import RUNNER_ACTIVE_WORKER_TYPE
+        from archivebox.machine.models import Machine, PROCESS_PID_NAMESPACE_KEY, Process, get_current_pid_namespace
+        from archivebox.tests.test_orm_helpers import use_archivebox_db
+
+        env = cli_env(PLUGINS="__archivebox_test_no_plugins__")
+        with use_archivebox_db(initialized_archive):
+            stopped_runner = Process.objects.create(
+                machine=Machine.current(),
+                process_type=Process.TypeChoices.ORCHESTRATOR,
+                worker_type=RUNNER_ACTIVE_WORKER_TYPE,
+                status=Process.StatusChoices.RUNNING,
+                pwd=str(initialized_archive),
+                pid=1,
+                started_at=timezone.now(),
+                env={PROCESS_PID_NAMESPACE_KEY: f"{get_current_pid_namespace()}-stopped-container"},
+            )
+        queued = run_archivebox_cmd(["crawl", "create", create_test_url()], cwd=initialized_archive, env=env, timeout=60)
+        assert queued.returncode == 0, queued.stderr or queued.stdout
+
+        daemon_log = initialized_archive / "run-daemon-stopped-container.log"
+        daemon_log_handle = daemon_log.open("w", encoding="utf-8")
+        replacement = run_archivebox_cmd(
+            ["run", "--daemon"],
+            cwd=initialized_archive,
+            env=env,
+            stdin=subprocess.DEVNULL,
+            stdout=daemon_log_handle,
+            stderr=subprocess.STDOUT,
+            start_new_session=True,
+            wait=False,
+        )
+        daemon_log_handle.close()
+
+        try:
+            wait_for_log(daemon_log, "[Crawl#", timeout=10)
+            assert "Multiple orchestrators sharing a single collection is not officially supported" in daemon_log.read_text()
+            with use_archivebox_db(initialized_archive):
+                stopped_runner.refresh_from_db()
+                assert stopped_runner.status == Process.StatusChoices.EXITED
+                active_runner = Process.objects.get(
+                    process_type=Process.TypeChoices.ORCHESTRATOR,
+                    worker_type=RUNNER_ACTIVE_WORKER_TYPE,
+                    status=Process.StatusChoices.RUNNING,
+                    pwd=str(initialized_archive),
+                )
+                assert active_runner.pid == replacement.pid
+        finally:
+            cleanup_process_group(replacement.pid)
+            replacement.wait(timeout=15)
+
 
 @pytest.mark.django_db
 class TestRecoverOrchestratorState:
@@ -852,43 +902,6 @@ class TestRecoverOrchestratorState:
         assert sealed_snapshot.status == Snapshot.StatusChoices.SEALED
         assert sealed_snapshot.retry_at is not None
 
-    def test_recover_orchestrator_state_requeues_backoff_archiveresults(self):
-        from archivebox.base_models.models import get_or_create_system_user_pk
-        from archivebox.core.models import ArchiveResult, Snapshot
-        from archivebox.core.recovery_util import recover_orchestrator_state
-        from archivebox.crawls.models import Crawl
-
-        crawl = Crawl.objects.create(
-            urls="https://example.com",
-            created_by_id=get_or_create_system_user_pk(),
-            status=Crawl.StatusChoices.SEALED,
-            retry_at=None,
-        )
-        snapshot = Snapshot.objects.create(
-            url="https://example.com",
-            crawl=crawl,
-            status=Snapshot.StatusChoices.SEALED,
-            retry_at=None,
-        )
-        result = ArchiveResult.objects.create(
-            snapshot=snapshot,
-            plugin="search_backend_sqlite",
-            hook_name="on_Snapshot__90_index_sqlite",
-            status=ArchiveResult.StatusChoices.BACKOFF,
-        )
-
-        recovered = recover_orchestrator_state()
-
-        result.refresh_from_db()
-        snapshot.refresh_from_db()
-        crawl.refresh_from_db()
-
-        assert recovered["archiveresults_backoff"] == 1
-        assert result.status == ArchiveResult.StatusChoices.QUEUED
-        assert snapshot.status == Snapshot.StatusChoices.SEALED
-        assert snapshot.retry_at is not None
-        assert crawl.status == Crawl.StatusChoices.SEALED
-
     def test_recover_orchestrator_state_leaves_due_queued_snapshot_for_runner_even_with_final_results(self):
         from archivebox.base_models.models import get_or_create_system_user_pk
         from archivebox.core.models import ArchiveResult, Snapshot
@@ -927,6 +940,7 @@ class TestRecoverOrchestratorState:
         assert crawl.status == Crawl.StatusChoices.QUEUED
         assert crawl.retry_at is not None
 
+    @pytest.mark.django_db(transaction=True)
     def test_recover_orchestrator_state_leaves_stale_queued_final_rows_for_runner(self):
         from datetime import timedelta
 
@@ -936,12 +950,13 @@ class TestRecoverOrchestratorState:
         from archivebox.core.models import ArchiveResult, Snapshot
         from archivebox.core.recovery_util import recover_orchestrator_state
         from archivebox.crawls.models import Crawl
-        from archivebox.services.runner import run_due_crawl, run_due_snapshot
+        from archivebox.services.runner import run_due_snapshot
 
         old = timezone.now() - timedelta(hours=13)
         crawl = Crawl.objects.create(
             urls="https://example.com",
             created_by_id=get_or_create_system_user_pk(),
+            config={"PLUGINS": "__archivebox_test_no_plugins__"},
             status=Crawl.StatusChoices.QUEUED,
             retry_at=old,
         )
@@ -980,16 +995,11 @@ class TestRecoverOrchestratorState:
 
         assert snapshot.status == Snapshot.StatusChoices.SEALED
         assert snapshot.retry_at is None
-        assert crawl.status == Crawl.StatusChoices.QUEUED
-        assert crawl.retry_at == old
-
-        assert run_due_crawl(crawl, lock_seconds=60) is True
-        crawl.refresh_from_db()
-
         assert crawl.status == Crawl.StatusChoices.SEALED
         assert crawl.retry_at is None
 
-    def test_run_due_snapshot_seals_queued_snapshot_with_final_results(self):
+    @pytest.mark.django_db(transaction=True)
+    def test_run_due_snapshot_runs_snapshot_without_consulting_final_result_rows(self):
         from django.utils import timezone
 
         from archivebox.base_models.models import get_or_create_system_user_pk
@@ -1000,6 +1010,7 @@ class TestRecoverOrchestratorState:
         crawl = Crawl.objects.create(
             urls="https://example.com",
             created_by_id=get_or_create_system_user_pk(),
+            config={"PLUGINS": "__archivebox_test_no_plugins__"},
             status=Crawl.StatusChoices.STARTED,
             retry_at=timezone.now(),
         )
@@ -1023,68 +1034,6 @@ class TestRecoverOrchestratorState:
         assert snapshot.retry_at is None
         assert snapshot.downloaded_at is not None
 
-    def test_create_pending_archiveresults_uses_canonical_hook_names(self):
-        from django.utils import timezone
-
-        from archivebox.base_models.models import get_or_create_system_user_pk
-        from archivebox.core.models import ArchiveResult, Snapshot
-        from archivebox.crawls.models import Crawl
-
-        crawl = Crawl.objects.create(
-            urls="https://example.com",
-            created_by_id=get_or_create_system_user_pk(),
-            status=Crawl.StatusChoices.STARTED,
-            retry_at=timezone.now(),
-        )
-        snapshot = Snapshot.objects.create(
-            url="https://example.com",
-            crawl=crawl,
-            status=Snapshot.StatusChoices.QUEUED,
-            retry_at=timezone.now(),
-        )
-
-        snapshot.create_pending_archiveresults()
-
-        hook_names = list(ArchiveResult.objects.filter(snapshot=snapshot).values_list("hook_name", flat=True))
-        assert hook_names
-        assert all(not hook_name.endswith((".py", ".js", ".sh")) for hook_name in hook_names)
-
-    def test_snapshot_hooks_for_pending_archiveresults_respects_disabled_plugins_when_plugins_empty(self):
-        from django.utils import timezone
-
-        from archivebox.base_models.models import get_or_create_system_user_pk
-        from archivebox.core.models import Snapshot
-        from archivebox.crawls.models import Crawl
-        from archivebox.services.runner import snapshot_hooks_for_pending_archiveresults
-
-        crawl = Crawl.objects.create(
-            urls="https://example.com",
-            created_by_id=get_or_create_system_user_pk(),
-            config={
-                "CLAUDECHROME_ENABLED": False,
-                "CLAUDECODEEXTRACT_ENABLED": False,
-                "CLAUDECODECLEANUP_ENABLED": False,
-                "SEARCH_BACKEND_SQLITE_ENABLED": False,
-            },
-            status=Crawl.StatusChoices.STARTED,
-            retry_at=timezone.now(),
-        )
-        snapshot = Snapshot.objects.create(
-            url="https://example.com",
-            crawl=crawl,
-            status=Snapshot.StatusChoices.QUEUED,
-            retry_at=timezone.now(),
-        )
-
-        hooks = snapshot_hooks_for_pending_archiveresults(snapshot)
-        queued_plugins = {plugin for plugin, _hook_name in hooks}
-
-        assert "title" in queued_plugins
-        assert "claudechrome" not in queued_plugins
-        assert "claudecodeextract" not in queued_plugins
-        assert "claudecodecleanup" not in queued_plugins
-        assert "search_backend_sqlite" not in queued_plugins
-
     def test_run_due_snapshot_pauses_child_when_parent_is_paused(self):
         from django.utils import timezone
 
@@ -1119,7 +1068,7 @@ class TestRecoverOrchestratorState:
         result.refresh_from_db()
         assert snapshot.status == Snapshot.StatusChoices.PAUSED
         assert snapshot.retry_at == RETRY_AT_MAX
-        assert result.status == ArchiveResult.StatusChoices.PAUSED
+        assert result.status == ArchiveResult.StatusChoices.QUEUED
         assert snapshot.archiveresult_set.count() == 1
 
     def test_parent_status_transitions_schedule_children_to_follow_parent_status(self):
@@ -1177,7 +1126,7 @@ class TestRecoverOrchestratorState:
         sealed_started_child.refresh_from_db()
         assert paused_child.status == Snapshot.StatusChoices.PAUSED
         assert paused_child.retry_at == RETRY_AT_MAX
-        assert paused_result.status == ArchiveResult.StatusChoices.PAUSED
+        assert paused_result.status == ArchiveResult.StatusChoices.QUEUED
         assert sealed_child.status == Snapshot.StatusChoices.PAUSED
         assert sealed_child.retry_at is not None
         assert sealed_child.retry_at <= timezone.now()
@@ -1574,7 +1523,7 @@ class TestRecoverOrchestratorState:
         assert snapshot.status == Snapshot.StatusChoices.SEALED
         assert snapshot.fs_version == Snapshot._fs_current_version()
         assert snapshot.output_dir.joinpath("index.html").read_text(encoding="utf-8") == "legacy archive"
-        assert legacy_dir.is_symlink()
+        assert not legacy_dir.exists()
 
     @pytest.mark.django_db(transaction=True)
     def test_run_due_snapshot_migrates_filesystem_after_sealed_parent_reconciliation(self):
@@ -1616,154 +1565,7 @@ class TestRecoverOrchestratorState:
         assert snapshot.status == Snapshot.StatusChoices.SEALED
         assert snapshot.fs_version == Snapshot._fs_current_version()
         assert snapshot.output_dir.joinpath("index.html").read_text(encoding="utf-8") == "legacy archive"
-        assert legacy_dir.is_symlink()
-
-    @pytest.mark.django_db(transaction=True)
-    def test_run_due_snapshot_runs_queued_plugin_after_fs_migration(self):
-        from django.utils import timezone
-
-        from archivebox.base_models.models import get_or_create_system_user_pk
-        from archivebox.core.models import ArchiveResult, Snapshot
-        from archivebox.crawls.models import Crawl
-        from archivebox.services.runner import run_due_snapshot
-
-        crawl = Crawl.objects.create(
-            urls="https://example.com",
-            created_by_id=get_or_create_system_user_pk(),
-            status=Crawl.StatusChoices.SEALED,
-            retry_at=None,
-        )
-        snapshot = Snapshot.objects.create(
-            url="https://example.com",
-            crawl=crawl,
-            status=Snapshot.StatusChoices.SEALED,
-            retry_at=timezone.now(),
-        )
-        Snapshot.objects.filter(pk=snapshot.pk).update(fs_version="0.9.0")
-        snapshot.refresh_from_db()
-        snapshot.output_dir.mkdir(parents=True, exist_ok=True)
-        title_dir = snapshot.output_dir / "title"
-        title_dir.mkdir(parents=True, exist_ok=True)
-        (title_dir / "title.txt").write_text("Example Domain\n", encoding="utf-8")
-        result = ArchiveResult.objects.create(
-            snapshot=snapshot,
-            plugin="search_backend_sqlite",
-            hook_name="on_Snapshot__90_index_sqlite",
-            status=ArchiveResult.StatusChoices.QUEUED,
-        )
-
-        assert run_due_snapshot(snapshot, lock_seconds=60) is True
-
-        snapshot.refresh_from_db()
-        result.refresh_from_db()
-        assert snapshot.fs_version == Snapshot._fs_current_version()
-        assert result.status in ArchiveResult.FINAL_STATES
-        assert result.status != ArchiveResult.StatusChoices.QUEUED
-        assert result.start_ts is not None
-        assert result.end_ts is not None
-
-    @pytest.mark.django_db(transaction=True)
-    def test_run_due_snapshot_fails_obsolete_queued_hook_name(self):
-        from django.utils import timezone
-
-        from archivebox.base_models.models import get_or_create_system_user_pk
-        from archivebox.core.models import ArchiveResult, Snapshot
-        from archivebox.crawls.models import Crawl
-        from archivebox.services.runner import run_due_snapshot
-
-        crawl = Crawl.objects.create(
-            urls="https://example.com",
-            created_by_id=get_or_create_system_user_pk(),
-            status=Crawl.StatusChoices.SEALED,
-            retry_at=None,
-        )
-        snapshot = Snapshot.objects.create(
-            url="https://example.com",
-            crawl=crawl,
-            status=Snapshot.StatusChoices.SEALED,
-            retry_at=timezone.now(),
-        )
-        result = ArchiveResult.objects.create(
-            snapshot=snapshot,
-            plugin="singlefile",
-            hook_name="on_Snapshot__50_singlefile.py",
-            status=ArchiveResult.StatusChoices.QUEUED,
-        )
-
-        assert run_due_snapshot(snapshot, lock_seconds=60) is True
-
-        result.refresh_from_db()
-        snapshot.refresh_from_db()
-        assert result.status == ArchiveResult.StatusChoices.FAILED
-        assert result.output_str == "Queued hook is no longer available in the installed plugin"
-        assert snapshot.retry_at is None
-
-    @pytest.mark.django_db(transaction=True)
-    def test_run_due_snapshot_skips_disabled_queued_plugins_and_seals_started_snapshot(self):
-        from django.utils import timezone
-
-        from archivebox.base_models.models import get_or_create_system_user_pk
-        from archivebox.core.models import ArchiveResult, Snapshot
-        from archivebox.crawls.models import Crawl
-        from archivebox.services.runner import run_due_snapshot
-
-        crawl = Crawl.objects.create(
-            urls="https://example.com",
-            created_by_id=get_or_create_system_user_pk(),
-            config={
-                "CLAUDECHROME_ENABLED": False,
-                "CLAUDECODEEXTRACT_ENABLED": False,
-                "CLAUDECODECLEANUP_ENABLED": False,
-                "SEARCH_BACKEND_SQLITE_ENABLED": False,
-            },
-            status=Crawl.StatusChoices.STARTED,
-            retry_at=timezone.now(),
-        )
-        snapshot = Snapshot.objects.create(
-            url="https://example.com",
-            crawl=crawl,
-            status=Snapshot.StatusChoices.STARTED,
-            retry_at=timezone.now(),
-            downloaded_at=timezone.now(),
-        )
-        ArchiveResult.objects.create(
-            snapshot=snapshot,
-            plugin="title",
-            hook_name="on_Snapshot__01_title",
-            status=ArchiveResult.StatusChoices.SUCCEEDED,
-            output_str="Example Domain",
-        )
-        stale_results = [
-            ArchiveResult.objects.create(
-                snapshot=snapshot,
-                plugin=plugin,
-                hook_name=hook_name,
-                status=ArchiveResult.StatusChoices.QUEUED,
-            )
-            for plugin, hook_name in (
-                ("claudechrome", "on_Snapshot__47_claudechrome"),
-                ("claudecodeextract", "on_Snapshot__58_claudecodeextract"),
-                ("claudecodecleanup", "on_Snapshot__92_claudecodecleanup"),
-                ("search_backend_sqlite", "on_Snapshot__90_index_sqlite"),
-            )
-        ]
-        stale_result_ids = [result.id for result in stale_results]
-        stale_plugins = [result.plugin for result in stale_results]
-
-        assert run_due_snapshot(snapshot, lock_seconds=60) is True
-
-        snapshot.refresh_from_db()
-        assert snapshot.status == Snapshot.StatusChoices.SEALED
-        assert snapshot.retry_at is None
-        assert not snapshot.archiveresult_set.filter(
-            plugin__in=stale_plugins,
-            status=ArchiveResult.StatusChoices.QUEUED,
-        ).exists()
-        for result in ArchiveResult.objects.filter(id__in=stale_result_ids):
-            assert result.status == ArchiveResult.StatusChoices.SKIPPED
-            assert result.start_ts is not None
-            assert result.end_ts is not None
-            assert "disabled by this Snapshot/Crawl config" in result.output_str
+        assert not legacy_dir.exists()
 
     @pytest.mark.django_db(transaction=True)
     @pytest.mark.timeout(300)
@@ -1898,6 +1700,7 @@ class TestRecoverOrchestratorState:
         assert crawl.status == Crawl.StatusChoices.SEALED
         assert crawl.retry_at is None
 
+    @pytest.mark.django_db(transaction=True)
     def test_recover_orchestrator_state_unlocks_started_snapshot_with_final_results_for_runner(self):
         from archivebox.base_models.models import get_or_create_system_user_pk
         from archivebox.core.models import ArchiveResult, Snapshot
@@ -1908,6 +1711,7 @@ class TestRecoverOrchestratorState:
         crawl = Crawl.objects.create(
             urls="https://example.com",
             created_by_id=get_or_create_system_user_pk(),
+            config={"PLUGINS": "__archivebox_test_no_plugins__"},
             status=Crawl.StatusChoices.STARTED,
             retry_at=None,
         )
@@ -1998,7 +1802,7 @@ class TestRunDueCrawlState:
         assert crawl.retry_at == now
         assert crawl.snapshot_set.count() == 0
 
-    def test_maintenance_only_runner_ignores_disabled_queued_results_on_sealed_snapshots(self):
+    def test_maintenance_only_runner_clears_snapshot_tick_without_scheduling_archive_results(self):
         from django.utils import timezone
 
         from archivebox.base_models.models import get_or_create_system_user_pk
@@ -2032,7 +1836,7 @@ class TestRunDueCrawlState:
         result.refresh_from_db()
         assert snapshot.status == Snapshot.StatusChoices.SEALED
         assert snapshot.fs_version == Snapshot._fs_current_version()
-        assert snapshot.retry_at is not None
+        assert snapshot.retry_at is None
         assert result.status == ArchiveResult.StatusChoices.QUEUED
 
     def test_snapshot_start_writes_short_future_lease(self):
@@ -2055,138 +1859,13 @@ class TestRunDueCrawlState:
             retry_at=timezone.now(),
         )
 
-        snapshot.sm.tick()
+        snapshot.advance_lifecycle()
         snapshot.refresh_from_db()
 
         assert snapshot.status == Snapshot.StatusChoices.STARTED
         assert snapshot.retry_at is not None
         assert snapshot.retry_at > timezone.now()
 
-    def test_abandoned_started_snapshot_results_are_reset_locally_for_resume(self):
-        from django.utils import timezone
-
-        from archivebox.base_models.models import get_or_create_system_user_pk
-        from archivebox.core.models import ArchiveResult, Snapshot
-        from archivebox.crawls.models import Crawl
-
-        crawl = Crawl.objects.create(
-            urls="https://example.com",
-            created_by_id=get_or_create_system_user_pk(),
-            status=Crawl.StatusChoices.STARTED,
-            retry_at=timezone.now(),
-        )
-        snapshot = Snapshot.objects.create(
-            url="https://example.com",
-            crawl=crawl,
-            status=Snapshot.StatusChoices.STARTED,
-            retry_at=timezone.now(),
-        )
-        abandoned = ArchiveResult.objects.create(
-            snapshot=snapshot,
-            plugin="title",
-            hook_name="on_Snapshot__01_title",
-            status=ArchiveResult.StatusChoices.STARTED,
-            output_str="partial output should be cleared",
-            output_files={"partial.txt": {"size": 12}},
-            output_size=12,
-            start_ts=timezone.now(),
-        )
-        queued = ArchiveResult.objects.create(
-            snapshot=snapshot,
-            plugin="wget",
-            hook_name="on_Snapshot__40_wget",
-            status=ArchiveResult.StatusChoices.QUEUED,
-        )
-        finished = ArchiveResult.objects.create(
-            snapshot=snapshot,
-            plugin="favicon",
-            hook_name="on_Snapshot__01_favicon",
-            status=ArchiveResult.StatusChoices.SUCCEEDED,
-            output_str="keep me",
-            output_files={"favicon.ico": {"size": 1}},
-            output_size=1,
-        )
-
-        snapshot.reset_abandoned_results()
-
-        abandoned.refresh_from_db()
-        queued.refresh_from_db()
-        finished.refresh_from_db()
-
-        assert abandoned.status == ArchiveResult.StatusChoices.QUEUED
-        assert abandoned.output_str == ""
-        assert abandoned.output_files == {}
-        assert abandoned.output_size == 0
-        assert queued.status == ArchiveResult.StatusChoices.QUEUED
-        assert finished.status == ArchiveResult.StatusChoices.SUCCEEDED
-        assert finished.output_str == "keep me"
-        assert finished.output_files == {"favicon.ico": {"size": 1}}
-
-    def test_finished_parser_result_projects_children_before_resume_seals_snapshot(self):
-        from importlib.resources import files
-        from pathlib import Path
-
-        from django.utils import timezone
-
-        from archivebox.base_models.models import get_or_create_system_user_pk
-        from archivebox.core.models import ArchiveResult, Snapshot
-        from archivebox.crawls.models import Crawl
-        from archivebox.plugins.hooks import extract_records_from_process, run_hook
-        from archivebox.services.runner import run_due_snapshot
-
-        crawl = Crawl.objects.create(
-            urls="Plain text import containing https://example.org/\n",
-            max_depth=1,
-            created_by_id=get_or_create_system_user_pk(),
-            status=Crawl.StatusChoices.STARTED,
-            retry_at=timezone.now(),
-        )
-        root = Snapshot.objects.create(
-            url=Snapshot.INTERNAL_INPUT_URL,
-            crawl=crawl,
-            depth=0,
-            status=Snapshot.StatusChoices.STARTED,
-            retry_at=timezone.now(),
-        )
-        staticfile_dir = root.output_dir / "staticfile"
-        parser_dir = root.output_dir / "parse_txt_urls"
-        staticfile_dir.mkdir(parents=True, exist_ok=True)
-        parser_dir.mkdir(parents=True, exist_ok=True)
-        (staticfile_dir / "input.txt").write_text(
-            "Plain text import containing https://example.org/\n",
-            encoding="utf-8",
-        )
-        hook_path = Path(str(files("abx_plugins.plugins.parse_txt_urls").joinpath("on_Snapshot__71_parse_txt_urls.py")))
-        process = run_hook(
-            hook_path,
-            parser_dir,
-            config={"ABXPKG_LIB_DIR": str(root.output_dir.parent.parent / "lib"), "SNAP_DIR": str(root.output_dir)},
-            timeout=30,
-            url=root.url,
-            depth=root.depth,
-            snapshot_id=str(root.id),
-        )
-        process.refresh_from_db()
-        assert process.exit_code == 0, process.stderr
-        result_record = next(record for record in extract_records_from_process(process) if record.get("type") == "ArchiveResult")
-        ArchiveResult.objects.create(
-            snapshot=root,
-            process=process,
-            plugin=result_record["plugin"],
-            hook_name=result_record["hook_name"],
-            status=result_record["status"],
-            output_str=result_record.get("output_str", ""),
-            output_files={"urls.jsonl": {"size": (parser_dir / "urls.jsonl").stat().st_size}},
-        )
-
-        assert run_due_snapshot(root, lock_seconds=60)
-
-        root.refresh_from_db()
-        child = Snapshot.objects.get(crawl=crawl, url="https://example.org/")
-        assert root.status == Snapshot.StatusChoices.SEALED
-        assert child.parent_snapshot_id == root.id
-        assert child.status == Snapshot.StatusChoices.QUEUED
-
     def test_due_started_snapshot_with_live_child_extends_lease_without_reset(self):
         import os
         from datetime import datetime
@@ -2525,7 +2204,7 @@ class TestRecoverOrchestratorStateRedFailureModes:
         assert crawl.status == Crawl.StatusChoices.STARTED
         assert crawl.retry_at == future
 
-    def test_recovery_requeues_started_archiveresult_without_process(self):
+    def test_recovery_closes_interrupted_result_and_requeues_parent_snapshot(self):
         from archivebox.base_models.models import get_or_create_system_user_pk
         from archivebox.core.models import ArchiveResult, Snapshot
         from archivebox.core.recovery_util import recover_orchestrator_state
@@ -2553,9 +2232,12 @@ class TestRecoverOrchestratorStateRedFailureModes:
         recover_orchestrator_state()
 
         result.refresh_from_db()
-        assert result.status == ArchiveResult.StatusChoices.QUEUED
+        snapshot.refresh_from_db()
+        assert result.status == ArchiveResult.StatusChoices.FAILED
+        assert snapshot.status == Snapshot.StatusChoices.STARTED
+        assert snapshot.retry_at is not None
 
-    def test_recovery_requeues_started_archiveresult_with_exited_process(self):
+    def test_recovery_closes_started_archiveresult_with_exited_process(self):
         from django.utils import timezone
 
         from archivebox.base_models.models import get_or_create_system_user_pk
@@ -2594,9 +2276,11 @@ class TestRecoverOrchestratorStateRedFailureModes:
         recover_orchestrator_state()
 
         result.refresh_from_db()
-        assert result.status == ArchiveResult.StatusChoices.QUEUED
+        snapshot.refresh_from_db()
+        assert result.status == ArchiveResult.StatusChoices.FAILED
+        assert snapshot.retry_at is not None
 
-    def test_recovery_requeues_sealed_snapshot_started_result_with_exited_process_result_too(self):
+    def test_recovery_does_not_reopen_sealed_snapshot_for_interrupted_result_projection(self):
         from django.utils import timezone
 
         from archivebox.base_models.models import get_or_create_system_user_pk
@@ -2642,10 +2326,10 @@ class TestRecoverOrchestratorStateRedFailureModes:
         snapshot.refresh_from_db()
         result.refresh_from_db()
         assert snapshot.status == Snapshot.StatusChoices.SEALED
-        assert snapshot.retry_at is not None
-        assert result.status == ArchiveResult.StatusChoices.QUEUED
+        assert snapshot.retry_at is None
+        assert result.status == ArchiveResult.StatusChoices.FAILED
 
-    def test_recovery_requeues_started_snapshot_result_before_unlocking_snapshot(self):
+    def test_recovery_closes_result_projection_before_unlocking_snapshot(self):
         from archivebox.base_models.models import get_or_create_system_user_pk
         from archivebox.core.models import ArchiveResult, Snapshot
         from archivebox.core.recovery_util import recover_orchestrator_state
@@ -2674,7 +2358,7 @@ class TestRecoverOrchestratorStateRedFailureModes:
 
         snapshot.refresh_from_db()
         result.refresh_from_db()
-        assert result.status == ArchiveResult.StatusChoices.QUEUED
+        assert result.status == ArchiveResult.StatusChoices.FAILED
         assert snapshot.retry_at is not None
 
     def test_crawl_runner_load_run_state_does_not_return_future_retry_snapshots(self):
@@ -2941,7 +2625,7 @@ class TestRecoverOrchestratorStateRedFailureModes:
             modified_at=now + timedelta(seconds=1),
         )
 
-        snapshot.sm.seal()
+        snapshot.seal()
         snapshot.refresh_from_db()
         assert snapshot.status == Snapshot.StatusChoices.SEALED
         assert snapshot.retry_at is None

archivebox/tests/test_cli_schedule.py

@@ -56,9 +56,11 @@ def test_schedule_without_import_path_creates_maintenance_schedule(initialized_a
     assert "Created scheduled maintenance update" in result.stdout
 
     with use_archivebox_db(initialized_archive):
-        row = Crawl.objects.order_by("-created_at").values_list("urls", "status").first()
+        schedule_row = CrawlSchedule.objects.select_related("template").get()
 
-    assert row == ("archivebox://update", "sealed")
+    assert schedule_row.kind == "update"
+    assert schedule_row.template.urls == ""
+    assert schedule_row.template.status == "sealed"
 
 
 def test_schedule_creates_enabled_db_schedule(initialized_archive):

archivebox/tests/test_cli_server.py

@@ -206,6 +206,99 @@ def test_sqlite_connections_use_explicit_busy_timeout():
     assert "PRAGMA journal_mode = WAL;" in SQLITE_CONNECTION_OPTIONS["OPTIONS"]["init_command"]
 
 
+def test_docker_sqlite_never_uses_wal_on_host_shared_collection(tmp_path):
+    """Docker collections must not expose a WAL database through /data.
+
+    Docker Desktop/OrbStack bind mounts cross the Linux VM/host locking
+    boundary. A container WAL writer plus a host-side sqlite reader can make
+    the writer SIGBUS and leave index.sqlite3 malformed; the reader does not
+    need to write. Run a fresh real settings process with Docker identity so a
+    host test cannot accidentally pass by reusing this process's non-Docker
+    config imports.
+    """
+    env = os.environ.copy()
+    env["IN_DOCKER"] = "True"
+    env["DJANGO_SETTINGS_MODULE"] = "archivebox.core.settings"
+    result = subprocess.run(
+        [
+            sys.executable,
+            "-c",
+            (
+                "import django;"
+                "django.setup();"
+                "from django.db import connection;"
+                "cursor=connection.cursor();"
+                "cursor.execute('CREATE TABLE journal_probe (value INTEGER)');"
+                "cursor.execute('INSERT INTO journal_probe VALUES (1)');"
+                "cursor.execute('PRAGMA journal_mode');"
+                "print(cursor.fetchone()[0]);"
+                "cursor.close();"
+                "connection.close()"
+            ),
+        ],
+        cwd=tmp_path,
+        env=env,
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+
+    assert result.stdout.strip().lower() == "delete"
+    assert not tmp_path.joinpath("index.sqlite3-wal").exists()
+    assert not tmp_path.joinpath("index.sqlite3-shm").exists()
+
+
+def test_docker_rejects_explicit_wal_override(tmp_path):
+    """An old config or environment override must not bypass the invariant."""
+    env = os.environ.copy()
+    env["IN_DOCKER"] = "True"
+    result = subprocess.run(
+        [
+            sys.executable,
+            "-c",
+            ("from archivebox.config.common import DatabaseConfig;DatabaseConfig(SQLITE_JOURNAL_MODE='WAL')"),
+        ],
+        cwd=tmp_path,
+        env=env,
+        capture_output=True,
+        text=True,
+    )
+
+    assert result.returncode != 0
+    assert "WAL is unsafe for Docker collections" in result.stderr
+
+
+def test_docker_postgres_ignores_irrelevant_sqlite_wal_override(tmp_path):
+    """The Docker SQLite safety invariant must not reject PostgreSQL.
+
+    Operators can switch an existing deployment to PostgreSQL while an old
+    SQLITE_JOURNAL_MODE setting remains in ArchiveBox.conf or the environment.
+    PostgreSQL never consumes that SQLite pragma, so rejecting the otherwise
+    valid configuration would prevent ArchiveBox from starting without making
+    any database safer.
+    """
+    env = os.environ.copy()
+    env["IN_DOCKER"] = "True"
+    result = subprocess.run(
+        [
+            sys.executable,
+            "-c",
+            (
+                "from archivebox.config.common import DatabaseConfig;"
+                "config=DatabaseConfig(DATABASE_ENGINE='postgres',SQLITE_JOURNAL_MODE='WAL');"
+                "print(config.DATABASE_ENGINE)"
+            ),
+        ],
+        cwd=tmp_path,
+        env=env,
+        capture_output=True,
+        text=True,
+    )
+
+    assert result.returncode == 0, result.stderr
+    assert result.stdout.strip() == "postgres"
+
+
 def test_server_shows_usage_info(initialized_archive):
     """Test that server command shows usage or starts."""
 

archivebox/tests/test_cli_update.py

@@ -24,7 +24,7 @@ def test_update_runs_successfully_on_empty_archive(initialized_archive):
 
     assert result.returncode == 0, output
     assert "Phase 1: Draining old archive/ directories" in output
-    assert "Phase 2: Processing all database snapshots" in output
+    assert "Phase 2: Selecting database snapshots with stale filesystem versions" in output
     assert "Updated DB rows:  0" in output
     assert "Sealed crawls:    0" in output
 
@@ -52,7 +52,7 @@ def test_update_reconciles_existing_snapshots(initialized_archive):
 
     output = result.stdout + result.stderr
     assert result.returncode == 0, output
-    assert "Phase 2: Processing all database snapshots" in output
+    assert "Phase 2: Selecting database snapshots with stale filesystem versions" in output
     assert "Updated DB rows:" in output
 
     with use_archivebox_db(initialized_archive):
@@ -147,7 +147,7 @@ def test_update_seals_migrated_snapshots(initialized_archive):
 
     output = result.stdout + result.stderr
     assert result.returncode == 0, output
-    assert "Phase 2: Processing all database snapshots" in output
+    assert "Phase 2: Selecting database snapshots with stale filesystem versions" in output
     assert "Reindexing" not in output
 
     # Check that snapshot remains archived instead of being queued for a full re-crawl.

archivebox/tests/test_cli_update_reindex_snapshots.py

@@ -1,6 +1,8 @@
 import json
 import os
 from datetime import datetime, timedelta
+from pathlib import Path
+
 from archivebox.tests.conftest import cli_env, run_archivebox_cmd
 
 import pytest
@@ -13,84 +15,6 @@ from archivebox.tests.test_orm_helpers import use_archivebox_db
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
-def test_only_sealed_search_backfill_bypasses_snapshot_lifecycle():
-    from archivebox.base_models.models import get_or_create_system_user_pk
-    from archivebox.cli.archivebox_extract import run_plugins
-    from archivebox.core.models import ArchiveResult
-    from archivebox.crawls.models import Crawl
-
-    search_crawl = Crawl.objects.create(
-        urls="https://example.com/search",
-        created_by_id=get_or_create_system_user_pk(),
-        status=Crawl.StatusChoices.SEALED,
-    )
-    search_snapshot = Snapshot.objects.create(
-        url="https://example.com/search",
-        crawl=search_crawl,
-        status=Snapshot.StatusChoices.SEALED,
-    )
-    extract_crawl = Crawl.objects.create(
-        urls="https://example.com/extract",
-        created_by_id=get_or_create_system_user_pk(),
-        status=Crawl.StatusChoices.SEALED,
-    )
-    extract_snapshot = Snapshot.objects.create(
-        url="https://example.com/extract",
-        crawl=extract_crawl,
-        status=Snapshot.StatusChoices.SEALED,
-    )
-
-    assert (
-        run_plugins(
-            args=(),
-            records=[
-                {
-                    "type": "ArchiveResult",
-                    "snapshot_id": str(search_snapshot.id),
-                    "plugin": "search_backend_sqlite",
-                },
-            ],
-            wait=False,
-            emit_results=False,
-            show_progress=False,
-        )
-        == 0
-    )
-    search_crawl.refresh_from_db()
-    search_snapshot.refresh_from_db()
-    assert search_crawl.status == Crawl.StatusChoices.SEALED
-    assert search_snapshot.status == Snapshot.StatusChoices.SEALED
-    assert search_snapshot.archiveresult_set.filter(
-        plugin="search_backend_sqlite",
-        status=ArchiveResult.StatusChoices.QUEUED,
-    ).exists()
-
-    assert (
-        run_plugins(
-            args=(),
-            records=[
-                {
-                    "type": "ArchiveResult",
-                    "snapshot_id": str(extract_snapshot.id),
-                    "plugin": "wget",
-                },
-            ],
-            wait=False,
-            emit_results=False,
-            show_progress=False,
-        )
-        == 0
-    )
-    extract_crawl.refresh_from_db()
-    extract_snapshot.refresh_from_db()
-    assert extract_crawl.status == Crawl.StatusChoices.QUEUED
-    assert extract_snapshot.status == Snapshot.StatusChoices.QUEUED
-    assert extract_snapshot.archiveresult_set.filter(
-        plugin="wget",
-        status=ArchiveResult.StatusChoices.QUEUED,
-    ).exists()
-
-
 def test_update_imports_orphaned_snapshots(tmp_path, initialized_archive):
     """Test that archivebox update imports real legacy archive directories."""
     env = cli_env(disable_extractors=True)
@@ -120,12 +44,12 @@ def test_update_imports_orphaned_snapshots(tmp_path, initialized_archive):
     assert update_process.returncode == 0, update_process.stderr
 
     with use_archivebox_db(tmp_path):
-        row = Snapshot.objects.values_list("url", "fs_version").get()
+        migrated_snapshot = Snapshot.objects.get()
+        row = (migrated_snapshot.url, migrated_snapshot.fs_version)
+        migrated_dir = Path(migrated_snapshot.output_dir)
 
     assert row == ("https://example.com", Snapshot._fs_current_version())
-    assert legacy_dir.is_symlink()
-
-    migrated_dir = legacy_dir.resolve()
+    assert not legacy_dir.exists()
     assert migrated_dir.exists()
     assert '{"type":"Process","id":"incomplete"}\n' in (migrated_dir / "index.jsonl").read_text()
     assert (migrated_dir / "singlefile.html").exists()
@@ -194,14 +118,15 @@ def test_update_migrates_every_declared_filesystem_version(tmp_path, initialized
         assert snapshot.status == Snapshot.StatusChoices.QUEUED
         assert snapshot.retry_at is not None
         result.refresh_from_db()
-        assert result.output_files
+        # Migration projects filesystem facts without changing result lifecycle.
+        assert "singlefile.html" in result.output_files
         assert result.output_size > 0
         if legacy_layout:
             assert (migrated_dir / "existing-user-output.bin").read_bytes() == b"preserve interrupted migration output"
 
     assert {path: migrated_tree.get(path) for path in original_tree} == original_tree
     if legacy_layout:
-        assert source_dir.is_symlink()
+        assert not source_dir.exists()
 
     update_process = run_archivebox_cmd(["update", "--migrate-only"], env=env, timeout=90)
     assert update_process.returncode == 0, f"Idempotency update failed: {update_process.stderr}"
@@ -213,15 +138,16 @@ def test_update_migrates_every_declared_filesystem_version(tmp_path, initialized
 
 
 @pytest.mark.django_db(transaction=True)
-def test_reindex_snapshots_resets_existing_search_results_and_reruns_requested_plugins():
+def test_reindex_snapshots_runs_only_missing_sealed_search_indexes():
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.cli.archivebox_update import reindex_snapshots
     from archivebox.core.models import ArchiveResult, Snapshot
     from archivebox.crawls.models import Crawl
 
     crawl = Crawl.objects.create(
-        urls="https://example.com",
+        urls="https://example.com\nhttps://example.org",
         created_by_id=get_or_create_system_user_pk(),
+        status=Crawl.StatusChoices.SEALED,
     )
     snapshot = Snapshot.objects.create(
         url="https://example.com",
@@ -248,12 +174,20 @@ def test_reindex_snapshots_resets_existing_search_results_and_reruns_requested_p
     (output_dir / "title" / "title.txt").write_text("Example Domain")
     (output_dir / "dom").mkdir(parents=True, exist_ok=True)
     (output_dir / "dom" / "output.html").write_text("<html><body>Example searchable text</body></html>")
+    missing_snapshot = Snapshot.objects.create(
+        url="https://example.org",
+        crawl=crawl,
+        status=Snapshot.StatusChoices.SEALED,
+    )
+    missing_output_dir = missing_snapshot.output_dir
+    (missing_output_dir / "title").mkdir(parents=True, exist_ok=True)
+    (missing_output_dir / "title" / "title.txt").write_text("Missing Search Index")
 
     original_engine = os.environ.get("SEARCH_BACKEND_ENGINE")
     os.environ["SEARCH_BACKEND_ENGINE"] = "sqlite"
     try:
         stats = reindex_snapshots(
-            Snapshot.objects.filter(id__in=(snapshot.id, paused_snapshot.id)),
+            Snapshot.objects.filter(id__in=(snapshot.id, missing_snapshot.id, paused_snapshot.id)).select_related("crawl"),
             search_plugins=["search_backend_sqlite"],
             batch_size=10,
         )
@@ -265,15 +199,92 @@ def test_reindex_snapshots_resets_existing_search_results_and_reruns_requested_p
 
     result.refresh_from_db()
 
+    snapshot.refresh_from_db()
+    missing_snapshot.refresh_from_db()
     assert stats["processed"] == 1
-    assert stats["queued"] == 1
-    assert stats["reindexed"] == 0
-    assert result.status == ArchiveResult.StatusChoices.QUEUED
-    assert result.output_str == ""
-    assert result.output_json is None
+    assert stats["requested"] == 1
+    assert stats["queued"] == 0
+    assert stats["reindexed"] == 1
+    assert result.status == ArchiveResult.StatusChoices.SUCCEEDED
+    assert result.output_str == "old index hit"
+    assert result.output_json == {"indexed": True}
+    assert snapshot.status == Snapshot.StatusChoices.SEALED
+    assert missing_snapshot.status == Snapshot.StatusChoices.SEALED
+    assert missing_snapshot.archiveresult_set.filter(
+        plugin="search_backend_sqlite",
+        status__in=[ArchiveResult.StatusChoices.SUCCEEDED, ArchiveResult.StatusChoices.NORESULTS],
+    ).exists()
     assert not paused_snapshot.archiveresult_set.exists()
 
 
+@pytest.mark.django_db(transaction=True)
+def test_update_schedule_backfills_missing_search_index_without_reopening_snapshot():
+    from archivebox.base_models.models import get_or_create_system_user_pk
+    from archivebox.core.models import ArchiveResult, Snapshot
+    from archivebox.crawls.models import Crawl, CrawlSchedule
+
+    user_id = get_or_create_system_user_pk()
+    template = Crawl.objects.create(
+        urls="",
+        created_by_id=user_id,
+        status=Crawl.StatusChoices.SEALED,
+    )
+    schedule = CrawlSchedule.objects.create(
+        template=template,
+        schedule="daily",
+        config={"SCHEDULE_KIND": "update"},
+        created_by_id=user_id,
+    )
+    crawl = Crawl.objects.create(
+        urls="https://example.com/scheduled-search",
+        created_by_id=user_id,
+        status=Crawl.StatusChoices.SEALED,
+    )
+    snapshot = Snapshot.objects.create(
+        url="https://example.com/scheduled-search",
+        crawl=crawl,
+        status=Snapshot.StatusChoices.SEALED,
+        fs_version=Snapshot._fs_current_version(),
+    )
+    stale_snapshot = Snapshot.objects.create(
+        url="https://example.com/stale-scheduled-search",
+        crawl=crawl,
+        status=Snapshot.StatusChoices.SEALED,
+        fs_version=next(iter(Snapshot._FS_VERSION_MIGRATION_PATHS)),
+        retry_at=None,
+    )
+    (snapshot.output_dir / "title").mkdir(parents=True, exist_ok=True)
+    (snapshot.output_dir / "title" / "title.txt").write_text("Scheduled Search Index")
+
+    original_engine = os.environ.get("SEARCH_BACKEND_ENGINE")
+    os.environ["SEARCH_BACKEND_ENGINE"] = "sqlite"
+    try:
+        assert schedule.dispatch() is None
+        first_modified_at = schedule.modified_at
+        assert schedule.dispatch() is None
+    finally:
+        if original_engine is None:
+            os.environ.pop("SEARCH_BACKEND_ENGINE", None)
+        else:
+            os.environ["SEARCH_BACKEND_ENGINE"] = original_engine
+
+    snapshot.refresh_from_db()
+    stale_snapshot.refresh_from_db()
+    assert snapshot.status == Snapshot.StatusChoices.SEALED
+    assert (
+        ArchiveResult.objects.filter(
+            snapshot=snapshot,
+            plugin="search_backend_sqlite",
+            status__in=[ArchiveResult.StatusChoices.SUCCEEDED, ArchiveResult.StatusChoices.NORESULTS],
+        ).count()
+        == 1
+    )
+    assert not ArchiveResult.objects.filter(snapshot=snapshot, status=ArchiveResult.StatusChoices.QUEUED).exists()
+    assert stale_snapshot.retry_at is not None
+    assert not ArchiveResult.objects.filter(snapshot=stale_snapshot, plugin="search_backend_sqlite").exists()
+    assert schedule.modified_at >= first_modified_at
+
+
 @pytest.mark.django_db
 def test_build_filtered_snapshots_queryset_respects_resume_cutoff():
     from archivebox.base_models.models import get_or_create_system_user_pk

archivebox/tests/test_config_MAX_limits.py

@@ -67,7 +67,7 @@ def test_enqueue_discovered_snapshots_refreshes_crawl_limits(tmp_path):
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.crawls.models import Crawl
     from archivebox.core.models import Snapshot
-    from archivebox.plugins.hooks import run_hook
+    from archivebox.tests.conftest import run_test_hook
     from archivebox.services.runner import CrawlRunner
 
     crawl = Crawl.objects.create(
@@ -92,7 +92,7 @@ def test_enqueue_discovered_snapshots_refreshes_crawl_limits(tmp_path):
         encoding="utf-8",
     )
     hook_path = Path(str(files("abx_plugins.plugins.parse_txt_urls").joinpath("on_Snapshot__71_parse_txt_urls.py")))
-    process = run_hook(
+    process = run_test_hook(
         hook_path,
         parser_dir,
         config={"ABXPKG_LIB_DIR": str(tmp_path / "lib"), "SNAP_DIR": str(snap_dir)},
@@ -195,7 +195,7 @@ def test_seal_snapshot_cancels_queued_descendants_after_crawl_max_size():
     from archivebox.crawls.models import Crawl
     from archivebox.core.models import Snapshot
     from archivebox.services.snapshot_service import SnapshotService
-    from abx_dl.events import SnapshotCompletedEvent
+    from abx_dl.events import SnapshotCompletedEvent, SnapshotEvent
     from abx_dl.orchestrator import create_bus
 
     crawl = Crawl.objects.create(
@@ -231,17 +231,25 @@ def test_seal_snapshot_cancels_queued_descendants_after_crawl_max_size():
     )
 
     bus = create_bus(name=f"test_snapshot_limit_cancel_{str(crawl.id).replace('-', '_')}")
-    service = SnapshotService(bus, crawl_id=str(crawl.id))
+    SnapshotService(bus, crawl_id=str(crawl.id))
     try:
 
         async def emit_event() -> None:
-            await service.on_SnapshotCompletedEvent(
-                SnapshotCompletedEvent(
+            snapshot_event = bus.emit(
+                SnapshotEvent(
                     url=root.url,
                     snapshot_id=str(root.id),
                     output_dir=str(root.output_dir),
                 ),
             )
+            await snapshot_event.now()
+            completed_event = SnapshotCompletedEvent(
+                url=root.url,
+                snapshot_id=str(root.id),
+                output_dir=str(root.output_dir),
+            )
+            completed_event.event_parent_id = snapshot_event.event_id
+            await bus.emit(completed_event).now()
 
         asyncio.run(emit_event())
     finally:

archivebox/tests/test_crawl_runner.py

@@ -11,6 +11,20 @@ from archivebox.tests.conftest import install_real_binary, resolve_abxpkg_binary
 pytestmark = pytest.mark.django_db
 
 
+@pytest.mark.django_db(transaction=True)
+def test_crawl_runner_creates_progress_reporter_without_a_tty(crawl):
+    from archivebox.services.runner import CrawlRunner
+
+    runner = CrawlRunner(crawl)
+    runner.load_run_state()
+
+    live_ui = runner._create_live_ui()
+
+    assert live_ui is not None
+    assert live_ui.interactive_tty is False
+    asyncio.run(runner.bus.destroy(clear=False))
+
+
 @pytest.mark.django_db(transaction=True)
 def test_cancelled_crawl_projection_emits_abort_event_from_runner_bus():
     from archivebox.base_models.models import get_or_create_system_user_pk
@@ -247,7 +261,9 @@ def test_snapshot_started_state_keeps_retry_at_lease():
         retry_at=before,
     )
 
-    assert snapshot.tick_claimed(lock_seconds=60) is True
+    assert snapshot.claim_processing_lock(lock_seconds=60) is True
+    snapshot.refresh_from_db()
+    assert snapshot.advance_lifecycle() is True
 
     snapshot.refresh_from_db()
     assert snapshot.status == Snapshot.StatusChoices.STARTED
@@ -602,6 +618,35 @@ def test_crawl_runner_empty_plugin_selection_emits_lifecycle_and_seals_crawl(tmp
     assert snapshot.archiveresult_set.count() == 0
 
 
+@pytest.mark.django_db(transaction=True)
+def test_crawl_runner_preflights_plugins_before_crawl_lifecycle(tmp_path):
+    from abx_dl.events import CrawlEvent, CrawlSetupEvent, InstallEvent
+    from archivebox.base_models.models import get_or_create_system_user_pk
+    from archivebox.crawls.models import Crawl
+    from archivebox.services.runner import CrawlRunner
+
+    crawl = Crawl.objects.create(
+        urls="https://example.com",
+        config={
+            "ABXPKG_LIB_DIR": str(tmp_path / "lib"),
+            "PLUGINS": "base",
+            "CHROME_BINARY": "",
+        },
+        created_by_id=get_or_create_system_user_pk(),
+    )
+    runner = CrawlRunner(crawl)
+    event_order = []
+    runner.bus.on(InstallEvent, lambda _event: event_order.append(InstallEvent))
+    runner.bus.on(CrawlEvent, lambda _event: event_order.append(CrawlEvent))
+    runner.bus.on(CrawlSetupEvent, lambda _event: event_order.append(CrawlSetupEvent))
+
+    asyncio.run(runner.run())
+
+    assert event_order.count(InstallEvent) == 1
+    assert event_order.index(InstallEvent) < event_order.index(CrawlEvent)
+    assert event_order.index(InstallEvent) < event_order.index(CrawlSetupEvent)
+
+
 @pytest.mark.django_db(transaction=True)
 def test_crawl_runner_resolves_persona_and_crawl_config_for_each_live_snapshot():
     from abx_dl.events import SnapshotCompletedEvent
@@ -1068,60 +1113,13 @@ def test_crawl_completed_event_seals_finished_crawl():
     assert crawl.retry_at is None
 
 
-@pytest.mark.django_db(transaction=True)
-def test_snapshot_completed_event_defers_finished_crawl_seal():
-    from archivebox.base_models.models import get_or_create_system_user_pk
-    from archivebox.crawls.models import Crawl
-    from archivebox.core.models import Snapshot
-    from archivebox.services.snapshot_service import SnapshotService
-    from abx_dl.events import SnapshotCompletedEvent
-    from abx_dl.orchestrator import create_bus
-    from django.utils import timezone
-
-    crawl = Crawl.objects.create(
-        urls="https://example.com",
-        created_by_id=get_or_create_system_user_pk(),
-        status=Crawl.StatusChoices.STARTED,
-        retry_at=timezone.now(),
-    )
-    snapshot = Snapshot.objects.create(
-        url="https://example.com",
-        crawl=crawl,
-        status=Snapshot.StatusChoices.STARTED,
-        retry_at=None,
-    )
-
-    bus = create_bus(name=f"test_snapshot_completed_finished_crawl_{str(crawl.id).replace('-', '_')}")
-    service = SnapshotService(bus, crawl_id=str(crawl.id))
-    try:
-
-        async def emit_completed() -> None:
-            await service.on_SnapshotCompletedEvent(
-                SnapshotCompletedEvent(
-                    url="https://example.com",
-                    snapshot_id=str(snapshot.id),
-                    output_dir=str(snapshot.output_dir),
-                ),
-            )
-
-        asyncio.run(emit_completed())
-    finally:
-        asyncio.run(bus.destroy())
-
-    snapshot.refresh_from_db()
-    crawl.refresh_from_db()
-    assert snapshot.status == Snapshot.StatusChoices.SEALED
-    assert crawl.status == Crawl.StatusChoices.STARTED
-    assert crawl.retry_at is not None
-
-
 @pytest.mark.django_db(transaction=True)
 def test_snapshot_completed_event_bus_defers_finished_crawl_seal():
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.crawls.models import Crawl
     from archivebox.core.models import Snapshot
     from archivebox.services.snapshot_service import SnapshotService
-    from abx_dl.events import SnapshotCompletedEvent
+    from abx_dl.events import SnapshotCompletedEvent, SnapshotEvent
     from abx_dl.orchestrator import create_bus
     from django.utils import timezone
 
@@ -1139,20 +1137,25 @@ def test_snapshot_completed_event_bus_defers_finished_crawl_seal():
     )
 
     bus = create_bus(name=f"test_snapshot_completed_bus_finished_crawl_{str(crawl.id).replace('-', '_')}")
-    service = SnapshotService(bus, crawl_id=str(crawl.id))
-    assert service is not None
+    SnapshotService(bus, crawl_id=str(crawl.id))
     try:
 
         async def emit_completed() -> None:
-            emitted = bus.emit(
-                SnapshotCompletedEvent(
+            snapshot_event = bus.emit(
+                SnapshotEvent(
                     url="https://example.com",
                     snapshot_id=str(snapshot.id),
                     output_dir=str(snapshot.output_dir),
                 ),
             )
-            await emitted.wait()
-            await emitted.event_results_list()
+            await snapshot_event.now()
+            completed_event = SnapshotCompletedEvent(
+                url="https://example.com",
+                snapshot_id=str(snapshot.id),
+                output_dir=str(snapshot.output_dir),
+            )
+            completed_event.event_parent_id = snapshot_event.event_id
+            await bus.emit(completed_event).now()
 
         asyncio.run(emit_completed())
     finally:
@@ -1164,3 +1167,58 @@ def test_snapshot_completed_event_bus_defers_finished_crawl_seal():
     assert snapshot.status == Snapshot.StatusChoices.SEALED
     assert crawl.status == Crawl.StatusChoices.STARTED
     assert crawl.retry_at is not None
+
+
+@pytest.mark.django_db(transaction=True)
+def test_delayed_snapshot_completion_cannot_seal_new_run():
+    from datetime import timedelta
+
+    from archivebox.base_models.models import get_or_create_system_user_pk
+    from archivebox.crawls.models import Crawl
+    from archivebox.core.models import Snapshot
+    from archivebox.services.snapshot_service import SnapshotService
+    from abx_dl.events import SnapshotCompletedEvent, SnapshotEvent
+    from abx_dl.orchestrator import create_bus
+    from asgiref.sync import sync_to_async
+    from django.utils import timezone
+
+    crawl = Crawl.objects.create(urls="https://example.com", created_by_id=get_or_create_system_user_pk())
+    old_retry_at = timezone.now() + timedelta(minutes=5)
+    new_retry_at = old_retry_at + timedelta(minutes=5)
+    snapshot = Snapshot.objects.create(
+        url="https://example.com",
+        crawl=crawl,
+        status=Snapshot.StatusChoices.STARTED,
+        retry_at=old_retry_at,
+    )
+
+    bus = create_bus(name=f"test_delayed_snapshot_completion_{str(crawl.id).replace('-', '_')}")
+    SnapshotService(bus, crawl_id=str(crawl.id))
+    try:
+
+        async def emit_runs() -> None:
+            old_event = bus.emit(
+                SnapshotEvent(url=snapshot.url, snapshot_id=str(snapshot.id), output_dir=str(snapshot.output_dir)),
+            )
+            await old_event.now()
+            await sync_to_async(Snapshot.objects.filter(pk=snapshot.pk).update, thread_sensitive=True)(retry_at=new_retry_at)
+            new_event = bus.emit(
+                SnapshotEvent(url=snapshot.url, snapshot_id=str(snapshot.id), output_dir=str(snapshot.output_dir)),
+            )
+            await new_event.now()
+            completed_event = SnapshotCompletedEvent(
+                url=snapshot.url,
+                snapshot_id=str(snapshot.id),
+                output_dir=str(snapshot.output_dir),
+            )
+            completed_event.event_parent_id = old_event.event_id
+            await bus.emit(completed_event).now()
+
+        asyncio.run(emit_runs())
+    finally:
+        asyncio.run(bus.wait_until_idle())
+        asyncio.run(bus.destroy())
+
+    snapshot.refresh_from_db()
+    assert snapshot.status == Snapshot.StatusChoices.STARTED
+    assert snapshot.retry_at == new_retry_at

archivebox/tests/test_crawl_service.py

@@ -80,7 +80,7 @@ def test_crawl_service_run_processes_queued_crawl_and_applies_crawl_config(tmp_p
     assert queued_state["retry_at"] is not None
     assert queued_state["config"]["PLUGINS"] == "wget,parse_html_urls"
     assert queued_state["config"]["URL_DENYLIST"] == "/contact$"
-    # add --bg seeds Crawl.urls as CrawlSeed JSONL and returns; the runner
+    # add --bg stores the submitted URL list in Crawl.urls and returns; the runner
     # materializes Snapshot rows + applies URL_DENYLIST when it claims the
     # crawl, not at add time. The post-run assertions below verify those.
     assert queued_state["snapshots"] == []

archivebox/tests/test_hooks.py

@@ -80,14 +80,14 @@ class TestJSONLParsing:
 
     @staticmethod
     def run_hashes_hook(tmp_path):
-        from archivebox.plugins.hooks import run_hook
+        from archivebox.tests.conftest import run_test_hook
 
         snap_dir = tmp_path / "hash-snapshot"
         output_dir = snap_dir / "hashes"
         output_dir.mkdir(parents=True)
         (snap_dir / "source.txt").write_text("real parser input", encoding="utf-8")
         hook_path = Path(str(files("abx_plugins.plugins.hashes").joinpath("on_Snapshot__93_hashes.py")))
-        process = run_hook(
+        process = run_test_hook(
             hook_path,
             output_dir,
             config={"ABXPKG_LIB_DIR": str(tmp_path / "lib"), "SNAP_DIR": str(snap_dir)},
@@ -100,7 +100,7 @@ class TestJSONLParsing:
 
     @staticmethod
     def run_parser_hook(tmp_path):
-        from archivebox.plugins.hooks import run_hook
+        from archivebox.tests.conftest import run_test_hook
 
         snap_dir = tmp_path / "parser-snapshot"
         staticfile_dir = snap_dir / "staticfile"
@@ -112,7 +112,7 @@ class TestJSONLParsing:
             encoding="utf-8",
         )
         hook_path = Path(str(files("abx_plugins.plugins.parse_txt_urls").joinpath("on_Snapshot__71_parse_txt_urls.py")))
-        process = run_hook(
+        process = run_test_hook(
             hook_path,
             output_dir,
             config={"ABXPKG_LIB_DIR": str(tmp_path / "lib"), "SNAP_DIR": str(snap_dir)},
@@ -417,7 +417,7 @@ class TestHookExecution:
 
     @pytest.mark.django_db(transaction=True)
     def test_real_js_hook_runs_through_abxpkg_shebang(self, tmp_path, hermetic_lib_dir):
-        from archivebox.plugins.hooks import run_hook
+        from archivebox.tests.conftest import run_test_hook
         from archivebox.services.runner import run_install
 
         lib_dir = hermetic_lib_dir
@@ -427,7 +427,7 @@ class TestHookExecution:
         snap_dir = crawl_dir / "snapshot"
         hook_path = Path(str(files("abx_plugins.plugins.chrome").joinpath("on_CrawlSetup__89_chrome_kill_zombies.js")))
 
-        process = run_hook(
+        process = run_test_hook(
             hook_path,
             crawl_dir / "chrome",
             config={
@@ -522,7 +522,7 @@ class TestSnapshotHookOutput:
         [(True, "succeeded"), (False, "skipped")],
     )
     def test_hashes_hook_emits_real_archive_result(self, tmp_path, enabled, expected_status):
-        from archivebox.plugins.hooks import extract_records_from_process, run_hook
+        from archivebox.tests.conftest import run_test_hook
 
         snap_dir = tmp_path / f"snapshot-{expected_status}"
         output_dir = snap_dir / "hashes"
@@ -530,7 +530,7 @@ class TestSnapshotHookOutput:
         (snap_dir / "source.txt").write_text("real hook protocol input", encoding="utf-8")
         hook_path = Path(str(files("abx_plugins.plugins.hashes").joinpath("on_Snapshot__93_hashes.py")))
 
-        process = run_hook(
+        process = run_test_hook(
             hook_path,
             output_dir,
             config={
@@ -544,51 +544,21 @@ class TestSnapshotHookOutput:
         process.refresh_from_db()
 
         assert process.exit_code == 0, process.stderr
-        records = extract_records_from_process(process)
+        records = process.get_records()
         assert len(records) == 1
         assert records[0]["type"] == "ArchiveResult"
         assert records[0]["status"] == expected_status
-        assert records[0]["plugin"] == "hashes"
-        assert records[0]["hook_name"] == hook_path.name
-        assert records[0]["plugin_hook"] == str(hook_path)
+        assert process.cmd[0] == str(hook_path)
         if enabled:
             assert (output_dir / "hashes.json").is_file()
         else:
             assert records[0]["output_str"] == "HASHES_ENABLED=False"
 
 
-class TestPluginMetadata:
-    """Test that plugin metadata is added to JSONL records."""
-
-    @pytest.mark.django_db(transaction=True)
-    def test_python_hook_metadata_comes_from_executed_shipped_hook(self, tmp_path):
-        from archivebox.plugins.hooks import extract_records_from_process, run_hook
-
-        snap_dir = tmp_path / "snapshot-metadata"
-        output_dir = snap_dir / "hashes"
-        output_dir.mkdir(parents=True)
-        (snap_dir / "source.txt").write_text("metadata", encoding="utf-8")
-        script = Path(str(files("abx_plugins.plugins.hashes").joinpath("on_Snapshot__93_hashes.py")))
-        process = run_hook(
-            script,
-            output_dir,
-            config={"ABXPKG_LIB_DIR": str(tmp_path / "lib"), "SNAP_DIR": str(snap_dir)},
-            timeout=30,
-            url="https://example.com/metadata",
-        )
-        process.refresh_from_db()
-
-        assert process.exit_code == 0, process.stderr
-        records = extract_records_from_process(process)
-        assert records[0]["plugin"] == "hashes"
-        assert records[0]["hook_name"] == script.name
-        assert records[0]["plugin_hook"] == str(script)
-
-
 @pytest.mark.django_db(transaction=True)
-def test_run_hook_exports_singular_node_modules_dir_with_colon_node_path(tmp_path, hermetic_lib_dir):
+def test_abx_dl_hook_execution_exports_singular_node_modules_dir_with_colon_node_path(tmp_path, hermetic_lib_dir):
     """Hook subprocesses must get a real NODE_MODULES_DIR even when NODE_PATH has multiple entries."""
-    from archivebox.plugins.hooks import run_hook
+    from archivebox.tests.conftest import run_test_hook
     from archivebox.services.runner import run_install
 
     lib_dir = hermetic_lib_dir
@@ -603,7 +573,7 @@ def test_run_hook_exports_singular_node_modules_dir_with_colon_node_path(tmp_pat
     crawl_dir = tmp_path / "crawl"
     output_dir = crawl_dir / "chrome"
     hook_path = Path(str(files("abx_plugins.plugins.chrome").joinpath("on_CrawlSetup__89_chrome_kill_zombies.js")))
-    process = run_hook(
+    process = run_test_hook(
         hook_path,
         output_dir,
         config={
@@ -625,16 +595,16 @@ def test_run_hook_exports_singular_node_modules_dir_with_colon_node_path(tmp_pat
 
 
 @pytest.mark.django_db(transaction=True)
-def test_run_hook_executes_python_hooks_through_abxpkg_shebang(tmp_path):
+def test_abx_dl_executes_python_hooks_through_abxpkg_shebang(tmp_path):
     """ArchiveBox treats Python hooks as opaque abxpkg-launched executables."""
-    from archivebox.plugins.hooks import run_hook
+    from archivebox.tests.conftest import run_test_hook
 
     snap_dir = tmp_path / "snapshot"
     output_dir = snap_dir / "hashes"
     output_dir.mkdir(parents=True)
     (snap_dir / "source.txt").write_text("real runtime hook input", encoding="utf-8")
     hook_path = Path(str(files("abx_plugins.plugins.hashes").joinpath("on_Snapshot__93_hashes.py")))
-    process = run_hook(
+    process = run_test_hook(
         hook_path,
         output_dir,
         config={

archivebox/tests/test_machine_models.py

@@ -4,8 +4,8 @@ Unit tests for machine module models: Machine, NetworkInterface, Binary, Process
 Tests cover:
 1. Machine model creation and current() method
 2. NetworkInterface model and network detection
-3. Binary model lifecycle and state machine
-4. Process model lifecycle, hierarchy, and state machine
+3. Binary model lifecycle
+4. Process model lifecycle and hierarchy
 5. JSONL serialization/deserialization
 6. Manager methods
 7. Process tracking methods (replacing pid_utils)
@@ -29,8 +29,6 @@ from archivebox.machine.models import (
     NetworkInterface,
     Binary,
     Process,
-    BinaryMachine,
-    ProcessMachine,
     MACHINE_RECHECK_INTERVAL,
     PID_REUSE_WINDOW,
     PROCESS_TIMEOUT_GRACE,
@@ -509,27 +507,19 @@ class TestBinaryModel:
         assert symlink.resolve() == source.resolve()
 
 
-class TestBinaryStateMachine:
-    """Test the BinaryMachine state machine."""
+class TestBinaryLifecycle:
+    """Test Binary lifecycle prerequisites."""
 
     @pytest.fixture(autouse=True)
     def setup_binary(self, binary):
         self.binary = binary
 
-    def test_binary_state_machine_initial_state(self):
-        """BinaryMachine should start in queued state."""
-        sm = BinaryMachine(self.binary)
-        assert sm.current_state_value == Binary.StatusChoices.QUEUED
-
-    def test_binary_state_machine_can_start(self):
-        """BinaryMachine.can_start() should check name and binproviders."""
-        sm = BinaryMachine(self.binary)
-        assert sm.can_install()
+    def test_binary_can_install_checks_name_and_binproviders(self):
+        assert self.binary.can_install
 
         self.binary.binproviders = ""
         self.binary.save()
-        sm = BinaryMachine(self.binary)
-        assert not sm.can_install()
+        assert not self.binary.can_install
 
 
 class TestProcessModel:
@@ -1002,38 +992,5 @@ class TestProcessClassMethods:
         assert child.exit_code == 143
 
 
-class TestProcessStateMachine:
-    """Test the ProcessMachine state machine."""
-
-    @pytest.fixture(autouse=True)
-    def setup_process(self, process):
-        self.process = process
-
-    def test_process_state_machine_initial_state(self):
-        """ProcessMachine should start in queued state."""
-        sm = ProcessMachine(self.process)
-        assert sm.current_state_value == Process.StatusChoices.QUEUED
-
-    def test_process_state_machine_can_start(self):
-        """ProcessMachine.can_start() should check cmd and machine."""
-        sm = ProcessMachine(self.process)
-        assert sm.can_start()
-
-        self.process.cmd = []
-        self.process.save()
-        sm = ProcessMachine(self.process)
-        assert not sm.can_start()
-
-    def test_process_state_machine_is_exited(self):
-        """ProcessMachine.is_exited() should check exit_code."""
-        sm = ProcessMachine(self.process)
-        assert not sm.is_exited()
-
-        self.process.exit_code = 0
-        self.process.save()
-        sm = ProcessMachine(self.process)
-        assert sm.is_exited()
-
-
 if __name__ == "__main__":
     pytest.main([__file__, "-v"])

archivebox/tests/test_machine_service.py

@@ -13,6 +13,7 @@ pytestmark = pytest.mark.django_db(transaction=True)
 def _runtime_env(data_dir: Path, **extra: str) -> dict[str, str]:
     return {
         "ABXPKG_LIB_DIR": str(data_dir / "lib"),
+        "PLUGINS": "liteparse",
         "LITEPARSE_ENABLED": "True",
         "TIMEOUT": "180",
         "ABXPKG_INSTALL_TIMEOUT": "180",

archivebox/tests/test_migrations_04_to_09.py

@@ -6,6 +6,7 @@ import sqlite3
 from .migrations_helpers import (
     SCHEMA_0_4,
     create_data_dir_structure,
+    current_snapshot_dir,
     filesystem_manifest,
     run_archivebox_migration_cmd,
     seed_0_4_data,
@@ -94,8 +95,8 @@ def test_oldest_django_collection_migrates_end_to_end_without_data_loss(tmp_path
 
     for timestamp, expected_tree in original_trees.items():
         legacy_dir = tmp_path / "archive" / timestamp
-        assert legacy_dir.is_symlink()
-        migrated_tree = filesystem_manifest(legacy_dir.resolve())
+        assert not legacy_dir.exists()
+        migrated_tree = filesystem_manifest(current_snapshot_dir(tmp_path, db_path, timestamp))
         assert {path: migrated_tree.get(path) for path in expected_tree} == expected_tree
 
     with sqlite3.connect(db_path) as connection:

archivebox/tests/test_migrations_07_to_09.py

@@ -403,7 +403,7 @@ def test_update_saves_migrated_snapshots_without_foreign_key_errors(archive_07):
     result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
     assert result.returncode == 0, f"Init failed: {result.stderr}"
 
-    result = run_archivebox_migration_cmd(work_dir, ["update"], timeout=60)
+    result = run_archivebox_migration_cmd(work_dir, ["update", "--migrate-only"], timeout=60)
     output = result.stdout + result.stderr
     assert result.returncode == 0, f"Update failed after migration: {result.stderr}"
     assert "FOREIGN KEY constraint failed" not in output

archivebox/tests/test_migrations_08_to_09.py

@@ -6,7 +6,7 @@ Migration tests from 0.8.x to 0.9.x.
 - Crawl model for grouping URLs
 - Seed model (removed in 0.9.x)
 - UUID primary keys for Snapshot
-- Status fields for state machine
+- Status fields for queued lifecycle processing
 - New fields like depth, retry_at, etc.
 """
 
@@ -21,6 +21,7 @@ import pytest
 from .migrations_helpers import (
     SCHEMA_0_7,
     SCHEMA_0_8,
+    current_snapshot_dir,
     filesystem_manifest,
     seed_0_8_data,
     seed_0_7_data,
@@ -91,7 +92,7 @@ def convert_legacy_tags_to_uuid(conn: sqlite3.Connection) -> None:
 
 
 @pytest.mark.parametrize("uuid_tags", (False, True), ids=("integer-tags", "uuid-tags"))
-def test_migration_preserves_extended_08_metadata(migration_08_data, uuid_tags):
+def test_migration_preserves_supported_extended_08_metadata(migration_08_data, uuid_tags):
     work_dir, db_path, original_data = migration_08_data
     snapshot = original_data["snapshots"][1]
     parent = original_data["snapshots"][0]
@@ -151,11 +152,12 @@ def test_migration_preserves_extended_08_metadata(migration_08_data, uuid_tags):
         snapshot_row = conn.execute(
             """
             SELECT depth, config, notes, num_uses_failed, num_uses_succeeded,
-                   parent_snapshot_id, current_step, fs_version
+                   parent_snapshot_id, fs_version
             FROM core_snapshot WHERE id = ?
             """,
             (snapshot["id"],),
         ).fetchone()
+        snapshot_columns = {row[1] for row in conn.execute("PRAGMA table_info(core_snapshot)")}
         result_notes = conn.execute(
             "SELECT notes FROM core_archiveresult WHERE id = REPLACE(?, '-', '')",
             (archiveresult["uuid"],),
@@ -173,9 +175,9 @@ def test_migration_preserves_extended_08_metadata(migration_08_data, uuid_tags):
         metadata["num_uses_failed"],
         metadata["num_uses_succeeded"],
         parent["id"],
-        metadata["current_step"],
         metadata["fs_version"],
     )
+    assert "current_step" not in snapshot_columns
     assert result_notes == ("legacy archive result notes",)
     assert migrated_tag == tag_metadata
     assert migrated_counts == expected_counts
@@ -481,7 +483,7 @@ def test_hyphenated_crawl_ids_are_normalized_before_snapshot_saves(migration_08_
     assert hyphenated_crawls == 0
     assert hyphenated_snapshot_refs == 0
 
-    result = run_archivebox_migration_cmd(work_dir, ["update"], timeout=60)
+    result = run_archivebox_migration_cmd(work_dir, ["update", "--migrate-only"], timeout=60)
     output = result.stdout + result.stderr
     assert result.returncode == 0, f"Update failed after migration: {result.stderr}"
     assert "FOREIGN KEY constraint failed" not in output
@@ -800,7 +802,7 @@ def test_update_preserves_legacy_folder_timestamp_over_index_float_variant(tmp_p
     conn.close()
 
     assert row == (timestamp,)
-    assert (work_dir / "archive" / timestamp).is_symlink()
+    assert not (work_dir / "archive" / timestamp).exists()
     assert not (work_dir / "archive" / f"{timestamp}.0").exists()
     assert not (work_dir / "invalid").exists()
 
@@ -842,8 +844,8 @@ def test_update_preserves_distinct_legacy_dirs_with_integer_and_float_timestamps
     conn.close()
 
     assert rows == [("1508259732",), ("1508259732.0",)]
-    assert (work_dir / "archive" / "1508259732").is_symlink()
-    assert (work_dir / "archive" / "1508259732.0").is_symlink()
+    assert not (work_dir / "archive" / "1508259732").exists()
+    assert not (work_dir / "archive" / "1508259732.0").exists()
     assert not (work_dir / "invalid").exists()
 
 
@@ -991,8 +993,8 @@ def test_07_filesystem_hop_preserves_complete_output_tree(tmp_path):
 
     for timestamp, expected_tree in original_trees.items():
         legacy_dir = tmp_path / "archive" / timestamp
-        assert legacy_dir.is_symlink()
-        migrated_tree = filesystem_manifest(legacy_dir.resolve())
+        assert not legacy_dir.exists()
+        migrated_tree = filesystem_manifest(current_snapshot_dir(tmp_path, db_path, timestamp))
         assert {path: migrated_tree.get(path) for path in expected_tree} == expected_tree
     assert (destination / "preexisting-output.bin").read_bytes() == b"destination-only output"
 
@@ -1065,7 +1067,10 @@ def test_each_declared_filesystem_hop_preserves_outputs(migration_08_data, fs_ve
     result = run_archivebox_migration_cmd(work_dir, ["update", "--migrate-only"], timeout=180)
     assert result.returncode == 0, result.stderr
 
-    migrated_dir = source_dir.resolve()
+    migrated_dir = current_snapshot_dir(work_dir, db_path, snapshot["timestamp"])
     assert {path: filesystem_manifest(migrated_dir).get(path) for path in expected_tree} == expected_tree
+    if fs_version in ("0.7.0", "0.8.0", "0.8.5"):
+        assert not source_dir.exists()
+        assert not source_dir.is_symlink()
     with sqlite3.connect(db_path) as connection:
         assert connection.execute("SELECT fs_version FROM core_snapshot WHERE id = ?", (snapshot["id"],)).fetchone() == ("0.9.4",)

archivebox/tests/test_opencode_agent.py

@@ -1,13 +1,19 @@
+import asyncio
 import os
+import re
+import shutil
 import socket
+from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
 from types import SimpleNamespace
-from urllib.parse import quote
+from urllib.parse import parse_qs, quote, urlsplit
 
 import pytest
 import requests
+from asgiref.testing import ApplicationCommunicator
 
 from archivebox.tests.conftest import ADMIN_TEST_HOST, run_archivebox_cmd
+from archivebox.config.common import get_config
 
 
 pytestmark = pytest.mark.django_db(transaction=True)
@@ -72,8 +78,8 @@ def opencode_archive_config(initialized_archive):
 
 
 @pytest.fixture
-def live_opencode(opencode_archive_config):
-    from archivebox.opencode import views
+def installed_opencode(opencode_archive_config):
+    from abx_plugins.plugins.opencode import runtime
 
     install = run_archivebox_cmd(
         ["install", "opencode", "--binproviders=env,pnpm"],
@@ -84,45 +90,61 @@ def live_opencode(opencode_archive_config):
     assert install.returncode == 0, install.stderr or install.stdout
     _reset_runtime_config()
 
-    config = views._machine_config()
-    settings = views._settings(config)
+    config = get_config().model_dump(mode="json")
+    settings = runtime._settings(config, opencode_archive_config.data_dir)
     settings["archivebox_base_url"] = "http://admin.archivebox.localhost:8000"
     settings["archivebox_admin_url"] = "http://admin.archivebox.localhost:8000/admin"
     settings["archivebox_api_url"] = "http://admin.archivebox.localhost:8000/api/"
-    binary, _, binary_env = views._resolve_binary(settings["binary"], settings["config"])
+    binary, binary_env = runtime._resolve_binary(settings["binary"], settings["config"])
     version = binary.exec(
         cmd=("--version",),
         env={**os.environ, **binary_env},
         timeout=120,
     )
     assert version.returncode == 0, version.stderr or version.stdout
-    ok, error = views._ensure_opencode(settings)
+    return SimpleNamespace(config=opencode_archive_config, settings=settings)
+
+
+@pytest.fixture
+def live_opencode(installed_opencode):
+    from abx_plugins.plugins.opencode import runtime
+
+    settings = installed_opencode.settings
+    ok, error = runtime._ensure_opencode(settings)
     assert ok, error
 
-    process = views._PROCESS
+    process = runtime._PROCESS
     assert process is not None
     try:
-        yield SimpleNamespace(config=opencode_archive_config, settings=settings, process=process)
+        yield SimpleNamespace(config=installed_opencode.config, settings=settings, process=process)
     finally:
-        if views._PROCESS and views._PROCESS.poll() is None:
-            views._PROCESS.terminate()
-            views._PROCESS.wait(timeout=10)
-        views._PROCESS = None
+        runtime._stop_owned_process()
 
 
 def test_opencode_disabled_route_does_not_start_server(client, initialized_archive):
     from archivebox.machine.models import Machine
-    from archivebox.opencode import views
+    from abx_plugins.plugins.opencode import runtime
 
     os.chdir(initialized_archive)
     Machine.from_json({"config": {"OPENCODE_ENABLED": False}})
     _reset_runtime_config()
-    assert views._machine_config()["OPENCODE_ENABLED"] is False
+    assert get_config().model_dump(mode="json")["OPENCODE_ENABLED"] is False
 
     response = client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
 
     assert response.status_code == 404
-    assert views._PROCESS is None or views._PROCESS.poll() is not None
+    assert runtime._PROCESS is None or runtime._PROCESS.poll() is not None
+
+
+def test_opencode_disabled_via_cli_stays_disabled(admin_client, initialized_archive):
+    _set_archivebox_config(initialized_archive, "OPENCODE_ENABLED=False")
+
+    assert get_config().OPENCODE_ENABLED is False
+    assert admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST).status_code == 404
+    for path in ("/add/", "/admin/core/snapshot/"):
+        response = admin_client.get(path, HTTP_HOST=ADMIN_TEST_HOST)
+        assert response.status_code == 200
+        assert b'href="/admin/agent"' not in response.content
 
 
 def test_opencode_agent_requires_superuser_when_enabled(client, db, django_user_model, live_opencode):
@@ -130,6 +152,11 @@ def test_opencode_agent_requires_superuser_when_enabled(client, db, django_user_
     assert response.status_code == 302
     assert "/admin/login/" in response.headers["Location"]
 
+    next_path = "/admin/agent?x=1&next=https://example.com"
+    response = client.get(next_path, HTTP_HOST=ADMIN_TEST_HOST)
+    assert response.status_code == 302
+    assert parse_qs(urlsplit(response.headers["Location"]).query) == {"next": [next_path]}
+
     user = django_user_model.objects.create_user(username="regular", password="testpassword")
     client.force_login(user)
     response = client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
@@ -161,29 +188,52 @@ def test_opencode_proxy_blocks_cross_site_fetch_metadata(admin_client, db, live_
 
 
 def test_opencode_agent_superuser_gets_admin_wrapper(admin_client, live_opencode):
-    from archivebox.opencode import views
+    from abx_plugins.plugins.opencode import runtime
 
     response = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
-    recent_session_id = views._recent_session_id(live_opencode.settings)
+    recent_session_id = response.context["recent_session_id"]
+    session_path = runtime._project_route(live_opencode.config.data_dir, recent_session_id)
 
     assert response.status_code == 200
     assert recent_session_id
-    assert f'<iframe src="{views._project_route(live_opencode.config.data_dir, recent_session_id)}"'.encode() in response.content
+    assert f'<iframe src="{session_path}"'.encode() in response.content
     assert b'id="header"' in response.content
     assert b'id="progress-monitor"' in response.content
+    assert b'<a href="/admin/agent" class="navbar-item navbar-ai">' in response.content
+    add_page = admin_client.get("/add/", HTTP_HOST=ADMIN_TEST_HOST)
+    assert add_page.status_code == 200
+    assert '<a href="/admin/agent">💬 Crawl with AI</a>'.encode() in add_page.content
+    assert response.context["proxy_prefix"] == runtime._PROXY_PREFIX
+    assert b"/_archivebox/health" not in response.content
+    assert b"window.setInterval(check, 3000)" not in response.content
+    assert response.headers["X-Frame-Options"] == "DENY"
+    assert response.headers["Content-Security-Policy"] == "frame-ancestors 'none'"
+
+    session = admin_client.get(
+        session_path,
+        HTTP_HOST=ADMIN_TEST_HOST,
+        HTTP_SEC_FETCH_SITE="same-origin",
+    )
+    assert session.status_code == 200
+    assert session.headers["X-Frame-Options"] == "SAMEORIGIN"
+    assert session.headers["Content-Security-Policy"] == "frame-ancestors 'self'"
 
 
 def test_opencode_proxy_serves_real_project_and_session(admin_client, live_opencode):
     workdir = str(live_opencode.config.data_dir.resolve())
     encoded_workdir = quote(workdir)
 
+    agent = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
+    assert agent.status_code == 200
+
     project = admin_client.get(
         f"/admin/agent/opencode/project/current?directory={encoded_workdir}",
         HTTP_HOST=ADMIN_TEST_HOST,
         HTTP_SEC_FETCH_SITE="same-origin",
     )
     assert project.status_code == 200
-    assert workdir.encode() in project.content
+    assert project.json()["id"] == "global"
+    assert not project.json().get("vcs")
 
     path = admin_client.get(
         f"/admin/agent/opencode/path?directory={encoded_workdir}",
@@ -191,7 +241,7 @@ def test_opencode_proxy_serves_real_project_and_session(admin_client, live_openc
         HTTP_SEC_FETCH_SITE="same-origin",
     )
     assert path.status_code == 200
-    assert workdir.encode() in path.content
+    assert path.json()["directory"] == workdir
 
     sessions = admin_client.get(
         f"/admin/agent/opencode/session?directory={encoded_workdir}&roots=true&limit=55",
@@ -199,7 +249,133 @@ def test_opencode_proxy_serves_real_project_and_session(admin_client, live_openc
         HTTP_SEC_FETCH_SITE="same-origin",
     )
     assert sessions.status_code == 200
-    assert b"id" in sessions.content
+    assert any(session["id"] == agent.context["recent_session_id"] and session["directory"] == workdir for session in sessions.json())
+    assert not (Path(workdir) / ".git").exists()
+
+
+def test_opencode_proxy_restarts_server_for_an_existing_agent_page(admin_client, live_opencode):
+    from abx_plugins.plugins.opencode import runtime
+
+    old_process = runtime._PROCESS
+    runtime._stop_owned_process()
+
+    response = admin_client.get(
+        "/admin/agent/opencode/global/health",
+        HTTP_HOST=ADMIN_TEST_HOST,
+        HTTP_SEC_FETCH_SITE="same-origin",
+    )
+
+    assert response.status_code == 200
+    assert runtime._PROCESS is not None
+    assert runtime._PROCESS is not old_process
+    assert runtime._PROCESS.poll() is None
+
+
+def test_concurrent_opencode_startup_waits_until_server_is_ready(live_opencode):
+    from abx_plugins.plugins.opencode import runtime
+
+    runtime._stop_owned_process()
+
+    with ThreadPoolExecutor(max_workers=2) as executor:
+        results = list(executor.map(runtime._ensure_opencode, [live_opencode.settings] * 2))
+
+    assert results == [(True, ""), (True, "")]
+    assert runtime._health(live_opencode.settings)
+
+
+def test_opencode_does_not_probe_or_replace_a_ready_owned_process(live_opencode):
+    from abx_plugins.plugins.opencode import runtime
+
+    process = runtime._PROCESS
+    settings = {**live_opencode.settings, "port": _free_port()}
+    settings["origin"] = f"http://{settings['host']}:{settings['port']}"
+
+    ok, error = runtime._ensure_opencode(settings)
+
+    assert ok, error
+    assert process is not None
+    assert runtime._PROCESS is process
+    assert process.poll() is None
+
+
+def test_opencode_proxy_does_not_wait_for_recovery_lock(admin_client, live_opencode):
+    from abx_plugins.plugins.opencode import runtime
+
+    workdir = quote(str(live_opencode.config.data_dir.resolve()))
+    assert runtime._owned_process_ready()
+    executor = ThreadPoolExecutor(max_workers=1)
+    runtime._PROCESS_LOCK.acquire()
+    try:
+        request = executor.submit(
+            admin_client.get,
+            f"/admin/agent/opencode/path?directory={workdir}",
+            HTTP_HOST=ADMIN_TEST_HOST,
+            HTTP_SEC_FETCH_SITE="same-origin",
+        )
+        response = request.result(timeout=5)
+    finally:
+        runtime._PROCESS_LOCK.release()
+        executor.shutdown(wait=True)
+
+    assert response.status_code == 200
+    assert str(live_opencode.config.data_dir.resolve()).encode() in response.content
+
+
+def test_opencode_proxy_waits_for_owned_process_readiness(admin_client, live_opencode):
+    from abx_plugins.plugins.opencode import runtime
+
+    process = runtime._PROCESS
+    assert process is not None
+    runtime._PROCESS_READY = None
+    workdir = quote(str(live_opencode.config.data_dir.resolve()))
+
+    response = admin_client.get(
+        f"/admin/agent/opencode/path?directory={workdir}",
+        HTTP_HOST=ADMIN_TEST_HOST,
+        HTTP_SEC_FETCH_SITE="same-origin",
+    )
+
+    assert response.status_code == 200
+    assert runtime._PROCESS is process
+    assert runtime._PROCESS_READY is process
+
+
+def test_opencode_proxy_preserves_protocol_headers(admin_client, live_opencode):
+    headers = {"HTTP_HOST": ADMIN_TEST_HOST, "HTTP_SEC_FETCH_SITE": "same-origin"}
+    created = admin_client.post(
+        "/admin/agent/opencode/pty",
+        data={"command": "/bin/sh", "args": []},
+        content_type="application/json",
+        **headers,
+    )
+    assert created.status_code == 200
+    path = f"/admin/agent/opencode/pty/{created.json()['id']}"
+    try:
+        response = admin_client.post(path + "/connect-token", HTTP_X_OPENCODE_TICKET="1", **headers)
+        assert response.status_code == 200, response.content
+        assert response.json()["ticket"]
+    finally:
+        deleted = admin_client.delete(path, **headers)
+    assert deleted.status_code == 200, deleted.content
+
+
+def test_opencode_static_assets_cache_privately_and_revalidate(admin_client, live_opencode):
+    headers = {"HTTP_HOST": ADMIN_TEST_HOST, "HTTP_SEC_FETCH_SITE": "same-origin"}
+    page = admin_client.get("/admin/agent/opencode/", **headers)
+    assert page.status_code == 200
+    asset = re.search(rb'src="(/admin/agent/opencode/assets/[^\"]+\.js)"', page.content)
+    assert asset is not None
+    path = asset[1].decode()
+    response = admin_client.get(path, **headers)
+    assert response.status_code == 200
+    assert "private" in response.headers["Cache-Control"]
+    assert "max-age=" in response.headers["Cache-Control"]
+    assert b"/admin/agent/opencode" in response.content
+    repeated = admin_client.get(path, HTTP_IF_NONE_MATCH=response.headers["ETag"], **headers)
+    assert repeated.status_code == 304
+    assert repeated.content == b""
+    assert repeated.headers["ETag"] == response.headers["ETag"]
+    assert admin_client.get("/admin/agent/opencode/path", **headers).headers["Cache-Control"] == "no-store"
 
 
 def test_opencode_proxy_sse_response_is_unbuffered(admin_client, live_opencode):
@@ -211,14 +387,93 @@ def test_opencode_proxy_sse_response_is_unbuffered(admin_client, live_opencode):
 
     assert response.status_code == 200
     assert response.streaming
+    assert response.is_async
     assert response.headers["X-Accel-Buffering"] == "no"
     assert response.headers["Cache-Control"] == "no-store"
 
 
-def test_opencode_starts_with_isolated_state(live_opencode):
+def test_opencode_proxy_sse_delivers_first_event_immediately(admin_client, live_opencode):
+    response = admin_client.get(
+        "/admin/agent/opencode/global/event",
+        HTTP_HOST=ADMIN_TEST_HOST,
+        HTTP_SEC_FETCH_SITE="same-origin",
+    )
+    assert response.status_code == 200
+
+    async def first_event():
+        stream = response.streaming_content
+        try:
+            async with asyncio.timeout(2):
+                return await anext(stream)
+        finally:
+            await stream.aclose()
+
+    assert b"server.connected" in asyncio.run(first_event())
+
+
+def test_opencode_proxy_sse_returns_headers_before_restart_finishes(admin_client, live_opencode):
+    from archivebox.core.asgi import application
+    from abx_plugins.plugins.opencode import runtime
+    from django.conf import settings as django_settings
+
+    owned_process = runtime._PROCESS
+    assert owned_process is not None
+    session_cookie_name = django_settings.SESSION_COOKIE_NAME
+    session_cookie = admin_client.cookies[session_cookie_name].value
+    path = "/admin/agent/opencode/global/event"
+
+    async def request_event_stream():
+        communicator = ApplicationCommunicator(
+            application,
+            {
+                "type": "http",
+                "asgi": {"version": "3.0"},
+                "http_version": "1.1",
+                "method": "GET",
+                "scheme": "http",
+                "path": path,
+                "raw_path": path.encode(),
+                "query_string": b"",
+                "headers": [
+                    (b"host", ADMIN_TEST_HOST.encode()),
+                    (b"cookie", f"{session_cookie_name}={session_cookie}".encode()),
+                    (b"sec-fetch-site", b"same-origin"),
+                ],
+                "client": ("127.0.0.1", 12345),
+                "server": ("127.0.0.1", 8000),
+            },
+        )
+        runtime._PROCESS_LOCK.acquire()
+        runtime._PROCESS = None
+        try:
+            await communicator.send_input({"type": "http.request", "body": b"", "more_body": False})
+            response_start = await communicator.receive_output(timeout=2)
+            assert response_start["type"] == "http.response.start"
+            assert response_start["status"] == 200
+        finally:
+            try:
+                await communicator.send_input({"type": "http.disconnect"})
+                await communicator.wait(timeout=5)
+            finally:
+                runtime._PROCESS = owned_process
+                runtime._PROCESS_LOCK.release()
+                await asyncio.get_running_loop().shutdown_default_executor()
+
+    asyncio.run(request_event_stream())
+    assert runtime._PROCESS is owned_process
+    assert owned_process.poll() is None
+
+
+def test_opencode_starts_with_isolated_state(admin_client, live_opencode):
     workdir = str(live_opencode.config.data_dir.resolve())
     state_dir = live_opencode.config.state_dir
 
+    assert not (Path(workdir) / ".git").exists()
+    agent = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
+    assert agent.status_code == 200
+    assert agent.context["recent_session_id"]
+    assert not (Path(workdir) / ".git").exists()
+
     project = requests.get(
         f"{live_opencode.settings['origin']}/project/current",
         params={"directory": workdir},
@@ -226,41 +481,119 @@ def test_opencode_starts_with_isolated_state(live_opencode):
     )
     project.raise_for_status()
 
+    config = requests.get(
+        f"{live_opencode.settings['origin']}/global/config",
+        timeout=live_opencode.settings["timeout"],
+    )
+    config.raise_for_status()
+
     assert Path(live_opencode.settings["workdir"]).resolve() == Path(workdir)
-    assert Path(str(project.json()["worktree"])).resolve() == Path(workdir)
+    assert project.json()["id"] == "global"
+    assert not project.json().get("vcs")
+    assert not config.json().get("model")
+    assert config.json()["snapshot"] is False
     assert live_opencode.process.poll() is None
-    assert (live_opencode.config.data_dir / ".git").is_dir()
+    path = requests.get(
+        f"{live_opencode.settings['origin']}/path",
+        params={"directory": workdir},
+        timeout=live_opencode.settings["timeout"],
+    )
+    path.raise_for_status()
+    assert Path(path.json()["directory"]).resolve() == Path(workdir)
+
+    diff = requests.get(
+        f"{live_opencode.settings['origin']}/vcs/diff",
+        params={"directory": workdir, "mode": "git"},
+        timeout=5,
+    )
+    diff.raise_for_status()
+    assert diff.json() == []
     assert (state_dir / "data" / "opencode" / "opencode.db").is_file()
     assert (state_dir / "SKILL.md").is_file()
     assert (state_dir / "config" / "opencode" / "skills" / "archivebox" / "SKILL.md").resolve() == state_dir / "SKILL.md"
 
 
-def test_opencode_state_dir_is_separate_from_workdir(tmp_path):
-    from archivebox.opencode import views
+def test_opencode_invalid_state_does_not_break_archivebox(admin_client, live_opencode):
+    from abx_plugins.plugins.opencode import runtime
 
-    workdir = tmp_path / "data"
-    settings = views._settings({"OPENCODE_WORKDIR": str(workdir)})
-    views._ensure_project_files(settings)
+    runtime._stop_owned_process()
+    invalid_state = live_opencode.config.state_dir / "config"
+    invalid_state.rename(live_opencode.config.state_dir / "saved-config")
+    invalid_state.write_text("Preserve this file.")
 
-    assert settings["workdir"] == workdir
-    assert settings["opencode_dir"] == workdir / "opencode"
-    assert settings["config_home"] == workdir / "opencode" / "config"
-    assert settings["data_home"] == workdir / "opencode" / "data"
-    assert settings["state_home"] == workdir / "opencode" / "state"
-    editable_skill = workdir / "opencode" / "SKILL.md"
-    loaded_skill = workdir / "opencode" / "config" / "opencode" / "skills" / "archivebox" / "SKILL.md"
-    assert editable_skill.exists()
-    assert loaded_skill.is_symlink()
-    assert loaded_skill.resolve() == editable_skill.resolve()
-    assert f"ArchiveBox collection directory: {workdir.resolve()}" in editable_skill.read_text()
+    for url in ("/admin/agent", "/admin/agent/opencode/global/health"):
+        response = admin_client.get(url, HTTP_HOST=ADMIN_TEST_HOST)
+        assert response.status_code == 503
+        assert str(invalid_state).encode() not in response.content
+
+    stream = admin_client.get("/admin/agent/opencode/global/event", HTTP_HOST=ADMIN_TEST_HOST)
+    assert stream.status_code == 200
+
+    async def read_failure():
+        return b"".join([chunk async for chunk in stream.streaming_content])
+
+    assert asyncio.run(read_failure()) == b'event: error\ndata: {"error":"OpenCode unavailable"}\n\n'
+
+    for url in ("/health/", "/add/", "/admin/core/snapshot/"):
+        response = admin_client.get(url, HTTP_HOST=ADMIN_TEST_HOST)
+        assert response.status_code == 200
+    assert invalid_state.read_text() == "Preserve this file."
 
 
-def test_opencode_rewrites_vite_preload_assets():
-    from archivebox.opencode import views
+@pytest.mark.parametrize(
+    ("damaged_file", "missing"),
+    [
+        ("runtime.py", True),
+        ("templates/agent.html", True),
+        ("templates/agent.html", False),
+        ("templates/navigation.html", False),
+        ("templates/add.html", False),
+    ],
+)
+def test_opencode_incomplete_install_does_not_break_archivebox(installed_opencode, tmp_path, damaged_file, missing):
+    import abx_plugins
 
-    body = b'const BL="modulepreload",UL=function(t){return"/"+t};const icon="/assets/sprite.svg#anthropic"'
-    rewritten = views._rewrite_text(body, {"origin": "http://127.0.0.1:4096"}).decode()
-
-    assert 'return"/"+t' not in rewritten
-    assert 'return"/admin/agent/opencode/"+t' in rewritten
-    assert '"/admin/agent/opencode/assets/sprite.svg#anthropic"' in rewritten
+    # Exercise a genuinely incomplete installation in a separate process;
+    # never alter the shared package or intercept Python imports.
+    site = tmp_path / "site"
+    installed = site / "abx_plugins"
+    shutil.copytree(Path(abx_plugins.__file__).parent, installed, ignore=shutil.ignore_patterns("__pycache__", "tests"))
+    damaged_path = installed / "plugins" / "opencode" / damaged_file
+    if missing:
+        damaged_path.unlink()
+    else:
+        damaged_path.write_text("{% invalid_template_tag %}")
+    expected_status = 200 if damaged_file in {"templates/navigation.html", "templates/add.html"} else 503
+    script = f"""
+import sys
+from pathlib import Path
+import abx_plugins
+from django.test import Client
+from django.contrib.auth import get_user_model
+assert Path(abx_plugins.__file__).is_relative_to({str(site)!r})
+user = get_user_model().objects.create_superuser(username='optional-service-test')
+client = Client(HTTP_HOST={ADMIN_TEST_HOST!r})
+client.force_login(user)
+for path in ('/health/', '/add/', '/admin/core/snapshot/'):
+    assert client.get(path).status_code == 200, path
+assert 'abx_plugins.plugins.opencode.runtime' not in sys.modules
+response = client.get('/admin/agent')
+assert response.status_code == {expected_status}, response.status_code
+if response.status_code == 503:
+    assert response.content == b'AI service unavailable. See server logs.'
+if {damaged_file != "runtime.py"!r}:
+    from abx_plugins.plugins.opencode import runtime
+    assert runtime._PROCESS is not None
+    assert runtime._PROCESS.poll() is None
+for path in ('/health/', '/add/', '/admin/core/snapshot/'):
+    assert client.get(path).status_code == 200, path
+print('OPTIONAL_SERVICE_FAILURE_ISOLATED')
+"""
+    result = run_archivebox_cmd(
+        ["shell", "-c", script],
+        cwd=installed_opencode.config.data_dir,
+        env={**installed_opencode.config.env, "PYTHONPATH": str(site)},
+        timeout=90,
+    )
+    assert result.returncode == 0, result.stderr or result.stdout
+    assert "OPTIONAL_SERVICE_FAILURE_ISOLATED" in result.stdout

archivebox/tests/test_opencode_browser.py

@@ -0,0 +1,371 @@
+"""Exercise the real agent wrapper with Chromium's native storage failures."""
+
+import asyncio
+import json
+import os
+import re
+import subprocess
+
+import pytest
+import requests
+
+from .conftest import get_free_port, run_archivebox_cmd, start_archivebox_server, stop_archivebox_process
+from .test_opencode_agent import _set_archivebox_config
+from .test_opencode_agent import installed_opencode as installed_opencode
+from .test_opencode_agent import live_opencode as live_opencode
+from .test_opencode_agent import opencode_archive_config as opencode_archive_config
+from .test_server_security_browser import browser_runtime as browser_runtime
+
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+def test_agent_navigation_stays_inside_mount(agent_server, browser_runtime):
+    server_url, _, _ = agent_server
+    script = r"""
+const assert = require('node:assert/strict');
+const puppeteer = require('puppeteer');
+const config = JSON.parse(require('node:fs').readFileSync(0, 'utf8'));
+(async () => {
+  const browser = await puppeteer.launch({executablePath: config.chrome, headless: true,
+    args: ['--no-sandbox', '--disable-frame-rate-limit']});
+  try {
+    const page = await browser.newPage();
+    await page.setViewport({width: 1440, height: 900});
+    await page.goto(config.url + '/admin/login/', {waitUntil: 'domcontentloaded'});
+    await page.locator('#login-form input[name="username"]').fill('agent-browser-test');
+    await page.locator('#login-form input[name="password"]').fill('test-password');
+    await Promise.all([page.waitForNavigation({waitUntil: 'domcontentloaded'}),
+      page.locator('#login-form input[type="submit"]').click()]);
+    assert.equal((await page.goto(config.url + '/admin/agent', {waitUntil: 'domcontentloaded'})).status(), 200);
+    await page.locator('#opencode-agent-welcome-dismiss').click();
+    const frame = await (await page.waitForSelector('iframe')).contentFrame();
+    await frame.waitForSelector('a[href*="/session"]');
+    const links = await frame.$$eval('a[href*="/session"]', nodes => nodes.map(node => node.getAttribute('href')));
+    assert.ok(links.length, 'OpenCode must expose session navigation');
+    for (const href of links) assert.ok(href.startsWith('/admin/agent/opencode/'), href);
+    await frame.locator('::-p-aria(New session[role="button"])').click();
+    await frame.waitForSelector('[contenteditable="true"]');
+    let navigation;
+    for (const link of await frame.$$('a[href*="/session"]')) {
+      if (await link.isVisible() && await link.evaluate(node => node.href) !== frame.url()) {
+        navigation = link;
+        break;
+      }
+    }
+    assert.ok(navigation, 'A visible link must navigate to another session route');
+    await navigation.click();
+    await frame.waitForSelector('[contenteditable="true"]');
+    assert.ok(new URL(frame.url()).pathname.startsWith('/admin/agent/opencode/'), frame.url());
+    const sessionUrl = frame.url();
+    assert.equal((await frame.goto(sessionUrl, {waitUntil: 'domcontentloaded'})).status(), 200);
+    await frame.waitForSelector('[contenteditable="true"]');
+    assert.ok(!(await frame.$eval('body', node => node.innerText)).includes('Something went wrong'));
+    // Exercise the actual public PTY API and native browser WebSocket. No
+    // intercepted traffic or replacement server: this runs a real shell.
+    const terminal = await frame.evaluate(async () => {
+      const base = location.origin + '/admin/agent/opencode';
+      const created = await fetch(base + '/pty', {
+        method: 'POST', headers: {'Content-Type': 'application/json'},
+        body: JSON.stringify({command: '/bin/sh', args: []}),
+      });
+      if (!created.ok) throw new Error('PTY create: ' + created.status);
+      const pty = await created.json();
+      try {
+        const authorized = await fetch(base + '/pty/' + pty.id + '/connect-token', {
+          method: 'POST', headers: {'x-opencode-ticket': '1'},
+        });
+        if (authorized.status !== 200) throw new Error('PTY ticket: ' + authorized.status);
+        const {ticket} = await authorized.json();
+        if (!ticket) throw new Error('PTY ticket missing');
+        return await new Promise((resolve, reject) => {
+          const url = new URL(base + '/pty/' + pty.id + '/connect');
+          url.searchParams.set('ticket', ticket);
+          url.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:';
+          const socket = new WebSocket(url);
+          const timer = setTimeout(() => { socket.close(); reject(new Error('PTY output timed out')); }, 10000);
+          let output = '';
+          socket.onopen = () => socket.send("printf 'ABX_%s\\n' TERMINAL_OK\n");
+          socket.onmessage = async event => {
+            output += typeof event.data === 'string' ? event.data : await event.data.text();
+            if (output.includes('ABX_TERMINAL_OK')) {
+              clearTimeout(timer); socket.close(); resolve(output);
+            }
+          };
+          socket.onerror = () => { clearTimeout(timer); reject(new Error('PTY WebSocket failed')); };
+        });
+      } finally { await fetch(base + '/pty/' + pty.id, {method: 'DELETE'}); }
+    });
+    assert.ok(terminal.includes('ABX_TERMINAL_OK'), terminal);
+    assert.equal((await page.goto(config.url + '/add/', {waitUntil: 'domcontentloaded'})).status(), 200);
+    console.log('AGENT_NAVIGATION_OK');
+  } finally { await browser.close(); }
+})().catch(error => { console.error(error); process.exitCode = 1; });
+"""
+    result = subprocess.run(
+        [str(browser_runtime["node_binary"]), "-e", script],
+        input=json.dumps({"chrome": str(browser_runtime["chrome_binary"]), "url": server_url}),
+        env={**os.environ, "NODE_PATH": browser_runtime["node_path"]},
+        text=True,
+        capture_output=True,
+        timeout=120,
+    )
+    assert result.returncode == 0, result.stderr or result.stdout
+    assert "AGENT_NAVIGATION_OK" in result.stdout
+
+
+@pytest.fixture
+def agent_server(installed_opencode, browser_runtime):
+    port = get_free_port()
+    url = f"http://localhost:{port}"
+    config = installed_opencode.config
+    _set_archivebox_config(
+        config.data_dir,
+        f"BASE_URL={url}",
+        "SERVER_SECURITY_MODE=safe-onedomain-nojsreplay",
+    )
+    user = run_archivebox_cmd(
+        [
+            "shell",
+            "-c",
+            "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(username='agent-browser-test', password='test-password'); User.objects.create_user(username='agent-regular-test', password='test-password', is_staff=True)",
+        ],
+        cwd=config.data_dir,
+        env=config.env,
+    )
+    assert user.returncode == 0, user.stderr or user.stdout
+    process = start_archivebox_server(config.data_dir, port=port, env=config.env, log_name="agent-browser-server.log")
+    try:
+        yield url, config.data_dir, process
+    finally:
+        if process.poll() is None:
+            stop_archivebox_process(process)
+
+
+def _login_cookie(server_url, username="agent-browser-test"):
+    session = requests.Session()
+    login_url = server_url + "/admin/login/"
+    page = session.get(login_url, timeout=10)
+    assert page.status_code == 200
+    token = re.search(r'name="csrfmiddlewaretoken" value="([^"]+)"', page.text)
+    assert token is not None
+    response = session.post(
+        login_url,
+        data={"username": username, "password": "test-password", "csrfmiddlewaretoken": token[1]},
+        headers={"Referer": login_url},
+        allow_redirects=False,
+        timeout=10,
+    )
+    assert response.status_code == 302
+    return "; ".join(f"{key}={value}" for key, value in session.cookies.items())
+
+
+def test_agent_websocket_rejects_unauthorized_access(agent_server):
+    from websockets.asyncio.client import connect
+    from websockets.exceptions import InvalidStatus
+
+    server_url, _, _ = agent_server
+    admin_cookie = _login_cookie(server_url)
+    regular_cookie = _login_cookie(server_url, "agent-regular-test")
+
+    async def check_denials():
+        for path, cookie, origin in (
+            ("/admin/agent/opencode/pty/unknown/connect", "", server_url),
+            ("/admin/agent/opencode/pty/unknown/connect", regular_cookie, server_url),
+            ("/admin/agent/opencode/pty/unknown/connect", admin_cookie, "https://untrusted.example"),
+            ("/admin/agent/opencode/pty/unknown/connect", admin_cookie, None),
+            ("/health/", admin_cookie, server_url),
+        ):
+            with pytest.raises(InvalidStatus) as error:
+                async with connect(
+                    server_url.replace("http", "ws", 1) + path,
+                    origin=origin,
+                    additional_headers={"Cookie": cookie},
+                    proxy=None,
+                ):
+                    pytest.fail("Unauthorized WebSocket was accepted")
+            assert error.value.response.status_code == 403
+
+    asyncio.run(check_denials())
+    assert requests.get(server_url + "/health/", timeout=10).status_code == 200
+
+
+@pytest.mark.parametrize(
+    "mode,proxy_whitelist,use_cookie,http_status",
+    [
+        ("unsafe-onedomain-noadmin", "", True, 403),
+        ("safe-onedomain-nojsreplay", "127.0.0.1/32", False, 200),
+        ("safe-onedomain-nojsreplay", "192.0.2.0/24", False, 302),
+    ],
+)
+def test_agent_websocket_matches_http_security_policy(agent_server, live_opencode, mode, proxy_whitelist, use_cookie, http_status):
+    from urllib.parse import urlsplit
+    from websockets.asyncio.client import connect
+    from websockets.exceptions import InvalidStatus
+
+    server_url, data_dir, process = agent_server
+    cookie = _login_cookie(server_url)
+    created = requests.post(
+        server_url + "/admin/agent/opencode/pty",
+        headers={"Cookie": cookie, "Origin": server_url},
+        json={"command": "/bin/sh", "args": []},
+        timeout=10,
+    )
+    assert created.status_code == 200
+    pty_id = created.json()["id"]
+    stop_archivebox_process(process)
+    _set_archivebox_config(
+        data_dir,
+        f"SERVER_SECURITY_MODE={mode}",
+        f"REVERSE_PROXY_WHITELIST={proxy_whitelist}",
+        "REVERSE_PROXY_USER_HEADER=Remote-User",
+    )
+    restarted = start_archivebox_server(
+        data_dir,
+        port=urlsplit(server_url).port,
+        env=live_opencode.config.env,
+        log_name="agent-security-server.log",
+    )
+    headers = {"Cookie": cookie} if use_cookie else {"Remote-User": "agent-browser-test"}
+    try:
+        response = requests.get(server_url + "/admin/agent/opencode/global/health", headers=headers, allow_redirects=False, timeout=10)
+        assert response.status_code == http_status
+
+        async def check_socket():
+            connection = connect(
+                server_url.replace("http", "ws", 1) + f"/admin/agent/opencode/pty/{pty_id}/connect",
+                origin=server_url,
+                additional_headers=headers,
+                proxy=None,
+            )
+            if http_status != 200:
+                with pytest.raises(InvalidStatus) as error:
+                    async with connection:
+                        pytest.fail("Disabled control plane or untrusted proxy accepted a WebSocket")
+                assert error.value.response.status_code == 403
+                return
+            async with connection as socket, asyncio.timeout(10):
+                await socket.send("printf 'ABX_%s\\n' PROXY_OK\n")
+                output = ""
+                while "ABX_PROXY_OK" not in output:
+                    chunk = await socket.recv()
+                    output += chunk.decode() if isinstance(chunk, bytes) else chunk
+                assert "ABX_PROXY_OK" in output
+
+        asyncio.run(check_socket())
+        assert requests.get(server_url + "/health/", timeout=10).status_code == 200
+    finally:
+        deleted = requests.delete(live_opencode.settings["origin"] + f"/pty/{pty_id}", timeout=10)
+        assert deleted.status_code == 200
+        stop_archivebox_process(restarted)
+
+
+def test_agent_preserves_projects_and_survives_storage_failure(agent_server, browser_runtime):
+    server_url, data_dir, _ = agent_server
+    script = r"""
+const assert = require('node:assert/strict');
+const puppeteer = require('puppeteer');
+const config = JSON.parse(require('node:fs').readFileSync(0, 'utf8'));
+(async () => {
+  for (const disabled of [false, true]) {
+    const browser = await puppeteer.launch({
+      executablePath: config.chrome,
+      headless: true,
+      // Headless interaction must not depend on the desktop display clock.
+      args: ['--no-sandbox', '--disable-frame-rate-limit', ...(disabled ? ['--disable-local-storage'] : [])],
+    });
+    try {
+      const page = await browser.newPage();
+      await page.goto(new URL('/admin/login/', config.url).href, {waitUntil: 'domcontentloaded'});
+      await page.locator('#login-form input[name="username"]').fill('agent-browser-test');
+      await page.locator('#login-form input[name="password"]').fill('test-password');
+      assert.equal(await page.$eval('#login-form input[name="username"]', element => element.value), 'agent-browser-test');
+      assert.equal(await page.$eval('#login-form input[name="password"]', element => element.value), 'test-password');
+      await Promise.all([
+        page.waitForNavigation({waitUntil: 'domcontentloaded'}),
+        page.locator('#login-form input[type="submit"]').click(),
+      ]);
+      assert.ok(!page.url().includes('/admin/login/'), page.url());
+      assert.equal((await page.goto(config.url, {waitUntil: 'domcontentloaded'})).status(), 200);
+      const welcome = '#opencode-agent-welcome';
+      await page.waitForSelector(welcome, {visible: true});
+      if (disabled) {
+        assert.equal(await page.evaluate(() => {
+          try { return localStorage === null; }
+          catch (error) { if (error.name !== 'SecurityError') throw error; return true; }
+        }), true);
+      } else {
+        await page.click('#opencode-agent-welcome-dismiss');
+        await page.waitForSelector(welcome, {hidden: true});
+        await page.reload({waitUntil: 'domcontentloaded'});
+        assert.equal(await page.$eval(welcome, element => element.hidden), true);
+        const existing = {
+          list: [{type: 'http', http: {url: 'http://other.example'}}],
+          projects: {
+            local: [{worktree: '/other-project', expanded: false}],
+            'http://other.example': [{worktree: '/remote-project', expanded: true}],
+          },
+          lastProject: {'http://other.example': '/remote-project'},
+        };
+        await page.evaluate(value => localStorage.setItem('opencode.global.dat:server', JSON.stringify(value)), existing);
+        await page.reload({waitUntil: 'domcontentloaded'});
+        await page.waitForFunction(workdir => {
+          const state = JSON.parse(localStorage.getItem('opencode.global.dat:server'));
+          return state.projects.local?.some(project => project.worktree === workdir);
+        }, {}, config.workdir);
+        const saved = await page.evaluate(() => JSON.parse(localStorage.getItem('opencode.global.dat:server')));
+        assert.deepEqual(saved.list, existing.list);
+        assert.deepEqual(saved.projects['http://other.example'], existing.projects['http://other.example']);
+        assert.equal(saved.lastProject['http://other.example'], '/remote-project');
+        const projects = saved.projects.local;
+        assert.ok(Array.isArray(projects), JSON.stringify({url: page.url(), saved, existing}));
+        assert.deepEqual(projects.find(project => project.worktree === '/other-project'), {worktree: '/other-project', expanded: false});
+        assert.equal(projects.filter(project => project.worktree === config.workdir).length, 1);
+        await page.reload({waitUntil: 'domcontentloaded'});
+        assert.equal(await page.evaluate(workdir => {
+          const state = JSON.parse(localStorage.getItem('opencode.global.dat:server'));
+          return state.projects.local.filter(project => project.worktree === workdir).length;
+        }, config.workdir), 1);
+
+        // Fill the actual browser quota; do not replace or intercept Storage methods.
+        const failure = await page.evaluate(() => {
+          localStorage.clear();
+          let low = 0, high = 8 * 1024 * 1024;
+          while (low < high) {
+            const size = Math.ceil((low + high) / 2);
+            try { localStorage.setItem('padding', 'x'.repeat(size)); low = size; }
+            catch (error) { if (error.name !== 'QuotaExceededError') throw error; high = size - 1; }
+          }
+          try { localStorage.setItem('extra', '1'); return null; }
+          catch (error) { return error.name; }
+        });
+        assert.equal(failure, 'QuotaExceededError');
+        await page.reload({waitUntil: 'domcontentloaded'});
+        await page.waitForSelector(welcome, {visible: true});
+      }
+      await page.click('#opencode-agent-welcome-dismiss');
+      await page.waitForSelector(welcome, {hidden: true});
+      assert.equal((await page.goto(new URL('/add/', config.url).href, {waitUntil: 'domcontentloaded'})).status(), 200);
+    } finally {
+      await browser.close();
+    }
+  }
+  console.log('AGENT_STORAGE_FAILURE_ISOLATED');
+})().catch(error => { console.error(error); process.exitCode = 1; });
+"""
+    result = subprocess.run(
+        [str(browser_runtime["node_binary"]), "-e", script],
+        input=json.dumps(
+            {
+                "chrome": str(browser_runtime["chrome_binary"]),
+                "url": f"{server_url}/admin/agent",
+                "workdir": str(data_dir.resolve()),
+            },
+        ),
+        env={**os.environ, "NODE_PATH": browser_runtime["node_path"]},
+        text=True,
+        capture_output=True,
+        timeout=120,
+    )
+    assert result.returncode == 0, result.stderr or result.stdout
+    assert "AGENT_STORAGE_FAILURE_ISOLATED" in result.stdout

archivebox/tests/test_process_runtime_paths.py

@@ -11,14 +11,14 @@ pytestmark = pytest.mark.django_db(transaction=True)
 
 class TestProcessRuntimePaths:
     def test_hook_processes_use_isolated_runtime_dir(self, tmp_path):
-        from archivebox.plugins.hooks import run_hook
+        from archivebox.tests.conftest import run_test_hook
 
         snap_dir = tmp_path / "snapshot"
         output_dir = snap_dir / "hashes"
         output_dir.mkdir(parents=True)
         (snap_dir / "source.txt").write_text("real runtime path input", encoding="utf-8")
         hook_path = Path(str(files("abx_plugins.plugins.hashes").joinpath("on_Snapshot__93_hashes.py")))
-        process = run_hook(
+        process = run_test_hook(
             hook_path,
             output_dir,
             config={"ABXPKG_LIB_DIR": str(tmp_path / "lib"), "SNAP_DIR": str(snap_dir)},