release: v0.9.35rc114

.github/SECURITY.md

@@ -68,4 +68,5 @@ Non-logged in users should be tightly limited to prevent attacks like SQL/shell
 
 [`SERVER_SECURITY_MODE`](https://github.com/archiveBox/archiveBox/wiki/configuration#server_security_mode) + CSRF/XSS defenses were only added in the new 0.9.0 `dev` work and are planned to land in `main` in mid-2026.
 
-Note archivebox is designed to be able to archive any URL the server can reach, including internal intranet URLs. The only URLs that are blocked are URLs hosted by the archivebox server itself.
+Note archivebox is designed to be able to archive any URL the server can reach, including internal intranet URLs and localhost. Users are expected to configure `URL_ALLOWLIST`/`URL_DENYLIST` accordingly if they need to limit what the server should archive.
+It does a basic check to try and avoid recursively archiving snapshots hosted by the archivebox server itself, but it's not a strict security check, it's just to prevent wasted CPU/disk from recursively archiving snapshots in a browser that has the ArchiveBox Browser Extension turned on.

.github/scripts/clone_abx_repo.sh

@@ -4,14 +4,6 @@ set -Eeuo pipefail
 repo_name="$1"
 target_dir="${2:-$repo_name}"
 
-case "$repo_name" in
-    abxpkg)
-        version="1.11.230"
-        ;;
-    abx-plugins)
-        version="1.11.243"
-        ;;
-    *)
 version="$(
 python3 - "$repo_name" <<'PY'
 import re
@@ -30,8 +22,6 @@ if not match:
 print(match.group(1))
 PY
 )"
-        ;;
-esac
 
 echo "Cloning ArchiveBox/${repo_name}@v${version} into ${target_dir}"
 git clone --depth=1 --branch "v${version}" "https://github.com/ArchiveBox/${repo_name}.git" "$target_dir"

.github/workflows/pip.yml

@@ -134,8 +134,6 @@ jobs:
 
       - name: Publish new package wheels and sdists to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1
-        # ignore when publish to PyPI fails due to duplicate tag
-        continue-on-error: true
 
       - name: UV install archivebox and archivebox/pkgs/* locally for tests
         env:

.github/workflows/test-parallel.yml

@@ -188,7 +188,7 @@ jobs:
             echo "ArchiveBox did not install an executable Chromium binary: ${candidate:-not found}" >&2
             exit 1
           fi
-          version="$("$candidate" --version || true)"
+          version="$("$candidate" --version)"
           major="$(printf '%s\n' "$version" | sed -E 's/.* ([0-9]+)\..*/\1/' | head -1)"
           case "$major" in
             ''|*[!0-9]*) major=0 ;;
@@ -226,7 +226,7 @@ jobs:
             echo "ArchiveBox did not install an executable Sonic binary: ${candidate:-not found}" >&2
             exit 1
           fi
-          version="$("$candidate" --version || true)"
+          version="$("$candidate" --version)"
           echo "Using ArchiveBox-installed Sonic: $candidate (${version:-version unknown})"
           echo "SONIC_BINARY=$candidate" >> "$GITHUB_ENV"
           echo "$(dirname "$candidate")" >> "$GITHUB_PATH"
@@ -234,15 +234,7 @@ jobs:
       - name: Run test - ${{ matrix.test.name }}
         run: |
           mkdir -p tests/out
-          set +e
-          uv run --no-sync --no-sources pytest -xvs "${{ matrix.test.path }}" --basetemp=tests/out --ignore=archivebox/pkgs
-          status=$?
-          set -e
-          if [ "$status" -eq 5 ]; then
-            echo "No tests collected from ${{ matrix.test.path }}; treating as an empty test module."
-            exit 0
-          fi
-          exit "$status"
+          uv run --no-sync --no-sources pytest -xvs "${{ matrix.test.path }}" --basetemp=tests/out
 
   plugin-tests:
     name: ${{ matrix.plugin.name }}
@@ -330,7 +322,7 @@ jobs:
             echo "ArchiveBox did not install an executable Chromium binary: ${candidate:-not found}" >&2
             exit 1
           fi
-          version="$("$candidate" --version || true)"
+          version="$("$candidate" --version)"
           major="$(printf '%s\n' "$version" | sed -E 's/.* ([0-9]+)\..*/\1/' | head -1)"
           case "$major" in
             ''|*[!0-9]*) major=0 ;;
@@ -368,7 +360,7 @@ jobs:
             echo "ArchiveBox did not install an executable Sonic binary: ${candidate:-not found}" >&2
             exit 1
           fi
-          version="$("$candidate" --version || true)"
+          version="$("$candidate" --version)"
           echo "Using ArchiveBox-installed Sonic: $candidate (${version:-version unknown})"
           echo "SONIC_BINARY=$candidate" >> "$GITHUB_ENV"
           echo "$(dirname "$candidate")" >> "$GITHUB_PATH"

.github/workflows/test.yml

@@ -20,8 +20,7 @@ jobs:
 
     strategy:
       matrix:
-        os: [ubuntu-22.04]
-        # os: [ubuntu-22.04, macos-latest, windows-latest]
+        os: [ubuntu-24.04]
         python: ["3.13"]
 
     steps:
@@ -92,14 +91,14 @@ jobs:
           uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox status
 
       - name: Test built package with pytest
-        # TODO: remove this exception for windows once we get tests passing on that platform
-        if: ${{ !contains(matrix.os, 'windows') }}
         run: |
           mkdir -p tests/out
-          uv run --no-sync --no-sources pytest -s archivebox/tests --basetemp=tests/out --ignore=archivebox/pkgs
+          uv run --no-sync --no-sources pytest -s archivebox/tests --basetemp=tests/out
+
+      - name: Run documentation code blocks
+        run: uv run --no-sync --no-sources pytest -vv --tb=long README.md AGENTS.md skills docs
 
       - name: Run plugin tests
-        if: ${{ !contains(matrix.os, 'windows') }}
         env:
           ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
           TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
@@ -116,9 +115,6 @@ jobs:
           submodules: true
           fetch-depth: 1
 
-      # TODO: as of 2020-11 this helper layer broke, upgrade and re-enable this once it's usable again
-      # - uses: satackey/action-docker-layer-caching@v0.0.8
-
       - name: Build image
         run: |
           docker build . -t "$DOCKER_IMAGE"
@@ -148,14 +144,14 @@ jobs:
 
       - name: Start docker-compose stack
         run: |
-          docker-compose run archivebox init
-          docker-compose up -d
+          docker compose run --rm archivebox init
+          docker compose up -d
           sleep 5
           curl --silent --location 'http://127.0.0.1:8000' | grep 'ArchiveBox'
           curl --silent --location 'http://127.0.0.1:8000/static/admin/js/jquery.init.js' | grep 'window.django'
 
       - name: Check added urls show up in index
         run: |
-          docker-compose run archivebox add 'http://example.com/#test_docker' --index-only
+          docker compose run --rm archivebox add 'http://example.com/#test_docker' --index-only
           curl --silent --location 'http://127.0.0.1:8000' | grep 'http://example.com/#test_docker'
-          docker-compose down || true
+          docker compose down

AGENTS.md

@@ -38,12 +38,12 @@ uv run --project .. archivebox run
 Recommended CLI install:
 
 ```bash
-uv tool install archivebox
-mkdir -p ~/archivebox/data
-cd ~/archivebox/data
+uv tool install --force .
+export PLUGINS=parse_txt_urls
+archivebox_data="$(mktemp -d)"
+cd "$archivebox_data"
 archivebox init --install
-archivebox add 'https://example.com'
-archivebox server 0.0.0.0:8000
+archivebox add --plugins=parse_txt_urls 'https://example.com'
 ```
 
 Alternative install methods:
@@ -55,13 +55,13 @@ Alternative install methods:
 
 ## Basic Usage
 
+<!--pytest-codeblocks:cont-->
 ```bash
 archivebox version
 archivebox help
 archivebox status
 archivebox install
-archivebox add 'https://example.com'
-archivebox add --extract=title,screenshot,pdf 'https://example.com'
+archivebox add --plugins=parse_txt_urls 'https://example.com/docs-basic-usage'
 archivebox list --json --with-headers
 archivebox search 'example'
 archivebox update --filter-type=domain example.com
@@ -73,15 +73,13 @@ archivebox run
 
 Use targeted tests for focused work:
 
-<!--pytest.mark.skip(reason="pytest invocation")-->
-```bash
+```console
 uv run pytest archivebox/tests/test_cli_add.py -q
 uv run prek run --all-files
 ```
 
 Use the full release/deploy loop only when requested:
 
-<!--pytest.mark.skip(reason="release/deploy script")-->
-```bash
+```console
 ./bin/release_dev_stack.sh
 ```

Dockerfile

@@ -241,7 +241,8 @@ RUN chmod +x "$CODE_DIR"/bin/*.sh \
     && chmod g+w "$TMP_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"
 
 RUN --mount=type=cache,target=/tmp/abxpkg-cache,sharing=locked,mode=1777 \
-    for forbidden_bin in gcc g++ make; do ! abxpkg load --binproviders=env "$forbidden_bin" >/dev/null 2>&1 || (echo "Unexpected build tool in runtime: $forbidden_bin" >&2 && exit 1); done \
+    chmod 1777 /tmp/abxpkg-cache \
+    && for forbidden_bin in gcc g++ make; do ! abxpkg load --binproviders=env "$forbidden_bin" >/dev/null 2>&1 || (echo "Unexpected build tool in runtime: $forbidden_bin" >&2 && exit 1); done \
     && stat -c "%U:%G %a %n" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
     && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$CONFIG_DIR" \
     && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$ABXPKG_LIB_DIR" \

README.md

@@ -29,7 +29,7 @@ Without active preservation effort, everything on the internet eventually disapp
 *ArchiveBox is an open source tool that lets organizations & individuals archive both public & private web content while retaining control over their data. It can be used to save copies of bookmarks, preserve evidence for legal cases, backup photos from FB/Insta/Flickr or media from YT/Soundcloud/etc., save research papers, and more...*
 <br/>
 
-> ➡️ Get ArchiveBox with `pip install archivebox>=0.9.5rc1` on [Linux](#quickstart)/[macOS](#quickstart), or via **[Docker](#quickstart)** ⭐️ on any OS.  
+> ➡️ Get ArchiveBox with `uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'` on [Linux](#quickstart)/[macOS](#quickstart), or via **[Docker](#quickstart)** ⭐️ on any OS.
 
 *Once installed, you can interact with it through the: [Browser Extension](https://github.com/ArchiveBox/archivebox-browser-extension), [CLI](#usage), [self-hosted web interface](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive), [Python API](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage), or [filesystem](#static-archive-exporting).*
 
@@ -52,7 +52,7 @@ It also detects any content featured *inside* pages & extracts it out into a fol
 - 💾 **Github**/**Gitlab**/etc. links ➡️ `clone of GIT source code`, `README`, `images`, ...
 - ✨ *and more, see [Output Formats](#output-formats) below...*
 
-You can run ArchiveBox as a Docker web app to manage these snapshots, or continue accessing the same collection using the `pip`-installed CLI, Python API, and SQLite3 APIs. 
+You can run ArchiveBox as a Docker web app to manage these snapshots, or continue accessing the same collection using the `uv`-installed CLI, Python API, and SQLite3 APIs.
 All the ways of using it are equivalent, and provide matching features like adding tags, scheduling regular crawls, viewing logs, and more...
 
 <br/>
@@ -68,7 +68,7 @@ The goal is to sleep soundly knowing the part of the internet you care about wil
 <br/>
 
 
-**📦&nbsp; Install ArchiveBox using your preferred method: `docker` / `pip` / `apt` / etc. ([see full Quickstart below](#quickstart)).**
+**📦&nbsp; Install ArchiveBox using your preferred method: `docker` / `uv` / `brew` / `apt` / etc. ([see full Quickstart below](#quickstart)).**
 
 
 <details>
@@ -85,14 +85,14 @@ docker compose run archivebox init
 <br/>
 # Option B: Or use it as a plain Docker container:
 mkdir -p ~/archivebox/data && cd ~/archivebox/data
-docker run -it -v $PWD:/data archivebox/archivebox init
-# docker run -it -v $PWD:/data archivebox/archivebox add 'https://example.com'
-# docker run -it -v $PWD:/data archivebox/archivebox help
-# docker run -it -v $PWD:/data -p 8000:8000 archivebox/archivebox
+docker run -it -v $PWD:/data archivebox/archivebox:dev init
+# docker run -it -v $PWD:/data archivebox/archivebox:dev add 'https://example.com'
+# docker run -it -v $PWD:/data archivebox/archivebox:dev help
+# docker run -it -v $PWD:/data -p 8000:8000 archivebox/archivebox:dev
 <br/>
 <br/>
-# Option C: Or install it with your preferred pkg manager (see Quickstart below for apt, brew, and more)
-pip install 'archivebox>=0.9.5rc1'
+# Option C: Or install it with uv (see Quickstart below for apt, brew, and more)
+uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
 mkdir -p ~/archivebox/data && cd ~/archivebox/data
 archivebox init
 archivebox install
@@ -214,14 +214,14 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <li>Install <a href="https://docs.docker.com/get-docker/">Docker</a> on your system (if not already installed).</li>
 <li>Create a new empty directory and initialize your collection (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
-docker run -v $PWD:/data -it archivebox/archivebox init
+docker run -v $PWD:/data -it archivebox/archivebox:dev init
 </code></pre>
 </li>
 <li>Optional: Start the server then login to the Web UI <a href="http://archivebox.localhost:8000">http://archivebox.localhost:8000</a> ⇢ Admin.
-<pre lang="bash"><code style="white-space: pre-line">docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox
+<pre lang="bash"><code style="white-space: pre-line">docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox:dev
 # completely optional, CLI can always be used without running a server
 # docker run -v $PWD:/data -it [subcommand] [--help]
-docker run -v $PWD:/data -it archivebox/archivebox help
+docker run -v $PWD:/data -it archivebox/archivebox:dev help
 </code></pre>
 <i>For more info, see <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-a-docker--docker-compose-setup-%EF%B8%8F">Install: Docker Compose</a> in the Wiki. ➡️</i>
 </li>
@@ -256,13 +256,13 @@ See <a href="https://docs.sweeting.me/s/against-curl-sh">"Against curl | sh as a
 
 
 <details>
-<summary><b><img src="https://user-images.githubusercontent.com/511499/117447613-ba4c5d80-af0b-11eb-8f89-1d98e31b6a79.png" alt="Pip" height="28px" align="top"/> <code>pip</code></b> (macOS/Linux/BSD)</summary>
+<summary><b><img src="https://user-images.githubusercontent.com/511499/117447613-ba4c5d80-af0b-11eb-8f89-1d98e31b6a79.png" alt="uv" height="28px" align="top"/> <code>uv</code></b> (macOS/Linux/BSD)</summary>
 <br/>
 <ol>
 
-<li>Install <a href="https://realpython.com/installing-python/">Python >= v3.13</a> and <a href="https://nodejs.org/en/download/package-manager/">Node >= v22</a> on your system (if not already installed).</li>
-<li>Install the ArchiveBox package using <code>pip3</code> (or <a href="https://docs.astral.sh/uv/guides/tools/#running-tools"><code>uvx</code></a>).
-<pre lang="bash"><code style="white-space: pre-line">pip3 install --upgrade archivebox
+<li>Install <a href="https://docs.astral.sh/uv/getting-started/installation/">uv</a> on your system (if not already installed).</li>
+<li>Install the ArchiveBox package using <code>uv</code>.
+<pre lang="bash"><code style="white-space: pre-line">uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
 archivebox version
 # install any missing extras shown using apt/brew/pkg/etc. see Wiki for instructions
 #    python@3.13 node curl wget git ripgrep ...
@@ -305,7 +305,7 @@ archivebox version                         # make sure the package is installed
 <li>Create a new empty directory and initialize your collection (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
 archivebox init
-sudo archivebox install
+archivebox install
 archivebox add 'https://example.com'
 </code></pre>
 <br/>
@@ -319,7 +319,7 @@ archivebox help
 </li>
 </ol>
 See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.<br/>
-<sub>See the <a href="https://github.com/ArchiveBox/debian-archivebox"><code>debian-archivebox</code></a> repo for more details about this distribution.</sub>
+<sub>The apt package is a thin dev-channel wrapper around the normal Python install flow. See the <a href="https://github.com/ArchiveBox/debian-archivebox"><code>debian-archivebox</code></a> repo for details, and run <code>sudo archivebox install</code> only if you want it to install missing system packages via apt.</sub>
 <br/><br/>
 </details>
 
@@ -481,6 +481,7 @@ ArchiveBox commands can be run in a terminal [directly on your host](https://git
 set -euo pipefail
 __archivebox_docs_home="$(mktemp -d)"
 export HOME="$__archivebox_docs_home"
+export PLUGINS=parse_txt_urls
 mkdir -p ~/archivebox/data
 cd ~/archivebox/data
 archivebox init
@@ -491,8 +492,7 @@ test -d archive
 rm -rf "$__archivebox_docs_home"
 ```
 -->
-<!--pytest.mark.skip(reason='Mixed host/Docker alternatives include interactive Docker commands')-->
-```bash
+```console
 mkdir -p ~/archivebox/data   # create a new data dir anywhere
 cd ~/archivebox/data         # IMPORTANT: cd into the directory
 
@@ -503,8 +503,8 @@ archivebox help
 # equivalent: docker compose run archivebox [subcommand] [--help]
 docker compose run archivebox help
 
-# equivalent: docker run -it -v $PWD:/data archivebox/archivebox [subcommand] [--help]
-docker run -it -v $PWD:/data archivebox/archivebox help
+# equivalent: docker run -it -v $PWD:/data archivebox/archivebox:dev [subcommand] [--help]
+docker run -it -v $PWD:/data archivebox/archivebox:dev help
 
 # optional: import your browser cookies into a persona for logged-in archiving
 archivebox persona create --import=chrome personal
@@ -525,12 +525,12 @@ archivebox persona create --import=chrome personal
 <summary><img src="https://user-images.githubusercontent.com/511499/117456282-08665e80-af16-11eb-91a1-8102eff54091.png" alt="curl sh automatic setup script" height="22px" align="top"/> <b>CLI Usage Examples: non-Docker</b></summary>
 <br/>
 <pre lang="bash"><code style="white-space: pre-line">
-# make sure you have pip-installed ArchiveBox and it's available in your $PATH first  
+# make sure you have uv-installed ArchiveBox and it's available in your $PATH first
 <br/>
 # archivebox [subcommand] [--help]
 mkdir -p ~/archivebox/data && cd ~/archivebox/data
 archivebox init
-sudo archivebox install
+archivebox install
 archivebox add 'https://example.com'
 archivebox version           # get archivebox version info + check dependencies
 archivebox help              # get list of archivebox subcommands that can be run
@@ -565,13 +565,13 @@ docker compose run archivebox add 'https://example.com'
 <pre lang="bash"><code style="white-space: pre-line">
 # make sure you create and cd into in a new empty directory first  
 <br/>
-# docker run -it -v $PWD:/data archivebox/archivebox [subcommand] [--help]
-docker run -v $PWD:/data -it archivebox/archivebox init
-docker run -v $PWD:/data -it archivebox/archivebox install
-docker run -v $PWD:/data -it archivebox/archivebox version
-docker run -v $PWD:/data -it archivebox/archivebox help
-docker run -v $PWD:/data -it archivebox/archivebox add 'https://example.com'
-# to start webserver: docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox
+# docker run -it -v $PWD:/data archivebox/archivebox:dev [subcommand] [--help]
+docker run -v $PWD:/data -it archivebox/archivebox:dev init
+docker run -v $PWD:/data -it archivebox/archivebox:dev install
+docker run -v $PWD:/data -it archivebox/archivebox:dev version
+docker run -v $PWD:/data -it archivebox/archivebox:dev help
+docker run -v $PWD:/data -it archivebox/archivebox:dev add 'https://example.com'
+# to start webserver: docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox:dev
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage-1">Usage: Docker CLI</a> wiki. ➡️</i>
 </details>
@@ -603,8 +603,8 @@ nano docker-compose.yml                        # setup initial ADMIN_USERNAME &
 docker compose up                              # start the server
 <br/>
 # Or with a Docker container:
-docker run -v $PWD:/data -it archivebox/archivebox archivebox manage createsuperuser
-docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox
+docker run -v $PWD:/data -it archivebox/archivebox:dev archivebox manage createsuperuser
+docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox:dev
 </code></pre>
 
 <sup>Open <a href="http://web.archivebox.localhost:8000"><code>http://web.archivebox.localhost:8000</code></a> for the public UI and <a href="http://admin.archivebox.localhost:8000"><code>http://admin.archivebox.localhost:8000</code></a> for the admin UI ➡️</sup><br/>
@@ -630,15 +630,15 @@ docker compose run archivebox config --set ...
 
 > [!TIP]
 > Whether in Docker or not, ArchiveBox commands work the same way, and can be used to access the same data on-disk.
-> For example, you could run the Web UI in Docker Compose, and run one-off commands with `pip`-installed ArchiveBox.
+> For example, you could run the Web UI in Docker Compose, and run one-off commands with `uv`-installed ArchiveBox.
 
 <details>
 <summary><i>Expand to show comparison...</i></summary><br/>
 
 <pre lang="bash"><code style="white-space: pre-line">
-archivebox add --depth=1 'https://example.com'                     # add a URL with pip-installed archivebox on the host
+archivebox add --depth=1 'https://example.com'                     # add a URL with uv-installed archivebox on the host
 docker compose run archivebox add --depth=1 'https://example.com'                       # or w/ Docker Compose
-docker run -it -v $PWD:/data archivebox/archivebox add --depth=1 'https://example.com'  # or w/ Docker, all equivalent
+docker run -it -v $PWD:/data archivebox/archivebox:dev add --depth=1 'https://example.com'  # or w/ Docker, all equivalent
 </code></pre>
 
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker">Docker</a> wiki. ➡️</i>
@@ -724,14 +724,14 @@ EOF
 <!--pytest-codeblocks:cont-->
 ```bash
 # archivebox add --help
-archivebox add 'https://example.com/some/page'
-archivebox add --depth=1 --plugins=parse_rss_urls "file://$HOME/Downloads/some_feed.xml"
-archivebox add --depth=1 'https://news.ycombinator.com#2020-12-12'
-echo 'http://example.com' | archivebox add
-echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox add
+archivebox add --plugins=parse_txt_urls 'https://example.com/some/page'
+archivebox add --depth=1 --plugins=parse_rss_urls < "$HOME/Downloads/some_feed.xml"
+archivebox add --plugins=parse_txt_urls 'https://example.com/docs-example'
+echo 'http://example.com' | archivebox add --plugins=parse_txt_urls
+echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox add --plugins=parse_txt_urls
 
 # if using Docker, add -i when piping stdin:
-# echo 'https://example.com' | docker run -v $PWD:/data -i archivebox/archivebox add
+# echo 'https://example.com' | docker run -v $PWD:/data -i archivebox/archivebox:dev add
 # if using Docker Compose, add -T when piping stdin / stdout:
 # echo 'https://example.com' | docker compose run -T archivebox add
 ```
@@ -741,10 +741,10 @@ echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox a
 archivebox list --json > snapshots.json
 grep -q 'https://example.com/some/page' snapshots.json
 grep -q 'https://example.com/from-feed' snapshots.json
-grep -q 'https://news.ycombinator.com#2020-12-12' snapshots.json
+grep -q 'https://example.com/docs-example' snapshots.json
 grep -q 'http://example.com' snapshots.json
-test -d archive
-test "$(find archive -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ')" -ge 4
+test -d archive/users/system/crawls
+test "$(grep -c '\"url\"' snapshots.json)" -ge 4
 rm -rf "$__archivebox_docs_home"
 ```
 -->
@@ -882,10 +882,8 @@ You don't need to install every dependency to use ArchiveBox. ArchiveBox will au
   
 If not using Docker, make sure to keep the dependencies up-to-date yourself and check that ArchiveBox isn't reporting any incompatibility with the versions you install.
 
-<pre lang="bash"><code style="white-space: pre-line">#install python3 and archivebox with your system package manager
-# apt/brew/pip/etc install ... (see Quickstart instructions above)
+<pre lang="bash"><code style="white-space: pre-line"># install uv + archivebox first (see Quickstart instructions above)
 <br/>
-which -a archivebox    # see where you have installed archivebox
 archivebox install     # auto install all the extractors and extras
 archivebox --version   # see info and check validity of installed dependencies
 </code></pre>
@@ -1406,8 +1404,7 @@ For low hanging fruit / easy first tickets, see: <a href="https://github.com/Arc
 
 First make sure you have `uv` installed: https://docs.astral.sh/uv/getting-started/installation/
 
-<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
-```bash
+```console
 git clone https://github.com/ArchiveBox/monorepo
 cd monorepo
 ./bin/setup.sh
@@ -1428,8 +1425,7 @@ Repos included in monorepo setup:
 
 #### 2. Option A: Install the Python, JS, and system dependencies directly on your machine
 
-<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
-```bash
+```console
 # Install ArchiveBox runtime dependencies
 mkdir -p data && cd data
 archivebox init
@@ -1444,14 +1440,14 @@ archivebox server 0.0.0.0:8000
 
 #### 2. Option B: Build the docker container and use that for development instead
 
-<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
-```bash
+```console
 # Optional: develop via docker by mounting the code dir into the container
 # if you edit e.g. ./archivebox/core/models.py on the docker host, runserver
 # inside the container will reload and pick up your changes
 ./bin/build_docker.sh dev
 
-docker run -it -v $PWD/data:/data archivebox/archivebox:dev init --install
+docker run -it -v $PWD/data:/data archivebox/archivebox:dev init
+docker run -it -v $PWD/data:/data archivebox/archivebox:dev install
 
 # Run the development server w/ autoreloading (but no bg workers)
 docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server --debug --reload 0.0.0.0:8000
@@ -1474,8 +1470,7 @@ You can also run all these in Docker. For more examples see the GitHub Actions C
 
 <details><summary><i>Click to expand...</i></summary>
 
-<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
-```bash
+```console
 # set up persistent DEBUG=True for all runs
 archivebox config --set DEBUG=True
 
@@ -1499,8 +1494,7 @@ https://stackoverflow.com/questions/1074212/how-can-i-see-the-raw-sql-queries-dj
 
 If you're looking for the latest `dev` Docker image, it's often available pre-built on Docker Hub, simply pull and use `archivebox/archivebox:dev`.
 
-<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
-```bash
+```console
 docker pull archivebox/archivebox:dev
 docker run archivebox/archivebox:dev version
 # verify the BUILD_TIME and COMMIT_HASH in the output are recent
@@ -1510,8 +1504,7 @@ docker run archivebox/archivebox:dev version
 
 You can also build and run any branch yourself from source, for example to build & use `dev` locally:
 
-<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
-```bash
+```console
 # docker-compose.yml:
 services:
     archivebox:
@@ -1535,8 +1528,7 @@ archivebox install
 
 <details><summary><i>Click to expand...</i></summary>
 
-<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
-```bash
+```console
 ./bin/lint.sh
 ./bin/test.sh
 ```
@@ -1549,8 +1541,7 @@ archivebox install
 
 <details><summary><i>Click to expand...</i></summary>
 
-<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
-```bash
+```console
 # generate the database migrations after changes to models.py
 cd archivebox/
 ./manage.py makemigrations
@@ -1614,8 +1605,7 @@ Copy a similar plugin as a template to modify, then open a new PR to add it in t
 <details><summary><i>Click to expand...</i></summary>
 
 (Normally CI takes care of this, but these scripts can be run to do it manually)
-<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
-```bash
+```console
 ./bin/build.sh
 
 # or individually:
@@ -1631,8 +1621,7 @@ Copy a similar plugin as a template to modify, then open a new PR to add it in t
 <details><summary><i>Click to expand...</i></summary>
 
 (Normally CI takes care of this, but these scripts can be run to do it manually)
-<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
-```bash
+```console
 ./bin/release.sh
 
 # or individually:

archivebox/api/v1_cli.py

@@ -54,6 +54,7 @@ class AddCommandSchema(Schema):
     snapshot_max_size: int = 0
     parser: str = "auto"
     plugins: str = ""
+    persona: str = "Default"
     only_new: bool | None = None
     update: bool = False
     overwrite: bool = False
@@ -145,6 +146,7 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
         snapshot_max_size=args.snapshot_max_size,
         index_only=args.index_only,
         plugins=args.plugins,
+        persona=args.persona,
         parser=args.parser,
         bg=True,  # Always run in background for API calls
         created_by_id=request.user.pk,

archivebox/cli/archivebox_extract.py

@@ -46,7 +46,6 @@ def process_archiveresult_by_id(archiveresult_id: str) -> int:
     through the shared crawl runner with the corresponding plugin selected.
     """
     from rich import print as rprint
-    from django.utils import timezone
     from archivebox.core.models import ArchiveResult
     from archivebox.api.v1_core import _uuid_ref_query
     from archivebox.services.runner import run_crawl
@@ -60,22 +59,9 @@ def process_archiveresult_by_id(archiveresult_id: str) -> int:
     rprint(f"[blue]Extracting {archiveresult.plugin} for {archiveresult.snapshot.url}[/blue]", file=sys.stderr)
 
     try:
-        was_paused = archiveresult.snapshot.is_paused
         archiveresult.reset_for_retry()
         snapshot = archiveresult.snapshot
-        if not was_paused:
-            snapshot.queue_for_extraction()
-        else:
-            # A paused snapshot may still accept explicit maintenance for one
-            # ArchiveResult, but this path must not transition it back to
-            # queued/startable work. Guard: only set retry_at while the row is
-            # still paused — concurrent resume would otherwise see a stale
-            # retry_at marker.
-            snapshot.safe_update(
-                {"retry_at": timezone.now()},
-                refresh=False,
-                extra_filter={"status": snapshot.StatusChoices.PAUSED},
-            )
+        snapshot.queue_for_extraction()
         crawl = snapshot.crawl
         if not crawl.claim_processing_lock(lock_seconds=10):
             rprint(
@@ -84,11 +70,7 @@ def process_archiveresult_by_id(archiveresult_id: str) -> int:
             )
             return 1
 
-        try:
-            run_crawl(str(snapshot.crawl_id), snapshot_ids=[str(snapshot.id)], selected_plugins=[archiveresult.plugin])
-        finally:
-            if was_paused:
-                snapshot.restore_paused_scheduler_marker()
+        run_crawl(str(snapshot.crawl_id), snapshot_ids=[str(snapshot.id)], selected_plugins=[archiveresult.plugin])
         archiveresult.refresh_from_db()
 
         if archiveresult.status == ArchiveResult.StatusChoices.SUCCEEDED:
@@ -290,12 +272,9 @@ def run_plugins(
     queue_at = timezone.now()
     if existing_snapshot_ids:
         if requested_rows:
-            # Targeted ArchiveResult retries use retry_at as the scheduling
-            # signal and keep sealed snapshots sealed so extractors are not
-            # re-run outside the explicitly queued plugin rows. Paused snapshots
-            # also keep status=paused here: `retry_at` only asks the orchestrator
-            # to process the queued plugin rows, and run_due_snapshot restores
-            # retry_at=MAX afterward instead of resuming the snapshot lifecycle.
+            # Search indexing on a sealed Snapshot is the only targeted hook
+            # allowed to bypass the normal lifecycle. Every other requested
+            # plugin requeues its Snapshot through the unified state machine.
             affected_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in rows_to_queue}
             if preserve_queued and queued_rows:
                 queued_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in queued_rows}
@@ -312,14 +291,30 @@ def run_plugins(
                         flat=True,
                     )
                 )
+            requested_plugins_by_id: dict[str, set[str]] = defaultdict(set)
+            for snapshot_id, plugin_name, _hook_name in requested_rows:
+                requested_plugins_by_id[snapshot_id].add(plugin_name)
             for snapshot in Snapshot.objects.filter(id__in=affected_snapshot_ids).only("id", "status", "modified_at"):
                 # Guard the read-time status so we never bump retry_at on a
                 # row that's been re-queued / started by a concurrent runner.
-                snapshot.safe_update(
-                    {"retry_at": queue_at, "modified_at": queue_at},
-                    refresh=False,
-                    extra_filter={"status": snapshot.status},
+                plugin_names = requested_plugins_by_id.get(str(snapshot.id), set())
+                sealed_search_backfill = (
+                    snapshot.status == Snapshot.StatusChoices.SEALED
+                    and plugin_names
+                    and all(plugin_name.startswith("search_backend_") for plugin_name in plugin_names)
                 )
+                if sealed_search_backfill:
+                    snapshot.safe_update(
+                        {"retry_at": queue_at, "modified_at": queue_at},
+                        refresh=False,
+                        extra_filter={"status": snapshot.status},
+                    )
+                else:
+                    snapshot.update_and_requeue(
+                        status=Snapshot.StatusChoices.QUEUED,
+                        retry_at=queue_at,
+                        current_step=0,
+                    )
         else:
             # No plugin rows were requested, so this is a full snapshot retry.
             for snapshot in Snapshot.objects.filter(id__in=existing_snapshot_ids).only("id", "status", "retry_at", "modified_at"):

archivebox/cli/archivebox_run.py

@@ -348,7 +348,7 @@ def run_runner(
 @click.option("--crawl-id", help="Run the crawl runner for a specific crawl only")
 @click.option("--snapshot-id", help="Run one snapshot through its crawl")
 @click.option("--binary-id", help="Run one queued binary install directly on the bus")
-@click.option("--maintenance-only", is_flag=True, help="Only process due maintenance ticks on sealed/paused snapshots")
+@click.option("--maintenance-only", is_flag=True, help="Only process sealed Snapshot maintenance and search-index backfills")
 @click.option(
     "--maintenance-batch-size",
     type=int,

archivebox/cli/archivebox_update.py

@@ -76,9 +76,14 @@ def reindex_snapshots(
     wait_for_turn=None,
 ) -> dict[str, Any]:
     from archivebox.cli.archivebox_extract import run_plugins
-    from archivebox.core.models import ArchiveResult
+    from archivebox.core.models import ArchiveResult, Snapshot
     from abx_dl.models import discover_plugins
 
+    # Search backfill is the one maintenance hook allowed to execute without
+    # reopening a Snapshot. Restrict that exception to already-sealed rows;
+    # every open lifecycle state remains owned by the normal state machine.
+    snapshots = snapshots.filter(status=Snapshot.StatusChoices.SEALED)
+
     stats: dict[str, Any] = {"processed": 0, "requested": 0, "queued": 0, "skipped_queued": 0, "reindexed": 0, "snapshot_ids": []}
     records: list[dict[str, str]] = []
     plugins_by_name = discover_plugins(runtime="archivebox")
@@ -365,7 +370,7 @@ def update(
                         # Snapshot rows: Snapshot.save() moves archive/<ts> to
                         # the current output_dir and preserves the lifecycle
                         # status. Drain those retry_at ticks before queuing
-                        # search backfill below. Otherwise the sealed/paused
+                        # search backfill below. Otherwise the sealed search
                         # runner branch correctly sees queued ArchiveResult
                         # rows first, runs the targeted plugins, and may leave
                         # the fs_version maintenance tick hidden behind that
@@ -430,9 +435,7 @@ def update(
                                 now = timezone.now()
                                 queued_result_for_snapshot = queued_index_results.filter(snapshot_id=OuterRef("pk"))
                                 snapshots_to_wake = (
-                                    Snapshot.objects.filter(
-                                        status__in=(Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED),
-                                    )
+                                    Snapshot.objects.filter(status=Snapshot.StatusChoices.SEALED)
                                     .annotate(
                                         has_queued_index_result=Exists(queued_result_for_snapshot),
                                     )
@@ -475,8 +478,8 @@ def update(
 
                 if do_run_until_idle and (do_index or not ran_post_migrate_runner):
                     # Search/index backfill intentionally queues targeted
-                    # ArchiveResult rows without reopening sealed/paused
-                    # snapshots. This second runner pass drains those plugin
+                    # ArchiveResult rows without reopening sealed snapshots.
+                    # This second runner pass drains those plugin
                     # rows after filesystem maintenance has had its own turn.
                     # For a normal unfiltered `archivebox update`, keep the
                     # historical final pass broad enough to resume genuinely

archivebox/cli/archivebox_version.py

@@ -355,13 +355,19 @@ def version(
                 ):
                     continue
 
-                any_rows = True
                 installed = db_binaries.get(logical_name) if db_available else None
                 if _binary_record_matches_runtime(installed, config.ABXPKG_LIB_DIR):
                     abspath = installed.abspath
                     version_str = (installed.version or "unknown")[:15]
                     provider = (installed.binprovider or "env")[:8]
                     valid = True
+                elif not plugin_enabled and not requested_names:
+                    # `archivebox version` is expected to verify the active runtime, not
+                    # cold-load every optional plugin provider. Migration and status
+                    # checks often run with PLUGINS narrowed to a tiny set; resolving
+                    # disabled plugin binaries there can spend most of the command on
+                    # providers the current collection will never execute.
+                    continue
                 else:
                     loaded = load_binary(actual_record)
                     abspath = str(loaded.loaded_abspath or "")
@@ -369,6 +375,7 @@ def version(
                     provider = (loaded.loaded_binprovider.name if loaded.loaded_binprovider else "env")[:8]
                     valid = loaded.is_valid
 
+                any_rows = True
                 if valid:
                     display_path = (
                         _format_binary_abspath(

archivebox/config/common.py

@@ -507,6 +507,32 @@ def _plugin_user_config(config: Mapping[str, object]) -> dict[str, str]:
     return {key: _plugin_user_config_value(value) for key, value in config.items()}
 
 
+def _plugin_input_config(config: Mapping[str, object]) -> dict[str, object]:
+    """Return raw config inputs understood by core or plugin schemas."""
+    input_names = _archivebox_config_input_names()
+    return {str(key): value for key, value in config.items() if str(key) in input_names}
+
+
+def _explicit_plugin_enabled_keys(config: Mapping[str, object]) -> set[str]:
+    """Return canonical ``*_ENABLED`` keys explicitly provided by the user.
+
+    ``PLUGINS`` is a selector, not an override for concrete plugin enable flags.
+    If a user passes ``PLUGINS=wget`` and ``SAVE_WGET=False`` together, the
+    alias-resolved ``WGET_ENABLED=False`` must survive the model validator that
+    expands ``PLUGINS`` into concrete booleans.
+    """
+    provided = {str(key) for key in config}
+    explicit: set[str] = set()
+    for key, prop in _plugin_config_properties(PLUGIN_CONFIG_SCHEMAS).items():
+        if not key.endswith("_ENABLED") or not isinstance(prop, Mapping):
+            continue
+        aliases = prop.get("x-aliases") if isinstance(prop, Mapping) else None
+        input_names = {key, *(str(alias) for alias in aliases if str(alias).strip())} if isinstance(aliases, list) else {key}
+        if provided & input_names:
+            explicit.add(key)
+    return explicit
+
+
 def _discover_plugin_config_schemas() -> PluginSchemaDocuments:
     from archivebox.plugins.discovery import discover_plugin_configs
 
@@ -752,15 +778,35 @@ class ArchiveBoxBaseConfig(
 
     @model_validator(mode="after")
     def derive_plugin_enabled_config(self):
+        self._derive_plugin_enabled_config(respect_current_enabled=False)
+        return self
+
+    def _derive_plugin_enabled_config(self, *, respect_current_enabled: bool) -> None:
         plugin_names = _normalize_plugins_config_value(self.PLUGINS)
-        selected_plugins = _plugins_with_required_plugins(plugin_names) if plugin_names else set()
+        enabled_config_keys = _plugin_enabled_config_keys()
+        disabled_plugins = []
+        if respect_current_enabled:
+            disabled_plugins = [plugin_name for plugin_name, enabled_key in enabled_config_keys.items() if not getattr(self, enabled_key)]
+        selected_plugin_roots = plugin_names
+        if selected_plugin_roots:
+            from abx_dl.models import discover_plugins, filter_plugins
+
+            selected_plugins = set(
+                filter_plugins(
+                    discover_plugins(runtime="archivebox"),
+                    sorted(selected_plugin_roots),
+                    include_providers=True,
+                    disabled_names=disabled_plugins,
+                ),
+            )
+        else:
+            selected_plugins = set()
         search_backend = self.SEARCH_BACKEND_ENGINE.strip().lower()
         if search_backend:
             selected_plugins.add(f"search_backend_{search_backend}")
-        for plugin_name, enabled_key in _plugin_enabled_config_keys().items():
+        for plugin_name, enabled_key in enabled_config_keys.items():
             if plugin_names or plugin_name in selected_plugins:
                 setattr(self, enabled_key, plugin_name in selected_plugins)
-        return self
 
 
 def _build_archivebox_config_model(plugin_schemas: PluginSchemaDocuments) -> type[ArchiveBoxBaseConfig]:
@@ -817,23 +863,6 @@ def _plugin_enabled_config_keys() -> dict[str, str]:
     return enabled_keys
 
 
-def _plugins_with_required_plugins(plugin_names: set[str]) -> set[str]:
-    selected = set(plugin_names)
-    pending = list(selected)
-    while pending:
-        plugin_name = pending.pop()
-        schema = PLUGIN_CONFIG_SCHEMAS.get(plugin_name, {})
-        required_plugins = schema.get("required_plugins") if isinstance(schema, dict) else None
-        if not isinstance(required_plugins, list):
-            continue
-        for required_plugin in required_plugins:
-            required_plugin_name = str(required_plugin).strip().lower()
-            if required_plugin_name and required_plugin_name not in selected:
-                selected.add(required_plugin_name)
-                pending.append(required_plugin_name)
-    return selected
-
-
 def get_live_config_url(key: str) -> str:
     return f"{LIVE_CONFIG_BASE_URL}{quote(key)}/"
 
@@ -1007,9 +1036,11 @@ def get_config(
     config_data: ConfigPayload = dict(defaults or {})
     config_data["PERSONAS_DIR"] = str(CONSTANTS.PERSONAS_DIR)
     base_config_payload: ConfigPayload = {}
+    base_config_model = ArchiveBoxConfig()
+
     if crawl_config_base:
         config_data.update(
-            normalize_runtime_config(ArchiveBoxConfig().model_dump(mode="json"), exclude_runtime_derived=True, json_safe=False),
+            normalize_runtime_config(base_config_model.model_dump(mode="json"), exclude_runtime_derived=True, json_safe=False),
         )
         config_data.update(normalize_runtime_config(dict(crawl.config or {}), exclude_crawl_execution=True, json_safe=False))
     elif base_config is not None:
@@ -1020,7 +1051,7 @@ def get_config(
         config_data.update(normalize_runtime_config(base_config_payload, exclude_runtime_derived=True, json_safe=False))
     else:
         config_data.update(
-            normalize_runtime_config(ArchiveBoxConfig().model_dump(mode="json"), exclude_runtime_derived=True, json_safe=False),
+            normalize_runtime_config(base_config_model.model_dump(mode="json"), exclude_runtime_derived=True, json_safe=False),
         )
         legacy_permissions = permissions_from_legacy_public_flags({**BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE), **os.environ})
         if legacy_permissions:
@@ -1072,22 +1103,29 @@ def get_config(
     archivebox_scope_overrides = {key: value for key, value in scope_overrides.items() if key in _archivebox_config_input_names()}
     config_data.update(archivebox_scope_overrides)
 
+    explicit_plugin_enabled_keys: set[str] = set()
     if resolve_plugins:
         plugin_schemas = {plugin_name: schema for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items() if isinstance(schema, dict)}
         plugin_global_config = {key: str(value) if isinstance(value, Path) else value for key, value in config_data.items()}
+        crawl_selected_plugins = crawl_config_base and bool(_normalize_plugins_config_value(dict(crawl.config or {}).get("PLUGINS")))
+        # A frozen crawl-level PLUGINS selector is the exact extractor set for
+        # this crawl. Env/file *_ENABLED values are ambient defaults and must
+        # not be re-applied after validation, or a later process env can change
+        # which hooks an already-queued crawl runs.
+        if not crawl_selected_plugins:
+            explicit_plugin_enabled_keys.update(_explicit_plugin_enabled_keys(os.environ))
+        explicit_plugin_enabled_keys.update(_explicit_plugin_enabled_keys(scope_overrides))
         plugin_user_config = _plugin_user_config(
             {
-                **normalize_runtime_config(config_data, only_crawl_execution=True, json_safe=False),
+                **_plugin_input_config(os.environ),
                 **scope_overrides,
             },
         )
         if not crawl_config_base:
+            file_config = BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE)
+            explicit_plugin_enabled_keys.update(_explicit_plugin_enabled_keys(file_config))
             plugin_user_config = {
-                **normalize_runtime_config(
-                    BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE),
-                    exclude_runtime_derived=True,
-                    json_safe=False,
-                ),
+                **_plugin_user_config(_plugin_input_config(file_config)),
                 **plugin_user_config,
             }
         plugin_sections = resolve_plugin_configs(
@@ -1112,6 +1150,13 @@ def get_config(
         if crawl_config_base:
             config_data.update(normalize_runtime_config(dict(crawl.config or {}), exclude_crawl_execution=True, json_safe=False))
         config_data.update(archivebox_scope_overrides)
+        if crawl_selected_plugins:
+            # Crawl PLUGINS is a frozen selector. Drop ambient enable flags from
+            # env/file/defaults before model validation so the selected plugin
+            # set is derived from the crawl, not from the worker process that
+            # happens to resume it.
+            for enabled_key in _plugin_enabled_config_keys().values():
+                config_data.pop(enabled_key, None)
 
     # Decode JSON-encoded complex values (dict/list fields) that came from
     # string-only sources before validation. ``IniConfigSettingsSource`` does
@@ -1137,6 +1182,11 @@ def get_config(
             )
 
     config = ArchiveBoxConfig.model_validate(config_data)
+    for key in explicit_plugin_enabled_keys:
+        if key in config_data:
+            setattr(config, key, config_data[key])
+    if config.PLUGINS:
+        config._derive_plugin_enabled_config(respect_current_enabled=True)
     if redact_sensitive:
         for key in type(config).model_fields:
             value = config[key]

archivebox/core/models.py

@@ -695,12 +695,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
     def restore_paused_scheduler_marker(self) -> None:
         """
-        Keep explicit maintenance from accidentally resuming paused snapshots.
-
-        Targeted jobs such as `archivebox update --index-only` may bump
-        retry_at so the orchestrator can run only queued search ArchiveResult
-        rows. After that maintenance pass, the lifecycle must remain PAUSED and
-        retry_at must go back to MAX until a real resume transition happens.
+        Restore the indefinite scheduler marker owned by the PAUSED lifecycle.
         """
         type(self).objects.filter(
             pk=self.pk,
@@ -2639,10 +2634,16 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             return
 
         # Delete ArchiveResults that produced no output files
-        empty_ars = self.archiveresult_set.filter(
-            output_files={},  # No output files
-        ).filter(
-            status__in=ArchiveResult.FINAL_STATES,  # Only delete finished ones
+        empty_ars = (
+            self.archiveresult_set.filter(
+                output_files={},  # No output files
+            )
+            .filter(
+                status__in=ArchiveResult.FINAL_STATES,  # Only delete finished ones
+            )
+            .exclude(
+                status=ArchiveResult.StatusChoices.FAILED,
+            )
         )
 
         if empty_ars.exists():

archivebox/core/views.py

@@ -93,7 +93,6 @@ from archivebox.core.routes_util import (
 from archivebox.core.forms import AddLinkForm
 from archivebox.plugins.forms import get_plugin_config_binary_urls
 from archivebox.crawls.models import Crawl
-from archivebox.plugins.discovery import discover_plugin_configs
 from archivebox.plugins.views import get_config_definition_link
 from archivebox.progressmonitor.views import live_progress_view, progress_endpoint
 
@@ -1424,7 +1423,6 @@ class AddView(UserPassesTestMixin, FormView):
         request_config = get_request_config(self.request, resolve_plugins=True)
         required_search_plugin = f"search_backend_{request_config.SEARCH_BACKEND_ENGINE}".strip()
         can_override_crawl_config = self._can_override_crawl_config()
-        plugin_configs = discover_plugin_configs() if can_override_crawl_config else {}
         public_persona_config_keys = {
             "CRAWL_MAX_CONCURRENT_SNAPSHOTS",
             "DELETE_AFTER",
@@ -1453,17 +1451,6 @@ class AddView(UserPassesTestMixin, FormView):
                 "binary_urls": binary_urls,
             }
         recent_personas = list(persona_queryset.order_by("-created_at", "name")[:5])
-        plugin_dependency_map = {}
-        if can_override_crawl_config:
-            plugin_dependency_map = {
-                plugin_name: [
-                    str(required_plugin).strip()
-                    for required_plugin in (schema.get("required_plugins") or [])
-                    if str(required_plugin).strip()
-                ]
-                for plugin_name, schema in plugin_configs.items()
-                if isinstance(schema.get("required_plugins"), list) and schema.get("required_plugins")
-            }
         return {
             **context,
             "title": "Create Crawl",
@@ -1473,7 +1460,6 @@ class AddView(UserPassesTestMixin, FormView):
             "VERSION": VERSION,
             "FOOTER_INFO": request_config.FOOTER_INFO,
             "required_search_plugin": required_search_plugin,
-            "plugin_dependency_map_json": json.dumps(plugin_dependency_map, sort_keys=True),
             "persona_config_map_json": json.dumps(persona_config_map, sort_keys=True, default=str),
             "recent_personas": recent_personas,
             "can_override_crawl_config": can_override_crawl_config,

archivebox/personas/importers.py

@@ -68,6 +68,55 @@ PERSONA_PROFILE_DIR_CANDIDATES = (
 )
 
 
+def _path_exists(path: Path) -> bool:
+    try:
+        return path.exists()
+    except OSError:
+        return False
+
+
+def _path_is_dir(path: Path) -> bool:
+    try:
+        return path.is_dir()
+    except OSError:
+        return False
+
+
+def _iter_children(path: Path) -> list[Path]:
+    try:
+        return list(path.iterdir())
+    except OSError:
+        return []
+
+
+def _is_copyable_profile_entry(path: Path) -> bool:
+    try:
+        if path.is_symlink():
+            return True
+        if path.is_dir():
+            return os.access(path, os.R_OK | os.X_OK)
+        return os.access(path, os.R_OK)
+    except OSError:
+        return False
+
+
+def profile_copy_ignore(src: str, names: list[str]) -> set[str]:
+    ignored = set(shutil.ignore_patterns(*VOLATILE_PROFILE_COPY_PATTERNS)(src, names))
+    src_path = Path(src)
+    for name in names:
+        if name in ignored:
+            continue
+        if not _is_copyable_profile_entry(src_path / name):
+            # Chrome can leave generated profile components unreadable when a
+            # previous run used a different runtime uid, a mounted profile came
+            # from the host, or Chrome tightened permissions on cache/model
+            # directories. Those entries are not required to launch a cloned
+            # profile, and letting copytree abort here prevents the crawl from
+            # even creating its first Snapshot.
+            ignored.add(name)
+    return ignored
+
+
 @dataclass(frozen=True)
 class PersonaImportSource:
     kind: str
@@ -394,13 +443,16 @@ def discover_persona_template_profiles(personas_dir: Path | None = None) -> list
             continue
         seen_roots.add(resolved_root)
 
-        if not resolved_root.exists() or not resolved_root.is_dir():
+        if not _path_is_dir(resolved_root):
             continue
 
-        for persona_dir in sorted((path for path in resolved_root.iterdir() if path.is_dir()), key=lambda path: path.name.lower()):
+        for persona_dir in sorted(
+            (path for path in _iter_children(resolved_root) if _path_is_dir(path)),
+            key=lambda path: path.name.lower(),
+        ):
             for candidate_dir_name in PERSONA_PROFILE_DIR_CANDIDATES:
                 user_data_dir = persona_dir / candidate_dir_name
-                if not user_data_dir.exists() or not user_data_dir.is_dir():
+                if not _path_is_dir(user_data_dir):
                     continue
 
                 for profile_dir in _list_profile_names(user_data_dir):
@@ -594,7 +646,7 @@ def copy_browser_user_data_dir(source_dir: Path, destination_dir: Path) -> None:
         source_dir,
         destination_dir,
         symlinks=True,
-        ignore=shutil.ignore_patterns(*VOLATILE_PROFILE_COPY_PATTERNS),
+        ignore=profile_copy_ignore,
     )
 
 
@@ -707,12 +759,12 @@ def export_browser_state(
 
 
 def _list_profile_names(user_data_dir: Path) -> list[str]:
-    if not user_data_dir.exists() or not user_data_dir.is_dir():
+    if not _path_is_dir(user_data_dir):
         return []
 
     profiles: list[str] = []
-    for child in sorted(user_data_dir.iterdir(), key=lambda path: path.name.lower()):
-        if not child.is_dir():
+    for child in sorted(_iter_children(user_data_dir), key=lambda path: path.name.lower()):
+        if not _path_is_dir(child):
             continue
         if child.name == "System Profile":
             continue
@@ -726,7 +778,7 @@ def _list_profile_names(user_data_dir: Path) -> list[str]:
 
 
 def _looks_like_profile_dir(path: Path) -> bool:
-    if not path.exists() or not path.is_dir():
+    if not _path_is_dir(path):
         return False
 
     marker_paths = (
@@ -738,7 +790,7 @@ def _looks_like_profile_dir(path: Path) -> bool:
         path / "Session Storage",
     )
 
-    if any(marker.exists() for marker in marker_paths):
+    if any(_path_exists(marker) for marker in marker_paths):
         return True
 
     return any(path.name == prefix or path.name.startswith(prefix) for prefix in BROWSER_PROFILE_DIR_NAMES)

archivebox/personas/models.py

@@ -11,6 +11,7 @@ Each persona has its own:
 __package__ = "archivebox.personas"
 
 import shutil
+import os
 from contextlib import contextmanager
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
@@ -179,7 +180,16 @@ class Persona(ModelWithConfig):
         if not profile_dir.exists():
             return False
 
-        for path in profile_dir.rglob("*"):
+        def profile_paths():
+            # Chrome profiles are user-controlled filesystem state. If a cache
+            # or generated component is unreadable, cleanup should skip it and
+            # continue pruning the rest instead of blocking the crawl runner.
+            for dirpath, dirnames, filenames in os.walk(profile_dir, onerror=lambda _err: None):
+                current_dir = Path(dirpath)
+                for name in [*dirnames, *filenames]:
+                    yield current_dir / name
+
+        for path in profile_paths():
             if path.name in VOLATILE_PROFILE_FILE_NAMES:
                 try:
                     path.unlink()
@@ -188,13 +198,21 @@ class Persona(ModelWithConfig):
                     pass
 
         for dirname in VOLATILE_PROFILE_DIR_NAMES:
-            for path in profile_dir.rglob(dirname):
-                if not path.is_dir():
+            for path in profile_paths():
+                if path.name != dirname:
+                    continue
+                try:
+                    is_dir = path.is_dir()
+                except OSError:
+                    is_dir = False
+                if not is_dir:
                     continue
                 shutil.rmtree(path, ignore_errors=True)
                 cleaned = True
 
-        for path in profile_dir.rglob("*.log"):
+        for path in profile_paths():
+            if not path.match("*.log"):
+                continue
             try:
                 path.unlink()
                 cleaned = True
@@ -252,7 +270,7 @@ class Persona(ModelWithConfig):
         return self.runtime_root_for_snapshot(snapshot) / "chrome_downloads"
 
     def copy_chrome_profile(self, source_dir: Path, destination_dir: Path) -> None:
-        from archivebox.personas.importers import VOLATILE_PROFILE_COPY_PATTERNS
+        from archivebox.personas.importers import profile_copy_ignore
 
         destination_dir.parent.mkdir(parents=True, exist_ok=True)
         shutil.rmtree(destination_dir, ignore_errors=True)
@@ -260,7 +278,7 @@ class Persona(ModelWithConfig):
             source_dir,
             destination_dir,
             symlinks=True,
-            ignore=shutil.ignore_patterns(*VOLATILE_PROFILE_COPY_PATTERNS),
+            ignore=profile_copy_ignore,
         )
 
     def prepare_runtime_for_crawl(self, crawl, chrome_binary: str = "") -> dict[str, str]:

archivebox/plugins/discovery.py

@@ -64,6 +64,15 @@ def get_plugins() -> list[str]:
     return sorted(set(plugins))
 
 
+def get_plugin_models():
+    from abx_dl.models import discover_plugins
+
+    plugins = {}
+    for base_dir in (BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR):
+        plugins.update(discover_plugins(plugins_dir=base_dir, runtime="archivebox"))
+    return plugins
+
+
 def get_plugin_name(plugin: str) -> str:
     """
     Get the base plugin name without numeric prefix.
@@ -90,36 +99,18 @@ def get_enabled_plugins(config: ConfigLookup | None = None, **config_kwargs: Any
 
         config = get_config(**config_kwargs)
 
-    def normalize_enabled_plugins(value: Any) -> list[str]:
-        if value is None:
-            return []
-        if isinstance(value, str):
-            raw = value.strip()
-            if not raw:
-                return []
-            if raw.startswith("["):
-                try:
-                    parsed = json.loads(raw)
-                except json.JSONDecodeError:
-                    parsed = None
-                if isinstance(parsed, list):
-                    return [str(plugin).strip() for plugin in parsed if str(plugin).strip()]
-            return [plugin.strip() for plugin in raw.split(",") if plugin.strip()]
-        if isinstance(value, (list, tuple, set)):
-            return [str(plugin).strip() for plugin in value if str(plugin).strip()]
-        return [str(value).strip()] if str(value).strip() else []
-
-    plugins_override = config.get("PLUGINS")
-    if plugins_override:
-        return normalize_enabled_plugins(plugins_override)
-
     enabled = []
+    disabled = []
     for plugin in get_plugins():
         plugin_config = get_plugin_special_config(plugin, config)
         if plugin_config["enabled"]:
             enabled.append(plugin)
+        else:
+            disabled.append(plugin)
 
-    return enabled
+    from abx_dl.models import filter_plugins
+
+    return list(filter_plugins(get_plugin_models(), enabled, include_providers=True, disabled_names=disabled))
 
 
 def discover_plugins_that_provide_interface(
@@ -245,61 +236,12 @@ def get_plugin_special_config(plugin_name: str, config: ConfigLookup, _visited:
     """
     plugin_upper = plugin_name.upper()
 
-    plugins_whitelist = config.get("PLUGINS", "")
-    if plugins_whitelist:
-        plugin_configs = discover_plugin_configs()
-        plugin_names = {p.strip().lower() for p in plugins_whitelist.split(",") if p.strip()}
-        pending = list(plugin_names)
-
-        while pending:
-            current = pending.pop()
-            schema = plugin_configs.get(current, {})
-            required_plugins = schema.get("required_plugins", [])
-            if not isinstance(required_plugins, list):
-                continue
-
-            for required_plugin in required_plugins:
-                required_plugin_name = str(required_plugin).strip().lower()
-                if not required_plugin_name or required_plugin_name in plugin_names:
-                    continue
-                plugin_names.add(required_plugin_name)
-                pending.append(required_plugin_name)
-
-        if plugin_name.lower() not in plugin_names:
-            enabled = False
-        else:
-            enabled_key = f"{plugin_upper}_ENABLED"
-            enabled = config.get(enabled_key)
-            if enabled is None:
-                enabled = True
-            elif isinstance(enabled, str):
-                enabled = enabled.lower() not in ("false", "0", "no", "")
-    else:
-        enabled_key = f"{plugin_upper}_ENABLED"
-        enabled = config.get(enabled_key)
-        if enabled is None:
-            enabled = True
-        elif isinstance(enabled, str):
-            enabled = enabled.lower() not in ("false", "0", "no", "")
-
-    plugin_configs = discover_plugin_configs()
-    plugin_name_lower = plugin_name.lower()
-
-    if enabled:
-        visited = _visited or set()
-        if plugin_name_lower not in visited:
-            next_visited = visited | {plugin_name_lower}
-            schema = plugin_configs.get(plugin_name_lower, {})
-            required_plugins = schema.get("required_plugins", [])
-            if isinstance(required_plugins, list):
-                for required_plugin in required_plugins:
-                    required_plugin_name = str(required_plugin).strip()
-                    if not required_plugin_name:
-                        continue
-                    required_config = get_plugin_special_config(required_plugin_name, config, _visited=next_visited)
-                    if not required_config["enabled"]:
-                        enabled = False
-                        break
+    enabled_key = f"{plugin_upper}_ENABLED"
+    enabled = config.get(enabled_key)
+    if enabled is None:
+        enabled = True
+    elif isinstance(enabled, str):
+        enabled = enabled.lower() not in ("false", "0", "no", "")
 
     timeout_key = f"{plugin_upper}_TIMEOUT"
     timeout = config.get(timeout_key) or config.get("TIMEOUT", 300)

archivebox/plugins/hooks.py

@@ -55,6 +55,7 @@ from archivebox.plugins.discovery import (
     BUILTIN_PLUGINS_DIR,
     USER_PLUGINS_DIR,
     ConfigLookup,
+    get_enabled_plugins,
     get_plugin_special_config,
 )
 
@@ -215,6 +216,7 @@ def discover_hooks(
 
             config = get_config(**config_kwargs)
 
+        enabled_plugins = set(get_enabled_plugins(config=config))
         enabled_hooks = []
 
         for hook in hooks:
@@ -229,9 +231,7 @@ def discover_hooks(
                 enabled_hooks.append(hook)
                 continue
 
-            # Check if plugin is enabled
-            plugin_config = get_plugin_special_config(plugin_name, config)
-            if plugin_config["enabled"]:
+            if plugin_name in enabled_plugins:
                 enabled_hooks.append(hook)
 
         hooks = enabled_hooks

archivebox/services/archive_result_service.py

@@ -4,6 +4,7 @@ import asyncio
 import inspect
 import json
 import os
+import signal
 import sys
 import time
 from collections import defaultdict
@@ -240,6 +241,14 @@ def _should_update_snapshot_title(current_title: str, next_title: str, *, snapsh
 def _status_for_process_without_archive_result(event: ProcessCompletedEvent) -> str:
     if event.exit_code == PROCESS_EXIT_SKIPPED:
         return "skipped"
+    if event.exit_code in {128 + signal.SIGHUP, 128 + signal.SIGINT, 128 + signal.SIGTERM}:
+        # This fallback only runs when a snapshot hook exited before emitting a
+        # structured ArchiveResult. A polite shutdown signal means the runner
+        # was interrupted during ownership transfer, not that the extractor
+        # produced a durable negative result. Keep the hook queued so the next
+        # runner can retry the exact work item instead of sealing in a transient
+        # process-lifecycle failure.
+        return "queued"
     if event.exit_code != 0:
         return "failed"
     return "noresults"

archivebox/services/process_service.py

@@ -43,9 +43,10 @@ def normalize_process_env(env: dict) -> dict:
         if is_sensitive_config_key(key) or (key in config_input_names and key not in allowed_config_keys):
             normalized.pop(key, None)
     if selected_plugins:
-        from archivebox.config.common import _plugin_enabled_config_keys, _plugins_with_required_plugins
+        from abx_dl.models import discover_plugins, filter_plugins
+        from archivebox.config.common import _plugin_enabled_config_keys
 
-        selected_plugins = _plugins_with_required_plugins(selected_plugins)
+        selected_plugins = set(filter_plugins(discover_plugins(runtime="archivebox"), sorted(selected_plugins), include_providers=True))
         for plugin_name, enabled_key in _plugin_enabled_config_keys().items():
             normalized.setdefault(enabled_key, "True" if plugin_name in selected_plugins else "False")
     return normalized

archivebox/services/runner.py

@@ -59,7 +59,6 @@ from archivebox.config.common import (
     ArchiveBoxBaseConfig,
     normalize_runtime_config,
     _plugin_enabled_config_keys,
-    _plugins_with_required_plugins,
 )
 from archivebox.misc.db import run_db_analyze_batch
 from archivebox.core.shutdown_util import foreground_shutdown_signals, raise_if_shutdown_requested
@@ -318,16 +317,17 @@ class CrawlRunner:
 
     @property
     def allow_maintenance_on_inactive_crawl(self) -> bool:
-        """Run the requested hooks on a snapshot whose parent crawl is paused or sealed.
+        """Run targeted search indexing on an already-sealed snapshot.
 
-        Maintenance entry paths — direct ``snapshot_ids + selected_plugins`` invocations
-        for search backend backfill, fs migration, plugin-targeted updates — are
-        legitimately allowed to operate on finished/paused crawls. Without this gate,
-        ``crawl_is_cancelled`` would treat a SEALED parent as a cancellation signal
-        and short-circuit every guard before any hook ran, leaving the queued
-        ArchiveResult rows stuck and the orchestrator looping on them.
+        This is the singular hook-execution exception to the unified lifecycle.
+        All other work must queue the Snapshot and Crawl normally.
         """
-        return bool(self.initial_snapshot_ids and self.selected_plugins)
+        return bool(
+            self.initial_snapshot_ids
+            and self.selected_plugins
+            and self.crawl.status == self.crawl.StatusChoices.SEALED
+            and all(plugin.startswith("search_backend_") for plugin in self.selected_plugins),
+        )
 
     async def run(self) -> None:
         heartbeat = CrawlHeartbeat(
@@ -595,13 +595,12 @@ class CrawlRunner:
                     hook.parent.name for event_name in runtime_events for hook in discover_hooks(event_name, config=self.base_config)
                 }
                 self.selected_plugins = sorted(runtime_plugins) or None
-        if self.initial_snapshot_ids:
-            # Direct snapshot maintenance paths are allowed to name paused
-            # snapshots explicitly. The runner still requires selected_plugins
-            # later, so this does not restart the crawl lifecycle.
-            return [str(snapshot_id) for snapshot_id in self.initial_snapshot_ids]
         if self.crawl.is_paused:
             return []
+        if self.initial_snapshot_ids:
+            # Explicit ids select normal runnable work, except for the one
+            # sealed-search backfill admitted by allow_maintenance_on_inactive_crawl.
+            return [str(snapshot_id) for snapshot_id in self.initial_snapshot_ids]
         pending_snapshots = list(
             self.crawl.snapshot_set.filter(status__in=Snapshot.RUNNABLE_STATES)
             .filter(retry_at__lte=timezone.now())
@@ -811,7 +810,7 @@ class CrawlRunner:
         normalized_config = normalize_runtime_config(config)
         configured_plugins = [name.strip().lower() for name in str(normalized_config.get("PLUGINS") or "").split(",") if name.strip()]
         if configured_plugins:
-            selected_plugin_names = _plugins_with_required_plugins(set(configured_plugins))
+            selected_plugin_names = set(filter_plugins(self.plugins, configured_plugins, include_providers=True))
             for plugin_name, enabled_key in _plugin_enabled_config_keys().items():
                 normalized_config.setdefault(enabled_key, plugin_name in selected_plugin_names)
         return {
@@ -1146,7 +1145,14 @@ class CrawlRunner:
                     )(snapshot["id"])
                     if queued_plugins:
                         if snapshot_selected_plugins:
+                            original_queued_plugins = queued_plugins
                             queued_plugins = queued_plugins_selected_by_config(queued_plugins)
+                            disabled_queued_plugins = sorted(set(original_queued_plugins) - set(queued_plugins))
+                            if disabled_queued_plugins:
+                                await sync_to_async(skip_disabled_queued_plugins, thread_sensitive=True)(
+                                    snapshot["id"],
+                                    disabled_queued_plugins,
+                                )
                             selected_hooks_by_plugin = {
                                 plugin: hooks for plugin, hooks in (selected_hooks_by_plugin or {}).items() if plugin in queued_plugins
                             }
@@ -1158,7 +1164,14 @@ class CrawlRunner:
                 )(snapshot["id"])
                 if queued_plugins:
                     if snapshot_selected_plugins:
+                        original_queued_plugins = queued_plugins
                         queued_plugins = queued_plugins_selected_by_config(queued_plugins)
+                        disabled_queued_plugins = sorted(set(original_queued_plugins) - set(queued_plugins))
+                        if disabled_queued_plugins:
+                            await sync_to_async(skip_disabled_queued_plugins, thread_sensitive=True)(
+                                snapshot["id"],
+                                disabled_queued_plugins,
+                            )
                         selected_hooks_by_plugin = {
                             plugin: hooks for plugin, hooks in (selected_hooks_by_plugin or {}).items() if plugin in queued_plugins
                         }
@@ -1189,7 +1202,13 @@ class CrawlRunner:
                 if snapshot_selected_plugins and remaining_queued_plugins:
                     remaining_queued_plugins = queued_plugins_selected_by_config(remaining_queued_plugins)
                 if not remaining_queued_plugins:
-                    await sync_to_async(run_snapshot_maintenance, thread_sensitive=True)(snapshot_id, output_dir=output_dir)
+                    if snapshot["status"] in ("queued", "started"):
+                        await sync_to_async(finalize_completed_snapshot, thread_sensitive=True)(
+                            snapshot_id,
+                            output_dir=output_dir,
+                        )
+                    else:
+                        await sync_to_async(run_snapshot_maintenance, thread_sensitive=True)(snapshot_id, output_dir=output_dir)
                     return
                 snapshot_selected_plugins = remaining_queued_plugins
                 plugins = filter_plugins(self.plugins, snapshot_selected_plugins, include_providers=True)
@@ -1453,7 +1472,9 @@ def queued_plugins_for_snapshot(snapshot_id: str) -> list[str] | None:
 def config_overrides_for_queued_plugins(selected_plugins: list[str], **overrides: Any) -> dict[str, Any]:
     config_overrides = dict(overrides)
     config_overrides["PLUGINS"] = ",".join(selected_plugins)
-    selected_plugin_names = _plugins_with_required_plugins({plugin_name.lower() for plugin_name in selected_plugins})
+    selected_plugin_names = set(
+        filter_plugins(_discover_archivebox_plugins(), [plugin_name.lower() for plugin_name in selected_plugins], include_providers=True),
+    )
     for plugin_name, enabled_key in _plugin_enabled_config_keys().items():
         config_overrides[enabled_key] = plugin_name in selected_plugin_names
     for plugin_name in selected_plugins:
@@ -1498,6 +1519,28 @@ def fail_unavailable_queued_hooks(
         )
 
 
+def skip_disabled_queued_plugins(snapshot_id: str, plugin_names: list[str]) -> None:
+    from archivebox.core.models import ArchiveResult
+
+    if not plugin_names:
+        return
+    now = timezone.now()
+    # Queued ArchiveResult rows are durable scheduler state and can outlive a
+    # config change or deploy. If a plugin is no longer selected for this
+    # Snapshot/Crawl, leaving its old row queued keeps the Snapshot STARTED
+    # forever even though there is no runnable work left.
+    ArchiveResult.objects.filter(
+        snapshot_id=snapshot_id,
+        plugin__in=plugin_names,
+        status=ArchiveResult.StatusChoices.QUEUED,
+    ).update(
+        status=ArchiveResult.StatusChoices.SKIPPED,
+        start_ts=now,
+        end_ts=now,
+        output_str="Queued plugin is disabled by this Snapshot/Crawl config",
+    )
+
+
 def include_background_prerequisite_hooks(
     selected_hooks_by_plugin: dict[str, set[str] | None],
     plugins: dict[str, Plugin],
@@ -1530,12 +1573,13 @@ def include_background_prerequisite_hooks(
 def snapshot_hooks_for_pending_archiveresults(snapshot) -> list[tuple[str, str]]:
     from archivebox.config.common import get_config
     from archivebox.core.models import Snapshot
+    from archivebox.plugins.discovery import get_enabled_plugins
 
     config = get_config(crawl=snapshot.crawl, snapshot=snapshot)
     snapshot_plugin_names = [name.strip() for name in str((snapshot.config or {}).get("PLUGINS") or "").split(",") if name.strip()]
     crawl_plugin_names = [name.strip() for name in str((snapshot.crawl.config or {}).get("PLUGINS") or "").split(",") if name.strip()]
     config_plugin_names = [name.strip() for name in str(config.PLUGINS or "").split(",") if name.strip()]
-    plugin_names = snapshot_plugin_names or crawl_plugin_names or config_plugin_names
+    plugin_names = snapshot_plugin_names or crawl_plugin_names or config_plugin_names or get_enabled_plugins(config=config)
     plugins = (
         filter_plugins(_discover_archivebox_plugins(), plugin_names, include_providers=True)
         if plugin_names
@@ -1559,10 +1603,8 @@ def run_snapshot_maintenance(snapshot_id: str, *, output_dir: Path | None = None
     # queued ArchiveResult rows, so run it whenever this helper is called.
     # The only thing queued rows change is the next scheduler value:
     # - no queued rows left: clear retry_at because maintenance is done
-    # - queued rows remain: leave the Snapshot due so the sealed/paused runner
-    #   branch can process those targeted plugin rows on the next tick
-    # This avoids reopening final/paused snapshots while also avoiding stranded
-    # queued ArchiveResults that have no independent scheduler.
+    # - queued rows remain on a sealed Snapshot: leave it due so the search
+    #   backfill exception can process them on the next tick
     current_retry_at = snapshot.retry_at
     next_retry_at = timezone.now() if has_queued_results else None
     snapshot.retry_at = next_retry_at
@@ -1687,62 +1729,12 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo
         return parent_reconciled
 
     if snapshot.is_paused:
-        selected_plugins = queued_plugins_for_snapshot(str(snapshot.id))
-        if snapshot.fs_migration_needed and Snapshot.claim_for_worker(snapshot, lock_seconds=lock_seconds):
-            _runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot)
-            run_snapshot_maintenance(str(snapshot.id))
-            if not selected_plugins:
-                # No targeted plugin rows remain, so put paused snapshots back
-                # behind the indefinite retry_at marker. If queued plugin rows
-                # remain, continue into the targeted plugin path below and let
-                # its finally block restore the paused marker after completion.
-                snapshot.restore_paused_scheduler_marker()
-                return True
-            snapshot.refresh_from_db()
-        if not selected_plugins:
-            # Paused is a real lifecycle state; retry_at=MAX is only the
-            # orchestrator selection marker. If a direct maintenance/update
-            # command bumps retry_at on a paused snapshot but there are no
-            # targeted ArchiveResult rows to run, restore the scheduler marker
-            # without changing status.
-            snapshot.restore_paused_scheduler_marker()
-            return True
-        if not Snapshot.claim_for_worker(snapshot, lock_seconds=lock_seconds):
-            return False
-        try:
-            _runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot)
-            # Explicit maintenance, e.g. `archivebox update --index-only`, may
-            # need to run search/index hooks for a paused snapshot. That should
-            # not resume the crawl or make unrelated queued work runnable. The
-            # queued ArchiveResult rows are the durable maintenance request, so
-            # run that exact plugin set even when the paused crawl's normal
-            # PLUGINS config names a different extractor surface.
-            run_crawl(
-                str(snapshot.crawl_id),
-                snapshot_ids=[str(snapshot.id)],
-                selected_plugins=selected_plugins,
-                process_discovered_snapshots_inline=True,
-                interactive_interrupts=interactive_interrupts,
-                config_overrides=config_overrides_for_queued_plugins(selected_plugins),
-                selected_plugins_are_explicit=False,
-            )
-        finally:
-            # Targeted plugin rows can complete while the Snapshot remains
-            # paused. Put retry_at back at MAX only after the queued rows are
-            # gone; if a hook was interrupted before projection, keep the
-            # paused row due so the next runner can retry that targeted work
-            # without a user-visible resume transition.
-            if queued_plugins_for_snapshot(str(snapshot.id)):
-                now = timezone.now()
-                type(snapshot).objects.filter(
-                    pk=snapshot.pk,
-                    status=snapshot.StatusChoices.PAUSED,
-                ).update(
-                    retry_at=now,
-                    modified_at=now,
-                )
-            else:
-                snapshot.restore_paused_scheduler_marker()
+        # Paused work never executes out of band. Preserve the lifecycle marker
+        # until an explicit resume moves it through the normal state machine.
+        from archivebox.core.models import ArchiveResult
+
+        ArchiveResult.pause_queryset(snapshot.archiveresult_set.all())
+        snapshot.restore_paused_scheduler_marker()
         return True
     if snapshot.status == Snapshot.StatusChoices.SEALED:
         if not Snapshot.claim_for_worker(snapshot, lock_seconds=lock_seconds):
@@ -1761,17 +1753,24 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo
         selected_plugins = queued_plugins_for_snapshot(str(snapshot.id))
         if selected_plugins:
             search_only_plugins = all(plugin.startswith("search_backend_") for plugin in selected_plugins)
-            _runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot)
-            run_crawl(
-                str(snapshot.crawl_id),
-                snapshot_ids=[str(snapshot.id)],
-                selected_plugins=selected_plugins,
-                process_discovered_snapshots_inline=True,
-                interactive_interrupts=interactive_interrupts,
-                config_overrides=config_overrides_for_queued_plugins(selected_plugins),
-                selected_plugins_are_explicit=False,
-            )
-            if search_only_plugins:
+            if not search_only_plugins:
+                snapshot.update_and_requeue(
+                    status=Snapshot.StatusChoices.QUEUED,
+                    retry_at=timezone.now(),
+                    current_step=0,
+                )
+                snapshot.refresh_from_db()
+            else:
+                _runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot)
+                run_crawl(
+                    str(snapshot.crawl_id),
+                    snapshot_ids=[str(snapshot.id)],
+                    selected_plugins=selected_plugins,
+                    process_discovered_snapshots_inline=True,
+                    interactive_interrupts=interactive_interrupts,
+                    config_overrides=config_overrides_for_queued_plugins(selected_plugins),
+                    selected_plugins_are_explicit=False,
+                )
                 from archivebox.core.models import ArchiveResult
 
                 has_queued_results = ArchiveResult.objects.filter(
@@ -1794,10 +1793,11 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo
                         retry_at=timezone.now(),
                         modified_at=timezone.now(),
                     )
-            return True
-        if maintenance_ran:
-            return True
-        return run_snapshot_maintenance(str(snapshot.id))
+                return True
+        if snapshot.status == Snapshot.StatusChoices.SEALED:
+            if maintenance_ran:
+                return True
+            return run_snapshot_maintenance(str(snapshot.id))
 
     if snapshot.status == Snapshot.StatusChoices.STARTED:
         _reset_count, running_count = snapshot.reset_abandoned_results()
@@ -1809,17 +1809,18 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo
         return False
     snapshot.refresh_from_db()
     if snapshot.status == Snapshot.StatusChoices.QUEUED:
-        if snapshot.archiveresult_set.exists() and snapshot.is_finished_processing():
+        has_archiveresults = snapshot.archiveresult_set.exists()
+        if has_archiveresults and snapshot.is_finished_processing():
             snapshot.sm.tick()
             snapshot.refresh_from_db()
             if snapshot.status == Snapshot.StatusChoices.SEALED:
                 _runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot, status="SEALED")
                 return True
-        # The runner owns queued Snapshot setup. Create missing enabled hook
-        # rows before ticking so maintenance-only final rows, e.g. search
-        # backfill on a paused snapshot, cannot make queued -> sealed skip the
-        # real extraction work after resume.
-        snapshot.create_pending_archiveresults(hooks=snapshot_hooks_for_pending_archiveresults(snapshot))
+        # A Snapshot with no hook rows is fresh lifecycle work; materialize its
+        # configured hook set. Existing rows are already the durable requested
+        # work set and must not be broadened during retry/recovery.
+        if not has_archiveresults:
+            snapshot.create_pending_archiveresults(hooks=snapshot_hooks_for_pending_archiveresults(snapshot))
         snapshot.sm.tick()
         snapshot.refresh_from_db()
         if snapshot.status == Snapshot.StatusChoices.SEALED:
@@ -1831,6 +1832,17 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo
         if snapshot.status == Snapshot.StatusChoices.SEALED:
             _runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot, status="SEALED")
             return True
+    if snapshot.status == Snapshot.StatusChoices.STARTED:
+        queued_plugins, selected_hooks_by_plugin = queued_plugins_and_hooks_for_snapshot(str(snapshot.id))
+        if queued_plugins and selected_hooks_by_plugin:
+            fail_unavailable_queued_hooks(
+                str(snapshot.id),
+                selected_hooks_by_plugin,
+                _discover_archivebox_plugins(),
+            )
+            if not queued_plugins_for_snapshot(str(snapshot.id)):
+                finalize_completed_snapshot(str(snapshot.id), output_dir=Path(snapshot.output_dir))
+                return True
     _runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot)
     run_crawl(
         str(snapshot.crawl_id),
@@ -2066,7 +2078,7 @@ def _run_due_queued_plugin_result(
             status=ArchiveResult.StatusChoices.QUEUED,
             plugin__in=plugin_names,
             snapshot__retry_at__lte=now,
-            snapshot__status__in=(Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED),
+            snapshot__status=Snapshot.StatusChoices.SEALED,
         )
         .filter(**({"snapshot__crawl_id": crawl_id} if crawl_id else {}))
         .values("snapshot_id", "snapshot__crawl_id")[:1]
@@ -2076,15 +2088,6 @@ def _run_due_queued_plugin_result(
         return False
     root_crawl_id = str(first_due_results[0]["snapshot__crawl_id"])
 
-    first_due_snapshot = Snapshot.objects.filter(pk=first_due_results[0]["snapshot_id"]).first()
-    if first_due_snapshot and first_due_snapshot.status == Snapshot.StatusChoices.PAUSED:
-        return run_due_snapshot(
-            first_due_snapshot,
-            lock_seconds=lock_seconds,
-            interactive_interrupts=interactive_interrupts,
-            runtime_config=runtime_config,
-        )
-
     due_snapshots = Snapshot.objects.filter(
         retry_at__lte=now,
         status=Snapshot.StatusChoices.SEALED,
@@ -2238,18 +2241,11 @@ def run_pending_crawls(
     from archivebox.config.common import get_config
     from archivebox.crawls.models import Crawl, CrawlSchedule
     from archivebox.core.models import ArchiveResult, Snapshot
-    from archivebox.plugins.discovery import discover_plugin_configs
     from archivebox.plugins.hooks import discover_hooks
     from archivebox.machine.models import Process
 
     crawl_claim_lock_seconds = 10
     runtime_config = get_config()
-    plugin_configs = discover_plugin_configs()
-    download_plugin_names = frozenset(
-        plugin_name
-        for plugin_name, plugin_config in plugin_configs.items()
-        if plugin_config.get("output_mimetypes") and not plugin_name.startswith("search_backend_")
-    )
     last_recovery_at = 0.0
     last_retention_at = 0.0
     last_retention_repair_at = 0.0
@@ -2277,19 +2273,6 @@ def run_pending_crawls(
                 if schedule.is_due(now):
                     schedule.enqueue(queued_at=now)
 
-        # Final-state download rows are always first: they have no parent crawl
-        # scheduler of their own, and leaving them behind makes the global
-        # counters report stale queued work while new crawls continue.
-        if _run_due_queued_plugin_result(
-            download_plugin_names,
-            crawl_id=crawl_id,
-            lock_seconds=60,
-            interactive_interrupts=interactive_interrupts,
-            runtime_config=runtime_config,
-            batch_size=maintenance_batch_size,
-        ):
-            continue
-
         if _fast_forward_same_path_snapshot_fs_versions():
             continue
 
@@ -2365,21 +2348,20 @@ def run_pending_crawls(
         # Final active-state fallback uses only the retry_at scheduler index and
         # selects an id first. Keep final SEALED rows out of this broad path so
         # large filesystem/index backfills cannot starve newly queued crawls.
-        due_snapshots = Snapshot.objects.filter(
-            retry_at__lte=timezone.now(),
-            status__in=Snapshot.OPEN_STATES,
-        )
-        if maintenance_only:
-            due_snapshots = due_snapshots.filter(status=Snapshot.StatusChoices.PAUSED)
-        if crawl_id:
-            due_snapshots = due_snapshots.filter(crawl_id=crawl_id)
-        if _run_due_snapshot_query(
-            due_snapshots,
-            lock_seconds=60,
-            interactive_interrupts=interactive_interrupts,
-            runtime_config=runtime_config,
-        ):
-            continue
+        if not maintenance_only:
+            due_snapshots = Snapshot.objects.filter(
+                retry_at__lte=timezone.now(),
+                status__in=Snapshot.OPEN_STATES,
+            )
+            if crawl_id:
+                due_snapshots = due_snapshots.filter(crawl_id=crawl_id)
+            if _run_due_snapshot_query(
+                due_snapshots,
+                lock_seconds=60,
+                interactive_interrupts=interactive_interrupts,
+                runtime_config=runtime_config,
+            ):
+                continue
 
         # Search backend selection is live crawl-execution config, not an
         # installed-plugin list. Old queued rows for a backend that is disabled

archivebox/templates/core/add.html

@@ -48,7 +48,9 @@
                 <div class="crawl-explanation">
                     <p class="crawl-tip" style="user-select: none;">
                         <span style="float: right">
-                            <a href="/admin/agent">💬 Crawl with AI</a> &nbsp; | &nbsp;
+                            {% if user.is_authenticated and user.is_superuser and request.archivebox_config.OPENCODE_ENABLED %}
+                                <a href="/admin/agent">💬 Crawl with AI</a> &nbsp; | &nbsp;
+                            {% endif %}
                             <a href="https://github.com/ArchiveBox/archivebox-browser-extension"><img src="{% static 'chrome_extension_icon.png' %}" alt="Chrome Extension" style="height: 33px; margin-top: -5px;">&nbsp; Get the extension</a>
                         </span>
                         💡 <strong>Tip:</strong> Instantly save a single URL by visiting:
@@ -372,7 +374,6 @@
                 if (sameDomainToggleSlot && shortcutToggles.length) {
                     shortcutToggles.forEach((toggle) => sameDomainToggleSlot.appendChild(toggle));
                 }
-                const pluginDependencyMap = JSON.parse('{{ plugin_dependency_map_json|default:"{}"|escapejs }}');
                 const personaConfigMap = JSON.parse('{{ persona_config_map_json|default:"{}"|escapejs }}');
                 const personaSelect = document.querySelector('select[name="persona"]');
 
@@ -404,22 +405,6 @@
                     return document.querySelector(`.plugin-section-toggle[value="${requiredSearchPlugin}"]`);
                 }
 
-                function getPluginCheckbox(pluginName) {
-                    if (!pluginName) return null;
-                    return document.querySelector(`.plugin-section-toggle[value="${pluginName}"]`);
-                }
-
-                function getRequiredPlugins(pluginName) {
-                    const requiredPlugins = pluginDependencyMap[pluginName];
-                    return Array.isArray(requiredPlugins) ? requiredPlugins : [];
-                }
-
-                function getDependentPlugins(pluginName) {
-                    return Object.entries(pluginDependencyMap)
-                        .filter(([, requiredPlugins]) => Array.isArray(requiredPlugins) && requiredPlugins.includes(pluginName))
-                        .map(([dependentPlugin]) => dependentPlugin);
-                }
-
                 function getConfigEditorRows() {
                     return document.getElementById('id_config_rows');
                 }
@@ -555,47 +540,6 @@
                     updateURLPreview();
                 }
 
-                function ensureRequiredPluginsChecked(pluginName, visited = new Set()) {
-                    if (!pluginName || visited.has(pluginName)) {
-                        return;
-                    }
-                    visited.add(pluginName);
-
-                    getRequiredPlugins(pluginName).forEach(requiredPlugin => {
-                        const requiredCheckbox = getPluginCheckbox(requiredPlugin);
-                        if (!requiredCheckbox) {
-                            return;
-                        }
-                        requiredCheckbox.checked = true;
-                        dispatchChange(requiredCheckbox);
-                        ensureRequiredPluginsChecked(requiredPlugin, visited);
-                    });
-                }
-
-                function uncheckDependentPlugins(pluginName, visited = new Set()) {
-                    if (!pluginName || visited.has(pluginName)) {
-                        return;
-                    }
-                    visited.add(pluginName);
-
-                    getDependentPlugins(pluginName).forEach(dependentPlugin => {
-                        const dependentCheckbox = getPluginCheckbox(dependentPlugin);
-                        if (!dependentCheckbox) {
-                            return;
-                        }
-                        dependentCheckbox.checked = false;
-                        dispatchChange(dependentCheckbox);
-                        uncheckDependentPlugins(dependentPlugin, visited);
-                    });
-                }
-
-                function normalizePluginSelections() {
-                    const checkedPlugins = Array.from(document.querySelectorAll('.plugin-section-toggle:checked'))
-                        .map(cb => cb.value)
-                        .filter(Boolean);
-                    checkedPlugins.forEach(pluginName => ensureRequiredPluginsChecked(pluginName));
-                }
-
                 function applyRequiredSearchPlugin(preferredValue = undefined) {
                     const requiredCheckbox = getRequiredSearchCheckbox();
                     if (!requiredCheckbox) return;
@@ -1245,7 +1189,6 @@
                             });
                         }
 
-                        normalizePluginSelections();
                         applyRequiredSearchPlugin(requiredSearchPreference);
                         syncEnabledPluginsConfig();
                         updateChromeToggleButton();
@@ -1279,7 +1222,6 @@
                             dispatchChange(cb);
                         });
 
-                        normalizePluginSelections();
                         applyRequiredSearchPlugin(requiredSearchPreference);
                         syncEnabledPluginsConfig();
                         updateChromeToggleButton();
@@ -1289,11 +1231,6 @@
 
                 document.querySelectorAll('.plugin-section-toggle').forEach(checkbox => {
                     checkbox.addEventListener('change', function() {
-                        if (this.checked) {
-                            ensureRequiredPluginsChecked(this.value);
-                        } else {
-                            uncheckDependentPlugins(this.value);
-                        }
                         syncEnabledPluginsConfig();
                         updateChromeToggleButton();
                         saveFormState();
@@ -1348,7 +1285,6 @@
                         if (personaSelect) {
                             applyPersonaConfig(personaSelect.value);
                         }
-                        normalizePluginSelections();
                         applyRequiredSearchPlugin();
                         syncEnabledPluginsConfig();
                         updateChromeToggleButton();

archivebox/templates/core/navigation.html

@@ -6,7 +6,7 @@
     <a href="{% url 'Home' %}">Snapshots</a> |
     <a href="/admin/core/archiveresult/?o=-1">Log</a> |
     <a href="/admin/core/tag/">Tags</a> &nbsp; &nbsp;
-    {% if user.is_authenticated and user.is_superuser %}
+    {% if user.is_authenticated and user.is_superuser and request.archivebox_config.OPENCODE_ENABLED %}
         <a href="/admin/agent">💬 AI</a> |
     {% endif %}
     <a href="{% url 'Docs' %}" target="_blank" rel="noopener noreferrer">Docs</a> |

archivebox/tests/conftest.py

@@ -228,6 +228,8 @@ def run_archivebox_cmd(
                     "SAVE_HTMLTOTEXT": "False",
                 },
             )
+            if args and args[0] == "version":
+                run_env["PLUGINS"] = "__archivebox_test_no_plugins__"
         if env:
             run_env.update(env)
     _set_test_source_pythonpath(run_env)
@@ -434,27 +436,45 @@ def recursive_test_site():
         "/about": """
             <html>
               <body>
-                <a href="/deep/about">Deep About</a>
+                <main>
+                  <article>
+                    <h1>About</h1>
+                    <p>This local article describes the ArchiveBox recursive crawl fixture with enough semantic content for real article extractors.</p>
+                    <a href="/deep/about">Deep About</a>
+                  </article>
+                </main>
               </body>
             </html>
         """.strip().encode("utf-8"),
         "/blog": """
             <html>
               <body>
-                <a href="/deep/blog">Deep Blog</a>
+                <main>
+                  <article>
+                    <h1>Blog</h1>
+                    <p>This local article exercises real browser, text extraction, and recursive discovery behavior without depending on a public website.</p>
+                    <a href="/deep/blog">Deep Blog</a>
+                  </article>
+                </main>
               </body>
             </html>
         """.strip().encode("utf-8"),
         "/contact": """
             <html>
               <body>
-                <a href="/deep/contact">Deep Contact</a>
+                <main>
+                  <article>
+                    <h1>Contact</h1>
+                    <p>This local article keeps the deterministic crawl graph content-rich while preserving the original contact and deep-contact URLs.</p>
+                    <a href="/deep/contact">Deep Contact</a>
+                  </article>
+                </main>
               </body>
             </html>
         """.strip().encode("utf-8"),
-        "/deep/about": b"<html><body><h1>Deep About</h1></body></html>",
-        "/deep/blog": b"<html><body><h1>Deep Blog</h1></body></html>",
-        "/deep/contact": b"<html><body><h1>Deep Contact</h1></body></html>",
+        "/deep/about": b"<html><body><main><article><h1>Deep About</h1><p>This depth-two article verifies that recursive snapshots retain extractable semantic content and their exact parent relationship.</p></article></main></body></html>",
+        "/deep/blog": b"<html><body><main><article><h1>Deep Blog</h1><p>This depth-two article provides deterministic local content for every real extraction plugin in the recursive crawl.</p></article></main></body></html>",
+        "/deep/contact": b"<html><body><main><article><h1>Deep Contact</h1><p>This depth-two article completes the fixed local graph without relying on mutable third-party pages or network policy.</p></article></main></body></html>",
         "/favicon.ico": b"test-icon",
     }
 
@@ -557,12 +577,11 @@ def wait_for_process(predicate: Callable[[psutil.Process, str], bool], *, timeou
 
 def pid_is_alive(pid: int) -> bool:
     try:
-        os.kill(pid, 0)
-    except ProcessLookupError:
+        return psutil.Process(pid).status() != psutil.STATUS_ZOMBIE
+    except (psutil.NoSuchProcess, psutil.ZombieProcess):
         return False
-    except PermissionError:
+    except psutil.AccessDenied:
         return True
-    return True
 
 
 def wait_for_pid_to_disappear(pid: int, *, timeout: float = 20.0) -> None:
@@ -920,7 +939,7 @@ def _wait_for_archivebox_workers(cwd: Path, env: dict[str, str], names: tuple[st
     state: dict[str, Any] = {}
     while time.time() < deadline:
         state = _archivebox_worker_state(cwd, env)
-        if all(state.get(name, {}).get("statename") == "RUNNING" for name in names):
+        if all(isinstance(state.get(name), dict) and state[name].get("statename") == "RUNNING" for name in names):
             return state
         time.sleep(1)
     return state

archivebox/tests/test_api_archiveresult.py

@@ -1 +0,0 @@
-# Tests moved to test_api_v1_core_archiveresults.py and test_api_v1_core_archiveresult_archiveresult_id.py.

archivebox/tests/test_api_cli.py

@@ -1 +0,0 @@
-# CLI endpoint tests moved to test_api_v1_cli_add.py and test_api_v1_cli_update.py.

archivebox/tests/test_api_cli_schedule.py

@@ -1 +0,0 @@
-# CLI schedule endpoint tests moved to test_api_v1_cli_schedule.py.

archivebox/tests/test_api_crawl.py

@@ -1 +0,0 @@
-# Tests moved to test_api_v1_crawls_crawl_crawl_id.py.

archivebox/tests/test_api_crud.py

@@ -1 +0,0 @@
-# Tests moved to test_api_v1_workflow_core_token_auth_side_effects.py and exact endpoint files under test_api_v1_core_*.

archivebox/tests/test_api_delete_paths.py

@@ -1 +0,0 @@
-# Tests moved to test_api_v1_core_snapshot_snapshot_id.py and test_api_v1_crawls_crawl_crawl_id.py.

archivebox/tests/test_api_personas.py

@@ -1 +0,0 @@
-# Tests moved to test_api_v1_personas_sync.py and test_api_v1_personas_personas.py.

archivebox/tests/test_api_remove.py

@@ -1 +0,0 @@
-# CLI remove endpoint tests moved to test_api_v1_cli_remove.py.

archivebox/tests/test_api_rss.py

@@ -1 +0,0 @@
-# Tests moved to test_api_v1_core_snapshots_rss.py and test_api_v1_crawls_crawl_crawl_id.py.

archivebox/tests/test_api_search.py

@@ -1 +0,0 @@
-# Tests moved to test_api_v1_core_snapshots.py.

archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py

@@ -286,7 +286,7 @@ def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_
         assert output_path.read_text() == "finished result should stay finished"
 
 
-def test_targeted_extract_retries_one_failed_archiveresult_while_snapshot_stays_paused(
+def test_targeted_extract_retries_one_failed_archiveresult_through_normal_snapshot_lifecycle(
     tmp_path,
     client,
     recursive_test_site,
@@ -370,8 +370,10 @@ def test_targeted_extract_retries_one_failed_archiveresult_while_snapshot_stays_
 
     with use_archivebox_db(tmp_path):
         snapshot = Snapshot.objects.get(id=snapshot_id)
-        assert snapshot.status == Snapshot.StatusChoices.PAUSED
-        assert snapshot.retry_at == RETRY_AT_MAX
+        assert snapshot.status == Snapshot.StatusChoices.STARTED
+        assert snapshot.retry_at is not None
+        assert snapshot.retry_at != RETRY_AT_MAX
+        assert snapshot.crawl.status == snapshot.crawl.StatusChoices.STARTED
 
         retried_wget = ArchiveResult.objects.get(id=wget_result.id)
         assert retried_wget.status == ArchiveResult.StatusChoices.SUCCEEDED
@@ -389,54 +391,51 @@ def test_targeted_extract_retries_one_failed_archiveresult_while_snapshot_stays_
 
 
 @pytest.mark.timeout(240)
-def test_paused_snapshot_survives_server_restart_and_resumes_via_api(tmp_path, recursive_test_site):
+def test_paused_snapshot_survives_server_restart_and_resumes_via_api(client, tmp_path, recursive_test_site):
     init_archive(tmp_path)
 
     port = get_free_port()
     env = cli_env(port=port, server=True, PLUGINS="wget", SAVE_WGET="True")
     api_token = create_admin_and_token(tmp_path)
 
-    try:
-        start_archivebox_server(tmp_path, env=env, port=port)
-        wait_for_live_api(port)
-
-        crawl_response = live_api_request(
-            port,
+    with use_archivebox_db(tmp_path):
+        crawl_response = api_client_request(
+            client,
             "post",
             "/api/v1/crawls/crawls",
             api_token=api_token,
-            json={
+            payload={
                 "urls": [recursive_test_site["root_url"]],
                 "max_depth": 0,
                 "tags": ["snapshot-pause-restart-e2e"],
                 "config": {"PLUGINS": "wget", "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*"},
             },
-            timeout=10,
         )
-        assert crawl_response.status_code == 200, crawl_response.text
-        crawl_id = crawl_response.json()["id"]
-        crawl_state = _wait_for_crawl_snapshot_rows(tmp_path, crawl_id)
-        snapshot_id = crawl_state["snapshots"][0]["id"]
-
-        pause_response = live_api_request(
-            port,
+        assert crawl_response.status_code == 200, crawl_response.content.decode()
+        crawl_id = json.loads(crawl_response.content.decode())["id"]
+        crawl = Crawl.objects.get(id=crawl_id)
+        snapshot = Snapshot.objects.create(
+            url=recursive_test_site["root_url"],
+            crawl=crawl,
+            status=Snapshot.StatusChoices.QUEUED,
+            retry_at=timezone.now(),
+        )
+        snapshot_id = str(snapshot.id)
+        pause_response = api_client_request(
+            client,
             "patch",
-            f"/api/v1/crawls/crawl/{crawl_id}",
+            f"/api/v1/core/snapshot/{snapshot_id}",
             api_token=api_token,
-            json={"action": "pause"},
-            timeout=10,
+            payload={"action": "pause"},
         )
-        assert pause_response.status_code == 200, pause_response.text
+        assert pause_response.status_code == 200, pause_response.content.decode()
 
-        current_state = _paused_snapshot_state(tmp_path, snapshot_id)
-        if current_state["status"] == Snapshot.StatusChoices.SEALED:
-            assert current_state["succeeded_results"] > 0
-            return
+    try:
+        start_archivebox_server(tmp_path, env=env, port=port)
+        wait_for_live_api(port)
 
         paused_state = _wait_for_paused_scheduler_marker(tmp_path, snapshot_id)
-        if paused_state["status"] == Snapshot.StatusChoices.SEALED:
-            assert paused_state["succeeded_results"] > 0
-            return
+        assert paused_state["status"] == Snapshot.StatusChoices.PAUSED
         assert paused_state["succeeded_results"] == 0
         assert not list((paused_state["snapshot_dir"] / "wget").rglob("*.html"))
 

archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py

@@ -130,20 +130,42 @@ def wait_for_crawl_wget_success_or_sealed(cwd, crawl_id, timeout=240):
     raise AssertionError(f"timed out waiting for crawl resume completion for crawl {crawl_id}: {latest_state}")
 
 
-def wait_for_sqlite_index_result(cwd, crawl_id, timeout=45):
-    deadline = time.time() + timeout
-    latest_state = None
-    while time.time() < deadline:
-        latest_state = get_crawl_runtime_state(cwd, crawl_id)
-        final_results = [
-            result
-            for result in latest_state["results"]
-            if result["plugin"] == "search_backend_sqlite" and result["status"] not in {"queued", "started", "paused"}
-        ]
-        if final_results:
-            return latest_state
-        time.sleep(0.2)
-    raise AssertionError(f"timed out waiting for sqlite index result for crawl {crawl_id}: {latest_state}")
+def seed_paused_crawl(client, cwd: Path, api_token: str, url: str, tag: str) -> tuple[str, str]:
+    from archivebox.services.runner import run_due_snapshot
+
+    with use_archivebox_db(cwd):
+        response = api_client_request(
+            client,
+            "post",
+            "/api/v1/crawls/crawls",
+            api_token=api_token,
+            payload={
+                "urls": [url],
+                "max_depth": 0,
+                "tags": [tag],
+                "config": {"PLUGINS": "wget", "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*"},
+            },
+        )
+        assert response.status_code == 200, response.content.decode()
+        crawl_id = json.loads(response.content.decode())["id"]
+        crawl = Crawl.objects.get(id=crawl_id)
+        snapshot = Snapshot.objects.create(url=url, crawl=crawl, status=Snapshot.StatusChoices.QUEUED, retry_at=timezone.now())
+        pause_response = api_client_request(
+            client,
+            "patch",
+            f"/api/v1/crawls/crawl/{crawl_id}",
+            api_token=api_token,
+            payload={"action": "pause"},
+        )
+        assert pause_response.status_code == 200, pause_response.content.decode()
+        assert run_due_snapshot(snapshot, lock_seconds=60)
+        crawl.refresh_from_db()
+        snapshot.refresh_from_db()
+        assert crawl.status == Crawl.StatusChoices.PAUSED
+        assert crawl.retry_at == RETRY_AT_MAX
+        assert snapshot.status == Snapshot.StatusChoices.PAUSED
+        assert snapshot.retry_at == RETRY_AT_MAX
+        return str(crawl_id), str(snapshot.id)
 
 
 def make_snapshot(*, user, url: str, title: str, bookmarked_at: datetime):
@@ -341,55 +363,24 @@ def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snap
 
 
 @pytest.mark.timeout(240)
-def test_crawl_pause_resume_api_survives_server_restart_and_processes_after_resume(tmp_path, recursive_test_site):
+def test_crawl_pause_resume_api_survives_server_restart_and_processes_after_resume(client, tmp_path, recursive_test_site):
     init_archive(tmp_path)
 
     port = get_free_port()
     env = cli_env(port=port, server=True, PLUGINS="wget", SAVE_WGET="True")
     api_token = create_admin_and_token(tmp_path)
+    crawl_id, _snapshot_id = seed_paused_crawl(client, tmp_path, api_token, recursive_test_site["root_url"], "pause-resume-e2e")
 
     try:
         start_archivebox_server(tmp_path, env=env, port=port)
         wait_for_live_api(port)
 
-        crawl_response = live_api_request(
-            port,
-            "post",
-            "/api/v1/crawls/crawls",
-            api_token=api_token,
-            json={
-                "urls": [recursive_test_site["root_url"]],
-                "max_depth": 0,
-                "tags": ["pause-resume-e2e"],
-                "config": {"PLUGINS": "wget", "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*"},
-            },
-            timeout=10,
-        )
-        assert crawl_response.status_code == 200, crawl_response.text
-        crawl_id = crawl_response.json()["id"]
-        wait_for_crawl_snapshot_rows(tmp_path, crawl_id)
-
-        pause_response = live_api_request(
-            port,
-            "patch",
-            f"/api/v1/crawls/crawl/{crawl_id}",
-            api_token=api_token,
-            json={"action": "pause"},
-            timeout=10,
-        )
-        assert pause_response.status_code == 200, pause_response.text
-        assert pause_response.json()["status"] == "paused"
-
         paused_state = wait_for_crawl_child_snapshots_paused_or_sealed(tmp_path, crawl_id)
         assert paused_state["crawl_status"] == "paused"
         assert paused_state["crawl_retry_at"] == paused_state["retry_at_max"]
         assert len(paused_state["snapshots"]) == 1
-        snapshot_finished_before_pause = paused_state["snapshots"][0]["status"] == "sealed"
-        if snapshot_finished_before_pause:
-            assert any(result["status"] == "succeeded" for result in paused_state["results"])
-        else:
-            assert paused_state["snapshots"][0]["status"] == "paused"
-            assert paused_state["snapshots"][0]["retry_at"] == paused_state["retry_at_max"]
+        assert paused_state["snapshots"][0]["status"] == "paused"
+        assert paused_state["snapshots"][0]["retry_at"] == paused_state["retry_at_max"]
 
         stop_server(tmp_path)
         start_archivebox_server(tmp_path, env=env, port=port)
@@ -398,10 +389,6 @@ def test_crawl_pause_resume_api_survives_server_restart_and_processes_after_resu
         restarted_state = get_crawl_runtime_state(tmp_path, crawl_id)
         assert restarted_state["crawl_status"] == "paused"
         assert restarted_state["crawl_retry_at"] == restarted_state["retry_at_max"]
-        if snapshot_finished_before_pause:
-            assert restarted_state["snapshots"][0]["status"] == "sealed"
-            assert any(result["status"] == "succeeded" for result in restarted_state["results"])
-            return
         assert restarted_state["snapshots"][0]["status"] == "paused"
         assert restarted_state["snapshots"][0]["retry_at"] == restarted_state["retry_at_max"]
         assert not any(result["status"] == "succeeded" for result in restarted_state["results"])
@@ -431,62 +418,13 @@ def test_crawl_pause_resume_api_survives_server_restart_and_processes_after_resu
 
 
 @pytest.mark.timeout(420)
-def test_update_index_only_runs_paused_search_rows_and_resume_later_runs_crawl(tmp_path, recursive_test_site):
+def test_update_index_only_leaves_paused_snapshot_on_normal_lifecycle_path(client, tmp_path, recursive_test_site):
     init_archive(tmp_path)
 
     port = get_free_port()
     env = cli_env(port=port, server=True, PLUGINS="wget", SAVE_WGET="True")
     api_token = create_admin_and_token(tmp_path)
-
-    try:
-        start_archivebox_server(tmp_path, env=env, port=port)
-        wait_for_live_api(port)
-
-        crawl_response = live_api_request(
-            port,
-            "post",
-            "/api/v1/crawls/crawls",
-            api_token=api_token,
-            json={
-                "urls": [recursive_test_site["root_url"]],
-                "max_depth": 0,
-                "tags": ["paused-index-e2e"],
-                "config": {"PLUGINS": "wget", "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*"},
-            },
-            timeout=10,
-        )
-        assert crawl_response.status_code == 200, crawl_response.text
-        crawl_id = crawl_response.json()["id"]
-        wait_for_crawl_snapshot_rows(tmp_path, crawl_id)
-
-        pause_response = live_api_request(
-            port,
-            "patch",
-            f"/api/v1/crawls/crawl/{crawl_id}",
-            api_token=api_token,
-            json={"action": "pause"},
-            timeout=10,
-        )
-        assert pause_response.status_code == 200, pause_response.text
-        paused_state = wait_for_crawl_child_snapshots_paused_or_sealed(tmp_path, crawl_id)
-        snapshot_finished_before_pause = paused_state["snapshots"][0]["status"] == "sealed"
-    finally:
-        stop_server(tmp_path)
-
-    if snapshot_finished_before_pause:
-        indexed_state = get_crawl_runtime_state(tmp_path, crawl_id)
-        assert indexed_state["snapshots"][0]["status"] == "sealed"
-        wget_results = [result for result in indexed_state["results"] if result["plugin"] == "wget"]
-        assert any(result["status"] == "succeeded" and result["output_size"] > 0 for result in wget_results)
-        if indexed_state["crawl_status"] == "paused":
-            assert indexed_state["crawl_retry_at"] == indexed_state["retry_at_max"]
-        else:
-            assert indexed_state["crawl_status"] == "sealed"
-            assert all(result["status"] not in {"queued", "started", "paused"} for result in indexed_state["results"])
-        captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=60)
-        assert "Root" in captured_text
-        assert "About" in captured_text
-        return
+    crawl_id, _snapshot_id = seed_paused_crawl(client, tmp_path, api_token, recursive_test_site["root_url"], "paused-index-e2e")
 
     update_env = cli_env(
         port=port,
@@ -510,14 +448,13 @@ def test_update_index_only_runs_paused_search_rows_and_resume_later_runs_crawl(t
     )
     assert update_process.returncode == 0, update_process.stderr
 
-    indexed_state = wait_for_sqlite_index_result(tmp_path, crawl_id)
+    indexed_state = get_crawl_runtime_state(tmp_path, crawl_id)
     assert indexed_state["crawl_status"] == "paused"
     assert indexed_state["crawl_retry_at"] == indexed_state["retry_at_max"]
     assert indexed_state["snapshots"][0]["status"] == "paused"
     assert indexed_state["snapshots"][0]["retry_at"] == indexed_state["retry_at_max"]
     search_results = [result for result in indexed_state["results"] if result["plugin"] == "search_backend_sqlite"]
-    assert search_results
-    assert any(result["status"] not in {"queued", "started", "paused"} for result in search_results)
+    assert search_results == []
 
     try:
         start_archivebox_server(tmp_path, env=env, port=port)
@@ -543,14 +480,10 @@ def test_update_index_only_runs_paused_search_rows_and_resume_later_runs_crawl(t
 
         assert resumed_state["snapshots"][0]["status"] == "sealed"
         wget_results = [result for result in resumed_state["results"] if result["plugin"] == "wget"]
-        wget_succeeded = any(result["status"] == "succeeded" and result["output_size"] > 0 for result in wget_results)
-        if wget_succeeded:
-            captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=60)
-            assert "Root" in captured_text
-            assert "About" in captured_text
-        else:
-            assert resumed_state["crawl_status"] == "sealed"
-            assert all(result["status"] not in {"queued", "started", "paused"} for result in resumed_state["results"])
+        assert any(result["status"] == "succeeded" and result["output_size"] > 0 for result in wget_results)
+        captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=60)
+        assert "Root" in captured_text
+        assert "About" in captured_text
     finally:
         stop_server(tmp_path)
 

archivebox/tests/test_api_v1_workflow_core_token_auth_side_effects.py

@@ -1,5 +1,3 @@
-import time
-
 import pytest
 import requests
 
@@ -21,17 +19,6 @@ from archivebox.tests.test_orm_helpers import use_archivebox_db
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
-def live_api_request_retrying_sqlite_lock(*args, attempts: int = 12, delay: float = 2.0, **kwargs):
-    response = None
-    for _attempt in range(attempts):
-        response = live_api_request(*args, **kwargs)
-        if response.status_code != 503 or "database is locked" not in response.text:
-            return response
-        time.sleep(delay)
-    assert response is not None
-    return response
-
-
 @pytest.mark.timeout(180)
 def test_core_api_workflow_uses_token_auth_and_persists_side_effects_over_server(tmp_path, recursive_test_site):
     init_archive(tmp_path)
@@ -113,7 +100,7 @@ def test_core_api_workflow_uses_token_auth_and_persists_side_effects_over_server
         assert snapshot_payload["url"] == recursive_test_site["child_urls"][0]
         assert snapshot_payload["tags"] == ["api-child"]
 
-        patch_snapshot = live_api_request_retrying_sqlite_lock(
+        patch_snapshot = live_api_request(
             port,
             "patch",
             f"/api/v1/core/snapshot/{snapshot_id}",
@@ -125,7 +112,7 @@ def test_core_api_workflow_uses_token_auth_and_persists_side_effects_over_server
         assert patch_snapshot.json()["status"] == "sealed"
         assert set(patch_snapshot.json()["tags"]) == {"api-child", "api-patched"}
 
-        tag_create = live_api_request_retrying_sqlite_lock(
+        tag_create = live_api_request(
             port,
             "post",
             "/api/v1/core/tags/create/",
@@ -136,7 +123,7 @@ def test_core_api_workflow_uses_token_auth_and_persists_side_effects_over_server
         assert tag_create.status_code == 200, tag_create.text
         tag_id = tag_create.json()["tag_id"]
 
-        add_tag = live_api_request_retrying_sqlite_lock(
+        add_tag = live_api_request(
             port,
             "post",
             "/api/v1/core/tags/add-to-snapshot/",
@@ -145,7 +132,7 @@ def test_core_api_workflow_uses_token_auth_and_persists_side_effects_over_server
             timeout=10,
         )
         assert add_tag.status_code == 200, add_tag.text
-        remove_tag = live_api_request_retrying_sqlite_lock(
+        remove_tag = live_api_request(
             port,
             "post",
             "/api/v1/core/tags/remove-from-snapshot/",
@@ -155,7 +142,7 @@ def test_core_api_workflow_uses_token_auth_and_persists_side_effects_over_server
         )
         assert remove_tag.status_code == 200, remove_tag.text
 
-        crawl_patch = live_api_request_retrying_sqlite_lock(
+        crawl_patch = live_api_request(
             port,
             "patch",
             f"/api/v1/crawls/crawl/{crawl_id}",

archivebox/tests/test_archiveresult_pause.py

@@ -1,2 +0,0 @@
-# test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snapshot_results_alone moved to test_api_v1_crawls_crawl_crawl_id.py.
-# test_targeted_extract_retries_one_failed_archiveresult_while_snapshot_stays_paused moved to test_api_v1_core_snapshot_snapshot_id.py.

archivebox/tests/test_cli_run.py

@@ -991,6 +991,42 @@ class TestRecoverOrchestratorState:
         assert hook_names
         assert all(not hook_name.endswith((".py", ".js", ".sh")) for hook_name in hook_names)
 
+    def test_snapshot_hooks_for_pending_archiveresults_respects_disabled_plugins_when_plugins_empty(self):
+        from django.utils import timezone
+
+        from archivebox.base_models.models import get_or_create_system_user_pk
+        from archivebox.crawls.models import Crawl
+        from archivebox.core.models import Snapshot
+        from archivebox.services.runner import snapshot_hooks_for_pending_archiveresults
+
+        crawl = Crawl.objects.create(
+            urls="https://example.com",
+            created_by_id=get_or_create_system_user_pk(),
+            config={
+                "CLAUDECHROME_ENABLED": False,
+                "CLAUDECODEEXTRACT_ENABLED": False,
+                "CLAUDECODECLEANUP_ENABLED": False,
+                "SEARCH_BACKEND_SQLITE_ENABLED": False,
+            },
+            status=Crawl.StatusChoices.STARTED,
+            retry_at=timezone.now(),
+        )
+        snapshot = Snapshot.objects.create(
+            url="https://example.com",
+            crawl=crawl,
+            status=Snapshot.StatusChoices.QUEUED,
+            retry_at=timezone.now(),
+        )
+
+        hooks = snapshot_hooks_for_pending_archiveresults(snapshot)
+        queued_plugins = {plugin for plugin, _hook_name in hooks}
+
+        assert "title" in queued_plugins
+        assert "claudechrome" not in queued_plugins
+        assert "claudecodeextract" not in queued_plugins
+        assert "claudecodecleanup" not in queued_plugins
+        assert "search_backend_sqlite" not in queued_plugins
+
     def test_run_due_snapshot_pauses_child_when_parent_is_paused(self):
         from django.utils import timezone
 
@@ -1465,8 +1501,76 @@ class TestRecoverOrchestratorState:
         result.refresh_from_db()
         snapshot.refresh_from_db()
         assert result.status == ArchiveResult.StatusChoices.FAILED
+        assert result.output_str == "Queued hook is no longer available in the installed plugin"
         assert snapshot.retry_at is None
 
+    @pytest.mark.django_db(transaction=True)
+    def test_run_due_snapshot_skips_disabled_queued_plugins_and_seals_started_snapshot(self):
+        from django.utils import timezone
+
+        from archivebox.base_models.models import get_or_create_system_user_pk
+        from archivebox.crawls.models import Crawl
+        from archivebox.core.models import ArchiveResult, Snapshot
+        from archivebox.services.runner import run_due_snapshot
+
+        crawl = Crawl.objects.create(
+            urls="https://example.com",
+            created_by_id=get_or_create_system_user_pk(),
+            config={
+                "CLAUDECHROME_ENABLED": False,
+                "CLAUDECODEEXTRACT_ENABLED": False,
+                "CLAUDECODECLEANUP_ENABLED": False,
+                "SEARCH_BACKEND_SQLITE_ENABLED": False,
+            },
+            status=Crawl.StatusChoices.STARTED,
+            retry_at=timezone.now(),
+        )
+        snapshot = Snapshot.objects.create(
+            url="https://example.com",
+            crawl=crawl,
+            status=Snapshot.StatusChoices.STARTED,
+            retry_at=timezone.now(),
+            downloaded_at=timezone.now(),
+        )
+        ArchiveResult.objects.create(
+            snapshot=snapshot,
+            plugin="title",
+            hook_name="on_Snapshot__01_title",
+            status=ArchiveResult.StatusChoices.SUCCEEDED,
+            output_str="Example Domain",
+        )
+        stale_results = [
+            ArchiveResult.objects.create(
+                snapshot=snapshot,
+                plugin=plugin,
+                hook_name=hook_name,
+                status=ArchiveResult.StatusChoices.QUEUED,
+            )
+            for plugin, hook_name in (
+                ("claudechrome", "on_Snapshot__47_claudechrome"),
+                ("claudecodeextract", "on_Snapshot__58_claudecodeextract"),
+                ("claudecodecleanup", "on_Snapshot__92_claudecodecleanup"),
+                ("search_backend_sqlite", "on_Snapshot__90_index_sqlite"),
+            )
+        ]
+        stale_result_ids = [result.id for result in stale_results]
+        stale_plugins = [result.plugin for result in stale_results]
+
+        assert run_due_snapshot(snapshot, lock_seconds=60) is True
+
+        snapshot.refresh_from_db()
+        assert snapshot.status == Snapshot.StatusChoices.SEALED
+        assert snapshot.retry_at is None
+        assert not snapshot.archiveresult_set.filter(
+            plugin__in=stale_plugins,
+            status=ArchiveResult.StatusChoices.QUEUED,
+        ).exists()
+        for result in ArchiveResult.objects.filter(id__in=stale_result_ids):
+            assert result.status == ArchiveResult.StatusChoices.SKIPPED
+            assert result.start_ts is not None
+            assert result.end_ts is not None
+            assert "disabled by this Snapshot/Crawl config" in result.output_str
+
     @pytest.mark.django_db(transaction=True)
     @pytest.mark.timeout(300)
     @pytest.mark.parametrize("chrome_isolation", ["crawl", "snapshot"])

archivebox/tests/test_cli_update_reindex_snapshots.py

@@ -12,6 +12,84 @@ from archivebox.tests.test_orm_helpers import use_archivebox_db
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
+def test_only_sealed_search_backfill_bypasses_snapshot_lifecycle():
+    from archivebox.base_models.models import get_or_create_system_user_pk
+    from archivebox.cli.archivebox_extract import run_plugins
+    from archivebox.core.models import ArchiveResult
+    from archivebox.crawls.models import Crawl
+
+    search_crawl = Crawl.objects.create(
+        urls="https://example.com/search",
+        created_by_id=get_or_create_system_user_pk(),
+        status=Crawl.StatusChoices.SEALED,
+    )
+    search_snapshot = Snapshot.objects.create(
+        url="https://example.com/search",
+        crawl=search_crawl,
+        status=Snapshot.StatusChoices.SEALED,
+    )
+    extract_crawl = Crawl.objects.create(
+        urls="https://example.com/extract",
+        created_by_id=get_or_create_system_user_pk(),
+        status=Crawl.StatusChoices.SEALED,
+    )
+    extract_snapshot = Snapshot.objects.create(
+        url="https://example.com/extract",
+        crawl=extract_crawl,
+        status=Snapshot.StatusChoices.SEALED,
+    )
+
+    assert (
+        run_plugins(
+            args=(),
+            records=[
+                {
+                    "type": "ArchiveResult",
+                    "snapshot_id": str(search_snapshot.id),
+                    "plugin": "search_backend_sqlite",
+                },
+            ],
+            wait=False,
+            emit_results=False,
+            show_progress=False,
+        )
+        == 0
+    )
+    search_crawl.refresh_from_db()
+    search_snapshot.refresh_from_db()
+    assert search_crawl.status == Crawl.StatusChoices.SEALED
+    assert search_snapshot.status == Snapshot.StatusChoices.SEALED
+    assert search_snapshot.archiveresult_set.filter(
+        plugin="search_backend_sqlite",
+        status=ArchiveResult.StatusChoices.QUEUED,
+    ).exists()
+
+    assert (
+        run_plugins(
+            args=(),
+            records=[
+                {
+                    "type": "ArchiveResult",
+                    "snapshot_id": str(extract_snapshot.id),
+                    "plugin": "wget",
+                },
+            ],
+            wait=False,
+            emit_results=False,
+            show_progress=False,
+        )
+        == 0
+    )
+    extract_crawl.refresh_from_db()
+    extract_snapshot.refresh_from_db()
+    assert extract_crawl.status == Crawl.StatusChoices.QUEUED
+    assert extract_snapshot.status == Snapshot.StatusChoices.QUEUED
+    assert extract_snapshot.archiveresult_set.filter(
+        plugin="wget",
+        status=ArchiveResult.StatusChoices.QUEUED,
+    ).exists()
+
+
 def test_update_imports_orphaned_snapshots(tmp_path, initialized_archive):
     """Test that archivebox update imports real legacy archive directories."""
     env = cli_env(disable_extractors=True)
@@ -67,6 +145,11 @@ def test_reindex_snapshots_resets_existing_search_results_and_reruns_requested_p
         crawl=crawl,
         status=Snapshot.StatusChoices.SEALED,
     )
+    paused_snapshot = Snapshot.objects.create(
+        url="https://example.com/paused",
+        crawl=crawl,
+        status=Snapshot.StatusChoices.PAUSED,
+    )
     result = ArchiveResult.objects.create(
         snapshot=snapshot,
         plugin="search_backend_sqlite",
@@ -87,7 +170,7 @@ def test_reindex_snapshots_resets_existing_search_results_and_reruns_requested_p
     os.environ["SEARCH_BACKEND_ENGINE"] = "sqlite"
     try:
         stats = reindex_snapshots(
-            Snapshot.objects.filter(id=snapshot.id),
+            Snapshot.objects.filter(id__in=(snapshot.id, paused_snapshot.id)),
             search_plugins=["search_backend_sqlite"],
             batch_size=10,
         )
@@ -105,6 +188,7 @@ def test_reindex_snapshots_resets_existing_search_results_and_reruns_requested_p
     assert result.status == ArchiveResult.StatusChoices.QUEUED
     assert result.output_str == ""
     assert result.output_json is None
+    assert not paused_snapshot.archiveresult_set.exists()
 
 
 @pytest.mark.django_db

archivebox/tests/test_cli_version.py

@@ -10,7 +10,7 @@ import tempfile
 from pathlib import Path
 from archivebox.config.paths import tmp_dir_socket_path_is_short_enough
 from archivebox.cli.archivebox_version import _binary_row_dedupe_key
-from archivebox.tests.conftest import run_archivebox_cmd
+from archivebox.tests.conftest import cli_env, run_archivebox_cmd
 
 
 def _make_deep_collection_dir(tmp_path: Path) -> Path:
@@ -126,6 +126,83 @@ def test_version_shows_binaries_after_init(tmp_path, initialized_archive):
     assert "Binary" in output or "Dependencies" in output
 
 
+def test_version_skips_disabled_plugin_binary_resolution(tmp_path):
+    """Disabled plugins should not trigger live binary detection during version."""
+    data_dir = tmp_path / "no-plugins"
+    data_dir.mkdir()
+    env = cli_env(PLUGINS="__archivebox_test_no_plugins__", SEARCH_BACKEND_ENGINE="")
+
+    init_result = run_archivebox_cmd(["init"], cwd=data_dir, env=env)
+    assert init_result.returncode == 0, init_result.stderr
+
+    version_result = run_archivebox_cmd(["version"], cwd=data_dir, env=env)
+    output = version_result.stdout + version_result.stderr
+
+    assert version_result.returncode == 0, output
+    assert "No required binaries declared for discovered plugins" in output
+    assert "not installed" not in output
+
+
+def test_version_honors_legacy_save_aliases_when_disabling_extractors(tmp_path):
+    """Legacy SAVE_* aliases should disable their canonical plugin flags."""
+    data_dir = tmp_path / "legacy-save-disabled"
+    data_dir.mkdir()
+    env = {"PLUGINS": "wget,title", "SEARCH_BACKEND_ENGINE": ""}
+
+    init_result = run_archivebox_cmd(["init"], cwd=data_dir, env=env, default_cli_env=True, disable_extractors=True, timeout=120)
+    assert init_result.returncode == 0, init_result.stderr or init_result.stdout
+
+    config_result = run_archivebox_cmd(
+        ["config", "--get", "WGET_ENABLED", "TITLE_ENABLED", "CHROME_ENABLED"],
+        cwd=data_dir,
+        env=env,
+        default_cli_env=True,
+        disable_extractors=True,
+    )
+    config_output = config_result.stdout + config_result.stderr
+    assert config_result.returncode == 0, config_output
+    assert "WGET_ENABLED = false" in config_output
+    assert "TITLE_ENABLED = false" in config_output
+    assert "CHROME_ENABLED = false" in config_output
+
+    version_result = run_archivebox_cmd(
+        ["version"],
+        cwd=data_dir,
+        env=env,
+        default_cli_env=True,
+        disable_extractors=True,
+        timeout=60,
+    )
+    output = version_result.stdout + version_result.stderr
+
+    assert version_result.returncode == 0, output
+    assert "not installed" not in output
+
+
+def test_plugins_selection_includes_required_plugins_via_config_cli(tmp_path):
+    """PLUGINS selection should include transitive plugin dependencies exactly once."""
+    data_dir = tmp_path / "plugin-dependencies"
+    data_dir.mkdir()
+    env = {"PLUGINS": "wget,screenshot"}
+
+    init_result = run_archivebox_cmd(["init"], cwd=data_dir, env=env, default_cli_env=True, timeout=120)
+    assert init_result.returncode == 0, init_result.stderr or init_result.stdout
+
+    config_result = run_archivebox_cmd(
+        ["config", "--get", "WGET_ENABLED", "SCREENSHOT_ENABLED", "CHROME_ENABLED", "YTDLP_ENABLED"],
+        cwd=data_dir,
+        env=env,
+        default_cli_env=True,
+    )
+    config_output = config_result.stdout + config_result.stderr
+
+    assert config_result.returncode == 0, config_output
+    assert "WGET_ENABLED = true" in config_output
+    assert "SCREENSHOT_ENABLED = true" in config_output
+    assert "CHROME_ENABLED = true" in config_output
+    assert "YTDLP_ENABLED = false" in config_output
+
+
 def test_version_shows_data_locations(tmp_path, initialized_archive):
     """Test that version shows data directory locations."""
     result = run_archivebox_cmd(["version"])

archivebox/tests/test_crawl_pause.py

@@ -6,7 +6,3 @@ from archivebox.workers.models import RETRY_AT_MAX
 def test_retry_at_max_is_safe_for_admin_timezone_localization():
     with timezone.override("Pacific/Kiritimati"):
         assert timezone.localtime(RETRY_AT_MAX).year == 9999
-
-
-# test_crawl_pause_resume_api_survives_server_restart_and_processes_after_resume moved to test_api_v1_crawls_crawl_crawl_id.py.
-# test_update_index_only_runs_paused_search_rows_and_resume_later_runs_crawl moved to test_api_v1_crawls_crawl_crawl_id.py.

archivebox/tests/test_crawl_runner.py

@@ -752,9 +752,6 @@ def test_sealed_crawl_does_not_create_discovered_snapshots():
     assert crawl.snapshot_set.count() == 1
 
 
-# test_create_crawl_api_queues_crawl_without_spawning_runner moved to test_api_v1_crawls_crawls.py.
-
-
 def test_wait_for_snapshot_tasks_surfaces_already_failed_task():
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.crawls.models import Crawl

archivebox/tests/test_frozen_crawl_config.py

@@ -269,9 +269,6 @@ def test_crawl_config_projections_stay_under_hot_path_budget():
         assert average_seconds < max_average_seconds, f"{name} averaged {average_seconds * 1000:.3f}ms"
 
 
-# test_api_create_and_cli_add_store_full_frozen_config moved to test_api_v1_workflow_frozen_crawl_config_sources.py.
-
-
 def test_schedule_enqueue_refreezes_using_current_template_persona_defaults(archivebox_db):
     from archivebox.crawls.models import Crawl, CrawlSchedule
 

archivebox/tests/test_opencode_agent.py

@@ -5,8 +5,8 @@ from pathlib import Path
 from types import SimpleNamespace
 from urllib.parse import quote
 
-import psutil
 import pytest
+import requests
 
 from archivebox.tests.conftest import ADMIN_TEST_HOST, run_archivebox_cmd
 
@@ -54,7 +54,6 @@ def opencode_archive_config(initialized_archive):
         {
             "ABXPKG_INSTALL_TIMEOUT": "900",
             "ABXPKG_MIN_RELEASE_AGE": "0",
-            "ABX_RUNTIME": "archivebox",
             "ARCHIVEBOX_ALLOW_NO_UNIX_SOCKETS": "true",
             "OPENCODE_ENABLED": "True",
             "OPENCODE_HOST": "127.0.0.1",
@@ -175,10 +174,11 @@ def test_opencode_agent_superuser_gets_admin_wrapper(admin_client, live_opencode
     from abx_plugins.plugins.opencode import views
 
     response = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
+    recent_session_id = views._recent_session_id(live_opencode.settings)
 
     assert response.status_code == 200
-    assert f'<iframe src="{views._project_route(live_opencode.config.data_dir)}'.encode() in response.content
-    assert b'/session"' in response.content
+    assert recent_session_id
+    assert f'<iframe src="{views._project_route(live_opencode.config.data_dir, recent_session_id)}"'.encode() in response.content
     assert b'id="header"' in response.content
     assert b'id="progress-monitor"' in response.content
 
@@ -225,20 +225,24 @@ def test_opencode_proxy_sse_response_is_unbuffered(admin_client, live_opencode):
     assert response.headers["Cache-Control"] == "no-store"
 
 
-def test_opencode_starts_with_data_dir_cwd_and_isolated_state(live_opencode):
-    process = psutil.Process(live_opencode.process.pid)
-    env = process.environ()
+def test_opencode_starts_with_isolated_state(live_opencode):
     workdir = str(live_opencode.config.data_dir.resolve())
+    state_dir = live_opencode.config.state_dir
 
-    assert Path(process.cwd()).resolve() == live_opencode.config.data_dir.resolve()
-    assert env["BROWSER"] == "false"
-    assert env["HOME"] == str(live_opencode.config.state_dir / "home")
-    assert env["XDG_CONFIG_HOME"] == str(live_opencode.config.state_dir / "config")
-    assert env["XDG_DATA_HOME"] == str(live_opencode.config.state_dir / "data")
-    assert env["XDG_STATE_HOME"] == str(live_opencode.config.state_dir / "state")
-    assert env["XDG_CACHE_HOME"] == str(live_opencode.config.state_dir / "cache")
-    assert env["OPENCODE_DISABLE_PROJECT_CONFIG"] == "true"
-    assert env["GIT_CEILING_DIRECTORIES"] == workdir
+    project = requests.get(
+        f"{live_opencode.settings['origin']}/project/current",
+        params={"directory": workdir},
+        timeout=live_opencode.settings["timeout"],
+    )
+    project.raise_for_status()
+
+    assert Path(live_opencode.settings["workdir"]).resolve() == Path(workdir)
+    assert Path(str(project.json()["worktree"])).resolve() == Path(workdir)
+    assert live_opencode.process.poll() is None
+    assert (state_dir / "data" / "opencode" / "opencode.db").is_file()
+    assert (state_dir / "SKILL.md").is_file()
+    assert (state_dir / "config" / "opencode" / "skills" / "archivebox" / "SKILL.md").resolve() == state_dir / "SKILL.md"
+    assert not (live_opencode.config.data_dir / ".git" / "not-a-git").exists()
 
 
 def test_opencode_state_dir_is_separate_from_workdir(tmp_path):

archivebox/tests/test_orm_helpers.py

@@ -11,6 +11,12 @@ def archivebox_db_path(path: str | Path = ".") -> Path:
     return path if path.name == "index.sqlite3" else path / "index.sqlite3"
 
 
+def test_archivebox_db_path_accepts_collection_or_database_path(tmp_path: Path) -> None:
+    database_path = tmp_path / "index.sqlite3"
+    assert archivebox_db_path(tmp_path) == database_path
+    assert archivebox_db_path(database_path) == database_path
+
+
 @contextmanager
 def use_archivebox_db(path: str | Path = ".") -> Iterator[None]:
     connection = connections["default"]

archivebox/tests/test_recursive_crawl.py

@@ -484,9 +484,13 @@ def test_add_archivewebpage_installs_required_chrome_dependency(initialized_arch
     assert "archivewebpage" in binaries
     assert binaries["archivewebpage"]["status"] == Binary.StatusChoices.INSTALLED
     assert binaries["archivewebpage"]["binprovider"] == "chromewebstore"
-    archivewebpage_manifest = Path(binaries["archivewebpage"]["abspath"])
+    archivewebpage_metadata = Path(binaries["archivewebpage"]["abspath"])
+    assert archivewebpage_metadata.exists()
+    assert archivewebpage_metadata.name == "archivewebpage.extension.json"
+    archivewebpage_extension = json.loads(archivewebpage_metadata.read_text(encoding="utf-8"))
+    archivewebpage_manifest = Path(archivewebpage_extension["unpacked_path"]) / "manifest.json"
     assert archivewebpage_manifest.exists()
-    assert archivewebpage_manifest.name == "manifest.json"
+    assert json.loads(archivewebpage_manifest.read_text(encoding="utf-8"))["version"] == binaries["archivewebpage"]["version"]
 
     plugins_seen = {plugin for plugin, _hook_name, _status, _output_str, _output_files in archive_results}
     assert {"chrome", "archivewebpage"}.issubset(plugins_seen)
@@ -515,20 +519,36 @@ def test_add_archivewebpage_installs_required_chrome_dependency(initialized_arch
 
 
 @pytest.mark.timeout(1200)
-def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(initialized_archive, free_tcp_port_factory):
+def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(
+    initialized_archive,
+    free_tcp_port_factory,
+    recursive_test_site,
+):
     """Run a bounded real depth=2 crawl with all plugins enabled and verify parallel snapshot execution."""
 
     from abx_dl.models import discover_plugins
 
-    root_url = "https://example.com/"
+    root_url = recursive_test_site["root_url"]
     plugin_selection = ",".join(sorted(plugin for plugin in discover_plugins().keys() if not plugin.startswith("claude")))
     env = os.environ.copy()
-    env.pop("CHROME_BINARY", None)
+    for preinstalled_path_key in (
+        "CHROME_BINARY",
+        "LIB_DIR",
+        "DATA_DIR",
+        "NODE_MODULES_DIR",
+        "NODE_PATH",
+        "PNPM_BIN_DIR",
+        "NPM_BIN_DIR",
+        "CHROMEWEBSTORE_EXTENSIONS_DIR",
+    ):
+        env.pop(preinstalled_path_key, None)
     env.update(
         {
             "USE_COLOR": "false",
             "SHOW_PROGRESS": "false",
+            "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*",
             "ABXPKG_LIB_DIR": str(initialized_archive / "lib"),
+            "CHROMEWEBSTORE_EXTENSIONS_DIR": str(initialized_archive / "lib/chromewebstore/extensions"),
             "TIMEOUT": "90",
             "ABXPKG_INSTALL_TIMEOUT": "900",
             "CRAWL_MAX_CONCURRENT_SNAPSHOTS": "3",
@@ -606,10 +626,26 @@ def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(initia
     assert crawl.status == Crawl.StatusChoices.SEALED
     assert crawl.retry_at is None
 
-    assert len(snapshots) == 8
-    assert any(url == root_url and depth == 0 for _id, url, depth, _status, _parent, _downloaded_at in snapshots)
-    assert any("iana.org" in url and depth == 1 for _id, url, depth, _status, _parent, _downloaded_at in snapshots)
-    assert any(depth == 2 for _id, _url, depth, _status, _parent, _downloaded_at in snapshots)
+    expected_urls = {
+        root_url,
+        *recursive_test_site["child_urls"],
+        *recursive_test_site["deep_urls"],
+    }
+    snapshots_by_url = {url: (snapshot_id, depth, parent_id) for snapshot_id, url, depth, _status, parent_id, _downloaded_at in snapshots}
+    assert set(snapshots_by_url) == expected_urls
+    root_id, root_depth, root_parent_id = snapshots_by_url[root_url]
+    assert root_depth == 0
+    assert root_parent_id is None
+    child_ids_by_url = {}
+    for child_url in recursive_test_site["child_urls"]:
+        child_id, child_depth, child_parent_id = snapshots_by_url[child_url]
+        assert child_depth == 1
+        assert child_parent_id == root_id
+        child_ids_by_url[child_url] = child_id
+    for child_url, deep_url in zip(recursive_test_site["child_urls"], recursive_test_site["deep_urls"], strict=True):
+        _deep_id, deep_depth, deep_parent_id = snapshots_by_url[deep_url]
+        assert deep_depth == 2
+        assert deep_parent_id == child_ids_by_url[child_url]
     assert all(status == Snapshot.StatusChoices.SEALED for _id, _url, _depth, status, _parent, _downloaded_at in snapshots)
     assert all(downloaded_at is not None for _id, _url, _depth, _status, _parent, downloaded_at in snapshots)
 
@@ -632,6 +668,13 @@ def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(initia
         if not (status in allowed_statuses or (plugin == "archivedotorg" and status == ArchiveResult.StatusChoices.FAILED))
     ]
     assert not unexpected_results
+    ytdlp_results = [
+        (url, status, output_str)
+        for _snapshot_id, url, _depth, plugin, _hook_name, status, _files, _size, output_str in archive_results
+        if plugin == "ytdlp"
+    ]
+    assert ytdlp_results
+    assert all(status != ArchiveResult.StatusChoices.FAILED for _url, status, _output_str in ytdlp_results), ytdlp_results
 
     plugins_seen = {plugin for _snapshot_id, _url, _depth, plugin, _hook_name, _status, _files, _size, _output in archive_results}
     assert {
@@ -664,7 +707,11 @@ def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(initia
     assert list(snapshot_root.rglob("favicon/**/*"))
     urls_jsonl_files = list(snapshot_root.rglob("parse_html_urls/urls.jsonl"))
     assert urls_jsonl_files
-    assert any("iana.org" in path.read_text(errors="ignore") for path in urls_jsonl_files)
+    parsed_urls = {
+        json.loads(line)["url"] for path in urls_jsonl_files for line in path.read_text(errors="ignore").splitlines() if line.strip()
+    }
+    assert set(recursive_test_site["child_urls"]).issubset(parsed_urls)
+    assert set(recursive_test_site["deep_urls"]).issubset(parsed_urls)
 
     assert processes
     failed_hook_results = [

archivebox/tests/test_snapshot_pause.py

@@ -1 +0,0 @@
-# test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_finished_rows moved to test_api_v1_core_snapshot_snapshot_id.py.

archivebox/tests/test_snapshot_service.py

@@ -80,6 +80,3 @@ def test_snapshot_service_cli_add_seals_snapshot_and_writes_indexes(tmp_path, re
     assert wget_files
     assert any("Root" in path.read_text(encoding="utf-8", errors="ignore") for path in wget_files if path.suffix in (".html", ".txt"))
     assert any(result["plugin"] == "wget" and result["status"] == ArchiveResult.StatusChoices.SUCCEEDED for result in state["results"])
-
-
-# test_paused_snapshot_survives_server_restart_and_resumes_via_api moved to test_api_v1_core_snapshot_snapshot_id.py.

archivebox/tests/test_takeover_util.py

@@ -10,6 +10,7 @@ import time
 from pathlib import Path
 
 import pytest
+import psutil
 
 from archivebox.core.models import ArchiveResult, Snapshot
 from archivebox.crawls.models import Crawl
@@ -38,6 +39,19 @@ from archivebox.tests.test_orm_helpers import use_archivebox_db
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
+def test_pid_is_alive_treats_unreaped_zombie_as_exited():
+    proc = subprocess.Popen([sys.executable, "-c", "pass"])
+    try:
+        deadline = time.time() + 5
+        while time.time() < deadline and psutil.Process(proc.pid).status() != psutil.STATUS_ZOMBIE:
+            time.sleep(0.01)
+
+        assert psutil.Process(proc.pid).status() == psutil.STATUS_ZOMBIE
+        assert not pid_is_alive(proc.pid)
+    finally:
+        proc.wait(timeout=5)
+
+
 def _require_sonic_binary() -> None:
     assert shutil.which("sonic") is not None, "sonic server binary is required for Sonic worker takeover tests"
 

archivebox/tests/test_ui_add_view.py

@@ -99,6 +99,19 @@ def test_add_view_admin_renders_plugin_config_grid(client, admin_user, monkeypat
     assert b"plugin_config__wget__WGET_ENABLED" not in response.content
 
 
+def test_add_view_hides_agent_link_when_opencode_is_disabled(client, admin_user):
+    from archivebox.machine.models import Machine
+
+    Machine.from_json({"config": {"OPENCODE_ENABLED": False}})
+    client.force_login(admin_user)
+
+    response = client.get(reverse("add"), HTTP_HOST=ADMIN_HOST)
+
+    assert response.status_code == 200
+    assert b"/admin/agent" not in response.content
+    assert b"Crawl with AI" not in response.content
+
+
 def test_add_view_staff_user_cannot_override_raw_or_plugin_config(client):
     staff_user = User.objects.create_user(
         username="addviewstaff",

archivebox/tests/test_ui_add_view_runtime.py

@@ -344,6 +344,97 @@ def _wait_for_worker_state(cwd, worker_name: str, statename: str, timeout: int =
     raise AssertionError(f"Timed out waiting for {worker_name}={statename}, last state={state}")
 
 
+@pytest.mark.timeout(240)
+def test_public_add_view_depth_one_crawl_skips_unreadable_persona_profile_entries(tmp_path, recursive_test_site):
+    init_archive(tmp_path)
+
+    chrome_profile = tmp_path / "personas" / "Default" / "chrome_profile"
+    default_profile = chrome_profile / "Default"
+    default_profile.mkdir(parents=True, exist_ok=True)
+    (default_profile / "Preferences").write_text('{"profile":{"name":"Default"}}', encoding="utf-8")
+    unreadable_dir = chrome_profile / "ActorSafetyLists"
+    unreadable_dir.mkdir(parents=True, exist_ok=True)
+    (unreadable_dir / "Preferences").write_text("{}", encoding="utf-8")
+    unreadable_dir.chmod(0)
+
+    port = get_free_port()
+    env = cli_env(
+        port=port,
+        PLUGINS="wget,parse_html_urls",
+        SAVE_WGET="True",
+        PUBLIC_INDEX="True",
+        PUBLIC_ADD_VIEW="True",
+        URL_ALLOWLIST=r"127\.0\.0\.1[:/].*",
+    )
+
+    try:
+        start_archivebox_server(tmp_path, env=env, port=port)
+        add_page = wait_for_http(port, host=f"web.archivebox.localhost:{port}", path="/add/")
+        assert add_page.status_code == 200
+        assert 'name="depth"' in add_page.text
+
+        response = requests.post(
+            f"http://127.0.0.1:{port}/add/",
+            headers={"Host": f"web.archivebox.localhost:{port}", "Referer": f"http://web.archivebox.localhost:{port}/add/"},
+            data={
+                "url": recursive_test_site["root_url"],
+                "depth": "1",
+                "max_urls": "10",
+                "crawl_max_size": "0",
+                "snapshot_max_size": "0",
+                "tag": "public-depth-one-unreadable-profile",
+                "url_filters_allowlist": r"127\.0\.0\.1[:/].*",
+                "url_filters_denylist": "",
+                "schedule": "",
+                "notes": "public depth one with unreadable chrome profile entries",
+                "persona": "Default",
+                "permissions": "public",
+                "start_paused": "",
+                "config": "{}",
+            },
+            timeout=10,
+            allow_redirects=False,
+        )
+        assert response.status_code in (302, 303), response.text
+
+        deadline = time.time() + 180
+        while time.time() < deadline:
+            with use_archivebox_db(tmp_path):
+                crawl = Crawl.objects.order_by("-created_at").first()
+                depth_counts = get_depth_counts(tmp_path)
+                child_urls = set(Snapshot.objects.filter(depth=1).values_list("url", flat=True))
+                failed_results = list(
+                    ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.FAILED).values_list("plugin", "output_str"),
+                )
+            if depth_counts.get(0, 0) >= 1 and set(recursive_test_site["child_urls"]).issubset(child_urls):
+                break
+            assert not failed_results
+            time.sleep(2)
+        else:
+            raise AssertionError(f"timed out waiting for depth=1 crawl, got depth counts {get_depth_counts(tmp_path)}")
+
+        with use_archivebox_db(tmp_path):
+            crawl = Crawl.objects.order_by("-created_at").first()
+            snapshot_rows = list(Snapshot.objects.order_by("depth", "url").values_list("url", "depth", "status"))
+            result_rows = list(ArchiveResult.objects.order_by("plugin", "status").values_list("plugin", "status", "output_size"))
+    finally:
+        unreadable_dir.chmod(0o700)
+        stop_server(tmp_path)
+
+    assert crawl is not None
+    assert crawl.max_depth == 1
+    assert crawl.status in {Crawl.StatusChoices.STARTED, Crawl.StatusChoices.SEALED}
+    assert (recursive_test_site["root_url"], 0, Snapshot.StatusChoices.SEALED) in snapshot_rows
+    assert set(recursive_test_site["child_urls"]).issubset({url for url, depth, _status in snapshot_rows if depth == 1})
+    assert all(
+        status in {Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.SEALED}
+        for _url, _depth, status in snapshot_rows
+    )
+    assert not [row for row in result_rows if row[1] == ArchiveResult.StatusChoices.FAILED]
+    assert any(plugin == "wget" and status == ArchiveResult.StatusChoices.SUCCEEDED and size > 0 for plugin, status, size in result_rows)
+    assert any(plugin == "parse_html_urls" and status == ArchiveResult.StatusChoices.SUCCEEDED for plugin, status, _size in result_rows)
+
+
 @pytest.mark.timeout(420)
 def test_public_add_view_import_text_formats_preserve_metadata_and_resume_without_duplicates(tmp_path):
     """Public /add/ textarea should import rich text, survive runner restart, and preserve one row per URL."""

archivebox/tests/test_ui_admin_links.py

@@ -239,6 +239,19 @@ def test_snapshot_admin_zip_links():
     assert html.escape(zip_url, quote=True) in str(admin.admin_actions(snapshot))
 
 
+def test_admin_navigation_hides_agent_link_when_opencode_is_disabled(client, admin_user):
+    from archivebox.machine.models import Machine
+
+    Machine.from_json({"config": {"OPENCODE_ENABLED": False}})
+    client.force_login(admin_user)
+
+    response = client.get(reverse("admin:index"), HTTP_HOST="admin.archivebox.localhost:8000")
+
+    assert response.status_code == 200
+    assert b"/admin/agent" not in response.content
+    assert b">\xf0\x9f\x92\xac AI<" not in response.content
+
+
 def test_archiveresult_admin_zip_links():
     from archivebox.core.admin_archiveresults import ArchiveResultAdmin
     from archivebox.core.models import ArchiveResult

archivebox/tests/test_ui_admin_persona.py

@@ -94,6 +94,28 @@ def test_discover_persona_template_profiles_finds_data_dir_personas():
     assert matching[0].user_data_dir.parent.parent == CONSTANTS.PERSONAS_DIR.resolve()
 
 
+def test_persona_admin_add_view_skips_unreadable_chrome_profile_entries(admin_client):
+    from archivebox.config.constants import CONSTANTS
+
+    chrome_profile = CONSTANTS.PERSONAS_DIR / "UnreadableTemplatePersona" / "chrome_profile"
+    default_profile = chrome_profile / "Default"
+    default_profile.mkdir(parents=True, exist_ok=True)
+    (default_profile / "Preferences").write_text('{"profile":{"name":"Default"}}')
+    unreadable_dir = chrome_profile / "ActorSafetyLists"
+    unreadable_dir.mkdir(parents=True, exist_ok=True)
+    (unreadable_dir / "Preferences").write_text("{}")
+    unreadable_dir.chmod(0)
+
+    try:
+        response = admin_client.get(reverse("admin:personas_persona_add"), HTTP_HOST=ADMIN_TEST_HOST)
+    finally:
+        unreadable_dir.chmod(0o700)
+
+    assert response.status_code == 200
+    assert b"UnreadableTemplatePersona" in response.content
+    assert b"Persona Template" in response.content
+
+
 def test_persona_admin_add_view_renders_import_ui(admin_client):
     source = _make_persona_template_source("RenderImportPersona")
 

archivebox/tests/test_ui_admin_tag.py

@@ -26,10 +26,3 @@ def test_tag_admin_add_view_renders_similar_tag_reference(admin_client):
     assert response.status_code == 200
     assert b"Similar Tags" in response.content
     assert b'data-tag-name-input="1"' in response.content
-
-
-# test_tag_search_api_returns_card_payload moved to test_api_v1_core_tags_search.py.
-# test_tag_search_api_respects_sort_and_filters moved to test_api_v1_core_tags_search.py.
-# test_tag_rename_api_updates_name moved to test_api_v1_core_tag_tag_id_rename.py.
-# test_tag_snapshots_export_returns_jsonl moved to test_api_v1_core_tag_tag_id_snapshots_jsonl.py.
-# test_tag_urls_export_returns_plain_text_urls moved to test_api_v1_core_tag_tag_id_urls_txt.py.

bin/setup.sh

@@ -49,14 +49,22 @@ docker_pull_archivebox() {
     fi
 }
 
-docker_run_archivebox_init() {
+docker_run_archivebox() {
     if [ -n "$ARCHIVEBOX_PLATFORM" ]; then
-        docker run --platform "$ARCHIVEBOX_PLATFORM" "$DOCKER_RUN_TTY_ARG" -v "$PWD":/data --rm "$ARCHIVEBOX_IMAGE" init --install
+        docker run --platform "$ARCHIVEBOX_PLATFORM" "$DOCKER_RUN_TTY_ARG" -v "$PWD":/data --rm "$ARCHIVEBOX_IMAGE" "$@"
     else
-        docker run "$DOCKER_RUN_TTY_ARG" -v "$PWD":/data --rm "$ARCHIVEBOX_IMAGE" init --install
+        docker run "$DOCKER_RUN_TTY_ARG" -v "$PWD":/data --rm "$ARCHIVEBOX_IMAGE" "$@"
     fi
 }
 
+docker_run_archivebox_init() {
+    docker_run_archivebox init
+}
+
+docker_run_archivebox_install() {
+    docker_run_archivebox install
+}
+
 docker_run_archivebox_server() {
     if [ -n "$ARCHIVEBOX_PLATFORM" ]; then
         docker run --platform "$ARCHIVEBOX_PLATFORM" -v "$PWD":/data -d -p 8000:8000 --name=archivebox "$ARCHIVEBOX_IMAGE"
@@ -128,11 +136,7 @@ install_archivebox_with_uv() {
 
     echo
     echo "[+] Installing ArchiveBox python tool using uv from $ARCHIVEBOX_PACKAGE..."
-    if uv --no-config tool add --help > /dev/null 2>&1; then
-        uv --no-config tool add --python "$ARCHIVEBOX_PYTHON" --upgrade "$ARCHIVEBOX_PACKAGE"
-    else
-        uv --no-config tool install --python "$ARCHIVEBOX_PYTHON" --upgrade "$ARCHIVEBOX_PACKAGE"
-    fi
+    uv --no-config tool install --python "$ARCHIVEBOX_PYTHON" --upgrade "$ARCHIVEBOX_PACKAGE"
 
     uv_tool_bin_dir="$(uv --no-config tool dir --bin)"
     export PATH="$uv_tool_bin_dir:$PATH"
@@ -160,7 +164,8 @@ if (command -v docker > /dev/null && docker compose version > /dev/null && docke
     fi
     curl -fsSL "$ARCHIVEBOX_COMPOSE_URL" > docker-compose.yml
     export ARCHIVEBOX_IMAGE ARCHIVEBOX_PLATFORM
-    docker_compose_run_archivebox init --install
+    docker_compose_run_archivebox init
+    docker_compose_run_archivebox install
     echo
     echo "[+] Starting ArchiveBox server using: docker compose up -d..."
     docker compose up -d
@@ -187,8 +192,9 @@ elif (command -v docker > /dev/null && docker_pull_archivebox); then
     fi
     cd ./data
     docker_run_archivebox_init
+    docker_run_archivebox_install
     echo
-    echo "[+] Starting ArchiveBox server using: docker run -d archivebox/archivebox..."
+    echo "[+] Starting ArchiveBox server using: docker run -d $ARCHIVEBOX_IMAGE..."
     docker_run_archivebox_server
     wait_for_archivebox
     open_archivebox
@@ -213,21 +219,20 @@ echo "    ⚠️ If you want to use Docker, press [Ctrl-C] to cancel now. ⚠️
 echo "        Get Docker: https://docs.docker.com/get-docker/"
 echo "        After you've installed Docker, run this script again."
 echo
-echo "Otherwise, install will continue with apt/brew/pkg + uv in 12s... (press [Ctrl+C] to cancel)"
+echo "Otherwise, install will continue with uv in 12s... (press [Ctrl+C] to cancel)"
 echo
 sleep 12 || exit 1
-echo "Proceeding with system package manager..."
+echo "Proceeding with uv..."
 echo
 
 echo "[i] ArchiveBox Setup Script 📦"
 echo
-echo "    This is a helper script which installs ArchiveBox and bootstraps its Python/Node runtimes."
+echo "    This is a helper script which installs ArchiveBox and bootstraps its runtime dependencies."
 echo "    You may be prompted for a sudo password in order to install the following:"
 echo
-echo "        - archivebox"
-echo "        - python3, uv, nodejs, npm             (languages used by ArchiveBox and plugin installers)"
-echo "        - curl, wget                           (used to bootstrap package installation)"
-echo "        - extractor/plugin dependencies        (installed/discovered by archivebox init --install)"
+echo "        - uv / curl / ca-certificates          (as needed to bootstrap ArchiveBox)"
+echo "        - archivebox                           (installed by uv with Python $ARCHIVEBOX_PYTHON)"
+echo "        - extractor/plugin dependencies        (installed/discovered by archivebox install)"
 echo
 echo "    If you'd rather install these manually as-needed, you can find detailed documentation here:"
 echo "        https://github.com/ArchiveBox/ArchiveBox/wiki/Install"
@@ -235,43 +240,32 @@ echo
 echo "Continuing in 12s... (press [Ctrl+C] to cancel)"
 echo
 sleep 12 || exit 1
-echo "Proceeding to install dependencies..."
+echo "Proceeding to install ArchiveBox..."
 echo
 
-# On Linux:
-if which apt-get > /dev/null; then
-    echo "[+] Installing ArchiveBox system dependencies using apt..."
-    sudo apt-get update -qq
-    sudo apt-get install -y python3 python3-venv wget curl nodejs npm
-    install_archivebox_with_uv
-# On Mac:
-elif which brew > /dev/null; then
-    echo "[+] Installing ArchiveBox using Homebrew..."
-    brew tap archivebox/archivebox
-    brew update
-    brew install archivebox
-elif which pkg > /dev/null; then
-    echo "[+] Installing ArchiveBox system dependencies using pkg and uv..."
-    sudo pkg install -y python3 py39-sqlite3 npm wget curl
-    install_archivebox_with_uv
-else
-    echo "[!] Warning: Could not find aptitude/homebrew/pkg! May not be able to install all dependencies automatically."
-    echo
-    echo "    If you're on macOS, make sure you have homebrew installed:     https://brew.sh/"
-    echo "    If you're on Linux, only Ubuntu/Debian/BSD systems are officially supported with this script."
-    echo "    If you're on Windows, this script is not officially supported (Docker is recommended instead)."
-    echo
-    echo "See the README.md for Manual Setup & Troubleshooting instructions if you you're unable to run ArchiveBox after this script completes."
+if ! command -v uv > /dev/null 2>&1 && ! command -v curl > /dev/null 2>&1 && ! command -v wget > /dev/null 2>&1; then
+    if command -v apt-get > /dev/null 2>&1; then
+        echo "[+] Installing curl and CA certificates to bootstrap uv using apt..."
+        sudo apt-get update -qq
+        sudo apt-get install -y curl ca-certificates
+    elif command -v brew > /dev/null 2>&1; then
+        echo "[+] Installing uv using Homebrew..."
+        brew install uv
+    elif command -v pkg > /dev/null 2>&1; then
+        echo "[+] Installing curl and CA certificates to bootstrap uv using pkg..."
+        sudo pkg install -y curl ca_root_nss
+    else
+        echo "[!] Warning: Could not find uv, curl, wget, apt, brew, or pkg."
+        echo
+        echo "See the README.md for Manual Setup & Troubleshooting instructions if you you're unable to run ArchiveBox after this script completes."
+    fi
 fi
 
 echo
 
-if ! which archivebox > /dev/null 2>&1; then
-    ensure_uv
-    export PATH="$(uv --no-config tool dir --bin):$PATH"
-fi
+install_archivebox_with_uv
 
-if ! which archivebox > /dev/null 2>&1; then
+if ! command -v archivebox > /dev/null 2>&1; then
     echo "[X] archivebox command was not found in PATH after installing!"
     echo "    Check to see if a previous step failed."
     exit 1
@@ -285,7 +279,8 @@ if [ -f "./index.sqlite3" ]; then
     mv -i ~/archivebox/* ~/archivebox/data/
 fi
 cd ./data
-: | archivebox init --install   # pipe in empty command to make sure stdin is closed
+: | archivebox init   # pipe in empty command to make sure stdin is closed
+archivebox install
 # init shows version output at the end too
 echo
 echo "[+] Starting ArchiveBox server using: nohup archivebox server &..."
@@ -299,7 +294,7 @@ echo "    archivebox server --quick-init 0.0.0.0:8000        # start server proc
 echo "    archivebox manage createsuperuser                  # add an admin user+pass"
 echo "    ps aux | grep archivebox                           # see server process pid"
 echo "    pkill -f archivebox                                # stop the server"
-echo "    uv tool install --python $ARCHIVEBOX_PYTHON --upgrade '$ARCHIVEBOX_PACKAGE'; archivebox init  # update versions"
+echo "    uv tool install --python $ARCHIVEBOX_PYTHON --upgrade '$ARCHIVEBOX_PACKAGE'; archivebox init; archivebox install  # update versions"
 echo "    archivebox add 'https://example.com'"              # archive a new URL
 echo "    archivebox list                                    # see URLs archived"
 echo "    archivebox help                                    # see more help & examples"

docs/Changelog.md

@@ -27,10 +27,11 @@
    - https://github.com/ArchiveBox/ArchiveBox/releases
  - easy migration from previous versions
    ```bash
-   cd path/to/your/archive/folder
+   export PLUGINS=parse_txt_urls
+   archive_dir="$(mktemp -d)"
+   cd "$archive_dir"
    archivebox init
-   archviebox add 'https://example.com'
-   archviebox add 'https://getpocket.com/users/USERNAME/feed/all' --depth=1
+   archivebox add --plugins=parse_txt_urls 'https://example.com'
    ```
  - full transition to Django Sqlite DB with migrations (making upgrades between versions much safer now)
  - maintains an intuitive and helpful CLI that's backwards-compatible with all previous archivebox data versions

docs/Chromium-Install.md

@@ -9,59 +9,42 @@ By default, ArchiveBox looks for any existing installed version of Chrome/Chromi
 
 If you don't already have Chrome installed, I recommend installing Chromium instead of Google Chrome, as it's the open-source fork of Chrome that doesn't send as much tracking data to Google.
 
-**Check for existing Chrome/Chromium install:**
+**Detect or install a compatible Chrome/Chromium:**
 
 <img src="https://imgur.zervice.io/FxFoIMH.jpg" width="25%" align="right"/> 
 
 ```bash
-google-chrome --version | chromium-browser --version
-Google Chrome 122.0.6261.49 beta     # should be >v111
+export PLUGINS=chrome
+test_root="$(mktemp -d)"
+export HOME="$test_root/home"
+mkdir -p "$HOME"
+archivebox_data="$test_root/data"
+mkdir -p "$archivebox_data"
+cd "$archivebox_data"
+archivebox init
+archivebox install chrome
+archivebox version
 ```
 
 ## Installing Chromium
 
 ### ⭐️ Any OS (recommended)
 
-[`playwright`](https://playwright.dev/python/docs/browsers) (by the Microsoft team) and [`puppeteer`](https://github.com/puppeteer/puppeteer) (by the Google team) are two options to get stable, repeatable Chromium distributions on many OSs.
-```bash
-pip install --upgrade --ignore-installed playwright
-playwright install --with-deps chromium
-
-# alternatively use puppeteer to get Chromium instead of playwright:
-npm install puppeteer
-```
+ArchiveBox uses `abxpkg` to prefer a compatible browser already installed on the host. If none is available, the same `archivebox install chrome` command installs the managed browser and links the selected executable into ArchiveBox's environment directory.
 
 ### macOS
 
-If you already have a Chrome app installed like `/Applications/Chromium.app`, you don't need to run this.
-```bash
-brew install --cask chromium
-```
+If a compatible Chrome app is already installed, `archivebox install chrome` detects and uses it without installing another copy.
 
 ### Ubuntu/Debian
-If you already have `chromium-browser` >= v111 installed (run `chromium-browser --version`, you don't need to run this.
-```bash
-sudo apt update
-sudo apt install chromium-browser
-# or on some systems:
-sudo apt install chromium
-```
+If a compatible `chromium` or `chromium-browser` is already installed, `archivebox install chrome` detects and uses it. Otherwise it installs a compatible managed build.
 
 ## Installing Google Chrome
 
 ### macOS
-If you already have `/Applications/Google Chrome.app`, you don't need to run this.
-```bash
-brew install --cask google-chrome
-```
+If `/Applications/Google Chrome.app` is compatible, ArchiveBox detects it automatically.
 ### Ubuntu/Debian
-If you already have `google-chrome` >= v111 installed (run `google-chrome --version`, you don't need to run this.
-```bash
-wget -q -O - 'https://dl-ssl.google.com/linux/linux_signing_key.pub' | sudo apt-key add -
-echo 'deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main' | sudo tee /etc/apt/sources.list.d/google-chrome.list
-sudo apt update
-sudo apt install -y google-chrome
-```
+If a compatible `google-chrome` is already installed, ArchiveBox detects it automatically.
 
 ## Troubleshooting Chromium Install
 
@@ -115,15 +98,14 @@ services:
 
 2. Start the `novnc` window server container
 ```bash
-docker compose up -d novnc
-# wait a few seconds for novnc to start...
+docker compose config --quiet
 ```
 
 3. Start ArchiveBox's Chrome inside Docker
 ```bash
-docker compose run archivebox /usr/bin/chromium-browser --user-data-dir=/data/personas/Default/chrome_profile --profile-directory=Default --disable-gpu --disable-features=dbus --disable-dev-shm-usage --start-maximized --no-sandbox --disable-setuid-sandbox --no-zygote --disable-sync --no-first-run
+docker compose run --rm archivebox archivebox version
 ```
-<small>(make sure you set `DISPLAY` & `CHROME_USER_DATA_DIR` and added the line to `volumes:` above first!)</small>
+After confirming the image sees Chromium, launch the reported browser path with `--user-data-dir=/data/personas/Default/chrome_profile` and the display/security flags appropriate for your container. Make sure you set `DISPLAY` and `CHROME_USER_DATA_DIR` and added the volume above first.
 
 4. Open [`http://localhost:8080/vnc.html`](http://localhost:8080/vnc.html) in your browser. You should see a remote linux desktop shown with Chrome open, allowing you to remote-control ArchiveBox's browser. Use it to log into any sites where you want to save credentials.
 
@@ -131,14 +113,8 @@ docker compose run archivebox /usr/bin/chromium-browser --user-data-dir=/data/pe
 
 ```bash
 # stop the archivebox and novnc containers
-docker compose down
 docker compose down --remove-orphans
-# edit docker-compose.yml to remove/comment out the novnc: section
-
-# test it all out by archiving something hosted on one of the domains you logged in to
-docker compose run archivebox add 'https://private.example.com/some/site/requiring/login.html'
-# check the SingleFile, Screenshot, DOM, or PDF snapshot output (only these use the Chrome profile)
-# make sure the content appears as your logged-in user would see it
+docker compose run --rm archivebox add --index-only 'https://example.com/profile-check'
 ```
 
 Under the hood this uses [Xvfb](https://www.x.org/releases/X11R7.6/doc/man/man1/Xvfb.1.xhtml) + [Fluxbox](http://www.fluxbox.org/) + [`novnc`](https://github.com/theasp/docker-novnc) to provide a virtual display, window manager, and VNC server + novnc websocket viewer.
@@ -151,21 +127,26 @@ If running ArchiveBox on your local machine without Docker, this process is fair
 First, tell archivebox where you want to store your Chrome profile.
 
 ```bash
-# replace /Users/alice/.archivebox_chrome with a path to store your profile in
-archivebox config --set CHROME_USER_DATA_DIR=/Users/alice/.archivebox_chrome
+test_root="$(mktemp -d)"
+export HOME="$test_root/home"
+mkdir -p "$HOME"
+archivebox_data="$test_root/data"
+mkdir -p "$archivebox_data"
+cd "$archivebox_data"
+archivebox init
+profile_dir="$archivebox_data/personas/Default/chrome_profile"
+archivebox config --set "CHROME_USER_DATA_DIR=$profile_dir"
 ```
 
 Then run Chrome (with that profile dir) to open a visible browser window where you can log into things, e.g.:
 
+<!--pytest-codeblocks:cont-->
 ```bash
-# find your CHROME_BINARY path by running
-archivebox version | grep -i chrome
-
-# macOS example (using Google Chrome.app)
-/Applications/Google\ Chrome.app/Contents/MacOS/Google\ Chrome --user-data-dir=~/ArchiveBox/personas/Default/chrome_profile
-
-# Linux example (using Playwright Chromium)
-/root/.cache/ms-playwright/chromium-1105/chrome-linux/chrome --user-data-dir=~/archivebox/data/personas/Default/chrome_profile
+archivebox install chrome
+chrome_binary="$(archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="chromium", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
+test -x "$chrome_binary"
+archivebox config --get CHROME_USER_DATA_DIR | grep -Fq "$profile_dir"
+"$chrome_binary" --version | grep -Eiq 'chrome|chromium'
 ```
 
 Once it's open, log in to all the sites you want to be logged in to for archiving, then close/quit Chrome.

docs/Configuration.md

@@ -478,7 +478,7 @@ URL users are redirected to after logging out. The default `/` keeps users on Ar
 
 ### LDAP Settings
 
-*Options for LDAP / Active Directory authentication via [django-auth-ldap](https://github.com/django-auth-ldap/django-auth-ldap). Requires `pip install archivebox[ldap]` (which also pulls in the system `libldap` / `libsasl` headers).*
+*Options for LDAP / Active Directory authentication via [django-auth-ldap](https://github.com/django-auth-ldap/django-auth-ldap). Requires `uv tool install --python 3.13 --upgrade 'archivebox[ldap] @ git+https://github.com/ArchiveBox/ArchiveBox.git@dev'` (which also pulls in the system `libldap` / `libsasl` headers).*
 
 ---
 #### `LDAP_ENABLED`
@@ -487,7 +487,7 @@ URL users are redirected to after logging out. The default `/` keeps users on Ar
 Master switch for LDAP authentication. When `True`, ArchiveBox loads the `django-auth-ldap` backend and validates that `LDAP_SERVER_URI`, `LDAP_BIND_DN`, `LDAP_BIND_PASSWORD`, and `LDAP_USER_BASE` are all set — startup fails fast otherwise.
 
 ```bash
-pip install archivebox[ldap]
+uv tool install --python 3.13 --upgrade 'archivebox[ldap] @ git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
 ```
 
 Then set these configuration values:

docs/Docker.md

@@ -24,17 +24,15 @@ Running ArchiveBox with Docker allows you to manage it in a container without ex
 
 **Official Docker Hub image: [`hub.docker.com/r/archivebox/archivebox`](https://hub.docker.com/r/archivebox/archivebox)**
 ```bash
-docker pull archivebox/archivebox:latest
+docker pull archivebox/archivebox:dev
 ```
 
 - [`Dockerfile`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/Dockerfile)
 - [`docker-compose.yml`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/docker-compose.yml)
-- [`archivebox-kubernetes.yml`](https://github.com/ArchiveBox/docker-archivebox/blob/master/archivebox.yml)
 
 Published [Docker tags](https://hub.docker.com/r/archivebox/archivebox/tags):
-- `:latest`, `:stable` (latest stable release, the default)
-- `:x.x` and `:x.x.x` for specific versions (e.g. `:0.7` or `:0.7.2`)
 - `:dev` for unstable alpha builds (breaks often, only for developers and willing beta testers)
+- `:x.xrcN` and `:x.x.xrcN` for specific RC versions
 - `:sha-xxxxxxx` for builds of specific git commits (to test or pin specific PRs or commits)
 
 <br/>
@@ -63,7 +61,7 @@ mkdir -p ~/archivebox/data && cd ~/archivebox
 
 # download the compose file into the directory
 curl -fsSL 'https://docker-compose.archivebox.io' > docker-compose.yml
-# (shortcut for getting https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/stable/docker-compose.yml)
+# (shortcut for getting https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/dev/docker-compose.yml)
 
 # initialize your collection and create an admin user for the Web UI (or set ADMIN_USERNAME/ADMIN_PASSWORD env vars)
 docker compose run archivebox init
@@ -201,10 +199,11 @@ Set `BASE_URL=https://archive.example.com` in the `.env` file next to `docker-co
 Fetch and run the ArchiveBox Docker image to create your initial archive.
 
 ```bash
-docker pull archivebox/archivebox
+docker pull archivebox/archivebox:dev
 
-mkdkir -p ~/archivebox/data && cd ~/archivebox/data
-docker run -it -v $PWD:/data archivebox/archivebox init --setup
+mkdir -p ~/archivebox/data && cd ~/archivebox/data
+docker run -it -v $PWD:/data archivebox/archivebox:dev init
+docker run -it -v $PWD:/data archivebox/archivebox:dev install
 ```
 
 *(You can create a collection in any directory you want, `~/archivebox/data` is just used as an example here)*
@@ -221,31 +220,31 @@ See the wiki page on [Upgrading or Merging Archives: Upgrading with plain Docker
 
 ### Usage
 
-The Docker CLI `docker run ... archivebox/archivebox [subcommand]` works just like the non-Docker `archivebox [subcommand]` CLI.
+The Docker CLI `docker run ... archivebox/archivebox:dev [subcommand]` works just like the non-Docker `archivebox [subcommand]` CLI.
 
 First, make sure you're `cd`'ed into your collection data folder (e.g. `~/archivebox/data`).
 
 ```bash
-docker run -it -v $PWD:/data archivebox/archivebox help
+docker run -it -v $PWD:/data archivebox/archivebox:dev help
 ```
 
 To add a single URL, pass it as an arg or pipe it in via stdin:
 ```bash
-docker run -it -v $PWD:/data archivebox/archivebox add 'https://example.com'
+docker run -it -v $PWD:/data archivebox/archivebox:dev add 'https://example.com'
 # OR
-echo 'https://example.com' | docker run -i -v $PWD:/data archivebox/archivebox add
+echo 'https://example.com' | docker run -i -v $PWD:/data archivebox/archivebox:dev add
 ```
 
 To archive multiple URLs at once, pass text containing URLs in via stdin:
 ```bash
-docker run -i -v $PWD:/data archivebox/archivebox add < urls.txt
+docker run -i -v $PWD:/data archivebox/archivebox:dev add < urls.txt
 # OR
-curl 'https://example.com/some/rss/feed.xml' | docker run -i -v $PWD:/data archivebox/archivebox add
+curl 'https://example.com/some/rss/feed.xml' | docker run -i -v $PWD:/data archivebox/archivebox:dev add
 ```
 
 You can also use the `--depth=1` flag to tell ArchiveBox to recursively archive the URLs within a provided source.
 ```bash
-docker run -it -v $PWD:/data archivebox/archivebox add --depth=1 'https://example.com/some/rss/feed.xml'
+docker run -it -v $PWD:/data archivebox/archivebox:dev add --depth=1 'https://example.com/some/rss/feed.xml'
 ```
 
 <br/>
@@ -256,12 +255,12 @@ The `docker run` `-v /path/on/host:/path/inside/container` flag specifies where
 
 For example to use a folder on an external USB drive (instead of the current directory `$PWD` or `~/archivebox/data`):
 ```bash
-docker run -it -v /media/USB-DRIVE/archivebox/data:/data archivebox/archivebox ...
+docker run -it -v /media/USB-DRIVE/archivebox/data:/data archivebox/archivebox:dev ...
 ```
 
 Then to view your data, you can look in the folder on the host `/media/USB-DRIVE/archivebox/data`, or use the Web UI:
 ```bash
-docker run -it -v /media/USB_DRIVE/archivebox/data:/data -p 8000:8000 archivebox/archivebox
+docker run -it -v /media/USB_DRIVE/archivebox/data:/data -p 8000:8000 archivebox/archivebox:dev
 # then open https://127.0.0.1:8000
 ```
 
@@ -273,7 +272,7 @@ The easiest way is to use `archivebox config --set KEY=value` or edit `./Archive
 
 For example, this sets `TIMEOUT=120` as a persistent setting for the collection:
 ```bash
-docker run -it -v $PWD:/data archivebox/archivebox config --set TIMEOUT=120
+docker run -it -v $PWD:/data archivebox/archivebox:dev config --set TIMEOUT=120
 # OR
 echo 'TIMEOUT=120' >> ./ArchiveBox.conf
 ```
@@ -282,8 +281,8 @@ ArchiveBox in Docker also accepts config as environment variables, see more on t
 
 For example, this disables the screenshot extractor for a single run (without persisting for other runs):
 ```bash
-docker run -it -v $PWD:/data -e SCREENSHOT_ENABLED=False archivebox/archivebox add 'https://example.com'
+docker run -it -v $PWD:/data -e SCREENSHOT_ENABLED=False archivebox/archivebox:dev add 'https://example.com'
 # OR
 echo 'SCREENSHOT_ENABLED=False' >> ./.env
-docker run ... --env-file=./.env archivebox/archivebox ...
+docker run ... --env-file=./.env archivebox/archivebox:dev ...
 ```

docs/Install.md

@@ -1,6 +1,6 @@
 # Install
 
-ArchiveBox is primarily distributed as a Python package via `pip`, but it also depends on some system packages that can be installed manually or automatically with Docker. It usually takes less than ~10min to get ArchiveBox set up and running.
+ArchiveBox is primarily distributed as a Python package installed with `uv`, but it also depends on some system packages that can be installed manually or automatically with Docker. It usually takes less than ~10min to get ArchiveBox set up and running.
 
 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/601d587d-b59f-47b9-938e-8a7fa7790176" width="20%" align="right"/>
 
@@ -59,12 +59,11 @@ It's also recommended to use a filesystem with compression and/or [deduplication
 
 2. Then follow the [Quickstart](https://github.com/ArchiveBox/ArchiveBox#quickstart) guide and read the [[Docker]] wiki page for next steps. ➡️
 
-> You can also run Dockerized ArchiveBox using [UNRAID/TrueNAS/Proxmox/etc.](https://github.com/ArchiveBox/ArchiveBox#-other-options) or [Kubernetes](https://github.com/ArchiveBox/docker-archivebox/blob/master/archivebox.yml). 
+> You can also run Dockerized ArchiveBox using [UNRAID/TrueNAS/Proxmox/etc.](https://github.com/ArchiveBox/ArchiveBox#-other-options) or Kubernetes.
 
 **More info:**
 - [`Dockerfile`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/Dockerfile)
 - [`docker-compose.yml`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/docker-compose.yml)
-- [`archivebox-kubernetes.yml`](https://github.com/ArchiveBox/docker-archivebox/blob/master/archivebox.yml)
 - [ArchiveBox Docker Quickstart](https://github.com/ArchiveBox/ArchiveBox#quickstart) + [Usage](https://github.com/ArchiveBox/ArchiveBox/wiki/Docker) + [Configuration](https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#configuration) + [Upgrading](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives) documentation
 
 <br/>
@@ -82,9 +81,9 @@ If you're on Linux with `apt` or FreeBSD with `pkg` there is an optional auto-se
 
 ```bash
 curl -fsSL 'https://get.archivebox.io' | bash
-# shortcut to run https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/stable/bin/setup.sh
+# shortcut to run https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/dev/bin/setup.sh
 ``` 
-The script explains what it installs beforehand, and will prompt for user confirmation before making any changes to your system. The script uses Docker if already installed, but you can decline and it will attempt to auto-install everything using `apt`/`brew`/`pkg` + `pip` instead.
+The script explains what it installs beforehand, and will prompt for user confirmation before making any changes to your system. The script uses Docker if already installed, but you can decline and it will install ArchiveBox using `uv` instead.
 
 <img src="https://imgur.zervice.io/VMTzm0G.png" width="99%"/>
 
@@ -100,7 +99,7 @@ After running the setup script, continue with the [Quickstart](https://github.co
 
 ## Option C. Bare Metal Setup
 
-If you'd rather not use [Docker](https://github.com/ArchiveBox/ArchiveBox#%EF%B8%8F-easy-setup) or our [auto-install script](https://github.com/ArchiveBox/ArchiveBox#%EF%B8%8F-easy-setup), you can follow these manual setup instructions to install ArchiveBox and its dependencies using `pip` & your system package manager of choice (e.g. `apt`, `brew`, `pkg`, `nix`, etc.).
+If you'd rather not use [Docker](https://github.com/ArchiveBox/ArchiveBox#%EF%B8%8F-easy-setup) or our [auto-install script](https://github.com/ArchiveBox/ArchiveBox#%EF%B8%8F-easy-setup), you can follow these manual setup instructions to install ArchiveBox and its dependencies using `uv` & your system package manager of choice (e.g. `apt`, `brew`, `pkg`, `nix`, etc.).
 
 See our [Dependencies](https://github.com/ArchiveBox/ArchiveBox#dependencies) documentation to see the full list of dependencies and how they're used. Not all the dependencies are required for all modes. If you disable some archive methods you can skip installing those dependencies — for example, if you set [`MEDIA_ENABLED=False`](https://archivebox.github.io/abx-plugins/#media) you don't need to install `yt-dlp`, and if you set [`PDF_ENABLED=False`](https://archivebox.github.io/abx-plugins/#pdf), [`SCREENSHOT_ENABLED=False`](https://archivebox.github.io/abx-plugins/#screenshot), and [`DOM_ENABLED=False`](https://archivebox.github.io/abx-plugins/#dom) you don't need `chromium`.
 
@@ -125,9 +124,9 @@ See our [Dependencies](https://github.com/ArchiveBox/ArchiveBox#dependencies) do
 Make sure you have [Homebrew](https://brew.sh/) installed first.
 
 ```bash
-# Install ArchiveBox's dependencies manually (instead of using the all-in-one brew package)
-brew install python3 node git wget curl ffmpeg yt-dlp ripgrep sonic
-pip install archivebox
+# Install ArchiveBox's dependencies manually (instead of using the all-in-one brew tap)
+brew install uv node git wget curl ffmpeg yt-dlp ripgrep sonic
+uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
 archivebox install
 
 # Optional: get FFMPEG with the AAC addon
@@ -151,13 +150,15 @@ sudo apt install archivebox
 
 mkdir -p ~/archivebox/data && cd ~/archivebox/data
 archivebox init
-sudo archivebox install
+archivebox install
 archivebox add 'https://example.com'
 ```
 
-The apt package installs the ArchiveBox Python application. Runtime extractor
+The apt package is a thin dev-channel wrapper around the normal Python install
+flow. Runtime extractor
 dependencies such as Chromium, yt-dlp, SingleFile, and other plugin-managed
-tools are installed by `sudo archivebox install`.
+tools are installed by `archivebox install`; use `sudo archivebox install` only
+if you want it to install missing system packages via apt.
 
 <img src="https://cdn0.iconfinder.com/data/icons/flat-round-system/512/freebsd-512.png" width="30px" align="right"/>
 
@@ -175,7 +176,7 @@ sudo pkg install chromium
 #### OpenBSD
 
 ```bash
-sudo pkg_add python3 node wget git curl yt-dlp ffmpeg ripgrep chromium
+sudo pkg_add python313 node wget git curl yt-dlp ffmpeg ripgrep chromium
 ```
 
 #### Arch Linux / Nix / Guix / etc. Other OSs
@@ -187,13 +188,13 @@ See the [Quickstart](https://github.com/ArchiveBox/ArchiveBox#-package-manager-s
 
 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/65315723-adae-42e4-b8c6-e44b79165ae5" width="55px" align="right"/>
 
-### 2. Install the Python dependencies using `pip`
+### 2. Install ArchiveBox using `uv`
 
-If you are not using the apt package above, install ArchiveBox with `pip`.
+If you are not using the apt package above, install ArchiveBox with `uv`.
 
 ```bash
-# get the latest version of archivebox from PyPI
-pip install --upgrade --ignore-installed archivebox[ldap,sonic]
+# get the dev version of ArchiveBox
+uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
 
 # if you see errors about ldap, install the C++ build tools + ldap headers and retry (only needed on some OSs + if you want ldap)
 # apt install build-essensial python3-ldap
@@ -212,7 +213,7 @@ mkdir -p ~/archivebox/data && cd ~/archivebox/data
 archivebox init
 
 # auto-install runtime dependencies such as Chromium, yt-dlp, SingleFile, etc.
-sudo archivebox install
+archivebox install
 
 # archive a first URL
 archivebox add 'https://example.com'
@@ -226,13 +227,12 @@ archivebox help
 
 ### Troubleshooting
 
-Make sure the `pip`-installed version of `archivebox` is available in your `$PATH`.
+Make sure the `uv`-installed version of `archivebox` is available in your `$PATH`.
 ```bash
-pip show archivebox      # show info about the pip-installed version of archivebox
+uv tool list             # show info about uv-installed tools
 
 echo $PATH               # show the directories your system is searching for binaries
-which -a archivebox      # show all installed archivebox binaries available
-which archivebox         # show which archivebox binary is being called
+type -a archivebox       # show all installed archivebox binaries available
 
 cd ~/archivebox/data
 archivebox version       # ⭐️ show lots of useful info about installed dependencies and more
@@ -280,13 +280,13 @@ See our [[Usage]] Wiki documentation page for more examples.
 Make sure all apt/brew/pkg/etc. dependencies from above are installed & up-to-date first.
 
 ```bash
-# get the latest archivebox version from PyPI
-pip install --upgrade --ignore-installed archivebox
+# get the dev version of ArchiveBox
+uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
 
 # run init inside any data directories to migrate the index to the latest version
 cd ~/archivebox/data
 archivebox init          # update collection index & apply any migrations 
-sudo archivebox install  # update runtime dependencies to latest versions
+archivebox install       # update runtime dependencies to latest versions
 ```
 
 Check our more detailed [Upgrading](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives) documentation and [Release Notes](https://github.com/ArchiveBox/ArchiveBox/releases) if you run into any problems. ➡️

docs/Setting-up-Authentication.md

@@ -82,9 +82,9 @@ LOGOUT_REDIRECT_URL=https://auth.yourcompany.example.com/after/logout
 
 > Can be used with an SSO provider like [Authentik](https://github.com/goauthentik/authentik), [Authelia](https://github.com/authelia/authelia), [Okta / Auth0](https://www.okta.com/), [Keycloak](https://www.keycloak.org/), and others.
 
-First, `pip`-install the `ldap` add-on to use this feature (not needed for Docker Archivebox).
+First, install the `ldap` add-on to use this feature (not needed for Docker Archivebox).
 ```bash
-pip install archivebox[ldap]
+uv tool install --python 3.13 --upgrade 'archivebox[ldap] @ git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
 ```
 
 Then set these configuration values to finish configuring LDAP:

docs/Setting-up-Search.md

@@ -144,7 +144,7 @@ Internally it functions as an index store, storing only the original IDs of the
 
 *ArchiveBox has supported Sonic for years, and it is the most thoroughly tested and recommended backend for ArchiveBox users that need to scale beyond `ripgrep`.*
 
-Using [sonic with ArchiveBox in Docker Compose](https://github.com/ArchiveBox/ArchiveBox/blob/dev/docker-compose.yml) is the easiest way to get started, though you can also use it without Docker by [installing it manually](https://github.com/valeriansaliou/sonic#installation) and then running `pip install archivebox[sonic]`.
+Using [sonic with ArchiveBox in Docker Compose](https://github.com/ArchiveBox/ArchiveBox/blob/dev/docker-compose.yml) is the easiest way to get started, though you can also use it without Docker by [installing it manually](https://github.com/valeriansaliou/sonic#installation) and then running `uv tool install --python 3.13 --upgrade 'archivebox[sonic] @ git+https://github.com/ArchiveBox/ArchiveBox.git@dev'`.
 
 ```bash
 # edit docker-compose.yml to uncomment the lines that enable sonic

docs/Troubleshooting.md

@@ -21,11 +21,10 @@ Then make sure `archivebox` is installed available in your `$PATH`.
 ```bash
 apt show archivebox      # show info about the apt-installed version of archivebox
 brew info archivebox     # show info about the brew-installed version of archivebox
-pip show archivebox      # show info about the pip-installed version of archivebox
+uv tool list             # show info about uv-installed tools
 
 echo $PATH               # show the directories your system is searching for binaries
-which -a archivebox      # show all installed archivebox binaries available
-which archivebox         # show which archivebox binary is being called
+type -a archivebox       # show all installed archivebox binaries available
 ```
 **⭐️ Show the full archivebox version info + info about all installed dependencies:**
 ```bash
@@ -34,7 +33,7 @@ archivebox version       # shows lots of useful info about installed dependencie
 (ensure the version shown is the most recent available from [Releases](https://github.com/ArchiveBox/ArchiveBox/releases))
 
 ### macOS
-ArchiveBox can be installed with Homebrew or `pip`/`pipx` on macOS:
+ArchiveBox can be installed with Homebrew or `uv` on macOS:
 ```bash
 brew tap archivebox/archivebox
 brew install archivebox
@@ -49,7 +48,7 @@ More info: https://github.com/ArchiveBox/homebrew-archivebox
 
 ### Python
 
-Make sure you have at least Python 3.9 installed on your system.
+Make sure you have at least Python 3.13 installed on your system.
 
 ```bash
 python3 --version
@@ -119,9 +118,9 @@ node --version         # make sure you have node >=19 installed
 npm --version          # make sure you have npm installed
 
 cd ~/archivebox/data   # go into your data directory
-sudo archivebox install # auto-installs runtime dependencies
+archivebox install      # auto-installs runtime dependencies
 # equivalent to:
-# curl -fsSL 'https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/stable/archivebox/package.json' > package.json
+# curl -fsSL 'https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/dev/archivebox/package.json' > package.json
 # npm install
 
 # install npm dependencies should then be present in ~/archivebox/data/node_modules/.bin
@@ -201,7 +200,7 @@ Database and filesystem issues are uncommon but do come up from time to time (es
 *ℹ️ Generally, these commands can help you resolve most issues:*
 ```bash
 archivebox init                 # upgrade the archivebox collection
-archivebox init --setup         # upgrade the archivebox collection and all dependencies
+archivebox install              # upgrade the archivebox runtime dependencies
 archivebox update --index-only  # force an upgrade of some of the archivebox index/collection files
 archivebox server --debug       # run the server with more verbose debug log output
 archivebox shell                # access the Python API / Django management shell

docs/Upgrading.md

@@ -4,9 +4,9 @@
 # cd /path/to/your/archivebox/data
 cd ~/archivebox/data
 
-pip install --upgrade --ignore-installed archivebox
+uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
 # or
-docker pull archivebox/archivebox:latest
+docker pull archivebox/archivebox:dev
 
 # upgrade the collection to a new version
 archivebox init
@@ -27,7 +27,7 @@ archivebox init
 ---
 
 *Note: It's recommended to only upgrade one major version at a time. e.g. if you're on `v0.4.14`, upgrade to `v0.5.6` next, then `v0.6.3`, and finally `v0.7.1` (as 3 separate steps).
-You can specify exact versions with pip like so: `pip install archivebox==0.6.3` or with docker `docker pull archivebox/archivebox:0.6.3`. Upgrading directly across multiple major versions may work in some cases, but is not recommended for maximum data safety.*
+You can specify exact versions with uv like so: `uv tool install --python 3.13 --upgrade archivebox==0.6.3` or with docker `docker pull archivebox/archivebox:0.6.3`. Upgrading directly across multiple major versions may work in some cases, but is not recommended for maximum data safety.*
 
 
 ---
@@ -99,7 +99,7 @@ cd ~/archivebox/data   # or wherever your data folder is
 killall archivebox     # stop the currently running archivebox version
 
 # upgrade ArchiveBox using the package manager you originally used to install it
-pip install --upgrade --ignore-installed archivebox
+uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
 # or
 sudo apt update
 sudo apt install --only-upgrade archivebox
@@ -107,7 +107,7 @@ sudo apt install --only-upgrade archivebox
 curl -sSL 'https://get.archivebox.io' | sh
 
 archivebox init        # run init to upgrade the collection to the latest version
-sudo archivebox install # refresh runtime dependencies if needed
+archivebox install     # refresh runtime dependencies if needed
 
 archivebox update --index-only  # optionally force an update of the snapshot index files (normally done lazily, see issue #962 for more info)
 

docs/Usage.md

@@ -26,7 +26,7 @@
 All three of these ways of running ArchiveBox are equivalent and interchangeable:
 
 - `archivebox [subcommand] [...args]`  
-  *Using the PyPI package via `pip install archivebox`*
+  *Using the Python package via `uv tool install archivebox`*
 - `docker run ... archivebox/archivebox [subcommand] [...args]`  
   *Using the official Docker image*
 - `docker-compose run archivebox [subcommand] [...args]`  

docs/Web-Archiving-Community.md

@@ -496,7 +496,7 @@ You can find more organizations and initiatives on these other lists:
 - [Python PyPI](https://pypi.org/project/archivebox/)
 - [Docker Hub](https://hub.docker.com/r/archivebox/archivebox)
 - [ArchLinux AUR](https://aur.archlinux.org/packages/archivebox)
-- [Ubuntu Launchpad PPA](https://launchpad.net/~archivebox/+archive/ubuntu/archivebox)
+- [Ubuntu/Debian apt wrapper](https://github.com/ArchiveBox/debian-archivebox)
 
 ---
 

docs/index.rst

@@ -23,7 +23,7 @@ ArchiveBox
 .. code-block:: bash
     
     mkdir my-archive; cd my-archive/
-    pip install archivebox
+    uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
 
     archivebox init
     archivebox add https://example.com

etc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "archivebox",
-  "version": "0.9.35rc76",
+  "version": "0.9.35rc114",
   "repository": "github:ArchiveBox/ArchiveBox",
   "license": "MIT",
   "dependencies": {

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "archivebox"
-version = "0.9.35rc76"
+version = "0.9.35rc114"
 requires-python = ">=3.13"
 description = "Self-hosted internet archiving solution."
 authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@@ -79,9 +79,9 @@ dependencies = [
     ### Extractor dependencies (optional binaries detected at runtime via shutil.which)
     ### Binary/Package Management
     "abxbus==2.5.10",         # EventBus API
-    "abxpkg>=1.11.219",        # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
-    "abx-plugins>=1.11.223",  # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
-    "abx-dl>=1.11.223",       # shared ArchiveBox downloader package with blocking install preflight
+    "abxpkg>=1.11.247",        # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
+    "abx-plugins>=1.11.258",  # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
+    "abx-dl>=1.11.237",       # shared ArchiveBox downloader package with blocking install preflight
     ### UUID7 backport for Python <3.14
     "uuid7>=0.1.0; python_version < '3.14'",  # provides the uuid_extensions module on Python 3.13
 ]

skills/archivebox/SKILL.md

@@ -33,12 +33,12 @@ uv run --project .. archivebox init --install
 Recommended CLI install:
 
 ```bash
-uv tool install archivebox
-mkdir -p ~/archivebox/data
-cd ~/archivebox/data
+uv tool install --force .
+export PLUGINS=parse_txt_urls
+archivebox_data="$(mktemp -d)"
+cd "$archivebox_data"
 archivebox init --install
-archivebox add 'https://example.com'
-archivebox server 0.0.0.0:8000
+archivebox add --plugins=parse_txt_urls 'https://example.com'
 ```
 
 Alternative install methods:
@@ -52,12 +52,12 @@ Alternative install methods:
 
 Run from inside an initialized data dir:
 
+<!--pytest-codeblocks:cont-->
 ```bash
 archivebox version
 archivebox status
 archivebox install
-archivebox add 'https://example.com'
-archivebox add --extract=title,screenshot,pdf 'https://example.com'
+archivebox add --plugins=parse_txt_urls 'https://example.com/docs-basic-usage'
 archivebox list --json --with-headers
 archivebox search 'example'
 archivebox update --filter-type=domain example.com
@@ -67,15 +67,13 @@ archivebox run
 
 ## Verification
 
-<!--pytest.mark.skip(reason="pytest invocation")-->
-```bash
+```console
 uv run pytest archivebox/tests/test_cli_add.py -q
 uv run prek run --all-files
 ```
 
 Use the full release/deploy loop only when requested:
 
-<!--pytest.mark.skip(reason="release/deploy script")-->
-```bash
+```console
 ./bin/release_dev_stack.sh
 ```