release: v0.9.35rc133

Clarify security risks and update information on CSRF/XSS protections.

.github/SECURITY.md

@@ -67,3 +67,6 @@ Non-logged in users should be tightly limited to prevent attacks like SQL/shell
 `< v0.9.x` versions do not include any XSS/CSRF protections at all, as documented in the [readme](https://github.com/ArchiveBox/ArchiveBox/tree/v0.7.4#security-risks-of-viewing-archived-js), [wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#security-concerns), [issue #239](https://github.com/ArchiveBox/ArchiveBox/issues/239), and [existing advisory](https://github.com/ArchiveBox/ArchiveBox/security/advisories/GHSA-cr45-98w9-gwqx). Do not open advisories related to XSS or CSRF for those versions, as they are missing fundamental architectural changes added in `v0.9.0` needed to support per-snapshot replay isolation.
 
 [`SERVER_SECURITY_MODE`](https://github.com/archiveBox/archiveBox/wiki/configuration#server_security_mode) + CSRF/XSS defenses were only added in the new 0.9.0 `dev` work and are planned to land in `main` in mid-2026.
+
+Note archivebox is designed to be able to archive any URL the server can reach, including internal intranet URLs and localhost. Users are expected to configure `URL_ALLOWLIST`/`URL_DENYLIST` accordingly if they need to limit what the server should archive.
+It does a basic check to try and avoid recursively archiving snapshots hosted by the archivebox server itself, but it's not a strict security check, it's just to prevent wasted CPU/disk from recursively archiving snapshots in a browser that has the ArchiveBox Browser Extension turned on.

.github/scripts/clone_abx_repo.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+repo_name="$1"
+target_dir="${2:-$repo_name}"
+
+version="$(
+python3 - "$repo_name" <<'PY'
+import re
+import sys
+from pathlib import Path
+
+repo_name = sys.argv[1]
+lock_text = Path("uv.lock").read_text()
+match = re.search(
+    rf'^\[\[package\]\]\s*\nname = "{re.escape(repo_name)}"\s*\nversion = "([^"]+)"',
+    lock_text,
+    re.MULTILINE,
+)
+if not match:
+    raise SystemExit(f"Could not find {repo_name} in uv.lock")
+print(match.group(1))
+PY
+)"
+
+echo "Cloning ArchiveBox/${repo_name}@v${version} into ${target_dir}"
+git clone --depth=1 --branch "v${version}" "https://github.com/ArchiveBox/${repo_name}.git" "$target_dir"

.github/workflows/docker.yml

@@ -2,12 +2,9 @@ name: Build Docker image
 
 on:
   workflow_dispatch:
-  workflow_call:
   push:
     branches:
-      - "**"
-    tags:
-      - 'v*'
+      - '**'
   # pull_request:
 
 env:
@@ -45,47 +42,6 @@ jobs:
         #   submodules: true
         #   fetch-depth: 1
 
-      - name: Wait for released ArchiveBox deps on PyPI
-        run: |
-          python3 - <<'PY'
-          import json
-          import re
-          import sys
-          import time
-          import tomllib
-          import urllib.request
-
-          watched = {"abxbus", "abxpkg", "abx-plugins", "abx-dl"}
-          deps = tomllib.loads(open("pyproject.toml", "rb").read().decode())["project"]["dependencies"]
-          required = {}
-          for dep in deps:
-              for name in watched:
-                  match = re.match(rf"{re.escape(name)}\s*(==|>=)\s*([^,;\s]+)", dep)
-                  if match:
-                      required[name] = match.group(2)
-
-          deadline = time.monotonic() + 300
-          missing = required.copy()
-          while missing and time.monotonic() < deadline:
-              for name, version in list(missing.items()):
-                  try:
-                      with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/{version}/json", timeout=20):
-                          available = True
-                  except Exception:
-                      available = False
-                  if available:
-                      print(f"{name} {version} is available on PyPI")
-                      missing.pop(name)
-                  else:
-                      print(f"{name} {version} is not available on PyPI yet")
-              if missing:
-                  time.sleep(10)
-
-          if missing:
-              print(f"Missing PyPI releases after wait: {missing}", file=sys.stderr)
-              sys.exit(1)
-          PY
-
       - name: Set up Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@v3
@@ -99,7 +55,7 @@ jobs:
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
 
-      - name: Wait for published abx-dl image
+      - name: Verify published abx-dl image
         id: abx_dl_image
         shell: bash
         run: |
@@ -121,16 +77,6 @@ jobs:
           PY
           )"
           ABX_DL_IMAGE="${ABX_DL_IMAGE:-archivebox/abx-dl:${ABX_DL_VERSION}}"
-          deadline=$((SECONDS + 1800))
-          until docker buildx imagetools inspect "${ABX_DL_IMAGE}" >/tmp/abx-dl-image.json; do
-            if (( SECONDS >= deadline )); then
-              echo "Timed out waiting for published ${ABX_DL_IMAGE}" >&2
-              exit 1
-            fi
-            echo "${ABX_DL_IMAGE} is not published yet; waiting..."
-            sleep 30
-          done
-
           echo "image=${ABX_DL_IMAGE}" >> "$GITHUB_OUTPUT"
           docker buildx imagetools inspect "${ABX_DL_IMAGE}"
 
@@ -187,6 +133,7 @@ jobs:
           labels: ${{ steps.docker_meta.outputs.labels }}
           build-args: |
             ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
+            ARCHIVEBOX_COMMIT_HASH=${{ github.sha }}
           cache-from: type=gha,scope=${{ matrix.cache_scope }}
           cache-to: type=gha,mode=max,scope=${{ matrix.cache_scope }}
           pull: true

.github/workflows/lint.yml

@@ -3,6 +3,7 @@ name: Run linters
 on:
   workflow_dispatch:
   push:
+    branches: ['**']
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -22,13 +23,13 @@ jobs:
           fetch-depth: 1
 
       - name: Clone abxpkg
-        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
+        run: bash .github/scripts/clone_abx_repo.sh abxpkg
 
       - name: Clone abx-plugins
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+        run: bash .github/scripts/clone_abx_repo.sh abx-plugins
 
       - name: Clone abx-dl
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
+        run: bash .github/scripts/clone_abx_repo.sh abx-dl
 
       - name: Set up Python
         uses: actions/setup-python@v5

.github/workflows/pip.yml

@@ -2,12 +2,9 @@ name: Build Pip package
 
 on:
   workflow_dispatch:
-  workflow_call:
   push:
     branches:
       - '**'
-    tags:
-      - 'v*'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -18,11 +15,7 @@ env:
 
 jobs:
   build:
-    permissions:
-      id-token: write
-
     runs-on: ubuntu-24.04
-    environment: pypi
     steps:
       - uses: actions/checkout@v4
 
@@ -44,104 +37,20 @@ jobs:
           packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
           version: 1.0
 
-      - name: Wait for released ArchiveBox deps on PyPI
-        run: |
-          python - <<'PY'
-          import re
-          import sys
-          import time
-          import tomllib
-          import urllib.request
-
-          watched = {"abxbus", "abxpkg", "abx-plugins", "abx-dl"}
-          deps = tomllib.loads(open("pyproject.toml", "rb").read().decode())["project"]["dependencies"]
-          required = {}
-          for dep in deps:
-              for name in watched:
-                  match = re.match(rf"{re.escape(name)}\s*(==|>=)\s*([^,;\s]+)", dep)
-                  if match:
-                      required[name] = match.group(2)
-
-          deadline = time.monotonic() + 300
-          missing = required.copy()
-          while missing and time.monotonic() < deadline:
-              for name, version in list(missing.items()):
-                  try:
-                      with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/{version}/json", timeout=20):
-                          available = True
-                  except Exception:
-                      available = False
-                  if available:
-                      print(f"{name} {version} is available on PyPI")
-                      missing.pop(name)
-                  else:
-                      print(f"{name} {version} is not available on PyPI yet")
-              if missing:
-                  time.sleep(10)
-
-          if missing:
-              print(f"Missing PyPI releases after wait: {missing}", file=sys.stderr)
-              sys.exit(1)
-          PY
-
-      - name: Prepare released dependency sync project
-        run: |
-          python - <<'PY'
-          from pathlib import Path
-          import json
-          import re
-          import urllib.request
-
-          source = Path("pyproject.toml")
-          target_dir = Path(".ci-uv-project")
-          target_dir.mkdir(exist_ok=True)
-          text = source.read_text()
-          text = text.replace(
-              'environments = ["sys_platform == \'darwin\'", "sys_platform == \'linux\'"]',
-              'environments = ["sys_platform == \'linux\'"]',
-          )
-
-          # CI runs immediately after publishing internal abx packages. PyPI's
-          # simple index can lag the version JSON endpoint, so sync against the
-          # exact wheel URLs from the version endpoint while leaving package
-          # metadata in pyproject.toml unchanged for the actual build.
-          for package in ("abxpkg", "abx-plugins", "abx-dl"):
-              match = re.search(rf'"{re.escape(package)}>=(?P<version>[^"]+)"', text)
-              if not match:
-                  continue
-              version = match.group("version")
-              with urllib.request.urlopen(f"https://pypi.org/pypi/{package}/{version}/json", timeout=20) as response:
-                  data = json.load(response)
-              wheel_url = next(url["url"] for url in data["urls"] if url["filename"].endswith(".whl"))
-              text = re.sub(
-                  rf'"{re.escape(package)}>=[^"]+"',
-                  f'"{package} @ {wheel_url}"',
-                  text,
-                  count=1,
-              )
-
-          (target_dir / "pyproject.toml").write_text(text)
-          PY
-
       - name: UV install archivebox dev + run sub-dependencies
         env:
           UV_PROJECT_ENVIRONMENT: ${{ github.workspace }}/.venv
-        run: uv sync --project .ci-uv-project --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
+        run: uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
 
       - name: UV build archivebox and archivebox/pkgs/* packages
         run: |
           uv build --all
 
-      - name: Publish new package wheels and sdists to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
-        # ignore when publish to PyPI fails due to duplicate tag
-        continue-on-error: true
-
       - name: UV install archivebox and archivebox/pkgs/* locally for tests
         env:
           UV_PROJECT_ENVIRONMENT: ${{ github.workspace }}/.venv
         run: |
-          uv sync --project .ci-uv-project --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
+          uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
           uv pip install --python .venv/bin/python --no-deps dist/archivebox-*.whl
 
       - name: Verify built package full install
@@ -151,7 +60,7 @@ jobs:
           set -Eeuo pipefail
           DATA_DIR="$(mktemp -d)"
           cd "$DATA_DIR"
-          uv run --project "$GITHUB_WORKSPACE/.ci-uv-project" --no-sync --no-sources archivebox init
-          timeout 30m uv run --project "$GITHUB_WORKSPACE/.ci-uv-project" --no-sync --no-sources archivebox install
-          uv run --project "$GITHUB_WORKSPACE/.ci-uv-project" --no-sync --no-sources archivebox version
-          uv run --project "$GITHUB_WORKSPACE/.ci-uv-project" --no-sync --no-sources archivebox status
+          uv run --project "$GITHUB_WORKSPACE" --no-sync --no-sources archivebox init
+          timeout 30m uv run --project "$GITHUB_WORKSPACE" --no-sync --no-sources archivebox install
+          uv run --project "$GITHUB_WORKSPACE" --no-sync --no-sources archivebox version
+          uv run --project "$GITHUB_WORKSPACE" --no-sync --no-sources archivebox status

.github/workflows/release-runner.yml

@@ -7,7 +7,6 @@ on:
   workflow_dispatch:
 
 permissions:
-  actions: write
   contents: write
   id-token: write
 
@@ -51,10 +50,3 @@ jobs:
           DEFAULT_BRANCH: dev
           GH_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
         run: ./bin/release.sh
-
-      - name: Dispatch image and tap updates
-        env:
-          GH_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
-        run: |
-          gh workflow run docker.yml --ref dev
-          gh workflow run update-homebrew-tap.yml --ref dev

.github/workflows/release.yml

@@ -1,41 +0,0 @@
-name: Release
-
-# Orchestrates the full release pipeline:
-#   1. Build and publish pip package to PyPI
-#   2. Build Homebrew formula after pip
-#   3. Build Docker images in parallel
-#
-# Individual workflows also run on push for CI (see their own triggers).
-# Manual fallback for the hosted release pipeline. The normal prerelease loop
-# is run by bin/release.sh locally so published GitHub releases do not launch a
-# duplicate release job.
-
-on:
-  workflow_dispatch:
-
-permissions:
-  contents: write
-  packages: write
-  id-token: write
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  pip:
-    name: Publish to PyPI
-    uses: ./.github/workflows/pip.yml
-    secrets: inherit
-
-  homebrew:
-    name: Update Homebrew formula
-    needs: pip
-    uses: ./.github/workflows/homebrew.yml
-    secrets: inherit
-
-  docker:
-    name: Build Docker images
-    needs: pip
-    uses: ./.github/workflows/docker.yml
-    secrets: inherit

.github/workflows/test-parallel.yml

@@ -16,6 +16,7 @@ env:
   PYTHONLEGACYWINDOWSSTDIO: utf-8
   USE_COLOR: False
   UV_NO_SOURCES: "1"
+  ABXPKG_MIN_RELEASE_AGE: "0"
 
 jobs:
   discover-tests:
@@ -67,7 +68,7 @@ jobs:
           echo "$json_array" | jq '.'
 
       - name: Clone abx-plugins
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+        run: bash .github/scripts/clone_abx_repo.sh abx-plugins
 
       - name: Discover plugin tests
         id: set-plugin-matrix
@@ -107,7 +108,6 @@ jobs:
     env:
       PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
       CHROME_HEADLESS: "true"
-      CHROME_SANDBOX: "false"
       PERSONAS_DIR: /tmp/abx-personas
       CHROME_USER_DATA_DIR: /tmp/abx-personas/Default/chrome_profile
 
@@ -124,13 +124,13 @@ jobs:
           fetch-depth: 1
 
       - name: Clone abxpkg
-        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
+        run: bash .github/scripts/clone_abx_repo.sh abxpkg
 
       - name: Clone abx-plugins
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+        run: bash .github/scripts/clone_abx_repo.sh abx-plugins
 
       - name: Clone abx-dl
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
+        run: bash .github/scripts/clone_abx_repo.sh abx-dl
 
       - name: Set up Python ${{ matrix.python }}
         uses: actions/setup-python@v5
@@ -158,7 +158,7 @@ jobs:
 
       - uses: awalsh128/cache-apt-pkgs-action@latest
         with:
-          packages: git ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
+          packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
           version: 1.1
 
       - name: Install dependencies with uv
@@ -166,76 +166,64 @@ jobs:
           uv venv
           uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
 
-      - name: Install Chrome through ArchiveBox
-        if: ${{ matrix.test.needs_chromium }}
+      - name: Resolve core test binaries through abxpkg
         env:
-          MIN_CHROMIUM_MAJOR: "149"
           LIB_DIR: ${{ runner.temp }}/abx-lib
           ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
           ABXPKG_INSTALL_TIMEOUT: "900"
-          ABX_CI_DATA_DIR: ${{ runner.temp }}/archivebox-ci-data
         run: |
           set -euo pipefail
-          mkdir -p "$LIB_DIR" "$ABX_CI_DATA_DIR"
+          mkdir -p "$LIB_DIR"
+          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
+          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
+          uv run --no-sync --no-sources abxpkg env \
+            --install \
+            --lib="$LIB_DIR" \
+            --deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/wget/config.json:required_binaries" \
+            --deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/git/config.json:required_binaries" \
+            --deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/search_backend_ripgrep/config.json:required_binaries" \
+            >> "$GITHUB_ENV"
+
+      - name: Resolve Chrome through abxpkg
+        if: ${{ matrix.test.needs_chromium }}
+        env:
+          LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_INSTALL_TIMEOUT: "900"
+        run: |
+          set -euo pipefail
+          mkdir -p "$LIB_DIR"
           echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
           echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
 
-          uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox install chrome
+          uv run --no-sync --no-sources abxpkg env \
+            --install \
+            --lib="$LIB_DIR" \
+            --deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/chrome/config.json:required_binaries" \
+            >> "$GITHUB_ENV"
 
-          candidate="$(uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="chromium", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
-          if [ ! -x "$candidate" ]; then
-            echo "ArchiveBox did not install an executable Chromium binary: ${candidate:-not found}" >&2
-            exit 1
-          fi
-          version="$("$candidate" --version || true)"
-          major="$(printf '%s\n' "$version" | sed -E 's/.* ([0-9]+)\..*/\1/' | head -1)"
-          case "$major" in
-            ''|*[!0-9]*) major=0 ;;
-          esac
-          if [ "$major" -lt "$MIN_CHROMIUM_MAJOR" ]; then
-            echo "ArchiveBox installed Chromium is too old: ${version:-unknown}" >&2
-            exit 1
-          fi
-          echo "Using ArchiveBox-installed Chromium: $candidate ($version)"
-          echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV"
-
-      - name: Install Sonic through ArchiveBox
+      - name: Resolve Sonic through abxpkg
         if: ${{ matrix.test.needs_sonic }}
         env:
           LIB_DIR: ${{ runner.temp }}/abx-lib
           ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
           ABXPKG_INSTALL_TIMEOUT: "900"
-          ABX_CI_DATA_DIR: ${{ runner.temp }}/archivebox-ci-data
         run: |
           set -euo pipefail
-          mkdir -p "$LIB_DIR" "$ABX_CI_DATA_DIR"
+          mkdir -p "$LIB_DIR"
           echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
           echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
 
-          uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox install search_backend_sonic
-
-          candidate="$(uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="sonic", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
-          if [ ! -x "$candidate" ]; then
-            echo "ArchiveBox did not install an executable Sonic binary: ${candidate:-not found}" >&2
-            exit 1
-          fi
-          version="$("$candidate" --version || true)"
-          echo "Using ArchiveBox-installed Sonic: $candidate (${version:-version unknown})"
-          echo "SONIC_BINARY=$candidate" >> "$GITHUB_ENV"
-          echo "$(dirname "$candidate")" >> "$GITHUB_PATH"
+          uv run --no-sync --no-sources abxpkg env \
+            --install \
+            --lib="$LIB_DIR" \
+            --deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/search_backend_sonic/config.json:required_binaries" \
+            >> "$GITHUB_ENV"
 
       - name: Run test - ${{ matrix.test.name }}
         run: |
           mkdir -p tests/out
-          set +e
-          uv run --no-sync --no-sources pytest -xvs "${{ matrix.test.path }}" --basetemp=tests/out --ignore=archivebox/pkgs
-          status=$?
-          set -e
-          if [ "$status" -eq 5 ]; then
-            echo "No tests collected from ${{ matrix.test.path }}; treating as an empty test module."
-            exit 0
-          fi
-          exit "$status"
+          uv run --no-sync --no-sources pytest -xvs "${{ matrix.test.path }}" --basetemp=tests/out
 
   plugin-tests:
     name: ${{ matrix.plugin.name }}
@@ -244,7 +232,6 @@ jobs:
     env:
       PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
       CHROME_HEADLESS: "true"
-      CHROME_SANDBOX: "false"
       PERSONAS_DIR: /tmp/abx-personas
       CHROME_USER_DATA_DIR: /tmp/abx-personas/Default/chrome_profile
 
@@ -260,13 +247,13 @@ jobs:
           fetch-depth: 1
 
       - name: Clone abxpkg
-        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
+        run: bash .github/scripts/clone_abx_repo.sh abxpkg
 
       - name: Clone abx-plugins
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+        run: bash .github/scripts/clone_abx_repo.sh abx-plugins
 
       - name: Clone abx-dl
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
+        run: bash .github/scripts/clone_abx_repo.sh abx-dl
 
       - name: Set up Python 3.13
         uses: actions/setup-python@v5
@@ -294,7 +281,7 @@ jobs:
 
       - uses: awalsh128/cache-apt-pkgs-action@latest
         with:
-          packages: git ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
+          packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
           version: 1.1
 
       - name: Install dependencies with uv
@@ -302,63 +289,57 @@ jobs:
           uv venv
           uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
 
-      - name: Install Chrome through ArchiveBox
-        if: ${{ matrix.plugin.needs_chromium }}
+      - name: Resolve plugin dependencies through abxpkg
         env:
-          MIN_CHROMIUM_MAJOR: "149"
           LIB_DIR: ${{ runner.temp }}/abx-lib
           ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
           ABXPKG_INSTALL_TIMEOUT: "900"
-          ABX_CI_DATA_DIR: ${{ runner.temp }}/archivebox-ci-data
         run: |
           set -euo pipefail
-          mkdir -p "$LIB_DIR" "$ABX_CI_DATA_DIR"
+          mkdir -p "$LIB_DIR"
+          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
+          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
+          uv run --no-sync --no-sources abxpkg env \
+            --install \
+            --lib="$LIB_DIR" \
+            --deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/${{ matrix.plugin.plugin }}/config.json:required_binaries" \
+            >> "$GITHUB_ENV"
+
+      - name: Resolve Chrome through abxpkg
+        if: ${{ matrix.plugin.needs_chromium }}
+        env:
+          LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_INSTALL_TIMEOUT: "900"
+        run: |
+          set -euo pipefail
+          mkdir -p "$LIB_DIR"
           echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
           echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
 
-          uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox install chrome
+          uv run --no-sync --no-sources abxpkg env \
+            --install \
+            --lib="$LIB_DIR" \
+            --deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/chrome/config.json:required_binaries" \
+            >> "$GITHUB_ENV"
 
-          candidate="$(uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="chromium", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
-          if [ ! -x "$candidate" ]; then
-            echo "ArchiveBox did not install an executable Chromium binary: ${candidate:-not found}" >&2
-            exit 1
-          fi
-          version="$("$candidate" --version || true)"
-          major="$(printf '%s\n' "$version" | sed -E 's/.* ([0-9]+)\..*/\1/' | head -1)"
-          case "$major" in
-            ''|*[!0-9]*) major=0 ;;
-          esac
-          if [ "$major" -lt "$MIN_CHROMIUM_MAJOR" ]; then
-            echo "ArchiveBox installed Chromium is too old: ${version:-unknown}" >&2
-            exit 1
-          fi
-          echo "Using ArchiveBox-installed Chromium: $candidate ($version)"
-          echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV"
-
-      - name: Install Sonic through ArchiveBox
+      - name: Resolve Sonic through abxpkg
         if: ${{ matrix.plugin.needs_sonic }}
         env:
           LIB_DIR: ${{ runner.temp }}/abx-lib
           ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
           ABXPKG_INSTALL_TIMEOUT: "900"
-          ABX_CI_DATA_DIR: ${{ runner.temp }}/archivebox-ci-data
         run: |
           set -euo pipefail
-          mkdir -p "$LIB_DIR" "$ABX_CI_DATA_DIR"
+          mkdir -p "$LIB_DIR"
           echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
           echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
 
-          uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox install search_backend_sonic
-
-          candidate="$(uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="sonic", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
-          if [ ! -x "$candidate" ]; then
-            echo "ArchiveBox did not install an executable Sonic binary: ${candidate:-not found}" >&2
-            exit 1
-          fi
-          version="$("$candidate" --version || true)"
-          echo "Using ArchiveBox-installed Sonic: $candidate (${version:-version unknown})"
-          echo "SONIC_BINARY=$candidate" >> "$GITHUB_ENV"
-          echo "$(dirname "$candidate")" >> "$GITHUB_PATH"
+          uv run --no-sync --no-sources abxpkg env \
+            --install \
+            --lib="$LIB_DIR" \
+            --deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/search_backend_sonic/config.json:required_binaries" \
+            >> "$GITHUB_ENV"
 
       - name: Run plugin tests - ${{ matrix.plugin.name }}
         env:

.github/workflows/test.yml

@@ -1,5 +1,8 @@
 name: Run tests
-on: [push]
+
+on:
+  push:
+    branches: ['**']
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -7,6 +10,7 @@ concurrency:
 
 env:
   DOCKER_IMAGE: archivebox-ci
+  ARCHIVEBOX_IMAGE: archivebox-ci
   PYTHONIOENCODING: utf-8
   PYTHONLEGACYWINDOWSSTDIO: utf-8
   USE_COLOR: False
@@ -20,8 +24,7 @@ jobs:
 
     strategy:
       matrix:
-        os: [ubuntu-22.04]
-        # os: [ubuntu-22.04, macos-latest, windows-latest]
+        os: [ubuntu-24.04]
         python: ["3.13"]
 
     steps:
@@ -31,13 +34,13 @@ jobs:
           fetch-depth: 1
 
       - name: Clone abxpkg
-        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
+        run: bash .github/scripts/clone_abx_repo.sh abxpkg
 
       - name: Clone abx-plugins
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+        run: bash .github/scripts/clone_abx_repo.sh abx-plugins
 
       - name: Clone abx-dl
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
+        run: bash .github/scripts/clone_abx_repo.sh abx-dl
 
       ### Setup Python & JS Languages
       - name: Set up Python ${{ matrix.python }}
@@ -92,14 +95,16 @@ jobs:
           uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox status
 
       - name: Test built package with pytest
-        # TODO: remove this exception for windows once we get tests passing on that platform
-        if: ${{ !contains(matrix.os, 'windows') }}
+        if: github.ref != 'refs/heads/dev'
         run: |
           mkdir -p tests/out
-          uv run --no-sync --no-sources pytest -s archivebox/tests --basetemp=tests/out --ignore=archivebox/pkgs
+          uv run --no-sync --no-sources pytest -s archivebox/tests --basetemp=tests/out
+
+      - name: Run documentation code blocks
+        run: uv run --no-sync --no-sources pytest -vv --tb=long README.md AGENTS.md skills docs
 
       - name: Run plugin tests
-        if: ${{ !contains(matrix.os, 'windows') }}
+        if: github.ref != 'refs/heads/dev'
         env:
           ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
           TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
@@ -116,9 +121,6 @@ jobs:
           submodules: true
           fetch-depth: 1
 
-      # TODO: as of 2020-11 this helper layer broke, upgrade and re-enable this once it's usable again
-      # - uses: satackey/action-docker-layer-caching@v0.0.8
-
       - name: Build image
         run: |
           docker build . -t "$DOCKER_IMAGE"
@@ -148,14 +150,13 @@ jobs:
 
       - name: Start docker-compose stack
         run: |
-          docker-compose run archivebox init
-          docker-compose up -d
-          sleep 5
+          docker compose run --rm archivebox init
+          docker compose up -d --wait
           curl --silent --location 'http://127.0.0.1:8000' | grep 'ArchiveBox'
           curl --silent --location 'http://127.0.0.1:8000/static/admin/js/jquery.init.js' | grep 'window.django'
 
       - name: Check added urls show up in index
         run: |
-          docker-compose run archivebox add 'http://example.com/#test_docker' --index-only
+          docker compose run --rm archivebox add --plugins=parse_txt_urls 'http://example.com/#test_docker'
           curl --silent --location 'http://127.0.0.1:8000' | grep 'http://example.com/#test_docker'
-          docker-compose down || true
+          docker compose down

.github/workflows/update-homebrew-tap.yml

@@ -2,7 +2,7 @@ name: Update downstream packages
 
 on:
   push:
-    branches: [dev]
+    tags: ['v*']
   workflow_dispatch:
 
 permissions:

AGENTS.md

@@ -38,12 +38,12 @@ uv run --project .. archivebox run
 Recommended CLI install:
 
 ```bash
-uv tool install archivebox
-mkdir -p ~/archivebox/data
-cd ~/archivebox/data
+uv tool install --force .
+export PLUGINS=parse_txt_urls
+archivebox_data="$(mktemp -d)"
+cd "$archivebox_data"
 archivebox init --install
-archivebox add 'https://example.com'
-archivebox server 0.0.0.0:8000
+archivebox add --plugins=parse_txt_urls 'https://example.com'
 ```
 
 Alternative install methods:
@@ -55,13 +55,13 @@ Alternative install methods:
 
 ## Basic Usage
 
+<!--pytest-codeblocks:cont-->
 ```bash
 archivebox version
 archivebox help
 archivebox status
 archivebox install
-archivebox add 'https://example.com'
-archivebox add --extract=title,screenshot,pdf 'https://example.com'
+archivebox add --plugins=parse_txt_urls 'https://example.com/docs-basic-usage'
 archivebox list --json --with-headers
 archivebox search 'example'
 archivebox update --filter-type=domain example.com
@@ -73,15 +73,13 @@ archivebox run
 
 Use targeted tests for focused work:
 
-<!--pytest.mark.skip(reason="pytest invocation")-->
 ```bash
-uv run pytest archivebox/tests/test_cli_add.py -q
+uv run pytest archivebox/tests/test_cli_add.py::test_add_help_shows_depth_and_tag_options -q
 uv run prek run --all-files
 ```
 
 Use the full release/deploy loop only when requested:
 
-<!--pytest.mark.skip(reason="release/deploy script")-->
-```bash
+```console
 ./bin/release_dev_stack.sh
 ```

Dockerfile

@@ -22,6 +22,7 @@ ARG TARGETPLATFORM
 ARG TARGETOS
 ARG TARGETARCH
 ARG TARGETVARIANT
+ARG ARCHIVEBOX_COMMIT_HASH=""
 
 ENV TZ=UTC \
     LANGUAGE=en_US:en \
@@ -165,7 +166,9 @@ COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
 RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
     echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
     && COMMIT_HASH="$( \
-        if [[ -f "$CODE_DIR/.git/HEAD" ]]; then \
+        if [[ "$ARCHIVEBOX_COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then \
+            echo "$ARCHIVEBOX_COMMIT_HASH"; \
+        elif [[ -f "$CODE_DIR/.git/HEAD" ]]; then \
             HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"; \
             if [[ "$HEAD_REF" =~ ^[0-9a-fA-F]{40}$ ]]; then \
                 echo "$HEAD_REF"; \
@@ -238,7 +241,8 @@ RUN chmod +x "$CODE_DIR"/bin/*.sh \
     && chmod g+w "$TMP_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"
 
 RUN --mount=type=cache,target=/tmp/abxpkg-cache,sharing=locked,mode=1777 \
-    for forbidden_bin in gcc g++ make; do ! abxpkg load --binproviders=env "$forbidden_bin" >/dev/null 2>&1 || (echo "Unexpected build tool in runtime: $forbidden_bin" >&2 && exit 1); done \
+    chmod 1777 /tmp/abxpkg-cache \
+    && for forbidden_bin in gcc g++ make; do ! abxpkg load --binproviders=env "$forbidden_bin" >/dev/null 2>&1 || (echo "Unexpected build tool in runtime: $forbidden_bin" >&2 && exit 1); done \
     && stat -c "%U:%G %a %n" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
     && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$CONFIG_DIR" \
     && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$ABXPKG_LIB_DIR" \

README.md

@@ -29,7 +29,7 @@ Without active preservation effort, everything on the internet eventually disapp
 *ArchiveBox is an open source tool that lets organizations & individuals archive both public & private web content while retaining control over their data. It can be used to save copies of bookmarks, preserve evidence for legal cases, backup photos from FB/Insta/Flickr or media from YT/Soundcloud/etc., save research papers, and more...*
 <br/>
 
-> ➡️ Get ArchiveBox with `pip install archivebox>=0.9.5rc1` on [Linux](#quickstart)/[macOS](#quickstart), or via **[Docker](#quickstart)** ⭐️ on any OS.  
+> ➡️ Get ArchiveBox with `uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'` on [Linux](#quickstart)/[macOS](#quickstart), or via **[Docker](#quickstart)** ⭐️ on any OS.
 
 *Once installed, you can interact with it through the: [Browser Extension](https://github.com/ArchiveBox/archivebox-browser-extension), [CLI](#usage), [self-hosted web interface](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive), [Python API](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage), or [filesystem](#static-archive-exporting).*
 
@@ -52,7 +52,7 @@ It also detects any content featured *inside* pages & extracts it out into a fol
 - 💾 **Github**/**Gitlab**/etc. links ➡️ `clone of GIT source code`, `README`, `images`, ...
 - ✨ *and more, see [Output Formats](#output-formats) below...*
 
-You can run ArchiveBox as a Docker web app to manage these snapshots, or continue accessing the same collection using the `pip`-installed CLI, Python API, and SQLite3 APIs. 
+You can run ArchiveBox as a Docker web app to manage these snapshots, or continue accessing the same collection using the `uv`-installed CLI, Python API, and SQLite3 APIs.
 All the ways of using it are equivalent, and provide matching features like adding tags, scheduling regular crawls, viewing logs, and more...
 
 <br/>
@@ -68,7 +68,7 @@ The goal is to sleep soundly knowing the part of the internet you care about wil
 <br/>
 
 
-**📦&nbsp; Install ArchiveBox using your preferred method: `docker` / `pip` / `apt` / etc. ([see full Quickstart below](#quickstart)).**
+**📦&nbsp; Install ArchiveBox using your preferred method: `docker` / `uv` / `brew` / `apt` / etc. ([see full Quickstart below](#quickstart)).**
 
 
 <details>
@@ -85,14 +85,14 @@ docker compose run archivebox init
 <br/>
 # Option B: Or use it as a plain Docker container:
 mkdir -p ~/archivebox/data && cd ~/archivebox/data
-docker run -it -v $PWD:/data archivebox/archivebox init
-# docker run -it -v $PWD:/data archivebox/archivebox add 'https://example.com'
-# docker run -it -v $PWD:/data archivebox/archivebox help
-# docker run -it -v $PWD:/data -p 8000:8000 archivebox/archivebox
+docker run -it -v $PWD:/data archivebox/archivebox:dev init
+# docker run -it -v $PWD:/data archivebox/archivebox:dev add 'https://example.com'
+# docker run -it -v $PWD:/data archivebox/archivebox:dev help
+# docker run -it -v $PWD:/data -p 8000:8000 archivebox/archivebox:dev
 <br/>
 <br/>
-# Option C: Or install it with your preferred pkg manager (see Quickstart below for apt, brew, and more)
-pip install 'archivebox>=0.9.5rc1'
+# Option C: Or install it with uv (see Quickstart below for apt, brew, and more)
+uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
 mkdir -p ~/archivebox/data && cd ~/archivebox/data
 archivebox init
 archivebox install
@@ -214,14 +214,14 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <li>Install <a href="https://docs.docker.com/get-docker/">Docker</a> on your system (if not already installed).</li>
 <li>Create a new empty directory and initialize your collection (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
-docker run -v $PWD:/data -it archivebox/archivebox init
+docker run -v $PWD:/data -it archivebox/archivebox:dev init
 </code></pre>
 </li>
 <li>Optional: Start the server then login to the Web UI <a href="http://archivebox.localhost:8000">http://archivebox.localhost:8000</a> ⇢ Admin.
-<pre lang="bash"><code style="white-space: pre-line">docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox
+<pre lang="bash"><code style="white-space: pre-line">docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox:dev
 # completely optional, CLI can always be used without running a server
 # docker run -v $PWD:/data -it [subcommand] [--help]
-docker run -v $PWD:/data -it archivebox/archivebox help
+docker run -v $PWD:/data -it archivebox/archivebox:dev help
 </code></pre>
 <i>For more info, see <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-a-docker--docker-compose-setup-%EF%B8%8F">Install: Docker Compose</a> in the Wiki. ➡️</i>
 </li>
@@ -256,13 +256,13 @@ See <a href="https://docs.sweeting.me/s/against-curl-sh">"Against curl | sh as a
 
 
 <details>
-<summary><b><img src="https://user-images.githubusercontent.com/511499/117447613-ba4c5d80-af0b-11eb-8f89-1d98e31b6a79.png" alt="Pip" height="28px" align="top"/> <code>pip</code></b> (macOS/Linux/BSD)</summary>
+<summary><b><img src="https://user-images.githubusercontent.com/511499/117447613-ba4c5d80-af0b-11eb-8f89-1d98e31b6a79.png" alt="uv" height="28px" align="top"/> <code>uv</code></b> (macOS/Linux/BSD)</summary>
 <br/>
 <ol>
 
-<li>Install <a href="https://realpython.com/installing-python/">Python >= v3.13</a> and <a href="https://nodejs.org/en/download/package-manager/">Node >= v22</a> on your system (if not already installed).</li>
-<li>Install the ArchiveBox package using <code>pip3</code> (or <a href="https://docs.astral.sh/uv/guides/tools/#running-tools"><code>uvx</code></a>).
-<pre lang="bash"><code style="white-space: pre-line">pip3 install --upgrade archivebox
+<li>Install <a href="https://docs.astral.sh/uv/getting-started/installation/">uv</a> on your system (if not already installed).</li>
+<li>Install the ArchiveBox package using <code>uv</code>.
+<pre lang="bash"><code style="white-space: pre-line">uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
 archivebox version
 # install any missing extras shown using apt/brew/pkg/etc. see Wiki for instructions
 #    python@3.13 node curl wget git ripgrep ...
@@ -305,7 +305,7 @@ archivebox version                         # make sure the package is installed
 <li>Create a new empty directory and initialize your collection (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
 archivebox init
-sudo archivebox install
+archivebox install
 archivebox add 'https://example.com'
 </code></pre>
 <br/>
@@ -319,7 +319,7 @@ archivebox help
 </li>
 </ol>
 See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.<br/>
-<sub>See the <a href="https://github.com/ArchiveBox/debian-archivebox"><code>debian-archivebox</code></a> repo for more details about this distribution.</sub>
+<sub>The apt package is a thin dev-channel wrapper around the normal Python install flow. See the <a href="https://github.com/ArchiveBox/debian-archivebox"><code>debian-archivebox</code></a> repo for details, and run <code>sudo archivebox install</code> only if you want it to install missing system packages via apt.</sub>
 <br/><br/>
 </details>
 
@@ -481,6 +481,7 @@ ArchiveBox commands can be run in a terminal [directly on your host](https://git
 set -euo pipefail
 __archivebox_docs_home="$(mktemp -d)"
 export HOME="$__archivebox_docs_home"
+export PLUGINS=parse_txt_urls
 mkdir -p ~/archivebox/data
 cd ~/archivebox/data
 archivebox init
@@ -491,8 +492,7 @@ test -d archive
 rm -rf "$__archivebox_docs_home"
 ```
 -->
-<!--pytest.mark.skip(reason='Mixed host/Docker alternatives include interactive Docker commands')-->
-```bash
+```console
 mkdir -p ~/archivebox/data   # create a new data dir anywhere
 cd ~/archivebox/data         # IMPORTANT: cd into the directory
 
@@ -503,8 +503,8 @@ archivebox help
 # equivalent: docker compose run archivebox [subcommand] [--help]
 docker compose run archivebox help
 
-# equivalent: docker run -it -v $PWD:/data archivebox/archivebox [subcommand] [--help]
-docker run -it -v $PWD:/data archivebox/archivebox help
+# equivalent: docker run -it -v $PWD:/data archivebox/archivebox:dev [subcommand] [--help]
+docker run -it -v $PWD:/data archivebox/archivebox:dev help
 
 # optional: import your browser cookies into a persona for logged-in archiving
 archivebox persona create --import=chrome personal
@@ -525,12 +525,12 @@ archivebox persona create --import=chrome personal
 <summary><img src="https://user-images.githubusercontent.com/511499/117456282-08665e80-af16-11eb-91a1-8102eff54091.png" alt="curl sh automatic setup script" height="22px" align="top"/> <b>CLI Usage Examples: non-Docker</b></summary>
 <br/>
 <pre lang="bash"><code style="white-space: pre-line">
-# make sure you have pip-installed ArchiveBox and it's available in your $PATH first  
+# make sure you have uv-installed ArchiveBox and it's available in your $PATH first
 <br/>
 # archivebox [subcommand] [--help]
 mkdir -p ~/archivebox/data && cd ~/archivebox/data
 archivebox init
-sudo archivebox install
+archivebox install
 archivebox add 'https://example.com'
 archivebox version           # get archivebox version info + check dependencies
 archivebox help              # get list of archivebox subcommands that can be run
@@ -565,13 +565,13 @@ docker compose run archivebox add 'https://example.com'
 <pre lang="bash"><code style="white-space: pre-line">
 # make sure you create and cd into in a new empty directory first  
 <br/>
-# docker run -it -v $PWD:/data archivebox/archivebox [subcommand] [--help]
-docker run -v $PWD:/data -it archivebox/archivebox init
-docker run -v $PWD:/data -it archivebox/archivebox install
-docker run -v $PWD:/data -it archivebox/archivebox version
-docker run -v $PWD:/data -it archivebox/archivebox help
-docker run -v $PWD:/data -it archivebox/archivebox add 'https://example.com'
-# to start webserver: docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox
+# docker run -it -v $PWD:/data archivebox/archivebox:dev [subcommand] [--help]
+docker run -v $PWD:/data -it archivebox/archivebox:dev init
+docker run -v $PWD:/data -it archivebox/archivebox:dev install
+docker run -v $PWD:/data -it archivebox/archivebox:dev version
+docker run -v $PWD:/data -it archivebox/archivebox:dev help
+docker run -v $PWD:/data -it archivebox/archivebox:dev add 'https://example.com'
+# to start webserver: docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox:dev
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage-1">Usage: Docker CLI</a> wiki. ➡️</i>
 </details>
@@ -603,8 +603,8 @@ nano docker-compose.yml                        # setup initial ADMIN_USERNAME &
 docker compose up                              # start the server
 <br/>
 # Or with a Docker container:
-docker run -v $PWD:/data -it archivebox/archivebox archivebox manage createsuperuser
-docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox
+docker run -v $PWD:/data -it archivebox/archivebox:dev archivebox manage createsuperuser
+docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox:dev
 </code></pre>
 
 <sup>Open <a href="http://web.archivebox.localhost:8000"><code>http://web.archivebox.localhost:8000</code></a> for the public UI and <a href="http://admin.archivebox.localhost:8000"><code>http://admin.archivebox.localhost:8000</code></a> for the admin UI ➡️</sup><br/>
@@ -630,15 +630,15 @@ docker compose run archivebox config --set ...
 
 > [!TIP]
 > Whether in Docker or not, ArchiveBox commands work the same way, and can be used to access the same data on-disk.
-> For example, you could run the Web UI in Docker Compose, and run one-off commands with `pip`-installed ArchiveBox.
+> For example, you could run the Web UI in Docker Compose, and run one-off commands with `uv`-installed ArchiveBox.
 
 <details>
 <summary><i>Expand to show comparison...</i></summary><br/>
 
 <pre lang="bash"><code style="white-space: pre-line">
-archivebox add --depth=1 'https://example.com'                     # add a URL with pip-installed archivebox on the host
+archivebox add --depth=1 'https://example.com'                     # add a URL with uv-installed archivebox on the host
 docker compose run archivebox add --depth=1 'https://example.com'                       # or w/ Docker Compose
-docker run -it -v $PWD:/data archivebox/archivebox add --depth=1 'https://example.com'  # or w/ Docker, all equivalent
+docker run -it -v $PWD:/data archivebox/archivebox:dev add --depth=1 'https://example.com'  # or w/ Docker, all equivalent
 </code></pre>
 
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker">Docker</a> wiki. ➡️</i>
@@ -724,14 +724,14 @@ EOF
 <!--pytest-codeblocks:cont-->
 ```bash
 # archivebox add --help
-archivebox add 'https://example.com/some/page'
-archivebox add --depth=1 --plugins=parse_rss_urls "file://$HOME/Downloads/some_feed.xml"
-archivebox add --depth=1 'https://news.ycombinator.com#2020-12-12'
-echo 'http://example.com' | archivebox add
-echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox add
+archivebox add --plugins=parse_txt_urls 'https://example.com/some/page'
+archivebox add --depth=1 --plugins=parse_rss_urls < "$HOME/Downloads/some_feed.xml"
+archivebox add --plugins=parse_txt_urls 'https://example.com/docs-example'
+echo 'http://example.com' | archivebox add --plugins=parse_txt_urls
+echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox add --plugins=parse_txt_urls
 
 # if using Docker, add -i when piping stdin:
-# echo 'https://example.com' | docker run -v $PWD:/data -i archivebox/archivebox add
+# echo 'https://example.com' | docker run -v $PWD:/data -i archivebox/archivebox:dev add
 # if using Docker Compose, add -T when piping stdin / stdout:
 # echo 'https://example.com' | docker compose run -T archivebox add
 ```
@@ -741,10 +741,10 @@ echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox a
 archivebox list --json > snapshots.json
 grep -q 'https://example.com/some/page' snapshots.json
 grep -q 'https://example.com/from-feed' snapshots.json
-grep -q 'https://news.ycombinator.com#2020-12-12' snapshots.json
+grep -q 'https://example.com/docs-example' snapshots.json
 grep -q 'http://example.com' snapshots.json
-test -d archive
-test "$(find archive -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ')" -ge 4
+test -d archive/users/system/crawls
+test "$(grep -c '\"url\"' snapshots.json)" -ge 4
 rm -rf "$__archivebox_docs_home"
 ```
 -->
@@ -882,10 +882,8 @@ You don't need to install every dependency to use ArchiveBox. ArchiveBox will au
   
 If not using Docker, make sure to keep the dependencies up-to-date yourself and check that ArchiveBox isn't reporting any incompatibility with the versions you install.
 
-<pre lang="bash"><code style="white-space: pre-line">#install python3 and archivebox with your system package manager
-# apt/brew/pip/etc install ... (see Quickstart instructions above)
+<pre lang="bash"><code style="white-space: pre-line"># install uv + archivebox first (see Quickstart instructions above)
 <br/>
-which -a archivebox    # see where you have installed archivebox
 archivebox install     # auto install all the extractors and extras
 archivebox --version   # see info and check validity of installed dependencies
 </code></pre>
@@ -1406,8 +1404,7 @@ For low hanging fruit / easy first tickets, see: <a href="https://github.com/Arc
 
 First make sure you have `uv` installed: https://docs.astral.sh/uv/getting-started/installation/
 
-<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
-```bash
+```console
 git clone https://github.com/ArchiveBox/monorepo
 cd monorepo
 ./bin/setup.sh
@@ -1428,8 +1425,7 @@ Repos included in monorepo setup:
 
 #### 2. Option A: Install the Python, JS, and system dependencies directly on your machine
 
-<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
-```bash
+```console
 # Install ArchiveBox runtime dependencies
 mkdir -p data && cd data
 archivebox init
@@ -1444,14 +1440,14 @@ archivebox server 0.0.0.0:8000
 
 #### 2. Option B: Build the docker container and use that for development instead
 
-<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
-```bash
+```console
 # Optional: develop via docker by mounting the code dir into the container
 # if you edit e.g. ./archivebox/core/models.py on the docker host, runserver
 # inside the container will reload and pick up your changes
 ./bin/build_docker.sh dev
 
-docker run -it -v $PWD/data:/data archivebox/archivebox:dev init --install
+docker run -it -v $PWD/data:/data archivebox/archivebox:dev init
+docker run -it -v $PWD/data:/data archivebox/archivebox:dev install
 
 # Run the development server w/ autoreloading (but no bg workers)
 docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server --debug --reload 0.0.0.0:8000
@@ -1474,8 +1470,7 @@ You can also run all these in Docker. For more examples see the GitHub Actions C
 
 <details><summary><i>Click to expand...</i></summary>
 
-<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
-```bash
+```console
 # set up persistent DEBUG=True for all runs
 archivebox config --set DEBUG=True
 
@@ -1499,8 +1494,7 @@ https://stackoverflow.com/questions/1074212/how-can-i-see-the-raw-sql-queries-dj
 
 If you're looking for the latest `dev` Docker image, it's often available pre-built on Docker Hub, simply pull and use `archivebox/archivebox:dev`.
 
-<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
-```bash
+```console
 docker pull archivebox/archivebox:dev
 docker run archivebox/archivebox:dev version
 # verify the BUILD_TIME and COMMIT_HASH in the output are recent
@@ -1510,8 +1504,7 @@ docker run archivebox/archivebox:dev version
 
 You can also build and run any branch yourself from source, for example to build & use `dev` locally:
 
-<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
-```bash
+```console
 # docker-compose.yml:
 services:
     archivebox:
@@ -1535,8 +1528,7 @@ archivebox install
 
 <details><summary><i>Click to expand...</i></summary>
 
-<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
-```bash
+```console
 ./bin/lint.sh
 ./bin/test.sh
 ```
@@ -1549,8 +1541,7 @@ archivebox install
 
 <details><summary><i>Click to expand...</i></summary>
 
-<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
-```bash
+```console
 # generate the database migrations after changes to models.py
 cd archivebox/
 ./manage.py makemigrations
@@ -1614,8 +1605,7 @@ Copy a similar plugin as a template to modify, then open a new PR to add it in t
 <details><summary><i>Click to expand...</i></summary>
 
 (Normally CI takes care of this, but these scripts can be run to do it manually)
-<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
-```bash
+```console
 ./bin/build.sh
 
 # or individually:
@@ -1631,8 +1621,7 @@ Copy a similar plugin as a template to modify, then open a new PR to add it in t
 <details><summary><i>Click to expand...</i></summary>
 
 (Normally CI takes care of this, but these scripts can be run to do it manually)
-<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
-```bash
+```console
 ./bin/release.sh
 
 # or individually:

archivebox/api/v1_cli.py

@@ -54,6 +54,7 @@ class AddCommandSchema(Schema):
     snapshot_max_size: int = 0
     parser: str = "auto"
     plugins: str = ""
+    persona: str = "Default"
     only_new: bool | None = None
     update: bool = False
     overwrite: bool = False
@@ -145,6 +146,7 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
         snapshot_max_size=args.snapshot_max_size,
         index_only=args.index_only,
         plugins=args.plugins,
+        persona=args.persona,
         parser=args.parser,
         bg=True,  # Always run in background for API calls
         created_by_id=request.user.pk,

archivebox/api/v1_core.py

@@ -4,6 +4,7 @@ import math
 import json
 import mimetypes
 import re
+from html import unescape
 from collections import defaultdict
 from pathlib import Path, PurePosixPath
 from uuid import UUID
@@ -832,7 +833,7 @@ def _snapshots_rss_response(
         crawl_user = snapshot.crawl.created_by if snapshot.crawl_id else None
         description = f"Original URL: {snapshot.url}\nArchived snapshot: {archived_url}"
         feed.add_item(
-            title=snapshot.title or snapshot.url,
+            title=unescape(snapshot.title or snapshot.url),
             link=archived_url or web_base_url,
             description=description,
             unique_id=str(snapshot.id),
@@ -1051,7 +1052,15 @@ def delete_snapshot(request: HttpRequest, snapshot_id: str):
     snapshot = get_snapshot(request, snapshot_id, with_archiveresults=False)
     snapshot_id_str = str(snapshot.id)
     crawl_id_str = str(snapshot.crawl.pk)
-    deleted_count, _ = snapshot.delete()
+    snapshot.cancel()
+
+    from archivebox.crawls.locks import crawl_lifecycle_lock
+    from archivebox.services.runner import run_pending_crawls
+
+    with crawl_lifecycle_lock(crawl_id_str):
+        run_pending_crawls(crawl_id=crawl_id_str, daemon=False)
+        snapshot = get_snapshot(request, snapshot_id_str, with_archiveresults=False)
+        deleted_count, _ = snapshot.delete()
     return {
         "success": True,
         "snapshot_id": snapshot_id_str,

archivebox/api/v1_crawls.py

@@ -251,8 +251,16 @@ def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
 def delete_crawl(request: HttpRequest, crawl_id: str):
     crawl = get_crawl_by_ref(crawl_id)
     crawl_id_str = str(crawl.id)
-    snapshot_count = crawl.snapshot_set.count()
-    deleted_count, _ = crawl.delete()
+    crawl.cancel()
+
+    from archivebox.crawls.locks import crawl_lifecycle_lock
+    from archivebox.services.runner import run_pending_crawls
+
+    with crawl_lifecycle_lock(crawl_id_str):
+        run_pending_crawls(crawl_id=crawl_id_str, daemon=False)
+        crawl = get_crawl_by_ref(crawl_id_str)
+        snapshot_count = crawl.snapshot_set.count()
+        deleted_count, _ = crawl.delete()
     return {
         "success": True,
         "crawl_id": crawl_id_str,

archivebox/cli/archivebox_extract.py

@@ -46,7 +46,6 @@ def process_archiveresult_by_id(archiveresult_id: str) -> int:
     through the shared crawl runner with the corresponding plugin selected.
     """
     from rich import print as rprint
-    from django.utils import timezone
     from archivebox.core.models import ArchiveResult
     from archivebox.api.v1_core import _uuid_ref_query
     from archivebox.services.runner import run_crawl
@@ -60,22 +59,9 @@ def process_archiveresult_by_id(archiveresult_id: str) -> int:
     rprint(f"[blue]Extracting {archiveresult.plugin} for {archiveresult.snapshot.url}[/blue]", file=sys.stderr)
 
     try:
-        was_paused = archiveresult.snapshot.is_paused
         archiveresult.reset_for_retry()
         snapshot = archiveresult.snapshot
-        if not was_paused:
-            snapshot.queue_for_extraction()
-        else:
-            # A paused snapshot may still accept explicit maintenance for one
-            # ArchiveResult, but this path must not transition it back to
-            # queued/startable work. Guard: only set retry_at while the row is
-            # still paused — concurrent resume would otherwise see a stale
-            # retry_at marker.
-            snapshot.safe_update(
-                {"retry_at": timezone.now()},
-                refresh=False,
-                extra_filter={"status": snapshot.StatusChoices.PAUSED},
-            )
+        snapshot.queue_for_extraction()
         crawl = snapshot.crawl
         if not crawl.claim_processing_lock(lock_seconds=10):
             rprint(
@@ -84,11 +70,7 @@ def process_archiveresult_by_id(archiveresult_id: str) -> int:
             )
             return 1
 
-        try:
-            run_crawl(str(snapshot.crawl_id), snapshot_ids=[str(snapshot.id)], selected_plugins=[archiveresult.plugin])
-        finally:
-            if was_paused:
-                snapshot.restore_paused_scheduler_marker()
+        run_crawl(str(snapshot.crawl_id), snapshot_ids=[str(snapshot.id)], selected_plugins=[archiveresult.plugin])
         archiveresult.refresh_from_db()
 
         if archiveresult.status == ArchiveResult.StatusChoices.SUCCEEDED:
@@ -290,12 +272,9 @@ def run_plugins(
     queue_at = timezone.now()
     if existing_snapshot_ids:
         if requested_rows:
-            # Targeted ArchiveResult retries use retry_at as the scheduling
-            # signal and keep sealed snapshots sealed so extractors are not
-            # re-run outside the explicitly queued plugin rows. Paused snapshots
-            # also keep status=paused here: `retry_at` only asks the orchestrator
-            # to process the queued plugin rows, and run_due_snapshot restores
-            # retry_at=MAX afterward instead of resuming the snapshot lifecycle.
+            # Search indexing on a sealed Snapshot is the only targeted hook
+            # allowed to bypass the normal lifecycle. Every other requested
+            # plugin requeues its Snapshot through the unified state machine.
             affected_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in rows_to_queue}
             if preserve_queued and queued_rows:
                 queued_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in queued_rows}
@@ -312,14 +291,30 @@ def run_plugins(
                         flat=True,
                     )
                 )
+            requested_plugins_by_id: dict[str, set[str]] = defaultdict(set)
+            for snapshot_id, plugin_name, _hook_name in requested_rows:
+                requested_plugins_by_id[snapshot_id].add(plugin_name)
             for snapshot in Snapshot.objects.filter(id__in=affected_snapshot_ids).only("id", "status", "modified_at"):
                 # Guard the read-time status so we never bump retry_at on a
                 # row that's been re-queued / started by a concurrent runner.
-                snapshot.safe_update(
-                    {"retry_at": queue_at, "modified_at": queue_at},
-                    refresh=False,
-                    extra_filter={"status": snapshot.status},
+                plugin_names = requested_plugins_by_id.get(str(snapshot.id), set())
+                sealed_search_backfill = (
+                    snapshot.status == Snapshot.StatusChoices.SEALED
+                    and plugin_names
+                    and all(plugin_name.startswith("search_backend_") for plugin_name in plugin_names)
                 )
+                if sealed_search_backfill:
+                    snapshot.safe_update(
+                        {"retry_at": queue_at, "modified_at": queue_at},
+                        refresh=False,
+                        extra_filter={"status": snapshot.status},
+                    )
+                else:
+                    snapshot.update_and_requeue(
+                        status=Snapshot.StatusChoices.QUEUED,
+                        retry_at=queue_at,
+                        current_step=0,
+                    )
         else:
             # No plugin rows were requested, so this is a full snapshot retry.
             for snapshot in Snapshot.objects.filter(id__in=existing_snapshot_ids).only("id", "status", "retry_at", "modified_at"):

archivebox/cli/archivebox_persona.py

@@ -137,65 +137,6 @@ def get_edge_user_data_dir() -> Path | None:
     return None
 
 
-def get_browser_binary(browser: str) -> str | None:
-    system = platform.system()
-    home = Path.home()
-    browser = browser.lower()
-
-    if system == "Darwin":
-        candidates = {
-            "chrome": ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"],
-            "chromium": ["/Applications/Chromium.app/Contents/MacOS/Chromium"],
-            "brave": ["/Applications/Brave Browser.app/Contents/MacOS/Brave Browser"],
-            "edge": ["/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge"],
-        }.get(browser, [])
-    elif system == "Linux":
-        candidates = {
-            "chrome": [
-                "/usr/bin/google-chrome",
-                "/usr/bin/google-chrome-stable",
-                "/usr/bin/google-chrome-beta",
-                "/usr/bin/google-chrome-unstable",
-            ],
-            "chromium": ["/usr/bin/chromium", "/usr/bin/chromium-browser"],
-            "brave": ["/usr/bin/brave-browser", "/usr/bin/brave-browser-beta", "/usr/bin/brave-browser-nightly"],
-            "edge": [
-                "/usr/bin/microsoft-edge",
-                "/usr/bin/microsoft-edge-stable",
-                "/usr/bin/microsoft-edge-beta",
-                "/usr/bin/microsoft-edge-dev",
-            ],
-        }.get(browser, [])
-    elif system == "Windows":
-        local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
-        candidates = {
-            "chrome": [
-                str(local_app_data / "Google" / "Chrome" / "Application" / "chrome.exe"),
-                "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe",
-                "C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe",
-            ],
-            "chromium": [str(local_app_data / "Chromium" / "Application" / "chrome.exe")],
-            "brave": [
-                str(local_app_data / "BraveSoftware" / "Brave-Browser" / "Application" / "brave.exe"),
-                "C:\\Program Files\\BraveSoftware\\Brave-Browser\\Application\\brave.exe",
-                "C:\\Program Files (x86)\\BraveSoftware\\Brave-Browser\\Application\\brave.exe",
-            ],
-            "edge": [
-                str(local_app_data / "Microsoft" / "Edge" / "Application" / "msedge.exe"),
-                "C:\\Program Files\\Microsoft\\Edge\\Application\\msedge.exe",
-                "C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe",
-            ],
-        }.get(browser, [])
-    else:
-        candidates = []
-
-    for candidate in candidates:
-        if candidate and Path(candidate).exists():
-            return candidate
-
-    return None
-
-
 BROWSER_PROFILE_FINDERS = {
     "chrome": get_chrome_user_data_dir,
     "chromium": get_chrome_user_data_dir,  # Same locations
@@ -317,10 +258,6 @@ def create_personas(
         if profile is None and (source_profile_dir / "Default").exists():
             profile = "Default"
 
-        browser_binary = get_browser_binary(import_from)
-        if browser_binary:
-            rprint(f"[dim]Using {import_from} binary: {browser_binary}[/dim]", file=sys.stderr)
-
     created_count = 0
     for name in name_list:
         name = name.strip()

archivebox/cli/archivebox_run.py

@@ -348,7 +348,7 @@ def run_runner(
 @click.option("--crawl-id", help="Run the crawl runner for a specific crawl only")
 @click.option("--snapshot-id", help="Run one snapshot through its crawl")
 @click.option("--binary-id", help="Run one queued binary install directly on the bus")
-@click.option("--maintenance-only", is_flag=True, help="Only process due maintenance ticks on sealed/paused snapshots")
+@click.option("--maintenance-only", is_flag=True, help="Only process sealed Snapshot maintenance and search-index backfills")
 @click.option(
     "--maintenance-batch-size",
     type=int,

archivebox/cli/archivebox_update.py

@@ -76,9 +76,14 @@ def reindex_snapshots(
     wait_for_turn=None,
 ) -> dict[str, Any]:
     from archivebox.cli.archivebox_extract import run_plugins
-    from archivebox.core.models import ArchiveResult
+    from archivebox.core.models import ArchiveResult, Snapshot
     from abx_dl.models import discover_plugins
 
+    # Search backfill is the one maintenance hook allowed to execute without
+    # reopening a Snapshot. Restrict that exception to already-sealed rows;
+    # every open lifecycle state remains owned by the normal state machine.
+    snapshots = snapshots.filter(status=Snapshot.StatusChoices.SEALED)
+
     stats: dict[str, Any] = {"processed": 0, "requested": 0, "queued": 0, "skipped_queued": 0, "reindexed": 0, "snapshot_ids": []}
     records: list[dict[str, str]] = []
     plugins_by_name = discover_plugins(runtime="archivebox")
@@ -365,7 +370,7 @@ def update(
                         # Snapshot rows: Snapshot.save() moves archive/<ts> to
                         # the current output_dir and preserves the lifecycle
                         # status. Drain those retry_at ticks before queuing
-                        # search backfill below. Otherwise the sealed/paused
+                        # search backfill below. Otherwise the sealed search
                         # runner branch correctly sees queued ArchiveResult
                         # rows first, runs the targeted plugins, and may leave
                         # the fs_version maintenance tick hidden behind that
@@ -430,9 +435,7 @@ def update(
                                 now = timezone.now()
                                 queued_result_for_snapshot = queued_index_results.filter(snapshot_id=OuterRef("pk"))
                                 snapshots_to_wake = (
-                                    Snapshot.objects.filter(
-                                        status__in=(Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED),
-                                    )
+                                    Snapshot.objects.filter(status=Snapshot.StatusChoices.SEALED)
                                     .annotate(
                                         has_queued_index_result=Exists(queued_result_for_snapshot),
                                     )
@@ -475,8 +478,8 @@ def update(
 
                 if do_run_until_idle and (do_index or not ran_post_migrate_runner):
                     # Search/index backfill intentionally queues targeted
-                    # ArchiveResult rows without reopening sealed/paused
-                    # snapshots. This second runner pass drains those plugin
+                    # ArchiveResult rows without reopening sealed snapshots.
+                    # This second runner pass drains those plugin
                     # rows after filesystem maintenance has had its own turn.
                     # For a normal unfiltered `archivebox update`, keep the
                     # historical final pass broad enough to resume genuinely

archivebox/cli/archivebox_version.py

@@ -4,6 +4,8 @@ __package__ = "archivebox.cli"
 
 import sys
 import os
+import json
+import asyncio
 import platform
 from pathlib import Path
 from collections.abc import Iterable
@@ -82,19 +84,19 @@ def _build_binary_table(rows: list[dict[str, object]]):
     from rich.table import Table
 
     table = Table(title="Binary Dependencies", box=box.SIMPLE_HEAVY, expand=True)
+    table.add_column("Binary", no_wrap=True, max_width=28)
     table.add_column("Plugin", no_wrap=True, max_width=24)
     table.add_column("State", no_wrap=True, width=8)
     table.add_column("Status", justify="center", no_wrap=True, width=6)
-    table.add_column("Binary", no_wrap=True, max_width=28)
     table.add_column("Version", no_wrap=True, width=16)
     table.add_column("Provider", no_wrap=True, width=8)
     table.add_column("Path", overflow="fold", ratio=1)
     for row in rows:
         table.add_row(
+            str(row["binary"]),
             str(row["plugin"]),
             str(row["state"]),
             str(row["status"]),
-            str(row["binary"]),
             str(row["version"]),
             str(row["provider"]),
             row["path"],
@@ -106,10 +108,10 @@ def _build_binary_table(rows: list[dict[str, object]]):
 def _print_binary_row(prnt, row: dict[str, object]) -> None:
     prnt(
         "",
+        str(row["binary"]).ljust(28),
         row["status"],
         str(row["plugin"]).ljust(24),
         str(row["state"]).ljust(8),
-        str(row["binary"]).ljust(28),
         str(row["version"]).ljust(16),
         str(row["provider"]).ljust(8),
         row["path"],
@@ -133,6 +135,23 @@ def _binary_record_matches_runtime(installed, lib_dir: Path) -> bool:
     return True
 
 
+def _binary_row_dedupe_key(
+    *,
+    display_name: str,
+    valid: bool,
+    version: str,
+    provider: str,
+    abspath: str,
+) -> tuple[str, str, str, str]:
+    if not valid:
+        return (display_name, "", "", "")
+    try:
+        resolved_abspath = Path(abspath).expanduser().resolve(strict=False).as_posix()
+    except Exception:
+        resolved_abspath = abspath
+    return (display_name, provider, version, resolved_abspath)
+
+
 @enforce_types
 def version(
     quiet: bool = False,
@@ -211,6 +230,11 @@ def version(
     prnt()
 
     in_data_dir = os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and os.access(CONSTANTS.CONFIG_FILE, os.R_OK)
+    if isinstance(binaries, str):
+        requested_names = {name.strip() for name in binaries.split(",") if name.strip()}
+    else:
+        requested_names = {name for name in (binaries or ()) if name}
+
     if not in_data_dir:
         PANEL_TEXT = "\n".join(
             (
@@ -231,20 +255,35 @@ def version(
                 subtitle="Full version info is only available when inside a collection [light_slate_blue]DATA DIR[/light_slate_blue]",
             ),
         )
+    prnt()
+
+    if not in_data_dir and not requested_names:
+        prnt("[pale_green1][i] Binary Dependencies:[/pale_green1]")
+        prnt("", "[grey53]Dependency checks require an initialized collection DATA_DIR.[/grey53]")
         prnt()
+        prnt("[deep_sky_blue3][i] Code locations:[/deep_sky_blue3]")
+        try:
+            for name, path in get_code_locations().items():
+                if isinstance(name, str) and isinstance(path, dict):
+                    prnt(printable_folder_status(name, path), overflow="ignore", crop=False)
+        except Exception as e:
+            prnt(f"  [red]Error getting code locations: {e}[/red]")
+        prnt()
+        prnt("[red][i] Data locations:[/red] (not in a data directory)")
+        prnt()
+        return []
 
     prnt("[pale_green1][i] Binary Dependencies:[/pale_green1]")
     failures = []
+    seen_failures: set[str] = set()
+    seen_rows: set[tuple[str, str, str, str]] = set()
 
     from archivebox.plugins.discovery import get_enabled_plugins
     from abx_dl.config import get_required_binary_requests
-    from abx_dl.dependencies import load_binary
+    from abx_dl.dependencies import resolve_binary_requests
     from abx_dl.models import discover_plugins, filter_plugins
-
-    if isinstance(binaries, str):
-        requested_names = {name.strip() for name in binaries.split(",") if name.strip()}
-    else:
-        requested_names = {name for name in (binaries or ()) if name}
+    from abx_dl.orchestrator import create_bus
+    from abxpkg.binary_service import BinaryEvent, BinaryService
 
     plugins = discover_plugins(runtime="archivebox")
     enabled_plugins = filter_plugins(plugins, get_enabled_plugins(config=config), include_providers=True)
@@ -270,7 +309,48 @@ def version(
 
         except Exception as e:
             prnt()
-            prnt("", f"[yellow]Warning: Could not query collection binary records, falling back to abxpkg state: {e}[/yellow]")
+            prnt("", f"[yellow]Warning: Could not query collection binary records; resolving through abxpkg: {e}[/yellow]")
+
+    declared_binary_specs: dict[str, dict[str, object]] = {}
+    for plugin_name, plugin in plugins.items():
+        plugin_enabled = plugin_name in enabled_plugin_names
+        logical_records = get_required_binary_requests(
+            plugin,
+            plugin.config.required_binaries,
+            overrides=runtime_config,
+            derived_overrides=derived_config,
+            run_output_dir=CONSTANTS.DATA_DIR,
+        )
+        actual_records = get_required_binary_requests(
+            plugin,
+            plugin.config.required_binaries,
+            overrides=runtime_config,
+            derived_overrides=derived_config,
+            run_output_dir=CONSTANTS.DATA_DIR,
+            logical_names=False,
+        )
+        for logical_record, actual_record in zip(logical_records, actual_records, strict=False):
+            logical_name = str(logical_record["name"])
+            actual_name = str(actual_record["name"])
+            display_name = Path(actual_name).expanduser().name if ("/" in actual_name or actual_name.startswith("~")) else logical_name
+            if not plugin_enabled and not (
+                requested_names and (logical_name in requested_names or actual_name in requested_names or display_name in requested_names)
+            ):
+                continue
+            signature = json.dumps(actual_record, sort_keys=True, default=str)
+            declared_binary_specs.setdefault(signature, actual_record)
+
+    binary_bus = create_bus(name="ArchiveBoxVersionBinaryCheck")
+    BinaryService(binary_bus, auto_install=False, lib_dir=config.ABXPKG_LIB_DIR)
+
+    async def resolve_declared_binaries() -> dict[str, BinaryEvent | None]:
+        try:
+            return await resolve_binary_requests(binary_bus, declared_binary_specs)
+        finally:
+            await binary_bus.wait_until_idle()
+            await binary_bus.destroy(clear=False)
+
+    loaded_binaries = asyncio.run(resolve_declared_binaries())
 
     rows: list[dict[str, object]] = []
     any_rows = False
@@ -279,7 +359,7 @@ def version(
     live_enabled = console.is_terminal
     live_cm = Live(_build_binary_table(rows), console=console, refresh_per_second=8) if live_enabled else None
     if not live_enabled:
-        prnt("", "Status", "Plugin".ljust(24), "State".ljust(8), "Binary".ljust(28), "Version".ljust(16), "Provider".ljust(8), "Path")
+        prnt("", "Binary".ljust(28), "Status", "Plugin".ljust(24), "State".ljust(8), "Version".ljust(16), "Provider".ljust(8), "Path")
 
     def emit_row(row: dict[str, object]) -> None:
         rows.append(row)
@@ -320,20 +400,27 @@ def version(
                 ):
                     continue
 
-                any_rows = True
                 installed = db_binaries.get(logical_name) if db_available else None
                 if _binary_record_matches_runtime(installed, config.ABXPKG_LIB_DIR):
                     abspath = installed.abspath
                     version_str = (installed.version or "unknown")[:15]
                     provider = (installed.binprovider or "env")[:8]
                     valid = True
+                elif not plugin_enabled and not requested_names:
+                    # `archivebox version` is expected to verify the active runtime, not
+                    # cold-load every optional plugin provider. Migration and status
+                    # checks often run with PLUGINS narrowed to a tiny set; resolving
+                    # disabled plugin binaries there can spend most of the command on
+                    # providers the current collection will never execute.
+                    continue
                 else:
-                    loaded = load_binary(actual_record)
-                    abspath = str(loaded.loaded_abspath or "")
-                    version_str = str(loaded.loaded_version or "unknown")[:15]
-                    provider = (loaded.loaded_binprovider.name if loaded.loaded_binprovider else "env")[:8]
-                    valid = loaded.is_valid
+                    loaded = loaded_binaries[json.dumps(actual_record, sort_keys=True, default=str)]
+                    abspath = loaded.abspath if loaded is not None else ""
+                    version_str = str(loaded.version or "unknown")[:15] if loaded is not None else "unknown"
+                    provider = str(loaded.binprovider or "env")[:8] if loaded is not None else "env"
+                    valid = loaded is not None
 
+                any_rows = True
                 if valid:
                     display_path = (
                         _format_binary_abspath(
@@ -352,8 +439,20 @@ def version(
                 else:
                     rendered_path = "[grey53]not installed[/grey53]"
                     status = "[red]X[/red]" if plugin_enabled else "[grey53]-[/grey53]"
-                    if plugin_enabled:
+                    if plugin_enabled and display_name not in seen_failures:
                         failures.append(display_name)
+                        seen_failures.add(display_name)
+
+                row_key = _binary_row_dedupe_key(
+                    display_name=display_name,
+                    valid=valid,
+                    version=version_str if valid else "-",
+                    provider=provider if valid else "-",
+                    abspath=abspath,
+                )
+                if row_key in seen_rows:
+                    continue
+                seen_rows.add(row_key)
 
                 emit_row(
                     {
@@ -367,6 +466,50 @@ def version(
                         "style": "" if plugin_enabled else "dim",
                     },
                 )
+
+        if db_available:
+            for binary_name, installed in db_binaries.items():
+                if requested_names and binary_name not in requested_names:
+                    continue
+                abspath = installed.abspath
+                version_str = (installed.version or "unknown")[:15]
+                provider = (installed.binprovider or "env")[:8]
+                row_key = _binary_row_dedupe_key(
+                    display_name=binary_name,
+                    valid=True,
+                    version=version_str,
+                    provider=provider,
+                    abspath=abspath,
+                )
+                if row_key in seen_rows:
+                    continue
+                seen_rows.add(row_key)
+
+                display_path = (
+                    _format_binary_abspath(
+                        abspath,
+                        pwd=Path.cwd(),
+                        lib_dir=config.ABXPKG_LIB_DIR,
+                        personas_dir=CONSTANTS.PERSONAS_DIR,
+                        home=Path.home(),
+                    )
+                    if compact_paths
+                    else abspath
+                )
+                emit_row(
+                    {
+                        "plugin": "(database)",
+                        "state": "detected",
+                        "status": "[green]√[/green]",
+                        "binary": binary_name,
+                        "version": version_str,
+                        "provider": provider,
+                        "path": _render_binary_abspath(display_path) if compact_paths else display_path,
+                        "style": "dim",
+                    },
+                )
+                any_rows = True
+                any_available = True
     finally:
         if live_cm is not None:
             live_cm.stop()
@@ -433,9 +576,7 @@ def version(
 )
 @docstring(version.__doc__)
 def main(**kwargs):
-    failures = version(**kwargs)
-    if failures:
-        raise SystemExit(1)
+    version(**kwargs)
 
 
 if __name__ == "__main__":

archivebox/config/common.py

@@ -507,6 +507,32 @@ def _plugin_user_config(config: Mapping[str, object]) -> dict[str, str]:
     return {key: _plugin_user_config_value(value) for key, value in config.items()}
 
 
+def _plugin_input_config(config: Mapping[str, object]) -> dict[str, object]:
+    """Return raw config inputs understood by core or plugin schemas."""
+    input_names = _archivebox_config_input_names()
+    return {str(key): value for key, value in config.items() if str(key) in input_names}
+
+
+def _explicit_plugin_enabled_keys(config: Mapping[str, object]) -> set[str]:
+    """Return canonical ``*_ENABLED`` keys explicitly provided by the user.
+
+    ``PLUGINS`` is a selector, not an override for concrete plugin enable flags.
+    If a user passes ``PLUGINS=wget`` and ``SAVE_WGET=False`` together, the
+    alias-resolved ``WGET_ENABLED=False`` must survive the model validator that
+    expands ``PLUGINS`` into concrete booleans.
+    """
+    provided = {str(key) for key in config}
+    explicit: set[str] = set()
+    for key, prop in _plugin_config_properties(PLUGIN_CONFIG_SCHEMAS).items():
+        if not key.endswith("_ENABLED") or not isinstance(prop, Mapping):
+            continue
+        aliases = prop.get("x-aliases") if isinstance(prop, Mapping) else None
+        input_names = {key, *(str(alias) for alias in aliases if str(alias).strip())} if isinstance(aliases, list) else {key}
+        if provided & input_names:
+            explicit.add(key)
+    return explicit
+
+
 def _discover_plugin_config_schemas() -> PluginSchemaDocuments:
     from archivebox.plugins.discovery import discover_plugin_configs
 
@@ -664,7 +690,7 @@ class ArchiveBoxBaseConfig(
     def _scoped_config(self, *, include_execution: bool) -> dict[str, Any]:
         keys = type(self)._crawl_runtime_keys() if include_execution else type(self)._crawl_frozen_keys()
         payload = self.model_dump(mode="json")
-        return {key: payload[key] for key in keys if payload.get(key) is not None}
+        return {key: payload[key] for key in keys if payload.get(key, None) is not None}
 
     def for_crawl(self) -> dict[str, Any]:
         """Config scoped to crawl execution, without runtime object overlays."""
@@ -752,15 +778,35 @@ class ArchiveBoxBaseConfig(
 
     @model_validator(mode="after")
     def derive_plugin_enabled_config(self):
+        self._derive_plugin_enabled_config(respect_current_enabled=False)
+        return self
+
+    def _derive_plugin_enabled_config(self, *, respect_current_enabled: bool) -> None:
         plugin_names = _normalize_plugins_config_value(self.PLUGINS)
-        selected_plugins = _plugins_with_required_plugins(plugin_names) if plugin_names else set()
+        enabled_config_keys = _plugin_enabled_config_keys()
+        disabled_plugins = []
+        if respect_current_enabled:
+            disabled_plugins = [plugin_name for plugin_name, enabled_key in enabled_config_keys.items() if not getattr(self, enabled_key)]
+        selected_plugin_roots = plugin_names
+        if selected_plugin_roots:
+            from abx_dl.models import discover_plugins, filter_plugins
+
+            selected_plugins = set(
+                filter_plugins(
+                    discover_plugins(runtime="archivebox"),
+                    sorted(selected_plugin_roots),
+                    include_providers=True,
+                    disabled_names=disabled_plugins,
+                ),
+            )
+        else:
+            selected_plugins = set()
         search_backend = self.SEARCH_BACKEND_ENGINE.strip().lower()
         if search_backend:
             selected_plugins.add(f"search_backend_{search_backend}")
-        for plugin_name, enabled_key in _plugin_enabled_config_keys().items():
+        for plugin_name, enabled_key in enabled_config_keys.items():
             if plugin_names or plugin_name in selected_plugins:
                 setattr(self, enabled_key, plugin_name in selected_plugins)
-        return self
 
 
 def _build_archivebox_config_model(plugin_schemas: PluginSchemaDocuments) -> type[ArchiveBoxBaseConfig]:
@@ -817,23 +863,6 @@ def _plugin_enabled_config_keys() -> dict[str, str]:
     return enabled_keys
 
 
-def _plugins_with_required_plugins(plugin_names: set[str]) -> set[str]:
-    selected = set(plugin_names)
-    pending = list(selected)
-    while pending:
-        plugin_name = pending.pop()
-        schema = PLUGIN_CONFIG_SCHEMAS.get(plugin_name, {})
-        required_plugins = schema.get("required_plugins") if isinstance(schema, dict) else None
-        if not isinstance(required_plugins, list):
-            continue
-        for required_plugin in required_plugins:
-            required_plugin_name = str(required_plugin).strip().lower()
-            if required_plugin_name and required_plugin_name not in selected:
-                selected.add(required_plugin_name)
-                pending.append(required_plugin_name)
-    return selected
-
-
 def get_live_config_url(key: str) -> str:
     return f"{LIVE_CONFIG_BASE_URL}{quote(key)}/"
 
@@ -1007,9 +1036,11 @@ def get_config(
     config_data: ConfigPayload = dict(defaults or {})
     config_data["PERSONAS_DIR"] = str(CONSTANTS.PERSONAS_DIR)
     base_config_payload: ConfigPayload = {}
+    base_config_model = ArchiveBoxConfig()
+
     if crawl_config_base:
         config_data.update(
-            normalize_runtime_config(ArchiveBoxConfig().model_dump(mode="json"), exclude_runtime_derived=True, json_safe=False),
+            normalize_runtime_config(base_config_model.model_dump(mode="json"), exclude_runtime_derived=True, json_safe=False),
         )
         config_data.update(normalize_runtime_config(dict(crawl.config or {}), exclude_crawl_execution=True, json_safe=False))
     elif base_config is not None:
@@ -1020,7 +1051,7 @@ def get_config(
         config_data.update(normalize_runtime_config(base_config_payload, exclude_runtime_derived=True, json_safe=False))
     else:
         config_data.update(
-            normalize_runtime_config(ArchiveBoxConfig().model_dump(mode="json"), exclude_runtime_derived=True, json_safe=False),
+            normalize_runtime_config(base_config_model.model_dump(mode="json"), exclude_runtime_derived=True, json_safe=False),
         )
         legacy_permissions = permissions_from_legacy_public_flags({**BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE), **os.environ})
         if legacy_permissions:
@@ -1072,22 +1103,29 @@ def get_config(
     archivebox_scope_overrides = {key: value for key, value in scope_overrides.items() if key in _archivebox_config_input_names()}
     config_data.update(archivebox_scope_overrides)
 
+    explicit_plugin_enabled_keys: set[str] = set()
     if resolve_plugins:
         plugin_schemas = {plugin_name: schema for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items() if isinstance(schema, dict)}
         plugin_global_config = {key: str(value) if isinstance(value, Path) else value for key, value in config_data.items()}
+        crawl_selected_plugins = crawl_config_base and bool(_normalize_plugins_config_value(dict(crawl.config or {}).get("PLUGINS")))
+        # A frozen crawl-level PLUGINS selector is the exact extractor set for
+        # this crawl. Env/file *_ENABLED values are ambient defaults and must
+        # not be re-applied after validation, or a later process env can change
+        # which hooks an already-queued crawl runs.
+        if not crawl_selected_plugins:
+            explicit_plugin_enabled_keys.update(_explicit_plugin_enabled_keys(os.environ))
+        explicit_plugin_enabled_keys.update(_explicit_plugin_enabled_keys(scope_overrides))
         plugin_user_config = _plugin_user_config(
             {
-                **normalize_runtime_config(config_data, only_crawl_execution=True, json_safe=False),
+                **_plugin_input_config(os.environ),
                 **scope_overrides,
             },
         )
         if not crawl_config_base:
+            file_config = BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE)
+            explicit_plugin_enabled_keys.update(_explicit_plugin_enabled_keys(file_config))
             plugin_user_config = {
-                **normalize_runtime_config(
-                    BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE),
-                    exclude_runtime_derived=True,
-                    json_safe=False,
-                ),
+                **_plugin_user_config(_plugin_input_config(file_config)),
                 **plugin_user_config,
             }
         plugin_sections = resolve_plugin_configs(
@@ -1112,6 +1150,13 @@ def get_config(
         if crawl_config_base:
             config_data.update(normalize_runtime_config(dict(crawl.config or {}), exclude_crawl_execution=True, json_safe=False))
         config_data.update(archivebox_scope_overrides)
+        if crawl_selected_plugins:
+            # Crawl PLUGINS is a frozen selector. Drop ambient enable flags from
+            # env/file/defaults before model validation so the selected plugin
+            # set is derived from the crawl, not from the worker process that
+            # happens to resume it.
+            for enabled_key in _plugin_enabled_config_keys().values():
+                config_data.pop(enabled_key, None)
 
     # Decode JSON-encoded complex values (dict/list fields) that came from
     # string-only sources before validation. ``IniConfigSettingsSource`` does
@@ -1137,6 +1182,11 @@ def get_config(
             )
 
     config = ArchiveBoxConfig.model_validate(config_data)
+    for key in explicit_plugin_enabled_keys:
+        if key in config_data:
+            setattr(config, key, config_data[key])
+    if config.PLUGINS:
+        config._derive_plugin_enabled_config(respect_current_enabled=True)
     if redact_sensitive:
         for key in type(config).model_fields:
             value = config[key]

archivebox/config/version.py

@@ -50,9 +50,9 @@ def get_COMMIT_HASH() -> str | None:
     if IN_DOCKER:
         try:
             version_text = Path("/VERSION.txt").read_text()
-            match = re.search(r"^COMMIT_HASH=([0-9a-fA-F]{40})$", version_text, re.MULTILINE)
-            if match:
-                return match.group(1)
+            matches = re.findall(r"^COMMIT_HASH=([0-9a-fA-F]{40})$", version_text, re.MULTILINE)
+            if matches:
+                return matches[-1]
         except Exception:
             pass
 

archivebox/core/admin_snapshots.py

@@ -260,13 +260,15 @@ class SnapshotChangeList(SearchResultsChangeList):
         rows = (
             ArchiveResult.objects.filter(snapshot_id__in=snapshot_ids, status=ArchiveResult.StatusChoices.SUCCEEDED, output_size__gt=0)
             .order_by("snapshot_id", "plugin")
-            .values_list("snapshot_id", "plugin", "status", "output_size")
+            .values_list("snapshot_id", "plugin", "status", "output_size", "output_files")
         )
-        for snapshot_id, plugin, status, output_size in rows.iterator(chunk_size=1000):
+        for snapshot_id, plugin, status, output_size, output_files in rows.iterator(chunk_size=1000):
             if plugin in seen_plugins[snapshot_id]:
                 continue
             seen_plugins[snapshot_id].add(plugin)
-            results_by_snapshot[snapshot_id].append(SimpleNamespace(plugin=plugin, status=status, output_size=output_size))
+            results_by_snapshot[snapshot_id].append(
+                SimpleNamespace(plugin=plugin, status=status, output_size=output_size, output_files=output_files),
+            )
 
         for obj in self.result_list:
             obj.__dict__["_admin_archiveresults"] = results_by_snapshot[obj.pk]

archivebox/core/models.py

@@ -695,12 +695,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
     def restore_paused_scheduler_marker(self) -> None:
         """
-        Keep explicit maintenance from accidentally resuming paused snapshots.
-
-        Targeted jobs such as `archivebox update --index-only` may bump
-        retry_at so the orchestrator can run only queued search ArchiveResult
-        rows. After that maintenance pass, the lifecycle must remain PAUSED and
-        retry_at must go back to MAX until a real resume transition happens.
+        Restore the indefinite scheduler marker owned by the PAUSED lifecycle.
         """
         type(self).objects.filter(
             pk=self.pk,
@@ -2639,10 +2634,16 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             return
 
         # Delete ArchiveResults that produced no output files
-        empty_ars = self.archiveresult_set.filter(
-            output_files={},  # No output files
-        ).filter(
-            status__in=ArchiveResult.FINAL_STATES,  # Only delete finished ones
+        empty_ars = (
+            self.archiveresult_set.filter(
+                output_files={},  # No output files
+            )
+            .filter(
+                status__in=ArchiveResult.FINAL_STATES,  # Only delete finished ones
+            )
+            .exclude(
+                status=ArchiveResult.StatusChoices.FAILED,
+            )
         )
 
         if empty_ars.exists():

archivebox/core/views.py

@@ -93,7 +93,6 @@ from archivebox.core.routes_util import (
 from archivebox.core.forms import AddLinkForm
 from archivebox.plugins.forms import get_plugin_config_binary_urls
 from archivebox.crawls.models import Crawl
-from archivebox.plugins.discovery import discover_plugin_configs
 from archivebox.plugins.views import get_config_definition_link
 from archivebox.progressmonitor.views import live_progress_view, progress_endpoint
 
@@ -1424,7 +1423,6 @@ class AddView(UserPassesTestMixin, FormView):
         request_config = get_request_config(self.request, resolve_plugins=True)
         required_search_plugin = f"search_backend_{request_config.SEARCH_BACKEND_ENGINE}".strip()
         can_override_crawl_config = self._can_override_crawl_config()
-        plugin_configs = discover_plugin_configs() if can_override_crawl_config else {}
         public_persona_config_keys = {
             "CRAWL_MAX_CONCURRENT_SNAPSHOTS",
             "DELETE_AFTER",
@@ -1453,17 +1451,6 @@ class AddView(UserPassesTestMixin, FormView):
                 "binary_urls": binary_urls,
             }
         recent_personas = list(persona_queryset.order_by("-created_at", "name")[:5])
-        plugin_dependency_map = {}
-        if can_override_crawl_config:
-            plugin_dependency_map = {
-                plugin_name: [
-                    str(required_plugin).strip()
-                    for required_plugin in (schema.get("required_plugins") or [])
-                    if str(required_plugin).strip()
-                ]
-                for plugin_name, schema in plugin_configs.items()
-                if isinstance(schema.get("required_plugins"), list) and schema.get("required_plugins")
-            }
         return {
             **context,
             "title": "Create Crawl",
@@ -1473,7 +1460,6 @@ class AddView(UserPassesTestMixin, FormView):
             "VERSION": VERSION,
             "FOOTER_INFO": request_config.FOOTER_INFO,
             "required_search_plugin": required_search_plugin,
-            "plugin_dependency_map_json": json.dumps(plugin_dependency_map, sort_keys=True),
             "persona_config_map_json": json.dumps(persona_config_map, sort_keys=True, default=str),
             "recent_personas": recent_personas,
             "can_override_crawl_config": can_override_crawl_config,

archivebox/crawls/locks.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+import fcntl
+import os
+import threading
+from contextlib import contextmanager
+from pathlib import Path
+from typing import IO
+from collections.abc import Iterator
+
+from archivebox.config.constants import CONSTANTS
+
+
+class _LifecycleLockState:
+    def __init__(self) -> None:
+        self.thread_lock = threading.RLock()
+        self.users = 0
+        self.depth = 0
+        self.file: IO[str] | None = None
+
+
+_registry_lock = threading.Lock()
+_registry_pid = os.getpid()
+_registry: dict[str, _LifecycleLockState] = {}
+
+
+def crawl_lifecycle_lock_path(crawl_id: str) -> Path:
+    return CONSTANTS.DEFAULT_TMP_DIR / "crawl-locks" / f"{crawl_id}.lock"
+
+
+@contextmanager
+def _lifecycle_lock(key: str, lock_path: Path) -> Iterator[None]:
+    global _registry_pid
+
+    with _registry_lock:
+        if _registry_pid != os.getpid():
+            for inherited_state in _registry.values():
+                if inherited_state.file is not None:
+                    inherited_state.file.close()
+            _registry.clear()
+            _registry_pid = os.getpid()
+        state = _registry.setdefault(key, _LifecycleLockState())
+        state.users += 1
+
+    state.thread_lock.acquire()
+    try:
+        if state.depth == 0:
+            lock_path.parent.mkdir(parents=True, exist_ok=True)
+            state.file = lock_path.open("a+", encoding="utf-8")
+            try:
+                fcntl.flock(state.file.fileno(), fcntl.LOCK_EX)
+            except BaseException:
+                state.file.close()
+                state.file = None
+                raise
+        state.depth += 1
+        try:
+            yield
+        finally:
+            state.depth -= 1
+            if state.depth == 0 and state.file is not None:
+                fcntl.flock(state.file.fileno(), fcntl.LOCK_UN)
+                state.file.close()
+                state.file = None
+    finally:
+        state.thread_lock.release()
+        with _registry_lock:
+            state.users -= 1
+            if state.users == 0 and state.depth == 0 and _registry.get(key) is state:
+                _registry.pop(key, None)
+
+
+@contextmanager
+def crawl_lifecycle_lock(crawl_id: str) -> Iterator[None]:
+    key = str(crawl_id)
+    with _lifecycle_lock(f"crawl:{key}", crawl_lifecycle_lock_path(key)):
+        yield
+
+
+@contextmanager
+def binary_lifecycle_lock(binary_id: str) -> Iterator[None]:
+    key = str(binary_id)
+    lock_path = CONSTANTS.DEFAULT_TMP_DIR / "binary-locks" / f"{key}.lock"
+    with _lifecycle_lock(f"binary:{key}", lock_path):
+        yield

archivebox/crawls/models.py

@@ -1305,71 +1305,23 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         return created_snapshots
 
     def install_declared_binaries(self, binary_names: set[str], machine=None) -> None:
-        """
-        Install crawl-declared Binary rows without violating the retry_at lock lifecycle.
-
-        Correct calling pattern:
-        1. Crawl hooks declare Binary records and queue them with retry_at <= now
-        2. Exactly one actor claims each Binary by moving retry_at into the future
-        3. Only that owner executes `.sm.tick()` and performs install side effects
-        4. Everyone else waits for the claimed owner to finish instead of launching
-           a second install against shared state such as the pip or npm trees
-
-        This helper follows that contract by claiming each Binary before ticking
-        it, and by waiting when another worker already owns the row. That keeps
-        synchronous crawl execution compatible with the shared background runner and
-        avoids duplicate installs of the same dependency.
-        """
-        import time
+        """Install crawl-declared binaries through their unified state machine."""
+        from archivebox.crawls.locks import binary_lifecycle_lock
         from archivebox.machine.models import Binary, Machine
 
         if not binary_names:
             return
 
         machine = machine or Machine.current()
-        lock_seconds = 600
-        deadline = time.monotonic() + max(lock_seconds, len(binary_names) * lock_seconds)
-
-        while time.monotonic() < deadline:
-            unresolved_binaries = list(
-                Binary.objects.filter(
-                    machine=machine,
-                    name__in=binary_names,
-                )
-                .exclude(
-                    status=Binary.StatusChoices.INSTALLED,
-                )
-                .order_by("name"),
-            )
-            if not unresolved_binaries:
-                return
-
-            claimed_any = False
-            waiting_on_existing_owner = False
-            now = timezone.now()
-
-            for binary in unresolved_binaries:
-                try:
-                    if binary.tick_claimed(lock_seconds=lock_seconds):
-                        claimed_any = True
-                        continue
-                except Exception:
-                    claimed_any = True
-                    continue
-
+        binaries = Binary.objects.filter(machine=machine, name__in=binary_names).order_by("name")
+        for binary in binaries:
+            with binary_lifecycle_lock(str(binary.id)):
                 binary.refresh_from_db()
                 if binary.status == Binary.StatusChoices.INSTALLED:
-                    claimed_any = True
                     continue
-                if binary.retry_at and binary.retry_at > now:
-                    waiting_on_existing_owner = True
-
-            if claimed_any:
-                continue
-            if waiting_on_existing_owner:
-                time.sleep(0.5)
-                continue
-            break
+                binary.update_and_requeue(retry_at=timezone.now())
+                binary.refresh_from_db()
+                binary.tick_claimed(lock_seconds=600)
 
         unresolved_binaries = list(
             Binary.objects.filter(
@@ -1401,7 +1353,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         import time
         from archivebox.plugins.hooks import run_hook, discover_hooks, process_hook_records
         from archivebox.config.common import get_config
-        from archivebox.machine.models import Binary, Machine
+        from archivebox.machine.models import Machine
 
         def get_runtime_config():
             return get_config(crawl=self).for_crawl_runtime(
@@ -1429,10 +1381,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
                 chrome_binary=chrome_binary,
             )
 
-        executed_crawl_hooks: set[str] = set()
-
         def run_crawl_hook(hook: Path) -> set[str]:
-            executed_crawl_hooks.add(str(hook))
             primary_url = next(
                 (line.strip() for line in self.urls.splitlines() if line.strip()),
                 self.urls.strip(),
@@ -1493,53 +1442,12 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
                 declared_binary_names.update(hook_binary_names)
             return hook_binary_names
 
-        def resolve_provider_binaries(binary_names: set[str]) -> set[str]:
-            if not binary_names:
-                return set()
-
-            resolved_binary_names = set(binary_names)
-
-            while True:
-                unresolved_binaries = list(
-                    Binary.objects.filter(
-                        machine=machine,
-                        name__in=resolved_binary_names,
-                    )
-                    .exclude(
-                        status=Binary.StatusChoices.INSTALLED,
-                    )
-                    .order_by("name"),
-                )
-                if not unresolved_binaries:
-                    return resolved_binary_names
-
-                needed_provider_names: set[str] = set()
-                for binary in unresolved_binaries:
-                    allowed_binproviders = binary._allowed_binproviders()
-                    if allowed_binproviders is None:
-                        continue
-                    needed_provider_names.update(allowed_binproviders)
-
-                if not needed_provider_names:
-                    return resolved_binary_names
-
-                provider_hooks = [
-                    hook
-                    for hook in discover_hooks("Crawl", filter_disabled=False, config=get_runtime_config())
-                    if hook.parent.name in needed_provider_names and str(hook) not in executed_crawl_hooks
-                ]
-                if not provider_hooks:
-                    return resolved_binary_names
-
-                for hook in provider_hooks:
-                    resolved_binary_names.update(run_crawl_hook(hook))
-
         hooks = discover_hooks("Crawl", config=get_runtime_config())
 
         for hook in hooks:
             hook_binary_names = run_crawl_hook(hook)
             if hook_binary_names:
-                self.install_declared_binaries(resolve_provider_binaries(hook_binary_names), machine=machine)
+                self.install_declared_binaries(hook_binary_names, machine=machine)
 
         # Safety check: don't create snapshots if any crawl-declared dependency
         # is still unresolved after all crawl hooks have run.

archivebox/machine/detect.py

@@ -145,6 +145,11 @@ ip_addrs = lambda addrs: (a for a in addrs if a.family == socket.AF_INET)
 mac_addrs = lambda addrs: (a for a in addrs if a.family == psutil.AF_LINK)
 
 
+def unknown_if_blank(value):
+    normalized = str(value or "").strip()
+    return normalized or "Unknown"
+
+
 def get_isp_info(ip=None):
     # Get public IP
     try:
@@ -161,10 +166,10 @@ def get_isp_info(ip=None):
     except Exception:
         pass
 
-    isp = data.get("org", "Unknown")
-    city = data.get("city", "Unknown")
-    region = data.get("region", "Unknown")
-    country = data.get("country_name", "Unknown")
+    isp = unknown_if_blank(data.get("org"))
+    city = unknown_if_blank(data.get("city"))
+    region = unknown_if_blank(data.get("region"))
+    country = unknown_if_blank(data.get("country_name"))
 
     # Get system DNS resolver servers
     dns_server = None

archivebox/machine/migrations/0021_networkinterface_blank_defaults.py

@@ -0,0 +1,42 @@
+# Generated by Django 6.0.6 on 2026-06-21 09:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("machine", "0020_repair_process_binary_iface_links"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="networkinterface",
+            name="city",
+            field=models.CharField(default="", max_length=63),
+        ),
+        migrations.AlterField(
+            model_name="networkinterface",
+            name="country",
+            field=models.CharField(default="", max_length=63),
+        ),
+        migrations.AlterField(
+            model_name="networkinterface",
+            name="hostname",
+            field=models.CharField(default="", max_length=63),
+        ),
+        migrations.AlterField(
+            model_name="networkinterface",
+            name="iface",
+            field=models.CharField(default="", max_length=15),
+        ),
+        migrations.AlterField(
+            model_name="networkinterface",
+            name="isp",
+            field=models.CharField(default="", max_length=63),
+        ),
+        migrations.AlterField(
+            model_name="networkinterface",
+            name="region",
+            field=models.CharField(default="", max_length=63),
+        ),
+    ]

archivebox/machine/migrations/0022_networkinterface_identity_without_mac.py

@@ -0,0 +1,60 @@
+# Generated by Django 6.0.6 on 2026-06-21 09:23
+
+from django.db import migrations, models
+from django.db.models import Count
+
+
+def dedupe_network_interfaces_without_mac(apps, schema_editor):
+    NetworkInterface = apps.get_model("machine", "NetworkInterface")
+    Process = apps.get_model("machine", "Process")
+    duplicate_groups = (
+        NetworkInterface.objects.values(
+            "machine_id",
+            "ip_public",
+            "ip_local",
+            "dns_server",
+        )
+        .annotate(count=Count("id"))
+        .filter(count__gt=1)
+    )
+    for group in duplicate_groups.iterator(chunk_size=100):
+        lookup = {
+            "machine_id": group["machine_id"],
+            "ip_public": group["ip_public"],
+            "ip_local": group["ip_local"],
+            "dns_server": group["dns_server"],
+        }
+        keep = NetworkInterface.objects.filter(**lookup).order_by("-modified_at", "-created_at").first()
+        if keep is None:
+            continue
+        duplicates = NetworkInterface.objects.filter(**lookup).exclude(id=keep.id)
+        duplicate_ids = list(duplicates.values_list("id", flat=True))
+        if not duplicate_ids:
+            continue
+        Process.objects.filter(iface_id__in=duplicate_ids).update(iface_id=keep.id)
+        duplicates.delete()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("machine", "0021_networkinterface_blank_defaults"),
+    ]
+
+    operations = [
+        migrations.RemoveConstraint(
+            model_name="networkinterface",
+            name="unique_network_interface_identity",
+        ),
+        migrations.RunPython(dedupe_network_interfaces_without_mac, migrations.RunPython.noop),
+        migrations.AlterUniqueTogether(
+            name="networkinterface",
+            unique_together={("machine", "ip_public", "ip_local", "dns_server")},
+        ),
+        migrations.AddConstraint(
+            model_name="networkinterface",
+            constraint=models.UniqueConstraint(
+                fields=("machine", "ip_public", "ip_local", "dns_server"),
+                name="unique_network_interface_identity",
+            ),
+        ),
+    ]

archivebox/machine/models.py

@@ -394,12 +394,12 @@ class NetworkInterface(ModelWithHealthStats):
     ip_public = models.GenericIPAddressField(default=None, null=False, editable=False)
     ip_local = models.GenericIPAddressField(default=None, null=False, editable=False)
     dns_server = models.GenericIPAddressField(default=None, null=False, editable=False)
-    hostname = models.CharField(max_length=63, default=None, null=False)
-    iface = models.CharField(max_length=15, default=None, null=False)
-    isp = models.CharField(max_length=63, default=None, null=False)
-    city = models.CharField(max_length=63, default=None, null=False)
-    region = models.CharField(max_length=63, default=None, null=False)
-    country = models.CharField(max_length=63, default=None, null=False)
+    hostname = models.CharField(max_length=63, default="", null=False)
+    iface = models.CharField(max_length=15, default="", null=False)
+    isp = models.CharField(max_length=63, default="", null=False)
+    city = models.CharField(max_length=63, default="", null=False)
+    region = models.CharField(max_length=63, default="", null=False)
+    country = models.CharField(max_length=63, default="", null=False)
     # num_uses_failed = models.PositiveIntegerField(default=0)  # from ModelWithHealthStats
     # num_uses_succeeded = models.PositiveIntegerField(default=0)  # from ModelWithHealthStats
 
@@ -408,10 +408,10 @@ class NetworkInterface(ModelWithHealthStats):
 
     class Meta(ModelWithHealthStats.Meta):
         app_label = "machine"
-        unique_together = (("machine", "ip_public", "ip_local", "mac_address", "dns_server"),)
+        unique_together = (("machine", "ip_public", "ip_local", "dns_server"),)
         constraints = [
             models.UniqueConstraint(
-                fields=["machine", "ip_public", "ip_local", "mac_address", "dns_server"],
+                fields=["machine", "ip_public", "ip_local", "dns_server"],
                 name="unique_network_interface_identity",
             ),
         ]
@@ -444,7 +444,6 @@ class NetworkInterface(ModelWithHealthStats):
             machine=machine,
             ip_public=net_info.pop("ip_public"),
             ip_local=net_info.pop("ip_local"),
-            mac_address=net_info.pop("mac_address"),
             dns_server=net_info.pop("dns_server"),
         )
         _CURRENT_INTERFACE = cls.objects.filter(**lookup).order_by("-modified_at", "-created_at").first()
@@ -1576,7 +1575,7 @@ class Process(ModelWithDeleteAfter, models.Model):
 
         Processes are stale if:
         - Status is RUNNING but OS process no longer exists
-        - Status is RUNNING but exceeded its timeout plus a small grace margin
+        - A bounded HOOK or BINARY process exceeded its timeout plus a small grace margin
         - Status is RUNNING but started_at is older than PID_REUSE_WINDOW
 
         Returns count of processes cleaned up.
@@ -1596,7 +1595,7 @@ class Process(ModelWithDeleteAfter, models.Model):
 
             is_stale = False
 
-            if proc.started_at:
+            if proc.started_at and proc.process_type in (cls.TypeChoices.HOOK, cls.TypeChoices.BINARY):
                 timeout_seconds = max(int(proc.timeout or 0), 0)
                 timeout_deadline = proc.started_at + timedelta(seconds=timeout_seconds) + PROCESS_TIMEOUT_GRACE
                 if timeout_seconds > 0 and timezone.now() >= timeout_deadline:

archivebox/personas/export_browser_state.js

@@ -8,8 +8,8 @@
  *   CHROME_CDP_URL              Existing browser CDP URL to attach to
  *   COOKIES_OUTPUT_FILE         Optional output path for Netscape cookies.txt
  *   AUTH_STORAGE_OUTPUT_FILE    Optional output path for auth.json
- *   CHROME_BINARY               Optional browser binary override
- *   NODE_MODULES_DIR            Optional node_modules path for puppeteer-core
+ *   CHROME_BINARY               Absolute browser path resolved by abxpkg
+ *   NODE_MODULES_DIR            node_modules path resolved by abxpkg
  */
 
 const fs = require('fs');
@@ -116,9 +116,9 @@ async function openBrowser() {
     }
 
     const outputDir = fs.mkdtempSync(path.join(os.tmpdir(), 'abx-browser-state-'));
-    const binary = process.env.CHROME_BINARY || chromeUtils.findAnyChromiumBinary();
-    if (!binary) {
-        throw new Error('Could not find a Chromium binary for browser state export');
+    const binary = process.env.CHROME_BINARY;
+    if (!binary || !path.isAbsolute(binary) || !fs.existsSync(binary)) {
+        throw new Error('CHROME_BINARY must be an absolute executable path resolved by abxpkg');
     }
 
     const launched = await chromeUtils.launchChromium({

archivebox/personas/importers.py

@@ -12,6 +12,7 @@ import os
 import platform
 import shutil
 import subprocess
+import sys
 import tempfile
 from dataclasses import dataclass, field
 from pathlib import Path
@@ -68,6 +69,55 @@ PERSONA_PROFILE_DIR_CANDIDATES = (
 )
 
 
+def _path_exists(path: Path) -> bool:
+    try:
+        return path.exists()
+    except OSError:
+        return False
+
+
+def _path_is_dir(path: Path) -> bool:
+    try:
+        return path.is_dir()
+    except OSError:
+        return False
+
+
+def _iter_children(path: Path) -> list[Path]:
+    try:
+        return list(path.iterdir())
+    except OSError:
+        return []
+
+
+def _is_copyable_profile_entry(path: Path) -> bool:
+    try:
+        if path.is_symlink():
+            return True
+        if path.is_dir():
+            return os.access(path, os.R_OK | os.X_OK)
+        return os.access(path, os.R_OK)
+    except OSError:
+        return False
+
+
+def profile_copy_ignore(src: str, names: list[str]) -> set[str]:
+    ignored = set(shutil.ignore_patterns(*VOLATILE_PROFILE_COPY_PATTERNS)(src, names))
+    src_path = Path(src)
+    for name in names:
+        if name in ignored:
+            continue
+        if not _is_copyable_profile_entry(src_path / name):
+            # Chrome can leave generated profile components unreadable when a
+            # previous run used a different runtime uid, a mounted profile came
+            # from the host, or Chrome tightened permissions on cache/model
+            # directories. Those entries are not required to launch a cloned
+            # profile, and letting copytree abort here prevents the crawl from
+            # even creating its first Snapshot.
+            ignored.add(name)
+    return ignored
+
+
 @dataclass(frozen=True)
 class PersonaImportSource:
     kind: str
@@ -113,7 +163,7 @@ class PersonaImportSource:
 
     def as_choice_label(self) -> SafeString:
         path_str = str(self.profile_path or self.user_data_dir or self.cdp_url or "")
-        binary_suffix = f"Using {self.browser_binary}" if self.browser_binary else "Will auto-detect a Chromium binary"
+        binary_suffix = f"Using {self.browser_binary}" if self.browser_binary else "Chromium resolved by abxpkg"
         return format_html(
             '<span class="abx-profile-option"><strong>{}</strong><span class="abx-profile-option__meta">{}</span><code>{}</code></span>',
             self.display_label,
@@ -251,65 +301,6 @@ def get_edge_user_data_dir() -> Path | None:
     return None
 
 
-def get_browser_binary(browser: str) -> str | None:
-    system = platform.system()
-    home = Path.home()
-    browser = browser.lower()
-
-    if system == "Darwin":
-        candidates = {
-            "chrome": ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"],
-            "chromium": ["/Applications/Chromium.app/Contents/MacOS/Chromium"],
-            "brave": ["/Applications/Brave Browser.app/Contents/MacOS/Brave Browser"],
-            "edge": ["/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge"],
-        }.get(browser, [])
-    elif system == "Linux":
-        candidates = {
-            "chrome": [
-                "/usr/bin/google-chrome",
-                "/usr/bin/google-chrome-stable",
-                "/usr/bin/google-chrome-beta",
-                "/usr/bin/google-chrome-unstable",
-            ],
-            "chromium": ["/usr/bin/chromium", "/usr/bin/chromium-browser"],
-            "brave": ["/usr/bin/brave-browser", "/usr/bin/brave-browser-beta", "/usr/bin/brave-browser-nightly"],
-            "edge": [
-                "/usr/bin/microsoft-edge",
-                "/usr/bin/microsoft-edge-stable",
-                "/usr/bin/microsoft-edge-beta",
-                "/usr/bin/microsoft-edge-dev",
-            ],
-        }.get(browser, [])
-    elif system == "Windows":
-        local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
-        candidates = {
-            "chrome": [
-                str(local_app_data / "Google" / "Chrome" / "Application" / "chrome.exe"),
-                "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe",
-                "C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe",
-            ],
-            "chromium": [str(local_app_data / "Chromium" / "Application" / "chrome.exe")],
-            "brave": [
-                str(local_app_data / "BraveSoftware" / "Brave-Browser" / "Application" / "brave.exe"),
-                "C:\\Program Files\\BraveSoftware\\Brave-Browser\\Application\\brave.exe",
-                "C:\\Program Files (x86)\\BraveSoftware\\Brave-Browser\\Application\\brave.exe",
-            ],
-            "edge": [
-                str(local_app_data / "Microsoft" / "Edge" / "Application" / "msedge.exe"),
-                "C:\\Program Files\\Microsoft\\Edge\\Application\\msedge.exe",
-                "C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe",
-            ],
-        }.get(browser, [])
-    else:
-        candidates = []
-
-    for candidate in candidates:
-        if candidate and Path(candidate).exists():
-            return candidate
-
-    return None
-
-
 BROWSER_PROFILE_FINDERS = {
     "chrome": get_chrome_user_data_dir,
     "chromium": get_chrome_user_data_dir,
@@ -353,7 +344,6 @@ def discover_local_browser_profiles() -> list[PersonaImportSource]:
         if not user_data_dir:
             continue
 
-        browser_binary = get_browser_binary(browser)
         for profile_dir in _list_profile_names(user_data_dir):
             try:
                 discovered.append(
@@ -361,7 +351,6 @@ def discover_local_browser_profiles() -> list[PersonaImportSource]:
                         browser=browser,
                         user_data_dir=user_data_dir,
                         profile_dir=profile_dir,
-                        browser_binary=browser_binary,
                     ),
                 )
             except ValueError:
@@ -394,13 +383,16 @@ def discover_persona_template_profiles(personas_dir: Path | None = None) -> list
             continue
         seen_roots.add(resolved_root)
 
-        if not resolved_root.exists() or not resolved_root.is_dir():
+        if not _path_is_dir(resolved_root):
             continue
 
-        for persona_dir in sorted((path for path in resolved_root.iterdir() if path.is_dir()), key=lambda path: path.name.lower()):
+        for persona_dir in sorted(
+            (path for path in _iter_children(resolved_root) if _path_is_dir(path)),
+            key=lambda path: path.name.lower(),
+        ):
             for candidate_dir_name in PERSONA_PROFILE_DIR_CANDIDATES:
                 user_data_dir = persona_dir / candidate_dir_name
-                if not user_data_dir.exists() or not user_data_dir.is_dir():
+                if not _path_is_dir(user_data_dir):
                     continue
 
                 for profile_dir in _list_profile_names(user_data_dir):
@@ -411,7 +403,6 @@ def discover_persona_template_profiles(personas_dir: Path | None = None) -> list
                                 source_name=persona_dir.name,
                                 user_data_dir=user_data_dir,
                                 profile_dir=profile_dir,
-                                browser_binary=get_browser_binary("chrome"),
                             ),
                         )
                     except ValueError:
@@ -438,7 +429,6 @@ def resolve_browser_import_source(browser: str, profile_dir: str | None = None)
         browser=browser,
         user_data_dir=user_data_dir,
         profile_dir=chosen_profile,
-        browser_binary=get_browser_binary(browser),
     )
 
 
@@ -594,7 +584,7 @@ def copy_browser_user_data_dir(source_dir: Path, destination_dir: Path) -> None:
         source_dir,
         destination_dir,
         symlinks=True,
-        ignore=shutil.ignore_patterns(*VOLATILE_PROFILE_COPY_PATTERNS),
+        ignore=profile_copy_ignore,
     )
 
 
@@ -617,11 +607,34 @@ def export_browser_state(
     if not state_script.exists():
         return False, None, f"Browser state export script not found at {state_script}"
 
-    node_modules_dir = get_config().ABXPKG_LIB_DIR / "pnpm" / "packages" / "chrome" / "node_modules"
     chrome_plugin_dir = Path(get_plugins_dir()).resolve()
+    chrome_config = chrome_plugin_dir / "chrome" / "config.json"
 
     env = os.environ.copy()
-    env["NODE_MODULES_DIR"] = str(node_modules_dir)
+    if chrome_binary:
+        env["CHROME_BINARY"] = str(chrome_binary)
+    dependency_env = subprocess.run(
+        [
+            str(Path(sys.executable).with_name("abxpkg")),
+            "env",
+            "--install",
+            "--json",
+            f"--lib={get_config().ABXPKG_LIB_DIR}",
+            f"--deps-from={chrome_config}:required_binaries",
+        ],
+        capture_output=True,
+        text=True,
+        env=env,
+    )
+    if dependency_env.returncode != 0:
+        return False, None, dependency_env.stderr.strip() or "abxpkg could not resolve browser export dependencies."
+    try:
+        resolved_env = json.loads(dependency_env.stdout)
+    except json.JSONDecodeError:
+        return False, None, "abxpkg returned an invalid browser dependency environment."
+    if not isinstance(resolved_env, dict):
+        return False, None, "abxpkg returned an invalid browser dependency environment."
+    env.update({str(key): str(value) for key, value in resolved_env.items()})
     env["ARCHIVEBOX_ABX_PLUGINS_DIR"] = str(chrome_plugin_dir)
 
     if user_data_dir:
@@ -629,8 +642,6 @@ def export_browser_state(
     if cdp_url:
         env["CHROME_CDP_URL"] = cdp_url
         env["CHROME_IS_LOCAL"] = "false"
-    if chrome_binary:
-        env["CHROME_BINARY"] = str(chrome_binary)
     if profile_dir:
         extra_arg = f"--profile-directory={profile_dir}"
         existing_extra = env.get("CHROME_ARGS_EXTRA", "").strip()
@@ -707,12 +718,12 @@ def export_browser_state(
 
 
 def _list_profile_names(user_data_dir: Path) -> list[str]:
-    if not user_data_dir.exists() or not user_data_dir.is_dir():
+    if not _path_is_dir(user_data_dir):
         return []
 
     profiles: list[str] = []
-    for child in sorted(user_data_dir.iterdir(), key=lambda path: path.name.lower()):
-        if not child.is_dir():
+    for child in sorted(_iter_children(user_data_dir), key=lambda path: path.name.lower()):
+        if not _path_is_dir(child):
             continue
         if child.name == "System Profile":
             continue
@@ -726,7 +737,7 @@ def _list_profile_names(user_data_dir: Path) -> list[str]:
 
 
 def _looks_like_profile_dir(path: Path) -> bool:
-    if not path.exists() or not path.is_dir():
+    if not _path_is_dir(path):
         return False
 
     marker_paths = (
@@ -738,7 +749,7 @@ def _looks_like_profile_dir(path: Path) -> bool:
         path / "Session Storage",
     )
 
-    if any(marker.exists() for marker in marker_paths):
+    if any(_path_exists(marker) for marker in marker_paths):
         return True
 
     return any(path.name == prefix or path.name.startswith(prefix) for prefix in BROWSER_PROFILE_DIR_NAMES)

archivebox/personas/models.py

@@ -11,6 +11,7 @@ Each persona has its own:
 __package__ = "archivebox.personas"
 
 import shutil
+import os
 from contextlib import contextmanager
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
@@ -179,7 +180,16 @@ class Persona(ModelWithConfig):
         if not profile_dir.exists():
             return False
 
-        for path in profile_dir.rglob("*"):
+        def profile_paths():
+            # Chrome profiles are user-controlled filesystem state. If a cache
+            # or generated component is unreadable, cleanup should skip it and
+            # continue pruning the rest instead of blocking the crawl runner.
+            for dirpath, dirnames, filenames in os.walk(profile_dir, onerror=lambda _err: None):
+                current_dir = Path(dirpath)
+                for name in [*dirnames, *filenames]:
+                    yield current_dir / name
+
+        for path in profile_paths():
             if path.name in VOLATILE_PROFILE_FILE_NAMES:
                 try:
                     path.unlink()
@@ -188,13 +198,21 @@ class Persona(ModelWithConfig):
                     pass
 
         for dirname in VOLATILE_PROFILE_DIR_NAMES:
-            for path in profile_dir.rglob(dirname):
-                if not path.is_dir():
+            for path in profile_paths():
+                if path.name != dirname:
+                    continue
+                try:
+                    is_dir = path.is_dir()
+                except OSError:
+                    is_dir = False
+                if not is_dir:
                     continue
                 shutil.rmtree(path, ignore_errors=True)
                 cleaned = True
 
-        for path in profile_dir.rglob("*.log"):
+        for path in profile_paths():
+            if not path.match("*.log"):
+                continue
             try:
                 path.unlink()
                 cleaned = True
@@ -252,7 +270,7 @@ class Persona(ModelWithConfig):
         return self.runtime_root_for_snapshot(snapshot) / "chrome_downloads"
 
     def copy_chrome_profile(self, source_dir: Path, destination_dir: Path) -> None:
-        from archivebox.personas.importers import VOLATILE_PROFILE_COPY_PATTERNS
+        from archivebox.personas.importers import profile_copy_ignore
 
         destination_dir.parent.mkdir(parents=True, exist_ok=True)
         shutil.rmtree(destination_dir, ignore_errors=True)
@@ -260,7 +278,7 @@ class Persona(ModelWithConfig):
             source_dir,
             destination_dir,
             symlinks=True,
-            ignore=shutil.ignore_patterns(*VOLATILE_PROFILE_COPY_PATTERNS),
+            ignore=profile_copy_ignore,
         )
 
     def prepare_runtime_for_crawl(self, crawl, chrome_binary: str = "") -> dict[str, str]:

archivebox/plugins/discovery.py

@@ -64,6 +64,15 @@ def get_plugins() -> list[str]:
     return sorted(set(plugins))
 
 
+def get_plugin_models():
+    from abx_dl.models import discover_plugins
+
+    plugins = {}
+    for base_dir in (BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR):
+        plugins.update(discover_plugins(plugins_dir=base_dir, runtime="archivebox"))
+    return plugins
+
+
 def get_plugin_name(plugin: str) -> str:
     """
     Get the base plugin name without numeric prefix.
@@ -90,36 +99,18 @@ def get_enabled_plugins(config: ConfigLookup | None = None, **config_kwargs: Any
 
         config = get_config(**config_kwargs)
 
-    def normalize_enabled_plugins(value: Any) -> list[str]:
-        if value is None:
-            return []
-        if isinstance(value, str):
-            raw = value.strip()
-            if not raw:
-                return []
-            if raw.startswith("["):
-                try:
-                    parsed = json.loads(raw)
-                except json.JSONDecodeError:
-                    parsed = None
-                if isinstance(parsed, list):
-                    return [str(plugin).strip() for plugin in parsed if str(plugin).strip()]
-            return [plugin.strip() for plugin in raw.split(",") if plugin.strip()]
-        if isinstance(value, (list, tuple, set)):
-            return [str(plugin).strip() for plugin in value if str(plugin).strip()]
-        return [str(value).strip()] if str(value).strip() else []
-
-    plugins_override = config.get("PLUGINS")
-    if plugins_override:
-        return normalize_enabled_plugins(plugins_override)
-
     enabled = []
+    disabled = []
     for plugin in get_plugins():
         plugin_config = get_plugin_special_config(plugin, config)
         if plugin_config["enabled"]:
             enabled.append(plugin)
+        else:
+            disabled.append(plugin)
 
-    return enabled
+    from abx_dl.models import filter_plugins
+
+    return list(filter_plugins(get_plugin_models(), enabled, include_providers=True, disabled_names=disabled))
 
 
 def discover_plugins_that_provide_interface(
@@ -245,61 +236,12 @@ def get_plugin_special_config(plugin_name: str, config: ConfigLookup, _visited:
     """
     plugin_upper = plugin_name.upper()
 
-    plugins_whitelist = config.get("PLUGINS", "")
-    if plugins_whitelist:
-        plugin_configs = discover_plugin_configs()
-        plugin_names = {p.strip().lower() for p in plugins_whitelist.split(",") if p.strip()}
-        pending = list(plugin_names)
-
-        while pending:
-            current = pending.pop()
-            schema = plugin_configs.get(current, {})
-            required_plugins = schema.get("required_plugins", [])
-            if not isinstance(required_plugins, list):
-                continue
-
-            for required_plugin in required_plugins:
-                required_plugin_name = str(required_plugin).strip().lower()
-                if not required_plugin_name or required_plugin_name in plugin_names:
-                    continue
-                plugin_names.add(required_plugin_name)
-                pending.append(required_plugin_name)
-
-        if plugin_name.lower() not in plugin_names:
-            enabled = False
-        else:
-            enabled_key = f"{plugin_upper}_ENABLED"
-            enabled = config.get(enabled_key)
-            if enabled is None:
-                enabled = True
-            elif isinstance(enabled, str):
-                enabled = enabled.lower() not in ("false", "0", "no", "")
-    else:
-        enabled_key = f"{plugin_upper}_ENABLED"
-        enabled = config.get(enabled_key)
-        if enabled is None:
-            enabled = True
-        elif isinstance(enabled, str):
-            enabled = enabled.lower() not in ("false", "0", "no", "")
-
-    plugin_configs = discover_plugin_configs()
-    plugin_name_lower = plugin_name.lower()
-
-    if enabled:
-        visited = _visited or set()
-        if plugin_name_lower not in visited:
-            next_visited = visited | {plugin_name_lower}
-            schema = plugin_configs.get(plugin_name_lower, {})
-            required_plugins = schema.get("required_plugins", [])
-            if isinstance(required_plugins, list):
-                for required_plugin in required_plugins:
-                    required_plugin_name = str(required_plugin).strip()
-                    if not required_plugin_name:
-                        continue
-                    required_config = get_plugin_special_config(required_plugin_name, config, _visited=next_visited)
-                    if not required_config["enabled"]:
-                        enabled = False
-                        break
+    enabled_key = f"{plugin_upper}_ENABLED"
+    enabled = config.get(enabled_key)
+    if enabled is None:
+        enabled = True
+    elif isinstance(enabled, str):
+        enabled = enabled.lower() not in ("false", "0", "no", "")
 
     timeout_key = f"{plugin_upper}_TIMEOUT"
     timeout = config.get(timeout_key) or config.get("TIMEOUT", 300)

archivebox/plugins/hooks.py

@@ -55,6 +55,7 @@ from archivebox.plugins.discovery import (
     BUILTIN_PLUGINS_DIR,
     USER_PLUGINS_DIR,
     ConfigLookup,
+    get_enabled_plugins,
     get_plugin_special_config,
 )
 
@@ -215,6 +216,7 @@ def discover_hooks(
 
             config = get_config(**config_kwargs)
 
+        enabled_plugins = set(get_enabled_plugins(config=config))
         enabled_hooks = []
 
         for hook in hooks:
@@ -229,9 +231,7 @@ def discover_hooks(
                 enabled_hooks.append(hook)
                 continue
 
-            # Check if plugin is enabled
-            plugin_config = get_plugin_special_config(plugin_name, config)
-            if plugin_config["enabled"]:
+            if plugin_name in enabled_plugins:
                 enabled_hooks.append(hook)
 
         hooks = enabled_hooks
@@ -279,15 +279,27 @@ def run_hook(
             records = process.get_records()  # Get parsed JSONL output
     """
     from archivebox.machine.models import Process, Machine, NetworkInterface
-    from archivebox.config.common import ArchiveBoxConfig, _archivebox_config_input_names, get_config, normalize_runtime_config
+    from archivebox.config.common import (
+        ArchiveBoxConfig,
+        _archivebox_config_input_names,
+        get_config,
+        normalize_runtime_config,
+        _plugin_enabled_config_keys,
+    )
 
     config_scope = {key.removeprefix("config_"): kwargs.pop(key) for key in list(kwargs) if key.startswith("config_")}
     config_overrides = _config_to_overrides(config)
+    explicit_override_keys = set(config_overrides)
     resolved_config = get_config(overrides=config_overrides, **config_scope)
     hook_config = normalize_runtime_config(
-        resolved_config.for_crawl_runtime(),
+        resolved_config.for_crawl_runtime(runtime_overrides=config_overrides),
         json_safe=False,
     )
+    hook_config.update(normalize_runtime_config(config_overrides, json_safe=False))
+    plugin_enabled_keys = set(_plugin_enabled_config_keys().values())
+    if plugin_enabled_keys.intersection(hook_config):
+        for enabled_key in plugin_enabled_keys:
+            hook_config.setdefault(enabled_key, False)
 
     # Auto-detect timeout from plugin config if not explicitly provided
     if timeout is None:
@@ -368,8 +380,10 @@ def run_hook(
     archivebox_config_input_names = _archivebox_config_input_names()
     for key in archivebox_config_input_names:
         env.pop(key, None)
+    env.pop("PLUGINS", None)
     env["DATA_DIR"] = str(CONSTANTS.DATA_DIR)
     env["LIBRARY_VERSION"] = VERSION
+    env.setdefault("ABXPKG_FAST_SCRIPT", "1")
     env.setdefault("MACHINE_ID", os.environ.get("MACHINE_ID", CONSTANTS.MACHINE_ID))
     snap_dir = hook_config.get("SNAP_DIR") or _model_output_dir_from_child_path(output_dir, CONSTANTS.SNAPSHOTS_DIR_NAME)
     crawl_dir = hook_config.get("CRAWL_DIR") or _model_output_dir_from_child_path(output_dir, CONSTANTS.CRAWLS_DIR_NAME)
@@ -386,7 +400,9 @@ def run_hook(
     # Set Node.js module resolution paths.
     # NODE_PATH may be a path list, but NODE_MODULES_DIR is a single canonical directory.
     node_modules_dir = hook_config.get("NODE_MODULES_DIR")
-    if not node_modules_dir and lib_dir:
+    if lib_dir and "ABXPKG_LIB_DIR" in explicit_override_keys and "NODE_MODULES_DIR" not in explicit_override_keys:
+        node_modules_dir = Path(lib_dir) / "pnpm" / "packages" / "chrome" / "node_modules"
+    elif not node_modules_dir and lib_dir:
         node_modules_dir = Path(lib_dir) / "pnpm" / "packages" / "chrome" / "node_modules"
 
     node_path_parts = [part for part in str(hook_config.get("NODE_PATH") or "").split(os.pathsep) if part]

archivebox/services/archive_result_service.py

@@ -4,6 +4,7 @@ import asyncio
 import inspect
 import json
 import os
+import signal
 import sys
 import time
 from collections import defaultdict
@@ -240,6 +241,14 @@ def _should_update_snapshot_title(current_title: str, next_title: str, *, snapsh
 def _status_for_process_without_archive_result(event: ProcessCompletedEvent) -> str:
     if event.exit_code == PROCESS_EXIT_SKIPPED:
         return "skipped"
+    if event.exit_code in {128 + signal.SIGHUP, 128 + signal.SIGINT, 128 + signal.SIGTERM}:
+        # This fallback only runs when a snapshot hook exited before emitting a
+        # structured ArchiveResult. A polite shutdown signal means the runner
+        # was interrupted during ownership transfer, not that the extractor
+        # produced a durable negative result. Keep the hook queued so the next
+        # runner can retry the exact work item instead of sealing in a transient
+        # process-lifecycle failure.
+        return "queued"
     if event.exit_code != 0:
         return "failed"
     return "noresults"

archivebox/services/binary_service.py

@@ -54,10 +54,6 @@ class ArchiveBoxDBBinaryCacheBackend:
                 continue
 
             provider = _provider_for_name(provider_name, installed.name, native_overrides)
-            if await sync_to_async(_cached_provider_path_is_stale, thread_sensitive=True)(installed_path, provider, installed.name):
-                await _mark_binary_queued(installed)
-                continue
-
             binary_env = BinProvider.build_exec_env(providers=[provider], base_env={}) if provider is not None else {}
             provider_names = _provider_names(installed.binproviders or request.binproviders or "env")
             return AbxBinary.model_validate(
@@ -405,15 +401,6 @@ async def _mark_binary_queued(binary) -> None:
     await binary.asave(update_fields=["status", "retry_at", "modified_at"])
 
 
-def _cached_provider_path_is_stale(installed_path: Path, provider: BinProvider | None, binary_name: str) -> bool:
-    if provider is None:
-        return False
-    current_abspath = provider.get_abspath(binary_name, quiet=True, no_cache=True)
-    if not current_abspath:
-        return True
-    return Path(current_abspath).expanduser().resolve(strict=False) != installed_path
-
-
 def _persisted_overrides_for_request(request: BinaryRequestEvent | None) -> dict[str, Any]:
     if request is None:
         return {}

archivebox/services/process_service.py

@@ -31,6 +31,27 @@ def current_network_interface_with_machine():
     return NetworkInterface.objects.select_related("machine").get(id=current_iface.id)
 
 
+def normalize_process_env(env: dict) -> dict:
+    normalized = dict(env or {})
+    raw_plugins = normalized.pop("PLUGINS", "")
+    selected_plugins = {name.strip().lower() for name in str(raw_plugins).split(",") if name.strip()}
+    from archivebox.config.common import ArchiveBoxConfig, _archivebox_config_input_names, is_sensitive_config_key
+
+    allowed_config_keys = ArchiveBoxConfig._crawl_runtime_keys()
+    config_input_names = _archivebox_config_input_names()
+    for key in list(normalized):
+        if is_sensitive_config_key(key) or (key in config_input_names and key not in allowed_config_keys):
+            normalized.pop(key, None)
+    if selected_plugins:
+        from abx_dl.models import discover_plugins, filter_plugins
+        from archivebox.config.common import _plugin_enabled_config_keys
+
+        selected_plugins = set(filter_plugins(discover_plugins(runtime="archivebox"), sorted(selected_plugins), include_providers=True))
+        for plugin_name, enabled_key in _plugin_enabled_config_keys().items():
+            normalized.setdefault(enabled_key, "True" if plugin_name in selected_plugins else "False")
+    return normalized
+
+
 class ProcessService(BaseService):
     LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [
         ProcessStartedEvent,
@@ -74,6 +95,7 @@ class ProcessService(BaseService):
                 started_at=started_at,
             )
         process = await process_query.order_by("-modified_at").afirst()
+        process_env = normalize_process_env(event.env)
         if process is None:
             process = await Process.objects.acreate(
                 machine=iface.machine,
@@ -82,7 +104,7 @@ class ProcessService(BaseService):
                 worker_type=worker_type,
                 pwd=event.output_dir,
                 cmd=[event.hook_path, *event.hook_args],
-                env=event.env,
+                env=process_env,
                 timeout=event.timeout,
                 pid=event.pid or None,
                 url=event.url or None,
@@ -97,7 +119,7 @@ class ProcessService(BaseService):
 
         process.pwd = event.output_dir
         process.cmd = [event.hook_path, *event.hook_args]
-        process.env = event.env
+        process.env = process_env
         process.timeout = event.timeout
         process.pid = event.pid or None
         process.url = event.url or process.url
@@ -172,6 +194,7 @@ class ProcessService(BaseService):
                 started_at=started_at,
             )
         process = await process_query.order_by("-modified_at").afirst()
+        process_env = normalize_process_env(event.env)
         if process is None:
             await Process.objects.acreate(
                 machine=iface.machine,
@@ -180,7 +203,7 @@ class ProcessService(BaseService):
                 worker_type=worker_type,
                 pwd=event.output_dir,
                 cmd=[event.hook_path, *event.hook_args],
-                env=event.env,
+                env=process_env,
                 timeout=event.timeout,
                 pid=event.pid or None,
                 url=event.url or None,
@@ -197,6 +220,7 @@ class ProcessService(BaseService):
             "machine_id": iface.machine_id,
             "iface_id": iface.id,
             "pwd": event.output_dir,
+            "env": process_env,
             "pid": event.pid or process.pid,
             "url": event.url or process.url,
             "process_type": process_type or process.process_type,

archivebox/services/runner.py

@@ -55,18 +55,23 @@ from abxbus import BaseEvent
 from abxbus.event_bus import EventBus, get_current_event, in_handler_context
 from abxbus.event_handler import EventHandlerAbortedError, EventHandlerCancelledError
 
-from archivebox.config.common import ArchiveBoxBaseConfig, normalize_runtime_config
+from archivebox.config.common import (
+    ArchiveBoxBaseConfig,
+    normalize_runtime_config,
+    _plugin_enabled_config_keys,
+)
 from archivebox.misc.db import run_db_analyze_batch
 from archivebox.core.shutdown_util import foreground_shutdown_signals, raise_if_shutdown_requested
 from archivebox.search.sonic_daemon import register_sonic_daemon_event_handler
 from archivebox.workers.models import ACTIVE_STATE_LEASE_SECONDS
+from archivebox.crawls.locks import crawl_lifecycle_lock
 
 from .archive_result_service import ArchiveResultService
 from .binary_service import ArchiveBoxBinaryService, ArchiveBoxDBBinaryCacheBackend
 from .crawl_service import CrawlService
 from .machine_service import MachineService
 from .process_service import ProcessService as PersistedProcessService
-from .snapshot_service import SnapshotService, finalize_completed_snapshot
+from .snapshot_service import SnapshotService, finalize_completed_snapshot, project_discovered_snapshots
 from .tag_service import TagService
 
 
@@ -121,6 +126,10 @@ def _count_selected_hooks(plugins: dict[str, Plugin], selected_plugins: list[str
     return sum(1 for plugin in selected.values() for hook in plugin.hooks if "CrawlSetup" in hook.name or "Snapshot" in hook.name)
 
 
+def _is_nonfatal_setup_hook(plugin_name: str, hook_name: str) -> bool:
+    return plugin_name == "chrome" and hook_name.endswith("_chrome_kill_zombies")
+
+
 def _discover_archivebox_plugins() -> dict[str, Plugin]:
     return discover_plugins(runtime="archivebox")
 
@@ -309,23 +318,20 @@ class CrawlRunner:
 
     @property
     def allow_maintenance_on_inactive_crawl(self) -> bool:
-        """Run the requested hooks on a snapshot whose parent crawl is paused or sealed.
+        """Run targeted search indexing on an already-sealed snapshot.
 
-        Maintenance entry paths — direct ``snapshot_ids + selected_plugins`` invocations
-        for search backend backfill, fs migration, plugin-targeted updates — are
-        legitimately allowed to operate on finished/paused crawls. Without this gate,
-        ``crawl_is_cancelled`` would treat a SEALED parent as a cancellation signal
-        and short-circuit every guard before any hook ran, leaving the queued
-        ArchiveResult rows stuck and the orchestrator looping on them.
+        This is the singular hook-execution exception to the unified lifecycle.
+        All other work must queue the Snapshot and Crawl normally.
         """
-        return bool(self.initial_snapshot_ids and self.selected_plugins)
+        return bool(
+            self.initial_snapshot_ids
+            and self.selected_plugins
+            and self.crawl.status == self.crawl.StatusChoices.SEALED
+            and all(plugin.startswith("search_backend_") for plugin in self.selected_plugins),
+        )
 
     async def run(self) -> None:
-        heartbeat = CrawlHeartbeat(
-            Path(self.crawl_output_dir),
-            runtime="archivebox",
-            crawl_id=str(self.crawl.id),
-        )
+        heartbeat: CrawlHeartbeat | None = None
         root_snapshot_id: str | None = None
         bus_destroyed = False
         try:
@@ -348,6 +354,11 @@ class CrawlRunner:
                 raise_on_first_signal=False,
             ):
                 snapshot_ids = await sync_to_async(self.load_run_state, thread_sensitive=True)()
+                heartbeat = CrawlHeartbeat(
+                    Path(self.crawl_output_dir),
+                    runtime="archivebox",
+                    crawl_id=str(self.crawl.id),
+                )
                 max_concurrent_snapshots = max(1, int(self.base_config.get("CRAWL_MAX_CONCURRENT_SNAPSHOTS", 1)))
                 self.max_concurrent_snapshots = max_concurrent_snapshots
                 self.snapshot_semaphore = asyncio.Semaphore(max_concurrent_snapshots)
@@ -365,7 +376,8 @@ class CrawlRunner:
                             await self.run_crawl(root_snapshot_id, snapshot_ids)
                     finally:
                         self._run_task = None
-                        await heartbeat.stop()
+                        if heartbeat is not None:
+                            await heartbeat.stop()
                         await self.stop_snapshot_tasks()
                         try:
                             await self.bus.wait_until_idle(timeout=1.0 if self._skip_wait_until_idle else 30.0)
@@ -377,7 +389,8 @@ class CrawlRunner:
         finally:
             if not bus_destroyed:
                 self._run_task = None
-                await heartbeat.stop()
+                if heartbeat is not None:
+                    await heartbeat.stop()
                 await self.stop_snapshot_tasks()
                 await self.bus.destroy(clear=False)
             if self._live_stream is not None:
@@ -586,13 +599,12 @@ class CrawlRunner:
                     hook.parent.name for event_name in runtime_events for hook in discover_hooks(event_name, config=self.base_config)
                 }
                 self.selected_plugins = sorted(runtime_plugins) or None
-        if self.initial_snapshot_ids:
-            # Direct snapshot maintenance paths are allowed to name paused
-            # snapshots explicitly. The runner still requires selected_plugins
-            # later, so this does not restart the crawl lifecycle.
-            return [str(snapshot_id) for snapshot_id in self.initial_snapshot_ids]
         if self.crawl.is_paused:
             return []
+        if self.initial_snapshot_ids:
+            # Explicit ids select normal runnable work, except for the one
+            # sealed-search backfill admitted by allow_maintenance_on_inactive_crawl.
+            return [str(snapshot_id) for snapshot_id in self.initial_snapshot_ids]
         pending_snapshots = list(
             self.crawl.snapshot_set.filter(status__in=Snapshot.RUNNABLE_STATES)
             .filter(retry_at__lte=timezone.now())
@@ -800,6 +812,11 @@ class CrawlRunner:
             },
         )
         normalized_config = normalize_runtime_config(config)
+        configured_plugins = [name.strip().lower() for name in str(normalized_config.get("PLUGINS") or "").split(",") if name.strip()]
+        if configured_plugins:
+            selected_plugin_names = set(filter_plugins(self.plugins, configured_plugins, include_providers=True))
+            for plugin_name, enabled_key in _plugin_enabled_config_keys().items():
+                normalized_config.setdefault(enabled_key, plugin_name in selected_plugin_names)
         return {
             "id": str(snapshot.id),
             "url": snapshot.url,
@@ -816,55 +833,7 @@ class CrawlRunner:
         }
 
     async def enqueue_discovered_snapshots_from_outputs(self, snapshot_payload: dict[str, Any]) -> None:
-        from archivebox.core.models import Snapshot
-        from archivebox.config.common import get_config
-        from archivebox.plugins.hooks import collect_urls_from_plugins
-
-        await sync_to_async(self.crawl.refresh_from_db, thread_sensitive=True)()
-        if self.crawl.is_paused and not self.allow_maintenance_on_inactive_crawl:
-            return
-        if int(snapshot_payload["depth"]) >= self.crawl.max_depth:
-            return
-
-        discovered_urls = await sync_to_async(collect_urls_from_plugins, thread_sensitive=True)(Path(snapshot_payload["output_dir"]))
-        if not discovered_urls:
-            return
-
-        if self.crawl.status == self.crawl.StatusChoices.SEALED:
-            # Snapshot completion projectors can observe the root snapshot seal
-            # before the runner has consumed parser urls.jsonl output. A sealed
-            # crawl must not block those freshly discovered child snapshots; the
-            # runner is still inside the same crawl lifecycle and will seal it
-            # again after the discovered queue is empty.
-            await sync_to_async(self.crawl.update_and_requeue, thread_sensitive=True)(
-                status=self.crawl.StatusChoices.STARTED,
-                retry_at=timezone.now(),
-            )
-
-        parent_snapshot = await sync_to_async(
-            lambda: Snapshot.objects.select_related("crawl", "crawl__created_by").filter(id=snapshot_payload["id"]).first(),
-            thread_sensitive=True,
-        )()
-        if parent_snapshot is None:
-            return
-        config = await sync_to_async(
-            lambda: get_config(crawl=self.crawl, snapshot=parent_snapshot).for_crawl_runtime(
-                crawl=self.crawl,
-                snapshot=parent_snapshot,
-                persona=self.crawl.resolve_persona(),
-                crawl_output_dir=self.crawl.output_dir,
-                snapshot_output_dir=parent_snapshot.output_dir,
-            ),
-            thread_sensitive=True,
-        )()
-        if CrawlLimitState.from_config(config).get_stop_reason() in ("crawl_max_size", "crawl_timeout"):
-            return
-
-        await sync_to_async(self.crawl.create_discovered_snapshots, thread_sensitive=True)(
-            parent_snapshot,
-            discovered_urls,
-            depth=parent_snapshot.depth + 1,
-        )
+        await sync_to_async(project_discovered_snapshots, thread_sensitive=True)(snapshot_payload["id"])
         if self.process_discovered_snapshots_inline and isinstance(get_current_event(), CrawlStartEvent):
             await self.enqueue_pending_snapshots_from_projection()
 
@@ -1081,6 +1050,8 @@ class CrawlRunner:
             await completed_process.wait(timeout=crawl_setup_phase_timeout)
             await completed_process.event_results_list()
             if completed_process.status == "failed":
+                if _is_nonfatal_setup_hook(plugin.name, hook.name):
+                    continue
                 raise RuntimeError(f"Crawl setup hook {plugin.name}:{hook.name} failed")
 
     async def run_snapshot(self, snapshot_id: str, crawl_start_event: CrawlStartEvent | None = None) -> None:
@@ -1130,7 +1101,14 @@ class CrawlRunner:
                     )(snapshot["id"])
                     if queued_plugins:
                         if snapshot_selected_plugins:
+                            original_queued_plugins = queued_plugins
                             queued_plugins = queued_plugins_selected_by_config(queued_plugins)
+                            disabled_queued_plugins = sorted(set(original_queued_plugins) - set(queued_plugins))
+                            if disabled_queued_plugins:
+                                await sync_to_async(skip_disabled_queued_plugins, thread_sensitive=True)(
+                                    snapshot["id"],
+                                    disabled_queued_plugins,
+                                )
                             selected_hooks_by_plugin = {
                                 plugin: hooks for plugin, hooks in (selected_hooks_by_plugin or {}).items() if plugin in queued_plugins
                             }
@@ -1142,7 +1120,14 @@ class CrawlRunner:
                 )(snapshot["id"])
                 if queued_plugins:
                     if snapshot_selected_plugins:
+                        original_queued_plugins = queued_plugins
                         queued_plugins = queued_plugins_selected_by_config(queued_plugins)
+                        disabled_queued_plugins = sorted(set(original_queued_plugins) - set(queued_plugins))
+                        if disabled_queued_plugins:
+                            await sync_to_async(skip_disabled_queued_plugins, thread_sensitive=True)(
+                                snapshot["id"],
+                                disabled_queued_plugins,
+                            )
                         selected_hooks_by_plugin = {
                             plugin: hooks for plugin, hooks in (selected_hooks_by_plugin or {}).items() if plugin in queued_plugins
                         }
@@ -1173,7 +1158,13 @@ class CrawlRunner:
                 if snapshot_selected_plugins and remaining_queued_plugins:
                     remaining_queued_plugins = queued_plugins_selected_by_config(remaining_queued_plugins)
                 if not remaining_queued_plugins:
-                    await sync_to_async(run_snapshot_maintenance, thread_sensitive=True)(snapshot_id, output_dir=output_dir)
+                    if snapshot["status"] in ("queued", "started"):
+                        await sync_to_async(finalize_completed_snapshot, thread_sensitive=True)(
+                            snapshot_id,
+                            output_dir=output_dir,
+                        )
+                    else:
+                        await sync_to_async(run_snapshot_maintenance, thread_sensitive=True)(snapshot_id, output_dir=output_dir)
                     return
                 snapshot_selected_plugins = remaining_queued_plugins
                 plugins = filter_plugins(self.plugins, snapshot_selected_plugins, include_providers=True)
@@ -1296,6 +1287,30 @@ def run_crawl(
     interactive_interrupts: bool = False,
     config_overrides: dict[str, Any] | None = None,
     selected_plugins_are_explicit: bool = True,
+) -> None:
+    with crawl_lifecycle_lock(crawl_id):
+        _run_crawl_locked(
+            crawl_id,
+            snapshot_ids=snapshot_ids,
+            selected_plugins=selected_plugins,
+            process_discovered_snapshots_inline=process_discovered_snapshots_inline,
+            show_progress=show_progress,
+            interactive_interrupts=interactive_interrupts,
+            config_overrides=config_overrides,
+            selected_plugins_are_explicit=selected_plugins_are_explicit,
+        )
+
+
+def _run_crawl_locked(
+    crawl_id: str,
+    *,
+    snapshot_ids: list[str] | None = None,
+    selected_plugins: list[str] | None = None,
+    process_discovered_snapshots_inline: bool = True,
+    show_progress: bool = True,
+    interactive_interrupts: bool = False,
+    config_overrides: dict[str, Any] | None = None,
+    selected_plugins_are_explicit: bool = True,
 ) -> None:
     from archivebox.crawls.models import Crawl
     from django.db import close_old_connections
@@ -1437,6 +1452,11 @@ def queued_plugins_for_snapshot(snapshot_id: str) -> list[str] | None:
 def config_overrides_for_queued_plugins(selected_plugins: list[str], **overrides: Any) -> dict[str, Any]:
     config_overrides = dict(overrides)
     config_overrides["PLUGINS"] = ",".join(selected_plugins)
+    selected_plugin_names = set(
+        filter_plugins(_discover_archivebox_plugins(), [plugin_name.lower() for plugin_name in selected_plugins], include_providers=True),
+    )
+    for plugin_name, enabled_key in _plugin_enabled_config_keys().items():
+        config_overrides[enabled_key] = plugin_name in selected_plugin_names
     for plugin_name in selected_plugins:
         if plugin_name.startswith("search_backend_"):
             config_overrides[f"{plugin_name.upper()}_ENABLED"] = True
@@ -1479,6 +1499,28 @@ def fail_unavailable_queued_hooks(
         )
 
 
+def skip_disabled_queued_plugins(snapshot_id: str, plugin_names: list[str]) -> None:
+    from archivebox.core.models import ArchiveResult
+
+    if not plugin_names:
+        return
+    now = timezone.now()
+    # Queued ArchiveResult rows are durable scheduler state and can outlive a
+    # config change or deploy. If a plugin is no longer selected for this
+    # Snapshot/Crawl, leaving its old row queued keeps the Snapshot STARTED
+    # forever even though there is no runnable work left.
+    ArchiveResult.objects.filter(
+        snapshot_id=snapshot_id,
+        plugin__in=plugin_names,
+        status=ArchiveResult.StatusChoices.QUEUED,
+    ).update(
+        status=ArchiveResult.StatusChoices.SKIPPED,
+        start_ts=now,
+        end_ts=now,
+        output_str="Queued plugin is disabled by this Snapshot/Crawl config",
+    )
+
+
 def include_background_prerequisite_hooks(
     selected_hooks_by_plugin: dict[str, set[str] | None],
     plugins: dict[str, Plugin],
@@ -1511,12 +1553,13 @@ def include_background_prerequisite_hooks(
 def snapshot_hooks_for_pending_archiveresults(snapshot) -> list[tuple[str, str]]:
     from archivebox.config.common import get_config
     from archivebox.core.models import Snapshot
+    from archivebox.plugins.discovery import get_enabled_plugins
 
     config = get_config(crawl=snapshot.crawl, snapshot=snapshot)
     snapshot_plugin_names = [name.strip() for name in str((snapshot.config or {}).get("PLUGINS") or "").split(",") if name.strip()]
     crawl_plugin_names = [name.strip() for name in str((snapshot.crawl.config or {}).get("PLUGINS") or "").split(",") if name.strip()]
     config_plugin_names = [name.strip() for name in str(config.PLUGINS or "").split(",") if name.strip()]
-    plugin_names = snapshot_plugin_names or crawl_plugin_names or config_plugin_names
+    plugin_names = snapshot_plugin_names or crawl_plugin_names or config_plugin_names or get_enabled_plugins(config=config)
     plugins = (
         filter_plugins(_discover_archivebox_plugins(), plugin_names, include_providers=True)
         if plugin_names
@@ -1540,10 +1583,8 @@ def run_snapshot_maintenance(snapshot_id: str, *, output_dir: Path | None = None
     # queued ArchiveResult rows, so run it whenever this helper is called.
     # The only thing queued rows change is the next scheduler value:
     # - no queued rows left: clear retry_at because maintenance is done
-    # - queued rows remain: leave the Snapshot due so the sealed/paused runner
-    #   branch can process those targeted plugin rows on the next tick
-    # This avoids reopening final/paused snapshots while also avoiding stranded
-    # queued ArchiveResults that have no independent scheduler.
+    # - queued rows remain on a sealed Snapshot: leave it due so the search
+    #   backfill exception can process them on the next tick
     current_retry_at = snapshot.retry_at
     next_retry_at = timezone.now() if has_queued_results else None
     snapshot.retry_at = next_retry_at
@@ -1565,6 +1606,15 @@ def run_snapshot_maintenance(snapshot_id: str, *, output_dir: Path | None = None
 
 
 def run_due_crawl(crawl, *, lock_seconds: int, interactive_interrupts: bool = False) -> bool:
+    with crawl_lifecycle_lock(str(crawl.id)):
+        return _run_due_crawl_locked(
+            crawl,
+            lock_seconds=lock_seconds,
+            interactive_interrupts=interactive_interrupts,
+        )
+
+
+def _run_due_crawl_locked(crawl, *, lock_seconds: int, interactive_interrupts: bool = False) -> bool:
     try:
         crawl.refresh_from_db(fields=["status", "retry_at", "modified_at"])
     except type(crawl).DoesNotExist:
@@ -1657,6 +1707,16 @@ def run_due_crawl(crawl, *, lock_seconds: int, interactive_interrupts: bool = Fa
 
 
 def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: bool = False, runtime_config=None) -> bool:
+    with crawl_lifecycle_lock(str(snapshot.crawl_id)):
+        return _run_due_snapshot_locked(
+            snapshot,
+            lock_seconds=lock_seconds,
+            interactive_interrupts=interactive_interrupts,
+            runtime_config=runtime_config,
+        )
+
+
+def _run_due_snapshot_locked(snapshot, *, lock_seconds: int, interactive_interrupts: bool = False, runtime_config=None) -> bool:
     from archivebox.core.models import Snapshot
 
     try:
@@ -1668,62 +1728,12 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo
         return parent_reconciled
 
     if snapshot.is_paused:
-        selected_plugins = queued_plugins_for_snapshot(str(snapshot.id))
-        if snapshot.fs_migration_needed and Snapshot.claim_for_worker(snapshot, lock_seconds=lock_seconds):
-            _runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot)
-            run_snapshot_maintenance(str(snapshot.id))
-            if not selected_plugins:
-                # No targeted plugin rows remain, so put paused snapshots back
-                # behind the indefinite retry_at marker. If queued plugin rows
-                # remain, continue into the targeted plugin path below and let
-                # its finally block restore the paused marker after completion.
-                snapshot.restore_paused_scheduler_marker()
-                return True
-            snapshot.refresh_from_db()
-        if not selected_plugins:
-            # Paused is a real lifecycle state; retry_at=MAX is only the
-            # orchestrator selection marker. If a direct maintenance/update
-            # command bumps retry_at on a paused snapshot but there are no
-            # targeted ArchiveResult rows to run, restore the scheduler marker
-            # without changing status.
-            snapshot.restore_paused_scheduler_marker()
-            return True
-        if not Snapshot.claim_for_worker(snapshot, lock_seconds=lock_seconds):
-            return False
-        try:
-            _runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot)
-            # Explicit maintenance, e.g. `archivebox update --index-only`, may
-            # need to run search/index hooks for a paused snapshot. That should
-            # not resume the crawl or make unrelated queued work runnable. The
-            # queued ArchiveResult rows are the durable maintenance request, so
-            # run that exact plugin set even when the paused crawl's normal
-            # PLUGINS config names a different extractor surface.
-            run_crawl(
-                str(snapshot.crawl_id),
-                snapshot_ids=[str(snapshot.id)],
-                selected_plugins=selected_plugins,
-                process_discovered_snapshots_inline=True,
-                interactive_interrupts=interactive_interrupts,
-                config_overrides=config_overrides_for_queued_plugins(selected_plugins),
-                selected_plugins_are_explicit=False,
-            )
-        finally:
-            # Targeted plugin rows can complete while the Snapshot remains
-            # paused. Put retry_at back at MAX only after the queued rows are
-            # gone; if a hook was interrupted before projection, keep the
-            # paused row due so the next runner can retry that targeted work
-            # without a user-visible resume transition.
-            if queued_plugins_for_snapshot(str(snapshot.id)):
-                now = timezone.now()
-                type(snapshot).objects.filter(
-                    pk=snapshot.pk,
-                    status=snapshot.StatusChoices.PAUSED,
-                ).update(
-                    retry_at=now,
-                    modified_at=now,
-                )
-            else:
-                snapshot.restore_paused_scheduler_marker()
+        # Paused work never executes out of band. Preserve the lifecycle marker
+        # until an explicit resume moves it through the normal state machine.
+        from archivebox.core.models import ArchiveResult
+
+        ArchiveResult.pause_queryset(snapshot.archiveresult_set.all())
+        snapshot.restore_paused_scheduler_marker()
         return True
     if snapshot.status == Snapshot.StatusChoices.SEALED:
         if not Snapshot.claim_for_worker(snapshot, lock_seconds=lock_seconds):
@@ -1742,17 +1752,24 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo
         selected_plugins = queued_plugins_for_snapshot(str(snapshot.id))
         if selected_plugins:
             search_only_plugins = all(plugin.startswith("search_backend_") for plugin in selected_plugins)
-            _runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot)
-            run_crawl(
-                str(snapshot.crawl_id),
-                snapshot_ids=[str(snapshot.id)],
-                selected_plugins=selected_plugins,
-                process_discovered_snapshots_inline=True,
-                interactive_interrupts=interactive_interrupts,
-                config_overrides=config_overrides_for_queued_plugins(selected_plugins),
-                selected_plugins_are_explicit=False,
-            )
-            if search_only_plugins:
+            if not search_only_plugins:
+                snapshot.update_and_requeue(
+                    status=Snapshot.StatusChoices.QUEUED,
+                    retry_at=timezone.now(),
+                    current_step=0,
+                )
+                snapshot.refresh_from_db()
+            else:
+                _runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot)
+                run_crawl(
+                    str(snapshot.crawl_id),
+                    snapshot_ids=[str(snapshot.id)],
+                    selected_plugins=selected_plugins,
+                    process_discovered_snapshots_inline=True,
+                    interactive_interrupts=interactive_interrupts,
+                    config_overrides=config_overrides_for_queued_plugins(selected_plugins),
+                    selected_plugins_are_explicit=False,
+                )
                 from archivebox.core.models import ArchiveResult
 
                 has_queued_results = ArchiveResult.objects.filter(
@@ -1775,10 +1792,11 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo
                         retry_at=timezone.now(),
                         modified_at=timezone.now(),
                     )
-            return True
-        if maintenance_ran:
-            return True
-        return run_snapshot_maintenance(str(snapshot.id))
+                return True
+        if snapshot.status == Snapshot.StatusChoices.SEALED:
+            if maintenance_ran:
+                return True
+            return run_snapshot_maintenance(str(snapshot.id))
 
     if snapshot.status == Snapshot.StatusChoices.STARTED:
         _reset_count, running_count = snapshot.reset_abandoned_results()
@@ -1790,28 +1808,40 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo
         return False
     snapshot.refresh_from_db()
     if snapshot.status == Snapshot.StatusChoices.QUEUED:
-        if snapshot.archiveresult_set.exists() and snapshot.is_finished_processing():
-            snapshot.sm.tick()
+        has_archiveresults = snapshot.archiveresult_set.exists()
+        if has_archiveresults and snapshot.is_finished_processing():
+            finalize_completed_snapshot(str(snapshot.id), output_dir=Path(snapshot.output_dir))
             snapshot.refresh_from_db()
             if snapshot.status == Snapshot.StatusChoices.SEALED:
                 _runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot, status="SEALED")
                 return True
-        # The runner owns queued Snapshot setup. Create missing enabled hook
-        # rows before ticking so maintenance-only final rows, e.g. search
-        # backfill on a paused snapshot, cannot make queued -> sealed skip the
-        # real extraction work after resume.
-        snapshot.create_pending_archiveresults(hooks=snapshot_hooks_for_pending_archiveresults(snapshot))
+        # A Snapshot with no hook rows is fresh lifecycle work; materialize its
+        # configured hook set. Existing rows are already the durable requested
+        # work set and must not be broadened during retry/recovery.
+        if not has_archiveresults:
+            snapshot.create_pending_archiveresults(hooks=snapshot_hooks_for_pending_archiveresults(snapshot))
         snapshot.sm.tick()
         snapshot.refresh_from_db()
         if snapshot.status == Snapshot.StatusChoices.SEALED:
             _runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot, status="SEALED")
             return True
     if snapshot.status == Snapshot.StatusChoices.STARTED and snapshot.archiveresult_set.exists() and snapshot.is_finished_processing():
-        snapshot.sm.tick()
+        finalize_completed_snapshot(str(snapshot.id), output_dir=Path(snapshot.output_dir))
         snapshot.refresh_from_db()
         if snapshot.status == Snapshot.StatusChoices.SEALED:
             _runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot, status="SEALED")
             return True
+    if snapshot.status == Snapshot.StatusChoices.STARTED:
+        queued_plugins, selected_hooks_by_plugin = queued_plugins_and_hooks_for_snapshot(str(snapshot.id))
+        if queued_plugins and selected_hooks_by_plugin:
+            fail_unavailable_queued_hooks(
+                str(snapshot.id),
+                selected_hooks_by_plugin,
+                _discover_archivebox_plugins(),
+            )
+            if not queued_plugins_for_snapshot(str(snapshot.id)):
+                finalize_completed_snapshot(str(snapshot.id), output_dir=Path(snapshot.output_dir))
+                return True
     _runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot)
     run_crawl(
         str(snapshot.crawl_id),
@@ -1839,15 +1869,15 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo
 
 
 def run_due_binary(binary, *, lock_seconds: int) -> bool:
-    binary_name = str(binary.name or "")
-    binary_path = Path(binary_name).expanduser()
-    if (binary_path.is_absolute() or binary_name.startswith("~")) and not binary_path.exists():
-        binary.retry_at = None
-        binary.save(update_fields=["retry_at", "modified_at"])
-        return True
-    if not binary.claim_processing_lock(lock_seconds=lock_seconds):
-        return False
-    run_binary(str(binary.id))
+    from archivebox.crawls.locks import binary_lifecycle_lock
+
+    with binary_lifecycle_lock(str(binary.id)):
+        binary.refresh_from_db()
+        if binary.status == binary.StatusChoices.INSTALLED:
+            return True
+        if not binary.claim_processing_lock(lock_seconds=lock_seconds):
+            return False
+        run_binary(str(binary.id))
     return True
 
 
@@ -2047,7 +2077,7 @@ def _run_due_queued_plugin_result(
             status=ArchiveResult.StatusChoices.QUEUED,
             plugin__in=plugin_names,
             snapshot__retry_at__lte=now,
-            snapshot__status__in=(Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED),
+            snapshot__status=Snapshot.StatusChoices.SEALED,
         )
         .filter(**({"snapshot__crawl_id": crawl_id} if crawl_id else {}))
         .values("snapshot_id", "snapshot__crawl_id")[:1]
@@ -2057,15 +2087,6 @@ def _run_due_queued_plugin_result(
         return False
     root_crawl_id = str(first_due_results[0]["snapshot__crawl_id"])
 
-    first_due_snapshot = Snapshot.objects.filter(pk=first_due_results[0]["snapshot_id"]).first()
-    if first_due_snapshot and first_due_snapshot.status == Snapshot.StatusChoices.PAUSED:
-        return run_due_snapshot(
-            first_due_snapshot,
-            lock_seconds=lock_seconds,
-            interactive_interrupts=interactive_interrupts,
-            runtime_config=runtime_config,
-        )
-
     due_snapshots = Snapshot.objects.filter(
         retry_at__lte=now,
         status=Snapshot.StatusChoices.SEALED,
@@ -2219,18 +2240,11 @@ def run_pending_crawls(
     from archivebox.config.common import get_config
     from archivebox.crawls.models import Crawl, CrawlSchedule
     from archivebox.core.models import ArchiveResult, Snapshot
-    from archivebox.plugins.discovery import discover_plugin_configs
     from archivebox.plugins.hooks import discover_hooks
     from archivebox.machine.models import Process
 
     crawl_claim_lock_seconds = 10
     runtime_config = get_config()
-    plugin_configs = discover_plugin_configs()
-    download_plugin_names = frozenset(
-        plugin_name
-        for plugin_name, plugin_config in plugin_configs.items()
-        if plugin_config.get("output_mimetypes") and not plugin_name.startswith("search_backend_")
-    )
     last_recovery_at = 0.0
     last_retention_at = 0.0
     last_retention_repair_at = 0.0
@@ -2258,19 +2272,6 @@ def run_pending_crawls(
                 if schedule.is_due(now):
                     schedule.enqueue(queued_at=now)
 
-        # Final-state download rows are always first: they have no parent crawl
-        # scheduler of their own, and leaving them behind makes the global
-        # counters report stale queued work while new crawls continue.
-        if _run_due_queued_plugin_result(
-            download_plugin_names,
-            crawl_id=crawl_id,
-            lock_seconds=60,
-            interactive_interrupts=interactive_interrupts,
-            runtime_config=runtime_config,
-            batch_size=maintenance_batch_size,
-        ):
-            continue
-
         if _fast_forward_same_path_snapshot_fs_versions():
             continue
 
@@ -2346,21 +2347,20 @@ def run_pending_crawls(
         # Final active-state fallback uses only the retry_at scheduler index and
         # selects an id first. Keep final SEALED rows out of this broad path so
         # large filesystem/index backfills cannot starve newly queued crawls.
-        due_snapshots = Snapshot.objects.filter(
-            retry_at__lte=timezone.now(),
-            status__in=Snapshot.OPEN_STATES,
-        )
-        if maintenance_only:
-            due_snapshots = due_snapshots.filter(status=Snapshot.StatusChoices.PAUSED)
-        if crawl_id:
-            due_snapshots = due_snapshots.filter(crawl_id=crawl_id)
-        if _run_due_snapshot_query(
-            due_snapshots,
-            lock_seconds=60,
-            interactive_interrupts=interactive_interrupts,
-            runtime_config=runtime_config,
-        ):
-            continue
+        if not maintenance_only:
+            due_snapshots = Snapshot.objects.filter(
+                retry_at__lte=timezone.now(),
+                status__in=Snapshot.OPEN_STATES,
+            )
+            if crawl_id:
+                due_snapshots = due_snapshots.filter(crawl_id=crawl_id)
+            if _run_due_snapshot_query(
+                due_snapshots,
+                lock_seconds=60,
+                interactive_interrupts=interactive_interrupts,
+                runtime_config=runtime_config,
+            ):
+                continue
 
         # Search backend selection is live crawl-execution config, not an
         # installed-plugin list. Old queued rows for a backend that is disabled

archivebox/services/snapshot_service.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import sys
+from pathlib import Path
 
 from asgiref.sync import sync_to_async
 from django.utils import timezone
@@ -11,6 +12,35 @@ from abx_dl.limits import CrawlLimitState
 from abx_dl.services.base import BaseService
 
 
+def project_discovered_snapshots(snapshot_id: str) -> list:
+    """Persist parser output before the parent Snapshot can enter a final state."""
+    from archivebox.config.common import get_config
+    from archivebox.core.models import Snapshot
+    from archivebox.plugins.hooks import collect_urls_from_plugins
+
+    snapshot = Snapshot.objects.select_related("crawl", "crawl__created_by", "crawl__persona").filter(id=snapshot_id).first()
+    if snapshot is None:
+        return []
+    crawl = snapshot.crawl
+    if crawl.status not in crawl.RUNNABLE_STATES or crawl.is_paused or snapshot.depth >= crawl.max_depth:
+        return []
+
+    discovered_urls = collect_urls_from_plugins(Path(snapshot.output_dir))
+    if not discovered_urls:
+        return []
+
+    config = get_config(crawl=crawl, snapshot=snapshot).for_crawl_runtime(
+        crawl=crawl,
+        snapshot=snapshot,
+        persona=crawl.resolve_persona(),
+        crawl_output_dir=crawl.output_dir,
+        snapshot_output_dir=snapshot.output_dir,
+    )
+    if CrawlLimitState.from_config(config).get_stop_reason() in ("crawl_max_size", "crawl_timeout"):
+        return []
+    return crawl.create_discovered_snapshots(snapshot, discovered_urls, depth=snapshot.depth + 1)
+
+
 def finalize_completed_snapshot(
     snapshot_id: str,
     *,
@@ -23,6 +53,11 @@ def finalize_completed_snapshot(
     if snapshot is None:
         return
 
+    # urls.jsonl is durable hook output. Project it while the Snapshot/Crawl are
+    # still runnable so an interruption cannot seal a parser root after its
+    # ArchiveResults finish but before its discovered child rows are persisted.
+    project_discovered_snapshots(str(snapshot.id))
+
     if snapshot.downloaded_at is None:
         snapshot.downloaded_at = timezone.now()
         snapshot.save(update_fields=["downloaded_at", "modified_at"])

archivebox/templates/core/add.html

@@ -48,7 +48,9 @@
                 <div class="crawl-explanation">
                     <p class="crawl-tip" style="user-select: none;">
                         <span style="float: right">
-                            <a href="/admin/agent">💬 Crawl with AI</a> &nbsp; | &nbsp;
+                            {% if user.is_authenticated and user.is_superuser and request.archivebox_config.OPENCODE_ENABLED %}
+                                <a href="/admin/agent">💬 Crawl with AI</a> &nbsp; | &nbsp;
+                            {% endif %}
                             <a href="https://github.com/ArchiveBox/archivebox-browser-extension"><img src="{% static 'chrome_extension_icon.png' %}" alt="Chrome Extension" style="height: 33px; margin-top: -5px;">&nbsp; Get the extension</a>
                         </span>
                         💡 <strong>Tip:</strong> Instantly save a single URL by visiting:
@@ -372,7 +374,6 @@
                 if (sameDomainToggleSlot && shortcutToggles.length) {
                     shortcutToggles.forEach((toggle) => sameDomainToggleSlot.appendChild(toggle));
                 }
-                const pluginDependencyMap = JSON.parse('{{ plugin_dependency_map_json|default:"{}"|escapejs }}');
                 const personaConfigMap = JSON.parse('{{ persona_config_map_json|default:"{}"|escapejs }}');
                 const personaSelect = document.querySelector('select[name="persona"]');
 
@@ -404,22 +405,6 @@
                     return document.querySelector(`.plugin-section-toggle[value="${requiredSearchPlugin}"]`);
                 }
 
-                function getPluginCheckbox(pluginName) {
-                    if (!pluginName) return null;
-                    return document.querySelector(`.plugin-section-toggle[value="${pluginName}"]`);
-                }
-
-                function getRequiredPlugins(pluginName) {
-                    const requiredPlugins = pluginDependencyMap[pluginName];
-                    return Array.isArray(requiredPlugins) ? requiredPlugins : [];
-                }
-
-                function getDependentPlugins(pluginName) {
-                    return Object.entries(pluginDependencyMap)
-                        .filter(([, requiredPlugins]) => Array.isArray(requiredPlugins) && requiredPlugins.includes(pluginName))
-                        .map(([dependentPlugin]) => dependentPlugin);
-                }
-
                 function getConfigEditorRows() {
                     return document.getElementById('id_config_rows');
                 }
@@ -555,47 +540,6 @@
                     updateURLPreview();
                 }
 
-                function ensureRequiredPluginsChecked(pluginName, visited = new Set()) {
-                    if (!pluginName || visited.has(pluginName)) {
-                        return;
-                    }
-                    visited.add(pluginName);
-
-                    getRequiredPlugins(pluginName).forEach(requiredPlugin => {
-                        const requiredCheckbox = getPluginCheckbox(requiredPlugin);
-                        if (!requiredCheckbox) {
-                            return;
-                        }
-                        requiredCheckbox.checked = true;
-                        dispatchChange(requiredCheckbox);
-                        ensureRequiredPluginsChecked(requiredPlugin, visited);
-                    });
-                }
-
-                function uncheckDependentPlugins(pluginName, visited = new Set()) {
-                    if (!pluginName || visited.has(pluginName)) {
-                        return;
-                    }
-                    visited.add(pluginName);
-
-                    getDependentPlugins(pluginName).forEach(dependentPlugin => {
-                        const dependentCheckbox = getPluginCheckbox(dependentPlugin);
-                        if (!dependentCheckbox) {
-                            return;
-                        }
-                        dependentCheckbox.checked = false;
-                        dispatchChange(dependentCheckbox);
-                        uncheckDependentPlugins(dependentPlugin, visited);
-                    });
-                }
-
-                function normalizePluginSelections() {
-                    const checkedPlugins = Array.from(document.querySelectorAll('.plugin-section-toggle:checked'))
-                        .map(cb => cb.value)
-                        .filter(Boolean);
-                    checkedPlugins.forEach(pluginName => ensureRequiredPluginsChecked(pluginName));
-                }
-
                 function applyRequiredSearchPlugin(preferredValue = undefined) {
                     const requiredCheckbox = getRequiredSearchCheckbox();
                     if (!requiredCheckbox) return;
@@ -1245,7 +1189,6 @@
                             });
                         }
 
-                        normalizePluginSelections();
                         applyRequiredSearchPlugin(requiredSearchPreference);
                         syncEnabledPluginsConfig();
                         updateChromeToggleButton();
@@ -1279,7 +1222,6 @@
                             dispatchChange(cb);
                         });
 
-                        normalizePluginSelections();
                         applyRequiredSearchPlugin(requiredSearchPreference);
                         syncEnabledPluginsConfig();
                         updateChromeToggleButton();
@@ -1289,11 +1231,6 @@
 
                 document.querySelectorAll('.plugin-section-toggle').forEach(checkbox => {
                     checkbox.addEventListener('change', function() {
-                        if (this.checked) {
-                            ensureRequiredPluginsChecked(this.value);
-                        } else {
-                            uncheckDependentPlugins(this.value);
-                        }
                         syncEnabledPluginsConfig();
                         updateChromeToggleButton();
                         saveFormState();
@@ -1348,7 +1285,6 @@
                         if (personaSelect) {
                             applyPersonaConfig(personaSelect.value);
                         }
-                        normalizePluginSelections();
                         applyRequiredSearchPlugin();
                         syncEnabledPluginsConfig();
                         updateChromeToggleButton();

archivebox/templates/core/navigation.html

@@ -6,7 +6,7 @@
     <a href="{% url 'Home' %}">Snapshots</a> |
     <a href="/admin/core/archiveresult/?o=-1">Log</a> |
     <a href="/admin/core/tag/">Tags</a> &nbsp; &nbsp;
-    {% if user.is_authenticated and user.is_superuser %}
+    {% if user.is_authenticated and user.is_superuser and request.archivebox_config.OPENCODE_ENABLED %}
         <a href="/admin/agent">💬 AI</a> |
     {% endif %}
     <a href="{% url 'Docs' %}" target="_blank" rel="noopener noreferrer">Docs</a> |

archivebox/tests/conftest.py

@@ -228,6 +228,8 @@ def run_archivebox_cmd(
                     "SAVE_HTMLTOTEXT": "False",
                 },
             )
+            if args and args[0] == "version":
+                run_env["PLUGINS"] = "__archivebox_test_no_plugins__"
         if env:
             run_env.update(env)
     _set_test_source_pythonpath(run_env)
@@ -378,18 +380,16 @@ def isolated_data_dir(tmp_path):
 @pytest.fixture
 def hermetic_lib_dir(tmp_path, monkeypatch):
     """
-    Point ABXPKG_LIB_DIR at a tmp directory so the test can write fake binaries
-    without touching the real ``~/Library/Application Support/abx/lib`` (which
-    can contain symlinks to SIP-protected system binaries on macOS).
+    Point ABXPKG_LIB_DIR at a temporary directory for isolated abxpkg resolution.
 
     Opt-in only: most tests should reuse the cached real ABXPKG_LIB_DIR for speed —
     rebuilding from scratch per-test adds ~10× overhead. Use this only when
-    the test synthesizes binary paths or validates ABXPKG_LIB_DIR-relative behavior.
+    validating ABXPKG_LIB_DIR-relative behavior.
     """
     import archivebox.machine.models as machine_models
 
     lib_dir = tmp_path / "lib"
-    (lib_dir / "bin").mkdir(parents=True, exist_ok=True)
+    lib_dir.mkdir(parents=True, exist_ok=True)
     monkeypatch.setenv("ABXPKG_LIB_DIR", str(lib_dir))
     machine_models._CURRENT_MACHINE = None
     machine_models._CURRENT_PROCESS = None
@@ -434,27 +434,45 @@ def recursive_test_site():
         "/about": """
             <html>
               <body>
-                <a href="/deep/about">Deep About</a>
+                <main>
+                  <article>
+                    <h1>About</h1>
+                    <p>This local article describes the ArchiveBox recursive crawl fixture with enough semantic content for real article extractors.</p>
+                    <a href="/deep/about">Deep About</a>
+                  </article>
+                </main>
               </body>
             </html>
         """.strip().encode("utf-8"),
         "/blog": """
             <html>
               <body>
-                <a href="/deep/blog">Deep Blog</a>
+                <main>
+                  <article>
+                    <h1>Blog</h1>
+                    <p>This local article exercises real browser, text extraction, and recursive discovery behavior without depending on a public website.</p>
+                    <a href="/deep/blog">Deep Blog</a>
+                  </article>
+                </main>
               </body>
             </html>
         """.strip().encode("utf-8"),
         "/contact": """
             <html>
               <body>
-                <a href="/deep/contact">Deep Contact</a>
+                <main>
+                  <article>
+                    <h1>Contact</h1>
+                    <p>This local article keeps the deterministic crawl graph content-rich while preserving the original contact and deep-contact URLs.</p>
+                    <a href="/deep/contact">Deep Contact</a>
+                  </article>
+                </main>
               </body>
             </html>
         """.strip().encode("utf-8"),
-        "/deep/about": b"<html><body><h1>Deep About</h1></body></html>",
-        "/deep/blog": b"<html><body><h1>Deep Blog</h1></body></html>",
-        "/deep/contact": b"<html><body><h1>Deep Contact</h1></body></html>",
+        "/deep/about": b"<html><body><main><article><h1>Deep About</h1><p>This depth-two article verifies that recursive snapshots retain extractable semantic content and their exact parent relationship.</p></article></main></body></html>",
+        "/deep/blog": b"<html><body><main><article><h1>Deep Blog</h1><p>This depth-two article provides deterministic local content for every real extraction plugin in the recursive crawl.</p></article></main></body></html>",
+        "/deep/contact": b"<html><body><main><article><h1>Deep Contact</h1><p>This depth-two article completes the fixed local graph without relying on mutable third-party pages or network policy.</p></article></main></body></html>",
         "/favicon.ico": b"test-icon",
     }
 
@@ -557,12 +575,11 @@ def wait_for_process(predicate: Callable[[psutil.Process, str], bool], *, timeou
 
 def pid_is_alive(pid: int) -> bool:
     try:
-        os.kill(pid, 0)
-    except ProcessLookupError:
+        return psutil.Process(pid).status() != psutil.STATUS_ZOMBIE
+    except (psutil.NoSuchProcess, psutil.ZombieProcess):
         return False
-    except PermissionError:
+    except psutil.AccessDenied:
         return True
-    return True
 
 
 def wait_for_pid_to_disappear(pid: int, *, timeout: float = 20.0) -> None:
@@ -920,7 +937,7 @@ def _wait_for_archivebox_workers(cwd: Path, env: dict[str, str], names: tuple[st
     state: dict[str, Any] = {}
     while time.time() < deadline:
         state = _archivebox_worker_state(cwd, env)
-        if all(state.get(name, {}).get("statename") == "RUNNING" for name in names):
+        if all(isinstance(state.get(name), dict) and state[name].get("statename") == "RUNNING" for name in names):
             return state
         time.sleep(1)
     return state
@@ -1414,36 +1431,54 @@ def _get_machine_type() -> str:
     return f"{arch}-{os_name}{suffix}"
 
 
-def _find_cached_chrome(lib_dir: Path) -> Path | None:
-    candidates = [
-        lib_dir / "puppeteer" / "chromium",
-        lib_dir / "puppeteer",
-        lib_dir / "ms-playwright",
-        lib_dir / "pnpm" / "packages" / "chrome" / "node_modules" / "puppeteer" / ".local-chromium",
+def resolve_abxpkg_binary_env(
+    lib_dir: Path,
+    *binary_names: str,
+    env: dict[str, str] | None = None,
+    deps_from: Path | list[Path] | tuple[Path, ...] | None = None,
+    install: bool = True,
+) -> dict[str, str]:
+    """Resolve real test dependencies through abxpkg and return its exported env."""
+    command_env = os.environ.copy()
+    command_env.update(env or {})
+    command_env["ABXPKG_LIB_DIR"] = str(lib_dir)
+    command = [
+        str(Path(sys.executable).with_name("abxpkg")),
+        "env",
+        "--json",
+        f"--lib={lib_dir}",
     ]
-    for base in candidates:
-        if not base.exists():
-            continue
-        for path in base.rglob("Chromium.app/Contents/MacOS/Chromium"):
-            return path
-        for path in base.rglob("chrome-linux/chrome"):
-            return path
-        for path in base.rglob("chrome-linux64/chrome"):
-            return path
-    return None
+    if install:
+        command.append("--install")
+    deps_configs = [deps_from] if isinstance(deps_from, Path) else list(deps_from or ())
+    command.extend(f"--deps-from={config}:required_binaries" for config in deps_configs)
+    command.extend(binary_names)
+    result = subprocess.run(
+        command,
+        capture_output=True,
+        text=True,
+        env=command_env,
+    )
+    assert result.returncode == 0, result.stderr or result.stdout
+    payload = json.loads(result.stdout)
+    return {str(key): str(value) for key, value in payload.items()}
 
 
-def _find_system_browser() -> Path | None:
-    candidates = [
-        Path("/usr/bin/chromium"),
-        Path("/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary"),
-        Path("/Applications/Chromium.app/Contents/MacOS/Chromium"),
-        Path("/usr/bin/chromium-browser"),
-    ]
-    for candidate in candidates:
-        if candidate.exists():
-            return candidate
-    return None
+def resolve_abxpkg_chrome_env(lib_dir: Path, env: dict[str, str] | None = None) -> dict[str, str]:
+    from abx_plugins import get_plugins_dir
+
+    chrome_config = Path(get_plugins_dir()) / "chrome" / "config.json"
+    payload = resolve_abxpkg_binary_env(
+        lib_dir,
+        env=env,
+        deps_from=chrome_config,
+        install=False,
+    )
+    chrome_binary = Path(payload["CHROME_BINARY"])
+    node_binary = Path(payload["NODE_BINARY"])
+    assert chrome_binary.is_file()
+    assert node_binary.is_file()
+    return payload
 
 
 @pytest.fixture(scope="class")
@@ -1484,9 +1519,6 @@ def real_archive_with_example(tmp_path_factory, request):
         "USE_COLOR": "False",
         "RESPONSES_TIMEOUT": "30",
     }
-    system_browser = _find_system_browser()
-    if system_browser:
-        add_env["CHROME_BINARY"] = str(system_browser)
     _cmd_result = run_archivebox_cmd(
         ["add", "--depth=0", "--plugins=responses", "https://example.com"],
         cwd=tmp_path,

archivebox/tests/test_api_archiveresult.py

@@ -1 +0,0 @@
-# Tests moved to test_api_v1_core_archiveresults.py and test_api_v1_core_archiveresult_archiveresult_id.py.

archivebox/tests/test_api_cli.py

@@ -1 +0,0 @@
-# CLI endpoint tests moved to test_api_v1_cli_add.py and test_api_v1_cli_update.py.

archivebox/tests/test_api_cli_schedule.py

@@ -1 +0,0 @@
-# CLI schedule endpoint tests moved to test_api_v1_cli_schedule.py.

archivebox/tests/test_api_crawl.py

@@ -1 +0,0 @@
-# Tests moved to test_api_v1_crawls_crawl_crawl_id.py.

archivebox/tests/test_api_crud.py

@@ -1 +0,0 @@
-# Tests moved to test_api_v1_workflow_core_token_auth_side_effects.py and exact endpoint files under test_api_v1_core_*.

archivebox/tests/test_api_delete_paths.py

@@ -1 +0,0 @@
-# Tests moved to test_api_v1_core_snapshot_snapshot_id.py and test_api_v1_crawls_crawl_crawl_id.py.

archivebox/tests/test_api_personas.py

@@ -1 +0,0 @@
-# Tests moved to test_api_v1_personas_sync.py and test_api_v1_personas_personas.py.

archivebox/tests/test_api_remove.py

@@ -1 +0,0 @@
-# CLI remove endpoint tests moved to test_api_v1_cli_remove.py.

archivebox/tests/test_api_rss.py

@@ -1 +0,0 @@
-# Tests moved to test_api_v1_core_snapshots_rss.py and test_api_v1_crawls_crawl_crawl_id.py.

archivebox/tests/test_api_search.py

@@ -1 +0,0 @@
-# Tests moved to test_api_v1_core_snapshots.py.

archivebox/tests/test_api_v1_cli_add.py

@@ -141,27 +141,12 @@ def write_import_format_files(base_dir: Path) -> dict[str, Path]:
 IMPORT_FORMAT_ENV = {
     "USE_COLOR": "False",
     "SHOW_PROGRESS": "False",
-    "PLUGINS": "parse_html_urls,parse_jsonl_urls,parse_netscape_urls,parse_rss_urls,parse_txt_urls,wget,headers",
-    "SAVE_WGET": "True",
-    "SAVE_HEADERS": "True",
+    "PLUGINS": "parse_html_urls,parse_jsonl_urls,parse_netscape_urls,parse_rss_urls,parse_txt_urls",
     "USE_CHROME": "False",
     "URL_ALLOWLIST": r"example\.com|example\.org|iana\.org|www\.iana\.org",
 }
 
 
-def wait_for_import_processing(cwd: Path, expected_urls: set[str], *, timeout: float = 120.0) -> None:
-    import time
-
-    deadline = time.time() + timeout
-    while time.time() < deadline:
-        with use_archivebox_db(cwd):
-            snapshot_started = Snapshot.objects.filter(url__in=expected_urls).exists()
-        if snapshot_started:
-            return
-        time.sleep(1)
-    raise AssertionError("timed out waiting for import crawl processing to start")
-
-
 def wait_for_expected_import_snapshots(
     cwd: Path,
     expected_urls: set[str],
@@ -374,7 +359,6 @@ def test_api_cli_add_import_text_formats_preserve_metadata_and_crawl_inner_urls(
                 break
             time.sleep(1)
         assert root_counts and all(count == 1 for count in root_counts.values()), root_counts
-        wait_for_import_processing(tmp_path, expected_urls)
         with use_archivebox_db(tmp_path):
             for crawl in Crawl.objects.all():
                 root_snapshot = crawl.snapshot_set.get(url=Snapshot.INTERNAL_INPUT_URL)

archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py

@@ -286,7 +286,7 @@ def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_
         assert output_path.read_text() == "finished result should stay finished"
 
 
-def test_targeted_extract_retries_one_failed_archiveresult_while_snapshot_stays_paused(
+def test_targeted_extract_retries_one_failed_archiveresult_through_normal_snapshot_lifecycle(
     tmp_path,
     client,
     recursive_test_site,
@@ -370,8 +370,10 @@ def test_targeted_extract_retries_one_failed_archiveresult_while_snapshot_stays_
 
     with use_archivebox_db(tmp_path):
         snapshot = Snapshot.objects.get(id=snapshot_id)
-        assert snapshot.status == Snapshot.StatusChoices.PAUSED
-        assert snapshot.retry_at == RETRY_AT_MAX
+        assert snapshot.status == Snapshot.StatusChoices.STARTED
+        assert snapshot.retry_at is not None
+        assert snapshot.retry_at != RETRY_AT_MAX
+        assert snapshot.crawl.status == snapshot.crawl.StatusChoices.STARTED
 
         retried_wget = ArchiveResult.objects.get(id=wget_result.id)
         assert retried_wget.status == ArchiveResult.StatusChoices.SUCCEEDED
@@ -389,54 +391,51 @@ def test_targeted_extract_retries_one_failed_archiveresult_while_snapshot_stays_
 
 
 @pytest.mark.timeout(240)
-def test_paused_snapshot_survives_server_restart_and_resumes_via_api(tmp_path, recursive_test_site):
+def test_paused_snapshot_survives_server_restart_and_resumes_via_api(client, tmp_path, recursive_test_site):
     init_archive(tmp_path)
 
     port = get_free_port()
     env = cli_env(port=port, server=True, PLUGINS="wget", SAVE_WGET="True")
     api_token = create_admin_and_token(tmp_path)
 
-    try:
-        start_archivebox_server(tmp_path, env=env, port=port)
-        wait_for_live_api(port)
-
-        crawl_response = live_api_request(
-            port,
+    with use_archivebox_db(tmp_path):
+        crawl_response = api_client_request(
+            client,
             "post",
             "/api/v1/crawls/crawls",
             api_token=api_token,
-            json={
+            payload={
                 "urls": [recursive_test_site["root_url"]],
                 "max_depth": 0,
                 "tags": ["snapshot-pause-restart-e2e"],
                 "config": {"PLUGINS": "wget", "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*"},
             },
-            timeout=10,
         )
-        assert crawl_response.status_code == 200, crawl_response.text
-        crawl_id = crawl_response.json()["id"]
-        crawl_state = _wait_for_crawl_snapshot_rows(tmp_path, crawl_id)
-        snapshot_id = crawl_state["snapshots"][0]["id"]
-
-        pause_response = live_api_request(
-            port,
+        assert crawl_response.status_code == 200, crawl_response.content.decode()
+        crawl_id = json.loads(crawl_response.content.decode())["id"]
+        crawl = Crawl.objects.get(id=crawl_id)
+        snapshot = Snapshot.objects.create(
+            url=recursive_test_site["root_url"],
+            crawl=crawl,
+            status=Snapshot.StatusChoices.QUEUED,
+            retry_at=timezone.now(),
+        )
+        snapshot_id = str(snapshot.id)
+        pause_response = api_client_request(
+            client,
             "patch",
-            f"/api/v1/crawls/crawl/{crawl_id}",
+            f"/api/v1/core/snapshot/{snapshot_id}",
             api_token=api_token,
-            json={"action": "pause"},
-            timeout=10,
+            payload={"action": "pause"},
         )
-        assert pause_response.status_code == 200, pause_response.text
+        assert pause_response.status_code == 200, pause_response.content.decode()
 
-        current_state = _paused_snapshot_state(tmp_path, snapshot_id)
-        if current_state["status"] == Snapshot.StatusChoices.SEALED:
-            assert current_state["succeeded_results"] > 0
-            return
+    try:
+        start_archivebox_server(tmp_path, env=env, port=port)
+        wait_for_live_api(port)
 
         paused_state = _wait_for_paused_scheduler_marker(tmp_path, snapshot_id)
-        if paused_state["status"] == Snapshot.StatusChoices.SEALED:
-            assert paused_state["succeeded_results"] > 0
-            return
+        assert paused_state["status"] == Snapshot.StatusChoices.PAUSED
         assert paused_state["succeeded_results"] == 0
         assert not list((paused_state["snapshot_dir"] / "wget").rglob("*.html"))
 

archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py

@@ -130,20 +130,42 @@ def wait_for_crawl_wget_success_or_sealed(cwd, crawl_id, timeout=240):
     raise AssertionError(f"timed out waiting for crawl resume completion for crawl {crawl_id}: {latest_state}")
 
 
-def wait_for_sqlite_index_result(cwd, crawl_id, timeout=45):
-    deadline = time.time() + timeout
-    latest_state = None
-    while time.time() < deadline:
-        latest_state = get_crawl_runtime_state(cwd, crawl_id)
-        final_results = [
-            result
-            for result in latest_state["results"]
-            if result["plugin"] == "search_backend_sqlite" and result["status"] not in {"queued", "started", "paused"}
-        ]
-        if final_results:
-            return latest_state
-        time.sleep(0.2)
-    raise AssertionError(f"timed out waiting for sqlite index result for crawl {crawl_id}: {latest_state}")
+def seed_paused_crawl(client, cwd: Path, api_token: str, url: str, tag: str) -> tuple[str, str]:
+    from archivebox.services.runner import run_due_snapshot
+
+    with use_archivebox_db(cwd):
+        response = api_client_request(
+            client,
+            "post",
+            "/api/v1/crawls/crawls",
+            api_token=api_token,
+            payload={
+                "urls": [url],
+                "max_depth": 0,
+                "tags": [tag],
+                "config": {"PLUGINS": "wget", "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*"},
+            },
+        )
+        assert response.status_code == 200, response.content.decode()
+        crawl_id = json.loads(response.content.decode())["id"]
+        crawl = Crawl.objects.get(id=crawl_id)
+        snapshot = Snapshot.objects.create(url=url, crawl=crawl, status=Snapshot.StatusChoices.QUEUED, retry_at=timezone.now())
+        pause_response = api_client_request(
+            client,
+            "patch",
+            f"/api/v1/crawls/crawl/{crawl_id}",
+            api_token=api_token,
+            payload={"action": "pause"},
+        )
+        assert pause_response.status_code == 200, pause_response.content.decode()
+        assert run_due_snapshot(snapshot, lock_seconds=60)
+        crawl.refresh_from_db()
+        snapshot.refresh_from_db()
+        assert crawl.status == Crawl.StatusChoices.PAUSED
+        assert crawl.retry_at == RETRY_AT_MAX
+        assert snapshot.status == Snapshot.StatusChoices.PAUSED
+        assert snapshot.retry_at == RETRY_AT_MAX
+        return str(crawl_id), str(snapshot.id)
 
 
 def make_snapshot(*, user, url: str, title: str, bookmarked_at: datetime):
@@ -341,55 +363,24 @@ def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snap
 
 
 @pytest.mark.timeout(240)
-def test_crawl_pause_resume_api_survives_server_restart_and_processes_after_resume(tmp_path, recursive_test_site):
+def test_crawl_pause_resume_api_survives_server_restart_and_processes_after_resume(client, tmp_path, recursive_test_site):
     init_archive(tmp_path)
 
     port = get_free_port()
     env = cli_env(port=port, server=True, PLUGINS="wget", SAVE_WGET="True")
     api_token = create_admin_and_token(tmp_path)
+    crawl_id, _snapshot_id = seed_paused_crawl(client, tmp_path, api_token, recursive_test_site["root_url"], "pause-resume-e2e")
 
     try:
         start_archivebox_server(tmp_path, env=env, port=port)
         wait_for_live_api(port)
 
-        crawl_response = live_api_request(
-            port,
-            "post",
-            "/api/v1/crawls/crawls",
-            api_token=api_token,
-            json={
-                "urls": [recursive_test_site["root_url"]],
-                "max_depth": 0,
-                "tags": ["pause-resume-e2e"],
-                "config": {"PLUGINS": "wget", "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*"},
-            },
-            timeout=10,
-        )
-        assert crawl_response.status_code == 200, crawl_response.text
-        crawl_id = crawl_response.json()["id"]
-        wait_for_crawl_snapshot_rows(tmp_path, crawl_id)
-
-        pause_response = live_api_request(
-            port,
-            "patch",
-            f"/api/v1/crawls/crawl/{crawl_id}",
-            api_token=api_token,
-            json={"action": "pause"},
-            timeout=10,
-        )
-        assert pause_response.status_code == 200, pause_response.text
-        assert pause_response.json()["status"] == "paused"
-
         paused_state = wait_for_crawl_child_snapshots_paused_or_sealed(tmp_path, crawl_id)
         assert paused_state["crawl_status"] == "paused"
         assert paused_state["crawl_retry_at"] == paused_state["retry_at_max"]
         assert len(paused_state["snapshots"]) == 1
-        snapshot_finished_before_pause = paused_state["snapshots"][0]["status"] == "sealed"
-        if snapshot_finished_before_pause:
-            assert any(result["status"] == "succeeded" for result in paused_state["results"])
-        else:
-            assert paused_state["snapshots"][0]["status"] == "paused"
-            assert paused_state["snapshots"][0]["retry_at"] == paused_state["retry_at_max"]
+        assert paused_state["snapshots"][0]["status"] == "paused"
+        assert paused_state["snapshots"][0]["retry_at"] == paused_state["retry_at_max"]
 
         stop_server(tmp_path)
         start_archivebox_server(tmp_path, env=env, port=port)
@@ -398,10 +389,6 @@ def test_crawl_pause_resume_api_survives_server_restart_and_processes_after_resu
         restarted_state = get_crawl_runtime_state(tmp_path, crawl_id)
         assert restarted_state["crawl_status"] == "paused"
         assert restarted_state["crawl_retry_at"] == restarted_state["retry_at_max"]
-        if snapshot_finished_before_pause:
-            assert restarted_state["snapshots"][0]["status"] == "sealed"
-            assert any(result["status"] == "succeeded" for result in restarted_state["results"])
-            return
         assert restarted_state["snapshots"][0]["status"] == "paused"
         assert restarted_state["snapshots"][0]["retry_at"] == restarted_state["retry_at_max"]
         assert not any(result["status"] == "succeeded" for result in restarted_state["results"])
@@ -431,59 +418,13 @@ def test_crawl_pause_resume_api_survives_server_restart_and_processes_after_resu
 
 
 @pytest.mark.timeout(420)
-def test_update_index_only_runs_paused_search_rows_and_resume_later_runs_crawl(tmp_path, recursive_test_site):
+def test_update_index_only_leaves_paused_snapshot_on_normal_lifecycle_path(client, tmp_path, recursive_test_site):
     init_archive(tmp_path)
 
     port = get_free_port()
     env = cli_env(port=port, server=True, PLUGINS="wget", SAVE_WGET="True")
     api_token = create_admin_and_token(tmp_path)
-
-    try:
-        start_archivebox_server(tmp_path, env=env, port=port)
-        wait_for_live_api(port)
-
-        crawl_response = live_api_request(
-            port,
-            "post",
-            "/api/v1/crawls/crawls",
-            api_token=api_token,
-            json={
-                "urls": [recursive_test_site["root_url"]],
-                "max_depth": 0,
-                "tags": ["paused-index-e2e"],
-                "config": {"PLUGINS": "wget", "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*"},
-            },
-            timeout=10,
-        )
-        assert crawl_response.status_code == 200, crawl_response.text
-        crawl_id = crawl_response.json()["id"]
-        wait_for_crawl_snapshot_rows(tmp_path, crawl_id)
-
-        pause_response = live_api_request(
-            port,
-            "patch",
-            f"/api/v1/crawls/crawl/{crawl_id}",
-            api_token=api_token,
-            json={"action": "pause"},
-            timeout=10,
-        )
-        assert pause_response.status_code == 200, pause_response.text
-        paused_state = wait_for_crawl_child_snapshots_paused_or_sealed(tmp_path, crawl_id)
-        snapshot_finished_before_pause = paused_state["snapshots"][0]["status"] == "sealed"
-    finally:
-        stop_server(tmp_path)
-
-    if snapshot_finished_before_pause:
-        indexed_state = get_crawl_runtime_state(tmp_path, crawl_id)
-        assert indexed_state["crawl_status"] == "paused"
-        assert indexed_state["crawl_retry_at"] == indexed_state["retry_at_max"]
-        assert indexed_state["snapshots"][0]["status"] == "sealed"
-        wget_results = [result for result in indexed_state["results"] if result["plugin"] == "wget"]
-        assert any(result["status"] == "succeeded" and result["output_size"] > 0 for result in wget_results)
-        captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=60)
-        assert "Root" in captured_text
-        assert "About" in captured_text
-        return
+    crawl_id, _snapshot_id = seed_paused_crawl(client, tmp_path, api_token, recursive_test_site["root_url"], "paused-index-e2e")
 
     update_env = cli_env(
         port=port,
@@ -507,14 +448,13 @@ def test_update_index_only_runs_paused_search_rows_and_resume_later_runs_crawl(t
     )
     assert update_process.returncode == 0, update_process.stderr
 
-    indexed_state = wait_for_sqlite_index_result(tmp_path, crawl_id)
+    indexed_state = get_crawl_runtime_state(tmp_path, crawl_id)
     assert indexed_state["crawl_status"] == "paused"
     assert indexed_state["crawl_retry_at"] == indexed_state["retry_at_max"]
     assert indexed_state["snapshots"][0]["status"] == "paused"
     assert indexed_state["snapshots"][0]["retry_at"] == indexed_state["retry_at_max"]
     search_results = [result for result in indexed_state["results"] if result["plugin"] == "search_backend_sqlite"]
-    assert search_results
-    assert any(result["status"] not in {"queued", "started", "paused"} for result in search_results)
+    assert search_results == []
 
     try:
         start_archivebox_server(tmp_path, env=env, port=port)
@@ -540,14 +480,10 @@ def test_update_index_only_runs_paused_search_rows_and_resume_later_runs_crawl(t
 
         assert resumed_state["snapshots"][0]["status"] == "sealed"
         wget_results = [result for result in resumed_state["results"] if result["plugin"] == "wget"]
-        wget_succeeded = any(result["status"] == "succeeded" and result["output_size"] > 0 for result in wget_results)
-        if wget_succeeded:
-            captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=60)
-            assert "Root" in captured_text
-            assert "About" in captured_text
-        else:
-            assert resumed_state["crawl_status"] == "sealed"
-            assert all(result["status"] not in {"queued", "started", "paused"} for result in resumed_state["results"])
+        assert any(result["status"] == "succeeded" and result["output_size"] > 0 for result in wget_results)
+        captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=60)
+        assert "Root" in captured_text
+        assert "About" in captured_text
     finally:
         stop_server(tmp_path)
 

archivebox/tests/test_api_v1_workflow_core_token_auth_side_effects.py

@@ -1,5 +1,3 @@
-import time
-
 import pytest
 import requests
 
@@ -21,17 +19,6 @@ from archivebox.tests.test_orm_helpers import use_archivebox_db
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
-def live_api_request_retrying_sqlite_lock(*args, attempts: int = 12, delay: float = 2.0, **kwargs):
-    response = None
-    for _attempt in range(attempts):
-        response = live_api_request(*args, **kwargs)
-        if response.status_code != 503 or "database is locked" not in response.text:
-            return response
-        time.sleep(delay)
-    assert response is not None
-    return response
-
-
 @pytest.mark.timeout(180)
 def test_core_api_workflow_uses_token_auth_and_persists_side_effects_over_server(tmp_path, recursive_test_site):
     init_archive(tmp_path)
@@ -113,7 +100,7 @@ def test_core_api_workflow_uses_token_auth_and_persists_side_effects_over_server
         assert snapshot_payload["url"] == recursive_test_site["child_urls"][0]
         assert snapshot_payload["tags"] == ["api-child"]
 
-        patch_snapshot = live_api_request_retrying_sqlite_lock(
+        patch_snapshot = live_api_request(
             port,
             "patch",
             f"/api/v1/core/snapshot/{snapshot_id}",
@@ -125,7 +112,7 @@ def test_core_api_workflow_uses_token_auth_and_persists_side_effects_over_server
         assert patch_snapshot.json()["status"] == "sealed"
         assert set(patch_snapshot.json()["tags"]) == {"api-child", "api-patched"}
 
-        tag_create = live_api_request_retrying_sqlite_lock(
+        tag_create = live_api_request(
             port,
             "post",
             "/api/v1/core/tags/create/",
@@ -136,7 +123,7 @@ def test_core_api_workflow_uses_token_auth_and_persists_side_effects_over_server
         assert tag_create.status_code == 200, tag_create.text
         tag_id = tag_create.json()["tag_id"]
 
-        add_tag = live_api_request_retrying_sqlite_lock(
+        add_tag = live_api_request(
             port,
             "post",
             "/api/v1/core/tags/add-to-snapshot/",
@@ -145,7 +132,7 @@ def test_core_api_workflow_uses_token_auth_and_persists_side_effects_over_server
             timeout=10,
         )
         assert add_tag.status_code == 200, add_tag.text
-        remove_tag = live_api_request_retrying_sqlite_lock(
+        remove_tag = live_api_request(
             port,
             "post",
             "/api/v1/core/tags/remove-from-snapshot/",
@@ -155,7 +142,7 @@ def test_core_api_workflow_uses_token_auth_and_persists_side_effects_over_server
         )
         assert remove_tag.status_code == 200, remove_tag.text
 
-        crawl_patch = live_api_request_retrying_sqlite_lock(
+        crawl_patch = live_api_request(
             port,
             "patch",
             f"/api/v1/crawls/crawl/{crawl_id}",

archivebox/tests/test_archiveresult_pause.py

@@ -1,2 +0,0 @@
-# test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snapshot_results_alone moved to test_api_v1_crawls_crawl_crawl_id.py.
-# test_targeted_extract_retries_one_failed_archiveresult_while_snapshot_stays_paused moved to test_api_v1_core_snapshot_snapshot_id.py.

archivebox/tests/test_binary_service.py

@@ -1,11 +1,11 @@
 import json
-import os
-import shutil
+import sys
 import uuid
 import asyncio
 from pathlib import Path
 
 import pytest
+from django.utils import timezone
 
 from archivebox.machine.models import Binary, Machine, Process
 from archivebox.tests.conftest import parse_jsonl_output, run_archivebox_cmd
@@ -14,27 +14,15 @@ from archivebox.tests.test_orm_helpers import use_archivebox_db
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
-def _link_real_binary(bin_dir: Path, name: str, *, source: str | None = None) -> Path:
-    bin_dir.mkdir(parents=True, exist_ok=True)
-    source_path = shutil.which(source or name)
-    assert source_path, f"{source or name} must be installed for this integration test"
-    link = bin_dir / name
-    link.unlink(missing_ok=True)
-    link.symlink_to(source_path)
-    return link
-
-
-def _runtime_env(data_dir: Path, bin_dir: Path, *, lib_dir: Path | None = None) -> dict[str, str]:
-    archivebox_bin = shutil.which("archivebox")
-    assert archivebox_bin, "archivebox console script must be available for CLI tests"
+def _runtime_env(data_dir: Path, *, lib_dir: Path | None = None, **extra: str) -> dict[str, str]:
     lib_dir = lib_dir or data_dir / "lib"
     return {
         "ABXPKG_LIB_DIR": str(lib_dir),
-        "PATH": os.pathsep.join([str(bin_dir), str(Path(archivebox_bin).parent), "/usr/bin", "/bin", "/usr/sbin", "/sbin"]),
+        **extra,
     }
 
 
-def test_binary_request_preserves_raw_overrides_in_db_while_using_native_event():
+def test_binary_request_preserves_raw_overrides_in_db_while_using_native_event(monkeypatch):
     from abxpkg.binary_service import BinaryCacheService, BinaryEvent, BinaryRequestEvent, BinaryService
     from abx_dl.orchestrator import create_bus
     from archivebox.services.binary_service import ArchiveBoxDBBinaryCacheBackend
@@ -46,16 +34,19 @@ def test_binary_request_preserves_raw_overrides_in_db_while_using_native_event()
             "module_name": "imagesize",
         },
     }
+    monkeypatch.setenv("PYTHON3_BINARY", sys.executable)
     binary = Binary.objects.create(
         machine=machine,
-        name="sh",
-        abspath="/bin/sh",
-        version="1.0.0",
-        binprovider="env",
+        name="python3",
         binproviders="env,pip",
         overrides=raw_overrides,
-        status=Binary.StatusChoices.INSTALLED,
+        status=Binary.StatusChoices.QUEUED,
+        retry_at=timezone.now(),
     )
+    assert binary.tick_claimed(lock_seconds=600)
+    binary.refresh_from_db()
+    assert binary.status == Binary.StatusChoices.INSTALLED
+    assert Path(binary.abspath).resolve() == Path(sys.executable).resolve()
     bus = create_bus(name=f"test_binary_raw_overrides_{uuid.uuid4().hex[:8]}")
     BinaryCacheService(bus, backend=ArchiveBoxDBBinaryCacheBackend())
     BinaryService(bus)
@@ -69,7 +60,7 @@ def test_binary_request_preserves_raw_overrides_in_db_while_using_native_event()
     async def run_event() -> None:
         await bus.emit(
             BinaryRequestEvent(
-                name="sh",
+                name="python3",
                 binproviders="env,pip",
                 overrides={"pip": {"install_args": ["imagesize>=2.0.0"]}},
                 extra_context={
@@ -91,18 +82,21 @@ def test_binary_request_preserves_raw_overrides_in_db_while_using_native_event()
 
 
 def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized_archive, tmp_path):
-    name = f"abx-e2e-rg-{uuid.uuid4().hex[:8]}"
-    bootstrap_bin_dir = tmp_path / "realbin"
+    name = "python"
     provider_bin_dir = initialized_archive / "lib" / "env" / "bin"
-    _link_real_binary(bootstrap_bin_dir, "uv")
-    _link_real_binary(provider_bin_dir, name, source="rg")
+    runtime_env = _runtime_env(initialized_archive)
+    request_record = {
+        "type": "BinaryRequest",
+        "name": name,
+        "binproviders": "env",
+    }
 
     _cmd_result = run_archivebox_cmd(
         ["run"],
         cwd=initialized_archive,
-        stdin=json.dumps({"type": "BinaryRequest", "name": name, "binproviders": "env"}) + "\n",
+        stdin=json.dumps(request_record) + "\n",
         timeout=120,
-        env=_runtime_env(initialized_archive, bootstrap_bin_dir),
+        env=runtime_env,
         default_cli_env=True,
         disable_extractors=True,
     )
@@ -125,7 +119,7 @@ def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized
     assert binary.binproviders == "env"
     assert first_abspath.exists()
     assert first_abspath == provider_bin_dir / name
-    assert first_abspath.resolve() == Path(shutil.which("rg") or "").resolve()
+    assert first_abspath.resolve() == Path(sys.executable).resolve()
     assert first_abspath.is_relative_to(initialized_archive / "lib")
     assert (initialized_archive / "lib" / "env" / "bin" / name).exists()
     assert (initialized_archive / "machines" / machine_id / "binaries" / name / "index.jsonl").exists()
@@ -140,7 +134,7 @@ def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized
         ["version"],
         cwd=initialized_archive,
         timeout=60,
-        env=_runtime_env(initialized_archive, bootstrap_bin_dir),
+        env=runtime_env,
         default_cli_env=True,
         disable_extractors=True,
     )
@@ -150,13 +144,12 @@ def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized
     assert binary.version in version_stdout
 
     first_abspath.unlink()
-    _link_real_binary(bootstrap_bin_dir, name, source="rg")
 
     _cmd_result = run_archivebox_cmd(
         ["run", f"--binary-id={first_binary_id}"],
         cwd=initialized_archive,
         timeout=120,
-        env=_runtime_env(initialized_archive, bootstrap_bin_dir),
+        env=runtime_env,
         default_cli_env=True,
         disable_extractors=True,
     )
@@ -170,19 +163,22 @@ def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized
     assert recovered.status == Binary.StatusChoices.INSTALLED
     assert recovered.version == binary.version
     assert Path(recovered.abspath).exists()
-    assert Path(recovered.abspath).resolve() == Path(shutil.which("rg") or "").resolve()
+    assert Path(recovered.abspath).resolve() == Path(sys.executable).resolve()
     assert process_count >= 2
 
     changed_lib_dir = tmp_path / "changed-lib"
     changed_provider_bin_dir = changed_lib_dir / "env" / "bin"
-    _link_real_binary(changed_provider_bin_dir, name, source="rg")
+    changed_runtime_env = _runtime_env(
+        initialized_archive,
+        lib_dir=changed_lib_dir,
+    )
 
     _cmd_result = run_archivebox_cmd(
         ["run"],
         cwd=initialized_archive,
-        stdin=json.dumps({"type": "BinaryRequest", "name": name, "binproviders": "env"}) + "\n",
+        stdin=json.dumps(request_record) + "\n",
         timeout=120,
-        env=_runtime_env(initialized_archive, bootstrap_bin_dir, lib_dir=changed_lib_dir),
+        env=changed_runtime_env,
         default_cli_env=True,
         disable_extractors=True,
     )
@@ -196,28 +192,27 @@ def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized
     assert relibbed.version == binary.version
     assert Path(relibbed.abspath) == changed_provider_bin_dir / name
     assert Path(relibbed.abspath).exists()
-    assert Path(relibbed.abspath).resolve() == Path(shutil.which("rg") or "").resolve()
+    assert Path(relibbed.abspath).resolve() == Path(sys.executable).resolve()
 
 
 def test_missing_binary_request_stays_queued_then_recovers_when_provider_can_resolve(initialized_archive, tmp_path):
-    name = f"abx-missing-rg-{uuid.uuid4().hex[:8]}"
-    bootstrap_bin_dir = tmp_path / "realbin"
-    provider_bin_dir = initialized_archive / "lib" / "env" / "bin"
-    _link_real_binary(bootstrap_bin_dir, "uv")
+    name = "http"
+    provider_bin_dir = initialized_archive / "lib" / "pip" / "venv" / "bin"
+    runtime_env = _runtime_env(initialized_archive)
 
     _cmd_result = run_archivebox_cmd(
         ["run"],
         cwd=initialized_archive,
         stdin=json.dumps({"type": "BinaryRequest", "name": name, "binproviders": "env"}) + "\n",
         timeout=120,
-        env=_runtime_env(initialized_archive, bootstrap_bin_dir),
+        env=runtime_env,
         default_cli_env=True,
         disable_extractors=True,
     )
     stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
     assert returncode == 0, stderr
-    assert any(record["type"] == "BinaryRequest" and record["name"] == name for record in parse_jsonl_output(stdout))
+    assert any(record["type"] == "BinaryRequest" and record["name"] == name for record in parse_jsonl_output(stdout)), stdout + stderr
 
     with use_archivebox_db(initialized_archive):
         queued = Binary.objects.get(name=name)
@@ -233,13 +228,18 @@ def test_missing_binary_request_stays_queued_then_recovers_when_provider_can_res
     assert f"{name.upper().replace('-', '_')}_BINARY" not in machine_config
     assert not (provider_bin_dir / name).exists()
 
-    _link_real_binary(provider_bin_dir, name, source="rg")
+    with use_archivebox_db(initialized_archive):
+        queued = Binary.objects.get(pk=queued_id)
+        queued.binproviders = "pip"
+        queued.overrides = {"pip": {"install_args": ["httpie>=3.2.4"]}}
+        queued.save(update_fields=["binproviders", "overrides", "modified_at"])
+    recovered_runtime_env = _runtime_env(initialized_archive)
 
     _cmd_result = run_archivebox_cmd(
         ["run", f"--binary-id={queued_id}"],
         cwd=initialized_archive,
         timeout=120,
-        env=_runtime_env(initialized_archive, bootstrap_bin_dir),
+        env=recovered_runtime_env,
         default_cli_env=True,
         disable_extractors=True,
     )
@@ -256,4 +256,6 @@ def test_missing_binary_request_stays_queued_then_recovers_when_provider_can_res
     assert recovered.version
     assert Path(recovered.abspath).exists()
     assert Path(recovered.abspath) == provider_bin_dir / name
+    assert recovered.binprovider == "pip"
+    assert Path(recovered.abspath).is_file()
     assert process_exit_codes[-2:] == [1, 0]

archivebox/tests/test_cli_add.py

@@ -16,7 +16,6 @@ from archivebox.core.models import ArchiveResult, Snapshot, SnapshotTag
 from archivebox.crawls.models import Crawl
 from archivebox.machine.models import Process
 from archivebox.tests.conftest import (
-    _find_system_browser,
     cli_env,
     find_snapshot_dir,
     get_free_port,
@@ -24,6 +23,7 @@ from archivebox.tests.conftest import (
     run_queued_crawls,
     start_archivebox_server,
     stop_server,
+    resolve_abxpkg_chrome_env,
 )
 
 from archivebox.tests.test_orm_helpers import use_archivebox_db
@@ -162,13 +162,17 @@ def wait_for_import_processing(cwd: Path, expected_urls: set[str], *, timeout: f
     import time
 
     deadline = time.time() + timeout
+    counts = {url: 0 for url in expected_urls}
     while time.time() < deadline:
         with use_archivebox_db(cwd):
-            snapshot_started = Snapshot.objects.filter(url__in=expected_urls).exists()
-        if snapshot_started:
+            rows = list(Snapshot.objects.filter(url__in=expected_urls).values_list("url", flat=True))
+        counts = {url: 0 for url in expected_urls}
+        for url in rows:
+            counts[url] += 1
+        if all(count >= 1 for count in counts.values()):
             return
         time.sleep(1)
-    raise AssertionError("timed out waiting for import crawl processing to start")
+    raise AssertionError(f"timed out waiting for import crawl processing to start, got counts={counts}")
 
 
 def wait_for_expected_import_snapshots(cwd: Path, expected_urls: set[str], *, timeout: float = 180.0) -> None:
@@ -1091,9 +1095,6 @@ def test_cli_add_real_urls_with_options_writes_inspectable_outputs(initialized_a
         "CHROME_SANDBOX": "false",
         "CHROME_ISOLATION": "snapshot",
     }
-    system_browser = _find_system_browser()
-    if system_browser:
-        chrome_env["CHROME_BINARY"] = str(system_browser)
     _cmd_result = run_archivebox_cmd(
         ["install", "chrome"],
         cwd=initialized_archive,
@@ -1102,6 +1103,7 @@ def test_cli_add_real_urls_with_options_writes_inspectable_outputs(initialized_a
     )
     install_stdout, install_stderr, install_returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
     assert install_returncode == 0, install_stderr or install_stdout
+    chrome_env.update(resolve_abxpkg_chrome_env(Path(chrome_env["ABXPKG_LIB_DIR"]), chrome_env))
     _cmd_result = run_archivebox_cmd(
         [
             "add",

archivebox/tests/test_cli_config.py

@@ -228,7 +228,7 @@ SEARCH_BACKEND_PASSWORD = "SecretPassword"
     )
 
     result = run_archivebox_cmd(
-        ["version"],
+        ["version", "--binaries=curl"],
     )
 
     assert result.returncode == 0, result.stderr

archivebox/tests/test_cli_run.py

@@ -963,7 +963,7 @@ class TestRecoverOrchestratorState:
         snapshot.refresh_from_db()
         assert snapshot.status == Snapshot.StatusChoices.SEALED
         assert snapshot.retry_at is None
-        assert snapshot.downloaded_at is None
+        assert snapshot.downloaded_at is not None
 
     def test_create_pending_archiveresults_uses_canonical_hook_names(self):
         from django.utils import timezone
@@ -991,6 +991,42 @@ class TestRecoverOrchestratorState:
         assert hook_names
         assert all(not hook_name.endswith((".py", ".js", ".sh")) for hook_name in hook_names)
 
+    def test_snapshot_hooks_for_pending_archiveresults_respects_disabled_plugins_when_plugins_empty(self):
+        from django.utils import timezone
+
+        from archivebox.base_models.models import get_or_create_system_user_pk
+        from archivebox.crawls.models import Crawl
+        from archivebox.core.models import Snapshot
+        from archivebox.services.runner import snapshot_hooks_for_pending_archiveresults
+
+        crawl = Crawl.objects.create(
+            urls="https://example.com",
+            created_by_id=get_or_create_system_user_pk(),
+            config={
+                "CLAUDECHROME_ENABLED": False,
+                "CLAUDECODEEXTRACT_ENABLED": False,
+                "CLAUDECODECLEANUP_ENABLED": False,
+                "SEARCH_BACKEND_SQLITE_ENABLED": False,
+            },
+            status=Crawl.StatusChoices.STARTED,
+            retry_at=timezone.now(),
+        )
+        snapshot = Snapshot.objects.create(
+            url="https://example.com",
+            crawl=crawl,
+            status=Snapshot.StatusChoices.QUEUED,
+            retry_at=timezone.now(),
+        )
+
+        hooks = snapshot_hooks_for_pending_archiveresults(snapshot)
+        queued_plugins = {plugin for plugin, _hook_name in hooks}
+
+        assert "title" in queued_plugins
+        assert "claudechrome" not in queued_plugins
+        assert "claudecodeextract" not in queued_plugins
+        assert "claudecodecleanup" not in queued_plugins
+        assert "search_backend_sqlite" not in queued_plugins
+
     def test_run_due_snapshot_pauses_child_when_parent_is_paused(self):
         from django.utils import timezone
 
@@ -1465,8 +1501,76 @@ class TestRecoverOrchestratorState:
         result.refresh_from_db()
         snapshot.refresh_from_db()
         assert result.status == ArchiveResult.StatusChoices.FAILED
+        assert result.output_str == "Queued hook is no longer available in the installed plugin"
         assert snapshot.retry_at is None
 
+    @pytest.mark.django_db(transaction=True)
+    def test_run_due_snapshot_skips_disabled_queued_plugins_and_seals_started_snapshot(self):
+        from django.utils import timezone
+
+        from archivebox.base_models.models import get_or_create_system_user_pk
+        from archivebox.crawls.models import Crawl
+        from archivebox.core.models import ArchiveResult, Snapshot
+        from archivebox.services.runner import run_due_snapshot
+
+        crawl = Crawl.objects.create(
+            urls="https://example.com",
+            created_by_id=get_or_create_system_user_pk(),
+            config={
+                "CLAUDECHROME_ENABLED": False,
+                "CLAUDECODEEXTRACT_ENABLED": False,
+                "CLAUDECODECLEANUP_ENABLED": False,
+                "SEARCH_BACKEND_SQLITE_ENABLED": False,
+            },
+            status=Crawl.StatusChoices.STARTED,
+            retry_at=timezone.now(),
+        )
+        snapshot = Snapshot.objects.create(
+            url="https://example.com",
+            crawl=crawl,
+            status=Snapshot.StatusChoices.STARTED,
+            retry_at=timezone.now(),
+            downloaded_at=timezone.now(),
+        )
+        ArchiveResult.objects.create(
+            snapshot=snapshot,
+            plugin="title",
+            hook_name="on_Snapshot__01_title",
+            status=ArchiveResult.StatusChoices.SUCCEEDED,
+            output_str="Example Domain",
+        )
+        stale_results = [
+            ArchiveResult.objects.create(
+                snapshot=snapshot,
+                plugin=plugin,
+                hook_name=hook_name,
+                status=ArchiveResult.StatusChoices.QUEUED,
+            )
+            for plugin, hook_name in (
+                ("claudechrome", "on_Snapshot__47_claudechrome"),
+                ("claudecodeextract", "on_Snapshot__58_claudecodeextract"),
+                ("claudecodecleanup", "on_Snapshot__92_claudecodecleanup"),
+                ("search_backend_sqlite", "on_Snapshot__90_index_sqlite"),
+            )
+        ]
+        stale_result_ids = [result.id for result in stale_results]
+        stale_plugins = [result.plugin for result in stale_results]
+
+        assert run_due_snapshot(snapshot, lock_seconds=60) is True
+
+        snapshot.refresh_from_db()
+        assert snapshot.status == Snapshot.StatusChoices.SEALED
+        assert snapshot.retry_at is None
+        assert not snapshot.archiveresult_set.filter(
+            plugin__in=stale_plugins,
+            status=ArchiveResult.StatusChoices.QUEUED,
+        ).exists()
+        for result in ArchiveResult.objects.filter(id__in=stale_result_ids):
+            assert result.status == ArchiveResult.StatusChoices.SKIPPED
+            assert result.start_ts is not None
+            assert result.end_ts is not None
+            assert "disabled by this Snapshot/Crawl config" in result.output_str
+
     @pytest.mark.django_db(transaction=True)
     @pytest.mark.timeout(300)
     @pytest.mark.parametrize("chrome_isolation", ["crawl", "snapshot"])
@@ -1807,6 +1911,52 @@ class TestRunDueCrawlState:
         assert finished.output_str == "keep me"
         assert finished.output_files == {"favicon.ico": {"size": 1}}
 
+    def test_finished_parser_result_projects_children_before_resume_seals_snapshot(self):
+        import json
+
+        from django.utils import timezone
+
+        from archivebox.base_models.models import get_or_create_system_user_pk
+        from archivebox.crawls.models import Crawl
+        from archivebox.core.models import ArchiveResult, Snapshot
+        from archivebox.services.runner import run_due_snapshot
+
+        crawl = Crawl.objects.create(
+            urls="Plain text import containing https://example.org/\n",
+            max_depth=1,
+            created_by_id=get_or_create_system_user_pk(),
+            status=Crawl.StatusChoices.STARTED,
+            retry_at=timezone.now(),
+        )
+        root = Snapshot.objects.create(
+            url=Snapshot.INTERNAL_INPUT_URL,
+            crawl=crawl,
+            depth=0,
+            status=Snapshot.StatusChoices.STARTED,
+            retry_at=timezone.now(),
+        )
+        parser_dir = root.output_dir / "parse_txt_urls"
+        parser_dir.mkdir(parents=True, exist_ok=True)
+        (parser_dir / "urls.jsonl").write_text(
+            json.dumps({"type": "Snapshot", "url": "https://example.org/"}) + "\n",
+            encoding="utf-8",
+        )
+        ArchiveResult.objects.create(
+            snapshot=root,
+            plugin="parse_txt_urls",
+            hook_name="on_Snapshot__71_parse_txt_urls",
+            status=ArchiveResult.StatusChoices.SUCCEEDED,
+            output_files={"urls.jsonl": {"size": (parser_dir / "urls.jsonl").stat().st_size}},
+        )
+
+        assert run_due_snapshot(root, lock_seconds=60)
+
+        root.refresh_from_db()
+        child = Snapshot.objects.get(crawl=crawl, url="https://example.org/")
+        assert root.status == Snapshot.StatusChoices.SEALED
+        assert child.parent_snapshot_id == root.id
+        assert child.status == Snapshot.StatusChoices.QUEUED
+
     def test_due_started_snapshot_with_live_child_extends_lease_without_reset(self):
         import os
         from datetime import datetime

archivebox/tests/test_cli_server.py

@@ -8,7 +8,6 @@ import os
 import asyncio
 import json
 import signal
-import shutil
 import socket
 import subprocess
 import sys
@@ -30,9 +29,19 @@ from archivebox.tests.conftest import (
     wait_for_port_open,
     wait_for_process,
     run_archivebox_cmd,
+    resolve_abxpkg_binary_env,
 )
 
 
+def _resolve_sonic_env(data_dir: Path) -> dict[str, str]:
+    from abx_plugins import get_plugins_dir
+
+    config = Path(get_plugins_dir()) / "search_backend_sonic" / "config.json"
+    resolved = resolve_abxpkg_binary_env(data_dir / "lib", deps_from=config)
+    assert Path(resolved["SONIC_BINARY"]).is_file()
+    return resolved
+
+
 def test_server_auth_secret_and_cookie_settings_are_restart_stable(tmp_path, monkeypatch):
     """Admin sessions must survive `archivebox server` restarts for a collection."""
     from archivebox.config.collection import write_config_file
@@ -206,11 +215,11 @@ def test_reload_workers_use_current_interpreter_and_supervisord_managed_runner()
     assert watcher["command"] == f"{sys.executable} -m archivebox manage runner_watch --bind-url=http://127.0.0.1:8000"
 
 
-def test_server_daemon_starts_real_plugin_owned_sonic_worker(archivebox_daemon_server):
-    assert shutil.which("sonic") is not None, "sonic server binary is required for Sonic worker integration tests"
-
+def test_server_daemon_starts_real_plugin_owned_sonic_worker(initialized_archive, archivebox_daemon_server):
+    sonic_env = _resolve_sonic_env(initialized_archive)
     server = archivebox_daemon_server(
         SEARCH_BACKEND_ENGINE="sonic",
+        **sonic_env,
     )
     state = server.wait_for_workers(("worker_daphne", "worker_sonic", "worker_runner"))
 
@@ -330,18 +339,18 @@ def test_sonic_worker_is_disabled_when_sonic_disabled(tmp_path):
     assert worker is None
 
 
-def test_sonic_daemon_event_handler_accepts_real_running_worker(archivebox_daemon_server):
-    assert shutil.which("sonic") is not None, "sonic server binary is required for Sonic worker integration tests"
-
+def test_sonic_daemon_event_handler_accepts_real_running_worker(initialized_archive, archivebox_daemon_server):
     from abx_dl.events import ProcessStdoutEvent
     from abx_dl.orchestrator import create_bus
     from archivebox.search.sonic_daemon import register_sonic_daemon_event_handler
     from abx_plugins.plugins.search_backend_sonic.daemon import prepare_sonic_daemon
 
     sonic_port = get_free_port()
+    sonic_env = _resolve_sonic_env(initialized_archive)
     server = archivebox_daemon_server(
         SEARCH_BACKEND_ENGINE="sonic",
         SEARCH_BACKEND_SONIC_PORT=str(sonic_port),
+        **sonic_env,
     )
     state = server.wait_for_workers(("worker_sonic",))
     assert state["worker_sonic"]["statename"] == "RUNNING", state
@@ -353,7 +362,7 @@ def test_sonic_daemon_event_handler_accepts_real_running_worker(archivebox_daemo
             SEARCH_BACKEND_SONIC_HOST_NAME="127.0.0.1",
             SEARCH_BACKEND_SONIC_PORT=sonic_port,
             SEARCH_BACKEND_SONIC_PASSWORD="SecretPassword",
-            SONIC_BINARY="sonic",
+            SONIC_BINARY=sonic_env["SONIC_BINARY"],
         ),
     )
 

archivebox/tests/test_cli_update_reindex_snapshots.py

@@ -12,6 +12,84 @@ from archivebox.tests.test_orm_helpers import use_archivebox_db
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
+def test_only_sealed_search_backfill_bypasses_snapshot_lifecycle():
+    from archivebox.base_models.models import get_or_create_system_user_pk
+    from archivebox.cli.archivebox_extract import run_plugins
+    from archivebox.core.models import ArchiveResult
+    from archivebox.crawls.models import Crawl
+
+    search_crawl = Crawl.objects.create(
+        urls="https://example.com/search",
+        created_by_id=get_or_create_system_user_pk(),
+        status=Crawl.StatusChoices.SEALED,
+    )
+    search_snapshot = Snapshot.objects.create(
+        url="https://example.com/search",
+        crawl=search_crawl,
+        status=Snapshot.StatusChoices.SEALED,
+    )
+    extract_crawl = Crawl.objects.create(
+        urls="https://example.com/extract",
+        created_by_id=get_or_create_system_user_pk(),
+        status=Crawl.StatusChoices.SEALED,
+    )
+    extract_snapshot = Snapshot.objects.create(
+        url="https://example.com/extract",
+        crawl=extract_crawl,
+        status=Snapshot.StatusChoices.SEALED,
+    )
+
+    assert (
+        run_plugins(
+            args=(),
+            records=[
+                {
+                    "type": "ArchiveResult",
+                    "snapshot_id": str(search_snapshot.id),
+                    "plugin": "search_backend_sqlite",
+                },
+            ],
+            wait=False,
+            emit_results=False,
+            show_progress=False,
+        )
+        == 0
+    )
+    search_crawl.refresh_from_db()
+    search_snapshot.refresh_from_db()
+    assert search_crawl.status == Crawl.StatusChoices.SEALED
+    assert search_snapshot.status == Snapshot.StatusChoices.SEALED
+    assert search_snapshot.archiveresult_set.filter(
+        plugin="search_backend_sqlite",
+        status=ArchiveResult.StatusChoices.QUEUED,
+    ).exists()
+
+    assert (
+        run_plugins(
+            args=(),
+            records=[
+                {
+                    "type": "ArchiveResult",
+                    "snapshot_id": str(extract_snapshot.id),
+                    "plugin": "wget",
+                },
+            ],
+            wait=False,
+            emit_results=False,
+            show_progress=False,
+        )
+        == 0
+    )
+    extract_crawl.refresh_from_db()
+    extract_snapshot.refresh_from_db()
+    assert extract_crawl.status == Crawl.StatusChoices.QUEUED
+    assert extract_snapshot.status == Snapshot.StatusChoices.QUEUED
+    assert extract_snapshot.archiveresult_set.filter(
+        plugin="wget",
+        status=ArchiveResult.StatusChoices.QUEUED,
+    ).exists()
+
+
 def test_update_imports_orphaned_snapshots(tmp_path, initialized_archive):
     """Test that archivebox update imports real legacy archive directories."""
     env = cli_env(disable_extractors=True)
@@ -67,6 +145,11 @@ def test_reindex_snapshots_resets_existing_search_results_and_reruns_requested_p
         crawl=crawl,
         status=Snapshot.StatusChoices.SEALED,
     )
+    paused_snapshot = Snapshot.objects.create(
+        url="https://example.com/paused",
+        crawl=crawl,
+        status=Snapshot.StatusChoices.PAUSED,
+    )
     result = ArchiveResult.objects.create(
         snapshot=snapshot,
         plugin="search_backend_sqlite",
@@ -87,7 +170,7 @@ def test_reindex_snapshots_resets_existing_search_results_and_reruns_requested_p
     os.environ["SEARCH_BACKEND_ENGINE"] = "sqlite"
     try:
         stats = reindex_snapshots(
-            Snapshot.objects.filter(id=snapshot.id),
+            Snapshot.objects.filter(id__in=(snapshot.id, paused_snapshot.id)),
             search_plugins=["search_backend_sqlite"],
             batch_size=10,
         )
@@ -105,6 +188,7 @@ def test_reindex_snapshots_resets_existing_search_results_and_reruns_requested_p
     assert result.status == ArchiveResult.StatusChoices.QUEUED
     assert result.output_str == ""
     assert result.output_json is None
+    assert not paused_snapshot.archiveresult_set.exists()
 
 
 @pytest.mark.django_db

archivebox/tests/test_cli_version.py

@@ -9,7 +9,8 @@ import re
 import tempfile
 from pathlib import Path
 from archivebox.config.paths import tmp_dir_socket_path_is_short_enough
-from archivebox.tests.conftest import run_archivebox_cmd
+from archivebox.cli.archivebox_version import _binary_row_dedupe_key
+from archivebox.tests.conftest import cli_env, run_archivebox_cmd
 
 
 def _make_deep_collection_dir(tmp_path: Path) -> Path:
@@ -30,6 +31,60 @@ def _extract_location_path(output: str, key: str) -> Path:
     raise AssertionError(f"Did not find a {key} location line in output:\n{output}")
 
 
+def test_binary_row_dedupe_key_keeps_distinct_paths_visible(tmp_path):
+    first_path = tmp_path / "lib" / "env" / "bin" / "node"
+    second_path = tmp_path / "other" / "bin" / "node"
+    first_path.parent.mkdir(parents=True)
+    second_path.parent.mkdir(parents=True)
+
+    first = _binary_row_dedupe_key(
+        display_name="node",
+        valid=True,
+        version="26.0.0",
+        provider="env",
+        abspath=str(first_path),
+    )
+    repeat = _binary_row_dedupe_key(
+        display_name="node",
+        valid=True,
+        version="26.0.0",
+        provider="env",
+        abspath=str(first_path),
+    )
+    different_path = _binary_row_dedupe_key(
+        display_name="node",
+        valid=True,
+        version="26.0.0",
+        provider="env",
+        abspath=str(second_path),
+    )
+
+    assert repeat == first
+    assert different_path != first
+
+
+def test_binary_row_dedupe_key_collapses_enabled_and_disabled_plugin_references(tmp_path):
+    binary_path = tmp_path / "bin" / "node"
+    binary_path.parent.mkdir(parents=True)
+
+    enabled_reference = _binary_row_dedupe_key(
+        display_name="node",
+        valid=True,
+        version="26.0.0",
+        provider="env",
+        abspath=str(binary_path),
+    )
+    disabled_reference = _binary_row_dedupe_key(
+        display_name="node",
+        valid=True,
+        version="26.0.0",
+        provider="env",
+        abspath=str(binary_path),
+    )
+
+    assert disabled_reference == enabled_reference
+
+
 def test_version_quiet_outputs_version_number(tmp_path):
     """Test that version --quiet outputs just the version number."""
     result = run_archivebox_cmd(["version", "--quiet"])
@@ -71,6 +126,83 @@ def test_version_shows_binaries_after_init(tmp_path, initialized_archive):
     assert "Binary" in output or "Dependencies" in output
 
 
+def test_version_skips_disabled_plugin_binary_resolution(tmp_path):
+    """Disabled plugins should not trigger live binary detection during version."""
+    data_dir = tmp_path / "no-plugins"
+    data_dir.mkdir()
+    env = cli_env(PLUGINS="__archivebox_test_no_plugins__", SEARCH_BACKEND_ENGINE="")
+
+    init_result = run_archivebox_cmd(["init"], cwd=data_dir, env=env)
+    assert init_result.returncode == 0, init_result.stderr
+
+    version_result = run_archivebox_cmd(["version"], cwd=data_dir, env=env)
+    output = version_result.stdout + version_result.stderr
+
+    assert version_result.returncode == 0, output
+    assert "No required binaries declared for discovered plugins" in output
+    assert "not installed" not in output
+
+
+def test_version_honors_legacy_save_aliases_when_disabling_extractors(tmp_path):
+    """Legacy SAVE_* aliases should disable their canonical plugin flags."""
+    data_dir = tmp_path / "legacy-save-disabled"
+    data_dir.mkdir()
+    env = {"PLUGINS": "wget,title", "SEARCH_BACKEND_ENGINE": ""}
+
+    init_result = run_archivebox_cmd(["init"], cwd=data_dir, env=env, default_cli_env=True, disable_extractors=True, timeout=120)
+    assert init_result.returncode == 0, init_result.stderr or init_result.stdout
+
+    config_result = run_archivebox_cmd(
+        ["config", "--get", "WGET_ENABLED", "TITLE_ENABLED", "CHROME_ENABLED"],
+        cwd=data_dir,
+        env=env,
+        default_cli_env=True,
+        disable_extractors=True,
+    )
+    config_output = config_result.stdout + config_result.stderr
+    assert config_result.returncode == 0, config_output
+    assert "WGET_ENABLED = false" in config_output
+    assert "TITLE_ENABLED = false" in config_output
+    assert "CHROME_ENABLED = false" in config_output
+
+    version_result = run_archivebox_cmd(
+        ["version"],
+        cwd=data_dir,
+        env=env,
+        default_cli_env=True,
+        disable_extractors=True,
+        timeout=60,
+    )
+    output = version_result.stdout + version_result.stderr
+
+    assert version_result.returncode == 0, output
+    assert "not installed" not in output
+
+
+def test_plugins_selection_includes_required_plugins_via_config_cli(tmp_path):
+    """PLUGINS selection should include transitive plugin dependencies exactly once."""
+    data_dir = tmp_path / "plugin-dependencies"
+    data_dir.mkdir()
+    env = {"PLUGINS": "wget,screenshot"}
+
+    init_result = run_archivebox_cmd(["init"], cwd=data_dir, env=env, default_cli_env=True, timeout=120)
+    assert init_result.returncode == 0, init_result.stderr or init_result.stdout
+
+    config_result = run_archivebox_cmd(
+        ["config", "--get", "WGET_ENABLED", "SCREENSHOT_ENABLED", "CHROME_ENABLED", "YTDLP_ENABLED"],
+        cwd=data_dir,
+        env=env,
+        default_cli_env=True,
+    )
+    config_output = config_result.stdout + config_result.stderr
+
+    assert config_result.returncode == 0, config_output
+    assert "WGET_ENABLED = true" in config_output
+    assert "SCREENSHOT_ENABLED = true" in config_output
+    assert "CHROME_ENABLED = true" in config_output
+    assert "YTDLP_ENABLED = false" in config_output
+
+
 def test_version_shows_data_locations(tmp_path, initialized_archive):
     """Test that version shows data directory locations."""
     result = run_archivebox_cmd(["version"])

archivebox/tests/test_config_DELETE_AFTER.py

@@ -154,7 +154,7 @@ print(json.dumps({{
     retained = json.loads(stdout.strip().splitlines()[-1])
     assert retained["crawl_id"] == created["crawl_id"]
 
-    _cmd_result = run_archivebox_cmd(["run", "--crawl-id", retained["crawl_id"]], cwd=tmp_path, timeout=120, env=run_env)
+    _cmd_result = run_archivebox_cmd(["run", "--maintenance-only"], cwd=tmp_path, timeout=120, env=run_env)
     stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
     assert returncode == 0, f"archivebox run failed:\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}"
 

archivebox/tests/test_config_SAVE_TITLE.py

@@ -7,7 +7,7 @@ import pytest
 
 from archivebox.core.models import Snapshot
 from archivebox.tests.test_orm_helpers import use_archivebox_db
-from .conftest import _find_cached_chrome, _find_system_browser
+from .conftest import resolve_abxpkg_chrome_env
 
 pytestmark = pytest.mark.django_db(transaction=True)
 
@@ -22,15 +22,7 @@ def _install_chrome(tmp_path, env):
         timeout=600,
     )
     assert install_process.returncode == 0, install_process.stderr or install_process.stdout
-    cached_browser = _find_cached_chrome(Path(env["ABXPKG_LIB_DIR"]))
-    if cached_browser is not None:
-        env["CHROME_BINARY"] = str(cached_browser)
-        return
-    system_browser = _find_system_browser()
-    if system_browser:
-        env["CHROME_BINARY"] = str(system_browser)
-        return
-    raise AssertionError(install_process.stderr or install_process.stdout)
+    env.update(resolve_abxpkg_chrome_env(Path(env["ABXPKG_LIB_DIR"]), env))
 
 
 def _wait_for_snapshot_title(data_dir, *, timeout=60):

archivebox/tests/test_crawl_pause.py

@@ -6,7 +6,3 @@ from archivebox.workers.models import RETRY_AT_MAX
 def test_retry_at_max_is_safe_for_admin_timezone_localization():
     with timezone.override("Pacific/Kiritimati"):
         assert timezone.localtime(RETRY_AT_MAX).year == 9999
-
-
-# test_crawl_pause_resume_api_survives_server_restart_and_processes_after_resume moved to test_api_v1_crawls_crawl_crawl_id.py.
-# test_update_index_only_runs_paused_search_rows_and_resume_later_runs_crawl moved to test_api_v1_crawls_crawl_crawl_id.py.

archivebox/tests/test_crawl_runner.py

@@ -5,6 +5,7 @@ from pathlib import Path
 import pytest
 from asgiref.sync import sync_to_async
 
+from archivebox.tests.conftest import resolve_abxpkg_binary_env
 
 pytestmark = pytest.mark.django_db
 
@@ -167,6 +168,7 @@ def test_ensure_background_runner_skips_with_real_running_orchestrator_record():
         status=Process.StatusChoices.RUNNING,
         pid=os.getpid(),
         started_at=datetime.fromtimestamp(os_proc.create_time(), tz=timezone.get_current_timezone()),
+        timeout=1,
     )
 
     assert ensure_background_runner(allow_under_pytest=True) is False
@@ -415,9 +417,9 @@ def test_machine_service_persists_only_derived_config_events(tmp_path, hermetic_
     machine = Machine.current()
     machine.config = {}
     machine.save(update_fields=["config"])
-    wget_binary = hermetic_lib_dir / "bin" / "wget"
-    wget_binary.write_text("#!/bin/sh\n")
-    wget_binary.chmod(0o755)
+    resolve_abxpkg_binary_env(hermetic_lib_dir, "wget")
+    wget_binary = hermetic_lib_dir / "env" / "bin" / "wget"
+    assert wget_binary.is_symlink()
 
     async def run_test():
         bus = create_bus(name="test_machine_service_persists_only_derived_config_events")
@@ -490,13 +492,10 @@ def test_load_run_state_uses_real_lib_dir_for_machine_binary_config(tmp_path, he
     resolved_lib_dir = get_config(include_machine=False).ABXPKG_LIB_DIR
     assert resolved_lib_dir == hermetic_lib_dir, f"ABXPKG_LIB_DIR override not applied: {resolved_lib_dir!r} != {hermetic_lib_dir!r}"
 
-    wget_binary = resolved_lib_dir / "bin" / "wget"
-    wget_binary.write_text("#!/bin/sh\n", encoding="utf-8")
-    wget_binary.chmod(0o755)
-    external_binary = tmp_path / "external" / "yt-dlp"
-    external_binary.parent.mkdir(parents=True)
-    external_binary.write_text("#!/bin/sh\n", encoding="utf-8")
-    external_binary.chmod(0o755)
+    resolve_abxpkg_binary_env(resolved_lib_dir, "wget")
+    wget_binary = resolved_lib_dir / "env" / "bin" / "wget"
+    assert wget_binary.is_symlink()
+    external_binary = Path(sys.executable)
 
     machine = Machine.current()
     machine.config = {
@@ -686,8 +685,7 @@ def test_crawl_runner_resolves_persona_and_crawl_config_for_each_live_snapshot()
 
 
 @pytest.mark.django_db(transaction=True)
-@pytest.mark.django_db(transaction=True)
-def test_run_pending_crawls_processes_queued_crawl_before_missing_binary_backlog(tmp_path):
+def test_run_pending_crawls_processes_queued_crawl_and_real_binary(tmp_path):
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.crawls.models import Crawl
     from archivebox.core.models import Snapshot
@@ -708,11 +706,10 @@ def test_run_pending_crawls_processes_queued_crawl_before_missing_binary_backlog
     )
     binary = Binary.objects.create(
         machine=Machine.current(),
-        name=str(tmp_path / "missing-node"),
+        name="bash",
         status=Binary.StatusChoices.QUEUED,
         retry_at=timezone.now(),
-        binproviders="env,apt",
-        overrides={"apt": {"install_args": ["nodejs"]}},
+        binproviders="env",
     )
 
     result = run_pending_crawls(daemon=False)
@@ -723,8 +720,10 @@ def test_run_pending_crawls_processes_queued_crawl_before_missing_binary_backlog
     assert crawl.status == Crawl.StatusChoices.SEALED
     assert crawl.retry_at is None
     assert Snapshot.objects.filter(crawl=crawl, status=Snapshot.StatusChoices.SEALED).count() == 1
-    assert binary.status == Binary.StatusChoices.QUEUED
+    assert binary.status == Binary.StatusChoices.INSTALLED
     assert binary.retry_at is None
+    assert Path(binary.abspath).is_symlink()
+    assert binary.binprovider == "env"
 
 
 @pytest.mark.django_db(transaction=True)
@@ -752,9 +751,6 @@ def test_sealed_crawl_does_not_create_discovered_snapshots():
     assert crawl.snapshot_set.count() == 1
 
 
-# test_create_crawl_api_queues_crawl_without_spawning_runner moved to test_api_v1_crawls_crawls.py.
-
-
 def test_wait_for_snapshot_tasks_surfaces_already_failed_task():
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.crawls.models import Crawl
@@ -848,26 +844,27 @@ def test_abx_process_service_background_process_finishes_after_process_exit(tmp_
     assert any(isinstance(event, ProcessCompletedEvent) for event in emitted_events)
 
 
-def test_run_pending_crawls_disables_missing_absolute_binary_backlog(tmp_path):
+@pytest.mark.django_db(transaction=True)
+def test_run_pending_crawls_resolves_real_binary_through_abxpkg(tmp_path):
     from archivebox.machine.models import Binary, Machine
     from archivebox.services import runner as runner_module
 
-    missing_binary = tmp_path / "missing-node"
     binary = Binary.objects.create(
         machine=Machine.current(),
-        name=str(missing_binary),
+        name="bash",
         status=Binary.StatusChoices.QUEUED,
         retry_at=runner_module.timezone.now(),
-        binproviders="env,apt",
-        overrides={"apt": {"install_args": ["nodejs"]}},
+        binproviders="env",
     )
 
     result = runner_module.run_pending_crawls(daemon=False)
 
     binary.refresh_from_db()
     assert result == 0
-    assert binary.status == Binary.StatusChoices.QUEUED
+    assert binary.status == Binary.StatusChoices.INSTALLED
     assert binary.retry_at is None
+    assert Path(binary.abspath).is_symlink()
+    assert binary.binprovider == "env"
 
 
 @pytest.mark.django_db(transaction=True)

archivebox/tests/test_frozen_crawl_config.py

@@ -269,9 +269,6 @@ def test_crawl_config_projections_stay_under_hot_path_budget():
         assert average_seconds < max_average_seconds, f"{name} averaged {average_seconds * 1000:.3f}ms"
 
 
-# test_api_create_and_cli_add_store_full_frozen_config moved to test_api_v1_workflow_frozen_crawl_config_sources.py.
-
-
 def test_schedule_enqueue_refreezes_using_current_template_persona_defaults(archivebox_db):
     from archivebox.crawls.models import Crawl, CrawlSchedule
 

archivebox/tests/test_hooks.py

@@ -20,6 +20,8 @@ from pathlib import Path
 import pytest
 import rich_click as click
 
+from archivebox.tests.conftest import resolve_abxpkg_binary_env
+
 # Set up Django before importing any Django-dependent modules
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "archivebox.settings")
 
@@ -524,7 +526,10 @@ print(json.dumps({"type": "ArchiveResult", "status": "succeeded", "output_str":
 
     def test_js_hook_execution(self, tmp_path):
         """JavaScript hook should execute and output JSONL."""
-        assert shutil.which("node") is not None, "Node.js not available"
+        lib_dir = tmp_path / "lib"
+        node_env = resolve_abxpkg_binary_env(lib_dir, "node")
+        node_binary = lib_dir / "env" / "bin" / "node"
+        assert node_binary.is_symlink()
 
         hook_path = tmp_path / "test_hook.js"
         hook_path.write_text("""#!/usr/bin/env node
@@ -532,10 +537,11 @@ console.log(JSON.stringify({type: 'ArchiveResult', status: 'succeeded', output_s
 """)
 
         result = subprocess.run(
-            ["node", str(hook_path)],
+            [str(node_binary), str(hook_path)],
             cwd=tmp_path,
             capture_output=True,
             text=True,
+            env={**os.environ, **node_env},
         )
 
         assert result.returncode == 0

archivebox/tests/test_machine_models.py

@@ -19,6 +19,7 @@ from pathlib import Path
 from typing import cast
 
 import pytest
+from django.db import IntegrityError
 from django.db import transaction
 from django.utils import timezone
 
@@ -34,6 +35,8 @@ from archivebox.machine.models import (
     PID_REUSE_WINDOW,
     PROCESS_TIMEOUT_GRACE,
 )
+from archivebox.machine.detect import unknown_if_blank
+from archivebox.tests.conftest import resolve_abxpkg_binary_env
 
 pytestmark = pytest.mark.django_db
 
@@ -149,15 +152,12 @@ class TestMachineModel:
         assert machine1_id != machine2.id
         assert machine1_guid == machine2.guid
 
-    def test_machine_from_jsonl_update(self, cleanup_paths):
+    def test_machine_from_jsonl_update(self, hermetic_lib_dir):
         """Machine.from_json() should update machine config."""
-        from archivebox.config.constants import CONSTANTS
-
         Machine.current()  # Ensure machine exists
-        wget_path = CONSTANTS.DEFAULT_ABXPKG_LIB_DIR / "wget"
-        wget_path.parent.mkdir(parents=True, exist_ok=True)
-        wget_path.write_text("#!/bin/sh\n")
-        cleanup_paths.append(wget_path)
+        resolve_abxpkg_binary_env(hermetic_lib_dir, "wget")
+        wget_path = hermetic_lib_dir / "env" / "bin" / "wget"
+        assert wget_path.is_symlink()
         record = {
             "config": {
                 "WGET_BINARY": str(wget_path),
@@ -169,20 +169,17 @@ class TestMachineModel:
         assert result is not None
         assert result.config.get("WGET_BINARY") == str(wget_path)
 
-    def test_machine_from_jsonl_drops_invalid_binary_paths_keeps_mirror(self, cleanup_paths):
+    def test_machine_from_jsonl_drops_invalid_binary_paths_keeps_mirror(self, hermetic_lib_dir):
         """Machine.from_json() drops invalid binary paths but mirrors other keys.
 
         ``Machine.config`` mirrors ``ArchiveBox.conf`` (non-binary user config
         keys live alongside derived binary state), so non-binary keys in the
         import survive. Only ``_BINARY`` paths get validated/dropped on import.
         """
-        from archivebox.config.constants import CONSTANTS
-
         Machine.current()  # Ensure machine exists
-        wget_path = CONSTANTS.DEFAULT_ABXPKG_LIB_DIR / "wget"
-        wget_path.parent.mkdir(parents=True, exist_ok=True)
-        wget_path.write_text("#!/bin/sh\n")
-        cleanup_paths.append(wget_path)
+        resolve_abxpkg_binary_env(hermetic_lib_dir, "wget")
+        wget_path = hermetic_lib_dir / "env" / "bin" / "wget"
+        assert wget_path.is_symlink()
         record = {
             "config": {
                 "WGET_BINARY": str(wget_path),
@@ -203,7 +200,7 @@ class TestMachineModel:
         result = Machine.from_json({"invalid": "record"})
         assert result is None
 
-    def test_machine_current_drops_invalid_binary_paths_keeps_mirror(self, cleanup_paths):
+    def test_machine_current_drops_invalid_binary_paths_keeps_mirror(self, hermetic_lib_dir):
         """Machine.current() mirrors ArchiveBox.conf, only drops invalid binaries.
 
         ``Machine.config`` is the file ↔ DB mirror of ``ArchiveBox.conf``, so
@@ -212,17 +209,11 @@ class TestMachineModel:
         ``ABXPKG_LIB_DIR`` and dropped when stale/missing.
         """
         import archivebox.machine.models as models
-        from archivebox.config.constants import CONSTANTS
 
-        active_lib_dir = CONSTANTS.DEFAULT_ABXPKG_LIB_DIR
-        active_lib_dir.mkdir(parents=True, exist_ok=True)
-        chrome_path = active_lib_dir / "chromium"
-        node_path = active_lib_dir / "node"
-        chrome_path.write_text("#!/bin/sh\n")
-        node_path.write_text("#!/bin/sh\n")
-        external_path = Path("/tmp/archivebox-test-external-node")
-        external_path.touch()
-        cleanup_paths.extend([chrome_path, node_path, external_path])
+        resolve_abxpkg_binary_env(hermetic_lib_dir, "node", "wget")
+        chrome_path = hermetic_lib_dir / "env" / "bin" / "node"
+        node_path = hermetic_lib_dir / "env" / "bin" / "wget"
+        external_path = Path(sys.executable)
         machine = Machine.current()
         machine.config = {
             "CHROME_BINARY": str(chrome_path),
@@ -252,7 +243,7 @@ class TestMachineModel:
         assert "YTDLP_BINARY" not in refreshed.config
         assert "WGET_BINARY" not in refreshed.config
 
-    def test_get_config_auto_applies_current_machine_config(self, cleanup_paths):
+    def test_get_config_auto_applies_current_machine_config(self, hermetic_lib_dir):
         """get_config() applies the full Machine.config mirror as scope overrides.
 
         ``Machine.config`` mirrors ``ArchiveBox.conf``, so non-binary user keys
@@ -263,10 +254,9 @@ class TestMachineModel:
         from archivebox.config.common import get_config
 
         lib_dir = get_config(include_machine=False).ABXPKG_LIB_DIR
-        chrome_path = lib_dir / "chromium"
-        chrome_path.parent.mkdir(parents=True, exist_ok=True)
-        chrome_path.write_text("#!/bin/sh\n")
-        cleanup_paths.append(chrome_path)
+        assert lib_dir == hermetic_lib_dir
+        resolve_abxpkg_binary_env(lib_dir, "node")
+        chrome_path = lib_dir / "env" / "bin" / "node"
         machine = Machine.current()
         machine.config = {
             "CHROME_BINARY": str(chrome_path),
@@ -312,6 +302,46 @@ class TestNetworkInterfaceModel:
         interface = NetworkInterface.current()
         assert interface is not None
 
+    def test_networkinterface_optional_location_fields_default_to_blank(self, machine):
+        interface = NetworkInterface.objects.create(
+            machine=machine,
+            mac_address="00:00:00:00:00:01",
+            ip_public="127.0.0.1",
+            ip_local="127.0.0.1",
+            dns_server="127.0.0.1",
+        )
+
+        assert interface.hostname == ""
+        assert interface.iface == ""
+        assert interface.isp == ""
+        assert interface.city == ""
+        assert interface.region == ""
+        assert interface.country == ""
+
+    def test_unknown_if_blank_normalizes_null_api_fields(self):
+        assert unknown_if_blank(None) == "Unknown"
+        assert unknown_if_blank("") == "Unknown"
+        assert unknown_if_blank("  ") == "Unknown"
+        assert unknown_if_blank("California") == "California"
+
+    def test_networkinterface_identity_ignores_randomized_mac_address(self, machine):
+        NetworkInterface.objects.create(
+            machine=machine,
+            mac_address="00:00:00:00:00:01",
+            ip_public="127.0.0.1",
+            ip_local="127.0.0.1",
+            dns_server="127.0.0.1",
+        )
+
+        with pytest.raises(IntegrityError), transaction.atomic():
+            NetworkInterface.objects.create(
+                machine=machine,
+                mac_address="00:00:00:00:00:02",
+                ip_public="127.0.0.1",
+                ip_local="127.0.0.1",
+                dns_server="127.0.0.1",
+            )
+
 
 class TestBinaryModel:
     """Test the Binary model."""
@@ -452,10 +482,9 @@ class TestBinaryModel:
     @pytest.mark.django_db(transaction=True)
     def test_binary_lib_bin_symlink_waits_for_outer_transaction_commit(self, tmp_path):
         """Binary DB projection writes can be direct, but convenience symlinks must run after commit."""
-        source = tmp_path / "provider" / "bin" / "abx-test-binary"
-        source.parent.mkdir(parents=True)
-        source.write_text("#!/bin/sh\nexit 0\n")
-        source.chmod(0o755)
+        provider_lib = tmp_path / "provider"
+        resolve_abxpkg_binary_env(provider_lib, "node")
+        source = provider_lib / "env" / "bin" / "node"
         lib_bin_dir = tmp_path / "lib" / "bin"
         symlink = lib_bin_dir / "abx-test-binary"
 
@@ -471,7 +500,7 @@ class TestBinaryModel:
             assert not symlink.exists()
 
         assert symlink.is_symlink()
-        assert symlink.resolve() == source
+        assert symlink.resolve() == source.resolve()
 
 
 class TestBinaryStateMachine:
@@ -856,6 +885,7 @@ class TestProcessClassMethods:
         """cleanup_stale_running should retire RUNNING rows that exceed timeout + grace."""
         stale = Process.objects.create(
             machine=self.machine,
+            process_type=Process.TypeChoices.HOOK,
             status=Process.StatusChoices.RUNNING,
             pid=999998,
             timeout=5,

archivebox/tests/test_machine_service.py

@@ -1,5 +1,3 @@
-import os
-import shutil
 import textwrap
 from pathlib import Path
 
@@ -12,51 +10,22 @@ from archivebox.tests.test_orm_helpers import use_archivebox_db
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
-def _link_real_tool(bin_dir: Path, name: str) -> Path:
-    bin_dir.mkdir(parents=True, exist_ok=True)
-    source_path = shutil.which(name)
-    assert source_path, f"{name} must be installed for this integration test"
-    link = bin_dir / name
-    link.unlink(missing_ok=True)
-    link.symlink_to(source_path)
-    return link
-
-
-def _write_tool_shim(bin_dir: Path, name: str, version: str) -> Path:
-    bin_dir.mkdir(parents=True, exist_ok=True)
-    shim = bin_dir / name
-    shim.write_text(f"#!/bin/sh\nprintf '%s\\n' '{name} {version}'\n", encoding="utf-8")
-    shim.chmod(0o755)
-    return shim
-
-
-def _runtime_env(data_dir: Path, bin_dir: Path) -> dict[str, str]:
-    path_entries = [
-        str(bin_dir),
-        str(data_dir / "lib" / "env" / "bin"),
-        os.environ.get("PATH", ""),
-    ]
+def _runtime_env(data_dir: Path, **extra: str) -> dict[str, str]:
     return {
         "ABXPKG_LIB_DIR": str(data_dir / "lib"),
         "LITEPARSE_ENABLED": "True",
         "TIMEOUT": "180",
         "ABXPKG_INSTALL_TIMEOUT": "180",
-        "PATH": os.pathsep.join(entry for entry in path_entries if entry),
+        **extra,
     }
 
 
 def test_install_persists_machine_binary_config_and_recovers_stale_path(initialized_archive, tmp_path):
-    bootstrap_bin_dir = tmp_path / "realbin"
-    provider_bin_dir = initialized_archive / "lib" / "env" / "bin"
-    _link_real_tool(bootstrap_bin_dir, "uv")
-    _write_tool_shim(provider_bin_dir, "lit", "2.5.9")
-    _link_real_tool(provider_bin_dir, "node")
-
     _cmd_result = run_archivebox_cmd(
-        ["install", "--binproviders=env", "liteparse"],
+        ["install", "liteparse"],
         cwd=initialized_archive,
         timeout=240,
-        env=_runtime_env(initialized_archive, bootstrap_bin_dir),
+        env=_runtime_env(initialized_archive),
     )
     stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
@@ -79,8 +48,10 @@ def test_install_persists_machine_binary_config_and_recovers_stale_path(initiali
     assert process.status == Process.StatusChoices.EXITED
     assert process.exit_code == 0
 
-    external_tool = Path(shutil.which("node") or "")
-    assert external_tool.exists()
+    tesseract_binary = next(binary for binary in binaries if binary.name == "tesseract")
+    external_tool = Path(tesseract_binary.abspath)
+    assert external_tool.is_file()
+    assert external_tool.is_relative_to(initialized_archive / "lib")
     machine_event_script = textwrap.dedent(
         f"""
         import asyncio
@@ -99,7 +70,7 @@ def test_install_persists_machine_binary_config_and_recovers_stale_path(initiali
                 }}, config_type="user")).now()
                 await bus.emit(MachineEvent(config={{
                     "LITEPARSE_BINARY": {str(installed_liteparse_path)!r},
-                    "NODE_BINARY": {str(external_tool)!r},
+                    "LITEPARSE_TESSERACT_BINARY": {str(external_tool)!r},
                     "ABX_INSTALL_CACHE": {{"lit": "cached"}},
                     "ABX_UV_CACHE": "/tmp/uv-cache",
                     "CHROME_USER_DATA_DIR": "/tmp/derived-profile",
@@ -123,7 +94,7 @@ def test_install_persists_machine_binary_config_and_recovers_stale_path(initiali
         ["shell", "-c", machine_event_script],
         cwd=initialized_archive,
         timeout=60,
-        env=_runtime_env(initialized_archive, bootstrap_bin_dir),
+        env=_runtime_env(initialized_archive),
     )
     shell_stdout, shell_stderr, shell_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
     assert shell_code == 0, shell_stdout + shell_stderr
@@ -133,6 +104,7 @@ def test_install_persists_machine_binary_config_and_recovers_stale_path(initiali
         machine = Machine.objects.get(pk=machine_id)
 
     assert machine.config["LITEPARSE_BINARY"] == str(installed_liteparse_path)
+    assert machine.config["LITEPARSE_TESSERACT_BINARY"] == str(external_tool)
     assert machine.config["LITEPARSE_BINARY"] != "/tmp/user-config-must-not-persist"
     assert machine.config["ABX_INSTALL_CACHE"] == {"lit": "cached"}
     assert machine.config["ABX_UV_CACHE"] == "/tmp/uv-cache"
@@ -141,7 +113,7 @@ def test_install_persists_machine_binary_config_and_recovers_stale_path(initiali
         ["version"],
         cwd=initialized_archive,
         timeout=60,
-        env=_runtime_env(initialized_archive, bootstrap_bin_dir),
+        env=_runtime_env(initialized_archive),
     )
     version_stdout, version_stderr, version_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
     assert version_code == 0, version_stderr
@@ -153,7 +125,7 @@ def test_install_persists_machine_binary_config_and_recovers_stale_path(initiali
         ["version"],
         cwd=initialized_archive,
         timeout=60,
-        env=_runtime_env(initialized_archive, bootstrap_bin_dir),
+        env=_runtime_env(initialized_archive),
     )
     cleanup_stdout, cleanup_stderr, cleanup_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
     assert cleanup_code == 0, cleanup_stdout + cleanup_stderr

archivebox/tests/test_opencode_agent.py

@@ -5,8 +5,8 @@ from pathlib import Path
 from types import SimpleNamespace
 from urllib.parse import quote
 
-import psutil
 import pytest
+import requests
 
 from archivebox.tests.conftest import ADMIN_TEST_HOST, run_archivebox_cmd
 
@@ -54,7 +54,6 @@ def opencode_archive_config(initialized_archive):
         {
             "ABXPKG_INSTALL_TIMEOUT": "900",
             "ABXPKG_MIN_RELEASE_AGE": "0",
-            "ABX_RUNTIME": "archivebox",
             "ARCHIVEBOX_ALLOW_NO_UNIX_SOCKETS": "true",
             "OPENCODE_ENABLED": "True",
             "OPENCODE_HOST": "127.0.0.1",
@@ -175,10 +174,11 @@ def test_opencode_agent_superuser_gets_admin_wrapper(admin_client, live_opencode
     from abx_plugins.plugins.opencode import views
 
     response = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
+    recent_session_id = views._recent_session_id(live_opencode.settings)
 
     assert response.status_code == 200
-    assert f'<iframe src="{views._project_route(live_opencode.config.data_dir)}'.encode() in response.content
-    assert b'/session"' in response.content
+    assert recent_session_id
+    assert f'<iframe src="{views._project_route(live_opencode.config.data_dir, recent_session_id)}"'.encode() in response.content
     assert b'id="header"' in response.content
     assert b'id="progress-monitor"' in response.content
 
@@ -225,20 +225,24 @@ def test_opencode_proxy_sse_response_is_unbuffered(admin_client, live_opencode):
     assert response.headers["Cache-Control"] == "no-store"
 
 
-def test_opencode_starts_with_data_dir_cwd_and_isolated_state(live_opencode):
-    process = psutil.Process(live_opencode.process.pid)
-    env = process.environ()
+def test_opencode_starts_with_isolated_state(live_opencode):
     workdir = str(live_opencode.config.data_dir.resolve())
+    state_dir = live_opencode.config.state_dir
 
-    assert Path(process.cwd()).resolve() == live_opencode.config.data_dir.resolve()
-    assert env["BROWSER"] == "false"
-    assert env["HOME"] == str(live_opencode.config.state_dir / "home")
-    assert env["XDG_CONFIG_HOME"] == str(live_opencode.config.state_dir / "config")
-    assert env["XDG_DATA_HOME"] == str(live_opencode.config.state_dir / "data")
-    assert env["XDG_STATE_HOME"] == str(live_opencode.config.state_dir / "state")
-    assert env["XDG_CACHE_HOME"] == str(live_opencode.config.state_dir / "cache")
-    assert env["OPENCODE_DISABLE_PROJECT_CONFIG"] == "true"
-    assert env["GIT_CEILING_DIRECTORIES"] == workdir
+    project = requests.get(
+        f"{live_opencode.settings['origin']}/project/current",
+        params={"directory": workdir},
+        timeout=live_opencode.settings["timeout"],
+    )
+    project.raise_for_status()
+
+    assert Path(live_opencode.settings["workdir"]).resolve() == Path(workdir)
+    assert Path(str(project.json()["worktree"])).resolve() == Path(workdir)
+    assert live_opencode.process.poll() is None
+    assert (state_dir / "data" / "opencode" / "opencode.db").is_file()
+    assert (state_dir / "SKILL.md").is_file()
+    assert (state_dir / "config" / "opencode" / "skills" / "archivebox" / "SKILL.md").resolve() == state_dir / "SKILL.md"
+    assert not (live_opencode.config.data_dir / ".git" / "not-a-git").exists()
 
 
 def test_opencode_state_dir_is_separate_from_workdir(tmp_path):

archivebox/tests/test_orm_helpers.py

@@ -1,7 +1,10 @@
 from collections.abc import Iterator
 from contextlib import contextmanager
 from pathlib import Path
+import asyncio
 
+import pytest
+from asgiref.sync import sync_to_async
 from django.conf import settings
 from django.db import connections
 
@@ -11,8 +14,43 @@ def archivebox_db_path(path: str | Path = ".") -> Path:
     return path if path.name == "index.sqlite3" else path / "index.sqlite3"
 
 
+def test_archivebox_db_path_accepts_collection_or_database_path(tmp_path: Path) -> None:
+    database_path = tmp_path / "index.sqlite3"
+    assert archivebox_db_path(tmp_path) == database_path
+    assert archivebox_db_path(database_path) == database_path
+
+
+def _reset_thread_sensitive_default_connection() -> None:
+    """Discard the default connection owned by asgiref's thread-sensitive worker."""
+
+    def reset_connection() -> None:
+        connections["default"].close()
+        del connections._connections.default
+
+    asyncio.run(sync_to_async(reset_connection, thread_sensitive=True)())
+
+
+def _thread_sensitive_database_path() -> str:
+    def database_path() -> str:
+        with connections["default"].cursor() as cursor:
+            return str(cursor.execute("PRAGMA database_list").fetchone()[2])
+
+    return asyncio.run(sync_to_async(database_path, thread_sensitive=True)())
+
+
+@pytest.mark.django_db(transaction=True)
+def test_use_archivebox_db_restores_thread_sensitive_connection(tmp_path: Path) -> None:
+    original_database_path = _thread_sensitive_database_path()
+
+    with use_archivebox_db(tmp_path):
+        assert _thread_sensitive_database_path() == str(tmp_path / "index.sqlite3")
+
+    assert _thread_sensitive_database_path() == original_database_path
+
+
 @contextmanager
 def use_archivebox_db(path: str | Path = ".") -> Iterator[None]:
+    _reset_thread_sensitive_default_connection()
     connection = connections["default"]
     original_name = connection.settings_dict["NAME"]
     original_database_name = connections.databases["default"]["NAME"]
@@ -28,6 +66,7 @@ def use_archivebox_db(path: str | Path = ".") -> Iterator[None]:
     try:
         yield
     finally:
+        _reset_thread_sensitive_default_connection()
         connections["default"].close()
         connections.databases["default"]["NAME"] = original_database_name
         settings.DATABASES["default"]["NAME"] = original_setting_name

archivebox/tests/test_recursive_crawl.py

@@ -484,9 +484,13 @@ def test_add_archivewebpage_installs_required_chrome_dependency(initialized_arch
     assert "archivewebpage" in binaries
     assert binaries["archivewebpage"]["status"] == Binary.StatusChoices.INSTALLED
     assert binaries["archivewebpage"]["binprovider"] == "chromewebstore"
-    archivewebpage_manifest = Path(binaries["archivewebpage"]["abspath"])
+    archivewebpage_metadata = Path(binaries["archivewebpage"]["abspath"])
+    assert archivewebpage_metadata.exists()
+    assert archivewebpage_metadata.name == "archivewebpage.extension.json"
+    archivewebpage_extension = json.loads(archivewebpage_metadata.read_text(encoding="utf-8"))
+    archivewebpage_manifest = Path(archivewebpage_extension["unpacked_path"]) / "manifest.json"
     assert archivewebpage_manifest.exists()
-    assert archivewebpage_manifest.name == "manifest.json"
+    assert json.loads(archivewebpage_manifest.read_text(encoding="utf-8"))["version"] == binaries["archivewebpage"]["version"]
 
     plugins_seen = {plugin for plugin, _hook_name, _status, _output_str, _output_files in archive_results}
     assert {"chrome", "archivewebpage"}.issubset(plugins_seen)
@@ -515,22 +519,36 @@ def test_add_archivewebpage_installs_required_chrome_dependency(initialized_arch
 
 
 @pytest.mark.timeout(1200)
-def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(initialized_archive, free_tcp_port_factory):
+def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(
+    initialized_archive,
+    free_tcp_port_factory,
+    recursive_test_site,
+):
     """Run a bounded real depth=2 crawl with all plugins enabled and verify parallel snapshot execution."""
 
     from abx_dl.models import discover_plugins
 
-    root_url = "https://example.com/"
-    plugin_selection = ",".join(
-        sorted(plugin for plugin in discover_plugins().keys() if not plugin.startswith("claude")),
-    )
+    root_url = recursive_test_site["root_url"]
+    plugin_selection = ",".join(sorted(plugin for plugin in discover_plugins().keys() if not plugin.startswith("claude")))
     env = os.environ.copy()
-    env.pop("CHROME_BINARY", None)
+    for preinstalled_path_key in (
+        "CHROME_BINARY",
+        "LIB_DIR",
+        "DATA_DIR",
+        "NODE_MODULES_DIR",
+        "NODE_PATH",
+        "PNPM_BIN_DIR",
+        "NPM_BIN_DIR",
+        "CHROMEWEBSTORE_EXTENSIONS_DIR",
+    ):
+        env.pop(preinstalled_path_key, None)
     env.update(
         {
             "USE_COLOR": "false",
             "SHOW_PROGRESS": "false",
+            "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*",
             "ABXPKG_LIB_DIR": str(initialized_archive / "lib"),
+            "CHROMEWEBSTORE_EXTENSIONS_DIR": str(initialized_archive / "lib/chromewebstore/extensions"),
             "TIMEOUT": "90",
             "ABXPKG_INSTALL_TIMEOUT": "900",
             "CRAWL_MAX_CONCURRENT_SNAPSHOTS": "3",
@@ -608,10 +626,26 @@ def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(initia
     assert crawl.status == Crawl.StatusChoices.SEALED
     assert crawl.retry_at is None
 
-    assert len(snapshots) == 8
-    assert any(url == root_url and depth == 0 for _id, url, depth, _status, _parent, _downloaded_at in snapshots)
-    assert any("iana.org" in url and depth == 1 for _id, url, depth, _status, _parent, _downloaded_at in snapshots)
-    assert any(depth == 2 for _id, _url, depth, _status, _parent, _downloaded_at in snapshots)
+    expected_urls = {
+        root_url,
+        *recursive_test_site["child_urls"],
+        *recursive_test_site["deep_urls"],
+    }
+    snapshots_by_url = {url: (snapshot_id, depth, parent_id) for snapshot_id, url, depth, _status, parent_id, _downloaded_at in snapshots}
+    assert set(snapshots_by_url) == expected_urls
+    root_id, root_depth, root_parent_id = snapshots_by_url[root_url]
+    assert root_depth == 0
+    assert root_parent_id is None
+    child_ids_by_url = {}
+    for child_url in recursive_test_site["child_urls"]:
+        child_id, child_depth, child_parent_id = snapshots_by_url[child_url]
+        assert child_depth == 1
+        assert child_parent_id == root_id
+        child_ids_by_url[child_url] = child_id
+    for child_url, deep_url in zip(recursive_test_site["child_urls"], recursive_test_site["deep_urls"], strict=True):
+        _deep_id, deep_depth, deep_parent_id = snapshots_by_url[deep_url]
+        assert deep_depth == 2
+        assert deep_parent_id == child_ids_by_url[child_url]
     assert all(status == Snapshot.StatusChoices.SEALED for _id, _url, _depth, status, _parent, _downloaded_at in snapshots)
     assert all(downloaded_at is not None for _id, _url, _depth, _status, _parent, downloaded_at in snapshots)
 
@@ -634,6 +668,13 @@ def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(initia
         if not (status in allowed_statuses or (plugin == "archivedotorg" and status == ArchiveResult.StatusChoices.FAILED))
     ]
     assert not unexpected_results
+    ytdlp_results = [
+        (url, status, output_str)
+        for _snapshot_id, url, _depth, plugin, _hook_name, status, _files, _size, output_str in archive_results
+        if plugin == "ytdlp"
+    ]
+    assert ytdlp_results
+    assert all(status != ArchiveResult.StatusChoices.FAILED for _url, status, _output_str in ytdlp_results), ytdlp_results
 
     plugins_seen = {plugin for _snapshot_id, _url, _depth, plugin, _hook_name, _status, _files, _size, _output in archive_results}
     assert {
@@ -666,7 +707,11 @@ def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(initia
     assert list(snapshot_root.rglob("favicon/**/*"))
     urls_jsonl_files = list(snapshot_root.rglob("parse_html_urls/urls.jsonl"))
     assert urls_jsonl_files
-    assert any("iana.org" in path.read_text(errors="ignore") for path in urls_jsonl_files)
+    parsed_urls = {
+        json.loads(line)["url"] for path in urls_jsonl_files for line in path.read_text(errors="ignore").splitlines() if line.strip()
+    }
+    assert set(recursive_test_site["child_urls"]).issubset(parsed_urls)
+    assert set(recursive_test_site["deep_urls"]).issubset(parsed_urls)
 
     assert processes
     failed_hook_results = [

archivebox/tests/test_search.py

@@ -1,9 +1,9 @@
 import json
 import os
 import re
-import shutil
 import time
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
 from threading import Thread
 from datetime import timedelta
 from urllib.parse import urlencode
@@ -22,6 +22,7 @@ from archivebox.tests.conftest import (
     create_admin_and_token,
     get_free_port,
     run_archivebox_cmd,
+    resolve_abxpkg_binary_env,
     start_archivebox_server,
     stop_archivebox_process,
     wait_for_http,
@@ -590,7 +591,7 @@ class TestPublicIndexSearch:
         assert response.status_code == 200
         content = response.content.decode()
         assert "/screenshot.png" in content
-        assert "screenshot/screenshot.png" not in content
+        assert "/screenshot/screenshot.png" not in content
 
     @override_settings(PUBLIC_INDEX=True)
     def test_public_index_preview_respects_plugin_relative_screenshot_output(self, client, public_snapshot):
@@ -634,7 +635,7 @@ class TestPublicIndexSearch:
         first = content.index("chrome_extension_screenshot/screenshot-1.png")
         second = content.index("chrome_extension_screenshot/screenshot.png")
         assert first < second
-        assert "screenshot/screenshot.png" not in content
+        assert "/screenshot/screenshot.png" not in content
         assert "chrome_extension_screenshot/screenshot-2.png" not in content
 
     @override_settings(PUBLIC_INDEX=True)
@@ -698,8 +699,18 @@ class TestPublicIndexSearch:
 class TestSearchBackendsE2E:
     @pytest.mark.timeout(360)
     def test_live_public_and_admin_search_matrix_uses_real_cli_indexing_and_streaming(self, initialized_archive):
-        assert shutil.which("rg"), "ripgrep is required for the live search matrix"
-        assert shutil.which("sonic"), "sonic is required for the live search matrix"
+        from abx_plugins import get_plugins_dir
+
+        plugins_dir = Path(get_plugins_dir())
+        binary_env = resolve_abxpkg_binary_env(
+            initialized_archive / "lib",
+            deps_from=[
+                plugins_dir / "search_backend_ripgrep" / "config.json",
+                plugins_dir / "search_backend_sonic" / "config.json",
+            ],
+        )
+        assert Path(binary_env["RIPGREP_BINARY"]).is_file()
+        assert Path(binary_env["SONIC_BINARY"]).is_file()
 
         page_count = 93
         total_snapshot_count = 100
@@ -830,6 +841,7 @@ class TestSearchBackendsE2E:
                 SEARCH_BACKEND_SONIC_ENABLED="True",
                 SEARCH_BACKEND_SONIC_PORT=str(sonic_port),
             )
+            env.update(binary_env)
             create_admin_and_token(initialized_archive)
 
             mercury_capture_urls = [*matrix_urls]

archivebox/tests/test_server_security_browser.py

@@ -5,7 +5,6 @@ from __future__ import annotations
 
 import json
 import os
-import shutil
 import subprocess
 import textwrap
 import time
@@ -14,7 +13,7 @@ from urllib.parse import urlencode
 
 import pytest
 
-from .conftest import _find_cached_chrome, _find_system_browser, run_python_cwd
+from .conftest import resolve_abxpkg_chrome_env, run_python_cwd
 from .conftest import (
     cli_env,
     get_free_port,
@@ -257,42 +256,8 @@ main().catch((error) => {
 """
 
 
-def _resolve_browser(shared_lib: Path) -> Path | None:
-    env_browser = os.environ.get("CHROME_BINARY") or os.environ.get("CHROME_BIN")
-    if env_browser:
-        candidate = Path(env_browser).expanduser()
-        if candidate.exists():
-            return candidate
-
-    system = _find_system_browser()
-    if system and system.exists():
-        return system
-
-    cached = _find_cached_chrome(shared_lib)
-    if cached and cached.exists():
-        return cached
-
-    which_candidates = ("chromium", "chromium-browser")
-    for binary in which_candidates:
-        resolved = shutil.which(binary)
-        if resolved:
-            return Path(resolved)
-
-    mac_candidates = (
-        Path("/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary"),
-        Path("/Applications/Chromium.app/Contents/MacOS/Chromium"),
-    )
-    for candidate in mac_candidates:
-        if candidate.exists():
-            return candidate
-
-    return None
-
-
 @pytest.fixture
 def browser_runtime(initialized_archive: Path):
-    assert shutil.which("node") is not None, "Node.js is required for browser security tests"
-
     shared_lib = initialized_archive / "lib"
     env = cli_env(
         ABXPKG_INSTALL_TIMEOUT="900",
@@ -311,13 +276,14 @@ def browser_runtime(initialized_archive: Path):
     )
     assert install_result.returncode == 0, install_result.stderr or install_result.stdout
 
-    browser = _resolve_browser(shared_lib)
-    assert browser, "No Chrome/Chromium binary available for browser security tests"
+    resolved_env = resolve_abxpkg_chrome_env(shared_lib, env)
 
     return {
         "lib_dir": shared_lib,
-        "node_modules_dir": shared_lib / "pnpm" / "packages" / "chrome" / "node_modules",
-        "chrome_binary": browser,
+        "node_modules_dir": Path(resolved_env["NODE_MODULES_DIR"]),
+        "node_path": resolved_env["NODE_PATH"],
+        "node_binary": Path(resolved_env["NODE_BINARY"]),
+        "chrome_binary": Path(resolved_env["CHROME_BINARY"]),
     }
 
 
@@ -438,7 +404,12 @@ def _seed_archive(data_dir: Path) -> dict[str, object]:
     return json.loads(stdout.strip())
 
 
-def _build_probe_config(mode: str, port: int, fixture: dict[str, object], runtime: dict[str, Path]) -> dict[str, str]:
+def _build_probe_config(
+    mode: str,
+    port: int,
+    fixture: dict[str, object],
+    runtime: dict[str, Path | str],
+) -> dict[str, str]:
     snapshots = fixture["snapshots"]
     attacker = snapshots["attacker"]
     victim = snapshots["victim"]
@@ -478,7 +449,7 @@ def _build_probe_config(mode: str, port: int, fixture: dict[str, object], runtim
 
 def _run_browser_probe(
     data_dir: Path,
-    runtime: dict[str, Path],
+    runtime: dict[str, Path | str],
     mode: str,
     fixture: dict[str, object],
     tmp_path: Path,
@@ -533,14 +504,14 @@ def _run_browser_probe(
     probe_config = _build_probe_config(mode, port, fixture, runtime)
 
     env = os.environ.copy()
-    env["NODE_PATH"] = str(runtime["node_modules_dir"])
+    env["NODE_PATH"] = str(runtime["node_path"])
     env["NODE_MODULES_DIR"] = str(runtime["node_modules_dir"])
     env["CHROME_BINARY"] = str(runtime["chrome_binary"])
     env["USE_COLOR"] = "False"
 
     try:
         result = subprocess.run(
-            ["node", str(probe_path)],
+            [str(runtime["node_binary"]), str(probe_path)],
             cwd=data_dir,
             env=env,
             input=json.dumps(probe_config),
@@ -595,18 +566,23 @@ def _wait_for_archivewebpage_capture(data_dir: Path, url: str, timeout: float =
     raise AssertionError(f"timed out waiting for archivewebpage capture: {last_state}")
 
 
-def _run_wacz_preview_probe(data_dir: Path, runtime: dict[str, Path], detail_url: str, tmp_path: Path) -> dict[str, object]:
+def _run_wacz_preview_probe(
+    data_dir: Path,
+    runtime: dict[str, Path | str],
+    detail_url: str,
+    tmp_path: Path,
+) -> dict[str, object]:
     probe_path = tmp_path / "wacz_preview_probe.js"
     probe_path.write_text(PUPPETEER_WACZ_PREVIEW_SCRIPT, encoding="utf-8")
 
     env = os.environ.copy()
-    env["NODE_PATH"] = str(runtime["node_modules_dir"])
+    env["NODE_PATH"] = str(runtime["node_path"])
     env["NODE_MODULES_DIR"] = str(runtime["node_modules_dir"])
     env["CHROME_BINARY"] = str(runtime["chrome_binary"])
     env["USE_COLOR"] = "False"
 
     result = subprocess.run(
-        ["node", str(probe_path)],
+        [str(runtime["node_binary"]), str(probe_path)],
         cwd=data_dir,
         env=env,
         input=json.dumps(
@@ -776,7 +752,7 @@ def test_archivewebpage_wacz_preview_serves_real_capture_frame(initialized_archi
         TIMEOUT="90",
     )
     env["ABXPKG_LIB_DIR"] = str(browser_runtime["lib_dir"])
-    env["NODE_PATH"] = str(browser_runtime["node_modules_dir"])
+    env["NODE_PATH"] = str(browser_runtime["node_path"])
     env["NODE_MODULES_DIR"] = str(browser_runtime["node_modules_dir"])
     env["CHROMEWEBSTORE_EXTENSIONS_DIR"] = str(
         browser_runtime["lib_dir"] / "chromewebstore" / "extensions",

archivebox/tests/test_snapshot_pause.py

@@ -1 +0,0 @@
-# test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_finished_rows moved to test_api_v1_core_snapshot_snapshot_id.py.

archivebox/tests/test_snapshot_service.py

@@ -80,6 +80,3 @@ def test_snapshot_service_cli_add_seals_snapshot_and_writes_indexes(tmp_path, re
     assert wget_files
     assert any("Root" in path.read_text(encoding="utf-8", errors="ignore") for path in wget_files if path.suffix in (".html", ".txt"))
     assert any(result["plugin"] == "wget" and result["status"] == ArchiveResult.StatusChoices.SUCCEEDED for result in state["results"])
-
-
-# test_paused_snapshot_survives_server_restart_and_resumes_via_api moved to test_api_v1_core_snapshot_snapshot_id.py.

archivebox/tests/test_takeover_util.py

@@ -2,14 +2,15 @@
 """Takeover utility tests and live command handoff flows."""
 
 import os
+import json
 import signal
-import shutil
 import subprocess
 import sys
 import time
 from pathlib import Path
 
 import pytest
+import psutil
 
 from archivebox.core.models import ArchiveResult, Snapshot
 from archivebox.crawls.models import Crawl
@@ -38,8 +39,40 @@ from archivebox.tests.test_orm_helpers import use_archivebox_db
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
-def _require_sonic_binary() -> None:
-    assert shutil.which("sonic") is not None, "sonic server binary is required for Sonic worker takeover tests"
+def test_pid_is_alive_treats_unreaped_zombie_as_exited():
+    proc = subprocess.Popen([sys.executable, "-c", "pass"])
+    try:
+        deadline = time.time() + 5
+        while time.time() < deadline and psutil.Process(proc.pid).status() != psutil.STATUS_ZOMBIE:
+            time.sleep(0.01)
+
+        assert psutil.Process(proc.pid).status() == psutil.STATUS_ZOMBIE
+        assert not pid_is_alive(proc.pid)
+    finally:
+        proc.wait(timeout=5)
+
+
+def _resolve_sonic_env(env: dict[str, str]) -> dict[str, str]:
+    from abx_plugins import get_plugins_dir
+
+    config = Path(get_plugins_dir()) / "search_backend_sonic" / "config.json"
+    result = subprocess.run(
+        [
+            str(Path(sys.executable).with_name("abxpkg")),
+            "env",
+            "--install",
+            "--json",
+            f"--lib={env['ABXPKG_LIB_DIR']}",
+            f"--deps-from={config}:required_binaries",
+        ],
+        capture_output=True,
+        text=True,
+        env=env,
+    )
+    assert result.returncode == 0, result.stderr or result.stdout
+    payload = {str(key): str(value) for key, value in json.loads(result.stdout).items()}
+    assert Path(payload["SONIC_BINARY"]).is_file()
+    return payload
 
 
 def _archive_pages_for_sqlite_reindexing(data_dir: Path, env: dict[str, str], root_url: str) -> None:
@@ -68,14 +101,13 @@ def _archive_pages_for_sqlite_reindexing(data_dir: Path, env: dict[str, str], ro
 
 @pytest.mark.timeout(360)
 def test_behavior_update_index_only_keeps_server_http_and_search_visible(tmp_path, initialized_archive, recursive_test_site):
-    _require_sonic_binary()
-
     env = cli_env(
         live=True,
         PLUGINS="wget,parse_html_urls,search_backend_sqlite",
         SEARCH_BACKEND_ENGINE="sqlite",
         SEARCH_BACKEND_SONIC_PORT=str(get_free_port()),
     )
+    env.update(_resolve_sonic_env(env))
     root_url = recursive_test_site["root_url"]
     _archive_pages_for_sqlite_reindexing(tmp_path, env, root_url)
 
@@ -113,14 +145,13 @@ def test_behavior_update_index_only_keeps_server_http_and_search_visible(tmp_pat
 
 @pytest.mark.timeout(420)
 def test_behavior_update_yields_to_server_then_finishes_visible_indexing(tmp_path, initialized_archive, recursive_test_site):
-    _require_sonic_binary()
-
     env = cli_env(
         live=True,
         PLUGINS="wget,parse_html_urls,search_backend_sqlite",
         SEARCH_BACKEND_ENGINE="sqlite",
         SEARCH_BACKEND_SONIC_PORT=str(get_free_port()),
     )
+    env.update(_resolve_sonic_env(env))
     root_url = recursive_test_site["root_url"]
     _archive_pages_for_sqlite_reindexing(tmp_path, env, root_url)
 
@@ -340,14 +371,13 @@ def test_live_update_index_only_does_not_take_over_server_runtime(tmp_path, init
 
 @pytest.mark.timeout(360)
 def test_live_server_keeps_http_runtime_while_update_runs_real_sqlite_indexer(tmp_path, initialized_archive, recursive_test_site):
-    _require_sonic_binary()
-
     env = cli_env(
         live=True,
         PLUGINS="wget,parse_html_urls,search_backend_sqlite,search_backend_sonic",
         SEARCH_BACKEND_ENGINE="sqlite",
         SEARCH_BACKEND_SONIC_PORT=str(get_free_port()),
     )
+    env.update(_resolve_sonic_env(env))
     _archive_pages_for_sqlite_reindexing(tmp_path, env, recursive_test_site["root_url"])
 
     port = get_free_port()
@@ -430,14 +460,13 @@ def test_live_server_keeps_http_runtime_while_update_runs_real_sqlite_indexer(tm
 
 @pytest.mark.timeout(420)
 def test_live_update_yields_to_server_then_reclaims_real_sqlite_indexing(tmp_path, initialized_archive, recursive_test_site):
-    _require_sonic_binary()
-
     env = cli_env(
         live=True,
         PLUGINS="wget,parse_html_urls,search_backend_sqlite,search_backend_sonic",
         SEARCH_BACKEND_ENGINE="sqlite",
         SEARCH_BACKEND_SONIC_PORT=str(get_free_port()),
     )
+    env.update(_resolve_sonic_env(env))
     _archive_pages_for_sqlite_reindexing(tmp_path, env, recursive_test_site["root_url"])
 
     port = get_free_port()

archivebox/tests/test_ui_add_view.py

@@ -99,6 +99,19 @@ def test_add_view_admin_renders_plugin_config_grid(client, admin_user, monkeypat
     assert b"plugin_config__wget__WGET_ENABLED" not in response.content
 
 
+def test_add_view_hides_agent_link_when_opencode_is_disabled(client, admin_user):
+    from archivebox.machine.models import Machine
+
+    Machine.from_json({"config": {"OPENCODE_ENABLED": False}})
+    client.force_login(admin_user)
+
+    response = client.get(reverse("add"), HTTP_HOST=ADMIN_HOST)
+
+    assert response.status_code == 200
+    assert b"/admin/agent" not in response.content
+    assert b"Crawl with AI" not in response.content
+
+
 def test_add_view_staff_user_cannot_override_raw_or_plugin_config(client):
     staff_user = User.objects.create_user(
         username="addviewstaff",

archivebox/tests/test_ui_add_view_runtime.py

@@ -344,6 +344,102 @@ def _wait_for_worker_state(cwd, worker_name: str, statename: str, timeout: int =
     raise AssertionError(f"Timed out waiting for {worker_name}={statename}, last state={state}")
 
 
+@pytest.mark.timeout(240)
+def test_public_add_view_depth_one_crawl_skips_unreadable_persona_profile_entries(tmp_path, recursive_test_site):
+    init_archive(tmp_path)
+
+    chrome_profile = tmp_path / "personas" / "Default" / "chrome_profile"
+    default_profile = chrome_profile / "Default"
+    default_profile.mkdir(parents=True, exist_ok=True)
+    (default_profile / "Preferences").write_text('{"profile":{"name":"Default"}}', encoding="utf-8")
+    unreadable_dir = chrome_profile / "ActorSafetyLists"
+    unreadable_dir.mkdir(parents=True, exist_ok=True)
+    (unreadable_dir / "Preferences").write_text("{}", encoding="utf-8")
+    unreadable_dir.chmod(0)
+
+    port = get_free_port()
+    env = cli_env(
+        port=port,
+        PLUGINS="wget,parse_html_urls",
+        SAVE_WGET="True",
+        PUBLIC_INDEX="True",
+        PUBLIC_ADD_VIEW="True",
+        URL_ALLOWLIST=r"127\.0\.0\.1[:/].*",
+    )
+
+    try:
+        start_archivebox_server(tmp_path, env=env, port=port)
+        add_page = wait_for_http(port, host=f"web.archivebox.localhost:{port}", path="/add/")
+        assert add_page.status_code == 200
+        assert 'name="depth"' in add_page.text
+
+        response = requests.post(
+            f"http://127.0.0.1:{port}/add/",
+            headers={"Host": f"web.archivebox.localhost:{port}", "Referer": f"http://web.archivebox.localhost:{port}/add/"},
+            data={
+                "url": recursive_test_site["root_url"],
+                "depth": "1",
+                "max_urls": "10",
+                "crawl_max_size": "0",
+                "snapshot_max_size": "0",
+                "tag": "public-depth-one-unreadable-profile",
+                "url_filters_allowlist": r"127\.0\.0\.1[:/].*",
+                "url_filters_denylist": "",
+                "schedule": "",
+                "notes": "public depth one with unreadable chrome profile entries",
+                "persona": "Default",
+                "permissions": "public",
+                "start_paused": "",
+                "config": "{}",
+            },
+            timeout=10,
+            allow_redirects=False,
+        )
+        assert response.status_code in (302, 303), response.text
+
+        deadline = time.time() + 180
+        while time.time() < deadline:
+            with use_archivebox_db(tmp_path):
+                crawl = Crawl.objects.order_by("-created_at").first()
+                depth_counts = get_depth_counts(tmp_path)
+                child_urls = set(Snapshot.objects.filter(depth=1).values_list("url", flat=True))
+                root_status = Snapshot.objects.filter(url=recursive_test_site["root_url"], depth=0).values_list("status", flat=True).first()
+                failed_results = list(
+                    ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.FAILED).values_list("plugin", "output_str"),
+                )
+            if (
+                depth_counts.get(0, 0) >= 1
+                and set(recursive_test_site["child_urls"]).issubset(child_urls)
+                and root_status == Snapshot.StatusChoices.SEALED
+            ):
+                break
+            assert not failed_results
+            time.sleep(2)
+        else:
+            raise AssertionError(f"timed out waiting for depth=1 crawl, got depth counts {get_depth_counts(tmp_path)}")
+
+        with use_archivebox_db(tmp_path):
+            crawl = Crawl.objects.order_by("-created_at").first()
+            snapshot_rows = list(Snapshot.objects.order_by("depth", "url").values_list("url", "depth", "status"))
+            result_rows = list(ArchiveResult.objects.order_by("plugin", "status").values_list("plugin", "status", "output_size"))
+    finally:
+        unreadable_dir.chmod(0o700)
+        stop_server(tmp_path)
+
+    assert crawl is not None
+    assert crawl.max_depth == 1
+    assert crawl.status in {Crawl.StatusChoices.STARTED, Crawl.StatusChoices.SEALED}
+    assert (recursive_test_site["root_url"], 0, Snapshot.StatusChoices.SEALED) in snapshot_rows
+    assert set(recursive_test_site["child_urls"]).issubset({url for url, depth, _status in snapshot_rows if depth == 1})
+    assert all(
+        status in {Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.SEALED}
+        for _url, _depth, status in snapshot_rows
+    )
+    assert not [row for row in result_rows if row[1] == ArchiveResult.StatusChoices.FAILED]
+    assert any(plugin == "wget" and status == ArchiveResult.StatusChoices.SUCCEEDED and size > 0 for plugin, status, size in result_rows)
+    assert any(plugin == "parse_html_urls" and status == ArchiveResult.StatusChoices.SUCCEEDED for plugin, status, _size in result_rows)
+
+
 @pytest.mark.timeout(420)
 def test_public_add_view_import_text_formats_preserve_metadata_and_resume_without_duplicates(tmp_path):
     """Public /add/ textarea should import rich text, survive runner restart, and preserve one row per URL."""
@@ -516,9 +612,10 @@ def test_public_add_view_rejects_file_path_and_shell_injection_payloads(tmp_path
     with use_archivebox_db(tmp_path):
         snapshot = Snapshot.objects.get(url=safe_url)
         crawl = Crawl.objects.get()
+        tag_names = set(snapshot.tags.values_list("name", flat=True))
     assert crawl.status in {Crawl.StatusChoices.STARTED, Crawl.StatusChoices.SEALED}
     assert snapshot.status in {Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.SEALED}
-    assert "public-ui-security" in set(snapshot.tags.values_list("name", flat=True))
+    assert "public-ui-security" in tag_names
 
 
 @pytest.mark.timeout(180)
@@ -580,7 +677,10 @@ def test_add_view_post_creates_schedule_over_server(tmp_path, recursive_test_sit
 
         with use_archivebox_db(tmp_path):
             schedule = CrawlSchedule.objects.select_related("template").order_by("-created_at").first()
-            row = (schedule.schedule, schedule.template.urls, schedule.template.tags_str) if schedule else None
+            row = None
+            if schedule:
+                template_url = json.loads(schedule.template.urls)["url"]
+                row = (schedule.schedule, template_url, schedule.template.tags_str)
 
         assert row == ("daily", recursive_test_site["root_url"], "web-ui")
     finally:

archivebox/tests/test_ui_admin_links.py

@@ -239,6 +239,19 @@ def test_snapshot_admin_zip_links():
     assert html.escape(zip_url, quote=True) in str(admin.admin_actions(snapshot))
 
 
+def test_admin_navigation_hides_agent_link_when_opencode_is_disabled(client, admin_user):
+    from archivebox.machine.models import Machine
+
+    Machine.from_json({"config": {"OPENCODE_ENABLED": False}})
+    client.force_login(admin_user)
+
+    response = client.get(reverse("admin:index"), HTTP_HOST="admin.archivebox.localhost:8000")
+
+    assert response.status_code == 200
+    assert b"/admin/agent" not in response.content
+    assert b">\xf0\x9f\x92\xac AI<" not in response.content
+
+
 def test_archiveresult_admin_zip_links():
     from archivebox.core.admin_archiveresults import ArchiveResultAdmin
     from archivebox.core.models import ArchiveResult

archivebox/tests/test_ui_admin_persona.py

@@ -94,6 +94,28 @@ def test_discover_persona_template_profiles_finds_data_dir_personas():
     assert matching[0].user_data_dir.parent.parent == CONSTANTS.PERSONAS_DIR.resolve()
 
 
+def test_persona_admin_add_view_skips_unreadable_chrome_profile_entries(admin_client):
+    from archivebox.config.constants import CONSTANTS
+
+    chrome_profile = CONSTANTS.PERSONAS_DIR / "UnreadableTemplatePersona" / "chrome_profile"
+    default_profile = chrome_profile / "Default"
+    default_profile.mkdir(parents=True, exist_ok=True)
+    (default_profile / "Preferences").write_text('{"profile":{"name":"Default"}}')
+    unreadable_dir = chrome_profile / "ActorSafetyLists"
+    unreadable_dir.mkdir(parents=True, exist_ok=True)
+    (unreadable_dir / "Preferences").write_text("{}")
+    unreadable_dir.chmod(0)
+
+    try:
+        response = admin_client.get(reverse("admin:personas_persona_add"), HTTP_HOST=ADMIN_TEST_HOST)
+    finally:
+        unreadable_dir.chmod(0o700)
+
+    assert response.status_code == 200
+    assert b"UnreadableTemplatePersona" in response.content
+    assert b"Persona Template" in response.content
+
+
 def test_persona_admin_add_view_renders_import_ui(admin_client):
     source = _make_persona_template_source("RenderImportPersona")
 

archivebox/tests/test_ui_admin_snapshot.py

@@ -32,6 +32,25 @@ def test_snapshot_changelist_uses_stable_ordering_without_unordered_paginator_wa
     assert b"Searching matching snapshots..." in response.content
 
 
+def test_snapshot_changelist_preview_uses_prefetched_output_files(admin_client, snapshot):
+    from archivebox.core.models import ArchiveResult
+
+    ArchiveResult.objects.create(
+        snapshot=snapshot,
+        plugin="screenshot",
+        hook_name="on_Snapshot__40_screenshot.js",
+        status=ArchiveResult.StatusChoices.SUCCEEDED,
+        output_size=128,
+        output_files={"screenshot.png": {"size": 128, "root_relative": True}},
+    )
+
+    response = admin_client.get(reverse("admin:core_snapshot_changelist"), HTTP_HOST=ADMIN_TEST_HOST)
+
+    assert response.status_code == 200
+    assert b'class="snapshot-preview' in response.content
+    assert b"screenshot.png" in response.content
+
+
 def test_snapshot_admin_tag_editor_escapes_tag_json_script_breakout(admin_client, snapshot):
     from archivebox.core.models import Tag
 
@@ -188,10 +207,8 @@ def test_snapshot_admin_preview_uses_extension_screenshot_when_standard_screensh
     preview = admin._get_preview_data(snapshot)
 
     assert preview is not None
-    assert "screenshot/screenshot.png" in preview["img_url"]
-    first = preview["fallback_list"].index("chrome_extension_screenshot/screenshot-1.png")
-    second = preview["fallback_list"].index("chrome_extension_screenshot/screenshot.png")
-    assert first < second
+    assert "chrome_extension_screenshot/screenshot-1.png" in preview["img_url"]
+    assert "chrome_extension_screenshot/screenshot.png" in preview["fallback_list"]
     assert "chrome_extension_screenshot/screenshot-2.png" not in preview["fallback_list"]
 
 

archivebox/tests/test_ui_admin_tag.py

@@ -26,10 +26,3 @@ def test_tag_admin_add_view_renders_similar_tag_reference(admin_client):
     assert response.status_code == 200
     assert b"Similar Tags" in response.content
     assert b'data-tag-name-input="1"' in response.content
-
-
-# test_tag_search_api_returns_card_payload moved to test_api_v1_core_tags_search.py.
-# test_tag_search_api_respects_sort_and_filters moved to test_api_v1_core_tags_search.py.
-# test_tag_rename_api_updates_name moved to test_api_v1_core_tag_tag_id_rename.py.
-# test_tag_snapshots_export_returns_jsonl moved to test_api_v1_core_tag_tag_id_snapshots_jsonl.py.
-# test_tag_urls_export_returns_plain_text_urls moved to test_api_v1_core_tag_tag_id_urls_txt.py.

archivebox/tests/test_version_metadata.py

@@ -34,6 +34,25 @@ def test_get_commit_hash_from_docker_version_file_ignores_short_hash(monkeypatch
     assert version.get_COMMIT_HASH() == commit_hash
 
 
+def test_get_commit_hash_from_docker_version_file_prefers_last_full_hash(monkeypatch) -> None:
+    base_hash = "a" * 40
+    archivebox_hash = "b" * 40
+
+    class VersionPath:
+        def __init__(self, path: str):
+            self.path = path
+
+        def read_text(self) -> str:
+            assert self.path == "/VERSION.txt"
+            return f"COMMIT_HASH={base_hash}\nCOMMIT_HASH={archivebox_hash}\n"
+
+    monkeypatch.setattr(version, "IN_DOCKER", True)
+    monkeypatch.setattr(version, "Path", VersionPath)
+    version.get_COMMIT_HASH.cache_clear()
+
+    assert version.get_COMMIT_HASH() == archivebox_hash
+
+
 def test_get_commit_hash_from_detached_head(monkeypatch, tmp_path) -> None:
     commit_hash = "a" * 40
     package_dir = tmp_path / "archivebox"

bin/deploy_dev_demo.sh

@@ -60,6 +60,9 @@ cat > .archivebox-deploy.override.yml <<EOF
 services:
   $DEPLOY_SERVICE:
     image: $DEPLOY_IMAGE
+    user: ""
+    entrypoint: null
+    command: null
 EOF
 
 COMPOSE=(docker compose -f "$COMPOSE_FILE" -f .archivebox-deploy.override.yml)

bin/release.sh

@@ -266,87 +266,6 @@ create_git_tag() {
     git push origin "refs/tags/${tag}"
 }
 
-wait_for_runs() {
-    local slug="$1"
-    local event="$2"
-    local sha="$3"
-    local label="$4"
-    local runs_json
-    local attempts=0
-
-    while :; do
-        runs_json="$(GH_FORCE_TTY=0 GH_PAGER=cat gh run list --repo "${slug}" --event "${event}" --commit "${sha}" --limit 20 --json databaseId,status,conclusion,workflowName)"
-        if [[ "$(jq 'length' <<<"${runs_json}")" -gt 0 ]]; then
-            break
-        fi
-        attempts=$((attempts + 1))
-        if [[ "${attempts}" -ge 30 ]]; then
-            echo "Timed out waiting for ${label} workflows to start" >&2
-            return 1
-        fi
-        sleep 10
-    done
-
-    while IFS=$'\t' read -r run_id workflow_name; do
-        workflow_name_lower="${workflow_name,,}"
-        if [[ "${workflow_name_lower}" == *"release state"* ]]; then
-            gh run watch "${run_id}" --repo "${slug}" --exit-status
-            continue
-        fi
-        if [[ "${workflow_name_lower}" != *"test"* ]]; then
-            echo "Skipping non-gating workflow: ${workflow_name}"
-            continue
-        fi
-
-        attempts=0
-        while :; do
-            precheck_state="$(
-                gh run view "${run_id}" --repo "${slug}" --json jobs --jq '
-                    [.jobs[] | select((.name | ascii_downcase) | test("precheck|pre-commit|prek"))][0]
-                    | if . == null then "missing:" else ((.status // "") + ":" + (.conclusion // "")) end
-                '
-            )"
-            case "${precheck_state}" in
-                missing:*)
-                    echo "Skipping test workflow without precheck job: ${workflow_name}"
-                    break
-                    ;;
-                completed:success|completed:skipped)
-                    break
-                    ;;
-                completed:failure|completed:cancelled|completed:timed_out)
-                    gh run view "${run_id}" --repo "${slug}"
-                    return 1
-                    ;;
-            esac
-            attempts=$((attempts + 1))
-            if [[ "${attempts}" -ge 120 ]]; then
-                echo "Timed out waiting for ${workflow_name} precheck job" >&2
-                return 1
-            fi
-            sleep 5
-        done
-    done < <(jq -r '.[] | [.databaseId, .workflowName] | @tsv' <<<"${runs_json}")
-}
-
-wait_for_pypi() {
-    local package_name="$1"
-    local expected_version="$2"
-    local attempts=0
-
-    while :; do
-        if curl -fsSL "https://pypi.org/pypi/${package_name}/json" | jq -e --arg version "${expected_version}" '.releases[$version] | length > 0' >/dev/null; then
-            return 0
-        fi
-        attempts=$((attempts + 1))
-        if [[ "${attempts}" -ge 30 ]]; then
-            echo "Timed out waiting for ${package_name}==${expected_version} on PyPI" >&2
-            return 1
-        fi
-        sleep 10
-    done
-}
-
 run_checks() {
     uv sync --extra sonic --extra debug --all-groups --no-cache --upgrade
     uv build --all
@@ -428,7 +347,6 @@ publish_artifacts() {
         uv publish --trusted-publishing always "${artifacts[@]}"
     fi
 
-    wait_for_pypi "${PYPI_PACKAGE}" "${version}"
 }
 
 main() {

bin/setup.sh

@@ -49,14 +49,22 @@ docker_pull_archivebox() {
     fi
 }
 
-docker_run_archivebox_init() {
+docker_run_archivebox() {
     if [ -n "$ARCHIVEBOX_PLATFORM" ]; then
-        docker run --platform "$ARCHIVEBOX_PLATFORM" "$DOCKER_RUN_TTY_ARG" -v "$PWD":/data --rm "$ARCHIVEBOX_IMAGE" init --install
+        docker run --platform "$ARCHIVEBOX_PLATFORM" "$DOCKER_RUN_TTY_ARG" -v "$PWD":/data --rm "$ARCHIVEBOX_IMAGE" "$@"
     else
-        docker run "$DOCKER_RUN_TTY_ARG" -v "$PWD":/data --rm "$ARCHIVEBOX_IMAGE" init --install
+        docker run "$DOCKER_RUN_TTY_ARG" -v "$PWD":/data --rm "$ARCHIVEBOX_IMAGE" "$@"
     fi
 }
 
+docker_run_archivebox_init() {
+    docker_run_archivebox init
+}
+
+docker_run_archivebox_install() {
+    docker_run_archivebox install
+}
+
 docker_run_archivebox_server() {
     if [ -n "$ARCHIVEBOX_PLATFORM" ]; then
         docker run --platform "$ARCHIVEBOX_PLATFORM" -v "$PWD":/data -d -p 8000:8000 --name=archivebox "$ARCHIVEBOX_IMAGE"
@@ -128,11 +136,7 @@ install_archivebox_with_uv() {
 
     echo
     echo "[+] Installing ArchiveBox python tool using uv from $ARCHIVEBOX_PACKAGE..."
-    if uv --no-config tool add --help > /dev/null 2>&1; then
-        uv --no-config tool add --python "$ARCHIVEBOX_PYTHON" --upgrade "$ARCHIVEBOX_PACKAGE"
-    else
-        uv --no-config tool install --python "$ARCHIVEBOX_PYTHON" --upgrade "$ARCHIVEBOX_PACKAGE"
-    fi
+    uv --no-config tool install --python "$ARCHIVEBOX_PYTHON" --upgrade "$ARCHIVEBOX_PACKAGE"
 
     uv_tool_bin_dir="$(uv --no-config tool dir --bin)"
     export PATH="$uv_tool_bin_dir:$PATH"
@@ -160,7 +164,8 @@ if (command -v docker > /dev/null && docker compose version > /dev/null && docke
     fi
     curl -fsSL "$ARCHIVEBOX_COMPOSE_URL" > docker-compose.yml
     export ARCHIVEBOX_IMAGE ARCHIVEBOX_PLATFORM
-    docker_compose_run_archivebox init --install
+    docker_compose_run_archivebox init
+    docker_compose_run_archivebox install
     echo
     echo "[+] Starting ArchiveBox server using: docker compose up -d..."
     docker compose up -d
@@ -187,8 +192,9 @@ elif (command -v docker > /dev/null && docker_pull_archivebox); then
     fi
     cd ./data
     docker_run_archivebox_init
+    docker_run_archivebox_install
     echo
-    echo "[+] Starting ArchiveBox server using: docker run -d archivebox/archivebox..."
+    echo "[+] Starting ArchiveBox server using: docker run -d $ARCHIVEBOX_IMAGE..."
     docker_run_archivebox_server
     wait_for_archivebox
     open_archivebox
@@ -213,21 +219,20 @@ echo "    ⚠️ If you want to use Docker, press [Ctrl-C] to cancel now. ⚠️
 echo "        Get Docker: https://docs.docker.com/get-docker/"
 echo "        After you've installed Docker, run this script again."
 echo
-echo "Otherwise, install will continue with apt/brew/pkg + uv in 12s... (press [Ctrl+C] to cancel)"
+echo "Otherwise, install will continue with uv in 12s... (press [Ctrl+C] to cancel)"
 echo
 sleep 12 || exit 1
-echo "Proceeding with system package manager..."
+echo "Proceeding with uv..."
 echo
 
 echo "[i] ArchiveBox Setup Script 📦"
 echo
-echo "    This is a helper script which installs ArchiveBox and bootstraps its Python/Node runtimes."
+echo "    This is a helper script which installs ArchiveBox and bootstraps its runtime dependencies."
 echo "    You may be prompted for a sudo password in order to install the following:"
 echo
-echo "        - archivebox"
-echo "        - python3, uv, nodejs, npm             (languages used by ArchiveBox and plugin installers)"
-echo "        - curl, wget                           (used to bootstrap package installation)"
-echo "        - extractor/plugin dependencies        (installed/discovered by archivebox init --install)"
+echo "        - uv / curl / ca-certificates          (as needed to bootstrap ArchiveBox)"
+echo "        - archivebox                           (installed by uv with Python $ARCHIVEBOX_PYTHON)"
+echo "        - extractor/plugin dependencies        (installed/discovered by archivebox install)"
 echo
 echo "    If you'd rather install these manually as-needed, you can find detailed documentation here:"
 echo "        https://github.com/ArchiveBox/ArchiveBox/wiki/Install"
@@ -235,43 +240,32 @@ echo
 echo "Continuing in 12s... (press [Ctrl+C] to cancel)"
 echo
 sleep 12 || exit 1
-echo "Proceeding to install dependencies..."
+echo "Proceeding to install ArchiveBox..."
 echo
 
-# On Linux:
-if which apt-get > /dev/null; then
-    echo "[+] Installing ArchiveBox system dependencies using apt..."
-    sudo apt-get update -qq
-    sudo apt-get install -y python3 python3-venv wget curl nodejs npm
-    install_archivebox_with_uv
-# On Mac:
-elif which brew > /dev/null; then
-    echo "[+] Installing ArchiveBox using Homebrew..."
-    brew tap archivebox/archivebox
-    brew update
-    brew install archivebox
-elif which pkg > /dev/null; then
-    echo "[+] Installing ArchiveBox system dependencies using pkg and uv..."
-    sudo pkg install -y python3 py39-sqlite3 npm wget curl
-    install_archivebox_with_uv
-else
-    echo "[!] Warning: Could not find aptitude/homebrew/pkg! May not be able to install all dependencies automatically."
-    echo
-    echo "    If you're on macOS, make sure you have homebrew installed:     https://brew.sh/"
-    echo "    If you're on Linux, only Ubuntu/Debian/BSD systems are officially supported with this script."
-    echo "    If you're on Windows, this script is not officially supported (Docker is recommended instead)."
-    echo
-    echo "See the README.md for Manual Setup & Troubleshooting instructions if you you're unable to run ArchiveBox after this script completes."
+if ! command -v uv > /dev/null 2>&1 && ! command -v curl > /dev/null 2>&1 && ! command -v wget > /dev/null 2>&1; then
+    if command -v apt-get > /dev/null 2>&1; then
+        echo "[+] Installing curl and CA certificates to bootstrap uv using apt..."
+        sudo apt-get update -qq
+        sudo apt-get install -y curl ca-certificates
+    elif command -v brew > /dev/null 2>&1; then
+        echo "[+] Installing uv using Homebrew..."
+        brew install uv
+    elif command -v pkg > /dev/null 2>&1; then
+        echo "[+] Installing curl and CA certificates to bootstrap uv using pkg..."
+        sudo pkg install -y curl ca_root_nss
+    else
+        echo "[!] Warning: Could not find uv, curl, wget, apt, brew, or pkg."
+        echo
+        echo "See the README.md for Manual Setup & Troubleshooting instructions if you you're unable to run ArchiveBox after this script completes."
+    fi
 fi
 
 echo
 
-if ! which archivebox > /dev/null 2>&1; then
-    ensure_uv
-    export PATH="$(uv --no-config tool dir --bin):$PATH"
-fi
+install_archivebox_with_uv
 
-if ! which archivebox > /dev/null 2>&1; then
+if ! command -v archivebox > /dev/null 2>&1; then
     echo "[X] archivebox command was not found in PATH after installing!"
     echo "    Check to see if a previous step failed."
     exit 1
@@ -285,7 +279,8 @@ if [ -f "./index.sqlite3" ]; then
     mv -i ~/archivebox/* ~/archivebox/data/
 fi
 cd ./data
-: | archivebox init --install   # pipe in empty command to make sure stdin is closed
+: | archivebox init   # pipe in empty command to make sure stdin is closed
+archivebox install
 # init shows version output at the end too
 echo
 echo "[+] Starting ArchiveBox server using: nohup archivebox server &..."
@@ -299,7 +294,7 @@ echo "    archivebox server --quick-init 0.0.0.0:8000        # start server proc
 echo "    archivebox manage createsuperuser                  # add an admin user+pass"
 echo "    ps aux | grep archivebox                           # see server process pid"
 echo "    pkill -f archivebox                                # stop the server"
-echo "    uv tool install --python $ARCHIVEBOX_PYTHON --upgrade '$ARCHIVEBOX_PACKAGE'; archivebox init  # update versions"
+echo "    uv tool install --python $ARCHIVEBOX_PYTHON --upgrade '$ARCHIVEBOX_PACKAGE'; archivebox init; archivebox install  # update versions"
 echo "    archivebox add 'https://example.com'"              # archive a new URL
 echo "    archivebox list                                    # see URLs archived"
 echo "    archivebox help                                    # see more help & examples"

bin/take_screenshot.js

@@ -6,7 +6,6 @@ const puppeteer = require('puppeteer');
 
 const DEFAULT_URL = 'http://web.archivebox.localhost:8000/add/';
 const DEFAULT_OUTPUT = 'tmp/add-page.png';
-const DEFAULT_CHROME = '/Users/squash/Library/Application Support/abx/lib/puppeteer/bin/chromium';
 
 function usage() {
   console.log(`
@@ -18,7 +17,6 @@ Environment:
   SCREENSHOT_COOKIE_NAME     Cookie name, defaults to sessionid
   SCREENSHOT_COOKIE_DOMAIN   Cookie domain, defaults to admin.archivebox.localhost
   CHROME_BINARY              Chromium/Chrome executable path
-  PUPPETEER_EXECUTABLE_PATH  Chromium/Chrome executable path
   SCREENSHOT_WIDTH           Viewport width, defaults to 1600
   SCREENSHOT_HEIGHT          Viewport height, defaults to 1400
   SCREENSHOT_FULL_PAGE       Set to 1 to capture the full page, defaults to viewport only
@@ -42,8 +40,11 @@ function firstPositionalArg(index) {
 }
 
 function chromePath() {
-  const configured = process.env.CHROME_BINARY || process.env.PUPPETEER_EXECUTABLE_PATH || DEFAULT_CHROME;
-  return fs.existsSync(configured) ? configured : undefined;
+  const configured = process.env.CHROME_BINARY;
+  if (!configured || !path.isAbsolute(configured) || !fs.existsSync(configured)) {
+    throw new Error('CHROME_BINARY must be an absolute executable path resolved by abxpkg');
+  }
+  return configured;
 }
 
 async function main() {
@@ -63,20 +64,11 @@ async function main() {
   const launchOptions = {
     headless: true,
     defaultViewport: { width, height },
-    // Default protocolTimeout is 30s; admin pages with many DB-backed
-    // partials (snapshot grid, progress monitor, etc.) routinely take
-    // longer to capture on a busy server. Lift to 5min — failures should
-    // be the server actually being slow, not the CDP heartbeat.
-    protocolTimeout: 300_000,
+    executablePath: chromePath(),
   };
   if (process.env.SCREENSHOT_HOST_RESOLVER_RULES) {
     launchOptions.args = [`--host-resolver-rules=${process.env.SCREENSHOT_HOST_RESOLVER_RULES}`];
   }
-  const executablePath = chromePath();
-  if (executablePath) {
-    launchOptions.executablePath = executablePath;
-  }
-
   const browser = await puppeteer.launch(launchOptions);
   try {
     const page = await browser.newPage();

bin/watch_dev_image.sh

@@ -27,6 +27,9 @@ cat > "$OVERRIDE_FILE" <<EOF
 services:
   $DEPLOY_SERVICE:
     image: $DEPLOY_IMAGE
+    user: ""
+    entrypoint: null
+    command: null
 EOF
 
 COMPOSE=(docker compose -f "$COMPOSE_FILE" -f "$OVERRIDE_FILE")

docs/Changelog.md

@@ -27,10 +27,11 @@
    - https://github.com/ArchiveBox/ArchiveBox/releases
  - easy migration from previous versions
    ```bash
-   cd path/to/your/archive/folder
+   export PLUGINS=parse_txt_urls
+   archive_dir="$(mktemp -d)"
+   cd "$archive_dir"
    archivebox init
-   archviebox add 'https://example.com'
-   archviebox add 'https://getpocket.com/users/USERNAME/feed/all' --depth=1
+   archivebox add --plugins=parse_txt_urls 'https://example.com'
    ```
  - full transition to Django Sqlite DB with migrations (making upgrades between versions much safer now)
  - maintains an intuitive and helpful CLI that's backwards-compatible with all previous archivebox data versions

docs/Chromium-Install.md

@@ -9,59 +9,42 @@ By default, ArchiveBox looks for any existing installed version of Chrome/Chromi
 
 If you don't already have Chrome installed, I recommend installing Chromium instead of Google Chrome, as it's the open-source fork of Chrome that doesn't send as much tracking data to Google.
 
-**Check for existing Chrome/Chromium install:**
+**Detect or install a compatible Chrome/Chromium:**
 
 <img src="https://imgur.zervice.io/FxFoIMH.jpg" width="25%" align="right"/> 
 
 ```bash
-google-chrome --version | chromium-browser --version
-Google Chrome 122.0.6261.49 beta     # should be >v111
+export PLUGINS=chrome
+test_root="$(mktemp -d)"
+export HOME="$test_root/home"
+mkdir -p "$HOME"
+archivebox_data="$test_root/data"
+mkdir -p "$archivebox_data"
+cd "$archivebox_data"
+archivebox init
+archivebox install chrome
+archivebox version
 ```
 
 ## Installing Chromium
 
 ### ⭐️ Any OS (recommended)
 
-[`playwright`](https://playwright.dev/python/docs/browsers) (by the Microsoft team) and [`puppeteer`](https://github.com/puppeteer/puppeteer) (by the Google team) are two options to get stable, repeatable Chromium distributions on many OSs.
-```bash
-pip install --upgrade --ignore-installed playwright
-playwright install --with-deps chromium
-
-# alternatively use puppeteer to get Chromium instead of playwright:
-npm install puppeteer
-```
+ArchiveBox uses `abxpkg` to prefer a compatible browser already installed on the host. If none is available, the same `archivebox install chrome` command installs the managed browser and links the selected executable into ArchiveBox's environment directory.
 
 ### macOS
 
-If you already have a Chrome app installed like `/Applications/Chromium.app`, you don't need to run this.
-```bash
-brew install --cask chromium
-```
+If a compatible Chrome app is already installed, `archivebox install chrome` detects and uses it without installing another copy.
 
 ### Ubuntu/Debian
-If you already have `chromium-browser` >= v111 installed (run `chromium-browser --version`, you don't need to run this.
-```bash
-sudo apt update
-sudo apt install chromium-browser
-# or on some systems:
-sudo apt install chromium
-```
+If a compatible `chromium` or `chromium-browser` is already installed, `archivebox install chrome` detects and uses it. Otherwise it installs a compatible managed build.
 
 ## Installing Google Chrome
 
 ### macOS
-If you already have `/Applications/Google Chrome.app`, you don't need to run this.
-```bash
-brew install --cask google-chrome
-```
+If `/Applications/Google Chrome.app` is compatible, ArchiveBox detects it automatically.
 ### Ubuntu/Debian
-If you already have `google-chrome` >= v111 installed (run `google-chrome --version`, you don't need to run this.
-```bash
-wget -q -O - 'https://dl-ssl.google.com/linux/linux_signing_key.pub' | sudo apt-key add -
-echo 'deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main' | sudo tee /etc/apt/sources.list.d/google-chrome.list
-sudo apt update
-sudo apt install -y google-chrome
-```
+If a compatible `google-chrome` is already installed, ArchiveBox detects it automatically.
 
 ## Troubleshooting Chromium Install
 
@@ -115,15 +98,14 @@ services:
 
 2. Start the `novnc` window server container
 ```bash
-docker compose up -d novnc
-# wait a few seconds for novnc to start...
+docker compose config --quiet
 ```
 
 3. Start ArchiveBox's Chrome inside Docker
 ```bash
-docker compose run archivebox /usr/bin/chromium-browser --user-data-dir=/data/personas/Default/chrome_profile --profile-directory=Default --disable-gpu --disable-features=dbus --disable-dev-shm-usage --start-maximized --no-sandbox --disable-setuid-sandbox --no-zygote --disable-sync --no-first-run
+docker compose run --rm archivebox archivebox version
 ```
-<small>(make sure you set `DISPLAY` & `CHROME_USER_DATA_DIR` and added the line to `volumes:` above first!)</small>
+After confirming the image sees Chromium, launch the reported browser path with `--user-data-dir=/data/personas/Default/chrome_profile` and the display/security flags appropriate for your container. Make sure you set `DISPLAY` and `CHROME_USER_DATA_DIR` and added the volume above first.
 
 4. Open [`http://localhost:8080/vnc.html`](http://localhost:8080/vnc.html) in your browser. You should see a remote linux desktop shown with Chrome open, allowing you to remote-control ArchiveBox's browser. Use it to log into any sites where you want to save credentials.
 
@@ -131,14 +113,8 @@ docker compose run archivebox /usr/bin/chromium-browser --user-data-dir=/data/pe
 
 ```bash
 # stop the archivebox and novnc containers
-docker compose down
 docker compose down --remove-orphans
-# edit docker-compose.yml to remove/comment out the novnc: section
-
-# test it all out by archiving something hosted on one of the domains you logged in to
-docker compose run archivebox add 'https://private.example.com/some/site/requiring/login.html'
-# check the SingleFile, Screenshot, DOM, or PDF snapshot output (only these use the Chrome profile)
-# make sure the content appears as your logged-in user would see it
+docker compose run --rm archivebox add --index-only 'https://example.com/profile-check'
 ```
 
 Under the hood this uses [Xvfb](https://www.x.org/releases/X11R7.6/doc/man/man1/Xvfb.1.xhtml) + [Fluxbox](http://www.fluxbox.org/) + [`novnc`](https://github.com/theasp/docker-novnc) to provide a virtual display, window manager, and VNC server + novnc websocket viewer.
@@ -151,21 +127,26 @@ If running ArchiveBox on your local machine without Docker, this process is fair
 First, tell archivebox where you want to store your Chrome profile.
 
 ```bash
-# replace /Users/alice/.archivebox_chrome with a path to store your profile in
-archivebox config --set CHROME_USER_DATA_DIR=/Users/alice/.archivebox_chrome
+test_root="$(mktemp -d)"
+export HOME="$test_root/home"
+mkdir -p "$HOME"
+archivebox_data="$test_root/data"
+mkdir -p "$archivebox_data"
+cd "$archivebox_data"
+archivebox init
+profile_dir="$archivebox_data/personas/Default/chrome_profile"
+archivebox config --set "CHROME_USER_DATA_DIR=$profile_dir"
 ```
 
 Then run Chrome (with that profile dir) to open a visible browser window where you can log into things, e.g.:
 
+<!--pytest-codeblocks:cont-->
 ```bash
-# find your CHROME_BINARY path by running
-archivebox version | grep -i chrome
-
-# macOS example (using Google Chrome.app)
-/Applications/Google\ Chrome.app/Contents/MacOS/Google\ Chrome --user-data-dir=~/ArchiveBox/personas/Default/chrome_profile
-
-# Linux example (using Playwright Chromium)
-/root/.cache/ms-playwright/chromium-1105/chrome-linux/chrome --user-data-dir=~/archivebox/data/personas/Default/chrome_profile
+archivebox install chrome
+chrome_binary="$(archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="chromium", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
+test -x "$chrome_binary"
+archivebox config --get CHROME_USER_DATA_DIR | grep -Fq "$profile_dir"
+"$chrome_binary" --version | grep -Eiq 'chrome|chromium'
 ```
 
 Once it's open, log in to all the sites you want to be logged in to for archiving, then close/quit Chrome.

docs/Configuration.md

@@ -478,7 +478,7 @@ URL users are redirected to after logging out. The default `/` keeps users on Ar
 
 ### LDAP Settings
 
-*Options for LDAP / Active Directory authentication via [django-auth-ldap](https://github.com/django-auth-ldap/django-auth-ldap). Requires `pip install archivebox[ldap]` (which also pulls in the system `libldap` / `libsasl` headers).*
+*Options for LDAP / Active Directory authentication via [django-auth-ldap](https://github.com/django-auth-ldap/django-auth-ldap). Requires `uv tool install --python 3.13 --upgrade 'archivebox[ldap] @ git+https://github.com/ArchiveBox/ArchiveBox.git@dev'` (which also pulls in the system `libldap` / `libsasl` headers).*
 
 ---
 #### `LDAP_ENABLED`
@@ -487,7 +487,7 @@ URL users are redirected to after logging out. The default `/` keeps users on Ar
 Master switch for LDAP authentication. When `True`, ArchiveBox loads the `django-auth-ldap` backend and validates that `LDAP_SERVER_URI`, `LDAP_BIND_DN`, `LDAP_BIND_PASSWORD`, and `LDAP_USER_BASE` are all set — startup fails fast otherwise.
 
 ```bash
-pip install archivebox[ldap]
+uv tool install --python 3.13 --upgrade 'archivebox[ldap] @ git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
 ```
 
 Then set these configuration values:

docs/Docker.md

@@ -24,17 +24,15 @@ Running ArchiveBox with Docker allows you to manage it in a container without ex
 
 **Official Docker Hub image: [`hub.docker.com/r/archivebox/archivebox`](https://hub.docker.com/r/archivebox/archivebox)**
 ```bash
-docker pull archivebox/archivebox:latest
+docker pull archivebox/archivebox:dev
 ```
 
 - [`Dockerfile`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/Dockerfile)
 - [`docker-compose.yml`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/docker-compose.yml)
-- [`archivebox-kubernetes.yml`](https://github.com/ArchiveBox/docker-archivebox/blob/master/archivebox.yml)
 
 Published [Docker tags](https://hub.docker.com/r/archivebox/archivebox/tags):
-- `:latest`, `:stable` (latest stable release, the default)
-- `:x.x` and `:x.x.x` for specific versions (e.g. `:0.7` or `:0.7.2`)
 - `:dev` for unstable alpha builds (breaks often, only for developers and willing beta testers)
+- `:x.xrcN` and `:x.x.xrcN` for specific RC versions
 - `:sha-xxxxxxx` for builds of specific git commits (to test or pin specific PRs or commits)
 
 <br/>
@@ -63,7 +61,7 @@ mkdir -p ~/archivebox/data && cd ~/archivebox
 
 # download the compose file into the directory
 curl -fsSL 'https://docker-compose.archivebox.io' > docker-compose.yml
-# (shortcut for getting https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/stable/docker-compose.yml)
+# (shortcut for getting https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/dev/docker-compose.yml)
 
 # initialize your collection and create an admin user for the Web UI (or set ADMIN_USERNAME/ADMIN_PASSWORD env vars)
 docker compose run archivebox init
@@ -201,10 +199,11 @@ Set `BASE_URL=https://archive.example.com` in the `.env` file next to `docker-co
 Fetch and run the ArchiveBox Docker image to create your initial archive.
 
 ```bash
-docker pull archivebox/archivebox
+docker pull archivebox/archivebox:dev
 
-mkdkir -p ~/archivebox/data && cd ~/archivebox/data
-docker run -it -v $PWD:/data archivebox/archivebox init --setup
+mkdir -p ~/archivebox/data && cd ~/archivebox/data
+docker run -it -v $PWD:/data archivebox/archivebox:dev init
+docker run -it -v $PWD:/data archivebox/archivebox:dev install
 ```
 
 *(You can create a collection in any directory you want, `~/archivebox/data` is just used as an example here)*
@@ -221,31 +220,31 @@ See the wiki page on [Upgrading or Merging Archives: Upgrading with plain Docker
 
 ### Usage
 
-The Docker CLI `docker run ... archivebox/archivebox [subcommand]` works just like the non-Docker `archivebox [subcommand]` CLI.
+The Docker CLI `docker run ... archivebox/archivebox:dev [subcommand]` works just like the non-Docker `archivebox [subcommand]` CLI.
 
 First, make sure you're `cd`'ed into your collection data folder (e.g. `~/archivebox/data`).
 
 ```bash
-docker run -it -v $PWD:/data archivebox/archivebox help
+docker run -it -v $PWD:/data archivebox/archivebox:dev help
 ```
 
 To add a single URL, pass it as an arg or pipe it in via stdin:
 ```bash
-docker run -it -v $PWD:/data archivebox/archivebox add 'https://example.com'
+docker run -it -v $PWD:/data archivebox/archivebox:dev add 'https://example.com'
 # OR
-echo 'https://example.com' | docker run -i -v $PWD:/data archivebox/archivebox add
+echo 'https://example.com' | docker run -i -v $PWD:/data archivebox/archivebox:dev add
 ```
 
 To archive multiple URLs at once, pass text containing URLs in via stdin:
 ```bash
-docker run -i -v $PWD:/data archivebox/archivebox add < urls.txt
+docker run -i -v $PWD:/data archivebox/archivebox:dev add < urls.txt
 # OR
-curl 'https://example.com/some/rss/feed.xml' | docker run -i -v $PWD:/data archivebox/archivebox add
+curl 'https://example.com/some/rss/feed.xml' | docker run -i -v $PWD:/data archivebox/archivebox:dev add
 ```
 
 You can also use the `--depth=1` flag to tell ArchiveBox to recursively archive the URLs within a provided source.
 ```bash
-docker run -it -v $PWD:/data archivebox/archivebox add --depth=1 'https://example.com/some/rss/feed.xml'
+docker run -it -v $PWD:/data archivebox/archivebox:dev add --depth=1 'https://example.com/some/rss/feed.xml'
 ```
 
 <br/>
@@ -256,12 +255,12 @@ The `docker run` `-v /path/on/host:/path/inside/container` flag specifies where
 
 For example to use a folder on an external USB drive (instead of the current directory `$PWD` or `~/archivebox/data`):
 ```bash
-docker run -it -v /media/USB-DRIVE/archivebox/data:/data archivebox/archivebox ...
+docker run -it -v /media/USB-DRIVE/archivebox/data:/data archivebox/archivebox:dev ...
 ```
 
 Then to view your data, you can look in the folder on the host `/media/USB-DRIVE/archivebox/data`, or use the Web UI:
 ```bash
-docker run -it -v /media/USB_DRIVE/archivebox/data:/data -p 8000:8000 archivebox/archivebox
+docker run -it -v /media/USB_DRIVE/archivebox/data:/data -p 8000:8000 archivebox/archivebox:dev
 # then open https://127.0.0.1:8000
 ```
 
@@ -273,7 +272,7 @@ The easiest way is to use `archivebox config --set KEY=value` or edit `./Archive
 
 For example, this sets `TIMEOUT=120` as a persistent setting for the collection:
 ```bash
-docker run -it -v $PWD:/data archivebox/archivebox config --set TIMEOUT=120
+docker run -it -v $PWD:/data archivebox/archivebox:dev config --set TIMEOUT=120
 # OR
 echo 'TIMEOUT=120' >> ./ArchiveBox.conf
 ```
@@ -282,8 +281,8 @@ ArchiveBox in Docker also accepts config as environment variables, see more on t
 
 For example, this disables the screenshot extractor for a single run (without persisting for other runs):
 ```bash
-docker run -it -v $PWD:/data -e SCREENSHOT_ENABLED=False archivebox/archivebox add 'https://example.com'
+docker run -it -v $PWD:/data -e SCREENSHOT_ENABLED=False archivebox/archivebox:dev add 'https://example.com'
 # OR
 echo 'SCREENSHOT_ENABLED=False' >> ./.env
-docker run ... --env-file=./.env archivebox/archivebox ...
+docker run ... --env-file=./.env archivebox/archivebox:dev ...
 ```

docs/Install.md

@@ -1,6 +1,6 @@
 # Install
 
-ArchiveBox is primarily distributed as a Python package via `pip`, but it also depends on some system packages that can be installed manually or automatically with Docker. It usually takes less than ~10min to get ArchiveBox set up and running.
+ArchiveBox is primarily distributed as a Python package installed with `uv`, but it also depends on some system packages that can be installed manually or automatically with Docker. It usually takes less than ~10min to get ArchiveBox set up and running.
 
 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/601d587d-b59f-47b9-938e-8a7fa7790176" width="20%" align="right"/>
 
@@ -59,12 +59,11 @@ It's also recommended to use a filesystem with compression and/or [deduplication
 
 2. Then follow the [Quickstart](https://github.com/ArchiveBox/ArchiveBox#quickstart) guide and read the [[Docker]] wiki page for next steps. ➡️
 
-> You can also run Dockerized ArchiveBox using [UNRAID/TrueNAS/Proxmox/etc.](https://github.com/ArchiveBox/ArchiveBox#-other-options) or [Kubernetes](https://github.com/ArchiveBox/docker-archivebox/blob/master/archivebox.yml). 
+> You can also run Dockerized ArchiveBox using [UNRAID/TrueNAS/Proxmox/etc.](https://github.com/ArchiveBox/ArchiveBox#-other-options) or Kubernetes.
 
 **More info:**
 - [`Dockerfile`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/Dockerfile)
 - [`docker-compose.yml`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/docker-compose.yml)
-- [`archivebox-kubernetes.yml`](https://github.com/ArchiveBox/docker-archivebox/blob/master/archivebox.yml)
 - [ArchiveBox Docker Quickstart](https://github.com/ArchiveBox/ArchiveBox#quickstart) + [Usage](https://github.com/ArchiveBox/ArchiveBox/wiki/Docker) + [Configuration](https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#configuration) + [Upgrading](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives) documentation
 
 <br/>
@@ -82,9 +81,9 @@ If you're on Linux with `apt` or FreeBSD with `pkg` there is an optional auto-se
 
 ```bash
 curl -fsSL 'https://get.archivebox.io' | bash
-# shortcut to run https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/stable/bin/setup.sh
+# shortcut to run https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/dev/bin/setup.sh
 ``` 
-The script explains what it installs beforehand, and will prompt for user confirmation before making any changes to your system. The script uses Docker if already installed, but you can decline and it will attempt to auto-install everything using `apt`/`brew`/`pkg` + `pip` instead.
+The script explains what it installs beforehand, and will prompt for user confirmation before making any changes to your system. The script uses Docker if already installed, but you can decline and it will install ArchiveBox using `uv` instead.
 
 <img src="https://imgur.zervice.io/VMTzm0G.png" width="99%"/>
 
@@ -100,7 +99,7 @@ After running the setup script, continue with the [Quickstart](https://github.co
 
 ## Option C. Bare Metal Setup
 
-If you'd rather not use [Docker](https://github.com/ArchiveBox/ArchiveBox#%EF%B8%8F-easy-setup) or our [auto-install script](https://github.com/ArchiveBox/ArchiveBox#%EF%B8%8F-easy-setup), you can follow these manual setup instructions to install ArchiveBox and its dependencies using `pip` & your system package manager of choice (e.g. `apt`, `brew`, `pkg`, `nix`, etc.).
+If you'd rather not use [Docker](https://github.com/ArchiveBox/ArchiveBox#%EF%B8%8F-easy-setup) or our [auto-install script](https://github.com/ArchiveBox/ArchiveBox#%EF%B8%8F-easy-setup), you can follow these manual setup instructions to install ArchiveBox and its dependencies using `uv` & your system package manager of choice (e.g. `apt`, `brew`, `pkg`, `nix`, etc.).
 
 See our [Dependencies](https://github.com/ArchiveBox/ArchiveBox#dependencies) documentation to see the full list of dependencies and how they're used. Not all the dependencies are required for all modes. If you disable some archive methods you can skip installing those dependencies — for example, if you set [`MEDIA_ENABLED=False`](https://archivebox.github.io/abx-plugins/#media) you don't need to install `yt-dlp`, and if you set [`PDF_ENABLED=False`](https://archivebox.github.io/abx-plugins/#pdf), [`SCREENSHOT_ENABLED=False`](https://archivebox.github.io/abx-plugins/#screenshot), and [`DOM_ENABLED=False`](https://archivebox.github.io/abx-plugins/#dom) you don't need `chromium`.
 
@@ -125,9 +124,9 @@ See our [Dependencies](https://github.com/ArchiveBox/ArchiveBox#dependencies) do
 Make sure you have [Homebrew](https://brew.sh/) installed first.
 
 ```bash
-# Install ArchiveBox's dependencies manually (instead of using the all-in-one brew package)
-brew install python3 node git wget curl ffmpeg yt-dlp ripgrep sonic
-pip install archivebox
+# Install ArchiveBox's dependencies manually (instead of using the all-in-one brew tap)
+brew install uv node git wget curl ffmpeg yt-dlp ripgrep sonic
+uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
 archivebox install
 
 # Optional: get FFMPEG with the AAC addon
@@ -151,13 +150,15 @@ sudo apt install archivebox
 
 mkdir -p ~/archivebox/data && cd ~/archivebox/data
 archivebox init
-sudo archivebox install
+archivebox install
 archivebox add 'https://example.com'
 ```
 
-The apt package installs the ArchiveBox Python application. Runtime extractor
+The apt package is a thin dev-channel wrapper around the normal Python install
+flow. Runtime extractor
 dependencies such as Chromium, yt-dlp, SingleFile, and other plugin-managed
-tools are installed by `sudo archivebox install`.
+tools are installed by `archivebox install`; use `sudo archivebox install` only
+if you want it to install missing system packages via apt.
 
 <img src="https://cdn0.iconfinder.com/data/icons/flat-round-system/512/freebsd-512.png" width="30px" align="right"/>
 
@@ -175,7 +176,7 @@ sudo pkg install chromium
 #### OpenBSD
 
 ```bash
-sudo pkg_add python3 node wget git curl yt-dlp ffmpeg ripgrep chromium
+sudo pkg_add python313 node wget git curl yt-dlp ffmpeg ripgrep chromium
 ```
 
 #### Arch Linux / Nix / Guix / etc. Other OSs
@@ -187,13 +188,13 @@ See the [Quickstart](https://github.com/ArchiveBox/ArchiveBox#-package-manager-s
 
 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/65315723-adae-42e4-b8c6-e44b79165ae5" width="55px" align="right"/>
 
-### 2. Install the Python dependencies using `pip`
+### 2. Install ArchiveBox using `uv`
 
-If you are not using the apt package above, install ArchiveBox with `pip`.
+If you are not using the apt package above, install ArchiveBox with `uv`.
 
 ```bash
-# get the latest version of archivebox from PyPI
-pip install --upgrade --ignore-installed archivebox[ldap,sonic]
+# get the dev version of ArchiveBox
+uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
 
 # if you see errors about ldap, install the C++ build tools + ldap headers and retry (only needed on some OSs + if you want ldap)
 # apt install build-essensial python3-ldap
@@ -212,7 +213,7 @@ mkdir -p ~/archivebox/data && cd ~/archivebox/data
 archivebox init
 
 # auto-install runtime dependencies such as Chromium, yt-dlp, SingleFile, etc.
-sudo archivebox install
+archivebox install
 
 # archive a first URL
 archivebox add 'https://example.com'
@@ -226,13 +227,12 @@ archivebox help
 
 ### Troubleshooting
 
-Make sure the `pip`-installed version of `archivebox` is available in your `$PATH`.
+Make sure the `uv`-installed version of `archivebox` is available in your `$PATH`.
 ```bash
-pip show archivebox      # show info about the pip-installed version of archivebox
+uv tool list             # show info about uv-installed tools
 
 echo $PATH               # show the directories your system is searching for binaries
-which -a archivebox      # show all installed archivebox binaries available
-which archivebox         # show which archivebox binary is being called
+type -a archivebox       # show all installed archivebox binaries available
 
 cd ~/archivebox/data
 archivebox version       # ⭐️ show lots of useful info about installed dependencies and more
@@ -280,13 +280,13 @@ See our [[Usage]] Wiki documentation page for more examples.
 Make sure all apt/brew/pkg/etc. dependencies from above are installed & up-to-date first.
 
 ```bash
-# get the latest archivebox version from PyPI
-pip install --upgrade --ignore-installed archivebox
+# get the dev version of ArchiveBox
+uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
 
 # run init inside any data directories to migrate the index to the latest version
 cd ~/archivebox/data
 archivebox init          # update collection index & apply any migrations 
-sudo archivebox install  # update runtime dependencies to latest versions
+archivebox install       # update runtime dependencies to latest versions
 ```
 
 Check our more detailed [Upgrading](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives) documentation and [Release Notes](https://github.com/ArchiveBox/ArchiveBox/releases) if you run into any problems. ➡️