release: v0.9.35rc93

Clarify security risks and update information on CSRF/XSS protections.

.claude/settings.json

@@ -1,5 +0,0 @@
-{
-  "enabledPlugins": {
-    "pyright-lsp@claude-plugins-official": true
-  }
-}

.claude/settings.local.json

@@ -1,51 +0,0 @@
-{
-  "permissions": {
-    "allow": [
-      "Read(**)",
-      "Glob(**)",
-      "Grep(**)",
-      "Bash(python -m archivebox:*)",
-      "Bash(ls:*)",
-      "Bash(xargs:*)",
-      "Bash(python -c:*)",
-      "Bash(printf:*)",
-      "Bash(pkill:*)",
-      "Bash(python3:*)",
-      "Bash(sqlite3:*)",
-      "WebFetch(domain:github.com)",
-      "Bash(uv add:*)",
-      "Bash(mkdir:*)",
-      "Bash(chmod:*)",
-      "Bash(python -m forum_dl:*)",
-      "Bash(archivebox manage migrate:*)",
-      "Bash(cat:*)",
-      "Bash(python archivebox/plugins/pip/on_Dependency__install_using_pip_provider.py:*)",
-      "Bash(forum-dl:*)",
-      "Bash(pip uninstall:*)",
-      "Bash(python:*)",
-      "Bash(source .venv/bin/activate)",
-      "Bash(mv:*)",
-      "Bash(echo:*)",
-      "Bash(grep:*)",
-      "WebFetch(domain:python-statemachine.readthedocs.io)",
-      "Bash(./bin/run_plugin_tests.sh:*)",
-      "Bash(done)",
-      "Bash(coverage erase:*)",
-      "Bash(gh api:*)"
-    ]
-  },
-  "hooks": {
-    "PreToolUse": [
-      {
-        "matcher": "Bash",
-        "hooks": [
-          {
-            "type": "command",
-            "command": "REPO_ROOT=$(git rev-parse --show-toplevel 2>/dev/null); if [ -n \"$REPO_ROOT\" ] && [ \"$PWD\" != \"$REPO_ROOT\" ]; then echo \"ERROR: Not in repo root ($REPO_ROOT). Current dir: $PWD\" >&2; exit 1; fi",
-            "statusMessage": "Checking working directory..."
-          }
-        ]
-      }
-    ]
-  }
-}

.dockerignore

@@ -13,10 +13,14 @@ __pycache__
 .pytest_cache/
 .ruff_cache/
 .uv-cache/
+.cache/
+.cache*/
 .github/
 .pdm-build/
+.pdm-build/**
 .pdm-python
 .eggs/
+*.egg-info/
 .git/
 !.git/
 .git/*
@@ -42,6 +46,7 @@ chromeprofile/
 chrome_profile/
 lib/
 out/
+tests/out/
 users/
 archive/
 crawls/
@@ -63,6 +68,8 @@ dist/
 brew_dist/
 deb_dist/
 pip_dist/
+cache/
+caches/
 assets/
 docker/
 website/

.github/ISSUE_TEMPLATE/1-bug_report.yml

@@ -69,7 +69,7 @@ body:
         0.8.6
         ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
         IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
-        EUID=502:20 UID=502:20 PUID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
+        EUID=502:20 UID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
         DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
         
         Binary Dependencies:

.github/ISSUE_TEMPLATE/2-feature_request.yml

@@ -71,7 +71,7 @@ body:
         0.8.6
         ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
         IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
-        EUID=502:20 UID=502:20 PUID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
+        EUID=502:20 UID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
         DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
         
         Binary Dependencies:

.github/SECURITY.md

@@ -2,18 +2,15 @@
 
 ---
 
-## Security Information
+## Security Documentation
 
-Please see this wiki page for important notices about ArchiveBox security, publishing your archives securely, and the dangers of executing archived JS:
-
-https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview
-
-Also see this section of the README about important caveats when running ArchiveBox:
-
-https://github.com/ArchiveBox/ArchiveBox?tab=readme-ov-file#caveats
-
-You can also read these pages for more information about ArchiveBox's internals, development environment, DB schema, and more:
+Please start by reading these links to understand our existing security model, known issues, API surfaces, and setup steps:
 
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview
+- https://github.com/ArchiveBox/ArchiveBox/security/advisories/
+- https://github.com/ArchiveBox/ArchiveBox/tree/v0.7.4#security-risks-of-viewing-archived-js
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#security-concerns
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-up-Authentication
 - https://github.com/ArchiveBox/ArchiveBox#archive-layout
 - https://github.com/ArchiveBox/ArchiveBox#archivebox-development
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives
@@ -32,3 +29,43 @@ We use Github's built-in [Private Reporting](https://docs.github.com/en/code-sec
 3. Fill out the form to submit the details of the report and it will be securely sent to the maintainers
 
 You can also contact the maintainers via our public [Zulip Chat Server zulip.archivebox.io](https://zulip.archivebox.io) or [Twitter DMs @ArchiveBoxApp](https://twitter.com/ArchiveBoxApp).
+
+---
+
+## CVEs and Report Credits
+
+> [!IMPORTANT]
+> **Reports that only affect `dev` or beta versions will be converted to normal Github issues and will not be issued CVEs.**  
+> Reporters **will still receive credit in release notes** however.
+
+> [!TIP]
+> Reports that affect **stable published versions will be issued CVEs** and published w/ full credit.
+
+Please read our existing published [security advisories](https://github.com/ArchiveBox/ArchiveBox/security/advisories), [Security Overview Docs](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview), and [issues](https://github.com/ArchiveBox/ArchiveBox/issues) and avoid creating duplicates.
+
+---
+
+## Threat Model
+
+### All admin users are assumed to be trusted
+
+ArchiveBox does not support granular user permissions, there are only two types of users available in the Admin UI & REST API:
+
+- logged in admin superusers (which are assumed to have `root` on the ArchiveBox host machine already)
+- non-logged in anonymous guest users
+
+Because admin / REST API / CLI / Python API users are already assumed to have `root` on the host machine, there are deliberately no protections against admins inputting malicious config values, attempts to attack other admin users, or destructive/malicious actions. Permissions for non-logged in users are managed via the [`PERMISSIONS`](https://github.com/archiveBox/archiveBox/wiki/configuration#permissions) + [`PUBLIC_INDEX` + `PUBLIC_ADD_VIEW`](https://github.com/archiveBox/archiveBox/wiki/configuration#public_index--public_add_view) + other config options.
+
+### Sanitization of Config Values, Filesystem Paths, DB Fields, etc.
+
+Non-logged in users can only choose between public Persona config presets on the `/add/` page, but they should never be allowed to directly set config options like `*_ARGS`, `*_BINARY` or DB fields like `url`, `title`, `tag.name`, `notes` without sanitization/filtering on save and/or use.
+
+Non-logged in users should be tightly limited to prevent attacks like SQL/shell injection, path traversal attacks, XSS, cookie leaks between admin/non-admin domains, etc. Any reports related to non-logged-in users will be prioritized as this is a surface we care a lot about hardening.
+
+### XSS, CSRF, CORS, CSP
+
+`< v0.9.x` versions do not include any XSS/CSRF protections at all, as documented in the [readme](https://github.com/ArchiveBox/ArchiveBox/tree/v0.7.4#security-risks-of-viewing-archived-js), [wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#security-concerns), [issue #239](https://github.com/ArchiveBox/ArchiveBox/issues/239), and [existing advisory](https://github.com/ArchiveBox/ArchiveBox/security/advisories/GHSA-cr45-98w9-gwqx). Do not open advisories related to XSS or CSRF for those versions, as they are missing fundamental architectural changes added in `v0.9.0` needed to support per-snapshot replay isolation.
+
+[`SERVER_SECURITY_MODE`](https://github.com/archiveBox/archiveBox/wiki/configuration#server_security_mode) + CSRF/XSS defenses were only added in the new 0.9.0 `dev` work and are planned to land in `main` in mid-2026.
+
+Note archivebox is designed to be able to archive any URL the server can reach, including internal intranet URLs. The only URLs that are blocked are URLs hosted by the archivebox server itself.

.github/scripts/clone_abx_repo.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+repo_name="$1"
+target_dir="${2:-$repo_name}"
+
+version="$(
+python3 - "$repo_name" <<'PY'
+import re
+import sys
+from pathlib import Path
+
+repo_name = sys.argv[1]
+lock_text = Path("uv.lock").read_text()
+match = re.search(
+    rf'^\[\[package\]\]\s*\nname = "{re.escape(repo_name)}"\s*\nversion = "([^"]+)"',
+    lock_text,
+    re.MULTILINE,
+)
+if not match:
+    raise SystemExit(f"Could not find {repo_name} in uv.lock")
+print(match.group(1))
+PY
+)"
+
+echo "Cloning ArchiveBox/${repo_name}@v${version} into ${target_dir}"
+git clone --depth=1 --branch "v${version}" "https://github.com/ArchiveBox/${repo_name}.git" "$target_dir"

.github/workflows/docker.yml

@@ -13,7 +13,6 @@ on:
 env:
   DOCKERHUB_IMAGE: archivebox/archivebox
   GHCR_IMAGE: ghcr.io/archivebox/archivebox
-  ABX_DL_IMAGE: archivebox/abx-dl:latest
 
 permissions:
   contents: read
@@ -69,9 +68,12 @@ jobs:
           missing = required.copy()
           while missing and time.monotonic() < deadline:
               for name, version in list(missing.items()):
-                  with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/json", timeout=20) as resp:
-                      releases = json.load(resp)["releases"]
-                  if version in releases:
+                  try:
+                      with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/{version}/json", timeout=20):
+                          available = True
+                  except Exception:
+                      available = False
+                  if available:
                       print(f"{name} {version} is available on PyPI")
                       missing.pop(name)
                   else:
@@ -102,6 +104,23 @@ jobs:
         shell: bash
         run: |
           set -Eeuo pipefail
+          ABX_DL_VERSION="$(python3 - <<'PY'
+          import re
+          import tomllib
+
+          with open("pyproject.toml", "rb") as f:
+              deps = tomllib.load(f)["project"]["dependencies"]
+
+          for dep in deps:
+              match = re.match(r"abx-dl\s*(?:==|>=)\s*([^,;\s]+)", dep)
+              if match:
+                  print(match.group(1))
+                  break
+          else:
+              raise SystemExit("Missing abx-dl dependency in pyproject.toml")
+          PY
+          )"
+          ABX_DL_IMAGE="${ABX_DL_IMAGE:-archivebox/abx-dl:${ABX_DL_VERSION}}"
           deadline=$((SECONDS + 1800))
           until docker buildx imagetools inspect "${ABX_DL_IMAGE}" >/tmp/abx-dl-image.json; do
             if (( SECONDS >= deadline )); then
@@ -168,14 +187,46 @@ jobs:
           labels: ${{ steps.docker_meta.outputs.labels }}
           build-args: |
             ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
+            ARCHIVEBOX_COMMIT_HASH=${{ github.sha }}
           cache-from: type=gha,scope=${{ matrix.cache_scope }}
           cache-to: type=gha,mode=max,scope=${{ matrix.cache_scope }}
+          pull: true
           platforms: ${{ matrix.platform }}
           outputs: type=image,push-by-digest=true,name-canonical=true,push=true
 
       - name: Image digest
         run: echo ${{ steps.docker_build.outputs.digest }}
 
+      - name: Validate pushed image version, commit, and size
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          IMAGE="${DOCKERHUB_IMAGE}@${{ steps.docker_build.outputs.digest }}"
+          SHORT_SHA="${GITHUB_SHA::7}"
+
+          DATA_DIR="$(mktemp -d)"
+          docker run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox init
+          docker run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox version | tee /tmp/archivebox-version.out
+          docker run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox status
+          docker run --rm --platform '${{ matrix.platform }}' --entrypoint cat "$IMAGE" /VERSION.txt > /tmp/archivebox-image-version.txt
+          grep -F "ArchiveBox v${{ steps.docker_meta.outputs.version }}" /tmp/archivebox-version.out
+          grep -F "COMMIT_HASH=${SHORT_SHA}" /tmp/archivebox-version.out
+          grep -Fx "COMMIT_HASH=${GITHUB_SHA}" /tmp/archivebox-image-version.txt
+
+          python3 - "$IMAGE" <<'PY'
+          import json
+          import subprocess
+          import sys
+
+          image = sys.argv[1]
+          limit = 780 * 1024 * 1024
+          manifest = json.loads(subprocess.check_output(["docker", "manifest", "inspect", image]))
+          total = manifest.get("config", {}).get("size", 0) + sum(layer.get("size", 0) for layer in manifest.get("layers", []))
+          print(f"{image} compressed_size={total / 1024 / 1024:.2f} MiB limit={limit / 1024 / 1024:.2f} MiB")
+          if total > limit:
+              raise SystemExit(f"{image} is over the compressed size limit")
+          PY
+
       - name: Export digest
         shell: bash
         run: |
@@ -320,8 +371,12 @@ jobs:
         shell: bash
         run: |
           set -Eeuo pipefail
-          docker buildx imagetools inspect "${DOCKERHUB_IMAGE}:sha-${{ steps.docker_meta.outputs.short_sha }}"
-          docker buildx imagetools inspect "${GHCR_IMAGE}:sha-${{ steps.docker_meta.outputs.short_sha }}"
+          while IFS= read -r tag; do
+            [[ -n "$tag" ]] && docker buildx imagetools inspect "$tag"
+          done <<< "${{ steps.docker_meta.outputs.dockerhub_tags }}"
+          while IFS= read -r tag; do
+            [[ -n "$tag" ]] && docker buildx imagetools inspect "$tag"
+          done <<< "${{ steps.docker_meta.outputs.ghcr_tags }}"
 
       - name: Update README
         uses: peter-evans/dockerhub-description@v4

.github/workflows/lint.yml

@@ -22,13 +22,13 @@ jobs:
           fetch-depth: 1
 
       - name: Clone abxpkg
-        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
+        run: bash .github/scripts/clone_abx_repo.sh abxpkg
 
       - name: Clone abx-plugins
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+        run: bash .github/scripts/clone_abx_repo.sh abx-plugins
 
       - name: Clone abx-dl
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
+        run: bash .github/scripts/clone_abx_repo.sh abx-dl
 
       - name: Set up Python
         uses: actions/setup-python@v5
@@ -39,6 +39,7 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v6
         with:
+          version: "0.10.6"
           enable-cache: false
 
       - name: Install dependencies with uv

.github/workflows/pip.yml

@@ -29,6 +29,7 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v6
         with:
+          version: "0.10.6"
           enable-cache: false
 
       - name: Set up Python
@@ -46,7 +47,6 @@ jobs:
       - name: Wait for released ArchiveBox deps on PyPI
         run: |
           python - <<'PY'
-          import json
           import re
           import sys
           import time
@@ -66,9 +66,12 @@ jobs:
           missing = required.copy()
           while missing and time.monotonic() < deadline:
               for name, version in list(missing.items()):
-                  with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/json", timeout=20) as resp:
-                      releases = json.load(resp)["releases"]
-                  if version in releases:
+                  try:
+                      with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/{version}/json", timeout=20):
+                          available = True
+                  except Exception:
+                      available = False
+                  if available:
                       print(f"{name} {version} is available on PyPI")
                       missing.pop(name)
                   else:
@@ -81,8 +84,49 @@ jobs:
               sys.exit(1)
           PY
 
+      - name: Prepare released dependency sync project
+        run: |
+          python - <<'PY'
+          from pathlib import Path
+          import json
+          import re
+          import urllib.request
+
+          source = Path("pyproject.toml")
+          target_dir = Path(".ci-uv-project")
+          target_dir.mkdir(exist_ok=True)
+          text = source.read_text()
+          text = text.replace(
+              'environments = ["sys_platform == \'darwin\'", "sys_platform == \'linux\'"]',
+              'environments = ["sys_platform == \'linux\'"]',
+          )
+
+          # CI runs immediately after publishing internal abx packages. PyPI's
+          # simple index can lag the version JSON endpoint, so sync against the
+          # exact wheel URLs from the version endpoint while leaving package
+          # metadata in pyproject.toml unchanged for the actual build.
+          for package in ("abxpkg", "abx-plugins", "abx-dl"):
+              match = re.search(rf'"{re.escape(package)}>=(?P<version>[^"]+)"', text)
+              if not match:
+                  continue
+              version = match.group("version")
+              with urllib.request.urlopen(f"https://pypi.org/pypi/{package}/{version}/json", timeout=20) as response:
+                  data = json.load(response)
+              wheel_url = next(url["url"] for url in data["urls"] if url["filename"].endswith(".whl"))
+              text = re.sub(
+                  rf'"{re.escape(package)}>=[^"]+"',
+                  f'"{package} @ {wheel_url}"',
+                  text,
+                  count=1,
+              )
+
+          (target_dir / "pyproject.toml").write_text(text)
+          PY
+
       - name: UV install archivebox dev + run sub-dependencies
-        run: uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
+        env:
+          UV_PROJECT_ENVIRONMENT: ${{ github.workspace }}/.venv
+        run: uv sync --project .ci-uv-project --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
 
       - name: UV build archivebox and archivebox/pkgs/* packages
         run: |
@@ -92,17 +136,22 @@ jobs:
         uses: pypa/gh-action-pypi-publish@release/v1
         # ignore when publish to PyPI fails due to duplicate tag
         continue-on-error: true
-        with:
-          password: ${{ secrets.PYPI_PAT_SECRET }}
 
       - name: UV install archivebox and archivebox/pkgs/* locally for tests
-        run: uv sync --all-extras --no-sources --no-cache
-
-      - name: UV run archivebox init + archivebox version
+        env:
+          UV_PROJECT_ENVIRONMENT: ${{ github.workspace }}/.venv
         run: |
-          mkdir -p data && cd data
-          uv run --no-sync --no-sources archivebox init \
-          && uv run --no-sync --no-sources archivebox version
-          # && uv run archivebox add 'https://example.com' \
-          # && uv run archivebox status \
-          # || (echo "UV Failed to run archivebox!" && exit 1)
+          uv sync --project .ci-uv-project --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
+          uv pip install --python .venv/bin/python --no-deps dist/archivebox-*.whl
+
+      - name: Verify built package full install
+        env:
+          UV_PROJECT_ENVIRONMENT: ${{ github.workspace }}/.venv
+        run: |
+          set -Eeuo pipefail
+          DATA_DIR="$(mktemp -d)"
+          cd "$DATA_DIR"
+          uv run --project "$GITHUB_WORKSPACE/.ci-uv-project" --no-sync --no-sources archivebox init
+          timeout 30m uv run --project "$GITHUB_WORKSPACE/.ci-uv-project" --no-sync --no-sources archivebox install
+          uv run --project "$GITHUB_WORKSPACE/.ci-uv-project" --no-sync --no-sources archivebox version
+          uv run --project "$GITHUB_WORKSPACE/.ci-uv-project" --no-sync --no-sources archivebox status

.github/workflows/release-runner.yml

@@ -3,26 +3,30 @@ name: Release State
 on:
   push:
     branches:
-      - '**'
+      - dev
   workflow_dispatch:
 
 permissions:
+  actions: write
   contents: write
   id-token: write
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+  cancel-in-progress: false
 
 jobs:
   release-state:
+    if: github.event_name != 'push' || !startsWith(github.event.head_commit.message, 'release:')
     runs-on: ubuntu-latest
+    environment: pypi
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
           submodules: true
           ref: ${{ github.ref_name }}
+          token: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
 
       - uses: actions/setup-python@v5
         with:
@@ -30,6 +34,7 @@ jobs:
 
       - uses: astral-sh/setup-uv@v6
         with:
+          version: "0.10.6"
           enable-cache: false
 
       - uses: actions/setup-node@v4
@@ -43,7 +48,13 @@ jobs:
 
       - name: Run release script
         env:
-          DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
-          GH_TOKEN: ${{ github.token }}
-          PYPI_PAT_SECRET: ${{ secrets.PYPI_PAT_SECRET }}
+          DEFAULT_BRANCH: dev
+          GH_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
         run: ./bin/release.sh
+
+      - name: Dispatch image and tap updates
+        env:
+          GH_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
+        run: |
+          gh workflow run docker.yml --ref dev
+          gh workflow run update-homebrew-tap.yml --ref dev

.github/workflows/test-parallel.yml

@@ -16,6 +16,7 @@ env:
   PYTHONLEGACYWINDOWSSTDIO: utf-8
   USE_COLOR: False
   UV_NO_SOURCES: "1"
+  ABXPKG_MIN_RELEASE_AGE: "0"
 
 jobs:
   discover-tests:
@@ -53,8 +54,12 @@ jobs:
             if grep -Eiq '(chrom|archivewebpage|PLUGINS=.*title|--plugins=.*title|SAVE_TITLE.*[Tt]rue)' "$test_file"; then
               needs_chromium=true
             fi
+            needs_sonic=false
+            if grep -Eiq "shutil\\.which\\([\"']sonic|SEARCH_BACKEND_ENGINE=.*sonic|worker_sonic" "$test_file"; then
+              needs_sonic=true
+            fi
 
-            json_array+="{\"path\":\"$test_file\",\"name\":\"$name\",\"needs_chromium\":$needs_chromium}"
+            json_array+="{\"path\":\"$test_file\",\"name\":\"$name\",\"needs_chromium\":$needs_chromium,\"needs_sonic\":$needs_sonic}"
           done
           json_array+="]"
 
@@ -63,7 +68,7 @@ jobs:
           echo "$json_array" | jq '.'
 
       - name: Clone abx-plugins
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+        run: bash .github/scripts/clone_abx_repo.sh abx-plugins
 
       - name: Discover plugin tests
         id: set-plugin-matrix
@@ -83,8 +88,12 @@ jobs:
             if grep -Riq "chrom" "abx-plugins/abx_plugins/plugins/$plugin_name"; then
               needs_chromium=true
             fi
+            needs_sonic=false
+            if [ "$plugin_name" = "search_backend_sonic" ]; then
+              needs_sonic=true
+            fi
 
-            json_array+="{\"plugin\":\"$plugin_name\",\"name\":\"plugin/$plugin_name\",\"needs_chromium\":$needs_chromium}"
+            json_array+="{\"plugin\":\"$plugin_name\",\"name\":\"plugin/$plugin_name\",\"needs_chromium\":$needs_chromium,\"needs_sonic\":$needs_sonic}"
           done
           json_array+="]"
 
@@ -116,13 +125,13 @@ jobs:
           fetch-depth: 1
 
       - name: Clone abxpkg
-        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
+        run: bash .github/scripts/clone_abx_repo.sh abxpkg
 
       - name: Clone abx-plugins
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+        run: bash .github/scripts/clone_abx_repo.sh abx-plugins
 
       - name: Clone abx-dl
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
+        run: bash .github/scripts/clone_abx_repo.sh abx-dl
 
       - name: Set up Python ${{ matrix.python }}
         uses: actions/setup-python@v5
@@ -133,7 +142,7 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v6
         with:
-          version: "latest"
+          version: "0.10.6"
 
       - name: Set up Node JS
         uses: actions/setup-node@v4
@@ -190,6 +199,37 @@ jobs:
           fi
           echo "Using ArchiveBox-installed Chromium: $candidate ($version)"
           echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV"
+          echo "DATA_DIR=$ABX_CI_DATA_DIR" >> "$GITHUB_ENV"
+          echo "NODE_MODULES_DIR=$LIB_DIR/pnpm/packages/chrome/node_modules" >> "$GITHUB_ENV"
+          echo "NODE_PATH=$LIB_DIR/pnpm/packages/chrome/node_modules" >> "$GITHUB_ENV"
+          echo "PNPM_BIN_DIR=$LIB_DIR/pnpm/packages/chrome/node_modules/.bin" >> "$GITHUB_ENV"
+          echo "NPM_BIN_DIR=$LIB_DIR/pnpm/packages/chrome/node_modules/.bin" >> "$GITHUB_ENV"
+          echo "CHROMEWEBSTORE_EXTENSIONS_DIR=$LIB_DIR/chromewebstore/extensions" >> "$GITHUB_ENV"
+
+      - name: Install Sonic through ArchiveBox
+        if: ${{ matrix.test.needs_sonic }}
+        env:
+          LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_INSTALL_TIMEOUT: "900"
+          ABX_CI_DATA_DIR: ${{ runner.temp }}/archivebox-ci-data
+        run: |
+          set -euo pipefail
+          mkdir -p "$LIB_DIR" "$ABX_CI_DATA_DIR"
+          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
+          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
+
+          uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox install search_backend_sonic
+
+          candidate="$(uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="sonic", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
+          if [ ! -x "$candidate" ]; then
+            echo "ArchiveBox did not install an executable Sonic binary: ${candidate:-not found}" >&2
+            exit 1
+          fi
+          version="$("$candidate" --version || true)"
+          echo "Using ArchiveBox-installed Sonic: $candidate (${version:-version unknown})"
+          echo "SONIC_BINARY=$candidate" >> "$GITHUB_ENV"
+          echo "$(dirname "$candidate")" >> "$GITHUB_PATH"
 
       - name: Run test - ${{ matrix.test.name }}
         run: |
@@ -227,13 +267,13 @@ jobs:
           fetch-depth: 1
 
       - name: Clone abxpkg
-        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
+        run: bash .github/scripts/clone_abx_repo.sh abxpkg
 
       - name: Clone abx-plugins
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+        run: bash .github/scripts/clone_abx_repo.sh abx-plugins
 
       - name: Clone abx-dl
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
+        run: bash .github/scripts/clone_abx_repo.sh abx-dl
 
       - name: Set up Python 3.13
         uses: actions/setup-python@v5
@@ -244,7 +284,7 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v6
         with:
-          version: "latest"
+          version: "0.10.6"
 
       - name: Set up Node JS
         uses: actions/setup-node@v4
@@ -301,6 +341,37 @@ jobs:
           fi
           echo "Using ArchiveBox-installed Chromium: $candidate ($version)"
           echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV"
+          echo "DATA_DIR=$ABX_CI_DATA_DIR" >> "$GITHUB_ENV"
+          echo "NODE_MODULES_DIR=$LIB_DIR/pnpm/packages/chrome/node_modules" >> "$GITHUB_ENV"
+          echo "NODE_PATH=$LIB_DIR/pnpm/packages/chrome/node_modules" >> "$GITHUB_ENV"
+          echo "PNPM_BIN_DIR=$LIB_DIR/pnpm/packages/chrome/node_modules/.bin" >> "$GITHUB_ENV"
+          echo "NPM_BIN_DIR=$LIB_DIR/pnpm/packages/chrome/node_modules/.bin" >> "$GITHUB_ENV"
+          echo "CHROMEWEBSTORE_EXTENSIONS_DIR=$LIB_DIR/chromewebstore/extensions" >> "$GITHUB_ENV"
+
+      - name: Install Sonic through ArchiveBox
+        if: ${{ matrix.plugin.needs_sonic }}
+        env:
+          LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_INSTALL_TIMEOUT: "900"
+          ABX_CI_DATA_DIR: ${{ runner.temp }}/archivebox-ci-data
+        run: |
+          set -euo pipefail
+          mkdir -p "$LIB_DIR" "$ABX_CI_DATA_DIR"
+          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
+          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
+
+          uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox install search_backend_sonic
+
+          candidate="$(uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="sonic", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
+          if [ ! -x "$candidate" ]; then
+            echo "ArchiveBox did not install an executable Sonic binary: ${candidate:-not found}" >&2
+            exit 1
+          fi
+          version="$("$candidate" --version || true)"
+          echo "Using ArchiveBox-installed Sonic: $candidate (${version:-version unknown})"
+          echo "SONIC_BINARY=$candidate" >> "$GITHUB_ENV"
+          echo "$(dirname "$candidate")" >> "$GITHUB_PATH"
 
       - name: Run plugin tests - ${{ matrix.plugin.name }}
         env:

.github/workflows/test.yml

@@ -31,13 +31,13 @@ jobs:
           fetch-depth: 1
 
       - name: Clone abxpkg
-        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
+        run: bash .github/scripts/clone_abx_repo.sh abxpkg
 
       - name: Clone abx-plugins
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+        run: bash .github/scripts/clone_abx_repo.sh abx-plugins
 
       - name: Clone abx-dl
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
+        run: bash .github/scripts/clone_abx_repo.sh abx-dl
 
       ### Setup Python & JS Languages
       - name: Set up Python ${{ matrix.python }}
@@ -49,7 +49,7 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v4
         with:
-          version: "latest"
+          version: "0.10.6"
 
       - name: Set up Node JS
         uses: actions/setup-node@v4
@@ -81,10 +81,15 @@ jobs:
           pwd
           ls
 
-      - name: Archivebox version
+      - name: ArchiveBox full install check
         run: |
-          mkdir -p tests/out/data
-          DATA_DIR="$PWD/tests/out/data" uv run --no-sync --no-sources archivebox version
+          set -Eeuo pipefail
+          DATA_DIR="$PWD/tests/out/data"
+          mkdir -p "$DATA_DIR"
+          uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox init
+          timeout 30m uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox install
+          uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox version
+          uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox status
 
       - name: Test built package with pytest
         # TODO: remove this exception for windows once we get tests passing on that platform

.github/workflows/update-homebrew-tap.yml

@@ -1,4 +1,4 @@
-name: Update Homebrew tap
+name: Update downstream packages
 
 on:
   push:
@@ -9,7 +9,7 @@ permissions:
   contents: read
 
 concurrency:
-  group: update-homebrew-tap-${{ github.ref }}
+  group: update-downstream-packages-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
@@ -17,19 +17,21 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - name: Dispatch tap update
+      - name: Dispatch downstream package updates
         env:
-          HOMEBREW_TAP_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }}
+          DOWNSTREAM_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || secrets.HOMEBREW_TAP_TOKEN }}
         run: |
-          if [ -z "$HOMEBREW_TAP_TOKEN" ]; then
-            echo "HOMEBREW_TAP_TOKEN is not configured; tap hourly fallback will update the formula."
+          if [ -z "$DOWNSTREAM_TOKEN" ]; then
+            echo "RELEASE_GH_TOKEN/HOMEBREW_TAP_TOKEN is not configured; downstream schedules will update packages."
             exit 0
           fi
 
-          curl --fail-with-body \
-            -X POST \
-            -H "Accept: application/vnd.github+json" \
-            -H "Authorization: Bearer ${HOMEBREW_TAP_TOKEN}" \
-            -H "X-GitHub-Api-Version: 2022-11-28" \
-            https://api.github.com/repos/ArchiveBox/homebrew-archivebox/dispatches \
-            -d '{"event_type":"archivebox-dev-updated"}'
+          for repo in homebrew-archivebox debian-archivebox; do
+            curl --fail-with-body \
+              -X POST \
+              -H "Accept: application/vnd.github+json" \
+              -H "Authorization: Bearer ${DOWNSTREAM_TOKEN}" \
+              -H "X-GitHub-Api-Version: 2022-11-28" \
+              "https://api.github.com/repos/ArchiveBox/${repo}/dispatches" \
+              -d '{"event_type":"archivebox-dev-updated","client_payload":{"ref":"dev"}}'
+          done

AGENTS.md

@@ -0,0 +1,87 @@
+# ArchiveBox Agent Guide
+
+ArchiveBox is the full self-hosted web archiving app. Keep this repo on the `dev` branch.
+
+## Shared Standards
+
+- Use `uv` and `uv run` for Python commands. Do not use system `python`, direct `.venv/bin/python`, or `pip` commands.
+- Prefer existing repo patterns, helper APIs, fixtures, scripts, and command surfaces.
+- Keep edits focused and minimal. Do not add wrappers, shims, aliases, or extra abstraction layers unless the current code path requires them.
+- Do not weaken assertions, skip tests, xfail tests, or accept flaky behavior.
+- No mocks, monkeypatches, fakes, simulated handlers, fake binaries, fake hooks, fake buses, or direct shortcuts around user-facing flows.
+- Tests and verification should use real CLI commands, REST/API calls, browser UI flows, real hooks, real installs, real subprocesses, real DB rows, real files, and existing fixtures.
+- Assertions must verify real correctness: exit codes, returned values, DB state, filesystem contents, field values, rendered output, and side effects.
+- Start behavior fixes with a red failing test when a test is requested or practical.
+- Trace root causes from observed behavior. Do not paper over failures with retries, wider timeouts, broad fallbacks, or looser assertions.
+- Read `README.md` for the full setup, CLI, Docker, API, and release surface.
+
+## Development Setup
+
+```bash
+uv sync --dev --all-extras
+mkdir -p data
+cd data
+uv run --project .. archivebox init --install
+```
+
+Run collection commands from inside an initialized data directory:
+
+```bash
+cd data
+uv run --project .. archivebox status
+uv run --project .. archivebox add 'https://example.com'
+uv run --project .. archivebox run
+```
+
+## User-Facing Setup
+
+Recommended CLI install:
+
+```bash
+uv tool install archivebox
+mkdir -p ~/archivebox/data
+cd ~/archivebox/data
+archivebox init --install
+archivebox add 'https://example.com'
+archivebox server 0.0.0.0:8000
+```
+
+Alternative install methods:
+
+- Docker Compose / Docker
+- Homebrew
+- Debian package
+- pip
+
+## Basic Usage
+
+```bash
+archivebox version
+archivebox help
+archivebox status
+archivebox install
+archivebox add 'https://example.com'
+archivebox add --extract=title,screenshot,pdf 'https://example.com'
+archivebox list --json --with-headers
+archivebox search 'example'
+archivebox update --filter-type=domain example.com
+archivebox remove --filter-type=exact 'https://example.com'
+archivebox run
+```
+
+## Verification
+
+Use targeted tests for focused work:
+
+<!--pytest.mark.skip(reason="pytest invocation")-->
+```bash
+uv run pytest archivebox/tests/test_cli_add.py -q
+uv run prek run --all-files
+```
+
+Use the full release/deploy loop only when requested:
+
+<!--pytest.mark.skip(reason="release/deploy script")-->
+```bash
+./bin/release_dev_stack.sh
+```

CLAUDE.md

@@ -1,497 +0,0 @@
-# Claude Code Development Guide for ArchiveBox
-
-## Quick Start
-
-```bash
-# Set up dev environment (always use uv, never pip directly)
-uv sync --dev --all-extras
-
-# Run tests as non-root user (required - ArchiveBox always refuses to run as root)
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/ -v'
-```
-
-## Development Environment Setup
-
-### Prerequisites
-- Python 3.11+ (3.13 recommended)
-- uv package manager
-- A non-root user for running tests (e.g., `testuser`)
-
-### Install Dependencies
-```bash
-uv sync --dev --all-extras  # Always use uv, never pip directly
-```
-
-### Activate Virtual Environment
-```bash
-source .venv/bin/activate
-```
-
-### Common Gotchas
-
-#### File Permissions
-New files created by root need permissions fixed for testuser:
-```bash
-chmod 644 archivebox/tests/test_*.py
-```
-
-#### DATA_DIR Environment Variable
-ArchiveBox commands must run inside a data directory. Tests use temp directories - the `run_archivebox()` helper sets `DATA_DIR` automatically.
-
-## Code Style Guidelines
-
-### Naming Conventions for Grep-ability
-Use consistent naming for everything to enable easy grep-ability and logical grouping:
-
-**Principle**: Fewest unique names. If you must create a new unique name, make it grep and group well.
-
-**Examples**:
-```python
-# Filesystem migration methods - all start with fs_
-def fs_migration_needed() -> bool: ...
-def fs_migrate() -> None: ...
-def _fs_migrate_from_0_7_0_to_0_8_0() -> None: ...
-def _fs_migrate_from_0_8_0_to_0_9_0() -> None: ...
-def _fs_next_version(current: str) -> str: ...
-
-# Logging methods - ALL must start with log_ or _log
-def log_migration_start(snapshot_id: str) -> None: ...
-def _log_error(message: str) -> None: ...
-def log_validation_result(ok: bool, msg: str) -> None: ...
-```
-
-**Rules**:
-- Group related functions with common prefixes
-- Use `_` prefix for internal/private helpers within the same family
-- ALL logging-related methods MUST start with `log_` or `_log`
-- Search for all migration functions: `grep -r "def.*fs_.*(" archivebox/`
-- Search for all logging: `grep -r "def.*log_.*(" archivebox/`
-
-### Minimize Unique Names and Data Structures
-**Do not invent new data structures, variable names, or keys if possible.** Try to use existing field names and data structures exactly to keep the total unique data structures and names in the codebase to an absolute minimum.
-
-**Example - GOOD**:
-```python
-# Binary has overrides field
-binary = Binary(overrides={'TIMEOUT': '60s'})
-
-# Binary reuses the same field name and structure
-class Binary(models.Model):
-    overrides = models.JSONField(default=dict)  # Same name, same structure
-```
-
-**Example - BAD**:
-```python
-# Don't invent new names like custom_bin_cmds, binary_overrides, etc.
-class Binary(models.Model):
-    custom_bin_cmds = models.JSONField(default=dict)  # ❌ New unique name
-```
-
-**Principle**: If you're storing the same conceptual data (e.g., `overrides`), use the same field name across all models and keep the internal structure identical. This makes the codebase predictable and reduces cognitive load.
-
-## Testing
-
-### CRITICAL: Never Run as Root
-ArchiveBox has a root check that prevents running as root user. All ArchiveBox commands (including tests) must run as non-root user inside a data directory:
-
-```bash
-# Run all migration tests
-sudo -u testuser bash -c 'source /path/to/.venv/bin/activate && python -m pytest archivebox/tests/test_migrations_*.py -v'
-
-# Run specific test file
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_08_to_09.py -v'
-
-# Run single test
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_fresh.py::TestFreshInstall::test_init_creates_database -xvs'
-```
-
-### Test File Structure
-```
-archivebox/tests/
-├── test_migrations_helpers.py    # Schemas, seeding functions, verification helpers
-├── test_migrations_fresh.py      # Fresh install tests
-├── test_migrations_04_to_09.py   # 0.4.x → 0.9.x migration tests
-├── test_migrations_07_to_09.py   # 0.7.x → 0.9.x migration tests
-└── test_migrations_08_to_09.py   # 0.8.x → 0.9.x migration tests
-```
-
-### Test Writing Standards
-
-#### NO MOCKS - Real Tests Only
-Tests must exercise real code paths:
-- Create real SQLite databases with version-specific schemas
-- Seed with realistic test data
-- Run actual `python -m archivebox` commands via subprocess
-- Query SQLite directly to verify results
-
-**If something is hard to test**: Modify the implementation to make it easier to test, or fix the underlying issue. Never mock, skip, simulate, or exit early from a test because you can't get something working inside the test.
-
-#### NO SKIPS
-Never use `@skip`, `skipTest`, or `pytest.mark.skip`. Every test must run. If a test is difficult, fix the code or test environment - don't disable the test.
-
-#### Strict Assertions
-- `init` command must return exit code 0 (not `[0, 1]`)
-- Verify ALL data is preserved, not just "at least one"
-- Use exact counts (`==`) not loose bounds (`>=`)
-
-### Example Test Pattern
-```python
-def test_migration_preserves_snapshots(self):
-    """Migration should preserve all snapshots."""
-    result = run_archivebox(self.work_dir, ['init'], timeout=45)
-    self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}")
-
-    ok, msg = verify_snapshot_count(self.db_path, expected_count)
-    self.assertTrue(ok, msg)
-```
-
-### Testing Gotchas
-
-#### Extractors Disabled for Speed
-Tests disable all extractors via environment variables for faster execution:
-```python
-env['SAVE_TITLE'] = 'False'
-env['SAVE_FAVICON'] = 'False'
-# ... etc
-```
-
-#### Timeout Settings
-Use appropriate timeouts for migration tests (45s for init, 60s default).
-
-### Plugin Testing & Code Coverage
-
-**Target: 80-90% coverage** for critical plugins (screenshot, chrome, singlefile, dom)
-
-```bash
-# Run plugin tests with coverage (both Python + JavaScript)
-bash bin/test_plugins.sh screenshot
-
-# View coverage reports
-bash bin/test_plugins.sh --coverage-report
-# Or individual reports:
-coverage report --show-missing --include='archivebox/plugins/*' --omit='*/tests/*'
-```
-
-#### Plugin Test Structure
-
-Tests are **completely isolated** from ArchiveBox - they replicate production directory structure in temp dirs:
-
-```python
-# Correct production paths:
-# Crawl:    DATA_DIR/users/{username}/crawls/YYYYMMDD/example.com/{crawl-id}/{plugin}/
-# Snapshot: DATA_DIR/users/{username}/snapshots/YYYYMMDD/example.com/{snapshot-uuid}/{plugin}/
-
-with tempfile.TemporaryDirectory() as tmpdir:
-    data_dir = Path(tmpdir)
-
-    # Crawl-level plugin (e.g., chrome launcher)
-    crawl_dir = data_dir / 'users' / 'testuser' / 'crawls' / '20240101' / 'example.com' / 'crawl-123'
-    chrome_dir = crawl_dir / 'chrome'
-    chrome_dir.mkdir(parents=True)
-
-    # Snapshot-level plugin (e.g., screenshot)
-    snapshot_dir = data_dir / 'users' / 'testuser' / 'snapshots' / '20240101' / 'example.com' / 'snap-456'
-    screenshot_dir = snapshot_dir / 'screenshot'
-    screenshot_dir.mkdir(parents=True)
-
-    # Run hook in its output directory
-    result = subprocess.run(
-        ['node', str(SCREENSHOT_HOOK), '--url=https://example.com'],
-        cwd=str(screenshot_dir),
-        env={**get_test_env(), 'EXTRA_CONTEXT': '{"snapshot_id":"snap-456"}'},
-        capture_output=True,
-        timeout=120
-    )
-```
-
-#### Coverage Improvement Loop
-
-To improve from ~20% to 80%+:
-
-1. **Run tests**: `bash bin/test_plugins.sh screenshot` → Shows: `19.1% (13/68 ranges)`
-2. **Identify gaps**: Check hook file for untested paths (session connection vs fallback, config branches, error cases)
-3. **Add tests**: Test both execution paths (connect to session + launch own browser), skip conditions, error cases, config variations
-4. **Verify**: Re-run tests → Should show: `85%+ (58+/68 ranges)`
-
-**Critical**: JavaScript hooks have TWO paths that both must be tested (connect to session ~50% + launch browser ~30% + shared ~20%). Testing only one path = max 50% coverage possible!
-
-## Database Migrations
-
-### Generate and Apply Migrations
-```bash
-# Generate migrations (run from archivebox subdirectory)
-cd archivebox
-./manage.py makemigrations
-
-# Apply migrations to test database
-cd data/
-archivebox init
-```
-
-### Schema Versions
-- **0.4.x**: First Django version. Tags as comma-separated string, no ArchiveResult model
-- **0.7.x**: Tag model with M2M, ArchiveResult model, AutoField PKs
-- **0.8.x**: Crawl/Seed models, UUID PKs, status fields, depth/retry_at
-- **0.9.x**: Seed model removed, seed_id FK removed from Crawl
-
-### Testing a Migration Path
-1. Create SQLite DB with source version schema (from `test_migrations_helpers.py`)
-2. Seed with realistic test data using `seed_0_X_data()`
-3. Run `archivebox init` to trigger migrations
-4. Verify data preservation with `verify_*` functions
-5. Test CLI commands work post-migration (`status`, `list`, `add`, etc.)
-
-### Squashed Migrations
-When testing 0.8.x (dev branch), you must record ALL replaced migrations:
-```python
-# The squashed migration replaces these - all must be recorded
-('core', '0023_alter_archiveresult_options_archiveresult_abid_and_more'),
-('core', '0024_auto_20240513_1143'),
-# ... all 52 migrations from 0023-0074 ...
-('core', '0023_new_schema'),  # Also record the squashed migration itself
-```
-
-### Migration Strategy
-- Squashed migrations for clean installs
-- Individual migrations recorded for upgrades from dev branch
-- `replaces` attribute in squashed migrations lists what they replace
-
-### Migration Gotchas
-
-#### Circular FK References in Schemas
-SQLite handles circular references with `IF NOT EXISTS`. Order matters less than in other DBs.
-
-## Plugin System Architecture
-
-### Plugin Dependency Rules
-
-Like other plugins, chrome plugins **ARE NOT ALLOWED TO DEPEND ON ARCHIVEBOX OR DJANGO**.
-However, they are allowed to depend on two shared files ONLY:
-- `archivebox/plugins/chrome/chrome_utils.js` ← source of truth API for all basic chrome ops
-- `archivebox/plugins/chrome/tests/chrome_test_utils.py` ← use for your tests, do not implement launching/killing/pid files/cdp/etc. in python, just extend this file as needed.
-
-### Chrome-Dependent Plugins
-
-Many plugins depend on Chrome/Chromium via CDP (Chrome DevTools Protocol). When checking for script name references or debugging Chrome-related issues, check these plugins:
-
-**Main puppeteer-based chrome installer + launcher plugin**:
-- `chrome` - Core Chrome integration (CDP, launch, navigation)
-
-**Metadata extraction using chrome/chrome_utils.js / CDP**:
-- `dns` - DNS resolution info
-- `ssl` - SSL certificate info
-- `headers` - HTTP response headers
-- `redirects` - Capture redirect chains
-- `staticfile` - Direct file downloads (e.g. if the url itself is a .png, .exe, .zip, etc.)
-- `responses` - Capture network responses
-- `consolelog` - Capture console.log output
-- `title` - Extract page title
-- `accessibility` - Extract accessibility tree
-- `seo` - Extract SEO metadata
-
-**Extensions installed using chrome/chrome_utils.js / controlled using CDP**:
-- `ublock` - uBlock Origin ad blocking
-- `istilldontcareaboutcookies` - Cookie banner dismissal
-- `twocaptcha` - 2captcha CAPTCHA solver integration
-
-**Page-alteration plugins to prepare the content for archiving**:
-- `modalcloser` - Modal dialog dismissal
-- `infiniscroll` - Infinite scroll handler
-
-**Main Extractor Outputs**:
-- `dom` - DOM snapshot extraction
-- `pdf` - Generate PDF snapshots
-- `screenshot` - Generate screenshots
-- `singlefile` - SingleFile archival, can be single-file-cli that launches chrome, or singlefile extension running inside chrome
-
-**Crawl URL parsers** (post-process dom.html, singlefile.html, staticfile, responses, headers, etc. for URLs to re-emit as new queued Snapshots during recursive crawling):
-- `parse_dom_outlinks` - Extract outlinks from DOM (special, uses CDP to directly query browser)
-- `parse_html_urls` - Parse URLs from HTML (doesn't use chrome directly, just reads dom.html)
-- `parse_jsonl_urls` - Parse URLs from JSONL (doesn't use chrome directly, just reads dom.html)
-- `parse_netscape_urls` - Parse Netscape bookmark format (doesn't use chrome directly, just reads dom.html)
-
-### Finding Chrome-Dependent Plugins
-
-```bash
-# Find all files containing "chrom" (case-insensitive)
-grep -ri "chrom" archivebox/plugins/*/on_*.* --include="*.*" 2>/dev/null | cut -d: -f1 | sort -u
-
-# Or get just the plugin names
-grep -ri "chrom" archivebox/plugins/*/on_*.* --include="*.*" 2>/dev/null | cut -d/ -f3 | sort -u
-```
-
-**Note**: This list may not be complete. Always run the grep command above when checking for Chrome-related script references or debugging Chrome integration issues.
-
-## Architecture Notes
-
-### Crawl Model (0.9.x)
-- Crawl groups multiple Snapshots from a single `add` command
-- Each `add` creates one Crawl with one or more Snapshots
-- Seed model was removed - crawls now store URLs directly
-
-## Code Coverage
-
-### Overview
-
-Coverage tracking is enabled for passive collection across all contexts:
-- Unit tests (pytest)
-- Integration tests
-- Dev server (manual testing)
-- CLI usage
-
-Coverage data accumulates in `.coverage` file and can be viewed/analyzed to find dead code.
-
-### Install Coverage Tools
-
-```bash
-uv sync --dev  # Installs pytest-cov and coverage
-```
-
-### Running with Coverage
-
-#### Unit Tests
-```bash
-# Run tests with coverage
-pytest --cov=archivebox --cov-report=term archivebox/tests/
-
-# Or run specific test file
-pytest --cov=archivebox --cov-report=term archivebox/tests/test_migrations_08_to_09.py
-```
-
-#### Dev Server with Coverage
-```bash
-# Start dev server with coverage tracking
-coverage run --parallel-mode -m archivebox server
-
-# Or CLI commands
-coverage run --parallel-mode -m archivebox init
-coverage run --parallel-mode -m archivebox add https://example.com
-```
-
-#### Manual Testing (Always-On)
-To enable coverage during ALL Python executions (passive tracking):
-
-```bash
-# Option 1: Use coverage run wrapper
-coverage run --parallel-mode -m archivebox [command]
-
-# Option 2: Set environment variable (tracks everything)
-export COVERAGE_PROCESS_START=pyproject.toml
-# Now all Python processes will track coverage
-archivebox server
-archivebox add https://example.com
-```
-
-### Viewing Coverage
-
-#### Text Report (Quick View)
-```bash
-# Combine all parallel coverage data
-coverage combine
-
-# View summary
-coverage report
-
-# View detailed report with missing lines
-coverage report --show-missing
-
-# View specific file
-coverage report --include="archivebox/core/models.py" --show-missing
-```
-
-#### JSON Report (LLM-Friendly)
-```bash
-# Generate JSON report
-coverage json
-
-# View the JSON
-cat coverage.json | jq '.files | keys'  # List all files
-
-# Find files with low coverage
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered < 50) | "\(.key): \(.value.summary.percent_covered)%"'
-
-# Find completely uncovered files (dead code candidates)
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered == 0) | .key'
-
-# Get missing lines for a specific file
-cat coverage.json | jq '.files["archivebox/core/models.py"].missing_lines'
-```
-
-#### HTML Report (Visual)
-```bash
-# Generate interactive HTML report
-coverage html
-
-# Open in browser
-open htmlcov/index.html
-```
-
-### Isolated Runs
-
-To measure coverage for specific scenarios:
-
-```bash
-# 1. Reset coverage data
-coverage erase
-
-# 2. Run your isolated test/scenario
-pytest --cov=archivebox archivebox/tests/test_migrations_fresh.py
-# OR
-coverage run --parallel-mode -m archivebox add https://example.com
-
-# 3. View results
-coverage combine
-coverage report --show-missing
-
-# 4. Optionally export for analysis
-coverage json
-```
-
-### Finding Dead Code
-
-```bash
-# 1. Run comprehensive tests + manual testing to build coverage
-pytest --cov=archivebox archivebox/tests/
-coverage run --parallel-mode -m archivebox server  # Use the app manually
-coverage combine
-
-# 2. Find files with 0% coverage (strong dead code candidates)
-coverage json
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered == 0) | .key'
-
-# 3. Find files with <10% coverage (likely dead code)
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered < 10) | "\(.key): \(.value.summary.percent_covered)%"' | sort -t: -k2 -n
-
-# 4. Generate detailed report for analysis
-coverage report --show-missing > coverage_report.txt
-```
-
-### Tips
-
-- **Parallel mode** (`--parallel-mode`): Allows multiple processes to track coverage simultaneously without conflicts
-- **Combine**: Always run `coverage combine` before viewing reports to merge parallel data
-- **Reset**: Use `coverage erase` to start fresh for isolated measurements
-- **Branch coverage**: Enabled by default - tracks if both branches of if/else are executed
-- **Exclude patterns**: Config in `pyproject.toml` excludes tests, migrations, type stubs
-
-## Debugging Tips
-
-### Check Migration State
-```bash
-sqlite3 /path/to/index.sqlite3 "SELECT app, name FROM django_migrations WHERE app='core' ORDER BY id;"
-```
-
-### Check Table Schema
-```bash
-sqlite3 /path/to/index.sqlite3 "PRAGMA table_info(core_snapshot);"
-```
-
-### Verbose Test Output
-```bash
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_08_to_09.py -xvs 2>&1 | head -200'
-```
-
-### Kill Zombie Chrome Processes
-```bash
-./bin/kill_chrome.sh
-```

Dockerfile

@@ -2,7 +2,7 @@
 
 # Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image.
 # abx-dl owns Python, Node, Chromium, and downloader plugin runtimes.
-# ArchiveBox owns ripgrep, sonic, supervisor, Django, and the app runtime.
+# ArchiveBox owns sonic, supervisor, Django, and the app runtime.
 # Build abx-dl first, then point this file at it:
 #   docker buildx build ../abx-dl -f ../abx-dl/Dockerfile \
 #       --build-context abxbus=../abxbus \
@@ -22,6 +22,7 @@ ARG TARGETPLATFORM
 ARG TARGETOS
 ARG TARGETARCH
 ARG TARGETVARIANT
+ARG ARCHIVEBOX_COMMIT_HASH=""
 
 ENV TZ=UTC \
     LANGUAGE=en_US:en \
@@ -41,17 +42,16 @@ ENV PYTHON_VERSION=3.13 \
     NODE_VERSION=24
 
 ENV ARCHIVEBOX_USER=archivebox \
-    DEFAULT_PUID=911 \
-    DEFAULT_PGID=911 \
+    DEFAULT_ARCHIVEBOX_UID=911 \
+    DEFAULT_ARCHIVEBOX_GID=911 \
     IN_DOCKER=True
 
 ENV CODE_DIR=/app \
     DATA_DIR=/data \
-    LIB_DIR=/opt/archivebox/lib \
+    CONFIG_DIR=/opt/archivebox \
     ABXPKG_LIB_DIR=/opt/archivebox/lib \
     PLAYWRIGHT_BROWSERS_PATH=/opt/archivebox/lib/playwright/cache \
     PERSONAS_DIR=/data/personas \
-    CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \
     CHROME_HEADLESS=true \
     CHROME_SANDBOX=false \
     CHROME_ISOLATION=crawl \
@@ -65,7 +65,7 @@ ENV TMP_DIR=/tmp/archivebox \
 
 ENV HOME=/home/archivebox \
     XDG_CONFIG_HOME=/home/archivebox/.config \
-    XDG_CACHE_HOME=/home/archivebox/.cache \
+    XDG_CACHE_HOME=/opt/archivebox/lib/cache \
     ABXPKG_INSTALL_TIMEOUT=600 \
     ABXPKG_POSTINSTALL_SCRIPTS=True \
     ABXPKG_MIN_RELEASE_AGE=0 \
@@ -77,7 +77,7 @@ ENV UV_COMPILE_BYTECODE=false \
     UV_LINK_MODE=copy \
     UV_PROJECT_ENVIRONMENT=/venv \
     VIRTUAL_ENV=/venv \
-    PATH="/venv/bin:/opt/node/bin:/opt/archivebox/lib/bin:$PATH"
+    PATH="/venv/bin:/opt/node/bin:$PATH"
 
 SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
 WORKDIR "$CODE_DIR"
@@ -88,50 +88,87 @@ RUN cp /VERSION.txt /ABX-DL-VERSION.txt \
     && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
     && uname -a \
     && sed -n '1,7p' /etc/os-release \
-    && which node && node --version \
-    && which uv && uv self version \
+    && abxpkg load --binproviders=env node \
+    && abxpkg load --binproviders=env uv \
     ) | tee -a /VERSION.txt
 
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    echo "[+] APT Installing ArchiveBox search dependency ripgrep for $TARGETPLATFORM..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends ripgrep \
-    && rm -rf /var/lib/apt/lists/*
-
 FROM archivebox-runtime-base AS archivebox-builder
 
 WORKDIR "$CODE_DIR"
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
     --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
     --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
-    echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..." \
-    && echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
-    && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends \
-    && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests \
-    && rm -f /etc/apt/apt.conf.d/docker-clean \
-    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends \
-        build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
-    && /usr/bin/uv venv --clear /venv --python "${PYTHON_VERSION}" \
-    && /usr/bin/uv pip install setuptools pip wheel \
-    && /usr/bin/uv sync \
-        --refresh \
-        --no-dev \
-        --inexact \
-        --no-install-project \
-        --no-install-workspace \
-        --no-sources \
-    && (find /venv/lib/python3.*/site-packages -type f -name '*.so' -exec strip --strip-unneeded {} + 2>/dev/null || true) \
-    && rm -f /venv/bin/uv /venv/bin/uvx \
-    && apt-get purge -y build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
-    && apt-get autoremove -y \
-    && rm -rf /var/lib/apt/lists/*
+    <<'EOF'
+echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..."
+echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache
+echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends
+echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests
+rm -f /etc/apt/apt.conf.d/docker-clean
+apt-get update -qq
+apt-get install -qq -y --no-install-recommends \
+    build-essential gcc libldap2-dev libsasl2-dev libssl-dev
+/usr/bin/uv venv --clear /venv --python "${PYTHON_VERSION}"
+/usr/bin/uv pip install setuptools pip wheel
+
+mkdir -p /tmp/archivebox-uv-project
+/venv/bin/python - <<'PY'
+from pathlib import Path
+import json
+import re
+import urllib.request
+
+source = Path("/app/pyproject.toml")
+target = Path("/tmp/archivebox-uv-project/pyproject.toml")
+text = source.read_text()
+text = text.replace(
+    'environments = ["sys_platform == \'darwin\'", "sys_platform == \'linux\'"]',
+    'environments = ["sys_platform == \'linux\'"]',
+)
+
+# Docker builds need the just-published internal abx wheels immediately, but
+# PyPI simple can lag the version JSON endpoints by tens of minutes. Generate a
+# Docker-only dependency view from the version JSON so the published package
+# metadata stays normal while image builds remain resumable after a release.
+for package in ("abxpkg", "abx-plugins", "abx-dl"):
+    match = re.search(rf'"{re.escape(package)}>=(?P<version>[^"]+)"', text)
+    if not match:
+        continue
+    version = match.group("version")
+    with urllib.request.urlopen(f"https://pypi.org/pypi/{package}/{version}/json", timeout=20) as response:
+        data = json.load(response)
+    wheel_url = next(url["url"] for url in data["urls"] if url["filename"].endswith(".whl"))
+    text = re.sub(
+        rf'"{re.escape(package)}>=[^"]+"',
+        f'"{package} @ {wheel_url}"',
+        text,
+        count=1,
+    )
+
+target.write_text(text)
+PY
+
+/usr/bin/uv sync \
+    --project /tmp/archivebox-uv-project \
+    --refresh \
+    --no-dev \
+    --inexact \
+    --no-install-project \
+    --no-install-workspace \
+    --no-sources
+(find /venv/lib/python3.*/site-packages -type f -name '*.so' -exec strip --strip-unneeded {} + 2>/dev/null || true)
+rm -f /venv/bin/uv /venv/bin/uvx
+apt-get purge -y build-essential gcc libldap2-dev libsasl2-dev libssl-dev
+apt-get autoremove -y
+rm -rf /var/lib/apt/lists/*
+EOF
 
 COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
 RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
     echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
     && COMMIT_HASH="$( \
-        if [[ -f "$CODE_DIR/.git/HEAD" ]]; then \
+        if [[ "$ARCHIVEBOX_COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then \
+            echo "$ARCHIVEBOX_COMMIT_HASH"; \
+        elif [[ -f "$CODE_DIR/.git/HEAD" ]]; then \
             HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"; \
             if [[ "$HEAD_REF" =~ ^[0-9a-fA-F]{40}$ ]]; then \
                 echo "$HEAD_REF"; \
@@ -143,8 +180,7 @@ RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$T
     && if [[ "$COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt; fi \
     && /usr/bin/uv pip install --no-deps "$CODE_DIR" \
     && rm -f /venv/bin/uv /venv/bin/uvx \
-    && (/usr/bin/uv pip show archivebox && which archivebox) | tee -a /VERSION.txt \
-    && rm -rf "$CODE_DIR/.git"
+    && /usr/bin/uv pip show archivebox | tee -a /VERSION.txt
 
 FROM archivebox-runtime-base
 
@@ -162,12 +198,13 @@ LABEL name="archivebox" \
 COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
 COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
 
+COPY --from=archivebox-builder /opt/uv/python /opt/uv/python
 COPY --from=archivebox-builder /venv /venv
 COPY --from=archivebox-builder /app /app
 COPY --from=archivebox-builder /VERSION.txt /VERSION.txt
 
-RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
-    && printf 'export PATH="/venv/bin:/opt/node/bin:/opt/archivebox/lib/bin:$PATH"\n' > /etc/profile.d/archivebox-path.sh \
+RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_ARCHIVEBOX_UID}..." \
+    && printf 'export PATH="/venv/bin:/opt/node/bin:$PATH"\n' > /etc/profile.d/archivebox-path.sh \
     && ln -sf /venv/bin/archivebox /usr/local/bin/archivebox \
     && ln -sf /venv/bin/daphne /usr/local/bin/daphne \
     && ln -sf /venv/bin/supervisord /usr/local/bin/supervisord \
@@ -175,42 +212,44 @@ RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
     && getent group "$ARCHIVEBOX_USER" >/dev/null || groupadd --system "$ARCHIVEBOX_USER" \
     && id -u "$ARCHIVEBOX_USER" >/dev/null 2>&1 || useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \
     && usermod --append --groups audio,video "$ARCHIVEBOX_USER" \
-    && [[ "$(id -u "$ARCHIVEBOX_USER")" == "$DEFAULT_PUID" ]] || usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
-    && [[ "$(id -g "$ARCHIVEBOX_USER")" == "$DEFAULT_PGID" ]] || groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
-    && (which sonic && sonic --version) | tee -a /VERSION.txt \
-    && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER" "/home/$ARCHIVEBOX_USER/.config" "/home/$ARCHIVEBOX_USER/.cache" \
-    && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER/.config/abx" "/home/$ARCHIVEBOX_USER/.cache/abxbus" "/home/$ARCHIVEBOX_USER/.cache/pnpm" "/home/$ARCHIVEBOX_USER/.cache/uv" \
-    && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
+    && [[ "$(id -u "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_UID" ]] || usermod -u "$DEFAULT_ARCHIVEBOX_UID" "$ARCHIVEBOX_USER" \
+    && [[ "$(id -g "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_GID" ]] || groupmod -g "$DEFAULT_ARCHIVEBOX_GID" "$ARCHIVEBOX_USER" \
+    && abxpkg load --binproviders=env sonic | tee -a /VERSION.txt \
+    && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$XDG_CACHE_HOME" "$PLAYWRIGHT_BROWSERS_PATH" \
+    && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "/home/$ARCHIVEBOX_USER" \
+    && chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" \
+    && chown -R "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "$ABXPKG_LIB_DIR" \
     && openssl rand -hex 16 > /etc/machine-id \
-    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u "$ARCHIVEBOX_USER") PGID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
-    && echo -e "TMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
+    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER ARCHIVEBOX_UID=$(id -u "$ARCHIVEBOX_USER") ARCHIVEBOX_GID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
+    && echo -e "TMP_DIR=$TMP_DIR\nABXPKG_LIB_DIR=$ABXPKG_LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
 
 WORKDIR "$DATA_DIR"
 RUN echo "[+] Initializing image collection..." \
     && find "$DATA_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} + \
-    && PUID=0 PGID=0 archivebox init \
-    && (chown "$DEFAULT_PUID:$DEFAULT_PGID" \
+    && HOME="$TMP_DIR" archivebox init \
+    && (chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" \
         "$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \
         "$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \
         "$DATA_DIR"/archive "$DATA_DIR"/archive/users "$DATA_DIR"/personas \
         "$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \
-        "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
+        "$CONFIG_DIR" "$CONFIG_DIR"/config.env "$CONFIG_DIR"/derived.env \
+        "$TMP_DIR" "$ABXPKG_LIB_DIR" "$XDG_CACHE_HOME" "$PLAYWRIGHT_BROWSERS_PATH" \
         2>/dev/null || true) \
     && find "$TMP_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} +
 
 RUN chmod +x "$CODE_DIR"/bin/*.sh \
-    && chmod g+w "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"
+    && chmod g+w "$TMP_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"
 
-RUN "$LIB_DIR/playwright/bin/chromium" --version | tee -a /VERSION.txt \
-    && "$LIB_DIR/uv/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \
-    && /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \
-    && /usr/local/bin/sonic --version | tee -a /VERSION.txt \
-    && /venv/bin/supervisord --version | tee -a /VERSION.txt \
-    && for forbidden_bin in gcc g++ make; do ! command -v "$forbidden_bin" || (echo "Unexpected build tool in runtime: $forbidden_bin=$(command -v "$forbidden_bin")" >&2 && exit 1); done \
-    && stat -c "%U:%G %a %n" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$LIB_DIR" \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \
+RUN --mount=type=cache,target=/tmp/abxpkg-cache,sharing=locked,mode=1777 \
+    for forbidden_bin in gcc g++ make; do ! abxpkg load --binproviders=env "$forbidden_bin" >/dev/null 2>&1 || (echo "Unexpected build tool in runtime: $forbidden_bin" >&2 && exit 1); done \
+    && stat -c "%U:%G %a %n" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$CONFIG_DIR" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$ABXPKG_LIB_DIR" \
+    && python3 -c 'from abx_dl.models import discover_plugins; [print(f"export {plugin.enabled_key}=True") for plugin in discover_plugins(runtime="archivebox").values() if plugin.enabled_key in plugin.config.properties]' > /tmp/archivebox-enable-plugins.env \
+    && sort /tmp/archivebox-enable-plugins.env | tee -a /VERSION.txt \
+    && source /tmp/archivebox-enable-plugins.env \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups env HOME=/tmp/abxpkg-cache ABXPKG_NO_CACHE=True ABXPKG_TMP_CACHE_DIR=/tmp/abxpkg-cache archivebox install \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups env HOME=/tmp/abxpkg-cache archivebox version 2>&1 | tee -a /VERSION.txt \
     && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
 
 RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully." \

Dockerfile.multistage

@@ -1,224 +0,0 @@
-# syntax=docker/dockerfile:1.7
-
-# Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image.
-# abx-dl owns Python, Node, Chromium, and downloader plugin runtimes.
-# ArchiveBox owns ripgrep, sonic, supervisor, Django, and the app runtime.
-# Build abx-dl first, then point this file at it:
-#   docker buildx build ../abx-dl -f ../abx-dl/Dockerfile \
-#       --build-context abxbus=../abxbus \
-#       --build-context abxpkg=../abxpkg \
-#       --build-context abx-plugins=../abx-plugins \
-#       -t archivebox/abx-dl:dev
-#   docker buildx build . -f Dockerfile \
-#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:latest \
-#       -t archivebox:multistage
-
-ARG ABX_DL_IMAGE=archivebox/abx-dl:latest
-
-FROM archivebox/sonic:1.4.9 AS sonic
-FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base
-
-ARG TARGETPLATFORM
-ARG TARGETOS
-ARG TARGETARCH
-ARG TARGETVARIANT
-
-ENV TZ=UTC \
-    LANGUAGE=en_US:en \
-    LC_ALL=C.UTF-8 \
-    LANG=C.UTF-8 \
-    DEBIAN_FRONTEND=noninteractive \
-    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
-    PYTHONIOENCODING=UTF-8 \
-    PYTHONUNBUFFERED=1 \
-    PYTHONDONTWRITEBYTECODE=1 \
-    PIP_DISABLE_PIP_VERSION_CHECK=1 \
-    PIP_NO_COMPILE=1 \
-    PIP_ONLY_BINARY=aiohttp \
-    npm_config_loglevel=error
-
-ENV PYTHON_VERSION=3.13 \
-    NODE_VERSION=22.22.3
-
-ENV ARCHIVEBOX_USER=archivebox \
-    DEFAULT_PUID=911 \
-    DEFAULT_PGID=911 \
-    IN_DOCKER=True
-
-ENV CODE_DIR=/app \
-    DATA_DIR=/data \
-    LIB_DIR=/opt/archivebox/lib \
-    ABXPKG_LIB_DIR=/opt/archivebox/lib \
-    PLAYWRIGHT_BROWSERS_PATH=/opt/archivebox/lib/playwright/cache \
-    PERSONAS_DIR=/data/personas \
-    CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \
-    CHROME_HEADLESS=true \
-    CHROME_SANDBOX=false \
-    CHROME_ISOLATION=crawl \
-    CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
-
-ENV TMP_DIR=/tmp/archivebox \
-    PIP_VENV_PYTHON=/venv/bin/python3 \
-    GOOGLE_API_KEY=no \
-    GOOGLE_DEFAULT_CLIENT_ID=no \
-    GOOGLE_DEFAULT_CLIENT_SECRET=no
-
-ENV HOME=/home/archivebox \
-    XDG_CONFIG_HOME=/home/archivebox/.config \
-    XDG_CACHE_HOME=/home/archivebox/.cache \
-    ABXPKG_INSTALL_TIMEOUT=600 \
-    ABXPKG_POSTINSTALL_SCRIPTS=True \
-    ABXPKG_MIN_RELEASE_AGE=0 \
-    TIMEOUT=600
-
-ENV UV_COMPILE_BYTECODE=false \
-    UV_PYTHON_PREFERENCE=managed \
-    UV_PYTHON_INSTALL_DIR=/opt/uv/python \
-    ARCHIVEBOX_PYTHON_INSTALL_DIR=/opt/archivebox/python \
-    UV_LINK_MODE=copy \
-    UV_PROJECT_ENVIRONMENT=/venv \
-    VIRTUAL_ENV=/venv \
-    PATH="/venv/bin:/opt/node/bin:/opt/archivebox/lib/bin:$PATH"
-
-SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
-WORKDIR "$CODE_DIR"
-
-RUN cp /VERSION.txt /ABX-DL-VERSION.txt \
-    && (echo "[i] Docker build for ArchiveBox multistage starting..." \
-    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
-    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
-    && uname -a \
-    && sed -n '1,7p' /etc/os-release \
-    && which node && node --version \
-    && which uv && uv self version \
-    ) | tee -a /VERSION.txt
-
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    echo "[+] APT Installing ArchiveBox search dependency ripgrep for $TARGETPLATFORM..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends ripgrep \
-    && rm -rf /var/lib/apt/lists/*
-
-FROM archivebox-runtime-base AS archivebox-builder
-
-WORKDIR "$CODE_DIR"
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
-    echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..." \
-    && export UV_PYTHON_INSTALL_DIR="$ARCHIVEBOX_PYTHON_INSTALL_DIR" \
-    && echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
-    && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends \
-    && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests \
-    && rm -f /etc/apt/apt.conf.d/docker-clean \
-    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends \
-        build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
-    && /usr/bin/uv venv --clear /venv --python "${PYTHON_VERSION}" \
-    && /usr/bin/uv pip install setuptools pip wheel \
-    && /usr/bin/uv sync \
-        --refresh \
-        --no-dev \
-        --inexact \
-        --all-extras \
-        --no-install-project \
-        --no-install-workspace \
-        --no-sources \
-    && apt-get purge -y build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
-    && apt-get autoremove -y \
-    && rm -rf /var/lib/apt/lists/*
-
-COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
-RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
-    && COMMIT_HASH="$( \
-        if [[ -f "$CODE_DIR/.git/HEAD" ]]; then \
-            HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"; \
-            if [[ "$HEAD_REF" =~ ^[0-9a-fA-F]{40}$ ]]; then \
-                echo "$HEAD_REF"; \
-            elif [[ "$HEAD_REF" == ref:\ * ]]; then \
-                REF_PATH="${HEAD_REF#ref: }"; \
-                cat "$CODE_DIR/.git/$REF_PATH" 2>/dev/null || awk -v ref="$REF_PATH" '$2 == ref {print $1}' "$CODE_DIR/.git/packed-refs" 2>/dev/null || true; \
-            fi; \
-        fi)" \
-    && if [[ "$COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt; fi \
-    && /usr/bin/uv pip install --no-deps "$CODE_DIR" \
-    && (/usr/bin/uv pip show archivebox && which archivebox) | tee -a /VERSION.txt \
-    && rm -rf "$CODE_DIR/.git"
-
-FROM archivebox-runtime-base
-
-LABEL name="archivebox" \
-    maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
-    description="All-in-one self-hosted internet archiving solution" \
-    homepage="https://github.com/ArchiveBox/ArchiveBox" \
-    documentation="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker" \
-    org.opencontainers.image.title="ArchiveBox" \
-    org.opencontainers.image.vendor="ArchiveBox" \
-    org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \
-    org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \
-    com.docker.image.source.entrypoint="Dockerfile"
-
-COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
-COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
-
-COPY --from=archivebox-builder /opt/archivebox/python /opt/archivebox/python
-COPY --from=archivebox-builder /venv /venv
-COPY --from=archivebox-builder /app /app
-COPY --from=archivebox-builder /VERSION.txt /VERSION.txt
-
-RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
-    && getent group "$ARCHIVEBOX_USER" >/dev/null || groupadd --system "$ARCHIVEBOX_USER" \
-    && id -u "$ARCHIVEBOX_USER" >/dev/null 2>&1 || useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \
-    && usermod --append --groups audio,video "$ARCHIVEBOX_USER" \
-    && [[ "$(id -u "$ARCHIVEBOX_USER")" == "$DEFAULT_PUID" ]] || usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
-    && [[ "$(id -g "$ARCHIVEBOX_USER")" == "$DEFAULT_PGID" ]] || groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
-    && (which sonic && sonic --version) | tee -a /VERSION.txt \
-    && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER" "/home/$ARCHIVEBOX_USER/.config" "/home/$ARCHIVEBOX_USER/.cache" \
-    && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER/.config/abx" "/home/$ARCHIVEBOX_USER/.cache/abxbus" "/home/$ARCHIVEBOX_USER/.cache/pnpm" "/home/$ARCHIVEBOX_USER/.cache/uv" \
-    && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && openssl rand -hex 16 > /etc/machine-id \
-    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u "$ARCHIVEBOX_USER") PGID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
-    && echo -e "TMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
-
-WORKDIR "$DATA_DIR"
-RUN echo "[+] Initializing image collection..." \
-    && find "$DATA_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} + \
-    && PUID=0 PGID=0 archivebox init \
-    && (chown "$DEFAULT_PUID:$DEFAULT_PGID" \
-        "$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \
-        "$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \
-        "$DATA_DIR"/archive "$DATA_DIR"/archive/users "$DATA_DIR"/personas \
-        "$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \
-        "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
-        2>/dev/null || true)
-
-RUN chmod +x "$CODE_DIR"/bin/*.sh \
-    && chmod g+w "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"
-
-RUN "$LIB_DIR/bin/chromium" --version | tee -a /VERSION.txt \
-    && "$LIB_DIR/uv/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \
-    && /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \
-    && /usr/local/bin/sonic --version | tee -a /VERSION.txt \
-    && /venv/bin/supervisord --version | tee -a /VERSION.txt \
-    && for forbidden_bin in gcc g++ make; do ! command -v "$forbidden_bin" || (echo "Unexpected build tool in runtime: $forbidden_bin=$(command -v "$forbidden_bin")" >&2 && exit 1); done \
-    && stat -c "%U:%G %a %n" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$LIB_DIR" \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \
-    && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
-
-RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully." \
-    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
-    && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
-    ) | tee -a /VERSION.txt
-
-WORKDIR "$DATA_DIR"
-VOLUME "$DATA_DIR"
-EXPOSE 8000
-
-HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
-    CMD curl --fail --silent --show-error --max-time 5 --connect-timeout 2 'http://admin.archivebox.localhost:8000/health/' | grep -q 'OK'
-
-ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh"]
-CMD ["archivebox", "server", "--init", "0.0.0.0:8000"]

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2024 Nick Sweeting
+Copyright (c) 2026 Nick Sweeting
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -29,7 +29,7 @@ Without active preservation effort, everything on the internet eventually disapp
 *ArchiveBox is an open source tool that lets organizations & individuals archive both public & private web content while retaining control over their data. It can be used to save copies of bookmarks, preserve evidence for legal cases, backup photos from FB/Insta/Flickr or media from YT/Soundcloud/etc., save research papers, and more...*
 <br/>
 
-> ➡️ Get ArchiveBox with `pip install archivebox` on [Linux](#quickstart)/[macOS](#quickstart), or via **[Docker](#quickstart)** ⭐️ on any OS.  
+> ➡️ Get ArchiveBox with `pip install archivebox>=0.9.5rc1` on [Linux](#quickstart)/[macOS](#quickstart), or via **[Docker](#quickstart)** ⭐️ on any OS.  
 
 *Once installed, you can interact with it through the: [Browser Extension](https://github.com/ArchiveBox/archivebox-browser-extension), [CLI](#usage), [self-hosted web interface](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive), [Python API](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage), or [filesystem](#static-archive-exporting).*
 
@@ -85,14 +85,14 @@ docker compose run archivebox init
 <br/>
 # Option B: Or use it as a plain Docker container:
 mkdir -p ~/archivebox/data && cd ~/archivebox/data
-docker run -it -v $PWD:/data archivebox/archivebox init
-# docker run -it -v $PWD:/data archivebox/archivebox add 'https://example.com'
-# docker run -it -v $PWD:/data archivebox/archivebox help
-# docker run -it -v $PWD:/data -p 8000:8000 archivebox/archivebox
+docker run -it -v $PWD:/data archivebox/archivebox:dev init
+# docker run -it -v $PWD:/data archivebox/archivebox:dev add 'https://example.com'
+# docker run -it -v $PWD:/data archivebox/archivebox:dev help
+# docker run -it -v $PWD:/data -p 8000:8000 archivebox/archivebox:dev
 <br/>
 <br/>
-# Option C: Or install it with your preferred pkg manager (see Quickstart below for apt, brew, and more)
-pip install archivebox
+# Option C: Or install it with uv (see Quickstart below for apt, brew, and more)
+uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
 mkdir -p ~/archivebox/data && cd ~/archivebox/data
 archivebox init
 archivebox install
@@ -214,14 +214,14 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <li>Install <a href="https://docs.docker.com/get-docker/">Docker</a> on your system (if not already installed).</li>
 <li>Create a new empty directory and initialize your collection (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
-docker run -v $PWD:/data -it archivebox/archivebox init
+docker run -v $PWD:/data -it archivebox/archivebox:dev init
 </code></pre>
 </li>
 <li>Optional: Start the server then login to the Web UI <a href="http://archivebox.localhost:8000">http://archivebox.localhost:8000</a> ⇢ Admin.
-<pre lang="bash"><code style="white-space: pre-line">docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox
+<pre lang="bash"><code style="white-space: pre-line">docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox:dev
 # completely optional, CLI can always be used without running a server
 # docker run -v $PWD:/data -it [subcommand] [--help]
-docker run -v $PWD:/data -it archivebox/archivebox help
+docker run -v $PWD:/data -it archivebox/archivebox:dev help
 </code></pre>
 <i>For more info, see <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-a-docker--docker-compose-setup-%EF%B8%8F">Install: Docker Compose</a> in the Wiki. ➡️</i>
 </li>
@@ -256,13 +256,13 @@ See <a href="https://docs.sweeting.me/s/against-curl-sh">"Against curl | sh as a
 
 
 <details>
-<summary><b><img src="https://user-images.githubusercontent.com/511499/117447613-ba4c5d80-af0b-11eb-8f89-1d98e31b6a79.png" alt="Pip" height="28px" align="top"/> <code>pip</code></b> (macOS/Linux/BSD)</summary>
+<summary><b><img src="https://user-images.githubusercontent.com/511499/117447613-ba4c5d80-af0b-11eb-8f89-1d98e31b6a79.png" alt="uv" height="28px" align="top"/> <code>uv</code></b> (macOS/Linux/BSD)</summary>
 <br/>
 <ol>
 
-<li>Install <a href="https://realpython.com/installing-python/">Python >= v3.13</a> and <a href="https://nodejs.org/en/download/package-manager/">Node >= v22</a> on your system (if not already installed).</li>
-<li>Install the ArchiveBox package using <code>pip3</code> (or <a href="https://docs.astral.sh/uv/guides/tools/#running-tools"><code>uvx</code></a>).
-<pre lang="bash"><code style="white-space: pre-line">pip3 install --upgrade archivebox
+<li>Install <a href="https://docs.astral.sh/uv/getting-started/installation/">uv</a> on your system (if not already installed).</li>
+<li>Install the ArchiveBox package using <code>uv</code>.
+<pre lang="bash"><code style="white-space: pre-line">uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
 archivebox version
 # install any missing extras shown using apt/brew/pkg/etc. see Wiki for instructions
 #    python@3.13 node curl wget git ripgrep ...
@@ -476,6 +476,22 @@ For more discussion on managed and paid hosting options see here: <a href="https
 ArchiveBox commands can be run in a terminal [directly on your host](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage), or via [Docker](https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage-1)/[Docker Compose](https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage).  
 <sup>(depending on how you chose to install it above)</sup>
 
+<!--
+```bash
+set -euo pipefail
+__archivebox_docs_home="$(mktemp -d)"
+export HOME="$__archivebox_docs_home"
+mkdir -p ~/archivebox/data
+cd ~/archivebox/data
+archivebox init
+archivebox version
+archivebox help
+test -f index.sqlite3
+test -d archive
+rm -rf "$__archivebox_docs_home"
+```
+-->
+<!--pytest.mark.skip(reason='Mixed host/Docker alternatives include interactive Docker commands')-->
 ```bash
 mkdir -p ~/archivebox/data   # create a new data dir anywhere
 cd ~/archivebox/data         # IMPORTANT: cd into the directory
@@ -487,8 +503,8 @@ archivebox help
 # equivalent: docker compose run archivebox [subcommand] [--help]
 docker compose run archivebox help
 
-# equivalent: docker run -it -v $PWD:/data archivebox/archivebox [subcommand] [--help]
-docker run -it -v $PWD:/data archivebox/archivebox help
+# equivalent: docker run -it -v $PWD:/data archivebox/archivebox:dev [subcommand] [--help]
+docker run -it -v $PWD:/data archivebox/archivebox:dev help
 
 # optional: import your browser cookies into a persona for logged-in archiving
 archivebox persona create --import=chrome personal
@@ -549,13 +565,13 @@ docker compose run archivebox add 'https://example.com'
 <pre lang="bash"><code style="white-space: pre-line">
 # make sure you create and cd into in a new empty directory first  
 <br/>
-# docker run -it -v $PWD:/data archivebox/archivebox [subcommand] [--help]
-docker run -v $PWD:/data -it archivebox/archivebox init
-docker run -v $PWD:/data -it archivebox/archivebox install
-docker run -v $PWD:/data -it archivebox/archivebox version
-docker run -v $PWD:/data -it archivebox/archivebox help
-docker run -v $PWD:/data -it archivebox/archivebox add 'https://example.com'
-# to start webserver: docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox
+# docker run -it -v $PWD:/data archivebox/archivebox:dev [subcommand] [--help]
+docker run -v $PWD:/data -it archivebox/archivebox:dev init
+docker run -v $PWD:/data -it archivebox/archivebox:dev install
+docker run -v $PWD:/data -it archivebox/archivebox:dev version
+docker run -v $PWD:/data -it archivebox/archivebox:dev help
+docker run -v $PWD:/data -it archivebox/archivebox:dev add 'https://example.com'
+# to start webserver: docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox:dev
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage-1">Usage: Docker CLI</a> wiki. ➡️</i>
 </details>
@@ -587,8 +603,8 @@ nano docker-compose.yml                        # setup initial ADMIN_USERNAME &
 docker compose up                              # start the server
 <br/>
 # Or with a Docker container:
-docker run -v $PWD:/data -it archivebox/archivebox archivebox manage createsuperuser
-docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox
+docker run -v $PWD:/data -it archivebox/archivebox:dev archivebox manage createsuperuser
+docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox:dev
 </code></pre>
 
 <sup>Open <a href="http://web.archivebox.localhost:8000"><code>http://web.archivebox.localhost:8000</code></a> for the public UI and <a href="http://admin.archivebox.localhost:8000"><code>http://admin.archivebox.localhost:8000</code></a> for the admin UI ➡️</sup><br/>
@@ -622,7 +638,7 @@ docker compose run archivebox config --set ...
 <pre lang="bash"><code style="white-space: pre-line">
 archivebox add --depth=1 'https://example.com'                     # add a URL with pip-installed archivebox on the host
 docker compose run archivebox add --depth=1 'https://example.com'                       # or w/ Docker Compose
-docker run -it -v $PWD:/data archivebox/archivebox add --depth=1 'https://example.com'  # or w/ Docker, all equivalent
+docker run -it -v $PWD:/data archivebox/archivebox:dev add --depth=1 'https://example.com'  # or w/ Docker, all equivalent
 </code></pre>
 
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker">Docker</a> wiki. ➡️</i>
@@ -680,19 +696,58 @@ docker run -it -v $PWD:/data archivebox/archivebox add --depth=1 'https://exampl
 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/e1e5bd78-b0b6-45dc-914c-e1046fee4bc4" width="330px" align="right" style="float: right"/>
 
 
+<!--
+```bash
+set -euo pipefail
+__archivebox_docs_home="$(mktemp -d)"
+export HOME="$__archivebox_docs_home"
+mkdir -p ~/archivebox/data ~/Downloads
+cd ~/archivebox/data
+archivebox init
+cat > ~/Downloads/some_feed.xml <<'EOF'
+<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0">
+  <channel>
+    <title>ArchiveBox docs test feed</title>
+    <link>https://example.com/</link>
+    <description>ArchiveBox docs test feed</description>
+    <item>
+      <title>Feed item</title>
+      <link>https://example.com/from-feed</link>
+      <guid>https://example.com/from-feed</guid>
+    </item>
+  </channel>
+</rss>
+EOF
+```
+-->
+<!--pytest-codeblocks:cont-->
 ```bash
 # archivebox add --help
 archivebox add 'https://example.com/some/page'
-archivebox add --parser=generic_rss < ~/Downloads/some_feed.xml
+archivebox add --depth=1 --plugins=parse_rss_urls "file://$HOME/Downloads/some_feed.xml"
 archivebox add --depth=1 'https://news.ycombinator.com#2020-12-12'
 echo 'http://example.com' | archivebox add
 echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox add
 
 # if using Docker, add -i when piping stdin:
-# echo 'https://example.com' | docker run -v $PWD:/data -i archivebox/archivebox add
+# echo 'https://example.com' | docker run -v $PWD:/data -i archivebox/archivebox:dev add
 # if using Docker Compose, add -T when piping stdin / stdout:
 # echo 'https://example.com' | docker compose run -T archivebox add
 ```
+<!--pytest-codeblocks:cont-->
+<!--
+```bash
+archivebox list --json > snapshots.json
+grep -q 'https://example.com/some/page' snapshots.json
+grep -q 'https://example.com/from-feed' snapshots.json
+grep -q 'https://news.ycombinator.com#2020-12-12' snapshots.json
+grep -q 'http://example.com' snapshots.json
+test -d archive
+test "$(find archive -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ')" -ge 4
+rm -rf "$__archivebox_docs_home"
+```
+-->
 
 See the [Usage: CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) page for documentation and examples.
 
@@ -827,10 +882,8 @@ You don't need to install every dependency to use ArchiveBox. ArchiveBox will au
   
 If not using Docker, make sure to keep the dependencies up-to-date yourself and check that ArchiveBox isn't reporting any incompatibility with the versions you install.
 
-<pre lang="bash"><code style="white-space: pre-line">#install python3 and archivebox with your system package manager
-# apt/brew/pip/etc install ... (see Quickstart instructions above)
+<pre lang="bash"><code style="white-space: pre-line"># install uv + archivebox first (see Quickstart instructions above)
 <br/>
-which -a archivebox    # see where you have installed archivebox
 archivebox install     # auto install all the extractors and extras
 archivebox --version   # see info and check validity of installed dependencies
 </code></pre>
@@ -1089,7 +1142,7 @@ Because ArchiveBox is designed to ingest a large volume of URLs with multiple co
 <li><strong>Don't store large collections on older filesystems like EXT3/FAT</strong> as they may not be able to handle more than 50k directory entries in the <code>data/archive/</code> folder.
 </li>
 <li><strong>Try to keep the <code>data/index.sqlite3</code> file on local drive (not a network mount)</strong> or SSD for maximum performance, however the <code>data/archive/</code> folder can be on a network mount or slower HDD.</li>
-<li>If using Docker or NFS/SMB/FUSE for the <code>data/archive/</code> folder, you may need to set <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#puid--pgid"><code>PUID</code> & <code>PGID</code></a> and <a href="https://github.com/ArchiveBox/ArchiveBox/issues/1304">disable <code>root_squash</code></a> on your fileshare server.
+<li>If using Docker or NFS/SMB/FUSE for the <code>data/archive/</code> folder, make sure the mounted data directory is writable by its intended owner and consider disabling <a href="https://github.com/ArchiveBox/ArchiveBox/issues/1304"><code>root_squash</code></a> on your fileshare server.
 </li>
 </ul>
 
@@ -1099,7 +1152,6 @@ Because ArchiveBox is designed to ingest a large volume of URLs with multiple co
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">Wiki: Usage (Disk Layout)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#output-folder">Wiki: Security Overview (Output-Folder)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#large-archives">Wiki: Usage (Large Archives)</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#puid--pgid">Wiki: Configuration (<code>PUID</code> & <code>GUID</code>)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#do-not-run-as-root">Wiki: Security Overview (Do Not Run as Root)</a></li>
 </ul>
 
@@ -1352,6 +1404,7 @@ For low hanging fruit / easy first tickets, see: <a href="https://github.com/Arc
 
 First make sure you have `uv` installed: https://docs.astral.sh/uv/getting-started/installation/
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 git clone https://github.com/ArchiveBox/monorepo
 cd monorepo
@@ -1373,6 +1426,7 @@ Repos included in monorepo setup:
 
 #### 2. Option A: Install the Python, JS, and system dependencies directly on your machine
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 # Install ArchiveBox runtime dependencies
 mkdir -p data && cd data
@@ -1388,13 +1442,15 @@ archivebox server 0.0.0.0:8000
 
 #### 2. Option B: Build the docker container and use that for development instead
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 # Optional: develop via docker by mounting the code dir into the container
 # if you edit e.g. ./archivebox/core/models.py on the docker host, runserver
 # inside the container will reload and pick up your changes
 ./bin/build_docker.sh dev
 
-docker run -it -v $PWD/data:/data archivebox/archivebox:dev init --install
+docker run -it -v $PWD/data:/data archivebox/archivebox:dev init
+docker run -it -v $PWD/data:/data archivebox/archivebox:dev install
 
 # Run the development server w/ autoreloading (but no bg workers)
 docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server --debug --reload 0.0.0.0:8000
@@ -1417,6 +1473,7 @@ You can also run all these in Docker. For more examples see the GitHub Actions C
 
 <details><summary><i>Click to expand...</i></summary>
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 # set up persistent DEBUG=True for all runs
 archivebox config --set DEBUG=True
@@ -1441,6 +1498,7 @@ https://stackoverflow.com/questions/1074212/how-can-i-see-the-raw-sql-queries-dj
 
 If you're looking for the latest `dev` Docker image, it's often available pre-built on Docker Hub, simply pull and use `archivebox/archivebox:dev`.
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 docker pull archivebox/archivebox:dev
 docker run archivebox/archivebox:dev version
@@ -1448,9 +1506,10 @@ docker run archivebox/archivebox:dev version
 ```
 
 ##### Build Branch from Source
-  
+
 You can also build and run any branch yourself from source, for example to build & use `dev` locally:
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 # docker-compose.yml:
 services:
@@ -1475,6 +1534,7 @@ archivebox install
 
 <details><summary><i>Click to expand...</i></summary>
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 ./bin/lint.sh
 ./bin/test.sh
@@ -1488,6 +1548,7 @@ archivebox install
 
 <details><summary><i>Click to expand...</i></summary>
 
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 # generate the database migrations after changes to models.py
 cd archivebox/
@@ -1552,6 +1613,7 @@ Copy a similar plugin as a template to modify, then open a new PR to add it in t
 <details><summary><i>Click to expand...</i></summary>
 
 (Normally CI takes care of this, but these scripts can be run to do it manually)
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 ./bin/build.sh
 
@@ -1568,6 +1630,7 @@ Copy a similar plugin as a template to modify, then open a new PR to add it in t
 <details><summary><i>Click to expand...</i></summary>
 
 (Normally CI takes care of this, but these scripts can be run to do it manually)
+<!--pytest.mark.skip(reason='Development workflow snippets are not part of docs codeblock tests')-->
 ```bash
 ./bin/release.sh
 

archivebox/api/v1_cli.py

@@ -54,6 +54,7 @@ class AddCommandSchema(Schema):
     snapshot_max_size: int = 0
     parser: str = "auto"
     plugins: str = ""
+    persona: str = "Default"
     only_new: bool | None = None
     update: bool = False
     overwrite: bool = False
@@ -121,14 +122,21 @@ def snapshot_filter_kwargs(args: SnapshotFilterCommandSchema, *, default_filter_
 @router.post("/add", response=CLICommandResponseSchema, summary="archivebox add [args] [urls]")
 def cli_add(request: HttpRequest, args: AddCommandSchema):
     from archivebox.cli.archivebox_add import add
+    from archivebox.misc.util import validate_url
 
     config_overrides: dict[str, object] = {}
     if args.only_new is not None:
         config_overrides["ONLY_NEW"] = bool(args.only_new)
     if args.update or args.overwrite:
         config_overrides["ONLY_NEW"] = False
+    submitted_urls: str | list[str] = args.urls
+    if len(args.urls) == 1:
+        try:
+            validate_url(args.urls[0])
+        except ValueError:
+            submitted_urls = args.urls[0]
     crawl, snapshots = add(
-        urls=args.urls,
+        urls=submitted_urls,
         snapshot_ids=args.snapshot_ids,
         tag=args.tag,
         depth=args.depth,
@@ -138,6 +146,7 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
         snapshot_max_size=args.snapshot_max_size,
         index_only=args.index_only,
         plugins=args.plugins,
+        persona=args.persona,
         parser=args.parser,
         bg=True,  # Always run in background for API calls
         created_by_id=request.user.pk,

archivebox/api/v1_core.py

@@ -4,6 +4,7 @@ import math
 import json
 import mimetypes
 import re
+from html import unescape
 from collections import defaultdict
 from pathlib import Path, PurePosixPath
 from uuid import UUID
@@ -832,7 +833,7 @@ def _snapshots_rss_response(
         crawl_user = snapshot.crawl.created_by if snapshot.crawl_id else None
         description = f"Original URL: {snapshot.url}\nArchived snapshot: {archived_url}"
         feed.add_item(
-            title=snapshot.title or snapshot.url,
+            title=unescape(snapshot.title or snapshot.url),
             link=archived_url or web_base_url,
             description=description,
             unique_id=str(snapshot.id),

archivebox/api/v1_crawls.py

@@ -1,5 +1,6 @@
 __package__ = "archivebox.api"
 
+import json
 from pathlib import Path
 from uuid import UUID
 from datetime import datetime
@@ -124,7 +125,7 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
     config = dict(data.config or {})
     config.setdefault("PERMISSIONS", str(get_config().PERMISSIONS))
     crawl = Crawl.objects.create(
-        urls="\n".join(urls),
+        urls="\n".join(json.dumps({"type": "CrawlSeed", "url": url, "depth": 0}, separators=(",", ":")) for url in urls),
         max_depth=data.max_depth,
         tags_str=",".join(tags),
         label=data.label,

archivebox/cli/__init__.py

@@ -18,10 +18,11 @@ if "--debug" in sys.argv:
 
 # Universal `--init` flag: when passed to ANY subcommand (e.g. `archivebox server --init`,
 # `archivebox add --init`, `archivebox shell --init`), run a `quick` archivebox init before
-# the subcommand executes. Strip it from argv here so each subcommand's own click parser
-# never sees it. Ignored for `help` and `init` themselves.
-if "--init" in sys.argv:
-    sys.argv = [arg for arg in sys.argv if arg != "--init"]
+# the subcommand executes. `--quick-init` is the old server spelling. Strip either from
+# argv here so each subcommand's own click parser never sees it. Ignored for `help` and
+# `init` themselves.
+if "--init" in sys.argv or "--quick-init" in sys.argv:
+    sys.argv = [arg for arg in sys.argv if arg not in ("--init", "--quick-init")]
     os.environ["ARCHIVEBOX_WANTS_INIT"] = "1"
 
 
@@ -32,6 +33,7 @@ class ArchiveBoxGroup(click.Group):
         "help": "archivebox.cli.archivebox_help.main",
         "version": "archivebox.cli.archivebox_version.main",
         "mcp": "archivebox.cli.archivebox_mcp.main",
+        "oneshot": "archivebox.cli.archivebox_oneshot.main",
     }
     setup_commands = {
         "init": "archivebox.cli.archivebox_init.main",

archivebox/cli/archivebox_add.py

@@ -5,14 +5,15 @@ __package__ = "archivebox.cli"
 __command__ = "archivebox add"
 
 import sys
-import json
 import os
+import json
 from pathlib import Path
 
 from typing import Any, TYPE_CHECKING
 
 import rich_click as click
 
+from archivebox.config import CONSTANTS
 from archivebox.misc.util import enforce_types, docstring
 from archivebox.misc.util import parse_filesize_to_bytes
 
@@ -23,21 +24,28 @@ if TYPE_CHECKING:
     from archivebox.core.models import Snapshot
 
 
-def _collect_input_urls(args: tuple[str, ...]) -> list[str]:
+def _collect_input_urls(args: tuple[str, ...], *, parser: str = "auto") -> list[str]:
     from archivebox.misc.jsonl import read_args_or_stdin
+    from archivebox.misc.util import validate_url
 
     urls: list[str] = []
     for record in read_args_or_stdin(args):
         url = record.get("url")
         if isinstance(url, str) and url:
-            urls.append(url)
+            try:
+                urls.append(validate_url(url))
+            except ValueError as err:
+                raise click.BadParameter(str(err), param_hint="URL") from err
 
         urls_field = record.get("urls")
         if isinstance(urls_field, str):
             for line in urls_field.splitlines():
                 line = line.strip()
                 if line and not line.startswith("#"):
-                    urls.append(line)
+                    try:
+                        urls.append(validate_url(line))
+                    except ValueError as err:
+                        raise click.BadParameter(str(err), param_hint="URL") from err
 
     return urls
 
@@ -82,7 +90,6 @@ def add(
     crawl_max_size = parse_filesize_to_bytes(crawl_max_size)
     crawl_timeout = int(crawl_timeout or 0)
     snapshot_max_size = parse_filesize_to_bytes(snapshot_max_size)
-    from archivebox import CONSTANTS
     from archivebox.config.permissions import USER, HOSTNAME
     from archivebox.config.common import get_config
 
@@ -108,7 +115,6 @@ def add(
 
     # import models once django is set up
     from archivebox.crawls.models import Crawl
-    from archivebox.core.models import Snapshot
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.personas.models import Persona
     from archivebox.misc.logging_util import printable_filesize
@@ -119,34 +125,12 @@ def add(
     created_by_id = created_by_id or get_or_create_system_user_pk()
     started_at = timezone.now()
 
-    if isinstance(urls, str):
-        url_list = [line.strip() for line in urls.splitlines() if line.strip()]
-    else:
-        url_list = [str(url).strip() for url in urls if str(url).strip()]
-    if snapshot_ids and len(snapshot_ids) != len(url_list):
-        raise ValueError("snapshot_ids length must match urls length")
-    admitted_urls: list[str] = []
-    admitted_snapshot_ids: list[str] | None = [] if snapshot_ids else None
-    for index, url in enumerate(url_list):
-        if Snapshot.is_archivebox_internal_url(url):
-            print(f"[yellow][!] Skipping internal ArchiveBox URL: {url}[/yellow]")
-            continue
-        admitted_urls.append(url)
-        if admitted_snapshot_ids is not None and snapshot_ids:
-            admitted_snapshot_ids.append(snapshot_ids[index])
-
-    # 1. Save the provided URLs to sources/2024-11-05__23-59-59__cli_add.txt
-    sources_file = CONSTANTS.SOURCES_DIR / f"{timezone.now().strftime('%Y-%m-%d__%H-%M-%S')}__cli_add.txt"
-    sources_file.parent.mkdir(parents=True, exist_ok=True)
-    if admitted_snapshot_ids is not None:
-        sources_file.write_text(
-            "\n".join(
-                json.dumps({"url": url, "id": admitted_snapshot_ids[index], "tags": tag, "depth": 0})
-                for index, url in enumerate(admitted_urls)
-            ),
-        )
-    else:
-        sources_file.write_text("\n".join(admitted_urls))
+    use_internal_input_root = isinstance(urls, str)
+    source_text = (
+        urls
+        if use_internal_input_root
+        else "\n".join(json.dumps({"type": "CrawlSeed", "url": str(url), "depth": 0}, separators=(",", ":")) for url in urls)
+    )
 
     # 2. Create a new Crawl with inline URLs
     cli_args = [*sys.argv]
@@ -156,10 +140,9 @@ def add(
 
     timestamp = timezone.now().strftime("%Y-%m-%d__%H-%M-%S")
 
-    urls_content = sources_file.read_text()
     persona_name = (persona or "Default").strip() or "Default"
     plugins = plugins or ""
-    persona_obj, _ = Persona.objects.get_or_create(name=persona_name)
+    persona_obj = Persona.get_or_create_named(persona_name)
     persona_obj.ensure_dirs()
     effective_persona_config = get_config(persona=persona_obj)
 
@@ -186,9 +169,18 @@ def add(
     # stripped by Crawl.save() and rederived when hooks run.
     crawl_config.update(config_overrides)
 
+    # Crawl.urls is the user's submitted source, not a derived work queue for
+    # this add path. Keeping it byte-for-byte readable is what lets API/UI/CLI
+    # callers audit or resume imports without losing RSS/Netscape/JSON metadata
+    # that is not representable as one plain URL per line.
     crawl = Crawl.objects.create(
-        urls=urls_content,
-        max_depth=depth,
+        urls=source_text,
+        # Stdin/import text gets an extra hop because the synthetic
+        # archivebox://internal root lives at depth 0 and parser-discovered
+        # URLs land at depth 1; direct URL args become the depth=0 input
+        # snapshots themselves so --depth=N matches the deepest hop the user
+        # asked for.
+        max_depth=depth + 1 if use_internal_input_root else depth,
         tags_str=tag,
         persona_id=persona_obj.id,
         label=f"{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]",
@@ -222,6 +214,9 @@ def add(
         print(
             "[yellow]\\[*] URLs queued. The background runner will process them (run `archivebox server` or `archivebox run --daemon` if not already running).[/yellow]",
         )
+        from archivebox.services.runner import ensure_background_runner
+
+        ensure_background_runner(allow_under_pytest=True)
     else:
         # Foreground mode: run full crawl runner until all work is done
         print("[green]\\[*] Starting crawl runner to process crawl...[/green]")
@@ -336,6 +331,7 @@ def add(
 @click.option("--url-denylist", "--domain-denylist", default="", help="Comma-separated URL/domain denylist for this crawl")
 @click.option("--parser", default="auto", help="Parser for reading input URLs (auto, txt, html, rss, json, jsonl, netscape, ...)")
 @click.option("--plugins", "-p", default="", help="Comma-separated list of plugins to run e.g. title,favicon,screenshot,singlefile,...")
+@click.option("--extract", default="", help="Comma-separated plugins or output types to extract e.g. title,pdf,text/html,image,...")
 @click.option("--persona", default="Default", help="Authentication profile to use when archiving")
 @click.option(
     "--only-new/--no-only-new",
@@ -357,7 +353,12 @@ def main(**kwargs):
 
     with foreground_shutdown_signals(), foreground_parent_watchdog():
         raw_urls = kwargs.pop("urls")
-        urls = _collect_input_urls(raw_urls)
+        if raw_urls:
+            urls = _collect_input_urls(raw_urls, parser=kwargs.get("parser", "auto"))
+        elif not sys.stdin.isatty():
+            urls = sys.stdin.read()
+        else:
+            urls = []
         if not urls:
             raise click.UsageError("No URLs provided. Pass URLs as arguments or via stdin.")
         if int(kwargs.get("max_urls") or 0) < 0:
@@ -380,10 +381,23 @@ def main(**kwargs):
         overwrite = kwargs.pop("overwrite", False)
         update = kwargs.pop("update", False)
         only_new = kwargs.pop("only_new", None)
+        extract = kwargs.pop("extract", "")
         if overwrite or update:
             only_new = False
         if only_new is not None:
             kwargs["config"] = {"ONLY_NEW": bool(only_new)}
+        if extract:
+            from abx_dl.models import discover_plugins, plugins_matching_output
+
+            all_plugins = discover_plugins()
+            tokens = [token.strip() for token in extract.split(",") if token.strip()]
+            plugin_names = {name.lower(): name for name in all_plugins}
+            selected = [plugin_names[token.lower()] for token in tokens if token.lower() in plugin_names]
+            selected += plugins_matching_output(all_plugins, tokens)
+            if not selected:
+                raise click.UsageError(f"No plugins found matching extract types: {extract}")
+            existing = [token.strip() for token in (kwargs.get("plugins") or "").split(",") if token.strip()]
+            kwargs["plugins"] = ",".join(dict.fromkeys([*existing, *selected]))
 
         add(urls=urls, **kwargs)
 

archivebox/cli/archivebox_crawl.py

@@ -65,6 +65,7 @@ def create_crawl(
         1: Failure
     """
     from archivebox.misc.jsonl import read_args_or_stdin, write_record, TYPE_CRAWL
+    from archivebox.misc.util import validate_url
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.crawls.models import Crawl
 
@@ -98,7 +99,11 @@ def create_crawl(
         # Collect URLs
         url = record.get("url")
         if url:
-            url_list.append(url)
+            try:
+                url_list.append(validate_url(str(url)))
+            except ValueError as err:
+                rprint(f"[red]Invalid URL: {err}[/red]", file=sys.stderr)
+                return 1
 
         # Handle 'urls' field (newline-separated)
         urls_field = record.get("urls")
@@ -106,7 +111,11 @@ def create_crawl(
             for line in urls_field.split("\n"):
                 line = line.strip()
                 if line and not line.startswith("#"):
-                    url_list.append(line)
+                    try:
+                        url_list.append(validate_url(line))
+                    except ValueError as err:
+                        rprint(f"[red]Invalid URL: {err}[/red]", file=sys.stderr)
+                        return 1
 
     # Output pass-through records first
     if not is_tty:

archivebox/cli/archivebox_init.py

@@ -153,8 +153,8 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
 
     config = get_config()
     config.TMP_DIR.mkdir(parents=True, exist_ok=True)
-    config.LIB_DIR.mkdir(parents=True, exist_ok=True)
-    (config.LIB_DIR / "bin").mkdir(parents=True, exist_ok=True)
+    config.ABXPKG_LIB_DIR.mkdir(parents=True, exist_ok=True)
+    (config.ABXPKG_LIB_DIR / "bin").mkdir(parents=True, exist_ok=True)
 
     working_tmp_dir = get_or_create_working_tmp_dir(autofix=True, quiet=True)
     if working_tmp_dir:
@@ -163,7 +163,6 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     working_lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True)
     if working_lib_dir:
         working_lib_dir.mkdir(parents=True, exist_ok=True)
-        (working_lib_dir / "bin").mkdir(parents=True, exist_ok=True)
 
     if install:
         from archivebox.cli.archivebox_install import install as install_method
@@ -187,7 +186,7 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
 @click.command()
 @click.option("--force", "-f", is_flag=True, help="Ignore unrecognized files in current directory and initialize anyway")
 @click.option("--quick", "-q", is_flag=True, help="Run any updates or migrations without rechecking all snapshot dirs")
-@click.option("--install", "-s", is_flag=True, help="Automatically install dependencies and extras used for archiving")
+@click.option("--install", "--setup", "-s", is_flag=True, help="Automatically install dependencies and extras used for archiving")
 @docstring(init.__doc__)
 def main(**kwargs) -> None:
     init(**kwargs)

archivebox/cli/archivebox_install.py

@@ -3,6 +3,8 @@
 __package__ = "archivebox.cli"
 
 import os
+import shutil
+from pathlib import Path
 
 import rich_click as click
 from rich import print
@@ -10,6 +12,38 @@ from rich import print
 from archivebox.misc.util import docstring, enforce_types
 
 
+def ensure_data_dir_lib_symlink(data_dir: Path, abxpkg_lib_dir: Path) -> Path | None:
+    """Expose the shared abxpkg lib dir at DATA_DIR/lib after install."""
+    link_path = data_dir / "lib"
+    target_config_path = abxpkg_lib_dir.expanduser().absolute()
+    target_path = abxpkg_lib_dir.expanduser().resolve()
+    from archivebox.config.paths import get_machine_type
+
+    scoped_data_lib_path = (link_path / get_machine_type()).absolute()
+
+    if target_config_path == scoped_data_lib_path:
+        if link_path.is_symlink() or link_path.is_file():
+            link_path.unlink()
+        target_config_path.mkdir(parents=True, exist_ok=True)
+        return None
+
+    if link_path.resolve(strict=False) == target_path:
+        return None
+
+    target_path.mkdir(parents=True, exist_ok=True)
+
+    if link_path.is_symlink():
+        if link_path.resolve(strict=False) == target_path:
+            return link_path
+        link_path.unlink()
+    elif link_path.exists():
+        shutil.rmtree(link_path)
+
+    relative_target = os.path.relpath(target_path, start=link_path.parent)
+    link_path.symlink_to(relative_target, target_is_directory=True)
+    return link_path
+
+
 @enforce_types
 def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bool = False) -> None:
     """Detect and install ArchiveBox dependencies by running the abx-dl install flow
@@ -67,6 +101,11 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo
 
     run_install(plugin_names=plugin_names or None)
 
+    from archivebox.config.common import get_config
+
+    config = get_config(include_machine=False)
+    ensure_data_dir_lib_symlink(CONSTANTS.DATA_DIR, config.ABXPKG_LIB_DIR)
+
     print()
 
     # Check for superuser

archivebox/cli/archivebox_oneshot.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+
+__package__ = "archivebox.cli"
+__command__ = "archivebox oneshot"
+
+import subprocess
+from pathlib import Path
+
+import rich_click as click
+
+from archivebox.config import CONSTANTS
+
+
+@click.command(add_help_option=False, context_settings=dict(ignore_unknown_options=True))
+@click.argument("args", nargs=-1)
+def main(args: tuple[str, ...] = ()) -> None:
+    """Download URLs using abx-dl"""
+    cwd = Path.cwd()
+    if any((path / CONSTANTS.SQL_INDEX_FILENAME).exists() for path in (cwd, *cwd.parents)):
+        raise click.ClickException(
+            "Refusing to run `archivebox oneshot` inside an ArchiveBox DATA_DIR. Use `archivebox add` here, or run oneshot from another directory.",
+        )
+    raise SystemExit(subprocess.run(["abx-dl", *args]).returncode)
+
+
+if __name__ == "__main__":
+    main()

archivebox/cli/archivebox_remove.py

@@ -34,6 +34,7 @@ def remove(
     after: float | None = None,
     before: float | None = None,
     yes: bool = False,
+    delete: bool = False,
     out_dir: Path = CONSTANTS.DATA_DIR,
     timeout: float | None = None,
     **kwargs,
@@ -142,6 +143,7 @@ def remove(
 
 @click.command()
 @click.option("--yes", is_flag=True, help="Remove links instantly without prompting to confirm")
+@click.option("--delete", is_flag=True, help="Compatibility no-op; archived content is deleted automatically")
 @click.option("--timeout", type=float, default=None, help="Maximum seconds to spend deleting snapshots")
 @snapshot_filter_options(default_filter_type="exact")
 @docstring(remove.__doc__)

archivebox/cli/archivebox_run.py

@@ -248,7 +248,12 @@ def process_stdin_records() -> int:
     return 0
 
 
-def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_only: bool = False) -> int:
+def run_runner(
+    daemon: bool = False,
+    crawl_id: str | None = None,
+    maintenance_only: bool = False,
+    maintenance_batch_size: int | None = None,
+) -> int:
     """
     Run the background runner loop.
 
@@ -282,7 +287,7 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on
         current.mark_exited()
         return 0
 
-    recover_orchestrator_state(include_chrome=True)
+    recover_orchestrator_state(include_chrome=crawl_id is None, crawl_id=crawl_id)
     if crawl_id:
         from django.utils import timezone
         from archivebox.crawls.models import Crawl
@@ -319,6 +324,7 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on
                 crawl_id=crawl_id,
                 maintenance_only=maintenance_only,
                 interactive_interrupts=interactive_interrupts,
+                **({"maintenance_batch_size": maintenance_batch_size} if maintenance_batch_size else {}),
             )
         return 0
     except KeyboardInterrupt:
@@ -343,8 +349,23 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on
 @click.option("--snapshot-id", help="Run one snapshot through its crawl")
 @click.option("--binary-id", help="Run one queued binary install directly on the bus")
 @click.option("--maintenance-only", is_flag=True, help="Only process due maintenance ticks on sealed/paused snapshots")
+@click.option(
+    "--maintenance-batch-size",
+    type=int,
+    default=None,
+    hidden=True,
+    help="Limit queued maintenance snapshots claimed per scheduler tick",
+)
 @click.option("--no-stdin", is_flag=True, hidden=True, help="Run the scheduler even when stdin is not a TTY")
-def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintenance_only: bool, no_stdin: bool):
+def main(
+    daemon: bool,
+    crawl_id: str,
+    snapshot_id: str,
+    binary_id: str,
+    maintenance_only: bool,
+    maintenance_batch_size: int | None,
+    no_stdin: bool,
+):
     """
     Process queued work.
 
@@ -367,7 +388,7 @@ def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintena
                 ),
                 foreground_parent_watchdog(enabled=False),
             ):
-                sys.exit(run_runner(daemon=True, maintenance_only=maintenance_only))
+                sys.exit(run_runner(daemon=True, maintenance_only=maintenance_only, maintenance_batch_size=maintenance_batch_size))
         except KeyboardInterrupt:
             sys.exit(0)
 
@@ -391,23 +412,38 @@ def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintena
                 sys.exit(1)
 
         if crawl_id:
-            sys.exit(run_runner(daemon=False, crawl_id=crawl_id, maintenance_only=maintenance_only))
+            sys.exit(
+                run_runner(
+                    daemon=False,
+                    crawl_id=crawl_id,
+                    maintenance_only=maintenance_only,
+                    maintenance_batch_size=maintenance_batch_size,
+                ),
+            )
 
         if maintenance_only:
-            sys.exit(run_runner(daemon=daemon, maintenance_only=True))
+            sys.exit(run_runner(daemon=daemon, maintenance_only=True, maintenance_batch_size=maintenance_batch_size))
 
         if not no_stdin and not sys.stdin.isatty():
             sys.exit(process_stdin_records())
         else:
-            sys.exit(run_runner(daemon=daemon, maintenance_only=maintenance_only))
+            sys.exit(run_runner(daemon=daemon, maintenance_only=maintenance_only, maintenance_batch_size=maintenance_batch_size))
 
 
 def run_snapshot_worker(snapshot_id: str) -> int:
+    from archivebox.config import CONSTANTS
+    from archivebox.core.takeover_util import enter_single_runner_gate
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
+    from archivebox.machine.models import Process
     from archivebox.core.models import Snapshot
     from archivebox.services.runner import run_due_snapshot
     from django.utils import timezone
 
+    current = Process.current()
+    if not enter_single_runner_gate(current, data_dir=CONSTANTS.DATA_DIR):
+        current.mark_exited()
+        return 0
+
     snapshot = None
     try:
         with foreground_shutdown_signals(), foreground_parent_watchdog():
@@ -437,6 +473,10 @@ def run_snapshot_worker(snapshot_id: str) -> int:
 
         traceback.print_exc()
         return 1
+    finally:
+        current.refresh_from_db()
+        if current.status != Process.StatusChoices.EXITED:
+            current.mark_exited()
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_server.py

@@ -231,6 +231,7 @@ def server(
     debug: bool = False,
     daemonize: bool = False,
     nothreading: bool = False,
+    createsuperuser: bool = False,
 ) -> None:
     """Run the ArchiveBox HTTP server"""
     from archivebox.config.common import get_config
@@ -244,6 +245,12 @@ def server(
 
     from django.contrib.auth.models import User
 
+    if createsuperuser:
+        from archivebox.cli.archivebox_manage import manage
+
+        manage(args=["createsuperuser"])
+        print()
+
     if not User.objects.filter(is_superuser=True).exclude(username="system").exists():
         print()
         print(
@@ -404,6 +411,7 @@ def server(
 @click.option("--debug", is_flag=True, help="Enable DEBUG=True mode with more verbose errors")
 @click.option("--nothreading", is_flag=True, help="Force runserver to run in single-threaded mode")
 @click.option("--daemonize", is_flag=True, help="Run the server in the background as a daemon")
+@click.option("--createsuperuser", is_flag=True, help="Run archivebox manage createsuperuser before starting the server")
 @docstring(server.__doc__)
 def main(**kwargs):
     server(**kwargs)

archivebox/cli/archivebox_snapshot.py

@@ -67,7 +67,7 @@ def create_snapshots(
         TYPE_SNAPSHOT,
         TYPE_CRAWL,
     )
-    from archivebox.misc.util import validate_url_length
+    from archivebox.misc.util import validate_url
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.core.models import Snapshot
     from archivebox.crawls.models import Crawl
@@ -112,7 +112,7 @@ def create_snapshots(
                 # Create snapshots for each URL in the crawl
                 for url in crawl.get_urls_list():
                     try:
-                        validate_url_length(url)
+                        validate_url(url)
                     except ValueError as err:
                         rprint(f"[red]Error creating snapshot: {err}[/red]", file=sys.stderr)
                         continue
@@ -135,7 +135,7 @@ def create_snapshots(
             elif record_type == TYPE_SNAPSHOT or record.get("url"):
                 # Input is a Snapshot or plain URL
                 if record.get("url"):
-                    validate_url_length(str(record["url"]))
+                    validate_url(str(record["url"]))
                 if tag and not record.get("tags"):
                     record["tags"] = tag
                 if status:

archivebox/cli/archivebox_update.py

@@ -235,6 +235,14 @@ def update(
 
     command = current_command(Process.TypeChoices.UPDATE, data_dir=CONSTANTS.DATA_DIR)
 
+    def still_owns_foreground_runner() -> bool:
+        from django.db import connections
+
+        try:
+            return command_owns_foreground_runner(command, data_dir=CONSTANTS.DATA_DIR)
+        finally:
+            connections.close_all()
+
     def wait_for_turn() -> None:
         raise_if_shutdown_requested()
         standby_until_foreground_runner_needed(command, data_dir=CONSTANTS.DATA_DIR)
@@ -245,7 +253,11 @@ def update(
             wait_for_turn()
             if ensure_daemon_reason:
                 ensure_daemon_stack(reason=ensure_daemon_reason)
-            exit_code = run_runner_worker(list(args), name=f"worker_runner_update_{os.getpid()}")
+            exit_code = run_runner_worker(
+                list(args),
+                name=f"worker_runner_update_{os.getpid()}",
+                keep_running=still_owns_foreground_runner,
+            )
             if exit_code == 0:
                 return
             if not command_owns_foreground_runner(command, data_dir=CONSTANTS.DATA_DIR):
@@ -374,7 +386,7 @@ def update(
                             for snapshot_id in sorted(touched_snapshot_ids):
                                 run_scoped_runner("--snapshot-id", snapshot_id)
                         else:
-                            run_scoped_runner("--maintenance-only")
+                            run_scoped_runner("--maintenance-only", "--maintenance-batch-size", str(batch_size))
                         ran_post_migrate_runner = True
 
                 if do_index:
@@ -479,14 +491,14 @@ def update(
                         pass
                     elif not runner_work_queued:
                         pass
-                    elif touched_snapshot_ids:
+                    elif touched_snapshot_ids and is_filtered_update:
                         if not touched_snapshot_ids:
                             print("[*] No matching snapshots queued work for the runner.")
                         for snapshot_id in sorted(touched_snapshot_ids):
                             run_scoped_runner("--snapshot-id", snapshot_id)
                     else:
                         run_scoped_runner(
-                            *(["--maintenance-only"] if index_only or migrate_only else []),
+                            *(["--maintenance-only", "--maintenance-batch-size", str(batch_size)] if index_only or migrate_only else []),
                             ensure_daemon_reason="search indexing" if do_index else None,
                         )
 

archivebox/cli/archivebox_version.py

@@ -29,8 +29,7 @@ def _format_binary_abspath(
 
     candidate_bases: tuple[tuple[Path, str], ...] = (
         (pwd, "./"),
-        (lib_dir, "LIB_DIR/"),
-        (Path(os.environ.get("LIB_DIR", "")), "LIB_DIR/") if os.environ.get("LIB_DIR") else (Path(), ""),
+        (lib_dir, "ABXPKG_LIB_DIR/"),
         (personas_dir, "PERSONAS_DIR/"),
         (home, "~/"),
     )
@@ -47,8 +46,8 @@ def _format_binary_abspath(
             relative_str = relative.as_posix()
             if prefix == "./":
                 return "." if not relative_str else f"./{relative_str}"
-            if prefix == "LIB_DIR/":
-                return "LIB_DIR" if not relative_str else f"LIB_DIR/{relative_str}"
+            if prefix == "ABXPKG_LIB_DIR/":
+                return "ABXPKG_LIB_DIR" if not relative_str else f"ABXPKG_LIB_DIR/{relative_str}"
             if prefix == "PERSONAS_DIR/":
                 return "PERSONAS_DIR" if not relative_str else f"PERSONAS_DIR/{relative_str}"
             return "~" if not relative_str else f"~/{relative_str}"
@@ -59,10 +58,10 @@ def _format_binary_abspath(
 def _render_binary_abspath(abspath: str):
     from rich.text import Text
 
-    if abspath.startswith("LIB_DIR/"):
-        return Text.assemble(("LIB_DIR", "bright_blue"), (abspath.removeprefix("LIB_DIR"), "green"))
-    if abspath == "LIB_DIR":
-        return Text("LIB_DIR", style="bright_blue")
+    if abspath.startswith("ABXPKG_LIB_DIR/"):
+        return Text.assemble(("ABXPKG_LIB_DIR", "bright_blue"), (abspath.removeprefix("ABXPKG_LIB_DIR"), "green"))
+    if abspath == "ABXPKG_LIB_DIR":
+        return Text("ABXPKG_LIB_DIR", style="bright_blue")
     if abspath.startswith("PERSONAS_DIR/"):
         return Text.assemble(("PERSONAS_DIR", "medium_purple"), (abspath.removeprefix("PERSONAS_DIR"), "green"))
     if abspath == "PERSONAS_DIR":
@@ -78,6 +77,79 @@ def _render_binary_abspath(abspath: str):
     return Text(abspath, style="green")
 
 
+def _build_binary_table(rows: list[dict[str, object]]):
+    from rich import box
+    from rich.table import Table
+
+    table = Table(title="Binary Dependencies", box=box.SIMPLE_HEAVY, expand=True)
+    table.add_column("Binary", no_wrap=True, max_width=28)
+    table.add_column("Plugin", no_wrap=True, max_width=24)
+    table.add_column("State", no_wrap=True, width=8)
+    table.add_column("Status", justify="center", no_wrap=True, width=6)
+    table.add_column("Version", no_wrap=True, width=16)
+    table.add_column("Provider", no_wrap=True, width=8)
+    table.add_column("Path", overflow="fold", ratio=1)
+    for row in rows:
+        table.add_row(
+            str(row["binary"]),
+            str(row["plugin"]),
+            str(row["state"]),
+            str(row["status"]),
+            str(row["version"]),
+            str(row["provider"]),
+            row["path"],
+            style=str(row.get("style") or ""),
+        )
+    return table
+
+
+def _print_binary_row(prnt, row: dict[str, object]) -> None:
+    prnt(
+        "",
+        str(row["binary"]).ljust(28),
+        row["status"],
+        str(row["plugin"]).ljust(24),
+        str(row["state"]).ljust(8),
+        str(row["version"]).ljust(16),
+        str(row["provider"]).ljust(8),
+        row["path"],
+        overflow="ignore",
+        crop=False,
+        style=str(row.get("style") or ""),
+    )
+
+
+def _binary_record_matches_runtime(installed, lib_dir: Path) -> bool:
+    if not installed or not installed.is_valid or not installed.version:
+        return False
+    try:
+        abspath = Path(installed.abspath).expanduser().resolve(strict=False)
+        if not abspath.exists():
+            return False
+        if installed.binprovider not in {"", "env", "apt", "brew"}:
+            abspath.relative_to(lib_dir)
+    except (OSError, ValueError):
+        return False
+    return True
+
+
+def _binary_row_dedupe_key(
+    *,
+    display_name: str,
+    valid: bool,
+    version: str,
+    provider: str,
+    abspath: str,
+) -> tuple[str, str, str, str]:
+    if not valid:
+        return (display_name, "", "", "")
+    try:
+        resolved_abspath = Path(abspath).expanduser().resolve(strict=False).as_posix()
+    except Exception:
+        resolved_abspath = abspath
+    return (display_name, provider, version, resolved_abspath)
+
+
 @enforce_types
 def version(
     quiet: bool = False,
@@ -94,13 +166,14 @@ def version(
 
     from rich.panel import Panel
     from rich.console import Console
+    from rich.live import Live
 
     from archivebox.config import CONSTANTS
     from archivebox.config.version import get_COMMIT_HASH, get_BUILD_TIME
     from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, RUNNING_AS_UID, RUNNING_AS_GID, IN_DOCKER
     from archivebox.config.paths import get_data_locations, get_code_locations
     from archivebox.misc.logging_util import printable_folder_status
-    from archivebox.config.common import get_config
+    from archivebox.config.common import get_config, normalize_runtime_config
 
     console = Console()
     prnt = console.print
@@ -133,7 +206,7 @@ def version(
     try:
         DATA_DIR_STAT = CONSTANTS.DATA_DIR.stat()
         prnt(
-            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} PUID={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
+            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} ARCHIVEBOX_USER={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
             f"FS_UID={DATA_DIR_STAT.st_uid}:{DATA_DIR_STAT.st_gid}",
             f"FS_PERMS={config.OUTPUT_PERMISSIONS}",
             f"FS_ATOMIC={config.ENFORCE_ATOMIC_WRITES}",
@@ -141,7 +214,7 @@ def version(
         )
     except Exception:
         prnt(
-            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} PUID={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
+            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} ARCHIVEBOX_USER={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
         )
 
     prnt(
@@ -154,7 +227,13 @@ def version(
     )
     prnt()
 
-    if not (os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and os.access(CONSTANTS.CONFIG_FILE, os.R_OK)):
+    in_data_dir = os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and os.access(CONSTANTS.CONFIG_FILE, os.R_OK)
+    if isinstance(binaries, str):
+        requested_names = {name.strip() for name in binaries.split(",") if name.strip()}
+    else:
+        requested_names = {name for name in (binaries or ()) if name}
+
+    if not in_data_dir:
         PANEL_TEXT = "\n".join(
             (
                 "",
@@ -174,93 +253,226 @@ def version(
                 subtitle="Full version info is only available when inside a collection [light_slate_blue]DATA DIR[/light_slate_blue]",
             ),
         )
+    prnt()
+
+    if not in_data_dir and not requested_names:
+        prnt("[pale_green1][i] Binary Dependencies:[/pale_green1]")
+        prnt("", "[grey53]Dependency checks require an initialized collection DATA_DIR.[/grey53]")
+        prnt()
+        prnt("[deep_sky_blue3][i] Code locations:[/deep_sky_blue3]")
+        try:
+            for name, path in get_code_locations().items():
+                if isinstance(name, str) and isinstance(path, dict):
+                    prnt(printable_folder_status(name, path), overflow="ignore", crop=False)
+        except Exception as e:
+            prnt(f"  [red]Error getting code locations: {e}[/red]")
+        prnt()
+        prnt("[red][i] Data locations:[/red] (not in a data directory)")
         prnt()
         return []
 
     prnt("[pale_green1][i] Binary Dependencies:[/pale_green1]")
     failures = []
+    seen_failures: set[str] = set()
+    seen_rows: set[tuple[str, str, str, str]] = set()
 
-    # Setup Django before importing models
+    from archivebox.plugins.discovery import get_enabled_plugins
+    from abx_dl.config import get_required_binary_requests
+    from abx_dl.dependencies import load_binary
+    from abx_dl.models import discover_plugins, filter_plugins
+
+    plugins = discover_plugins(runtime="archivebox")
+    enabled_plugins = filter_plugins(plugins, get_enabled_plugins(config=config), include_providers=True)
+    enabled_plugin_names = set(enabled_plugins)
+    runtime_config = normalize_runtime_config(config.for_crawl(), json_safe=False)
+    derived_config: dict[str, object] = {}
+    db_binaries = {}
+    db_available = False
+    if in_data_dir:
+        try:
+            from archivebox.config.django import setup_django
+
+            setup_django()
+
+            from archivebox.machine.models import Machine, Binary
+
+            machine = Machine.current()
+            derived_config = normalize_runtime_config(machine.config, json_safe=False)
+            for binary in Binary.objects.filter(machine=machine).order_by("name", "-modified_at"):
+                if _binary_record_matches_runtime(binary, config.ABXPKG_LIB_DIR):
+                    db_binaries.setdefault(binary.name, binary)
+            db_available = True
+
+        except Exception as e:
+            prnt()
+            prnt("", f"[yellow]Warning: Could not query collection binary records, falling back to abxpkg state: {e}[/yellow]")
+
+    rows: list[dict[str, object]] = []
+    any_rows = False
+    any_available = False
+    compact_paths = console.is_terminal
+    live_enabled = console.is_terminal
+    live_cm = Live(_build_binary_table(rows), console=console, refresh_per_second=8) if live_enabled else None
+    if not live_enabled:
+        prnt("", "Binary".ljust(28), "Status", "Plugin".ljust(24), "State".ljust(8), "Version".ljust(16), "Provider".ljust(8), "Path")
+
+    def emit_row(row: dict[str, object]) -> None:
+        rows.append(row)
+        if live_cm is not None:
+            live_cm.update(_build_binary_table(rows), refresh=True)
+        else:
+            _print_binary_row(prnt, row)
+
+    if live_cm is not None:
+        live_cm.start()
     try:
-        from archivebox.config.django import setup_django
-
-        setup_django()
-
-        from archivebox.machine.models import Machine, Binary
-
-        machine = Machine.current()
-
-        if isinstance(binaries, str):
-            requested_names = {name.strip() for name in binaries.split(",") if name.strip()}
-        else:
-            requested_names = {name for name in (binaries or ()) if name}
-
-        db_binaries: dict[str, Binary] = {}
-        for binary in Binary.objects.filter(machine=machine).order_by("name", "-modified_at"):
-            db_binaries.setdefault(binary.name, binary)
-
-        all_binary_names = sorted(requested_names or set(db_binaries.keys()))
-
-        if not all_binary_names:
-            prnt("", "[grey53]No binaries detected. Run [green]archivebox install[/green] to detect dependencies.[/grey53]")
-        else:
-            any_available = False
-            compact_paths = console.is_terminal
-            for name in all_binary_names:
-                if requested_names and name not in requested_names:
+        for plugin_name, plugin in plugins.items():
+            plugin_enabled = plugin_name in enabled_plugin_names
+            logical_records = get_required_binary_requests(
+                plugin,
+                plugin.config.required_binaries,
+                overrides=runtime_config,
+                derived_overrides=derived_config,
+                run_output_dir=CONSTANTS.DATA_DIR,
+            )
+            actual_records = get_required_binary_requests(
+                plugin,
+                plugin.config.required_binaries,
+                overrides=runtime_config,
+                derived_overrides=derived_config,
+                run_output_dir=CONSTANTS.DATA_DIR,
+                logical_names=False,
+            )
+            for logical_record, actual_record in zip(logical_records, actual_records, strict=False):
+                logical_name = str(logical_record["name"])
+                actual_name = str(actual_record["name"])
+                display_name = Path(actual_name).expanduser().name if ("/" in actual_name or actual_name.startswith("~")) else logical_name
+                if (
+                    requested_names
+                    and logical_name not in requested_names
+                    and actual_name not in requested_names
+                    and display_name not in requested_names
+                ):
                     continue
 
-                installed = db_binaries.get(name)
-                if installed and installed.is_valid:
-                    display_name = Path(name).expanduser().name if ("/" in name or name.startswith("~")) else name
+                installed = db_binaries.get(logical_name) if db_available else None
+                if _binary_record_matches_runtime(installed, config.ABXPKG_LIB_DIR):
+                    abspath = installed.abspath
+                    version_str = (installed.version or "unknown")[:15]
+                    provider = (installed.binprovider or "env")[:8]
+                    valid = True
+                elif not plugin_enabled and not requested_names:
+                    # `archivebox version` is expected to verify the active runtime, not
+                    # cold-load every optional plugin provider. Migration and status
+                    # checks often run with PLUGINS narrowed to a tiny set; resolving
+                    # disabled plugin binaries there can spend most of the command on
+                    # providers the current collection will never execute.
+                    continue
+                else:
+                    loaded = load_binary(actual_record)
+                    abspath = str(loaded.loaded_abspath or "")
+                    version_str = str(loaded.loaded_version or "unknown")[:15]
+                    provider = (loaded.loaded_binprovider.name if loaded.loaded_binprovider else "env")[:8]
+                    valid = loaded.is_valid
+
+                any_rows = True
+                if valid:
                     display_path = (
                         _format_binary_abspath(
-                            installed.abspath,
+                            abspath,
                             pwd=Path.cwd(),
-                            lib_dir=config.LIB_DIR,
+                            lib_dir=config.ABXPKG_LIB_DIR,
                             personas_dir=CONSTANTS.PERSONAS_DIR,
                             home=Path.home(),
                         )
                         if compact_paths
-                        else installed.abspath
+                        else abspath
                     )
                     rendered_path = _render_binary_abspath(display_path) if compact_paths else display_path
-                    version_str = (installed.version or "unknown")[:15]
-                    provider = (installed.binprovider or "env")[:8]
-                    prnt(
-                        "",
-                        "[green]√[/green]",
-                        "",
-                        display_name.ljust(18),
-                        version_str.ljust(16),
-                        provider.ljust(8),
-                        rendered_path,
-                        overflow="ignore",
-                        crop=False,
-                    )
+                    status = "[green]√[/green]" if plugin_enabled else "[grey53]-[/grey53]"
                     any_available = True
-                    continue
+                else:
+                    rendered_path = "[grey53]not installed[/grey53]"
+                    status = "[red]X[/red]" if plugin_enabled else "[grey53]-[/grey53]"
+                    if plugin_enabled and display_name not in seen_failures:
+                        failures.append(display_name)
+                        seen_failures.add(display_name)
 
-                status = (
-                    "[grey53]not recorded[/grey53]" if name in requested_names and installed is None else "[grey53]not installed[/grey53]"
+                row_key = _binary_row_dedupe_key(
+                    display_name=display_name,
+                    valid=valid,
+                    version=version_str if valid else "-",
+                    provider=provider if valid else "-",
+                    abspath=abspath,
                 )
-                prnt("", "[red]X[/red]", "", name.ljust(18), status, overflow="ignore", crop=False)
-                failures.append(name)
+                if row_key in seen_rows:
+                    continue
+                seen_rows.add(row_key)
 
-            if not any_available:
-                prnt("", "[grey53]No binaries detected. Run [green]archivebox install[/green] to detect dependencies.[/grey53]")
+                emit_row(
+                    {
+                        "plugin": plugin_name,
+                        "state": "enabled" if plugin_enabled else "disabled",
+                        "status": status,
+                        "binary": display_name,
+                        "version": version_str if valid else "-",
+                        "provider": provider if valid else "-",
+                        "path": rendered_path,
+                        "style": "" if plugin_enabled else "dim",
+                    },
+                )
 
-        # Show hint if no binaries are installed yet
-        has_any_installed = Binary.objects.filter(machine=machine).exclude(abspath="").exists()
-        if not has_any_installed:
-            prnt()
-            prnt("", "[grey53]Run [green]archivebox install[/green] to detect and install dependencies.[/grey53]")
+        if db_available:
+            for binary_name, installed in db_binaries.items():
+                if requested_names and binary_name not in requested_names:
+                    continue
+                abspath = installed.abspath
+                version_str = (installed.version or "unknown")[:15]
+                provider = (installed.binprovider or "env")[:8]
+                row_key = _binary_row_dedupe_key(
+                    display_name=binary_name,
+                    valid=True,
+                    version=version_str,
+                    provider=provider,
+                    abspath=abspath,
+                )
+                if row_key in seen_rows:
+                    continue
+                seen_rows.add(row_key)
 
-    except Exception as e:
-        # Handle database errors gracefully (locked, missing, etc.)
-        prnt()
-        prnt("", f"[yellow]Warning: Could not query binaries from database: {e}[/yellow]")
-        prnt("", "[grey53]Run [green]archivebox init[/green] and [green]archivebox install[/green] to set up dependencies.[/grey53]")
+                display_path = (
+                    _format_binary_abspath(
+                        abspath,
+                        pwd=Path.cwd(),
+                        lib_dir=config.ABXPKG_LIB_DIR,
+                        personas_dir=CONSTANTS.PERSONAS_DIR,
+                        home=Path.home(),
+                    )
+                    if compact_paths
+                    else abspath
+                )
+                emit_row(
+                    {
+                        "plugin": "(database)",
+                        "state": "detected",
+                        "status": "[green]√[/green]",
+                        "binary": binary_name,
+                        "version": version_str,
+                        "provider": provider,
+                        "path": _render_binary_abspath(display_path) if compact_paths else display_path,
+                        "style": "dim",
+                    },
+                )
+                any_rows = True
+                any_available = True
+    finally:
+        if live_cm is not None:
+            live_cm.stop()
+
+    if not any_rows:
+        prnt("", "[grey53]No required binaries declared for discovered plugins.[/grey53]")
+    elif not any_available:
+        prnt("", "[grey53]No binaries detected. Run [green]archivebox install[/green] to detect dependencies.[/grey53]")
 
     if not binaries:
         # Show code and data locations
@@ -319,9 +531,7 @@ def version(
 )
 @docstring(version.__doc__)
 def main(**kwargs):
-    failures = version(**kwargs)
-    if failures:
-        raise SystemExit(1)
+    version(**kwargs)
 
 
 if __name__ == "__main__":

archivebox/config/common.py

@@ -176,6 +176,10 @@ def build_crawl_config_snapshot(
     plugin_owned_keys = set(_plugin_config_properties(PLUGIN_CONFIG_SCHEMAS)) - set(ArchiveBoxBaseConfig.model_fields)
     effective = get_config(persona=persona, base_config=base_config)
     frozen = effective.for_crawl_frozen(persona=persona)
+    for key in ("BIND_ADDR", "BASE_URL", "CSRF_TRUSTED_ORIGINS", "SERVER_SECURITY_MODE"):
+        value = getattr(effective, key, None)
+        if value is not None:
+            frozen[key] = value
     if persona is not None:
         persona_config = persona.get_derived_config()
         for key in plugin_owned_keys - explicit_overrides:
@@ -185,6 +189,10 @@ def build_crawl_config_snapshot(
         resolved = get_config(base_config=frozen, overrides=overrides, include_machine=False)
         resolved_payload = normalize_runtime_config(resolved)
         frozen = resolved.for_crawl_frozen(persona=persona)
+        for key in ("BIND_ADDR", "BASE_URL", "CSRF_TRUSTED_ORIGINS", "SERVER_SECURITY_MODE"):
+            value = getattr(resolved, key, None)
+            if value is not None:
+                frozen[key] = value
         for key in plugin_owned_keys & explicit_overrides:
             if ArchiveBoxConfig.scope_for_key(key) == _SCOPE_CRAWL_FROZEN and key in resolved_payload:
                 frozen[key] = resolved_payload[key]
@@ -238,18 +246,14 @@ class StorageConfig(BaseConfigSet):
     _scope: str = PrivateAttr(default=_SCOPE_SERVER)
 
     # TMP_DIR must be a local, fast, readable/writable dir by archivebox user,
-    # must be a short path due to unix path length restrictions for socket files (<100 chars)
+    # must be a short path due to unix path length restrictions for socket files (<90 chars)
     # must be a local SSD/tmpfs for speed and because bind mounts/network mounts/FUSE dont support unix sockets
     TMP_DIR: Path = Field(default=CONSTANTS.DEFAULT_TMP_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
 
-    # LIB_DIR must be a local, fast, readable/writable dir by archivebox user,
+    # ABXPKG_LIB_DIR must be a local, fast, readable/writable dir by archivebox user,
     # must be able to contain executable binaries (up to 5GB size)
     # should not be a remote/network/FUSE mount for speed reasons, otherwise extractors will be slow
-    LIB_DIR: Path = Field(default=CONSTANTS.DEFAULT_LIB_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
-
-    # LIB_BIN_DIR is an optional human-facing symlink convenience directory.
-    # Runtime lookup must use provider-specific paths under LIB_DIR instead.
-    LIB_BIN_DIR: Path = Field(default=CONSTANTS.DEFAULT_LIB_BIN_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
+    ABXPKG_LIB_DIR: Path = Field(default=CONSTANTS.DEFAULT_ABXPKG_LIB_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
 
     OUTPUT_PERMISSIONS: str = Field(default="644")
     ENFORCE_ATOMIC_WRITES: bool = Field(default=True)
@@ -503,6 +507,32 @@ def _plugin_user_config(config: Mapping[str, object]) -> dict[str, str]:
     return {key: _plugin_user_config_value(value) for key, value in config.items()}
 
 
+def _plugin_input_config(config: Mapping[str, object]) -> dict[str, object]:
+    """Return raw config inputs understood by core or plugin schemas."""
+    input_names = _archivebox_config_input_names()
+    return {str(key): value for key, value in config.items() if str(key) in input_names}
+
+
+def _explicit_plugin_enabled_keys(config: Mapping[str, object]) -> set[str]:
+    """Return canonical ``*_ENABLED`` keys explicitly provided by the user.
+
+    ``PLUGINS`` is a selector, not an override for concrete plugin enable flags.
+    If a user passes ``PLUGINS=wget`` and ``SAVE_WGET=False`` together, the
+    alias-resolved ``WGET_ENABLED=False`` must survive the model validator that
+    expands ``PLUGINS`` into concrete booleans.
+    """
+    provided = {str(key) for key in config}
+    explicit: set[str] = set()
+    for key, prop in _plugin_config_properties(PLUGIN_CONFIG_SCHEMAS).items():
+        if not key.endswith("_ENABLED") or not isinstance(prop, Mapping):
+            continue
+        aliases = prop.get("x-aliases") if isinstance(prop, Mapping) else None
+        input_names = {key, *(str(alias) for alias in aliases if str(alias).strip())} if isinstance(aliases, list) else {key}
+        if provided & input_names:
+            explicit.add(key)
+    return explicit
+
+
 def _discover_plugin_config_schemas() -> PluginSchemaDocuments:
     from archivebox.plugins.discovery import discover_plugin_configs
 
@@ -532,6 +562,10 @@ def _archivebox_config_input_names() -> set[str]:
     for field in ArchiveBoxConfig.model_fields.values():
         if isinstance(field.alias, str):
             names.add(field.alias)
+    for prop in _plugin_config_properties(PLUGIN_CONFIG_SCHEMAS).values():
+        aliases = prop.get("x-aliases") if isinstance(prop, Mapping) else None
+        if isinstance(aliases, list):
+            names.update(str(alias) for alias in aliases if str(alias).strip())
     return names
 
 
@@ -643,11 +677,10 @@ class ArchiveBoxBaseConfig(
         runtime_derived_keys = {
             "ABX_INSTALL_CACHE",
             "ACTIVE_PERSONA",
-            "CHROME_DOWNLOADS_DIR",
-            "CHROME_USER_DATA_DIR",
             "CRAWL_DIR",
             "DEFAULT_PERSONA",
             "EXTRA_CONTEXT",
+            "PERSONAS_DIR",
             "SNAP_DIR",
         }
         return frozenset(
@@ -657,7 +690,7 @@ class ArchiveBoxBaseConfig(
     def _scoped_config(self, *, include_execution: bool) -> dict[str, Any]:
         keys = type(self)._crawl_runtime_keys() if include_execution else type(self)._crawl_frozen_keys()
         payload = self.model_dump(mode="json")
-        return {key: payload[key] for key in keys if payload.get(key) is not None}
+        return {key: payload[key] for key in keys if payload.get(key, None) is not None}
 
     def for_crawl(self) -> dict[str, Any]:
         """Config scoped to crawl execution, without runtime object overlays."""
@@ -693,9 +726,9 @@ class ArchiveBoxBaseConfig(
         config["DATA_DIR"] = str(CONSTANTS.DATA_DIR)
         scope_by_key = type(self)._scope_by_key()
         model_fields = type(self).model_fields
-        # ArchiveBox owns SEARCH_BACKEND_ENGINE and uses it during model
-        # validation to derive the selected backend's *_ENABLED flag. Hooks
-        # only receive the backend-local flags, never the selector itself.
+        # ArchiveBox-only selectors are consumed during model validation to
+        # derive hook-owned *_ENABLED flags. They are stripped below after
+        # crawl/snapshot overlays have had a chance to apply.
         config.pop("SEARCH_BACKEND_ENGINE", None)
         if persona is not None:
             for key, value in persona.get_derived_config().items():
@@ -717,6 +750,11 @@ class ArchiveBoxBaseConfig(
         if runtime_overrides:
             config.update(normalize_runtime_config(runtime_overrides, json_safe=False))
 
+        # Hooks should only see concrete plugin-local flags, never the
+        # ArchiveBox selectors used to derive them.
+        config.pop("PLUGINS", None)
+        config.pop("SEARCH_BACKEND_ENGINE", None)
+
         if extra_context:
             context: dict[str, Any] = {}
             if config.get("EXTRA_CONTEXT"):
@@ -731,17 +769,10 @@ class ArchiveBoxBaseConfig(
 
     @model_validator(mode="after")
     def resolve_runtime_paths(self):
-        lib_dir = self.LIB_DIR.expanduser()
+        lib_dir = self.ABXPKG_LIB_DIR.expanduser()
         if not lib_dir.is_absolute():
             lib_dir = CONSTANTS.DATA_DIR / lib_dir
-        self.LIB_DIR = lib_dir.resolve()
-
-        lib_bin_dir = self.LIB_BIN_DIR.expanduser()
-        if lib_bin_dir == CONSTANTS.DEFAULT_LIB_BIN_DIR and self.LIB_DIR != CONSTANTS.DEFAULT_LIB_DIR:
-            lib_bin_dir = self.LIB_DIR / "bin"
-        elif not lib_bin_dir.is_absolute():
-            lib_bin_dir = CONSTANTS.DATA_DIR / lib_bin_dir
-        self.LIB_BIN_DIR = lib_bin_dir.resolve()
+        self.ABXPKG_LIB_DIR = lib_dir.resolve()
 
         return self
 
@@ -1000,10 +1031,13 @@ def get_config(
         persona = crawl.resolve_persona()
 
     config_data: ConfigPayload = dict(defaults or {})
+    config_data["PERSONAS_DIR"] = str(CONSTANTS.PERSONAS_DIR)
     base_config_payload: ConfigPayload = {}
+    base_config_model = ArchiveBoxConfig()
+
     if crawl_config_base:
         config_data.update(
-            normalize_runtime_config(ArchiveBoxConfig().model_dump(mode="json"), exclude_runtime_derived=True, json_safe=False),
+            normalize_runtime_config(base_config_model.model_dump(mode="json"), exclude_runtime_derived=True, json_safe=False),
         )
         config_data.update(normalize_runtime_config(dict(crawl.config or {}), exclude_crawl_execution=True, json_safe=False))
     elif base_config is not None:
@@ -1014,7 +1048,7 @@ def get_config(
         config_data.update(normalize_runtime_config(base_config_payload, exclude_runtime_derived=True, json_safe=False))
     else:
         config_data.update(
-            normalize_runtime_config(ArchiveBoxConfig().model_dump(mode="json"), exclude_runtime_derived=True, json_safe=False),
+            normalize_runtime_config(base_config_model.model_dump(mode="json"), exclude_runtime_derived=True, json_safe=False),
         )
         legacy_permissions = permissions_from_legacy_public_flags({**BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE), **os.environ})
         if legacy_permissions:
@@ -1027,7 +1061,7 @@ def get_config(
 
         scope_overrides.update(
             normalize_runtime_config(
-                _sanitize_machine_config(machine.config, lib_dir=config_data.get("LIB_DIR")),
+                _sanitize_machine_config(machine.config, lib_dir=config_data.get("ABXPKG_LIB_DIR")),
                 only_crawl_execution=crawl_config_base,
                 exclude_runtime_derived=True,
                 json_safe=False,
@@ -1066,22 +1100,23 @@ def get_config(
     archivebox_scope_overrides = {key: value for key, value in scope_overrides.items() if key in _archivebox_config_input_names()}
     config_data.update(archivebox_scope_overrides)
 
+    explicit_plugin_enabled_keys: set[str] = set()
     if resolve_plugins:
         plugin_schemas = {plugin_name: schema for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items() if isinstance(schema, dict)}
         plugin_global_config = {key: str(value) if isinstance(value, Path) else value for key, value in config_data.items()}
+        explicit_plugin_enabled_keys.update(_explicit_plugin_enabled_keys(os.environ))
+        explicit_plugin_enabled_keys.update(_explicit_plugin_enabled_keys(scope_overrides))
         plugin_user_config = _plugin_user_config(
             {
-                **normalize_runtime_config(config_data, only_crawl_execution=True, json_safe=False),
+                **_plugin_input_config(os.environ),
                 **scope_overrides,
             },
         )
         if not crawl_config_base:
+            file_config = BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE)
+            explicit_plugin_enabled_keys.update(_explicit_plugin_enabled_keys(file_config))
             plugin_user_config = {
-                **normalize_runtime_config(
-                    BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE),
-                    exclude_runtime_derived=True,
-                    json_safe=False,
-                ),
+                **_plugin_user_config(_plugin_input_config(file_config)),
                 **plugin_user_config,
             }
         plugin_sections = resolve_plugin_configs(
@@ -1131,14 +1166,15 @@ def get_config(
             )
 
     config = ArchiveBoxConfig.model_validate(config_data)
+    for key in explicit_plugin_enabled_keys:
+        if key in config_data:
+            setattr(config, key, config_data[key])
     if redact_sensitive:
         for key in type(config).model_fields:
             value = config[key]
             if is_sensitive_config_key(key) and value not in (None, ""):
                 setattr(config, key, SENSITIVE_CONFIG_VALUE_REDACTED)
-    os.environ["LIB_DIR"] = str(config.LIB_DIR)
-    os.environ["LIB_BIN_DIR"] = str(config.LIB_BIN_DIR)
-    os.environ["ABXPKG_LIB_DIR"] = str(config.LIB_DIR)
+    os.environ["ABXPKG_LIB_DIR"] = str(config.ABXPKG_LIB_DIR)
     archiving_warning_key = (config.TIMEOUT, config.USE_COLOR)
     if archiving_warning_key not in _WARNED_ARCHIVING_CONFIGS:
         config.warn_if_invalid()

archivebox/config/constants.py

@@ -36,8 +36,8 @@ from .permissions import (
     IN_DOCKER,
     RUNNING_AS_UID,
     RUNNING_AS_GID,
-    DEFAULT_PUID,
-    DEFAULT_PGID,
+    DEFAULT_UID,
+    DEFAULT_GID,
     ARCHIVEBOX_USER,
     ARCHIVEBOX_GROUP,
 )
@@ -66,8 +66,8 @@ class ConstantsDict:
     ARCHIVEBOX_GROUP: int = ARCHIVEBOX_GROUP
     RUNNING_AS_UID: int = RUNNING_AS_UID
     RUNNING_AS_GID: int = RUNNING_AS_GID
-    DEFAULT_PUID: int = DEFAULT_PUID
-    DEFAULT_PGID: int = DEFAULT_PGID
+    DEFAULT_UID: int = DEFAULT_UID
+    DEFAULT_GID: int = DEFAULT_GID
     IS_INSIDE_VENV: bool = sys.prefix != sys.base_prefix
 
     # Source code dirs
@@ -113,12 +113,10 @@ class ConstantsDict:
     TMP_DIR_NAME: str = "tmp"
     DEFAULT_TMP_DIR: Path = DATA_DIR / TMP_DIR_NAME / MACHINE_ID  # ./data/tmp/abc3244323
 
-    LIB_DIR_NAME: str = "lib"
-    DEFAULT_LIB_DIR: Path = _env_path(
-        "LIB_DIR",
-        user_config_path("abx") / LIB_DIR_NAME,
+    DEFAULT_ABXPKG_LIB_DIR: Path = _env_path(
+        "ABXPKG_LIB_DIR",
+        user_config_path("abx") / "lib",
     )
-    DEFAULT_LIB_BIN_DIR: Path = _env_path("LIB_BIN_DIR", DEFAULT_LIB_DIR / "bin")
 
     RESERVED_ARCHIVE_DIR_NAMES: frozenset[str] = frozenset(
         (
@@ -241,7 +239,7 @@ class ConstantsDict:
             SOURCES_DIR_NAME,
             LOGS_DIR_NAME,
             CACHE_DIR_NAME,
-            LIB_DIR_NAME,
+            "lib",
             TMP_DIR_NAME,
             PERSONAS_DIR_NAME,
             CUSTOM_TEMPLATES_DIR_NAME,

archivebox/config/django.py

@@ -41,6 +41,12 @@ def setup_django(check_db=False, in_memory_db=False) -> None:
         # TODO: figure out why CLI entrypoints with init_pending are running this twice sometimes
         return
 
+    # SQLite creates index.sqlite3 during django.setup()/migrate. Apply the
+    # ArchiveBox file-mode policy before any DB connection can create the file,
+    # otherwise a permissive parent umask can expose a just-created DB until a
+    # later chmod runs.
+    os.umask(0o777 - (int(CONFIG.OUTPUT_PERMISSIONS, base=8) | 0o111))
+
     # Third-party patches are only needed once Django/apps are about to load.
     # Keeping them out of archivebox.__init__ avoids paying Django/Daphne setup
     # cost for cheap CLI startup paths like `archivebox <cmd> --help`.

archivebox/config/paths.py

@@ -20,6 +20,8 @@ if TYPE_CHECKING:
 
 PACKAGE_DIR: Path = Path(__file__).resolve().parent.parent  # archivebox source code dir
 DATA_DIR: Path = Path(os.getcwd()).resolve()  # archivebox user data dir
+MAX_TMP_SOCKET_URL_LENGTH = 90
+SUPERVISORD_SOCKET_FILENAME = "supervisord.sock"
 
 
 def _env_path(key: str, default: Path) -> Path:
@@ -120,7 +122,7 @@ def dir_is_writable(dir_path: Path, uid: int | None = None, gid: int | None = No
     try:
         with SudoPermission(uid=uid, fallback=fallback):
             test_file.exists()
-            test_file.write_text(f"Checking if PUID={uid} PGID={gid} can write to dir")
+            test_file.write_text(f"Checking if uid={uid} gid={gid} can write to dir")
             test_file.unlink()
             return True
     except (OSError, PermissionError):
@@ -176,33 +178,45 @@ def create_and_chown_dir(dir_path: Path) -> None:
 
 
 def tmp_dir_socket_path_is_short_enough(dir_path: Path) -> bool:
-    socket_file = dir_path.absolute().resolve() / "supervisord.sock"
-    return len(f"file://{socket_file}") <= 96
+    socket_file = dir_path.absolute().resolve() / SUPERVISORD_SOCKET_FILENAME
+    return len(f"file://{socket_file}") < MAX_TMP_SOCKET_URL_LENGTH
+
+
+def tmp_dir_candidates(config: "ArchiveBoxConfig") -> list[Path]:
+    from archivebox.config.constants import CONSTANTS
+
+    collection_id = get_collection_id()
+    collection_id_short = collection_id[:4]
+    system_tmp_dir = Path(tempfile.gettempdir())
+    candidates = [
+        config.TMP_DIR,  # <user-specified>
+        CONSTANTS.DEFAULT_TMP_DIR,  # ./data/tmp/<machine_id>
+        Path("/var/run/archivebox") / collection_id,
+        Path("/tmp") / "archivebox" / collection_id,
+        Path("~/.tmp/archivebox").expanduser() / collection_id,
+        system_tmp_dir / "archivebox" / collection_id,
+        system_tmp_dir / "archivebox" / collection_id_short,
+        system_tmp_dir / "abx" / collection_id_short,
+    ]
+    seen = set()
+    unique_candidates = []
+    for path in candidates:
+        path_key = str(path.expanduser().absolute())
+        if path_key in seen:
+            continue
+        seen.add(path_key)
+        unique_candidates.append(path)
+    return unique_candidates
 
 
 def get_or_create_working_tmp_dir(autofix=True, quiet=True, config: "ArchiveBoxConfig | None" = None, **config_kwargs):
-    from archivebox.config.constants import CONSTANTS
     from archivebox.config.common import get_config
     from archivebox.misc.checks import check_tmp_dir
 
     config = config or get_config(**config_kwargs)
-    # try a few potential directories in order of preference
-    CANDIDATES = [
-        config.TMP_DIR,  # <user-specified>
-        CONSTANTS.DEFAULT_TMP_DIR,  # ./data/tmp/<machine_id>
-        Path("/var/run/archivebox") / get_collection_id(),  # /var/run/archivebox/abc5d8512
-        Path("/tmp") / "archivebox" / get_collection_id(),  # /tmp/archivebox/abc5d8512
-        Path("~/.tmp/archivebox").expanduser() / get_collection_id(),  # ~/.tmp/archivebox/abc5d8512
-        Path(tempfile.gettempdir())
-        / "archivebox"
-        / get_collection_id(),  # /var/folders/qy/6tpfrpx100j1t4l312nz683m0000gn/T/archivebox/abc5d8512
-        Path(tempfile.gettempdir())
-        / "archivebox"
-        / get_collection_id()[:4],  # /var/folders/qy/6tpfrpx100j1t4l312nz683m0000gn/T/archivebox/abc5d
-        Path(tempfile.gettempdir()) / "abx" / get_collection_id()[:4],  # /var/folders/qy/6tpfrpx100j1t4l312nz683m0000gn/T/abx/abc5
-    ]
+    candidates = tmp_dir_candidates(config)
     fallback_candidate = None
-    for candidate in CANDIDATES:
+    for candidate in candidates:
         try:
             create_and_chown_dir(candidate)
         except Exception:
@@ -231,7 +245,7 @@ def get_or_create_working_tmp_dir(autofix=True, quiet=True, config: "ArchiveBoxC
         return fallback_candidate
 
     if not quiet:
-        raise OSError(f"ArchiveBox is unable to find a writable TMP_DIR, tried {CANDIDATES}!")
+        raise OSError(f"ArchiveBox is unable to find a writable TMP_DIR, tried {candidates}!")
 
 
 def get_or_create_working_lib_dir(autofix=True, quiet=False, config: "ArchiveBoxConfig | None" = None, **config_kwargs):
@@ -240,8 +254,8 @@ def get_or_create_working_lib_dir(autofix=True, quiet=False, config: "ArchiveBox
 
     config = config or get_config(**config_kwargs)
 
-    # LIB_DIR is either the shared platformdirs default or an explicit env/config override.
-    CANDIDATES = [config.LIB_DIR]
+    # ABXPKG_LIB_DIR is either the shared platformdirs default or an explicit env/config override.
+    CANDIDATES = [config.ABXPKG_LIB_DIR]
 
     for candidate in CANDIDATES:
         try:
@@ -249,12 +263,12 @@ def get_or_create_working_lib_dir(autofix=True, quiet=False, config: "ArchiveBox
         except Exception:
             pass
         if check_lib_dir(candidate, throw=False, quiet=True, must_exist=True):
-            if autofix and config.LIB_DIR != candidate:
-                os.environ["LIB_DIR"] = str(candidate)
+            if autofix and config.ABXPKG_LIB_DIR != candidate:
+                os.environ["ABXPKG_LIB_DIR"] = str(candidate)
             return candidate
 
     if not quiet:
-        raise OSError(f"ArchiveBox is unable to find a writable LIB_DIR, tried {CANDIDATES}!")
+        raise OSError(f"ArchiveBox is unable to find a writable ABXPKG_LIB_DIR, tried {CANDIDATES}!")
 
 
 def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs):
@@ -329,7 +343,10 @@ def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
             "TMP_DIR": {
                 "path": tmp_dir.resolve(),
                 "enabled": True,
-                "is_valid": os.path.isdir(tmp_dir) and os.access(tmp_dir, os.R_OK) and os.access(tmp_dir, os.W_OK),  # read + write
+                "is_valid": os.path.isdir(tmp_dir)
+                and os.access(tmp_dir, os.R_OK)
+                and os.access(tmp_dir, os.W_OK)
+                and tmp_dir_socket_path_is_short_enough(tmp_dir),
             },
             # "CACHE_DIR": {
             #     "path": CACHE_DIR.resolve(),
@@ -347,11 +364,9 @@ def get_code_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
 
     config = config or get_config(**config_kwargs)
     try:
-        lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True, config=config) or config.LIB_DIR
+        lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True, config=config) or config.ABXPKG_LIB_DIR
     except Exception:
-        lib_dir = config.LIB_DIR
-
-    lib_bin_dir = lib_dir / "bin"
+        lib_dir = config.ABXPKG_LIB_DIR
 
     return AttrDict(
         {
@@ -375,17 +390,10 @@ def get_code_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
                 "enabled": os.path.isdir(CONSTANTS.USER_PLUGINS_DIR),
                 "is_valid": os.path.isdir(CONSTANTS.USER_PLUGINS_DIR) and os.access(CONSTANTS.USER_PLUGINS_DIR, os.R_OK),  # read
             },
-            "LIB_DIR": {
+            "ABXPKG_LIB_DIR": {
                 "path": lib_dir.resolve(),
                 "enabled": True,
                 "is_valid": os.path.isdir(lib_dir) and os.access(lib_dir, os.R_OK) and os.access(lib_dir, os.W_OK),  # read + write
             },
-            "LIB_BIN_DIR": {
-                "path": lib_bin_dir.resolve(),
-                "enabled": True,
-                "is_valid": os.path.isdir(lib_bin_dir)
-                and os.access(lib_bin_dir, os.R_OK)
-                and os.access(lib_bin_dir, os.W_OK),  # read + write
-            },
         },
     )

archivebox/config/permissions.py

@@ -24,8 +24,8 @@ except PermissionError:
     DATA_DIR_UID = 0
     DATA_DIR_GID = 0
 
-DEFAULT_PUID = 911
-DEFAULT_PGID = 911
+DEFAULT_UID = 911
+DEFAULT_GID = 911
 RUNNING_AS_UID = os.getuid()
 RUNNING_AS_GID = os.getgid()
 EUID = os.geteuid()
@@ -42,23 +42,20 @@ IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE",
 
 FALLBACK_UID = RUNNING_AS_UID or SUDO_UID
 FALLBACK_GID = RUNNING_AS_GID or SUDO_GID
-if RUNNING_AS_UID == 0:
-    try:
-        # if we are running as root it's really hard to figure out what the correct archivebox user should be
-        # as a last resort instead of setting DATA_DIR ownership to 0:0 (which breaks it for non-root users)
-        # check if 911:911 archivebox user exists on host system, and use it instead of 0
-        if pwd.getpwuid(DEFAULT_PUID).pw_name == "archivebox":
-            FALLBACK_UID = DEFAULT_PUID
-            FALLBACK_GID = DEFAULT_PGID
-    except Exception:
-        pass
+try:
+    ARCHIVEBOX_ACCOUNT = pwd.getpwnam("archivebox")
+except KeyError:
+    ARCHIVEBOX_ACCOUNT = None
 
-
-os.environ.setdefault("PUID", str(DATA_DIR_UID or EUID or RUNNING_AS_UID or FALLBACK_UID))
-os.environ.setdefault("PGID", str(DATA_DIR_GID or EGID or RUNNING_AS_GID or FALLBACK_GID))
-
-ARCHIVEBOX_USER = int(os.environ["PUID"])
-ARCHIVEBOX_GROUP = int(os.environ["PGID"])
+if DATA_DIR_UID != 0:
+    ARCHIVEBOX_USER = DATA_DIR_UID
+    ARCHIVEBOX_GROUP = DATA_DIR_GID
+elif RUNNING_AS_UID == 0 and ARCHIVEBOX_ACCOUNT is not None:
+    ARCHIVEBOX_USER = ARCHIVEBOX_ACCOUNT.pw_uid
+    ARCHIVEBOX_GROUP = ARCHIVEBOX_ACCOUNT.pw_gid
+else:
+    ARCHIVEBOX_USER = EUID or RUNNING_AS_UID or FALLBACK_UID
+    ARCHIVEBOX_GROUP = EGID or RUNNING_AS_GID or FALLBACK_GID
 if not USER:
     try:
         # alternative method 1 to get username
@@ -94,13 +91,12 @@ except Exception:
 
 
 def drop_privileges():
-    """If running as root, drop privileges to the user that owns the data dir (or PUID)"""
+    """If running as root, drop privileges to the data dir owner or archivebox user."""
 
-    # always run archivebox as the user that owns the data dir, never as root
+    # Always run ArchiveBox as the user that owns the data dir, or as the
+    # archivebox service account when the data dir is root-owned.
     if os.getuid() == 0:
-        # drop permissions to the user that owns the data dir / provided PUID
         if os.geteuid() != ARCHIVEBOX_USER and ARCHIVEBOX_USER != 0 and ARCHIVEBOX_USER_EXISTS:
-            # drop our effective UID to the archivebox user's UID
             os.seteuid(ARCHIVEBOX_USER)
 
             # update environment variables so that subprocesses dont try to write to /root

archivebox/config/version.py

@@ -49,10 +49,10 @@ def get_COMMIT_HASH() -> str | None:
 
     if IN_DOCKER:
         try:
-            version_txt = Path("/VERSION.txt").read_text()
-            docker_commit_hashes = re.findall(r"COMMIT_HASH=([0-9a-fA-F]{40})", version_txt)
-            if docker_commit_hashes:
-                return docker_commit_hashes[-1]
+            version_text = Path("/VERSION.txt").read_text()
+            matches = re.findall(r"^COMMIT_HASH=([0-9a-fA-F]{40})$", version_text, re.MULTILINE)
+            if matches:
+                return matches[-1]
         except Exception:
             pass
 
@@ -106,13 +106,10 @@ def get_COMMIT_HASH() -> str | None:
 
 @cache
 def get_BUILD_TIME() -> str:
-    if IN_DOCKER:
-        try:
-            # if we're in the archivebox official docker image, /VERSION.txt will contain the build time
-            docker_build_end_time = Path("/VERSION.txt").read_text().rsplit("BUILD_END_TIME=")[-1].split("\n", 1)[0]
-            return docker_build_end_time
-        except Exception:
-            pass
+    for env_var in ("ARCHIVEBOX_BUILD_TIME", "BUILD_TIME"):
+        build_time = os.environ.get(env_var, "").strip()
+        if build_time:
+            return build_time
 
     src_last_modified_unix_timestamp = (PACKAGE_DIR / "README.md").stat().st_mtime
     return datetime.fromtimestamp(src_last_modified_unix_timestamp).strftime("%Y-%m-%d %H:%M:%S %s")

archivebox/config/views.py

@@ -18,7 +18,6 @@ from archivebox.misc.util import parse_date
 
 from archivebox.machine.models import Binary
 
-LIVE_CONFIG_BASE_URL = "/admin/environment/config/"
 ENVIRONMENT_BINARIES_BASE_URL = "/admin/environment/binaries/"
 INSTALLED_BINARIES_BASE_URL = "/admin/machine/binary/"
 
@@ -32,10 +31,6 @@ def format_parsed_datetime(value: object) -> str:
     return parsed.strftime("%Y-%m-%d %H:%M:%S") if parsed else ""
 
 
-def get_live_config_url(key: str) -> str:
-    return f"{LIVE_CONFIG_BASE_URL}{quote(key)}/"
-
-
 def get_environment_binary_url(name: str) -> str:
     return f"{ENVIRONMENT_BINARIES_BASE_URL}{quote(name)}/"
 

archivebox/core/admin_archiveresults.py

@@ -142,25 +142,31 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
         process = result.process_record
         process_display = "-"
         if process:
+            process_url = html.escape(reverse("admin:machine_process_change", args=[process.id]), quote=True)
+            process_label = html.escape(get_process_link_label(process), quote=True)
             process_display = f'''
-                <a href="{reverse("admin:machine_process_change", args=[process.id])}"
+                <a href="{process_url}"
                    style="color: #2563eb; text-decoration: none; font-family: ui-monospace, monospace; font-size: 12px;"
-                   title="View process">{get_process_link_label(process)}</a>
+                   title="View process">{process_label}</a>
             '''
 
         machine_display = "-"
         if process and process.machine_id:
+            machine_url = html.escape(reverse("admin:machine_machine_change", args=[process.machine_id]), quote=True)
+            machine_label = html.escape(str(process.machine.hostname or ""), quote=True)
             machine_display = f'''
-                <a href="{reverse("admin:machine_machine_change", args=[process.machine_id])}"
+                <a href="{machine_url}"
                    style="color: #2563eb; text-decoration: none; font-size: 12px;"
-                   title="View machine">{process.machine.hostname}</a>
+                   title="View machine">{machine_label}</a>
             '''
 
         # Truncate output for display
-        full_output = result.output_str_for_display() or "-"
-        output_display = full_output[:60]
-        if len(full_output) > 60:
-            output_display += "..."
+        full_output_raw = result.output_str_for_display() or "-"
+        output_display_raw = full_output_raw[:60]
+        if len(full_output_raw) > 60:
+            output_display_raw += "..."
+        full_output = html.escape(full_output_raw)
+        output_display = html.escape(output_display_raw)
 
         display_cmd = build_abx_dl_display_command(result)
         replay_cmd = build_abx_dl_replay_command(result, config=config)
@@ -174,9 +180,13 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
             output_link = build_snapshot_url(snapshot_id, embed_path, config=config)
         else:
             output_link = build_snapshot_url(snapshot_id, "", config=config)
+        output_link_attr = html.escape(output_link, quote=True)
 
         # Get version - try cmd_version field
-        version = result.cmd_version if result.cmd_version else "-"
+        version = html.escape(str(result.cmd_version if result.cmd_version else "-"), quote=True)
+        plugin_text = html.escape(str(result.plugin or ""), quote=True)
+        status_text = html.escape(str(status), quote=True)
+        pwd_text = html.escape(str(result.pwd or "-"), quote=True)
 
         # Unique ID for this row's expandable output
         row_id = f"output_{idx}_{str(result.id)[:8]}"
@@ -193,18 +203,18 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
                 <td style="padding: 10px 12px; white-space: nowrap;">
                     <span style="display: inline-block; padding: 3px 10px; border-radius: 12px;
                                  font-size: 11px; font-weight: 600; text-transform: uppercase;
-                                 color: {color}; background: {bg};">{status}</span>
+                                 color: {color}; background: {bg};">{status_text}</span>
                 </td>
-                <td style="padding: 10px 12px; white-space: nowrap; font-size: 20px;" title="{result.plugin}">
+                <td style="padding: 10px 12px; white-space: nowrap; font-size: 20px;" title="{plugin_text}">
                     {icon}
                 </td>
                 <td style="padding: 10px 12px; font-weight: 500; color: #334155;">
-                        <a href="{output_link}" target="_blank"
+                        <a href="{output_link_attr}" target="_blank"
                            style="color: #334155; text-decoration: none;"
                        title="View output fullscreen"
                        onmouseover="this.style.color='#2563eb'; this.style.textDecoration='underline';"
                        onmouseout="this.style.color='#334155'; this.style.textDecoration='none';">
-                        {result.plugin}
+                        {plugin_text}
                     </a>
                 </td>
                 <td style="padding: 10px 12px; max-width: 280px;">
@@ -231,7 +241,7 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
                 </td>
                 <td style="padding: 10px 8px; white-space: nowrap;">
                     <div style="display: flex; gap: 4px;">
-                        <a href="{output_link}" target="_blank"
+                        <a href="{output_link_attr}" target="_blank"
                            style="padding: 4px 8px; background: #f1f5f9; border-radius: 4px; color: #475569; text-decoration: none; font-size: 11px;"
                            title="View output">📄</a>
                         <a href="{reverse("admin:core_archiveresult_change", args=[result.id])}"
@@ -250,7 +260,7 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
                             <div style="font-size: 11px; color: #64748b; margin-bottom: 8px;">
                                 <span style="margin-right: 16px;"><b>ID:</b> <code>{str(result.id)}</code></span>
                                 <span style="margin-right: 16px;"><b>Version:</b> <code>{version}</code></span>
-                                <span style="margin-right: 16px;"><b>PWD:</b> <code>{result.pwd or "-"}</code></span>
+                                <span style="margin-right: 16px;"><b>PWD:</b> <code>{pwd_text}</code></span>
                             </div>
                             <div style="font-size: 11px; color: #64748b; margin-bottom: 8px;">
                                 <b>Output:</b>

archivebox/core/admin_site.py

@@ -3,7 +3,15 @@ __package__ = "archivebox.core"
 from typing import TYPE_CHECKING, Any
 
 from django.contrib import admin
+from django.contrib.auth import REDIRECT_FIELD_NAME
+from django.contrib.auth.decorators import login_not_required
+from django.contrib.auth.views import LoginView
 from django.db import DatabaseError, connection
+from django.http import HttpResponseRedirect
+from django.urls import reverse
+from django.utils.decorators import method_decorator
+from django.utils.translation import gettext as _
+from django.views.decorators.cache import never_cache
 from admin_data_views.admin import (
     admin_data_index_view as adv_admin_data_index_view,
     get_admin_data_urls as adv_get_admin_data_urls,
@@ -12,6 +20,7 @@ from admin_data_views.admin import (
 
 from archivebox.config import VERSION
 from archivebox.config.version import get_COMMIT_HASH
+from archivebox.core.routes_util import is_allowed_archivebox_redirect_url
 
 if TYPE_CHECKING:
     from django.http import HttpRequest
@@ -21,6 +30,17 @@ if TYPE_CHECKING:
     from admin_data_views.typing import AppDict
 
 
+class ArchiveBoxLoginView(LoginView):
+    def get_redirect_url(self) -> str:
+        redirect_to = self.request.POST.get(
+            self.redirect_field_name,
+            self.request.GET.get(self.redirect_field_name),
+        )
+        if is_allowed_archivebox_redirect_url(redirect_to, request=self.request):
+            return redirect_to
+        return ""
+
+
 class ArchiveBoxAdmin(admin.AdminSite):
     site_header = "ArchiveBox"
     index_title = "Admin Views"
@@ -72,6 +92,34 @@ class ArchiveBoxAdmin(admin.AdminSite):
     def admin_data_index_view(self, request: "HttpRequest", **kwargs: Any) -> "TemplateResponse":
         return adv_admin_data_index_view(self, request, **kwargs)
 
+    @method_decorator(never_cache)
+    @login_not_required
+    def login(self, request: "HttpRequest", extra_context: dict[str, Any] | None = None) -> "TemplateResponse":
+        if request.method == "GET" and self.has_permission(request):
+            return HttpResponseRedirect(reverse("admin:index", current_app=self.name))
+
+        from django.contrib.admin.forms import AdminAuthenticationForm
+
+        context = {
+            **self.each_context(request),
+            "title": _("Log in"),
+            "subtitle": None,
+            "app_path": request.get_full_path(),
+            "username": request.user.get_username(),
+        }
+        if REDIRECT_FIELD_NAME not in request.GET and REDIRECT_FIELD_NAME not in request.POST:
+            context[REDIRECT_FIELD_NAME] = reverse("admin:index", current_app=self.name)
+        context.update(extra_context or {})
+
+        index_path = reverse("admin:index", current_app=self.name)
+        request.current_app = self.name
+        return ArchiveBoxLoginView.as_view(
+            extra_context=context,
+            authentication_form=self.login_form or AdminAuthenticationForm,
+            template_name=self.login_template or "admin/login.html",
+            next_page=index_path,
+        )(request)
+
     def index(self, request: "HttpRequest", extra_context: dict[str, Any] | None = None) -> "TemplateResponse":
         response = super().index(request, extra_context)
         if connection.vendor != "sqlite":

archivebox/core/admin_snapshots.py

@@ -30,6 +30,7 @@ from archivebox.plugins.discovery import get_plugin_icon, get_plugin_name, get_p
 from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin
 
 from archivebox.core.models import Tag, Snapshot, ArchiveResult
+from archivebox.crawls.models import Crawl
 from archivebox.core.admin_archiveresults import render_archiveresults_list
 from archivebox.core.preview_util import EXTENSION_SCREENSHOT_PLUGIN
 from archivebox.progressmonitor.views import progress_endpoint
@@ -259,13 +260,15 @@ class SnapshotChangeList(SearchResultsChangeList):
         rows = (
             ArchiveResult.objects.filter(snapshot_id__in=snapshot_ids, status=ArchiveResult.StatusChoices.SUCCEEDED, output_size__gt=0)
             .order_by("snapshot_id", "plugin")
-            .values_list("snapshot_id", "plugin", "status", "output_size")
+            .values_list("snapshot_id", "plugin", "status", "output_size", "output_files")
         )
-        for snapshot_id, plugin, status, output_size in rows.iterator(chunk_size=1000):
+        for snapshot_id, plugin, status, output_size, output_files in rows.iterator(chunk_size=1000):
             if plugin in seen_plugins[snapshot_id]:
                 continue
             seen_plugins[snapshot_id].add(plugin)
-            results_by_snapshot[snapshot_id].append(SimpleNamespace(plugin=plugin, status=status, output_size=output_size))
+            results_by_snapshot[snapshot_id].append(
+                SimpleNamespace(plugin=plugin, status=status, output_size=output_size, output_files=output_files),
+            )
 
         for obj in self.result_list:
             obj.__dict__["_admin_archiveresults"] = results_by_snapshot[obj.pk]
@@ -274,13 +277,7 @@ class SnapshotChangeList(SearchResultsChangeList):
         super().get_results(request)
         if request.GET.get("_embedded") == "crawl":
             self.full_result_count = self.result_count
-        else:
-            self.full_result_count = self.model_admin.get_paginator(
-                request,
-                self.model._default_manager.all().order_by("-id"),
-                self.list_per_page,
-            ).count
-        self.show_full_result_count = True
+            self.show_full_result_count = True
         self._attach_archiveresult_summaries()
 
 
@@ -476,7 +473,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     paginator = AcceleratedPaginator
 
     save_on_top = True
-    show_full_result_count = False
+    show_full_result_count = True
 
     def get_changelist(self, request, **kwargs):
         return SnapshotChangeList
@@ -652,10 +649,22 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
                     ),
                 ),
             )
+        else:
+            prefetches.append(
+                Prefetch(
+                    "crawl",
+                    queryset=Crawl.objects.select_related("created_by").only(
+                        "id",
+                        "created_by_id",
+                        "created_by__id",
+                        "created_by__username",
+                    ),
+                ),
+            )
 
-        qs = super().get_queryset(request).select_related("crawl__created_by")
+        qs = super().get_queryset(request)
         if is_change_view:
-            qs = qs.defer("notes")
+            qs = qs.select_related("crawl__created_by").defer("notes")
         else:
             qs = qs.only(
                 "id",
@@ -669,12 +678,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
                 "fs_version",
                 "output_size",
                 "permissions",
-                "crawl__id",
-                "crawl__persona_id",
-                "crawl__status",
-                "crawl__created_by_id",
-                "crawl__created_by__id",
-                "crawl__created_by__username",
             )
         qs = qs.prefetch_related(*prefetches)
         if needs_files_sort:
@@ -939,38 +942,54 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         config = request.archivebox_config
         results = self._get_prefetched_results(obj)
         if results is not None:
-            has_screenshot = any(r.plugin == "screenshot" for r in results)
-            has_favicon = any(r.plugin == "favicon" for r in results)
-            has_extension_screenshot = any(r.plugin == EXTENSION_SCREENSHOT_PLUGIN for r in results)
+            results = [r for r in results if r.plugin in ("screenshot", EXTENSION_SCREENSHOT_PLUGIN, "favicon")]
         else:
-            available_plugins = set(
-                obj.archiveresult_set.filter(plugin__in=("screenshot", EXTENSION_SCREENSHOT_PLUGIN, "favicon")).values_list(
+            results = list(
+                obj.archiveresult_set.filter(plugin__in=("screenshot", EXTENSION_SCREENSHOT_PLUGIN, "favicon")).only(
+                    "snapshot_id",
                     "plugin",
-                    flat=True,
+                    "status",
+                    "output_files",
+                    "output_str",
                 ),
             )
-            has_screenshot = "screenshot" in available_plugins
-            has_favicon = "favicon" in available_plugins
-            has_extension_screenshot = EXTENSION_SCREENSHOT_PLUGIN in available_plugins
 
-        extension_screenshot_urls = [
-            build_snapshot_url(str(obj.id), f"{EXTENSION_SCREENSHOT_PLUGIN}/screenshot-1.png", request=request, config=config),
-            build_snapshot_url(str(obj.id), f"{EXTENSION_SCREENSHOT_PLUGIN}/screenshot.png", request=request, config=config),
-        ]
+        def result_output_path(result, filename: str) -> str | None:
+            output_files = result.output_files or {}
+            file_info = output_files.get(filename)
+            if not isinstance(file_info, dict) or int(file_info.get("size") or 0) <= 0:
+                return None
+            if file_info.get("root_relative"):
+                return filename
+            return f"{result.plugin}/{filename}"
 
-        if not has_screenshot and not has_extension_screenshot and not has_favicon:
+        def result_urls(plugin: str, filenames: tuple[str, ...]) -> list[str]:
+            urls: list[str] = []
+            for result in results:
+                if result.plugin != plugin or result.status != ArchiveResult.StatusChoices.SUCCEEDED:
+                    continue
+                for filename in filenames:
+                    output_path = result_output_path(result, filename)
+                    if output_path:
+                        urls.append(build_snapshot_url(str(obj.id), output_path, request=request, config=config))
+            return urls
+
+        screenshot_urls = result_urls("screenshot", ("screenshot.png",))
+        extension_screenshot_urls = result_urls(EXTENSION_SCREENSHOT_PLUGIN, ("screenshot-1.png", "screenshot.png"))
+        favicon_urls = result_urls("favicon", ("favicon.ico",))
+
+        if not screenshot_urls and not extension_screenshot_urls and not favicon_urls:
             return None
 
-        if has_screenshot or has_extension_screenshot:
-            img_url = build_snapshot_url(str(obj.id), "screenshot/screenshot.png", request=request, config=config)
-            fallbacks = extension_screenshot_urls
+        all_screenshot_urls = [*screenshot_urls, *extension_screenshot_urls]
+        if all_screenshot_urls:
+            img_url = all_screenshot_urls[0]
+            fallbacks = [*all_screenshot_urls[1:], *favicon_urls]
             img_alt = "Screenshot"
             preview_class = "screenshot"
         else:
-            img_url = build_snapshot_url(str(obj.id), "favicon/favicon.ico", request=request, config=config)
-            fallbacks = [
-                build_snapshot_url(str(obj.id), "favicon.ico", request=request, config=config),
-            ]
+            img_url = favicon_urls[0]
+            fallbacks = favicon_urls[1:]
             img_alt = "Favicon"
             preview_class = "favicon"
 
@@ -988,6 +1007,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
             "preview_class": preview_class,
             "onerror_js": onerror_js,
             "fallback_list": fallback_list,
+            "favicon_url": favicon_urls[0] if favicon_urls else "",
+            "favicon_fallback_list": ",".join(favicon_urls[1:]),
         }
 
     @admin.display(description="", empty_value="")
@@ -996,10 +1017,10 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         if not preview:
             return ""
 
-        request = self.request
-        config = request.archivebox_config
-        favicon_url = build_snapshot_url(str(obj.id), "favicon/favicon.ico", request=request, config=config)
-        fallback_list = ",".join([build_snapshot_url(str(obj.id), "favicon.ico", request=request, config=config)])
+        favicon_url = preview.get("favicon_url") or ""
+        if not favicon_url:
+            return ""
+        fallback_list = preview.get("favicon_fallback_list") or ""
         onerror_js = (
             "this.dataset.fallbacks && this.dataset.fallbacks.length ? "
             "(this.src=this.dataset.fallbacks.split(',').shift(), "

archivebox/core/forms.py

@@ -6,11 +6,10 @@ from decimal import Decimal, InvalidOperation, ROUND_CEILING
 from django import forms
 
 from archivebox.misc.util import URL_REGEX, find_all_urls, parse_filesize_to_bytes
-from taggit.utils import edit_string_for_tags, parse_tags
 from archivebox.base_models.admin import KeyValueWidget
 from archivebox.crawls.schedule_util import validate_schedule
 from archivebox.config.common import get_config, parse_delete_after
-from archivebox.core.permissions import PERMISSIONS_CHOICES, PERMISSIONS_PUBLIC, filter_personas_by_permissions, is_admin_user
+from archivebox.core.permissions import PERMISSIONS_CHOICES, PERMISSIONS_PUBLIC, filter_personas_by_permissions
 from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget
 from archivebox.plugins.discovery import get_plugins
 from archivebox.plugins.forms import (
@@ -30,13 +29,74 @@ DEPTH_CHOICES = (
 )
 
 
+def _split_strip(value: str, delimiter: str) -> list[str]:
+    return [part.strip() for part in value.split(delimiter) if part.strip()]
+
+
+def parse_tag_string(value: str | None) -> list[str]:
+    """Parse the legacy tag-editing format used by the admin tag field."""
+    if not value:
+        return []
+
+    if "," not in value and '"' not in value:
+        return sorted(set(_split_strip(value, " ")))
+
+    tags: list[str] = []
+    buffer: list[str] = []
+    deferred_chunks: list[str] = []
+    saw_unquoted_comma = False
+    in_quote = False
+    chars = iter(value)
+
+    try:
+        while True:
+            char = next(chars)
+            if char == '"':
+                if buffer:
+                    deferred_chunks.append("".join(buffer))
+                    buffer = []
+                in_quote = True
+                char = next(chars)
+                while char != '"':
+                    buffer.append(char)
+                    char = next(chars)
+                tag = "".join(buffer).strip()
+                if tag:
+                    tags.append(tag)
+                buffer = []
+                in_quote = False
+            else:
+                if not saw_unquoted_comma and char == ",":
+                    saw_unquoted_comma = True
+                buffer.append(char)
+    except StopIteration:
+        if buffer:
+            if in_quote and "," in buffer:
+                saw_unquoted_comma = True
+            deferred_chunks.append("".join(buffer))
+
+    delimiter = "," if saw_unquoted_comma else " "
+    for chunk in deferred_chunks:
+        tags.extend(_split_strip(chunk, delimiter))
+
+    return sorted(set(tags))
+
+
+def edit_string_for_tag_names(tags) -> str:
+    names = []
+    for tag in tags:
+        name = tag.name
+        names.append(f'"{name}"' if "," in name or " " in name else name)
+    return ", ".join(sorted(names))
+
+
 class AddLinkForm(PluginConfigFormMixin, forms.Form):
     allow_crawl_execution_config_fields = False
 
     # Basic fields
     url = forms.CharField(
         label="URLs",
-        strip=True,
+        strip=False,
         widget=forms.Textarea(
             attrs={
                 "data-url-regex": URL_REGEX.pattern,
@@ -239,7 +299,10 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
 
     def __init__(self, *args, **kwargs):
         self.request = kwargs.pop("request", None)
-        self.can_override_crawl_config = bool(self.request and is_admin_user(self.request))
+        self.can_override_crawl_config = False
+        if self.request is not None:
+            user = self.request.user
+            self.can_override_crawl_config = bool(user.is_authenticated and user.is_active and user.is_superuser)
         super().__init__(*args, **kwargs)
 
         default_persona = Persona.get_or_create_default()
@@ -315,12 +378,9 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
 
     def clean_url(self):
         value = self.cleaned_data.get("url") or ""
-        valid_urls = []
-        for url in find_all_urls(value):
-            valid_urls.append(url)
-        if not valid_urls:
+        if not list(find_all_urls(value)):
             raise forms.ValidationError("Enter at least one valid URL.")
-        return "\n".join(valid_urls)
+        return value
 
     def clean_url_filters(self):
         from archivebox.crawls.models import Crawl
@@ -426,7 +486,7 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
 class TagWidget(forms.TextInput):
     def format_value(self, value):
         if value is not None and not isinstance(value, str):
-            value = edit_string_for_tags(value)
+            value = edit_string_for_tag_names(value)
         return super().format_value(value)
 
 
@@ -435,12 +495,7 @@ class TagField(forms.CharField):
 
     def clean(self, value):
         value = super().clean(value)
-        try:
-            return parse_tags(value)
-        except ValueError:
-            raise forms.ValidationError(
-                "Please provide a comma-separated list of tags.",
-            )
+        return parse_tag_string(value)
 
     def has_changed(self, initial, data):
         # Always return False if the field is disabled since self.bound_data

archivebox/core/middleware.py

@@ -3,6 +3,7 @@ __package__ = "archivebox.core"
 import ipaddress
 import re
 from pathlib import Path
+from django.conf import settings
 from django.utils import timezone
 from django.contrib.auth.middleware import RemoteUserMiddleware
 from django.contrib.auth.models import AnonymousUser
@@ -24,7 +25,6 @@ from archivebox.core.routes_util import (
     get_base_host,
     get_listen_host,
     get_listen_subdomain,
-    get_public_host,
     get_web_host,
     host_matches,
     is_snapshot_subdomain,
@@ -42,7 +42,7 @@ def _admin_login_hint_cookie_domain(config) -> str | None:
     NOTE: this cookie carries only the single bit "user is logged in on
     admin somewhere"; it MUST NOT be confused with the session cookie,
     which stays admin-host-scoped (see core/settings.py
-    SESSION_COOKIE_DOMAIN comment — admin/public is a security boundary).
+    SESSION_COOKIE_DOMAIN comment — admin/web is a security boundary).
 
     Returns the hostname portion of ``get_base_host`` (which respects
     ``BASE_URL`` and falls back to the local-bind mapping). Strips the
@@ -85,6 +85,34 @@ def TimezoneMiddleware(get_response):
     return middleware
 
 
+def AdminCookieIsolationMiddleware(get_response):
+    def middleware(request):
+        response = get_response(request)
+
+        config = request.__dict__.get("archivebox_config")
+        if config is None:
+            config = get_config(resolve_plugins=False)
+            request.archivebox_config = config
+        if not config.USES_SUBDOMAIN_ROUTING:
+            return response
+
+        request_host = (request.get_host() or "").lower()
+        if host_matches(request_host, get_admin_host(config=config)):
+            return response
+
+        if host_matches(request_host, get_web_host(config=config)):
+            for cookie_name in tuple(response.cookies.keys()):
+                if cookie_name != ADMIN_LOGIN_HINT_COOKIE:
+                    response.cookies.pop(cookie_name, None)
+            return response
+
+        response.cookies.pop(settings.SESSION_COOKIE_NAME, None)
+        response.cookies.pop(settings.CSRF_COOKIE_NAME, None)
+        return response
+
+    return middleware
+
+
 def CacheControlMiddleware(get_response):
     snapshot_path_re = re.compile(r"^/[^/]+/\\d{8}/[^/]+/[0-9a-fA-F-]{8,36}/")
     static_cache_key = (get_COMMIT_HASH() or VERSION or "dev").strip()
@@ -176,7 +204,6 @@ def HostRoutingMiddleware(get_response):
         admin_host = get_admin_host(config=config)
         web_host = get_web_host(config=config)
         api_host = get_api_host(config=config)
-        public_host = get_public_host(config=config)
         listen_host = get_listen_host(config=config)
         subdomain = get_listen_subdomain(request_host, config=config)
 
@@ -278,12 +305,6 @@ def HostRoutingMiddleware(get_response):
             request._cached_user = request.user
             return get_response(request)
 
-        if host_matches(request_host, public_host):
-            if request.COOKIES.get(ADMIN_LOGIN_HINT_COOKIE) == "1" and (request.path == "/public" or request.path.startswith("/public/")):
-                target = build_admin_url("/admin/core/snapshot/", request=request)
-                return redirect(target)
-            return get_response(request)
-
         if subdomain:
             view = OriginalDomainHostView.as_view()
             return view(request, domain=subdomain, path=request.path.lstrip("/"))

archivebox/core/models.py

@@ -1,7 +1,7 @@
 __package__ = "archivebox.core"
 
 from typing import TYPE_CHECKING, Optional, Any
-from collections.abc import Iterable, Sequence
+from collections.abc import Iterable, Mapping, Sequence
 import uuid
 from archivebox.uuid_compat import CompactUUIDField, uuid7
 from datetime import datetime, timedelta
@@ -36,9 +36,10 @@ from archivebox.misc.util import (
     to_json,
     ts_to_date_str,
     urlencode,
-    htmlencode,
+    htmldecode,
+    sanitize_html_text,
     urldecode,
-    validate_url_length,
+    validate_url,
 )
 from archivebox.plugins.discovery import (
     get_plugins,
@@ -94,6 +95,12 @@ class Tag(ModelWithUUID):
     def __str__(self):
         return self.name
 
+    def save(self, *args, **kwargs):
+        update_fields = kwargs.get("update_fields")
+        if update_fields is None or "name" in update_fields:
+            self.name = sanitize_html_text(self.name).strip()
+        super().save(*args, **kwargs)
+
     @property
     def slug(self) -> str:
         """ASCII-safe slugified form of the tag name (derived, not stored)."""
@@ -136,7 +143,7 @@ class Tag(ModelWithUUID):
 
         # Auto-attach to snapshot if in overrides
         if overrides and "snapshot" in overrides and tag:
-            overrides["snapshot"].tags.add(tag)
+            overrides["snapshot"].add_tag_ids([tag.pk])
 
         return tag
 
@@ -489,6 +496,8 @@ class SnapshotManager(models.Manager.from_queryset(SnapshotQuerySet)):  # ty: ig
 
 
 class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHealthStats, ModelWithStateMachine):
+    INTERNAL_INPUT_URL = "archivebox://internal"
+
     id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     modified_at = models.DateTimeField(auto_now=True)
@@ -569,6 +578,15 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         @property
         def sm(self) -> "SnapshotMachine": ...
 
+    def add_tag_ids(self, tag_ids: Iterable[int | str]) -> None:
+        tag_ids = [tag_id for tag_id in dict.fromkeys(tag_ids) if tag_id]
+        if not tag_ids:
+            return
+        SnapshotTag.objects.bulk_create(
+            [SnapshotTag(snapshot_id=self.pk, tag_id=tag_id) for tag_id in tag_ids],
+            ignore_conflicts=True,
+        )
+
     class Meta(
         ModelWithDeleteAfter.Meta,
         ModelWithOutputDir.Meta,
@@ -782,7 +800,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         )
 
     @classmethod
-    def is_archivebox_internal_url(cls, url: str) -> bool:
+    def is_archivebox_internal_url(cls, url: str, *, config: Mapping[str, Any] | Any | None = None) -> bool:
         parsed = urlparse((url or "").strip())
         if parsed.scheme not in ("http", "https") or not parsed.hostname:
             return False
@@ -792,15 +810,29 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             get_api_host,
             get_base_host,
             get_listen_host,
-            get_public_host,
             get_web_host,
             split_host_port,
         )
 
-        config = get_config()
+        if config is None:
+            config = get_config()
+        elif isinstance(config, Mapping):
+            route_config = config
+
+            class RouteConfig:
+                BIND_ADDR = str(route_config.get("BIND_ADDR") or "")
+                BASE_URL = str(route_config.get("BASE_URL") or "")
+                CSRF_TRUSTED_ORIGINS = str(route_config.get("CSRF_TRUSTED_ORIGINS") or "")
+                SERVER_SECURITY_MODE = str(route_config.get("SERVER_SECURITY_MODE") or "")
+
+                @property
+                def USES_SUBDOMAIN_ROUTING(self) -> bool:
+                    return self.SERVER_SECURITY_MODE == "safe-subdomains-fullreplay"
+
+            config = RouteConfig()
         host = parsed.hostname.lower().strip(".")
         port = str(parsed.port) if parsed.port else None
-        protected_subdomains = {"admin", "web", "api", "public"}
+        protected_subdomains = {"admin", "web", "api"}
         protected_hosts: set[tuple[str, str | None]] = set()
         protected_roots: set[tuple[str, str | None]] = set()
         for host_value in (
@@ -809,7 +841,6 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             get_admin_host(config=config),
             get_web_host(config=config),
             get_api_host(config=config),
-            get_public_host(config=config),
         ):
             if not host_value:
                 continue
@@ -865,17 +896,15 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
     def ensure_permissions_config(self, crawl_permissions: str | None = None) -> bool:
         config = dict(self.config or {})
         permission = str(config.get("PERMISSIONS") or "").strip().lower()
-        from archivebox.core.permissions import PERMISSIONS_VALUES, normalize_permissions
+        from archivebox.core.permissions import PERMISSIONS_PUBLIC, PERMISSIONS_VALUES, normalize_permissions
 
         if permission not in PERMISSIONS_VALUES:
             if self.crawl_id:
-                crawl = getattr(self, "crawl", None)
-                crawl_permissions = crawl_permissions or getattr(crawl, "permissions", None)
                 if not crawl_permissions:
                     crawl_permissions = Crawl.objects.filter(pk=self.crawl_id).values_list("permissions", flat=True).first()
             config["PERMISSIONS"] = normalize_permissions(
                 crawl_permissions,
-                default=normalize_permissions(get_config(include_machine=True).PERMISSIONS),
+                default=PERMISSIONS_PUBLIC,
             )
             self.config = config
             return True
@@ -885,21 +914,38 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             return True
         return False
 
+    def validate_url_for_archiving(self, *, config: Mapping[str, Any] | Any | None = None) -> None:
+        if self.is_internal_input_url():
+            return
+
+        try:
+            validate_url(self.url or "")
+        except ValueError as err:
+            raise ValidationError({"url": str(err)}) from err
+
+        if self.is_archivebox_internal_url(self.url, config=config):
+            raise ValidationError({"url": "ArchiveBox cannot archive its own admin, web, api, or snapshot URLs."})
+
+    def is_internal_input_url(self) -> bool:
+        return (self.url or "").strip() == self.INTERNAL_INPUT_URL and self.depth == 0 and bool(self.crawl_id)
+
     def save(self, *args, **kwargs):
         update_fields = kwargs.get("update_fields")
         validate_url_field = self._state.adding or update_fields is None or "url" in update_fields
-        if self.ensure_permissions_config():
+        crawl_config_for_save = None
+        crawl_permissions_for_save = None
+        if self.crawl_id and validate_url_field:
+            crawl_row = Crawl.objects.filter(pk=self.crawl_id).values("config", "permissions").first()
+            if crawl_row:
+                crawl_config_for_save = crawl_row.get("config") or {}
+                crawl_permissions_for_save = crawl_row.get("permissions")
+
+        if self.ensure_permissions_config(crawl_permissions=crawl_permissions_for_save):
             if update_fields is not None:
                 kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "config"]))
 
         if validate_url_field:
-            try:
-                validate_url_length(self.url or "")
-            except ValueError as err:
-                raise ValidationError({"url": str(err)}) from err
-
-            if self.is_archivebox_internal_url(self.url):
-                raise ValidationError({"url": "ArchiveBox cannot archive its own admin, web, api, or snapshot URLs."})
+            self.validate_url_for_archiving(config=crawl_config_for_save if self.crawl_id else None)
 
         if not self.bookmarked_at:
             self.bookmarked_at = self.created_at or timezone.now()
@@ -908,14 +954,17 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
         if self._state.adding or update_fields is None or "title" in update_fields:
             self.title = self._normalize_title_candidate(self.title, snapshot_url=self.url or "") or None
+        if self._state.adding or update_fields is None or "notes" in update_fields:
+            self.notes = sanitize_html_text(self.notes)
 
         # Migrate filesystem if needed (happens automatically on save)
-        if self.pk and self.fs_migration_needed:
+        existing_snapshot = self.pk and not self._state.adding
+        if existing_snapshot and self.fs_migration_needed:
             self.migrate_filesystem_to_current_version()
             update_fields = kwargs.get("update_fields")
             if update_fields is not None:
                 kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "fs_version", "modified_at"]))
-        elif self.pk:
+        elif existing_snapshot:
             current_dir = self.get_storage_path_for_version(self._fs_current_version())
             source_dir = Path(self.output_dir)
             if source_dir.exists() and source_dir != current_dir and not source_dir.is_symlink():
@@ -928,8 +977,29 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         def finish_snapshot_save():
             self.ensure_legacy_archive_symlink()
             self.ensure_crawl_symlink()
-            crawl = self.crawl
-            if not crawl.url_passes_filters(self.url, snapshot=self):
+            crawl = Crawl.objects.filter(pk=self.crawl_id).first()
+            if crawl is None:
+                return
+            crawl_tag_names = crawl.current_tag_names()
+            if crawl_tag_names:
+                # Snapshots can be created by parser hook side-effect records,
+                # direct ORM creates, or legacy crawl URL expansion. Crawl tags
+                # are user-facing metadata on the whole import, so attach them
+                # at the Snapshot.save() boundary instead of relying on every
+                # caller to remember to duplicate this fanout logic.
+                tags_by_name = {tag.name: tag for tag in Tag.objects.filter(name__in=crawl_tag_names)}
+                missing_tags = [Tag(name=name) for name in crawl_tag_names if name not in tags_by_name]
+                if missing_tags:
+                    Tag.objects.bulk_create(missing_tags, ignore_conflicts=True)
+                    tags_by_name = {tag.name: tag for tag in Tag.objects.filter(name__in=crawl_tag_names)}
+                self.add_tag_ids([tag.pk for name in crawl_tag_names if (tag := tags_by_name.get(name))])
+            # Snapshot.save() normally appends newly created URLs to Crawl.urls
+            # so legacy/direct crawls can keep their queue text in sync. For
+            # internal-input crawls that would corrupt the original submitted
+            # import text; parsed URLs are represented by child Snapshot rows.
+            if crawl.has_internal_input_root():
+                return
+            if not crawl.url_passes_filters(self.url, snapshot=self, use_effective_config=False):
                 return
             # Best-effort skip if our URL is already recorded on the crawl;
             # the atomic UPDATE below is what actually prevents clobbering.
@@ -1638,7 +1708,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             with transaction.atomic():
                 for tag_name in new_tags:
                     tag, _ = Tag.objects.get_or_create(name=tag_name)
-                    self.tags.add(tag)
+                    self.add_tag_ids([tag.pk])
 
     def _merge_archive_results_from_index(self, index_data: dict, update_existing: bool = True):
         """Merge ArchiveResults one row per hook; retries update the existing row."""
@@ -2084,7 +2154,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
             # Merge tags
             for tag in dup.tags.all():
-                keeper.tags.add(tag)
+                keeper.add_tag_ids([tag.pk])
 
             # Move ArchiveResults
             ArchiveResult.objects.filter(snapshot=dup).update(
@@ -2257,7 +2327,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
     @staticmethod
     def _normalize_title_candidate(candidate: str | None, *, snapshot_url: str) -> str:
-        title = " ".join(line.strip() for line in str(candidate or "").splitlines() if line.strip()).strip()
+        title = " ".join(line.strip() for line in sanitize_html_text(candidate).splitlines() if line.strip()).strip()
         if not title:
             return ""
         if title.lower() in {"pending...", "no title found", "unable to detect page title"}:
@@ -2532,7 +2602,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
     def save_tags(self, tags: Iterable[str] = ()) -> None:
         tags_id = [Tag.objects.get_or_create(name=tag)[0].pk for tag in tags if tag.strip()]
         self.tags.clear()
-        self.tags.add(*tags_id)
+        self.add_tag_ids(tags_id)
 
     def pending_archiveresults(self) -> QuerySet["ArchiveResult"]:
         return self.archiveresult_set.exclude(status__in=ArchiveResult.FINAL_OR_ACTIVE_STATES)
@@ -2713,15 +2783,18 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
                 )
                 rprint(f"[red]⚠️  Snapshot.from_json auto-created new crawl {crawl.id} for url={url}[/red]", file=sys.stderr)
 
-        # Parse tags (accept either a list ["tag1", "tag2"] or a comma-separated string "tag1,tag2")
+        # Parser hooks emit child Snapshot records through this generic
+        # dispatcher after crawling an internal import root. Those child rows
+        # must inherit crawl-level tags just like direct Crawl.urls snapshots;
+        # otherwise `archivebox add --tag ... < bookmarks.html` loses the tag
+        # unless the parser format also happened to provide its own tags.
         tags_raw = record.get("tags", "")
-        tag_list = []
+        tag_list = list(crawl.current_tag_names()) if crawl else []
         if isinstance(tags_raw, list):
-            tag_list = list(dict.fromkeys(tag.strip() for tag in tags_raw if tag.strip()))
+            tag_list.extend(tag.strip() for tag in tags_raw if tag.strip())
         elif tags_raw:
-            tag_list = list(
-                dict.fromkeys(tag.strip() for tag in re.split(config.TAG_SEPARATOR_PATTERN, tags_raw) if tag.strip()),
-            )
+            tag_list.extend(tag.strip() for tag in re.split(config.TAG_SEPARATOR_PATTERN, tags_raw) if tag.strip())
+        tag_list = list(dict.fromkeys(tag_list))
 
         # Check for existing snapshot with same URL in same crawl
         # (URLs can exist in multiple crawls, but should be unique within a crawl)
@@ -2805,6 +2878,12 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         Creates one ArchiveResult per hook (not per plugin), with hook_name set.
         This enables step-based execution where all hooks in a step can run in parallel.
         """
+        try:
+            self.validate_url_for_archiving()
+        except ValidationError as err:
+            rprint(f"[yellow][!] Skipping blocked snapshot URL: {(self.url or '')[:120]}... ({err})[/yellow]")
+            return []
+
         if hooks is None:
             from archivebox.plugins.hooks import discover_hooks
             from archivebox.config.common import get_config
@@ -3355,8 +3434,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         context = {
             **self.to_dict(extended=True),
             "snapshot": self,
-            "title": htmlencode(self.resolved_title or (self.base_url if is_archived else TITLE_LOADING_MSG)),
-            "url_str": htmlencode(urldecode(self.base_url)),
+            "title": htmldecode(self.resolved_title or (self.base_url if is_archived else TITLE_LOADING_MSG)),
+            "url_str": htmldecode(urldecode(self.base_url)),
             "archive_url": urlencode(f"warc/{self.timestamp}" or (self.domain if is_archived else "")) or "about:blank",
             "extension": self.extension or "html",
             "tags": self.tags_str() or "untagged",

archivebox/core/recovery_util.py

@@ -10,7 +10,7 @@ def _is_signal_interrupted_exit(exit_code: int | None) -> bool:
     return exit_code is not None and (exit_code < 0 or exit_code >= 128)
 
 
-def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int]:
+def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str | None = None) -> dict[str, int]:
     from archivebox.crawls.models import Crawl
     from archivebox.core.models import ArchiveResult, Snapshot
     from archivebox.services.archive_result_service import _collect_output_metadata
@@ -21,10 +21,13 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
 
     now = timezone.now()
     recovery_console = Console(stderr=True, highlight=False, soft_wrap=True)
+    crawl_filter = {"id": crawl_id} if crawl_id else {}
+    snapshot_filter = {"crawl_id": crawl_id} if crawl_id else {}
+    result_filter = {"snapshot__crawl_id": crawl_id} if crawl_id else {}
     cleaned = {
-        "processes_stale_running": Process.cleanup_stale_running(),
-        "processes_orphaned_workers": Process.cleanup_orphaned_workers(),
-        "chrome_processes_orphaned": Process.cleanup_orphaned_chrome() if include_chrome else 0,
+        "processes_stale_running": 0 if crawl_id else Process.cleanup_stale_running(),
+        "processes_orphaned_workers": 0 if crawl_id else Process.cleanup_orphaned_workers(),
+        "chrome_processes_orphaned": Process.cleanup_orphaned_chrome() if include_chrome and not crawl_id else 0,
         "crawls_queued_without_retry_at": 0,
         "snapshots_queued_without_retry_at": 0,
         "archiveresults_backoff": 0,
@@ -58,14 +61,16 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
     cleaned["crawls_queued_without_retry_at"] = Crawl.objects.filter(
         status=Crawl.StatusChoices.QUEUED,
         retry_at__isnull=True,
+        **crawl_filter,
     ).update(retry_at=now, modified_at=now)
     cleaned["snapshots_queued_without_retry_at"] = Snapshot.objects.filter(
         status=Snapshot.StatusChoices.QUEUED,
         retry_at__isnull=True,
         crawl__status__in=Crawl.RUNNABLE_STATES,
+        **snapshot_filter,
     ).update(retry_at=now, modified_at=now)
-    backoff_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.BACKOFF)
-    orphaned_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.STARTED).exclude(
+    backoff_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.BACKOFF, **result_filter)
+    orphaned_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.STARTED, **result_filter).exclude(
         process__status=Process.StatusChoices.RUNNING,
     )
     # ArchiveResult has no retry_at scheduler. Wake only the parent Snapshots
@@ -92,7 +97,7 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
     # only after this runner has won the single-runner gate, so it can safely
     # unlock those stale plugin leases for immediate processing instead of
     # waiting out the previous owner's full lock timeout.
-    queued_plugin_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.QUEUED)
+    queued_plugin_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.QUEUED, **result_filter)
     cleaned["snapshots_queued_plugin_rows_waiting_on_stale_lease"] = (
         Snapshot.objects.filter(
             id__in=queued_plugin_results.values("snapshot_id"),
@@ -113,16 +118,33 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
         process_type=Process.TypeChoices.HOOK,
         archiveresult__isnull=True,
     ).exclude(status=Process.StatusChoices.RUNNING)
+    crawl_snapshot_ids = (
+        {str(snapshot_id) for snapshot_id in Snapshot.objects.filter(crawl_id=crawl_id).values_list("id", flat=True)} if crawl_id else None
+    )
+    if crawl_snapshot_ids is not None:
+        # Targeted `archivebox run --crawl-id ...` is used by foreground add/update
+        # commands and by resume of one existing crawl. It must still repair bad
+        # hook state for that crawl, but it must not scan historical orphaned hook
+        # rows from unrelated crawls before the first snapshot can run.
+        if crawl_snapshot_ids:
+            snapshot_pwd_filter = Q()
+            for snapshot_id in crawl_snapshot_ids:
+                snapshot_pwd_filter |= Q(pwd__contains=snapshot_id)
+            orphaned_hook_processes = orphaned_hook_processes.filter(snapshot_pwd_filter)
+        else:
+            orphaned_hook_processes = orphaned_hook_processes.none()
     for process in orphaned_hook_processes.only("id", "pwd", "cmd", "process_type", "status"):
         hook_script_name = process.hook_script_name
         if not hook_script_name or not process.pwd:
             continue
         plugin_dir = Path(process.pwd)
+        if crawl_snapshot_ids is not None and plugin_dir.parent.name not in crawl_snapshot_ids:
+            continue
         try:
             # Old or synthetic hook Process rows can point at arbitrary paths.
             # Only paths whose parent directory is a valid Snapshot id can be
             # reconstructed into ArchiveResult rows.
-            snapshot = Snapshot.objects.filter(id=plugin_dir.parent.name).first()
+            snapshot = Snapshot.objects.filter(id=plugin_dir.parent.name, **snapshot_filter).first()
         except ValidationError:
             continue
         if snapshot is None:
@@ -181,6 +203,7 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
             Snapshot.objects.filter(id=snapshot.id).update(retry_at=now, modified_at=now)
     started_snapshots = Snapshot.objects.filter(status=Snapshot.StatusChoices.STARTED).filter(
         Q(retry_at__isnull=True) | Q(retry_at__gt=now),
+        **snapshot_filter,
     )
 
     # Broken lock repair: STARTED + retry_at=NULL or retry_at in the future
@@ -205,6 +228,7 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
     # are already final.
     recoverable_started_crawls = Crawl.objects.filter(status=Crawl.StatusChoices.STARTED).filter(
         Q(retry_at__isnull=True) | Q(retry_at__gt=now),
+        **crawl_filter,
     )
 
     due_started_crawls = recoverable_started_crawls.annotate(has_due_child=Exists(due_child_snapshots)).filter(has_due_child=True)

archivebox/core/routes_util.py

@@ -4,12 +4,14 @@ import re
 from typing import Any
 from urllib.parse import urlparse
 
+from django.utils.http import url_has_allowed_host_and_scheme
+
 from archivebox.config.common import get_config
 
 
 _SNAPSHOT_ID_RE = re.compile(r"^[0-9a-fA-F-]{8,36}$")
 _SNAPSHOT_SUBDOMAIN_RE = re.compile(r"^snap-(?P<suffix>[0-9a-fA-F]{12})$")
-_ROLE_SUBDOMAIN_LABELS = ("admin", "web", "api", "public")
+_ROLE_SUBDOMAIN_LABELS = ("admin", "web", "api")
 
 
 def split_host_port(host: str) -> tuple[str, str | None]:
@@ -102,7 +104,7 @@ def _with_port(host: str, port: str | None) -> str:
 
 
 def strip_role_subdomain(host: str) -> str:
-    """Strip leading ``admin.`` / ``web.`` / ``api.`` / ``public.`` / ``snap-*.``
+    """Strip leading ``admin.`` / ``web.`` / ``api.`` / ``snap-*.``
     labels from a host (preserving the port). Strips repeatedly so an
     already-compounded host like ``snap-X.snap-X.<base>`` reduces all the
     way down to ``<base>``.
@@ -172,13 +174,12 @@ def get_base_url(request=None, config: dict[str, Any] | None = None, **config_kw
             return f"{scheme}://{_with_port('archivebox.localhost', req_port)}"
         # C) Per-request fallback: when ``BASE_URL`` is unset and CSRF didn't
         # give us a single origin, trust the request's Host header — but first
-        # peel off any ``admin.`` / ``web.`` / ``api.`` / ``public.`` /
-        # ``snap-*.`` label. Otherwise the URL builders below prepend their own
-        # role label onto a host that already carries one, producing the
-        # ``snap-X.snap-X.snap-X.<base>`` compounding bug. Django has already
-        # admitted the host via ALLOWED_HOSTS; the misconfig banner surfaces
-        # the case where the resulting URL doesn't match what the operator
-        # probably intended.
+        # peel off any ``admin.`` / ``web.`` / ``api.`` / ``snap-*.`` label.
+        # Otherwise the URL builders below prepend their own role label onto a
+        # host that already carries one, producing the ``snap-X.snap-X.snap-X``
+        # compounding bug. Django has already admitted the host via
+        # ALLOWED_HOSTS; the misconfig banner surfaces the case where the
+        # resulting URL doesn't match what the operator probably intended.
         canonical_host = strip_role_subdomain(request.get_host())
         return f"{scheme}://{canonical_host}"
 
@@ -220,13 +221,6 @@ def get_api_host(config: dict[str, Any] | None = None, **config_kwargs: Any) ->
     return _build_base_host("api", config=config)
 
 
-def get_public_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return get_base_host(config=config)
-    return _build_base_host("public", config=config)
-
-
 def get_snapshot_subdomain(snapshot_id: str) -> str:
     normalized = re.sub(r"[^0-9a-fA-F]", "", snapshot_id or "")
     suffix = (normalized[-12:] if len(normalized) >= 12 else normalized).lower()
@@ -292,6 +286,54 @@ def host_matches(request_host: str, target_host: str) -> bool:
     return True
 
 
+def is_allowed_archivebox_redirect_url(url: str | None, request=None, config: dict[str, Any] | None = None) -> bool:
+    """Allow login redirects to this host or to the explicit ArchiveBox BASE_URL family.
+
+    Django already rejects arbitrary absolute ``next=`` URLs for the current host.
+    ArchiveBox also has legitimate cross-subdomain handoffs (admin/web/api/snap-*),
+    but only when BASE_URL is pinned. Without an explicit BASE_URL we do not widen
+    trust based on a request Host header that may be user supplied.
+    """
+    if not url:
+        return False
+
+    require_https = bool(request and request.is_secure())
+    request_host = request.get_host() if request else ""
+    if url_has_allowed_host_and_scheme(url=url, allowed_hosts={request_host}, require_https=require_https):
+        return True
+
+    parsed = urlparse(url)
+    if not parsed.netloc:
+        return False
+    if parsed.scheme not in {"http", "https"}:
+        return False
+    if require_https and parsed.scheme != "https":
+        return False
+    if parsed.username or parsed.password:
+        return False
+
+    config = config or get_config()
+    base_url = _normalize_base_url(config.BASE_URL)
+    if not base_url:
+        return False
+
+    base_host, base_port = split_host_port(urlparse(base_url).netloc)
+    if not base_host:
+        return False
+
+    redirect_host = (parsed.hostname or "").lower()
+    if redirect_host != base_host and not redirect_host.endswith(f".{base_host}"):
+        return False
+
+    try:
+        redirect_port = str(parsed.port) if parsed.port else None
+    except ValueError:
+        return False
+    if base_port:
+        return redirect_port == base_port
+    return redirect_port is None
+
+
 def _scheme_from_request(request=None, config: dict[str, Any] | None = None) -> str:
     config = config or get_config()
     override = _normalize_base_url(config.BASE_URL)
@@ -330,13 +372,6 @@ def get_api_base_url(request=None, config: dict[str, Any] | None = None, **confi
     return _build_base_url_for_host(_build_base_host("api", request=request, config=config), request=request, config=config)
 
 
-def get_public_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return get_base_url(request=request, config=config)
-    return _build_base_url_for_host(_build_base_host("public", request=request, config=config), request=request, config=config)
-
-
 def get_snapshot_base_url(snapshot_id: str, request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
     config = config or get_config(**config_kwargs)
     if not config.USES_SUBDOMAIN_ROUTING:

archivebox/core/settings.py

@@ -13,7 +13,7 @@ import archivebox
 
 from archivebox.config.constants import CONSTANTS
 from archivebox.config.common import get_config
-from archivebox.core.routes_util import normalize_base_url, get_admin_base_url, get_api_base_url
+from archivebox.core.routes_util import get_api_base_url, get_admin_base_url, get_base_url, normalize_base_url
 from .settings_logging import SETTINGS_LOGGING
 
 
@@ -84,6 +84,7 @@ DJANGO_OBJECT_ACTIONS_DEFAULT_HTTP_METHOD = "POST"
 MIDDLEWARE = [
     "archivebox.core.middleware.TimezoneMiddleware",
     "django.middleware.security.SecurityMiddleware",
+    "archivebox.core.middleware.AdminCookieIsolationMiddleware",
     "django.contrib.sessions.middleware.SessionMiddleware",
     "django.middleware.common.CommonMiddleware",
     "archivebox.api.middleware.ApiCorsMiddleware",
@@ -419,13 +420,28 @@ SECURE_BROWSER_XSS_FILTER = True
 SECURE_CONTENT_TYPE_NOSNIFF = True
 SECURE_REFERRER_POLICY = "strict-origin-when-cross-origin"
 
-CSRF_COOKIE_SECURE = False
-SESSION_COOKIE_SECURE = False
+# When BASE_URL is an https:// URL the deployment is HTTPS end-to-end, typically
+# behind a TLS-terminating proxy/tunnel (the bundled traefik/cloudflared profiles,
+# or your own caddy/traefik/nginx) where the proxy -> archivebox hop is plain HTTP, so
+# request.is_secure() / request.scheme would otherwise report http. Honour the
+# proxy's X-Forwarded-Proto so request-derived schemes are correct, and mark the
+# admin session + CSRF cookies Secure so auth cookies are never sent in cleartext.
+# Derived from the RESOLVED base URL's scheme — no separate flag to keep in sync.
+# get_base_url() also covers deployments that only set CSRF_TRUSTED_ORIGINS (the
+# implicit-BASE_URL fallback used on 0.7.x->0.9.x upgrades), so HTTPS hardening
+# isn't lost until BASE_URL is migrated. A plain-http base (e.g. local
+# http://archivebox.localhost:8000) keeps the defaults below.
+BASE_URL_IS_HTTPS = get_base_url(config=CONFIG).strip().lower().startswith("https://")
+if BASE_URL_IS_HTTPS:
+    SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
+
+CSRF_COOKIE_SECURE = BASE_URL_IS_HTTPS
+SESSION_COOKIE_SECURE = BASE_URL_IS_HTTPS
 SESSION_COOKIE_HTTPONLY = True
 SESSION_COOKIE_NAME = f"archivebox_sessionid_{CONSTANTS.COLLECTION_ID}"
 CSRF_COOKIE_NAME = f"archivebox_csrftoken_{CONSTANTS.COLLECTION_ID}"
 # Auth cookies are intentionally scoped to the exact host that set them so
-# the admin session is NOT readable from public.* / web.* / api.* — that
+# the admin session is NOT readable from web.* / api.* — that
 # split is a security boundary, not a UX choice. Subdomains that need to
 # render an "is the user logged in?" indicator must use the single-bit
 # `archivebox_admin_logged_in` hint cookie set by core/middleware.py

archivebox/core/tag_util.py

@@ -12,6 +12,7 @@ from django.http import HttpRequest
 from django.urls import reverse
 
 from archivebox.config.common import get_config
+from archivebox.misc.util import sanitize_html_text
 from archivebox.core.routes_util import build_snapshot_url, build_web_url
 from archivebox.core.models import Snapshot, SnapshotTag, Tag
 
@@ -33,7 +34,7 @@ TAG_HAS_SNAPSHOTS_CHOICES = (
 
 
 def normalize_tag_name(name: str) -> str:
-    return (name or "").strip()
+    return sanitize_html_text(name).strip()
 
 
 def normalize_tag_sort(sort: str = "created_desc") -> str:

archivebox/core/templatetags/core_tags.py

@@ -20,7 +20,6 @@ from archivebox.plugins.discovery import (
 from archivebox.core.routes_util import (
     canonical_base_host_for_request,
     get_admin_base_url,
-    get_public_base_url,
     get_web_base_url,
     get_snapshot_base_url,
     build_snapshot_url,
@@ -498,11 +497,6 @@ def web_base_url(context) -> str:
     return get_web_base_url(request=context.get("request"), config=context.get("CONFIG"))
 
 
-@register.simple_tag(takes_context=True)
-def public_base_url(context) -> str:
-    return get_public_base_url(request=context.get("request"), config=context.get("CONFIG"))
-
-
 @register.simple_tag(takes_context=True)
 def snapshot_base_url(context, snapshot) -> str:
     snapshot_id = _snapshot_id(snapshot)
@@ -515,22 +509,59 @@ def snapshot_url(context, snapshot, path: str = "") -> str:
     return build_snapshot_url(str(snapshot_id), path, request=context.get("request"), config=context.get("CONFIG"))
 
 
+@register.simple_tag(takes_context=True)
+def snapshot_archiveresult_url(context, snapshot, plugin: str, filename: str) -> str:
+    snapshot_id = str(_snapshot_id(snapshot))
+    url_cache = snapshot.__dict__.setdefault("_snapshot_archiveresult_url_cache", {})
+    cache_key = (plugin, filename)
+    if cache_key in url_cache:
+        return url_cache[cache_key]
+
+    results = None
+    if "_admin_archiveresults" in snapshot.__dict__:
+        results = snapshot.__dict__["_admin_archiveresults"]
+    elif "archiveresult_set" in snapshot.__dict__.get("_prefetched_objects_cache", {}):
+        results = snapshot.archiveresult_set.all()
+    elif "_snapshot_archiveresult_url_results" in snapshot.__dict__:
+        results = snapshot.__dict__["_snapshot_archiveresult_url_results"]
+
+    if results is None:
+        from archivebox.core.models import ArchiveResult
+
+        results = list(
+            ArchiveResult.objects.filter(
+                snapshot_id=snapshot_id,
+                plugin__in=("screenshot", "chrome_extension_screenshot", "favicon"),
+                status=ArchiveResult.StatusChoices.SUCCEEDED,
+            ).only(
+                "plugin",
+                "status",
+                "output_files",
+            ),
+        )
+        snapshot.__dict__["_snapshot_archiveresult_url_results"] = results
+
+    for result in results:
+        if result.plugin != plugin or result.status != "succeeded":
+            continue
+        output_files = result.output_files or {}
+        file_info = output_files.get(filename)
+        if not isinstance(file_info, dict) or int(file_info.get("size") or 0) <= 0:
+            continue
+        output_path = filename if file_info.get("root_relative") else f"{plugin}/{filename}"
+        url_cache[cache_key] = build_snapshot_url(snapshot_id, output_path, request=context.get("request"), config=context.get("CONFIG"))
+        return url_cache[cache_key]
+
+    url_cache[cache_key] = ""
+    return ""
+
+
 @register.simple_tag(takes_context=True)
 def snapshot_index_row(context, link) -> str:
     snapshot_id = str(_snapshot_id(link))
     request = context.get("request")
     config = context.get("CONFIG")
     snapshot_base = get_snapshot_base_url(snapshot_id, request=request, config=config)
-    screenshot_plugin_url = build_snapshot_url(snapshot_id, "screenshot/screenshot.png", request=request, config=config)
-    extension_screenshot_1_url = build_snapshot_url(
-        snapshot_id,
-        "chrome_extension_screenshot/screenshot-1.png",
-        request=request,
-        config=config,
-    )
-    extension_screenshot_url = build_snapshot_url(snapshot_id, "chrome_extension_screenshot/screenshot.png", request=request, config=config)
-    favicon_plugin_url = build_snapshot_url(snapshot_id, "favicon/favicon.ico", request=request, config=config)
-    favicon_root_url = build_snapshot_url(snapshot_id, "favicon.ico", request=request, config=config)
 
     status = getattr(link, "status", None) or "unknown"
     bookmarked_at = getattr(link, "bookmarked_at", None)
@@ -571,11 +602,59 @@ def snapshot_index_row(context, link) -> str:
             f'<img src="{escape(static("spinner.gif"))}" alt="" decoding="async" loading="lazy">'
             "</span>"
         )
+    elif "_public_preview_paths" in link.__dict__:
+        preview_paths = list(getattr(link, "_public_preview_paths", []) or [])
+        if preview_paths:
+            preview_urls = [build_snapshot_url(snapshot_id, path, request=request, config=config) for path in preview_paths]
+            preview_html = (
+                f'<img src="{escape(preview_urls[0])}" '
+                f'data-fallbacks="{escape(",".join(preview_urls[1:]))}" '
+                'onerror="nextPublicSnapshotPreview(this)" class="snapshot-preview screenshot" alt="" decoding="async" loading="lazy">'
+            )
+        else:
+            preview_html = '<span class="snapshot-preview snapshot-preview-empty" aria-label="No preview available"></span>'
     else:
-        preview_html = (
-            f'<img src="{escape(screenshot_plugin_url)}" '
-            f'data-fallbacks="{escape(extension_screenshot_1_url)},{escape(extension_screenshot_url)}" '
-            'onerror="nextPublicSnapshotPreview(this)" class="snapshot-preview screenshot" alt="" decoding="async" loading="lazy">'
+        preview_urls = [
+            url
+            for url in (
+                snapshot_archiveresult_url(context, link, "screenshot", "screenshot.png"),
+                snapshot_archiveresult_url(context, link, "chrome_extension_screenshot", "screenshot-1.png"),
+                snapshot_archiveresult_url(context, link, "chrome_extension_screenshot", "screenshot.png"),
+            )
+            if url
+        ]
+        if preview_urls:
+            preview_html = (
+                f'<img src="{escape(preview_urls[0])}" '
+                f'data-fallbacks="{escape(",".join(preview_urls[1:]))}" '
+                'onerror="nextPublicSnapshotPreview(this)" class="snapshot-preview screenshot" alt="" decoding="async" loading="lazy">'
+            )
+        else:
+            preview_html = '<span class="snapshot-preview snapshot-preview-empty" aria-label="No preview available"></span>'
+
+    if "_public_favicon_paths" in link.__dict__:
+        favicon_paths = list(getattr(link, "_public_favicon_paths", []) or [])
+        if favicon_paths:
+            favicon_urls = [build_snapshot_url(snapshot_id, path, request=request, config=config) for path in favicon_paths]
+            favicon_html = (
+                f'<img src="{escape(favicon_urls[0])}" '
+                f'data-fallbacks="{escape(",".join(favicon_urls[1:]))}" '
+                'onerror="nextPublicSnapshotPreview(this)" '
+                'class="link-favicon" alt="" decoding="async" loading="lazy">'
+            )
+        else:
+            favicon_html = '<span class="link-favicon link-favicon-empty" aria-hidden="true"></span>'
+    else:
+        favicon_url = snapshot_archiveresult_url(context, link, "favicon", "favicon.ico")
+        favicon_html = (
+            (
+                f'<img src="{escape(favicon_url)}" '
+                'data-fallbacks="" '
+                'onerror="nextPublicSnapshotPreview(this)" '
+                'class="link-favicon" alt="" decoding="async" loading="lazy">'
+            )
+            if favicon_url
+            else '<span class="link-favicon link-favicon-empty" aria-hidden="true"></span>'
         )
 
     html = f"""
@@ -594,13 +673,7 @@ def snapshot_index_row(context, link) -> str:
     <td class="snapshot-title-cell" title="{escape(title or url)}">
         <div class="snapshot-title-line">
             <a href="{escape(snapshot_base)}/index.html" class="snapshot-favicon-link" title="Open archived snapshot">
-                <img src="{escape(favicon_plugin_url)}"
-                     data-fallbacks="{escape(favicon_root_url)}"
-                     onerror="nextPublicSnapshotPreview(this)"
-                     class="link-favicon"
-                     alt=""
-                     decoding="async"
-                     loading="lazy">
+                {favicon_html}
             </a>
             <a href="{escape(snapshot_base)}/index.html" class="snapshot-title">
                 {escape(Truncator(title_text).chars(110))}

archivebox/core/views.py

@@ -11,7 +11,7 @@ from urllib.parse import quote, urlparse
 
 from django.shortcuts import render, redirect
 from django.http import HttpRequest, HttpResponse, Http404, HttpResponseForbidden, QueryDict
-from django.utils.html import format_html
+from django.utils.html import format_html, format_html_join
 from django.utils.safestring import mark_safe
 from django.views import View
 from django.views.generic.list import ListView
@@ -51,9 +51,11 @@ from archivebox.misc.paginators import AcceleratedPaginator
 from archivebox.misc.util import (
     base_url,
     filter_queryset_by_uuid_substring,
-    htmlencode,
+    htmldecode,
+    sanitize_html_text,
     ts_to_date_str,
     urldecode,
+    validate_url,
     without_fragment,
 )
 from archivebox.misc.serve_static import serve_static_with_byterange_support
@@ -68,7 +70,9 @@ from archivebox.search.views import get_cached_public_search_state
 
 from archivebox.core.models import ArchiveResult, Snapshot, SnapshotTag
 from archivebox.core.permissions import (
+    PERMISSIONS_PRIVATE,
     PERMISSIONS_PUBLIC,
+    PERMISSIONS_UNLISTED,
     can_view_snapshot,
     direct_snapshots_queryset,
     filter_personas_by_permissions,
@@ -89,7 +93,6 @@ from archivebox.core.routes_util import (
 from archivebox.core.forms import AddLinkForm
 from archivebox.plugins.forms import get_plugin_config_binary_urls
 from archivebox.crawls.models import Crawl
-from archivebox.workers.models import RETRY_AT_MAX
 from archivebox.plugins.discovery import discover_plugin_configs
 from archivebox.plugins.views import get_config_definition_link
 from archivebox.progressmonitor.views import live_progress_view, progress_endpoint
@@ -438,7 +441,7 @@ class SnapshotView(View):
             "progress_endpoint": progress_endpoint("snapshot", snapshot.id),
             "url": snapshot.url,
             "archive_path": snapshot.archive_path_from_db,
-            "title": htmlencode(snapshot.resolved_title or (snapshot.base_url if is_archived else TITLE_LOADING_MSG)),
+            "title": htmldecode(snapshot.resolved_title or (snapshot.base_url if is_archived else TITLE_LOADING_MSG)),
             "extension": snapshot.extension or "html",
             "tags": snapshot.tags_str() or "untagged",
             "size": printable_filesize(output_size) if output_size else "—",
@@ -568,37 +571,43 @@ class SnapshotView(View):
                 # Snapshot dir exists but file within does not e.g. 124235.324234/screenshot.png
                 return HttpResponse(
                     format_html(
-                        (
-                            "<html><head>"
-                            "<title>Snapshot Not Found</title>"
-                            #'<script>'
-                            #'setTimeout(() => { window.location.reload(); }, 5000);'
-                            #'</script>'
-                            "</head><body>"
-                            "<center><br/><br/><br/>"
-                            f'Snapshot <a href="/{snapshot.archive_path}/index.html" target="_top"><b><code>[{snapshot.timestamp}]</code></b></a>: <a href="{snapshot.url}" target="_blank" rel="noreferrer">{snapshot.url}</a><br/>'
-                            f"was queued on {str(snapshot.bookmarked_at).split('.')[0]}, "
-                            f'but no files have been saved yet in:<br/><b><a href="/{snapshot.archive_path}/" target="_top"><code>{snapshot.timestamp}</code></a><code>/'
-                            "{}"
-                            f"</code></b><br/><br/>"
-                            "It's possible {} "
-                            f"during the last capture on {str(snapshot.bookmarked_at).split('.')[0]},<br/>or that the archiving process has not completed yet.<br/>"
-                            f"<pre><code># run this cmd to finish/retry archiving this Snapshot</code><br/>"
-                            f'<code style="user-select: all; color: #333">archivebox update -t timestamp {snapshot.timestamp}</code></pre><br/><br/>'
-                            '<div class="text-align: left; width: 100%; max-width: 400px">'
-                            "<i><b>Next steps:</i></b><br/>"
-                            f'- list all the <a href="/{snapshot.archive_path}/" target="_top">Snapshot files <code>.*</code></a><br/>'
-                            f'- view the <a href="/{snapshot.archive_path}/index.html" target="_top">Snapshot <code>./index.html</code></a><br/>'
-                            f'- go to the <a href="/admin/core/snapshot/{snapshot.pk}/change/" target="_top">Snapshot admin</a> to edit<br/>'
-                            f'- go to the <a href="/admin/core/snapshot/?id__exact={snapshot.id}" target="_top">Snapshot actions</a> to re-archive<br/>'
-                            '- or return to <a href="/" target="_top">the main index...</a></div>'
-                            "</center>"
-                            "</body></html>"
-                        ),
+                        """
+                        <html><head>
+                        <title>Snapshot Not Found</title>
+                        </head><body>
+                        <center><br/><br/><br/>
+                        Snapshot <a href="/{}/index.html" target="_top"><b><code>[{}]</code></b></a>: <a href="{}" target="_blank" rel="noreferrer">{}</a><br/>
+                        was queued on {}, but no files have been saved yet in:<br/><b><a href="/{}/" target="_top"><code>{}</code></a><code>/{}</code></b><br/><br/>
+                        It's possible {} during the last capture on {},<br/>or that the archiving process has not completed yet.<br/>
+                        <pre><code># run this cmd to finish/retry archiving this Snapshot</code><br/>
+                        <code style="user-select: all; color: #333">archivebox update -t timestamp {}</code></pre><br/><br/>
+                        <div class="text-align: left; width: 100%; max-width: 400px">
+                        <i><b>Next steps:</i></b><br/>
+                        - list all the <a href="/{}/" target="_top">Snapshot files <code>.*</code></a><br/>
+                        - view the <a href="/{}/index.html" target="_top">Snapshot <code>./index.html</code></a><br/>
+                        - go to the <a href="/admin/core/snapshot/{}/change/" target="_top">Snapshot admin</a> to edit<br/>
+                        - go to the <a href="/admin/core/snapshot/?id__exact={}" target="_top">Snapshot actions</a> to re-archive<br/>
+                        - or return to <a href="/" target="_top">the main index...</a></div>
+                        </center>
+                        </body></html>
+                        """,
+                        snapshot.archive_path,
+                        snapshot.timestamp,
+                        snapshot.url,
+                        snapshot.url,
+                        str(snapshot.bookmarked_at).split(".")[0],
+                        snapshot.archive_path,
+                        snapshot.timestamp,
                         archivefile if str(archivefile) != "None" else "",
                         f"the {archivefile} resource could not be fetched"
                         if str(archivefile) != "None"
                         else "the original site was not available",
+                        str(snapshot.bookmarked_at).split(".")[0],
+                        snapshot.timestamp,
+                        snapshot.archive_path,
+                        snapshot.archive_path,
+                        snapshot.pk,
+                        snapshot.id,
                     ),
                     content_type="text/html",
                     status=404,
@@ -811,6 +820,34 @@ def _safe_archive_relpath(path: str) -> str | None:
     return cleaned
 
 
+def _resolve_archiveresult_relpath(snapshot: Snapshot, rel_path: str) -> tuple[str, bool]:
+    """Resolve plugin-relative output paths through ArchiveResult.output_files."""
+    parts = Path(rel_path).parts
+    if len(parts) < 2:
+        return rel_path, False
+
+    plugin = parts[0]
+    plugin_relpath = posixpath.join(*parts[1:])
+    result = (
+        ArchiveResult.objects.filter(snapshot=snapshot, plugin=plugin, status=ArchiveResult.StatusChoices.SUCCEEDED)
+        .only("output_files")
+        .first()
+    )
+    if not result or not result.output_files:
+        return rel_path, False
+
+    output_files = result.output_files or {}
+    for candidate in (plugin_relpath, rel_path):
+        file_info = output_files.get(candidate)
+        if not isinstance(file_info, dict):
+            continue
+        if file_info.get("root_relative"):
+            return candidate, True
+        return rel_path, True
+
+    return rel_path, True
+
+
 def _coerce_sort_timestamp(value: str | float | None) -> float:
     if value is None:
         return 0.0
@@ -1004,6 +1041,8 @@ def _serve_snapshot_replay(request: HttpRequest, snapshot: Snapshot, path: str =
     if rel_path is None:
         raise Http404
 
+    rel_path, _is_known_plugin_output = _resolve_archiveresult_relpath(snapshot, rel_path)
+
     try:
         return serve_static_with_byterange_support(
             request,
@@ -1128,7 +1167,7 @@ class PublicIndexView(ListView):
     model = Snapshot
     ordering = ["-bookmarked_at", "-created_at"]
     paginator_class = AcceleratedPaginator
-    public_page_scan_chunk_size = 500
+    public_page_scan_chunk_size = 50
 
     def get_paginate_by(self, queryset):
         runtime_config = self.__dict__.get("runtime_config")
@@ -1155,7 +1194,7 @@ class PublicIndexView(ListView):
         target_count = page_number * page_size
         public_snapshots: list[Snapshot] = []
         scanned = 0
-        chunk_size = max(self.public_page_scan_chunk_size, page_size * 4)
+        chunk_size = max(self.public_page_scan_chunk_size, page_size)
         ordered_snapshots = Snapshot.objects.order_by(*self.ordering).only(*self._base_public_snapshot_fields())
 
         while len(public_snapshots) < target_count:
@@ -1217,6 +1256,8 @@ class PublicIndexView(ListView):
         snapshots = list(context.get("object_list") or ())
         icons_by_snapshot: dict[str, set[str]] = {str(snapshot.id): set() for snapshot in snapshots}
         tag_names_by_snapshot: dict[str, list[str]] = {str(snapshot.id): [] for snapshot in snapshots}
+        preview_paths_by_snapshot: dict[str, list[tuple[int, int, str]]] = {str(snapshot.id): [] for snapshot in snapshots}
+        favicon_paths_by_snapshot: dict[str, list[str]] = {str(snapshot.id): [] for snapshot in snapshots}
         progress_by_snapshot: dict[str, dict[str, int]] = {
             str(snapshot.id): {
                 "total": 0,
@@ -1237,20 +1278,51 @@ class PublicIndexView(ListView):
             ):
                 tag_names_by_snapshot[str(snapshot_id)].append(tag_name)
 
-            for snapshot_id, plugin, status in (
+            preview_plugin_order = {
+                "screenshot": 0,
+                "chrome_extension_screenshot": 1,
+            }
+            preview_candidates = {
+                "screenshot": ("screenshot.png",),
+                "chrome_extension_screenshot": ("screenshot-1.png", "screenshot.png"),
+            }
+
+            def result_output_path(result: ArchiveResult, filename: str) -> str | None:
+                output_files = result.output_files or {}
+                file_info = output_files.get(filename)
+                if not isinstance(file_info, dict) or int(file_info.get("size") or 0) <= 0:
+                    return None
+                if file_info.get("root_relative"):
+                    return filename
+                return f"{result.plugin}/{filename}"
+
+            for result in (
                 ArchiveResult.objects.filter(
                     snapshot_id__in=icons_by_snapshot.keys(),
                 )
                 .exclude(plugin="")
-                .values_list("snapshot_id", "plugin", "status")
+                .only("snapshot_id", "plugin", "status", "output_files", "output_str")
                 .iterator(chunk_size=1000)
             ):
+                snapshot_id = result.snapshot_id
+                plugin = result.plugin
+                status = result.status
                 snapshot_key = str(snapshot_id)
                 progress = progress_by_snapshot[snapshot_key]
                 progress["total"] += 1
                 if status == ArchiveResult.StatusChoices.SUCCEEDED:
                     icons_by_snapshot[snapshot_key].add(plugin)
                     progress["succeeded"] += 1
+                    if plugin in preview_candidates:
+                        plugin_rank = preview_plugin_order[plugin]
+                        for filename_rank, filename in enumerate(preview_candidates[plugin]):
+                            output_path = result_output_path(result, filename)
+                            if output_path:
+                                preview_paths_by_snapshot[snapshot_key].append((plugin_rank, filename_rank, output_path))
+                    elif plugin == "favicon":
+                        output_path = result_output_path(result, "favicon.ico")
+                        if output_path:
+                            favicon_paths_by_snapshot[snapshot_key].append(output_path)
                 elif status == ArchiveResult.StatusChoices.FAILED:
                     progress["failed"] += 1
                 elif status == ArchiveResult.StatusChoices.STARTED:
@@ -1266,12 +1338,18 @@ class PublicIndexView(ListView):
             snapshot._icons_progress_stats = progress_by_snapshot.get(str(snapshot.id), {})
             snapshot.num_outputs_cached = snapshot._icons_progress_stats.get("succeeded", 0)
             snapshot._tags_str_cached = ",".join(tag_names_by_snapshot.get(str(snapshot.id), []))
+            snapshot._public_preview_paths = [
+                output_path for _plugin_rank, _filename_rank, output_path in sorted(preview_paths_by_snapshot.get(str(snapshot.id), []))
+            ]
+            snapshot._public_favicon_paths = favicon_paths_by_snapshot.get(str(snapshot.id), [])
             snapshot._is_archived_cached = bool(snapshot.downloaded_at or snapshot.status == Snapshot.StatusChoices.SEALED)
         context["object_list"] = snapshots
         return context
 
     def get_exact_public_snapshot_count(self) -> int:
-        return Snapshot.objects.filter(permissions=PERMISSIONS_PUBLIC).count()
+        hidden_count = Snapshot.objects.filter(permissions=PERMISSIONS_PRIVATE).count()
+        hidden_count += Snapshot.objects.filter(permissions=PERMISSIONS_UNLISTED).count()
+        return Snapshot.objects.count() - hidden_count
 
     def get_queryset(self, **kwargs):
         qs = public_snapshots_queryset(super().get_queryset(**kwargs)).only(*self._base_public_snapshot_fields())
@@ -1327,7 +1405,8 @@ class AddView(UserPassesTestMixin, FormView):
         return get_request_config(self.request).PUBLIC_ADD_VIEW or self.request.user.is_authenticated
 
     def _can_override_crawl_config(self) -> bool:
-        return is_admin_user(self.request)
+        user = self.request.user
+        return bool(user.is_authenticated and user.is_active and user.is_superuser)
 
     def _get_custom_config_overrides(self, form: AddLinkForm) -> dict:
         custom_config = form.cleaned_data.get("config") or {}
@@ -1402,8 +1481,22 @@ class AddView(UserPassesTestMixin, FormView):
         }
 
     def _create_crawl_from_form(self, form, *, created_by_id=None) -> Crawl:
-        urls = form.cleaned_data["url"]
-        print(f"[+] Adding URL: {urls}")
+        from archivebox.cli.archivebox_add import add
+
+        urls_input = form.cleaned_data["url"]
+        urls = urls_input
+        submitted_lines = [line.strip() for line in urls_input.splitlines() if line.strip()]
+        if len(submitted_lines) == 1:
+            try:
+                # A lone URL pasted into /add/ is the same user-facing input as
+                # `archivebox add https://...`: queue that URL directly so a
+                # narrow plugin selection like `wget` can archive it without
+                # also needing parser plugins. Multi-line or formatted text
+                # remains verbatim import content for the internal parser root.
+                urls = [validate_url(submitted_lines[0])]
+            except ValueError:
+                pass
+        print(f"[+] Adding URL: {urls_input}")
 
         # Extract all form fields
         tag = form.cleaned_data["tag"]
@@ -1434,8 +1527,6 @@ class AddView(UserPassesTestMixin, FormView):
         if persona:
             persona.ensure_dirs()
 
-        from archivebox.config.permissions import HOSTNAME
-
         if created_by_id is None:
             if self.request.user.is_authenticated:
                 created_by_id = self.request.user.pk
@@ -1444,60 +1535,43 @@ class AddView(UserPassesTestMixin, FormView):
 
                 created_by_id = get_or_create_system_user_pk()
 
-        created_by_name = self.request.user.username if self.request.user.is_authenticated else "web"
-
-        # 1. save the provided urls to sources/2024-11-05__23-59-59__web_ui_add_by_user_<user_pk>.txt
-        sources_file = CONSTANTS.SOURCES_DIR / f"{timezone.now().strftime('%Y-%m-%d__%H-%M-%S')}__web_ui_add_by_user_{created_by_id}.txt"
-        sources_file.parent.mkdir(parents=True, exist_ok=True)
-        sources_file.write_text(urls if isinstance(urls, str) else "\n".join(urls))
-
-        # 2. create a new Crawl with the URLs from the file
-        timestamp = timezone.now().strftime("%Y-%m-%d__%H-%M-%S")
-        urls_content = sources_file.read_text()
-        # Store explicit crawl-scoped overrides; Crawl.save() freezes them
-        # over the resolved persona/user/machine defaults at creation time.
         config = {}
-        if plugins:
-            config["PLUGINS"] = plugins
         effective_config = get_config(persona=persona) if persona else get_config()
-        if crawl_max_concurrent_snapshots != int(effective_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS):
-            config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] = crawl_max_concurrent_snapshots
         if delete_after != str(effective_config.DELETE_AFTER):
             config["DELETE_AFTER"] = delete_after
-        config["PERMISSIONS"] = permissions
-        if max_urls:
-            config["CRAWL_MAX_URLS"] = max_urls
-        if crawl_max_size:
-            config["CRAWL_MAX_SIZE"] = crawl_max_size
-        if crawl_timeout:
-            config["CRAWL_TIMEOUT"] = crawl_timeout
         if timeout is not None and int(timeout) != int(effective_config.TIMEOUT):
             config["TIMEOUT"] = int(timeout)
-        if snapshot_max_size:
-            config["SNAPSHOT_MAX_SIZE"] = snapshot_max_size
+        if permissions:
+            config["PERMISSIONS"] = permissions
 
-        # Merge custom config overrides
         config.update(plugin_config)
         config.update(custom_config)
         if bool(url_filters.get("only_new")) != bool(effective_config.ONLY_NEW):
             config["ONLY_NEW"] = bool(url_filters.get("only_new"))
-        if url_filters.get("allowlist"):
-            config["URL_ALLOWLIST"] = url_filters["allowlist"]
-        if url_filters.get("denylist"):
-            config["URL_DENYLIST"] = url_filters["denylist"]
-
-        crawl = Crawl.create_scheduler_row(
-            urls=urls_content,
-            max_depth=depth,
-            tags_str=tag,
-            notes=notes,
-            label=f"{created_by_name}@{HOSTNAME}{self.request.path} {timestamp}",
+        crawl, _snapshots = add(
+            urls=urls,
+            depth=depth,
+            max_urls=max_urls,
+            crawl_max_size=crawl_max_size,
+            crawl_timeout=crawl_timeout,
+            snapshot_max_size=snapshot_max_size,
+            crawl_max_concurrent_snapshots=crawl_max_concurrent_snapshots,
+            tag=tag,
+            url_allowlist=url_filters.get("allowlist") or "",
+            url_denylist=url_filters.get("denylist") or "",
+            plugins=plugins,
+            persona=persona.name if persona else "Default",
+            bg=True,
             created_by_id=created_by_id,
             config=config,
-            persona_id=persona.id if persona else None,
-            status=Crawl.StatusChoices.PAUSED if start_paused else Crawl.StatusChoices.QUEUED,
-            retry_at=RETRY_AT_MAX if start_paused else timezone.now(),
         )
+        if notes:
+            crawl.safe_update({"notes": sanitize_html_text(notes)}, refresh=False)
+        if permissions and crawl.config.get("PERMISSIONS") != permissions:
+            next_config = {**crawl.config, "PERMISSIONS": permissions}
+            crawl.safe_update({"config": next_config}, refresh=True)
+        if start_paused:
+            crawl.pause()
 
         # 3. create a CrawlSchedule if schedule is provided
         if schedule:
@@ -1515,11 +1589,6 @@ class AddView(UserPassesTestMixin, FormView):
             crawl.schedule = crawl_schedule
             crawl.safe_update({"schedule": crawl_schedule}, refresh=False)
 
-        if not start_paused:
-            from archivebox.services.runner import ensure_background_runner
-
-            ensure_background_runner()
-
         return crawl
 
     def form_valid(self, form):
@@ -1529,15 +1598,17 @@ class AddView(UserPassesTestMixin, FormView):
         schedule = form.cleaned_data.get("schedule", "").strip()
         rough_url_count = len([url for url in urls.splitlines() if url.strip()])
 
-        # Build success message with schedule link if created
         schedule_msg = ""
         if schedule and crawl.schedule_id:
-            schedule_msg = f" and <a href='{crawl.schedule.admin_change_url}'>scheduled to repeat {schedule}</a>"
+            schedule_msg = format_html(" and <a href='{}'>scheduled to repeat {}</a>", crawl.schedule.admin_change_url, schedule)
 
         messages.success(
             self.request,
-            mark_safe(
-                f"Created crawl with {rough_url_count} starting URL(s){schedule_msg}. Snapshots will be created and archived in the background. <a href='{crawl.admin_change_url}'>View Crawl →</a>",
+            format_html(
+                "Created crawl with {} starting URL(s){}. Snapshots will be created and archived in the background. <a href='{}'>View Crawl →</a>",
+                rough_url_count,
+                schedule_msg,
+                crawl.admin_change_url,
             ),
         )
 
@@ -1675,7 +1746,7 @@ def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
 
             # Use merged config value (includes machine overrides)
             actual_value = merged_config.get(key, dict(section)[key])
-            rows["Value"].append(mark_safe(f"<code>{actual_value}</code>"))
+            rows["Value"].append(format_html("<code>{}</code>", actual_value))
 
             # Show where the value comes from
             source = find_config_source(key, merged_config)
@@ -1683,8 +1754,10 @@ def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
             rows["Source"].append(format_html('<code style="color: {}">{}</code>', source_colors.get(source, "gray"), source))
 
             rows["Default"].append(
-                mark_safe(
-                    f'<a href="https://github.com/search?q=repo%3AArchiveBox%2FArchiveBox+path%3Aconfig+{key}&type=code"><code style="text-decoration: underline">{find_config_default(key) or "See here..."}</code></a>',
+                format_html(
+                    '<a href="https://github.com/search?q=repo%3AArchiveBox%2FArchiveBox+path%3Aconfig+{}&type=code"><code style="text-decoration: underline">{}</code></a>',
+                    key,
+                    find_config_default(key) or "See here...",
                 ),
             )
             # rows['Documentation'].append(mark_safe(f'Wiki: <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#{key.lower()}">{key}</a>'))
@@ -1698,8 +1771,10 @@ def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
         rows["Value"].append(format_html("<code>{}</code>", redact_sensitive_config(CONSTANTS_CONFIG).get(key)))
         rows["Source"].append(mark_safe('<code style="color: gray">Constant</code>'))
         rows["Default"].append(
-            mark_safe(
-                f'<a href="https://github.com/search?q=repo%3AArchiveBox%2FArchiveBox+path%3Aconfig+{key}&type=code"><code style="text-decoration: underline">{find_config_default(key) or "See here..."}</code></a>',
+            format_html(
+                '<a href="https://github.com/search?q=repo%3AArchiveBox%2FArchiveBox+path%3Aconfig+{}&type=code"><code style="text-decoration: underline">{}</code></a>',
+                key,
+                find_config_default(key) or "See here...",
             ),
         )
         # rows['Documentation'].append(mark_safe(f'Wiki: <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#{key.lower()}">{key}</a>'))
@@ -1753,25 +1828,56 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont
     is_redacted = final_value == SENSITIVE_CONFIG_VALUE_REDACTED
 
     # Build sources display
-    sources_html = "<br/>".join([f'<b style="color: {color}">{source}:</b> <code>{value}</code>' for source, value, color in sources_info])
+    sources_html = format_html_join(
+        mark_safe("<br/>"),
+        '<b style="color: {}">{}:</b> <code>{}</code>',
+        ((color, source, value) for source, value, color in sources_info),
+    )
 
     # aliases = USER_CONFIG.get(key, {}).get("aliases", [])
     aliases = []
 
     if key in CONSTANTS_CONFIG:
-        section_header = mark_safe(
-            f'[CONSTANTS]   &nbsp; <b><code style="color: lightgray">{key}</code></b> &nbsp; <small>(read-only, hardcoded by ArchiveBox)</small>',
+        section_header = format_html(
+            '[CONSTANTS]   &nbsp; <b><code style="color: lightgray">{}</code></b> &nbsp; <small>(read-only, hardcoded by ArchiveBox)</small>',
+            key,
         )
     elif key in merged_config:
-        section_header = mark_safe(
-            f'data / ArchiveBox.conf &nbsp; [{find_config_section(key)}]  &nbsp; <b><code style="color: lightgray">{key}</code></b>',
+        section_header = format_html(
+            'data / ArchiveBox.conf &nbsp; [{}]  &nbsp; <b><code style="color: lightgray">{}</code></b>',
+            find_config_section(key),
+            key,
         )
     else:
-        section_header = mark_safe(
-            f'[DYNAMIC CONFIG]   &nbsp; <b><code style="color: lightgray">{key}</code></b> &nbsp; <small>(read-only, calculated at runtime)</small>',
+        section_header = format_html(
+            '[DYNAMIC CONFIG]   &nbsp; <b><code style="color: lightgray">{}</code></b> &nbsp; <small>(read-only, calculated at runtime)</small>',
+            key,
         )
 
     definition_url, definition_label = get_config_definition_link(key)
+    redacted_message = (
+        mark_safe(
+            '<b style="color: red">Value is redacted for your security. (Passwords, secrets, API tokens, etc. cannot be viewed in the Web UI)</b><br/><br/>',
+        )
+        if is_redacted
+        else ""
+    )
+    default_command_value = val.strip("'") if (val := find_config_default(key)) else str(final_value).strip("'")
+    machine_config_link = (
+        format_html('<br/><a href="{}">→ Edit <code>{}</code> in Machine.config for this server</a>', machine_admin_url, key)
+        if machine_admin_url
+        else ""
+    )
+    machine_config_tip = (
+        format_html(
+            '<br/><b>Tip:</b> To override <code>{}</code> on this machine, <a href="{}">edit the Machine.config field</a> and add:<br/><code>{}</code>',
+            key,
+            machine_admin_url,
+            f'{{"{key}": "your_value_here"}}',
+        )
+        if machine_admin_url and key not in CONSTANTS_CONFIG
+        else ""
+    )
 
     section_data = cast(
         SectionData,
@@ -1785,50 +1891,65 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont
                 "Currently read from": config_source,
             },
             "help_texts": {
-                "Key": mark_safe(f"""
-                <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#{key.lower()}">Documentation</a>  &nbsp;
-                <span style="display: {"inline" if aliases else "none"}">
-                    Aliases: {", ".join(aliases)}
+                "Key": format_html(
+                    """
+                <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#{}">Documentation</a>  &nbsp;
+                <span style="display: {}">
+                    Aliases: {}
                 </span>
-            """),
-                "Type": mark_safe(f'''
-                <a href="{definition_url}" target="_blank" rel="noopener noreferrer">
-                    See full definition in <code>{definition_label}</code>...
+            """,
+                    key.lower(),
+                    "inline" if aliases else "none",
+                    ", ".join(aliases),
+                ),
+                "Type": format_html(
+                    """
+                <a href="{}" target="_blank" rel="noopener noreferrer">
+                    See full definition in <code>{}</code>...
                 </a>
-            '''),
-                "Value": mark_safe(f'''
-                {
-                    '<b style="color: red">Value is redacted for your security. (Passwords, secrets, API tokens, etc. cannot be viewed in the Web UI)</b><br/><br/>'
-                    if is_redacted
-                    else ""
-                }
+            """,
+                    definition_url,
+                    definition_label,
+                ),
+                "Value": format_html(
+                    """
+                {}
                 <br/><hr/><br/>
                 <b>Configuration Sources (highest priority first):</b><br/><br/>
-                {sources_html}
+                {}
                 <br/><br/>
-                <p style="display: {"block" if key in merged_config and key not in CONSTANTS_CONFIG else "none"}">
+                <p style="display: {}">
                     <i>To change this value, edit <code>data/ArchiveBox.conf</code> or run:</i>
                     <br/><br/>
-                    <code>archivebox config --set {key}="{
-                    val.strip("'") if (val := find_config_default(key)) else str(final_value).strip("'")
-                }"</code>
+                    <code>archivebox config --set {}="{}"</code>
                 </p>
-            '''),
-                "Currently read from": mark_safe(f"""
-                The value shown in the "Value" field comes from the <b>{config_source}</b> source.
+            """,
+                    redacted_message,
+                    sources_html,
+                    "block" if key in merged_config and key not in CONSTANTS_CONFIG else "none",
+                    key,
+                    default_command_value,
+                ),
+                "Currently read from": format_html(
+                    """
+                The value shown in the "Value" field comes from the <b>{}</b> source.
                 <br/><br/>
                 Priority order (highest to lowest):
                 <ol>
                     <li><b style="color: purple">Machine</b> - Machine-specific overrides
-                        {f'<br/><a href="{machine_admin_url}">→ Edit <code>{key}</code> in Machine.config for this server</a>' if machine_admin_url else ""}
+                        {}
                     </li>
                     <li><b style="color: blue">Environment</b> - process defaults from environment variables</li>
                     <li><b style="color: green">File</b> - data/ArchiveBox.conf</li>
                     <li><b style="color: gray">Plugin Default</b> - Default value from plugin config.json</li>
                     <li><b style="color: gray">Default</b> - Default value from code</li>
                 </ol>
-                {f'<br/><b>Tip:</b> To override <code>{key}</code> on this machine, <a href="{machine_admin_url}">edit the Machine.config field</a> and add:<br/><code>{{"\\"{key}\\": "your_value_here"}}</code>' if machine_admin_url and key not in CONSTANTS_CONFIG else ""}
-            """),
+                {}
+            """,
+                    config_source,
+                    machine_config_link,
+                    machine_config_tip,
+                ),
             },
         },
     )

archivebox/core/widgets.py

@@ -40,6 +40,16 @@ class TagEditorWidget(forms.Widget):
             normalized = f"t_{normalized}"
         return normalized
 
+    def _json_for_inline_script(self, value):
+        """Serialize JSON so it cannot close the surrounding inline <script> tag."""
+        return json.dumps(value).translate(
+            {
+                ord(">"): "\\u003E",
+                ord("<"): "\\u003C",
+                ord("&"): "\\u0026",
+            },
+        )
+
     def _tag_style(self, value):
         """Compute a stable pastel color style for a tag value."""
         tag = (value or "").strip().lower()
@@ -106,6 +116,8 @@ class TagEditorWidget(forms.Widget):
                 </span>
             '''
 
+        tags_json = self._json_for_inline_script(tags)
+
         # Build the widget HTML
         html = f'''
         <div id="{widget_id}_container" class="tag-editor-container" onclick="focusTagInput_{widget_id}(event)">
@@ -128,7 +140,7 @@ class TagEditorWidget(forms.Widget):
 
         <script>
         (function() {{
-            var currentTags_{widget_id} = {json.dumps(tags)};
+            var currentTags_{widget_id} = {tags_json};
             var autocompleteTimeout_{widget_id} = null;
 
             window.focusTagInput_{widget_id} = function(event) {{

archivebox/crawls/models.py

@@ -2,7 +2,6 @@ __package__ = "archivebox.crawls"
 
 from typing import TYPE_CHECKING, Any
 from collections.abc import Iterable, Mapping
-from io import StringIO
 import uuid
 import json
 import re
@@ -35,7 +34,7 @@ from archivebox.base_models.models import (
 )
 from archivebox.workers.models import RETRY_AT_MAX, ModelWithStateMachine, BaseStateMachine
 from archivebox.crawls.schedule_util import next_run_for_schedule, validate_schedule
-from archivebox.misc.util import validate_url, validate_url_length
+from archivebox.misc.util import parse_date, sanitize_html_text, validate_url, validate_url_length
 
 if TYPE_CHECKING:
     from archivebox.core.models import Snapshot
@@ -70,9 +69,14 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes):
         return str(reverse_lazy("api-1:get_any", args=[self.id]))
 
     def save(self, *args, **kwargs):
+        update_fields = kwargs.get("update_fields")
+        if update_fields is None or "label" in update_fields:
+            self.label = sanitize_html_text(self.label).strip()
+        if update_fields is None or "notes" in update_fields:
+            self.notes = sanitize_html_text(self.notes)
         self.schedule = (self.schedule or "").strip()
         validate_schedule(self.schedule)
-        self.label = self.label or (self.template.label if self.template else "")
+        self.label = self.label or (sanitize_html_text(self.template.label).strip() if self.template else "")
         super().save(*args, **kwargs)
         if self.template:
             self.template.safe_update(
@@ -280,6 +284,12 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
 
     def save(self, *args, **kwargs):
         update_fields = kwargs.get("update_fields")
+        if update_fields is None or "label" in update_fields:
+            self.label = sanitize_html_text(self.label).strip()
+        if update_fields is None or "notes" in update_fields:
+            self.notes = sanitize_html_text(self.notes)
+        if update_fields is None or "tags_str" in update_fields:
+            self.tags_str = ",".join(self.parse_tag_names(self.tags_str or ""))
         sync_tags = update_fields is None or "tags_str" in update_fields
         old_crawl = type(self).objects.filter(pk=self.pk).first() if self.pk else None
         previous_tag_names = set()
@@ -376,7 +386,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         tag_names: list[str] = []
         seen: set[str] = set()
         for raw_tag in raw_tags:
-            tag_name = str(raw_tag or "").strip()
+            tag_name = sanitize_html_text(raw_tag).strip()
             if not tag_name:
                 continue
             lowered = tag_name.lower()
@@ -503,12 +513,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         from archivebox.core.models import Snapshot
 
         date_str = self.created_at.strftime("%Y%m%d")
-        first_url = ""
-        for raw_line in StringIO(self.urls or ""):
-            candidate = raw_line.strip()
-            if candidate and not candidate.startswith("#"):
-                first_url = candidate
-                break
+        first_url = next((url for url in self.get_urls_list() if url), "")
         domain = Snapshot.extract_domain_from_url(first_url) if first_url else "unknown"
 
         output_dir = CONSTANTS.USERS_DIR / self.created_by.username / CONSTANTS.CRAWLS_DIR_NAME / date_str / domain / str(self.id)
@@ -519,7 +524,19 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         """Get list of URLs from urls field, filtering out comments and empty lines."""
         if not self.urls:
             return []
-        return [url.strip() for url in self.urls.split("\n") if url.strip() and not url.strip().startswith("#")]
+        return [url for _raw_line, url in self._iter_url_lines() if url]
+
+    def has_internal_input_root(self) -> bool:
+        """Return True when Crawl.urls is preserved source text, not the work queue.
+
+        The runner creates a synthetic root snapshot for raw import text so
+        parser hooks use the same Snapshot lifecycle as every other extractor.
+        In that mode the submitted text must remain in Crawl.urls forever;
+        parsed URLs live as child Snapshot rows and should not be appended back.
+        """
+        from archivebox.core.models import Snapshot
+
+        return self.snapshot_set.filter(url=Snapshot.INTERNAL_INPUT_URL, depth=0).exists()
 
     @staticmethod
     def normalize_domain(value: str) -> str:
@@ -588,22 +605,29 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         except re.error:
             return False
 
+    def get_current_config(self, *, refresh: bool = False) -> dict[str, Any]:
+        if refresh and self.pk:
+            config = type(self).objects.filter(pk=self.pk).values_list("config", flat=True).first()
+            if config is not None:
+                self.config = config
+        return dict(self.config or {})
+
     def get_url_allowlist(self, *, use_effective_config: bool = False, snapshot=None) -> list[str]:
         if use_effective_config:
-            from archivebox.config.common import get_config
-
-            config = get_config(crawl=self, snapshot=snapshot)
+            config = self.get_current_config(refresh=True)
         else:
-            config = self.config or {}
+            config = self.get_current_config()
+        if snapshot is not None and snapshot.config:
+            config.update(snapshot.config)
         return self.split_filter_patterns(config.get("URL_ALLOWLIST", ""))
 
     def get_url_denylist(self, *, use_effective_config: bool = False, snapshot=None) -> list[str]:
         if use_effective_config:
-            from archivebox.config.common import get_config
-
-            config = get_config(crawl=self, snapshot=snapshot)
+            config = self.get_current_config(refresh=True)
         else:
-            config = self.config or {}
+            config = self.get_current_config()
+        if snapshot is not None and snapshot.config:
+            config.update(snapshot.config)
         return self.split_filter_patterns(config.get("URL_DENYLIST", ""))
 
     def url_passes_filters(self, url: str, *, snapshot=None, use_effective_config: bool = True) -> bool:
@@ -682,7 +706,14 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
                 continue
             try:
                 entry = json.loads(stripped)
-                entries.append((raw_line.rstrip(), str(entry.get("url", "") or "").strip()))
+                # Crawl.urls accepts plain lines and JSONL URL records. Other
+                # valid JSON values, e.g. a quoted string from hostile input,
+                # are not records and must stay inert text instead of raising
+                # during later Snapshot.save() bookkeeping.
+                if isinstance(entry, dict):
+                    entries.append((raw_line.rstrip(), str(entry.get("url", "") or "").strip()))
+                else:
+                    entries.append((raw_line.rstrip(), stripped))
             except json.JSONDecodeError:
                 entries.append((raw_line.rstrip(), stripped))
         return entries
@@ -704,9 +735,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         return len(urls)
 
     def remaining_url_capacity(self) -> int | None:
-        from archivebox.config.common import get_config
-
-        max_urls = int(get_config(crawl=self).CRAWL_MAX_URLS or 0)
+        max_urls = int(self._config_value(self.get_current_config(refresh=True), "CRAWL_MAX_URLS", 0) or 0)
         if max_urls <= 0:
             return None
         return max(max_urls - self.count_urls_for_limit(), 0)
@@ -716,9 +745,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         return remaining is None or remaining > 0
 
     def remaining_snapshot_capacity(self) -> int | None:
-        from archivebox.config.common import get_config
-
-        max_urls = int(get_config(crawl=self).CRAWL_MAX_URLS or 0)
+        max_urls = int(self._config_value(self.get_current_config(refresh=True), "CRAWL_MAX_URLS", 0) or 0)
         if max_urls <= 0:
             return None
         return max(max_urls - self.snapshot_set.count(), 0)
@@ -948,21 +975,27 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
             List of newly created Snapshot objects
         """
         from archivebox.core.models import Snapshot, Tag
-        from archivebox.config.common import get_config
         from archivebox.misc.util import fix_url_from_markdown, sanitize_extracted_url
 
         if self.status == self.StatusChoices.SEALED:
             return []
+        # Internal-input crawls preserve the submitted text verbatim in
+        # Crawl.urls. The root snapshot's parser hooks are the only supported
+        # path for turning that text into child snapshots, otherwise a later
+        # runner pass could reinterpret plain URL-looking lines as direct
+        # depth-0 work and bypass format-specific metadata parsing.
+        if self.has_internal_input_root():
+            return []
 
         created_snapshots = []
         crawl_tag_names = self.current_tag_names()
         tags_by_name: dict[str, Tag] = {}
-        config = get_config(crawl=self)
-        only_new_urls = bool(config.ONLY_NEW)
 
         for line in self.urls.splitlines():
             if not line.strip():
                 continue
+            config = self.get_current_config(refresh=True)
+            only_new_urls = bool(self._config_value(config, "ONLY_NEW", True))
 
             # Parse JSONL or plain URL
             try:
@@ -988,10 +1021,10 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
             except ValueError as err:
                 print(f"[yellow][!] Skipping invalid snapshot URL: {url[:120]}... ({err})[/yellow]")
                 continue
-            if Snapshot.is_archivebox_internal_url(url):
+            if Snapshot.is_archivebox_internal_url(url, config=config):
                 print(f"[yellow][!] Skipping internal ArchiveBox snapshot URL: {url}[/yellow]")
                 continue
-            if not self.url_passes_filters(url):
+            if not self.url_passes_filters(url, use_effective_config=False):
                 continue
             if only_new_urls and Snapshot.objects.filter(url=url).exists():
                 continue
@@ -1061,7 +1094,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
                         # relation without clearing any non-crawl snapshot tags.
                         Tag.objects.bulk_create(missing_tags, ignore_conflicts=True)
                         tags_by_name.update({tag.name: tag for tag in Tag.objects.filter(name__in=missing_names)})
-                snapshot.tags.add(*[tag.pk for tag_name in tag_names if (tag := tags_by_name.get(tag_name))])
+                snapshot.add_tag_ids([tag.pk for tag_name in tag_names if (tag := tags_by_name.get(tag_name))])
 
             # Symlink creation touches the filesystem and can be slow on remote disks.
             # Defer it until after any active DB transaction commits so SQLite does
@@ -1099,7 +1132,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
     ) -> list["Snapshot"]:
         """Create child snapshots from discovered URL records after filtering and deduping once."""
         from archivebox.core.models import Snapshot, SnapshotTag, Tag
-        from archivebox.config.common import get_config
         from archivebox.misc.util import fix_url_from_markdown, sanitize_extracted_url
 
         if self.status == self.StatusChoices.SEALED:
@@ -1109,30 +1141,71 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
             return []
 
         crawl_tag_names = self.current_tag_names()
-        config = get_config(crawl=self, snapshot=parent_snapshot)
+        config = self.get_current_config(refresh=True)
+        if parent_snapshot is not None and parent_snapshot.config:
+            config.update(parent_snapshot.config)
         allowlist = self.split_filter_patterns(config.get("URL_ALLOWLIST", ""))
         denylist = self.split_filter_patterns(config.get("URL_DENYLIST", ""))
 
+        def metadata_score(record: Mapping[str, Any]) -> int:
+            # Multiple parsers can discover the same URL from one import root.
+            # Keep the record with the richest user-facing metadata so generic
+            # text/HTML extraction does not erase RSS/Netscape/JSON fields.
+            return sum(bool(record.get(field)) for field in ("title", "bookmarked_at", "timestamp", "tags"))
+
         deduped_records: dict[str, Mapping[str, Any]] = {}
         for record in records:
             url = sanitize_extracted_url(fix_url_from_markdown(str(record.get("url") or "").strip()))
-            if not url or url in deduped_records:
+            if not url:
                 continue
             try:
                 validate_url(url)
             except ValueError as err:
                 print(f"[yellow][!] Skipping invalid discovered snapshot URL: {url[:120]}... ({err})[/yellow]")
                 continue
-            if Snapshot.is_archivebox_internal_url(url):
+            if Snapshot.is_archivebox_internal_url(url, config=config):
                 print(f"[yellow][!] Skipping internal ArchiveBox discovered snapshot URL: {url}[/yellow]")
                 continue
             if self.url_passes_compiled_filters(url, allowlist=allowlist, denylist=denylist):
-                deduped_records[url] = record
+                existing_record = deduped_records.get(url)
+                if existing_record is None or metadata_score(record) > metadata_score(existing_record):
+                    deduped_records[url] = record
 
         if not deduped_records:
             return []
 
-        existing_scope = Snapshot.objects if bool(config.ONLY_NEW) else self.snapshot_set
+        existing_in_crawl = {
+            snapshot.url: snapshot for snapshot in self.snapshot_set.prefetch_related("tags").filter(url__in=deduped_records.keys())
+        }
+        for url, snapshot in existing_in_crawl.items():
+            record = deduped_records[url]
+            update_fields = []
+            title = Snapshot._normalize_title_candidate(str(record.get("title") or "").strip()[:512], snapshot_url=url)
+            if title and (not snapshot.title or len(title) > len(snapshot.title or "")):
+                snapshot.title = title
+                update_fields.append("title")
+            bookmarked_at = None
+            try:
+                bookmarked_at = parse_date(record.get("bookmarked_at") or record.get("timestamp"))
+            except (TypeError, ValueError, OSError):
+                pass
+            if bookmarked_at and snapshot.bookmarked_at != bookmarked_at:
+                snapshot.bookmarked_at = bookmarked_at
+                update_fields.append("bookmarked_at")
+            if update_fields:
+                snapshot.save(update_fields=[*update_fields, "modified_at"])
+            tag_names = {
+                *crawl_tag_names,
+                *self.parse_tag_names(
+                    str(record.get("tags") or ""),
+                    pattern=self._config_value(config, "TAG_SEPARATOR_PATTERN", r"[,]"),
+                ),
+            }
+            if tag_names:
+                tag_ids = [Tag.objects.get_or_create(name=tag_name)[0].pk for tag_name in tag_names]
+                snapshot.add_tag_ids(tag_ids)
+
+        existing_scope = Snapshot.objects if bool(self._config_value(config, "ONLY_NEW", True)) else self.snapshot_set
         existing_urls = set(existing_scope.filter(url__in=deduped_records.keys()).values_list("url", flat=True))
         urls = [url for url in deduped_records.keys() if url not in existing_urls]
         remaining = self.remaining_snapshot_capacity()
@@ -1142,27 +1215,34 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
             return []
 
         now = timezone.now()
-        snapshots = [
-            Snapshot(
-                url=url,
-                timestamp=str((now + timedelta(microseconds=index)).timestamp()),
-                title=Snapshot._normalize_title_candidate(
-                    str(deduped_records[url].get("title") or "").strip()[:512],
-                    snapshot_url=url,
-                )
-                or None,
-                crawl=self,
-                parent_snapshot=parent_snapshot,
-                depth=depth,
-                status=Snapshot.StatusChoices.QUEUED,
-                retry_at=now,
-                bookmarked_at=now,
-                created_at=now,
+        snapshots = []
+        for index, url in enumerate(urls):
+            record = deduped_records[url]
+            bookmarked_at = now
+            try:
+                bookmarked_at = parse_date(record.get("bookmarked_at") or record.get("timestamp")) or now
+            except (TypeError, ValueError, OSError):
+                pass
+            snapshots.append(
+                Snapshot(
+                    url=url,
+                    timestamp=str((now + timedelta(microseconds=index)).timestamp()),
+                    title=Snapshot._normalize_title_candidate(
+                        str(record.get("title") or "").strip()[:512],
+                        snapshot_url=url,
+                    )
+                    or None,
+                    crawl=self,
+                    parent_snapshot=parent_snapshot,
+                    depth=depth,
+                    status=Snapshot.StatusChoices.QUEUED,
+                    retry_at=now,
+                    bookmarked_at=bookmarked_at,
+                    created_at=now,
+                ),
             )
-            for index, url in enumerate(urls)
-        ]
         for snapshot in snapshots:
-            snapshot.set_delete_at_from_config(config.DELETE_AFTER)
+            snapshot.set_delete_at_from_config(self._config_value(config, "DELETE_AFTER", "0"))
 
         created_snapshots = []
         for snapshot in snapshots:
@@ -1181,7 +1261,11 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
 
         crawl_urls = {url for _raw_line, url in self._iter_url_lines() if url}
         new_url_lines = [snapshot.url for snapshot in created_snapshots if snapshot.url not in crawl_urls]
-        if new_url_lines:
+        # For internal-input crawls, Crawl.urls is the immutable source text.
+        # Child snapshots are the parsed/indexed representation, so appending
+        # discovered URLs here would both duplicate state and destroy the exact
+        # import artifact users submitted through CLI/API/UI.
+        if new_url_lines and not self.has_internal_input_root():
             self.urls = (self.urls.rstrip() + "\n" + "\n".join(new_url_lines)).lstrip("\n")
             self.save(update_fields=["urls", "modified_at"])
 
@@ -1189,7 +1273,10 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         for snapshot in created_snapshots:
             tag_names = {
                 *crawl_tag_names,
-                *self.parse_tag_names(str(deduped_records[snapshot.url].get("tags") or ""), pattern=config.TAG_SEPARATOR_PATTERN),
+                *self.parse_tag_names(
+                    str(deduped_records[snapshot.url].get("tags") or ""),
+                    pattern=self._config_value(config, "TAG_SEPARATOR_PATTERN", r"[,]"),
+                ),
             }
             if tag_names:
                 tag_names_by_url[snapshot.url] = tag_names
@@ -1312,7 +1399,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
             The root Snapshot for this crawl, or None for system crawls that don't create snapshots
         """
         import time
-        from archivebox.plugins.hooks import run_hook, discover_hooks, process_hook_records, is_finite_background_hook
+        from archivebox.plugins.hooks import run_hook, discover_hooks, process_hook_records
         from archivebox.config.common import get_config
         from archivebox.machine.models import Binary, Machine
 
@@ -1370,19 +1457,15 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
                 print(f"[yellow]⏱️  Hook {hook.name} took {hook_elapsed:.2f}s[/yellow]")
 
             if process.status == process.StatusChoices.RUNNING:
-                if not is_finite_background_hook(hook.name):
-                    return set()
-                try:
-                    process.wait(timeout=process.timeout)
-                except Exception:
+                if process.poll() is None:
                     return set()
 
             from archivebox.plugins.hooks import extract_records_from_process
 
             records = []
-            # Finite background hooks can exit before their completed Process
-            # metadata is visible. Give successful hooks a brief chance to
-            # flush JSONL stdout into the Process row before downstream hooks.
+            # A hook can exit before its completed Process metadata is visible.
+            # Give successful hooks a brief chance to flush JSONL stdout into
+            # the Process row before downstream hooks.
             for delay in (0.0, 0.05, 0.1, 0.25, 0.5):
                 if delay:
                     time.sleep(delay)

archivebox/machine/detect.py

@@ -145,6 +145,11 @@ ip_addrs = lambda addrs: (a for a in addrs if a.family == socket.AF_INET)
 mac_addrs = lambda addrs: (a for a in addrs if a.family == psutil.AF_LINK)
 
 
+def unknown_if_blank(value):
+    normalized = str(value or "").strip()
+    return normalized or "Unknown"
+
+
 def get_isp_info(ip=None):
     # Get public IP
     try:
@@ -161,10 +166,10 @@ def get_isp_info(ip=None):
     except Exception:
         pass
 
-    isp = data.get("org", "Unknown")
-    city = data.get("city", "Unknown")
-    region = data.get("region", "Unknown")
-    country = data.get("country_name", "Unknown")
+    isp = unknown_if_blank(data.get("org"))
+    city = unknown_if_blank(data.get("city"))
+    region = unknown_if_blank(data.get("region"))
+    country = unknown_if_blank(data.get("country_name"))
 
     # Get system DNS resolver servers
     dns_server = None

archivebox/machine/migrations/0021_networkinterface_blank_defaults.py

@@ -0,0 +1,42 @@
+# Generated by Django 6.0.6 on 2026-06-21 09:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("machine", "0020_repair_process_binary_iface_links"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="networkinterface",
+            name="city",
+            field=models.CharField(default="", max_length=63),
+        ),
+        migrations.AlterField(
+            model_name="networkinterface",
+            name="country",
+            field=models.CharField(default="", max_length=63),
+        ),
+        migrations.AlterField(
+            model_name="networkinterface",
+            name="hostname",
+            field=models.CharField(default="", max_length=63),
+        ),
+        migrations.AlterField(
+            model_name="networkinterface",
+            name="iface",
+            field=models.CharField(default="", max_length=15),
+        ),
+        migrations.AlterField(
+            model_name="networkinterface",
+            name="isp",
+            field=models.CharField(default="", max_length=63),
+        ),
+        migrations.AlterField(
+            model_name="networkinterface",
+            name="region",
+            field=models.CharField(default="", max_length=63),
+        ),
+    ]

archivebox/machine/migrations/0022_networkinterface_identity_without_mac.py

@@ -0,0 +1,60 @@
+# Generated by Django 6.0.6 on 2026-06-21 09:23
+
+from django.db import migrations, models
+from django.db.models import Count
+
+
+def dedupe_network_interfaces_without_mac(apps, schema_editor):
+    NetworkInterface = apps.get_model("machine", "NetworkInterface")
+    Process = apps.get_model("machine", "Process")
+    duplicate_groups = (
+        NetworkInterface.objects.values(
+            "machine_id",
+            "ip_public",
+            "ip_local",
+            "dns_server",
+        )
+        .annotate(count=Count("id"))
+        .filter(count__gt=1)
+    )
+    for group in duplicate_groups.iterator(chunk_size=100):
+        lookup = {
+            "machine_id": group["machine_id"],
+            "ip_public": group["ip_public"],
+            "ip_local": group["ip_local"],
+            "dns_server": group["dns_server"],
+        }
+        keep = NetworkInterface.objects.filter(**lookup).order_by("-modified_at", "-created_at").first()
+        if keep is None:
+            continue
+        duplicates = NetworkInterface.objects.filter(**lookup).exclude(id=keep.id)
+        duplicate_ids = list(duplicates.values_list("id", flat=True))
+        if not duplicate_ids:
+            continue
+        Process.objects.filter(iface_id__in=duplicate_ids).update(iface_id=keep.id)
+        duplicates.delete()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("machine", "0021_networkinterface_blank_defaults"),
+    ]
+
+    operations = [
+        migrations.RemoveConstraint(
+            model_name="networkinterface",
+            name="unique_network_interface_identity",
+        ),
+        migrations.RunPython(dedupe_network_interfaces_without_mac, migrations.RunPython.noop),
+        migrations.AlterUniqueTogether(
+            name="networkinterface",
+            unique_together={("machine", "ip_public", "ip_local", "dns_server")},
+        ),
+        migrations.AddConstraint(
+            model_name="networkinterface",
+            constraint=models.UniqueConstraint(
+                fields=("machine", "ip_public", "ip_local", "dns_server"),
+                name="unique_network_interface_identity",
+            ),
+        ),
+    ]

archivebox/machine/models.py

@@ -140,7 +140,7 @@ def _sanitize_machine_config(config: dict[str, Any] | None, *, lib_dir: str | Pa
     """Validate ``Machine.config`` in place.
 
     Drops stale ``*_BINARY`` overrides whose path no longer exists or whose
-    path falls outside of ``LIB_DIR`` (so a binary uninstall or a lib_dir
+    path falls outside of ``ABXPKG_LIB_DIR`` (so a binary uninstall or a lib_dir
     move clears the override automatically). Non-``_BINARY`` keys
     (``BASE_URL``, ``SERVER_SECURITY_MODE``, plugin tunables, etc.) are
     pass-through — they're arbitrary config overrides and not ours to filter.
@@ -287,7 +287,7 @@ class Machine(ModelWithHealthStats):
     def _sanitize_config(cls, machine: Machine) -> Machine:
         from archivebox.config.common import get_config
 
-        sanitized = _sanitize_machine_config(machine.config, lib_dir=get_config(include_machine=False).LIB_DIR)
+        sanitized = _sanitize_machine_config(machine.config, lib_dir=get_config(include_machine=False).ABXPKG_LIB_DIR)
         current = machine.config or {}
         if sanitized != current:
             machine.config = sanitized
@@ -394,12 +394,12 @@ class NetworkInterface(ModelWithHealthStats):
     ip_public = models.GenericIPAddressField(default=None, null=False, editable=False)
     ip_local = models.GenericIPAddressField(default=None, null=False, editable=False)
     dns_server = models.GenericIPAddressField(default=None, null=False, editable=False)
-    hostname = models.CharField(max_length=63, default=None, null=False)
-    iface = models.CharField(max_length=15, default=None, null=False)
-    isp = models.CharField(max_length=63, default=None, null=False)
-    city = models.CharField(max_length=63, default=None, null=False)
-    region = models.CharField(max_length=63, default=None, null=False)
-    country = models.CharField(max_length=63, default=None, null=False)
+    hostname = models.CharField(max_length=63, default="", null=False)
+    iface = models.CharField(max_length=15, default="", null=False)
+    isp = models.CharField(max_length=63, default="", null=False)
+    city = models.CharField(max_length=63, default="", null=False)
+    region = models.CharField(max_length=63, default="", null=False)
+    country = models.CharField(max_length=63, default="", null=False)
     # num_uses_failed = models.PositiveIntegerField(default=0)  # from ModelWithHealthStats
     # num_uses_succeeded = models.PositiveIntegerField(default=0)  # from ModelWithHealthStats
 
@@ -408,10 +408,10 @@ class NetworkInterface(ModelWithHealthStats):
 
     class Meta(ModelWithHealthStats.Meta):
         app_label = "machine"
-        unique_together = (("machine", "ip_public", "ip_local", "mac_address", "dns_server"),)
+        unique_together = (("machine", "ip_public", "ip_local", "dns_server"),)
         constraints = [
             models.UniqueConstraint(
-                fields=["machine", "ip_public", "ip_local", "mac_address", "dns_server"],
+                fields=["machine", "ip_public", "ip_local", "dns_server"],
                 name="unique_network_interface_identity",
             ),
         ]
@@ -444,7 +444,6 @@ class NetworkInterface(ModelWithHealthStats):
             machine=machine,
             ip_public=net_info.pop("ip_public"),
             ip_local=net_info.pop("ip_local"),
-            mac_address=net_info.pop("mac_address"),
             dns_server=net_info.pop("dns_server"),
         )
         _CURRENT_INTERFACE = cls.objects.filter(**lookup).order_by("-modified_at", "-created_at").first()
@@ -681,7 +680,7 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
             )
             from archivebox.config.common import get_config
 
-            binary.symlink_to_lib_bin_after_commit(get_config().LIB_BIN_DIR)
+            binary.symlink_to_lib_bin_after_commit(get_config().ABXPKG_LIB_DIR / "bin")
             return binary
 
         # Case 2: From binaries.json - create queued binary (needs installation)
@@ -714,7 +713,7 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
             )
             from archivebox.config.common import get_config
 
-            binary.symlink_to_lib_bin_after_commit(get_config().LIB_BIN_DIR)
+            binary.symlink_to_lib_bin_after_commit(get_config().ABXPKG_LIB_DIR / "bin")
             return binary
 
         return None
@@ -750,15 +749,15 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
 
     def symlink_to_lib_bin(self, lib_bin_dir: str | Path) -> Path | None:
         """
-        Symlink this binary into LIB_BIN_DIR for human-facing convenience.
+        Symlink this binary into a derived lib/bin directory for human-facing convenience.
 
         After a binary is installed by any binprovider (pip, npm, brew, apt, etc),
         we can optionally expose a flat convenience directory for shell users.
-        ArchiveBox/abx-dl runtime lookup must use the provider-specific LIB_DIR
+        ArchiveBox/abx-dl runtime lookup must use the provider-specific ABXPKG_LIB_DIR
         paths, not this indirection.
 
         Args:
-            lib_bin_dir: Path to LIB_BIN_DIR (e.g., /data/lib/arm64-darwin/bin)
+            lib_bin_dir: Path to the derived convenience bin dir (e.g., /data/lib/bin)
 
         Returns:
             Path to the created symlink, or None if symlinking failed
@@ -782,14 +781,14 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
         except StopIteration:
             app_index = -1
 
-        # Create LIB_BIN_DIR if it doesn't exist
+        # Create the derived convenience bin dir if it doesn't exist.
         try:
             lib_bin_dir.mkdir(parents=True, exist_ok=True)
         except (OSError, PermissionError) as e:
-            print(f"Failed to create LIB_BIN_DIR {lib_bin_dir}: {e}", file=sys.stderr)
+            print(f"Failed to create lib/bin convenience dir {lib_bin_dir}: {e}", file=sys.stderr)
             return None
 
-        # Expose the canonical Binary.name in LIB_BIN_DIR. Some providers point
+        # Expose the canonical Binary.name in the convenience bin dir. Some providers point
         # abspath at implementation files like cli.js or manifest.json; those
         # are valid targets, but they are not user-facing binary names.
         binary_name = _canonical_binary_name(self.name) or binary_abspath.name
@@ -1544,9 +1543,11 @@ class Process(ModelWithDeleteAfter, models.Model):
         avoids any race window where the row briefly exists as ADD before a
         higher-level demotion.
         """
-        argv_str = " ".join(sys.argv).lower()
+        argv = [str(arg) for arg in sys.argv]
+        argv_str = " ".join(argv).lower()
+        executable = Path(argv[0]).name.lower() if argv else ""
 
-        if "supervisord" in argv_str:
+        if executable == "supervisord" or any(arg.startswith("--configuration=") for arg in argv[1:]):
             return cls.TypeChoices.SUPERVISORD
         elif "runner_watch" in argv_str:
             return cls.TypeChoices.WORKER

archivebox/misc/checks.py

@@ -220,14 +220,14 @@ def check_not_inside_source_dir():
 def check_data_dir_permissions(config=None, **config_kwargs):
     from archivebox import DATA_DIR
     from archivebox.misc.logging import STDERR
-    from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, DEFAULT_PUID, DEFAULT_PGID, IS_ROOT, USER
+    from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, DEFAULT_UID, DEFAULT_GID, IS_ROOT, USER
     from archivebox.config.paths import get_or_create_working_tmp_dir, get_or_create_working_lib_dir
 
     data_dir_stat = Path(DATA_DIR).stat()
     data_dir_uid, data_dir_gid = data_dir_stat.st_uid, data_dir_stat.st_gid
     data_owned_by_root = data_dir_uid == 0
 
-    # data_owned_by_default_user = data_dir_uid == DEFAULT_PUID or data_dir_gid == DEFAULT_PGID
+    # data_owned_by_default_user = data_dir_uid == DEFAULT_UID or data_dir_gid == DEFAULT_GID
     data_owner_doesnt_match = (data_dir_uid != ARCHIVEBOX_USER and data_dir_gid != ARCHIVEBOX_GROUP) if not IS_ROOT else False
     data_not_writable = not (os.path.isdir(DATA_DIR) and os.access(DATA_DIR, os.W_OK))
     if data_not_writable:
@@ -245,9 +245,9 @@ def check_data_dir_permissions(config=None, **config_kwargs):
 
     if data_not_writable:
         STDERR.print(
-            f"[violet]Hint:[/violet] Change the current ownership [red]{data_dir_uid}[/red]:{data_dir_gid} (PUID:PGID) to the user & group that will run ArchiveBox, e.g.:",
+            f"[violet]Hint:[/violet] Change the current ownership [red]{data_dir_uid}[/red]:{data_dir_gid} to the user & group that will run ArchiveBox, e.g.:",
         )
-        STDERR.print(f"    [grey53]sudo[/grey53] chown -R [blue]{DEFAULT_PUID}:{DEFAULT_PGID}[/blue] {DATA_DIR.resolve()}")
+        STDERR.print(f"    [grey53]sudo[/grey53] chown -R [blue]{DEFAULT_UID}:{DEFAULT_GID}[/blue] {DATA_DIR.resolve()}")
         STDERR.print("    Avoid recursive chown on very large archives unless you know the full tree needs repair.")
         STDERR.print()
         STDERR.print("[blue]More info:[/blue]")
@@ -273,9 +273,9 @@ def check_data_dir_permissions(config=None, **config_kwargs):
         tmp_dir = config.TMP_DIR
 
     try:
-        lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True, config=config) or config.LIB_DIR
+        lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True, config=config) or config.ABXPKG_LIB_DIR
     except Exception:
-        lib_dir = config.LIB_DIR
+        lib_dir = config.ABXPKG_LIB_DIR
 
     # Check /tmp dir permissions
     check_tmp_dir(tmp_dir, throw=False, must_exist=True, config=config)
@@ -289,7 +289,14 @@ def check_data_dir_permissions(config=None, **config_kwargs):
 
 
 def check_tmp_dir(tmp_dir=None, throw=False, quiet=False, must_exist=True, config=None, **config_kwargs):
-    from archivebox.config.paths import assert_dir_can_contain_unix_sockets, dir_is_writable, get_or_create_working_tmp_dir
+    from archivebox.config.paths import (
+        MAX_TMP_SOCKET_URL_LENGTH,
+        SUPERVISORD_SOCKET_FILENAME,
+        assert_dir_can_contain_unix_sockets,
+        dir_is_writable,
+        get_or_create_working_tmp_dir,
+        tmp_dir_socket_path_is_short_enough,
+    )
     from archivebox.misc.logging import STDERR
     from archivebox.misc.logging_util import pretty_path
     from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
@@ -297,20 +304,22 @@ def check_tmp_dir(tmp_dir=None, throw=False, quiet=False, must_exist=True, confi
 
     config = config or get_config(**config_kwargs)
     tmp_dir = tmp_dir or config.TMP_DIR
-    socket_file = tmp_dir.absolute().resolve() / "supervisord.sock"
+    socket_file = tmp_dir.absolute().resolve() / SUPERVISORD_SOCKET_FILENAME
 
     if not must_exist and not os.path.isdir(tmp_dir):
         # just check that its viable based on its length (because dir may not exist yet, we cant check if its writable)
-        return len(f"file://{socket_file}") <= 96
+        return tmp_dir_socket_path_is_short_enough(tmp_dir)
 
     tmp_is_valid = False
     try:
         tmp_is_valid = dir_is_writable(tmp_dir)
         if not config.ALLOW_NO_UNIX_SOCKETS:
             tmp_is_valid = tmp_is_valid and assert_dir_can_contain_unix_sockets(tmp_dir)
-        assert tmp_is_valid, f"ArchiveBox user PUID={ARCHIVEBOX_USER} PGID={ARCHIVEBOX_GROUP} is unable to write to TMP_DIR={tmp_dir}"
-        assert len(f"file://{socket_file}") <= 96, (
-            f"ArchiveBox TMP_DIR={tmp_dir} is too long, dir containing unix socket files must be <90 chars."
+        assert tmp_is_valid, f"ArchiveBox user {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP} is unable to write to TMP_DIR={tmp_dir}"
+        socket_url_len = len(f"file://{socket_file}")
+        assert tmp_dir_socket_path_is_short_enough(tmp_dir), (
+            f"ArchiveBox TMP_DIR={tmp_dir} is too long, file://{socket_file} is {socket_url_len} chars "
+            f"and must be <{MAX_TMP_SOCKET_URL_LENGTH} chars."
         )
         return True
     except Exception as e:
@@ -324,7 +333,7 @@ def check_tmp_dir(tmp_dir=None, throw=False, quiet=False, must_exist=True, confi
                     "",
                     "[blue]Info:[/blue] [grey53]The TMP_DIR is used for the supervisord unix socket file and other temporary files.",
                     "  - It [red]must[/red] be on a local drive (not inside a docker volume, remote network drive, or FUSE mount).",
-                    f"  - It [red]must[/red] be readable and writable by the ArchiveBox user (PUID={ARCHIVEBOX_USER}, PGID={ARCHIVEBOX_GROUP}).",
+                    f"  - It [red]must[/red] be readable and writable by the ArchiveBox user ({ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}).",
                     "  - It [red]must[/red] be a *short* path (less than 90 characters) due to UNIX path length restrictions for sockets.",
                     "  - It [yellow]should[/yellow] be able to hold at least 200MB of data (in-progress downloads can be large).[/grey53]",
                     "",
@@ -356,7 +365,7 @@ def check_lib_dir(lib_dir: Path | None = None, throw=False, quiet=False, must_ex
     from archivebox.config.common import get_config
 
     config = config or get_config(**config_kwargs)
-    lib_dir = lib_dir or config.LIB_DIR
+    lib_dir = lib_dir or config.ABXPKG_LIB_DIR
 
     if not must_exist and not os.path.isdir(lib_dir):
         return True
@@ -364,7 +373,7 @@ def check_lib_dir(lib_dir: Path | None = None, throw=False, quiet=False, must_ex
     lib_is_valid = False
     try:
         lib_is_valid = dir_is_writable(lib_dir)
-        assert lib_is_valid, f"ArchiveBox user PUID={ARCHIVEBOX_USER} PGID={ARCHIVEBOX_GROUP} is unable to write to LIB_DIR={lib_dir}"
+        assert lib_is_valid, f"ArchiveBox user {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP} is unable to write to ABXPKG_LIB_DIR={lib_dir}"
         return True
     except Exception as e:
         if not quiet:
@@ -372,16 +381,16 @@ def check_lib_dir(lib_dir: Path | None = None, throw=False, quiet=False, must_ex
             ERROR_TEXT = "\n".join(
                 (
                     "",
-                    f"[red]:cross_mark: ArchiveBox is unable to use LIB_DIR={pretty_path(lib_dir)}[/red]",
+                    f"[red]:cross_mark: ArchiveBox is unable to use ABXPKG_LIB_DIR={pretty_path(lib_dir)}[/red]",
                     f"   [yellow]{e}[/yellow]",
                     "",
-                    "[blue]Info:[/blue] [grey53]The LIB_DIR is used to store ArchiveBox auto-installed plugin library and binary dependencies.",
-                    f"  - It [red]must[/red] be readable and writable by the ArchiveBox user (PUID={ARCHIVEBOX_USER}, PGID={ARCHIVEBOX_GROUP}).",
+                    "[blue]Info:[/blue] [grey53]The ABXPKG_LIB_DIR is used to store ArchiveBox auto-installed plugin library and binary dependencies.",
+                    f"  - It [red]must[/red] be readable and writable by the ArchiveBox user ({ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}).",
                     "  - It [yellow]should[/yellow] be on a local (ideally fast) drive like an SSD or HDD (not on a network drive or external HDD).",
                     "  - It [yellow]should[/yellow] be able to hold at least 1GB of data (some dependencies like Chrome can be large).[/grey53]",
                     "",
-                    "[violet]Hint:[/violet] Fix it by setting LIB_DIR to a path that meets these requirements, e.g.:",
-                    f"      [green]archivebox config --set LIB_DIR={get_or_create_working_lib_dir(autofix=False, quiet=True) or config.LIB_DIR}[/green]",
+                    "[violet]Hint:[/violet] Fix it by setting ABXPKG_LIB_DIR to a path that meets these requirements, e.g.:",
+                    f"      [green]archivebox config --set ABXPKG_LIB_DIR={get_or_create_working_lib_dir(autofix=False, quiet=True) or config.ABXPKG_LIB_DIR}[/green]",
                     "",
                 ),
             )
@@ -390,11 +399,13 @@ def check_lib_dir(lib_dir: Path | None = None, throw=False, quiet=False, must_ex
                     ERROR_TEXT,
                     expand=False,
                     border_style="red",
-                    title="[red]:cross_mark: Error with configured LIB_DIR[/red]",
+                    title="[red]:cross_mark: Error with configured ABXPKG_LIB_DIR[/red]",
                     subtitle="[yellow]Dependencies may not auto-install properly until fixed.[/yellow]",
                 ),
             )
             STDERR.print()
         if throw:
-            raise OSError(f"LIB_DIR={lib_dir} is invalid, ArchiveBox is unable to use it and dependencies will fail to install.") from e
+            raise OSError(
+                f"ABXPKG_LIB_DIR={lib_dir} is invalid, ArchiveBox is unable to use it and dependencies will fail to install.",
+            ) from e
     return False

archivebox/misc/jsonl.py

@@ -18,6 +18,9 @@ Typed JSONL Format:
 Plain URLs (also supported):
     https://example.com
     https://foo.com
+
+Local filesystem paths and file:// URLs are intentionally not accepted here.
+Pipe file contents through stdin instead of passing a path/URI as a crawl URL.
 """
 
 __package__ = "archivebox.misc"
@@ -80,7 +83,7 @@ def parse_line(line: str) -> dict[str, Any] | None:
             pass
 
     # Treat as plain URL if it looks like one
-    if line.startswith("http://") or line.startswith("https://") or line.startswith("file://"):
+    if line.startswith("http://") or line.startswith("https://"):
         return {"type": TYPE_SNAPSHOT, "url": line}
 
     # Could be a snapshot ID (UUID with dashes or compact 32-char hex)
@@ -135,17 +138,13 @@ def read_args_or_stdin(args: Iterable[str], stream: TextIO | None = None) -> Ite
     Read from CLI arguments if provided, otherwise from stdin.
 
     Handles both URLs and JSONL from either source.
+    Does not expand local file path arguments; pipe file contents via stdin.
     """
     if args:
         for arg in args:
-            # Check if it's a file path
-            path = Path(arg)
-            if path.exists() and path.is_file():
-                yield from read_file(path)
-            else:
-                record = parse_line(arg)
-                if record:
-                    yield record
+            record = parse_line(arg)
+            if record:
+                yield record
     else:
         yield from read_stdin(stream)
 

archivebox/misc/util.py

@@ -66,6 +66,22 @@ htmlencode = lambda s: s and escape(s, quote=True)
 htmldecode = lambda s: s and unescape(s)
 
 
+def sanitize_html_text(value: Any) -> str:
+    """Strip all HTML from user-editable text before storing it."""
+    if value is None:
+        return ""
+    import bleach
+
+    return bleach.clean(
+        str(value),
+        tags=[],
+        attributes={},
+        protocols=[],
+        strip=True,
+        strip_comments=True,
+    )
+
+
 def ts_to_date_str(ts: Any) -> str | None:
     parsed = parse_date(ts)
     return None if parsed is None else parsed.strftime("%Y-%m-%d %H:%M")

archivebox/personas/importers.py

@@ -47,6 +47,12 @@ VOLATILE_PROFILE_COPY_PATTERNS = (
     "ShaderCache",
     "Service Worker",
     "GCM Store",
+    "chrome-extension_*",
+    "DNR Extension Rules",
+    "Extension Rules",
+    "Extension Scripts",
+    "Extension State",
+    "Local Extension Settings",
     "*.log",
     "Crashpad",
     "BrowserMetrics",
@@ -62,6 +68,55 @@ PERSONA_PROFILE_DIR_CANDIDATES = (
 )
 
 
+def _path_exists(path: Path) -> bool:
+    try:
+        return path.exists()
+    except OSError:
+        return False
+
+
+def _path_is_dir(path: Path) -> bool:
+    try:
+        return path.is_dir()
+    except OSError:
+        return False
+
+
+def _iter_children(path: Path) -> list[Path]:
+    try:
+        return list(path.iterdir())
+    except OSError:
+        return []
+
+
+def _is_copyable_profile_entry(path: Path) -> bool:
+    try:
+        if path.is_symlink():
+            return True
+        if path.is_dir():
+            return os.access(path, os.R_OK | os.X_OK)
+        return os.access(path, os.R_OK)
+    except OSError:
+        return False
+
+
+def profile_copy_ignore(src: str, names: list[str]) -> set[str]:
+    ignored = set(shutil.ignore_patterns(*VOLATILE_PROFILE_COPY_PATTERNS)(src, names))
+    src_path = Path(src)
+    for name in names:
+        if name in ignored:
+            continue
+        if not _is_copyable_profile_entry(src_path / name):
+            # Chrome can leave generated profile components unreadable when a
+            # previous run used a different runtime uid, a mounted profile came
+            # from the host, or Chrome tightened permissions on cache/model
+            # directories. Those entries are not required to launch a cloned
+            # profile, and letting copytree abort here prevents the crawl from
+            # even creating its first Snapshot.
+            ignored.add(name)
+    return ignored
+
+
 @dataclass(frozen=True)
 class PersonaImportSource:
     kind: str
@@ -378,7 +433,6 @@ def discover_persona_template_profiles(personas_dir: Path | None = None) -> list
         candidate_roots.extend(
             [
                 CONSTANTS.PERSONAS_DIR.expanduser(),
-                Path.home() / ".config" / "abx" / "personas",
             ],
         )
 
@@ -389,13 +443,16 @@ def discover_persona_template_profiles(personas_dir: Path | None = None) -> list
             continue
         seen_roots.add(resolved_root)
 
-        if not resolved_root.exists() or not resolved_root.is_dir():
+        if not _path_is_dir(resolved_root):
             continue
 
-        for persona_dir in sorted((path for path in resolved_root.iterdir() if path.is_dir()), key=lambda path: path.name.lower()):
+        for persona_dir in sorted(
+            (path for path in _iter_children(resolved_root) if _path_is_dir(path)),
+            key=lambda path: path.name.lower(),
+        ):
             for candidate_dir_name in PERSONA_PROFILE_DIR_CANDIDATES:
                 user_data_dir = persona_dir / candidate_dir_name
-                if not user_data_dir.exists() or not user_data_dir.is_dir():
+                if not _path_is_dir(user_data_dir):
                     continue
 
                 for profile_dir in _list_profile_names(user_data_dir):
@@ -589,7 +646,7 @@ def copy_browser_user_data_dir(source_dir: Path, destination_dir: Path) -> None:
         source_dir,
         destination_dir,
         symlinks=True,
-        ignore=shutil.ignore_patterns(*VOLATILE_PROFILE_COPY_PATTERNS),
+        ignore=profile_copy_ignore,
     )
 
 
@@ -612,7 +669,7 @@ def export_browser_state(
     if not state_script.exists():
         return False, None, f"Browser state export script not found at {state_script}"
 
-    node_modules_dir = get_config().LIB_DIR / "pnpm" / "packages" / "chrome" / "node_modules"
+    node_modules_dir = get_config().ABXPKG_LIB_DIR / "pnpm" / "packages" / "chrome" / "node_modules"
     chrome_plugin_dir = Path(get_plugins_dir()).resolve()
 
     env = os.environ.copy()
@@ -702,12 +759,12 @@ def export_browser_state(
 
 
 def _list_profile_names(user_data_dir: Path) -> list[str]:
-    if not user_data_dir.exists() or not user_data_dir.is_dir():
+    if not _path_is_dir(user_data_dir):
         return []
 
     profiles: list[str] = []
-    for child in sorted(user_data_dir.iterdir(), key=lambda path: path.name.lower()):
-        if not child.is_dir():
+    for child in sorted(_iter_children(user_data_dir), key=lambda path: path.name.lower()):
+        if not _path_is_dir(child):
             continue
         if child.name == "System Profile":
             continue
@@ -721,7 +778,7 @@ def _list_profile_names(user_data_dir: Path) -> list[str]:
 
 
 def _looks_like_profile_dir(path: Path) -> bool:
-    if not path.exists() or not path.is_dir():
+    if not _path_is_dir(path):
         return False
 
     marker_paths = (
@@ -733,7 +790,7 @@ def _looks_like_profile_dir(path: Path) -> bool:
         path / "Session Storage",
     )
 
-    if any(marker.exists() for marker in marker_paths):
+    if any(_path_exists(marker) for marker in marker_paths):
         return True
 
     return any(path.name == prefix or path.name.startswith(prefix) for prefix in BROWSER_PROFILE_DIR_NAMES)

archivebox/personas/models.py

@@ -11,14 +11,13 @@ Each persona has its own:
 __package__ = "archivebox.personas"
 
 import shutil
-import subprocess
-import sys
+import os
 from contextlib import contextmanager
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
 from collections.abc import Mapping
 
-from django.db import models
+from django.db import IntegrityError, models
 from django.db.models.fields.json import KT
 from django.conf import settings
 from django.utils import timezone
@@ -62,11 +61,7 @@ VOLATILE_PROFILE_FILE_NAMES = {
 
 def derive_persona_config(*, name: str, config: Mapping[str, Any] | None, persona_dir: Path) -> dict[str, Any]:
     derived = dict(config or {})
-
-    if "CHROME_USER_DATA_DIR" not in derived:
-        derived["CHROME_USER_DATA_DIR"] = str(persona_dir / "chrome_profile")
-    if "CHROME_DOWNLOADS_DIR" not in derived:
-        derived["CHROME_DOWNLOADS_DIR"] = str(persona_dir / "chrome_downloads")
+    derived["PERSONAS_DIR"] = str(persona_dir.parent)
 
     cookies_path = persona_dir / "cookies.txt"
     if "COOKIES_FILE" not in derived and cookies_path.exists():
@@ -165,8 +160,7 @@ class Persona(ModelWithConfig):
 
         Returns dict with:
         - All values from self.config JSONField
-        - CHROME_USER_DATA_DIR (derived from persona path)
-        - CHROME_DOWNLOADS_DIR (derived from persona path)
+        - PERSONAS_DIR (derived from DATA_DIR/personas)
         - COOKIES_FILE (derived from persona path, if file exists)
         - AUTH_STORAGE_FILE (derived from persona path, if file exists)
         - ACTIVE_PERSONA (set to this persona's name)
@@ -186,7 +180,16 @@ class Persona(ModelWithConfig):
         if not profile_dir.exists():
             return False
 
-        for path in profile_dir.rglob("*"):
+        def profile_paths():
+            # Chrome profiles are user-controlled filesystem state. If a cache
+            # or generated component is unreadable, cleanup should skip it and
+            # continue pruning the rest instead of blocking the crawl runner.
+            for dirpath, dirnames, filenames in os.walk(profile_dir, onerror=lambda _err: None):
+                current_dir = Path(dirpath)
+                for name in [*dirnames, *filenames]:
+                    yield current_dir / name
+
+        for path in profile_paths():
             if path.name in VOLATILE_PROFILE_FILE_NAMES:
                 try:
                     path.unlink()
@@ -195,13 +198,21 @@ class Persona(ModelWithConfig):
                     pass
 
         for dirname in VOLATILE_PROFILE_DIR_NAMES:
-            for path in profile_dir.rglob(dirname):
-                if not path.is_dir():
+            for path in profile_paths():
+                if path.name != dirname:
+                    continue
+                try:
+                    is_dir = path.is_dir()
+                except OSError:
+                    is_dir = False
+                if not is_dir:
                     continue
                 shutil.rmtree(path, ignore_errors=True)
                 cleaned = True
 
-        for path in profile_dir.rglob("*.log"):
+        for path in profile_paths():
+            if not path.match("*.log"):
+                continue
             try:
                 path.unlink()
                 cleaned = True
@@ -228,6 +239,18 @@ class Persona(ModelWithConfig):
                 if fcntl is not None:
                     fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
 
+    @classmethod
+    def get_or_create_named(cls, name: str) -> "Persona":
+        persona_name = (name or "Default").strip() or "Default"
+        persona = cls.objects.filter(name=persona_name).first()
+        if persona is not None:
+            return persona
+
+        try:
+            return cls.objects.create(name=persona_name)
+        except IntegrityError:
+            return cls.objects.get(name=persona_name)
+
     def runtime_root_for_crawl(self, crawl) -> Path:
         return Path(crawl.output_dir) / ".persona" / self.name
 
@@ -247,27 +270,16 @@ class Persona(ModelWithConfig):
         return self.runtime_root_for_snapshot(snapshot) / "chrome_downloads"
 
     def copy_chrome_profile(self, source_dir: Path, destination_dir: Path) -> None:
+        from archivebox.personas.importers import profile_copy_ignore
+
         destination_dir.parent.mkdir(parents=True, exist_ok=True)
         shutil.rmtree(destination_dir, ignore_errors=True)
-        destination_dir.mkdir(parents=True, exist_ok=True)
-
-        copy_cmd: list[str] | None = None
-        source_contents = f"{source_dir}/."
-
-        if sys.platform == "darwin":
-            copy_cmd = ["cp", "-cR", source_contents, str(destination_dir)]
-        elif sys.platform.startswith("linux"):
-            copy_cmd = ["cp", "-a", source_contents, str(destination_dir)]
-
-        if copy_cmd:
-            result = subprocess.run(copy_cmd, capture_output=True, text=True)
-            if result.returncode == 0:
-                return
-
-            shutil.rmtree(destination_dir, ignore_errors=True)
-            destination_dir.mkdir(parents=True, exist_ok=True)
-
-        shutil.copytree(source_dir, destination_dir, symlinks=True, dirs_exist_ok=True)
+        shutil.copytree(
+            source_dir,
+            destination_dir,
+            symlinks=True,
+            ignore=profile_copy_ignore,
+        )
 
     def prepare_runtime_for_crawl(self, crawl, chrome_binary: str = "") -> dict[str, str]:
         self.ensure_dirs()
@@ -294,8 +306,13 @@ class Persona(ModelWithConfig):
                 (runtime_root / "chrome_binary.txt").write_text(chrome_binary)
 
         return {
-            "CHROME_USER_DATA_DIR": str(runtime_profile_dir),
-            "CHROME_DOWNLOADS_DIR": str(runtime_downloads_dir),
+            # Hooks derive CHROME_USER_DATA_DIR/CHROME_DOWNLOADS_DIR from
+            # PERSONAS_DIR + ACTIVE_PERSONA. Point PERSONAS_DIR at the
+            # per-crawl runtime root here so CHROME_ISOLATION=crawl never
+            # leaks or reuses the template profile while keeping Chrome path
+            # derivation centralized in the Chrome plugin helpers.
+            "PERSONAS_DIR": str(runtime_root.parent),
+            "ACTIVE_PERSONA": self.name,
         }
 
     def prepare_runtime_for_snapshot(self, snapshot, chrome_binary: str = "") -> dict[str, str]:
@@ -321,8 +338,11 @@ class Persona(ModelWithConfig):
             (runtime_root / "chrome_binary.txt").write_text(chrome_binary)
 
         return {
-            "CHROME_USER_DATA_DIR": str(runtime_profile_dir),
-            "CHROME_DOWNLOADS_DIR": str(runtime_downloads_dir),
+            # See prepare_runtime_for_crawl(): snapshot isolation changes the
+            # persona root, not individual CHROME_* config keys, so standalone
+            # Chrome hooks and ArchiveBox-driven hooks resolve paths the same way.
+            "PERSONAS_DIR": str(runtime_root.parent),
+            "ACTIVE_PERSONA": self.name,
         }
 
     def cleanup_runtime_for_crawl(self, crawl) -> None:
@@ -331,8 +351,7 @@ class Persona(ModelWithConfig):
     @classmethod
     def get_or_create_default(cls) -> "Persona":
         """Get or create the Default persona."""
-        persona, _ = cls.objects.get_or_create(name="Default")
-        return persona
+        return cls.get_or_create_named("Default")
 
     @classmethod
     def cleanup_chrome_all(cls) -> int:

archivebox/plugins/hooks.py

@@ -32,7 +32,7 @@ Execution order:
     - After all foreground hooks complete, background hooks receive SIGTERM and must finalize
 
 Hook naming convention:
-    on_{EventFamily}__{run_order}_{description}[.finite.bg|.daemon.bg].{ext}
+    on_{EventFamily}__{run_order}_{description}[.bg].{ext}
 
 API:
     discover_hooks(event)     -> List[Path]     Find hook scripts for a hook-backed event family
@@ -104,11 +104,6 @@ def is_background_hook(hook_name: str) -> bool:
     return ".bg." in hook_name or "__background" in hook_name
 
 
-def is_finite_background_hook(hook_name: str) -> bool:
-    """Check if a background hook is finite-lived and should be awaited."""
-    return ".finite.bg." in hook_name
-
-
 def normalize_hook_event_name(event_name: str) -> str | None:
     """
     Normalize a hook event family or event class name to its on_* prefix.
@@ -284,15 +279,27 @@ def run_hook(
             records = process.get_records()  # Get parsed JSONL output
     """
     from archivebox.machine.models import Process, Machine, NetworkInterface
-    from archivebox.config.common import get_config, normalize_runtime_config
+    from archivebox.config.common import (
+        ArchiveBoxConfig,
+        _archivebox_config_input_names,
+        get_config,
+        normalize_runtime_config,
+        _plugin_enabled_config_keys,
+    )
 
     config_scope = {key.removeprefix("config_"): kwargs.pop(key) for key in list(kwargs) if key.startswith("config_")}
     config_overrides = _config_to_overrides(config)
+    explicit_override_keys = set(config_overrides)
     resolved_config = get_config(overrides=config_overrides, **config_scope)
     hook_config = normalize_runtime_config(
-        config_overrides if config is not None else resolved_config.for_crawl(),
+        resolved_config.for_crawl_runtime(runtime_overrides=config_overrides),
         json_safe=False,
     )
+    hook_config.update(normalize_runtime_config(config_overrides, json_safe=False))
+    plugin_enabled_keys = set(_plugin_enabled_config_keys().values())
+    if plugin_enabled_keys.intersection(hook_config):
+        for enabled_key in plugin_enabled_keys:
+            hook_config.setdefault(enabled_key, False)
 
     # Auto-detect timeout from plugin config if not explicitly provided
     if timeout is None:
@@ -370,8 +377,13 @@ def run_hook(
 
     # Set up environment with base paths
     env = os.environ.copy()
+    archivebox_config_input_names = _archivebox_config_input_names()
+    for key in archivebox_config_input_names:
+        env.pop(key, None)
+    env.pop("PLUGINS", None)
     env["DATA_DIR"] = str(CONSTANTS.DATA_DIR)
     env["LIBRARY_VERSION"] = VERSION
+    env.setdefault("ABXPKG_FAST_SCRIPT", "1")
     env.setdefault("MACHINE_ID", os.environ.get("MACHINE_ID", CONSTANTS.MACHINE_ID))
     snap_dir = hook_config.get("SNAP_DIR") or _model_output_dir_from_child_path(output_dir, CONSTANTS.SNAPSHOTS_DIR_NAME)
     crawl_dir = hook_config.get("CRAWL_DIR") or _model_output_dir_from_child_path(output_dir, CONSTANTS.CRAWLS_DIR_NAME)
@@ -380,16 +392,17 @@ def run_hook(
     if crawl_dir:
         env["CRAWL_DIR"] = str(crawl_dir)
 
-    # Export runtime library roots; abx-dl/abxpkg own executable lookup env.
-    lib_dir = hook_config.get("LIB_DIR")
+    # Export the runtime library root; abx-dl/abxpkg own executable lookup env.
+    lib_dir = hook_config.get("ABXPKG_LIB_DIR")
     if lib_dir:
-        env["LIB_DIR"] = str(lib_dir)
         env["ABXPKG_LIB_DIR"] = str(lib_dir)
 
     # Set Node.js module resolution paths.
     # NODE_PATH may be a path list, but NODE_MODULES_DIR is a single canonical directory.
     node_modules_dir = hook_config.get("NODE_MODULES_DIR")
-    if not node_modules_dir and lib_dir:
+    if lib_dir and "ABXPKG_LIB_DIR" in explicit_override_keys and "NODE_MODULES_DIR" not in explicit_override_keys:
+        node_modules_dir = Path(lib_dir) / "pnpm" / "packages" / "chrome" / "node_modules"
+    elif not node_modules_dir and lib_dir:
         node_modules_dir = Path(lib_dir) / "pnpm" / "packages" / "chrome" / "node_modules"
 
     node_path_parts = [part for part in str(hook_config.get("NODE_PATH") or "").split(os.pathsep) if part]
@@ -405,12 +418,10 @@ def run_hook(
         env["NODE_PATH"] = os.pathsep.join(node_path_parts)
 
     # Export all config values to environment (already merged by get_config())
-    # Skip keys we've already handled specially above (PATH, LIB_DIR, NODE_PATH, etc.)
+    # Skip keys we've already handled specially above (PATH, ABXPKG_LIB_DIR, NODE_PATH, etc.)
     SKIP_KEYS = {
         "PATH",
-        "LIB_DIR",
         "ABXPKG_LIB_DIR",
-        "LIB_BIN_DIR",
         "NODE_PATH",
         "NODE_MODULES_DIR",
         "NODE_MODULE_DIR",
@@ -419,9 +430,12 @@ def run_hook(
         "SNAP_DIR",
         "CRAWL_DIR",
     }
+    canonical_config_keys = set(ArchiveBoxConfig.model_fields)
     for key, value in hook_config.items():
         if key in SKIP_KEYS:
             continue  # Already handled specially above, don't overwrite
+        if key in archivebox_config_input_names and key not in canonical_config_keys:
+            continue
         if value is None:
             continue
         elif isinstance(value, bool):
@@ -434,8 +448,8 @@ def run_hook(
     # Create output directory if needed
     output_dir.mkdir(parents=True, exist_ok=True)
 
-    # Detect if this is a background hook (long-running daemon)
-    # Background hooks use the .daemon.bg. or .finite.bg. filename convention.
+    # Detect if this is a background hook.
+    # Background hooks use the .bg. filename marker.
     # Old convention: __background in stem (for backwards compatibility)
     is_background = ".bg." in script.name or "__background" in script.stem
 

archivebox/plugins/views.py

@@ -14,13 +14,13 @@ from django.utils.safestring import mark_safe
 from admin_data_views.typing import ItemContext, SectionData, TableContext
 from admin_data_views.utils import ItemLink, render_with_item_view, render_with_table_view
 
+from archivebox.config.common import get_live_config_url
 from archivebox.config.views import get_environment_binary_url, is_superuser
 from archivebox.plugins.discovery import BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR, discover_plugin_configs, iter_plugin_dirs
 
 
 ABX_PLUGINS_DOCS_BASE_URL = "https://archivebox.github.io/abx-plugins/"
 ABX_PLUGINS_GITHUB_BASE_URL = "https://github.com/ArchiveBox/abx-plugins/tree/main/abx_plugins/plugins/"
-LIVE_CONFIG_BASE_URL = "/admin/environment/config/"
 LIVE_PLUGIN_BASE_URL = "/admin/environment/plugins/"
 
 
@@ -73,10 +73,6 @@ def get_plugin_hook_source_url(plugin_name: str, hook_name: str) -> str:
     return f"{ABX_PLUGINS_GITHUB_BASE_URL}{quote(plugin_name)}/{quote(hook_name)}"
 
 
-def get_live_config_url(key: str) -> str:
-    return f"{LIVE_CONFIG_BASE_URL}{quote(key)}/"
-
-
 def get_machine_admin_url() -> str | None:
     from archivebox.machine.models import Machine
 

archivebox/progressmonitor/views.py

@@ -4,6 +4,7 @@ from functools import lru_cache
 from pathlib import Path
 from typing import Literal
 
+from django.conf import settings
 from django.db.models import CharField, Count, Q, Sum
 from django.db.models.functions import Cast
 from django.http import HttpResponse, JsonResponse
@@ -935,27 +936,26 @@ def live_progress_view(request):
         except ImportError:
             return JsonResponse(payload)
     except Exception as e:
-        import traceback
+        error_payload = {
+            "error": str(e),
+            "orchestrator_running": False,
+            "total_workers": 0,
+            "crawls_active": 0,
+            "crawls_queued": 0,
+            "crawls_recent": 0,
+            "snapshots_active": 0,
+            "snapshots_queued": 0,
+            "archiveresults_active": 0,
+            "archiveresults_queued": 0,
+            "downloads_active": 0,
+            "downloads_queued": 0,
+            "indexing_active": 0,
+            "indexing_queued": 0,
+            "active_crawls": [],
+            "server_time": timezone.now().isoformat(),
+        }
+        if settings.DEBUG:
+            import traceback
 
-        return JsonResponse(
-            {
-                "error": str(e),
-                "traceback": traceback.format_exc(),
-                "orchestrator_running": False,
-                "total_workers": 0,
-                "crawls_active": 0,
-                "crawls_queued": 0,
-                "crawls_recent": 0,
-                "snapshots_active": 0,
-                "snapshots_queued": 0,
-                "archiveresults_active": 0,
-                "archiveresults_queued": 0,
-                "downloads_active": 0,
-                "downloads_queued": 0,
-                "indexing_active": 0,
-                "indexing_queued": 0,
-                "active_crawls": [],
-                "server_time": timezone.now().isoformat(),
-            },
-            status=500,
-        )
+            error_payload["traceback"] = traceback.format_exc()
+        return JsonResponse(error_payload, status=500)

archivebox/search/admin.py

@@ -2,6 +2,7 @@ __package__ = "archivebox.search"
 
 from django.contrib import admin
 from django.contrib.admin.views.main import ChangeList
+from django.db.models import Case, IntegerField, Value, When
 
 from archivebox.search.config import (
     get_default_search_mode,
@@ -71,7 +72,13 @@ class SearchResultsAdminMixin(admin.ModelAdmin):
         if queryset.model._meta.label_lower == "core.snapshot" and request.GET.get("_embedded") != "crawl":
             cached_ids = get_cached_admin_search_ids(request)
             if cached_ids is not None:
-                return queryset.filter(pk__in=cached_ids) if cached_ids else queryset.none(), False
+                if not cached_ids:
+                    return queryset.none(), False
+                search_rank = Case(
+                    *(When(pk=snapshot_id, then=Value(index)) for index, snapshot_id in enumerate(cached_ids)),
+                    output_field=IntegerField(),
+                )
+                return queryset.filter(pk__in=cached_ids).annotate(search_rank=search_rank).order_by("search_rank"), False
             return queryset.none(), False
 
         if get_search_mode_base(search_mode, config=request.archivebox_config) == "meta":

archivebox/search/views.py

@@ -149,7 +149,7 @@ def iter_url_prefix_search_ids(prefix: str, queryset):
             yield snapshot_id
 
 
-def iter_admin_meta_search_ids(query, queryset):
+def iter_meta_search_ids(query, queryset):
     """Yield metadata search matches from a filtered Snapshot queryset."""
     seen = set()
     try:
@@ -173,8 +173,7 @@ def iter_admin_meta_search_ids(query, queryset):
         Q(url__icontains=query),
         Q(title__icontains=query),
         Q(tags__name__icontains=query),
-        Q(notes__icontains=query) | Q(crawl__notes__icontains=query) | Q(crawl__label__icontains=query),
-        Q(crawl__created_by__username=query),
+        Q(notes__icontains=query),
     ]
     for wave in waves:
         for pk in queryset.filter(wave).values_list("pk", flat=True).distinct().iterator(chunk_size=500):
@@ -183,6 +182,12 @@ def iter_admin_meta_search_ids(query, queryset):
             seen.add(pk)
             yield pk
 
+    crawl_metadata_wave = Q(crawl__notes__icontains=query) | Q(crawl__label__icontains=query) | Q(crawl__created_by__username=query)
+    if not seen:
+        for pk in queryset.filter(crawl_metadata_wave).values_list("pk", flat=True).distinct().iterator(chunk_size=500):
+            seen.add(pk)
+            yield pk
+
     config_wave = crawl_config_values_search_wave(query)
     if config_wave is not None and not seen:
         for pk in queryset.filter(config_wave).values_list("pk", flat=True).distinct().iterator(chunk_size=500):
@@ -190,30 +195,63 @@ def iter_admin_meta_search_ids(query, queryset):
             yield pk
 
 
-def iter_admin_backend_search_ids(iterator, queryset):
-    """Yield backend search IDs that still match the filtered queryset."""
+def normalize_search_result_id(snapshot_id) -> str | None:
+    """Return a compact Snapshot ID string from a search provider result."""
+    snapshot_id = str(snapshot_id).strip().lower().replace("-", "")
+    if len(snapshot_id) != 32:
+        return None
+    return snapshot_id
+
+
+def iter_filtered_search_result_ids(iterator, queryset, *, flush_max_delay=0.05):
+    """Yield provider IDs that still match the filtered queryset.
+
+    This is the single intersection/dedupe path used for metadata and every
+    search backend. It flushes by elapsed time so sparse providers stream rows
+    as soon as IDs are found instead of waiting for a fixed batch size.
+    """
     batch = []
     seen = set()
+    queued = set()
+    last_flush_at = 0.0
 
     def flush_batch():
-        valid = {str(pk) for pk in queryset.filter(pk__in=batch).values_list("pk", flat=True)}
-        for snapshot_id in batch:
+        nonlocal batch, queued, last_flush_at
+        if not batch:
+            return
+        batch_ids = batch
+        batch = []
+        queued = set()
+        last_flush_at = time.monotonic()
+        valid = {str(pk).replace("-", "") for pk in queryset.filter(pk__in=batch_ids).values_list("pk", flat=True)}
+        for snapshot_id in batch_ids:
             if snapshot_id in valid and snapshot_id not in seen:
                 seen.add(snapshot_id)
                 yield snapshot_id
 
     for snapshot_id in iterator:
-        snapshot_id = str(snapshot_id).strip().lower().replace("-", "")
-        if len(snapshot_id) != 32:
+        snapshot_id = normalize_search_result_id(snapshot_id)
+        if not snapshot_id or snapshot_id in seen or snapshot_id in queued:
             continue
         batch.append(snapshot_id)
-        if len(batch) >= (1 if not seen else 200):
+        queued.add(snapshot_id)
+        if not seen or time.monotonic() - last_flush_at >= flush_max_delay:
             yield from flush_batch()
-            batch = []
     if batch:
         yield from flush_batch()
 
 
+def iter_search_result_ids(query, base_queryset, *, search_mode, config):
+    """Yield filtered Snapshot IDs from the selected search provider."""
+    search_mode_base = get_search_mode_base(search_mode, config=config)
+    provider = (
+        iter_meta_search_ids(query, base_queryset)
+        if search_mode_base == "meta"
+        else iter_query_search_ids(query, search_mode=search_mode, config=config)
+    )
+    yield from iter_filtered_search_result_ids(provider, base_queryset)
+
+
 def snapshot_search_stream_response(query, base_queryset, *, search_mode, config, cache_key, thread_name):
     """Stream Snapshot search progress and cache matching IDs for a list view."""
     if not query:
@@ -224,7 +262,6 @@ def snapshot_search_stream_response(query, base_queryset, *, search_mode, config
         ids = []
         last_sent = 0
         last_sent_at = time.monotonic()
-        stream_batch_size = 3
         stream_max_delay = 0.05
         stream_padding = " " * 4096
         cache.set(cache_key, {"ids": [], "done": False}, SEARCH_RESULT_CACHE_TTL)
@@ -241,8 +278,7 @@ def snapshot_search_stream_response(query, base_queryset, *, search_mode, config
 
         def publish_count(done=False):
             nonlocal last_sent, last_sent_at
-            if done:
-                cache.set(cache_key, {"ids": list(ids), "done": True}, SEARCH_RESULT_CACHE_TTL)
+            cache.set(cache_key, {"ids": list(ids), "done": done}, SEARCH_RESULT_CACHE_TTL)
             last_sent = len(ids)
             last_sent_at = time.monotonic()
             emit(f"{last_sent}{stream_padding}\n")
@@ -250,24 +286,16 @@ def snapshot_search_stream_response(query, base_queryset, *, search_mode, config
         def run_search():
             iterator = None
             try:
-                search_mode_base = get_search_mode_base(search_mode, config=config)
-                iterator = (
-                    iter_admin_meta_search_ids(query, base_queryset)
-                    if search_mode_base == "meta"
-                    else iter_admin_backend_search_ids(
-                        iter_query_search_ids(query, search_mode=search_mode, config=config),
-                        base_queryset,
-                    )
-                )
+                iterator = iter_search_result_ids(query, base_queryset, search_mode=search_mode, config=config)
                 for snapshot_id in iterator:
                     if stop_event.is_set():
                         break
-                    snapshot_id = str(snapshot_id).strip().lower().replace("-", "")
-                    if len(snapshot_id) != 32 or snapshot_id in seen:
+                    snapshot_id = normalize_search_result_id(snapshot_id)
+                    if not snapshot_id or snapshot_id in seen:
                         continue
                     seen.add(snapshot_id)
                     ids.append(snapshot_id)
-                    if len(ids) == 1 or len(ids) - last_sent >= stream_batch_size or time.monotonic() - last_sent_at >= stream_max_delay:
+                    if len(ids) == 1 or time.monotonic() - last_sent_at >= stream_max_delay:
                         publish_count()
                 if not stop_event.is_set() and len(ids) != last_sent:
                     publish_count(done=True)

archivebox/services/archive_result_service.py

@@ -237,12 +237,12 @@ def _should_update_snapshot_title(current_title: str, next_title: str, *, snapsh
     return len(next_title) > len(current)
 
 
-def _has_content_files(output_files: Any) -> bool:
-    return any(Path(path).suffix not in {".log", ".pid", ".sh"} for path in _normalize_output_files(output_files))
-
-
-def _is_signal_interrupted_exit(exit_code: int) -> bool:
-    return exit_code < 0 or (exit_code >= 128 and exit_code != PROCESS_EXIT_SKIPPED)
+def _status_for_process_without_archive_result(event: ProcessCompletedEvent) -> str:
+    if event.exit_code == PROCESS_EXIT_SKIPPED:
+        return "skipped"
+    if event.exit_code != 0:
+        return "failed"
+    return "noresults"
 
 
 def _iter_archiveresult_records(stdout: str) -> list[dict]:
@@ -444,24 +444,14 @@ class ArchiveResultService(BaseService):
                     ).now()
             return
 
-        # TODO: consider moving this fallback derivation into abx-dl itself.
-        # First try both patterns: if the whole abx-dl process crashes, restarting
-        # the snapshot may be enough, but don't guess before validating it.
-        process_interrupted = _is_signal_interrupted_exit(event.exit_code)
-        process_failed = event.exit_code not in (0, PROCESS_EXIT_SKIPPED) and not process_interrupted
+        process_failed = _status_for_process_without_archive_result(event) == "failed"
         with _perf_span("archivebox.ArchiveResultService.on_ProcessCompletedEvent.emit_archive_result_fallback"):
             await event.emit(
                 ArchiveResultEvent(
                     snapshot_id=snapshot_event.snapshot_id,
                     plugin=event.plugin_name,
                     hook_name=event.hook_name,
-                    status=(
-                        "queued"
-                        if process_interrupted
-                        else "failed"
-                        if process_failed
-                        else ("succeeded" if _has_content_files(event.output_files) else "noresult")
-                    ),
+                    status=_status_for_process_without_archive_result(event),
                     output_str=event.stderr if process_failed else "",
                     output_files=event.output_files,
                     start_ts=event.start_ts,

archivebox/services/binary_service.py

@@ -16,21 +16,6 @@ from abxbus import BaseEvent, EventBus
 from abx_dl.services.base import BaseService
 
 
-_LIB_DIR_MANAGED_PROVIDERS = {
-    "bash",
-    "cargo",
-    "deno",
-    "gem",
-    "goget",
-    "chromewebstore",
-    "nix",
-    "npm",
-    "pip",
-    "puppeteer",
-    "uv",
-}
-
-
 class ArchiveBoxDBBinaryCacheBackend:
     """ArchiveBox machine.Binary projection backend for abxpkg BinaryCacheService."""
 
@@ -43,11 +28,62 @@ class ArchiveBoxDBBinaryCacheBackend:
         if not binary_name:
             return None
 
-        existing = await Binary.objects.filter(machine=machine, name=binary_name).afirst()
         persisted_overrides = _persisted_overrides_for_request(request)
         native_overrides = request.overrides or {}
-        cache_invalidated = False
-        if existing and existing.status == Binary.StatusChoices.INSTALLED:
+        requested_provider_names = _provider_names(request.binproviders)
+        await sync_to_async(get_config, thread_sensitive=True)()
+
+        installed_qs = (
+            Binary.objects.filter(machine=machine, name=binary_name, status=Binary.StatusChoices.INSTALLED)
+            .exclude(abspath="")
+            .exclude(abspath__isnull=True)
+            .order_by("-modified_at")
+        )
+        async for installed in installed_qs:
+            installed_path = Path(installed.abspath).expanduser().resolve(strict=False)
+            if not await sync_to_async(installed_path.exists, thread_sensitive=True)():
+                await _mark_binary_queued(installed)
+                continue
+            if persisted_overrides and installed.overrides != persisted_overrides:
+                await _mark_binary_queued(installed)
+                continue
+
+            provider_name = (installed.binprovider or installed.binproviders.split(",", 1)[0]).strip()
+            if provider_name and provider_name not in requested_provider_names:
+                await _mark_binary_queued(installed)
+                continue
+
+            provider = _provider_for_name(provider_name, installed.name, native_overrides)
+            if await sync_to_async(_cached_provider_path_is_stale, thread_sensitive=True)(installed_path, provider, installed.name):
+                await _mark_binary_queued(installed)
+                continue
+
+            binary_env = BinProvider.build_exec_env(providers=[provider], base_env={}) if provider is not None else {}
+            provider_names = _provider_names(installed.binproviders or request.binproviders or "env")
+            return AbxBinary.model_validate(
+                {
+                    "name": request.name,
+                    "description": request.description,
+                    "binproviders": _providers_for_names(provider_names),
+                    "overrides": native_overrides,
+                    "loaded_binprovider": provider,
+                    "loaded_abspath": installed.abspath,
+                    "loaded_version": installed.version or None,
+                    "loaded_sha256": installed.sha256 or None,
+                    "env": binary_env,
+                },
+            )
+
+        existing = await Binary.objects.filter(machine=machine, name=binary_name).order_by("-modified_at").afirst()
+        if existing is None:
+            await Binary.objects.acreate(
+                machine=machine,
+                name=binary_name,
+                binproviders=_binproviders_to_str(request.binproviders),
+                overrides=persisted_overrides,
+                status=Binary.StatusChoices.QUEUED,
+            )
+        else:
             changed = False
             requested_binproviders = _binproviders_to_str(request.binproviders)
             if requested_binproviders and existing.binproviders != requested_binproviders:
@@ -56,75 +92,15 @@ class ArchiveBoxDBBinaryCacheBackend:
             if persisted_overrides and existing.overrides != persisted_overrides:
                 existing.overrides = persisted_overrides
                 changed = True
-            if changed:
+            if existing.status != Binary.StatusChoices.QUEUED:
                 existing.status = Binary.StatusChoices.QUEUED
                 existing.retry_at = None
-                cache_invalidated = True
+                changed = True
+            if changed:
                 await existing.asave(update_fields=["binproviders", "overrides", "status", "retry_at", "modified_at"])
-        elif existing is None:
-            await Binary.objects.acreate(
-                machine=machine,
-                name=binary_name,
-                binproviders=_binproviders_to_str(request.binproviders),
-                overrides=persisted_overrides,
-                status=Binary.StatusChoices.QUEUED,
-            )
-
-        installed = None
-        if not cache_invalidated:
-            installed = (
-                await Binary.objects.filter(machine=machine, name=binary_name, status=Binary.StatusChoices.INSTALLED)
-                .exclude(abspath="")
-                .exclude(abspath__isnull=True)
-                .order_by("-modified_at")
-                .afirst()
-            )
-            if installed is not None and not await sync_to_async(Path(installed.abspath).expanduser().exists, thread_sensitive=True)():
-                installed.status = Binary.StatusChoices.QUEUED
-                installed.retry_at = None
-                await installed.asave(update_fields=["status", "retry_at", "modified_at"])
-                installed = None
-            if installed is not None and persisted_overrides and installed.overrides != persisted_overrides:
-                installed.status = Binary.StatusChoices.QUEUED
-                installed.retry_at = None
-                await installed.asave(update_fields=["status", "retry_at", "modified_at"])
-                installed = None
-        if installed is None:
-            return None
-
-        installed_path = Path(installed.abspath).expanduser().resolve(strict=False)
-        active_lib_dir = (
-            Path(str((await sync_to_async(get_config, thread_sensitive=True)()).get("LIB_DIR", ""))).expanduser().resolve(strict=False)
-        )
-        provider_name = (installed.binprovider or installed.binproviders.split(",", 1)[0]).strip()
-        if active_lib_dir and provider_name in _LIB_DIR_MANAGED_PROVIDERS:
-            try:
-                installed_path.relative_to(active_lib_dir)
-            except ValueError:
-                installed.status = Binary.StatusChoices.QUEUED
-                installed.retry_at = None
-                await installed.asave(update_fields=["status", "retry_at", "modified_at"])
-                return None
-
-        provider = _provider_for_name(provider_name, installed.name, native_overrides)
-        binary_env = BinProvider.build_exec_env(providers=[provider], base_env={}) if provider is not None else {}
-        provider_names = _provider_names(installed.binproviders or request.binproviders or "env")
-        return AbxBinary.model_validate(
-            {
-                "name": request.name,
-                "description": request.description,
-                "binproviders": _providers_for_names(provider_names),
-                "overrides": native_overrides,
-                "loaded_binprovider": provider,
-                "loaded_abspath": installed.abspath,
-                "loaded_version": installed.version or None,
-                "loaded_sha256": installed.sha256 or None,
-                "env": binary_env,
-            },
-        )
+        return None
 
     async def set(self, request: BinaryRequestEvent | None, binary: AbxBinary) -> None:
-        from archivebox.config.common import get_config
         from archivebox.machine.models import Binary, Machine, _canonical_binary_name
 
         machine = await sync_to_async(Machine.current, thread_sensitive=True)()
@@ -160,8 +136,6 @@ class ArchiveBoxDBBinaryCacheBackend:
         await existing.asave(
             update_fields=["abspath", "version", "sha256", "binproviders", "binprovider", "overrides", "status", "retry_at", "modified_at"],
         )
-        lib_bin_dir = await sync_to_async(lambda: get_config().LIB_BIN_DIR, thread_sensitive=True)()
-        await sync_to_async(existing.symlink_to_lib_bin_after_commit, thread_sensitive=True)(lib_bin_dir)
 
     async def invalidate(self, request: BinaryRequestEvent, binary: AbxBinary, reason: str) -> None:
         from archivebox.machine.models import Binary, Machine, _canonical_binary_name
@@ -421,6 +395,25 @@ def _provider_for_name(provider_name: str, binary_name: str, overrides: dict[str
     return provider
 
 
+async def _mark_binary_queued(binary) -> None:
+    from archivebox.machine.models import Binary
+
+    if binary.status == Binary.StatusChoices.QUEUED:
+        return
+    binary.status = Binary.StatusChoices.QUEUED
+    binary.retry_at = None
+    await binary.asave(update_fields=["status", "retry_at", "modified_at"])
+
+
+def _cached_provider_path_is_stale(installed_path: Path, provider: BinProvider | None, binary_name: str) -> bool:
+    if provider is None:
+        return False
+    current_abspath = provider.get_abspath(binary_name, quiet=True, no_cache=True)
+    if not current_abspath:
+        return True
+    return Path(current_abspath).expanduser().resolve(strict=False) != installed_path
+
+
 def _persisted_overrides_for_request(request: BinaryRequestEvent | None) -> dict[str, Any]:
     if request is None:
         return {}

archivebox/services/process_service.py

@@ -31,6 +31,26 @@ def current_network_interface_with_machine():
     return NetworkInterface.objects.select_related("machine").get(id=current_iface.id)
 
 
+def normalize_process_env(env: dict) -> dict:
+    normalized = dict(env or {})
+    raw_plugins = normalized.pop("PLUGINS", "")
+    selected_plugins = {name.strip().lower() for name in str(raw_plugins).split(",") if name.strip()}
+    from archivebox.config.common import ArchiveBoxConfig, _archivebox_config_input_names, is_sensitive_config_key
+
+    allowed_config_keys = ArchiveBoxConfig._crawl_runtime_keys()
+    config_input_names = _archivebox_config_input_names()
+    for key in list(normalized):
+        if is_sensitive_config_key(key) or (key in config_input_names and key not in allowed_config_keys):
+            normalized.pop(key, None)
+    if selected_plugins:
+        from archivebox.config.common import _plugin_enabled_config_keys, _plugins_with_required_plugins
+
+        selected_plugins = _plugins_with_required_plugins(selected_plugins)
+        for plugin_name, enabled_key in _plugin_enabled_config_keys().items():
+            normalized.setdefault(enabled_key, "True" if plugin_name in selected_plugins else "False")
+    return normalized
+
+
 class ProcessService(BaseService):
     LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [
         ProcessStartedEvent,
@@ -74,6 +94,7 @@ class ProcessService(BaseService):
                 started_at=started_at,
             )
         process = await process_query.order_by("-modified_at").afirst()
+        process_env = normalize_process_env(event.env)
         if process is None:
             process = await Process.objects.acreate(
                 machine=iface.machine,
@@ -82,7 +103,7 @@ class ProcessService(BaseService):
                 worker_type=worker_type,
                 pwd=event.output_dir,
                 cmd=[event.hook_path, *event.hook_args],
-                env=event.env,
+                env=process_env,
                 timeout=event.timeout,
                 pid=event.pid or None,
                 url=event.url or None,
@@ -97,7 +118,7 @@ class ProcessService(BaseService):
 
         process.pwd = event.output_dir
         process.cmd = [event.hook_path, *event.hook_args]
-        process.env = event.env
+        process.env = process_env
         process.timeout = event.timeout
         process.pid = event.pid or None
         process.url = event.url or process.url
@@ -172,6 +193,7 @@ class ProcessService(BaseService):
                 started_at=started_at,
             )
         process = await process_query.order_by("-modified_at").afirst()
+        process_env = normalize_process_env(event.env)
         if process is None:
             await Process.objects.acreate(
                 machine=iface.machine,
@@ -180,7 +202,7 @@ class ProcessService(BaseService):
                 worker_type=worker_type,
                 pwd=event.output_dir,
                 cmd=[event.hook_path, *event.hook_args],
-                env=event.env,
+                env=process_env,
                 timeout=event.timeout,
                 pid=event.pid or None,
                 url=event.url or None,
@@ -197,6 +219,7 @@ class ProcessService(BaseService):
             "machine_id": iface.machine_id,
             "iface_id": iface.id,
             "pwd": event.output_dir,
+            "env": process_env,
             "pid": event.pid or process.pid,
             "url": event.url or process.url,
             "process_type": process_type or process.process_type,

archivebox/services/runner.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import asyncio
 import contextvars
+import json
 import os
 import signal
 import shutil
@@ -54,7 +55,12 @@ from abxbus import BaseEvent
 from abxbus.event_bus import EventBus, get_current_event, in_handler_context
 from abxbus.event_handler import EventHandlerAbortedError, EventHandlerCancelledError
 
-from archivebox.config.common import ArchiveBoxBaseConfig, normalize_runtime_config
+from archivebox.config.common import (
+    ArchiveBoxBaseConfig,
+    normalize_runtime_config,
+    _plugin_enabled_config_keys,
+    _plugins_with_required_plugins,
+)
 from archivebox.misc.db import run_db_analyze_batch
 from archivebox.core.shutdown_util import foreground_shutdown_signals, raise_if_shutdown_requested
 from archivebox.search.sonic_daemon import register_sonic_daemon_event_handler
@@ -120,6 +126,10 @@ def _count_selected_hooks(plugins: dict[str, Plugin], selected_plugins: list[str
     return sum(1 for plugin in selected.values() for hook in plugin.hooks if "CrawlSetup" in hook.name or "Snapshot" in hook.name)
 
 
+def _is_nonfatal_setup_hook(plugin_name: str, hook_name: str) -> bool:
+    return plugin_name == "chrome" and hook_name.endswith("_chrome_kill_zombies")
+
+
 def _discover_archivebox_plugins() -> dict[str, Plugin]:
     return discover_plugins(runtime="archivebox")
 
@@ -181,6 +191,9 @@ def ensure_background_runner(*, allow_under_pytest: bool = False) -> bool:
     runner_worker = get_worker(supervisor, "worker_runner") if supervisor else None
     if runner_worker and runner_worker.get("statename") in ("STARTING", "RUNNING"):
         return False
+    if supervisor is not None:
+        start_worker(supervisor, RUNNER_WORKER)
+        return True
 
     machine = Machine.current()
     Process.cleanup_stale_running(machine=machine)
@@ -192,13 +205,6 @@ def ensure_background_runner(*, allow_under_pytest: bool = False) -> bool:
     if any(proc.is_running for proc in running_orchestrators):
         return False
 
-    if supervisor is not None:
-        try:
-            start_worker(supervisor, RUNNER_WORKER)
-            return True
-        except Exception:
-            pass
-
     return False
 
 
@@ -569,7 +575,7 @@ class CrawlRunner:
             current_process.machine = current_iface.machine
             current_process.save(update_fields=["iface", "machine", "modified_at"])
         self.persona = self.crawl.resolve_persona()
-        self.base_config = get_config(crawl=self.crawl)
+        self.base_config = get_config(crawl=self.crawl, overrides=self.config_overrides)
         self.derived_config = dict(Machine.current().config or {})
         self.crawl_output_dir = str(self.crawl.output_dir)
         if self.persona:
@@ -579,7 +585,6 @@ class CrawlRunner:
                     chrome_binary=self.base_config["CHROME_BINARY"],
                 ),
             )
-        self.base_config.update(self.config_overrides)
         if self.selected_plugins is None:
             raw_plugins = str(self.base_config.get("PLUGINS") or "").strip()
             if raw_plugins:
@@ -606,10 +611,74 @@ class CrawlRunner:
             return [str(snapshot.id) for snapshot in pending_snapshots]
         if self.crawl.snapshot_set.exclude(status__in=[Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED]).exists():
             return []
-        created = self.crawl.create_snapshots_from_urls()
-        snapshots = created or list(self.crawl.snapshot_set.filter(depth=0).order_by("created_at"))
+        created = self.create_initial_snapshots()
+        snapshots = created or list(self.crawl.snapshot_set.filter(depth__in=[0, 1]).order_by("depth", "created_at"))
         return [str(snapshot.id) for snapshot in snapshots]
 
+    def create_initial_snapshots(self) -> list:
+        from archivebox.core.models import Snapshot
+
+        if self.crawl.snapshot_set.exists():
+            return []
+
+        # Direct URL crawls (CLI add, REST, crawl create, schedule, ORM) seed
+        # Crawl.urls with either explicit CrawlSeed JSONL or one plain URL per
+        # line. Either shape becomes the input-layer (depth=0) snapshots so the
+        # rest of the system has one parsing convention. Anything else (RSS,
+        # Netscape HTML, JSONL imports, free-form text) falls through to the
+        # synthetic root path below for parser hooks to process.
+        records = []
+        for line in (self.crawl.urls or "").splitlines():
+            raw_line = line.strip()
+            if not raw_line or raw_line.startswith("#"):
+                continue
+            try:
+                record = json.loads(raw_line)
+            except json.JSONDecodeError:
+                if raw_line.startswith(("http://", "https://")):
+                    records.append({"type": "CrawlSeed", "url": raw_line, "depth": 0})
+                    continue
+                records = []
+                break
+            if not isinstance(record, dict) or record.get("type") != "CrawlSeed" or not record.get("url"):
+                records = []
+                break
+            records.append(record)
+
+        if records:
+            created = []
+            for record in records:
+                try:
+                    record["depth"] = int(record.get("depth") or 0)
+                except (TypeError, ValueError):
+                    record["depth"] = 0
+            for depth in sorted({record["depth"] for record in records}):
+                depth_records = [record for record in records if record["depth"] == depth]
+                created.extend(self.crawl.create_discovered_snapshots(None, depth_records, depth=depth))
+            return created
+
+        # Raw stdin/API/UI import text must remain verbatim in Crawl.urls so a
+        # resumed crawl sees the exact same source bytes and cannot reparse an
+        # overwritten temp file. Parser hooks still need the normal Snapshot
+        # lifecycle and SNAP_DIR/staticfile convention, so the runner creates a
+        # single synthetic root only after it has claimed the crawl. This keeps
+        # DB/FS side effects out of request/CLI add paths and lets child URLs be
+        # discovered by the usual parser -> CrawlService -> Snapshot flow.
+        root_snapshot = Snapshot(
+            url=Snapshot.INTERNAL_INPUT_URL,
+            crawl=self.crawl,
+            depth=0,
+            title="stdin.txt",
+            status=Snapshot.StatusChoices.QUEUED,
+            retry_at=timezone.now(),
+        )
+        root_snapshot.set_delete_at_from_config(self.base_config.get("DELETE_AFTER", "0"))
+        root_snapshot.save()
+        staticfile_dir = root_snapshot.output_dir / "staticfile"
+        staticfile_dir.mkdir(parents=True, exist_ok=True)
+        (staticfile_dir / "stdin.txt").write_text(self.crawl.urls, encoding="utf-8")
+        return [root_snapshot]
+
     def finalize_run_state(self) -> None:
         from archivebox.crawls.models import Crawl
         from archivebox.core.models import Snapshot
@@ -704,8 +773,7 @@ class CrawlRunner:
         snapshot = Snapshot.objects.select_related("crawl", "crawl__created_by").get(id=snapshot_id)
         self.crawl = snapshot.crawl
         self.persona = snapshot.crawl.resolve_persona()
-        self.base_config = get_config(crawl=snapshot.crawl, persona=self.persona)
-        self.base_config.update(self.config_overrides)
+        self.base_config = get_config(crawl=snapshot.crawl, persona=self.persona, overrides=self.config_overrides)
         self.crawl_output_dir = str(snapshot.crawl.output_dir)
         runtime_chrome_overrides = {}
         if self.persona:
@@ -721,20 +789,31 @@ class CrawlRunner:
                 crawl_downloads_dir.mkdir(parents=True, exist_ok=True)
                 runtime_chrome_overrides.update(
                     {
-                        "CHROME_USER_DATA_DIR": str(self.persona.runtime_profile_dir_for_crawl(snapshot.crawl)),
-                        "CHROME_DOWNLOADS_DIR": str(crawl_downloads_dir),
+                        "PERSONAS_DIR": str(self.persona.runtime_root_for_crawl(snapshot.crawl).parent),
+                        "ACTIVE_PERSONA": self.persona.name,
                     },
                 )
         snapshot_output_dir = str(snapshot.output_dir)
+        tags = snapshot.tags_str()
         config = self.base_config.for_crawl_runtime(
             crawl=snapshot.crawl,
             snapshot=snapshot,
             persona=self.persona,
             runtime_overrides=runtime_chrome_overrides,
-            extra_context={"snapshot_id": str(snapshot.id), "snapshot_depth": snapshot.depth},
+            extra_context={
+                "snapshot_id": str(snapshot.id),
+                "snapshot_depth": snapshot.depth,
+                "snapshot_url": snapshot.url,
+                "snapshot_title": snapshot.title or "",
+                "snapshot_tags": tags,
+            },
         )
-        tags = snapshot.tags_str()
         normalized_config = normalize_runtime_config(config)
+        configured_plugins = [name.strip().lower() for name in str(normalized_config.get("PLUGINS") or "").split(",") if name.strip()]
+        if configured_plugins:
+            selected_plugin_names = _plugins_with_required_plugins(set(configured_plugins))
+            for plugin_name, enabled_key in _plugin_enabled_config_keys().items():
+                normalized_config.setdefault(enabled_key, plugin_name in selected_plugin_names)
         return {
             "id": str(snapshot.id),
             "url": snapshot.url,
@@ -765,6 +844,17 @@ class CrawlRunner:
         if not discovered_urls:
             return
 
+        if self.crawl.status == self.crawl.StatusChoices.SEALED:
+            # Snapshot completion projectors can observe the root snapshot seal
+            # before the runner has consumed parser urls.jsonl output. A sealed
+            # crawl must not block those freshly discovered child snapshots; the
+            # runner is still inside the same crawl lifecycle and will seal it
+            # again after the discovered queue is empty.
+            await sync_to_async(self.crawl.update_and_requeue, thread_sensitive=True)(
+                status=self.crawl.StatusChoices.STARTED,
+                retry_at=timezone.now(),
+            )
+
         parent_snapshot = await sync_to_async(
             lambda: Snapshot.objects.select_related("crawl", "crawl__created_by").filter(id=snapshot_payload["id"]).first(),
             thread_sensitive=True,
@@ -1005,6 +1095,8 @@ class CrawlRunner:
             await completed_process.wait(timeout=crawl_setup_phase_timeout)
             await completed_process.event_results_list()
             if completed_process.status == "failed":
+                if _is_nonfatal_setup_hook(plugin.name, hook.name):
+                    continue
                 raise RuntimeError(f"Crawl setup hook {plugin.name}:{hook.name} failed")
 
     async def run_snapshot(self, snapshot_id: str, crawl_start_event: CrawlStartEvent | None = None) -> None:
@@ -1016,7 +1108,20 @@ class CrawlRunner:
             if snapshot["status"] == "sealed" and not self.selected_plugins:
                 await sync_to_async(run_snapshot_maintenance, thread_sensitive=True)(snapshot_id)
                 return
-            snapshot_selected_plugins = self.selected_plugins
+            config = normalize_runtime_config(snapshot["config"])
+            snapshot_config_plugins = [name.strip() for name in str(config.get("PLUGINS") or "").split(",") if name.strip()]
+            snapshot_selected_plugins = (
+                self.selected_plugins if self.selected_plugins_from_args else (snapshot_config_plugins or self.selected_plugins)
+            )
+
+            def queued_plugins_selected_by_config(queued_plugins: list[str]) -> list[str]:
+                if not snapshot_selected_plugins:
+                    return queued_plugins
+                expanded_selected_plugins = set(
+                    filter_plugins(self.plugins, snapshot_selected_plugins, include_providers=True).keys(),
+                )
+                return [plugin for plugin in queued_plugins if plugin in expanded_selected_plugins]
+
             selected_hooks_by_plugin = None
             if snapshot["status"] == "started":
                 _reset_count, running_count = await sync_to_async(snapshot["_snapshot"].reset_abandoned_results, thread_sensitive=True)()
@@ -1041,7 +1146,14 @@ class CrawlRunner:
                     )(snapshot["id"])
                     if queued_plugins:
                         if snapshot_selected_plugins:
-                            queued_plugins = [plugin for plugin in queued_plugins if plugin in snapshot_selected_plugins]
+                            original_queued_plugins = queued_plugins
+                            queued_plugins = queued_plugins_selected_by_config(queued_plugins)
+                            disabled_queued_plugins = sorted(set(original_queued_plugins) - set(queued_plugins))
+                            if disabled_queued_plugins:
+                                await sync_to_async(skip_disabled_queued_plugins, thread_sensitive=True)(
+                                    snapshot["id"],
+                                    disabled_queued_plugins,
+                                )
                             selected_hooks_by_plugin = {
                                 plugin: hooks for plugin, hooks in (selected_hooks_by_plugin or {}).items() if plugin in queued_plugins
                             }
@@ -1053,7 +1165,14 @@ class CrawlRunner:
                 )(snapshot["id"])
                 if queued_plugins:
                     if snapshot_selected_plugins:
-                        queued_plugins = [plugin for plugin in queued_plugins if plugin in snapshot_selected_plugins]
+                        original_queued_plugins = queued_plugins
+                        queued_plugins = queued_plugins_selected_by_config(queued_plugins)
+                        disabled_queued_plugins = sorted(set(original_queued_plugins) - set(queued_plugins))
+                        if disabled_queued_plugins:
+                            await sync_to_async(skip_disabled_queued_plugins, thread_sensitive=True)(
+                                snapshot["id"],
+                                disabled_queued_plugins,
+                            )
                         selected_hooks_by_plugin = {
                             plugin: hooks for plugin, hooks in (selected_hooks_by_plugin or {}).items() if plugin in queued_plugins
                         }
@@ -1064,7 +1183,6 @@ class CrawlRunner:
             ):
                 await sync_to_async(self.seal_snapshot_due_to_limit, thread_sensitive=True)(snapshot_id)
                 return
-            config = normalize_runtime_config(snapshot["config"])
             derived_config = normalize_runtime_config(self.derived_config)
             output_dir = Path(snapshot["output_dir"])
             plugins = (
@@ -1083,18 +1201,19 @@ class CrawlRunner:
                     thread_sensitive=True,
                 )(snapshot["id"])
                 if snapshot_selected_plugins and remaining_queued_plugins:
-                    remaining_queued_plugins = [plugin for plugin in remaining_queued_plugins if plugin in snapshot_selected_plugins]
+                    remaining_queued_plugins = queued_plugins_selected_by_config(remaining_queued_plugins)
                 if not remaining_queued_plugins:
-                    await sync_to_async(run_snapshot_maintenance, thread_sensitive=True)(snapshot_id, output_dir=output_dir)
+                    if snapshot["status"] in ("queued", "started"):
+                        await sync_to_async(finalize_completed_snapshot, thread_sensitive=True)(
+                            snapshot_id,
+                            output_dir=output_dir,
+                        )
+                    else:
+                        await sync_to_async(run_snapshot_maintenance, thread_sensitive=True)(snapshot_id, output_dir=output_dir)
                     return
                 snapshot_selected_plugins = remaining_queued_plugins
                 plugins = filter_plugins(self.plugins, snapshot_selected_plugins, include_providers=True)
-                # Queued ArchiveResult rows select plugin work, not a mid-plugin
-                # resume cursor. Rerun the full selected plugin lifecycle so
-                # ordered prerequisite hooks like chrome_tab run before barriers
-                # like chrome_wait, while search_backend_* maintenance remains
-                # targeted to only those selected plugins.
-                selected_hooks_by_plugin = None
+                selected_hooks_by_plugin = include_background_prerequisite_hooks(selected_hooks_by_plugin, plugins)
             abx_snapshot = AbxSnapshot(
                 id=snapshot["id"],
                 url=snapshot["url"],
@@ -1153,17 +1272,36 @@ class CrawlRunner:
                     await sync_to_async(run_snapshot_maintenance, thread_sensitive=True)(snapshot_id, output_dir=output_dir)
                     return
                 await self.enqueue_discovered_snapshots_from_outputs(snapshot)
-                await sync_to_async(
-                    lambda: (
-                        self.crawl.sm.seal()
-                        if self.crawl.status == self.crawl.StatusChoices.STARTED
-                        and not self.crawl.snapshot_set.filter(
-                            status__in=self.crawl.snapshot_set.model.OPEN_STATES,
-                        ).exists()
-                        else None
-                    ),
-                    thread_sensitive=True,
-                )()
+
+                def _seal_when_last_snapshot_finished() -> None:
+                    # run_snapshot replaces self.crawl per-snapshot, so multiple
+                    # concurrent tasks each load a fresh Crawl/SM pointing at the
+                    # same DB row. The "no open snapshots" check is non-atomic
+                    # with sm.seal(), so two tasks racing to finish the last
+                    # snapshot can both pass the guard. The first call drives
+                    # the SM to a final state (engine.running=False); the loser
+                    # then raises TransitionNotAllowed even though current_state
+                    # still reads STARTED off its stale model field. Re-read the
+                    # row right before the call and swallow the race so the
+                    # task that lost the lap doesn't fail the whole snapshot.
+                    from statemachine.exceptions import TransitionNotAllowed
+
+                    crawl = self.crawl
+                    crawl.refresh_from_db(fields=["status"])
+                    if crawl.status != crawl.StatusChoices.STARTED:
+                        return
+                    if crawl.snapshot_set.filter(
+                        status__in=crawl.snapshot_set.model.OPEN_STATES,
+                    ).exists():
+                        return
+                    try:
+                        crawl.sm.seal()
+                    except TransitionNotAllowed:
+                        # Another task sealed it between our refresh and the
+                        # SM call. Idempotent by design.
+                        pass
+
+                await sync_to_async(_seal_when_last_snapshot_finished, thread_sensitive=True)()
             finally:
                 snapshot_service.close()
 
@@ -1173,9 +1311,11 @@ class CrawlRunner:
         snapshot = Snapshot.objects.select_related("crawl", "crawl__created_by").filter(id=snapshot_id).first()
         if snapshot is None or snapshot.status == Snapshot.StatusChoices.SEALED:
             return
-        if snapshot.status == Snapshot.StatusChoices.STARTED:
-            snapshot.sm.seal()
-            return
+        # Limit stops are runner-owned cancellation decisions, not normal
+        # "all ArchiveResults finished" lifecycle seals. Updating the row
+        # directly avoids racing the state machine's in-memory state while
+        # concurrent snapshot tasks are stopping because the crawl-wide limit
+        # has already been reached.
         snapshot.update_and_requeue(
             status=Snapshot.StatusChoices.SEALED,
             retry_at=None,
@@ -1330,6 +1470,18 @@ def queued_plugins_for_snapshot(snapshot_id: str) -> list[str] | None:
     return queued_plugins
 
 
+def config_overrides_for_queued_plugins(selected_plugins: list[str], **overrides: Any) -> dict[str, Any]:
+    config_overrides = dict(overrides)
+    config_overrides["PLUGINS"] = ",".join(selected_plugins)
+    selected_plugin_names = _plugins_with_required_plugins({plugin_name.lower() for plugin_name in selected_plugins})
+    for plugin_name, enabled_key in _plugin_enabled_config_keys().items():
+        config_overrides[enabled_key] = plugin_name in selected_plugin_names
+    for plugin_name in selected_plugins:
+        if plugin_name.startswith("search_backend_"):
+            config_overrides[f"{plugin_name.upper()}_ENABLED"] = True
+    return config_overrides
+
+
 def fail_unavailable_queued_hooks(
     snapshot_id: str,
     selected_hooks_by_plugin: dict[str, set[str] | None],
@@ -1339,11 +1491,14 @@ def fail_unavailable_queued_hooks(
 
     now = timezone.now()
     for plugin_name, selected_hook_names in selected_hooks_by_plugin.items():
-        if selected_hook_names is None or plugin_name not in plugins:
+        if selected_hook_names is None:
             continue
-        available_hook_names = {
-            name for hook in plugins[plugin_name].filter_hooks("Snapshot") for name in (hook.name, Path(hook.name).stem)
-        }
+        if plugin_name in plugins:
+            available_hook_names = {
+                name for hook in plugins[plugin_name].filter_hooks("Snapshot") for name in (hook.name, Path(hook.name).stem)
+            }
+        else:
+            available_hook_names = set()
         missing_hook_names = [hook_name for hook_name in selected_hook_names if hook_name not in available_hook_names]
         if not missing_hook_names:
             continue
@@ -1363,16 +1518,74 @@ def fail_unavailable_queued_hooks(
         )
 
 
+def skip_disabled_queued_plugins(snapshot_id: str, plugin_names: list[str]) -> None:
+    from archivebox.core.models import ArchiveResult
+
+    if not plugin_names:
+        return
+    now = timezone.now()
+    # Queued ArchiveResult rows are durable scheduler state and can outlive a
+    # config change or deploy. If a plugin is no longer selected for this
+    # Snapshot/Crawl, leaving its old row queued keeps the Snapshot STARTED
+    # forever even though there is no runnable work left.
+    ArchiveResult.objects.filter(
+        snapshot_id=snapshot_id,
+        plugin__in=plugin_names,
+        status=ArchiveResult.StatusChoices.QUEUED,
+    ).update(
+        status=ArchiveResult.StatusChoices.SKIPPED,
+        start_ts=now,
+        end_ts=now,
+        output_str="Queued plugin is disabled by this Snapshot/Crawl config",
+    )
+
+
+def include_background_prerequisite_hooks(
+    selected_hooks_by_plugin: dict[str, set[str] | None],
+    plugins: dict[str, Plugin],
+) -> dict[str, set[str] | None]:
+    expanded: dict[str, set[str] | None] = {}
+    for plugin_name, selected_hook_names in selected_hooks_by_plugin.items():
+        if selected_hook_names is None or plugin_name not in plugins:
+            expanded[plugin_name] = selected_hook_names
+            continue
+        plugin_hooks = sorted(plugins[plugin_name].filter_hooks("Snapshot"), key=lambda hook: hook.sort_key)
+        selected_sort_keys = [
+            hook.sort_key for hook in plugin_hooks if hook.name in selected_hook_names or Path(hook.name).stem in selected_hook_names
+        ]
+        if not selected_sort_keys:
+            expanded[plugin_name] = set(selected_hook_names)
+            continue
+        first_selected_sort_key = min(selected_sort_keys)
+        expanded_hook_names = set(selected_hook_names)
+        # Earlier background hooks publish live resources (e.g. Chrome tabs)
+        # needed by later foreground hooks, but completed foreground hooks stay
+        # final and are not rerun during hook-level resume.
+        for hook in plugin_hooks:
+            if hook.is_background and hook.sort_key < first_selected_sort_key:
+                expanded_hook_names.add(hook.name)
+                expanded_hook_names.add(Path(hook.name).stem)
+        expanded[plugin_name] = expanded_hook_names
+    return expanded
+
+
 def snapshot_hooks_for_pending_archiveresults(snapshot) -> list[tuple[str, str]]:
     from archivebox.config.common import get_config
+    from archivebox.core.models import Snapshot
+    from archivebox.plugins.discovery import get_enabled_plugins
 
     config = get_config(crawl=snapshot.crawl, snapshot=snapshot)
-    plugin_names = [name.strip() for name in str(config.PLUGINS or "").split(",") if name.strip()]
+    snapshot_plugin_names = [name.strip() for name in str((snapshot.config or {}).get("PLUGINS") or "").split(",") if name.strip()]
+    crawl_plugin_names = [name.strip() for name in str((snapshot.crawl.config or {}).get("PLUGINS") or "").split(",") if name.strip()]
+    config_plugin_names = [name.strip() for name in str(config.PLUGINS or "").split(",") if name.strip()]
+    plugin_names = snapshot_plugin_names or crawl_plugin_names or config_plugin_names or get_enabled_plugins(config=config)
     plugins = (
         filter_plugins(_discover_archivebox_plugins(), plugin_names, include_providers=True)
         if plugin_names
         else _discover_archivebox_plugins()
     )
+    if snapshot.url == Snapshot.INTERNAL_INPUT_URL:
+        plugins = {name: plugin for name, plugin in plugins.items() if getattr(plugin.config, "x_accepts_internal_input", False)}
     return sorted((plugin.name, hook.name) for plugin in plugins.values() for hook in plugin.filter_hooks("Snapshot"))
 
 
@@ -1543,22 +1756,36 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo
             _runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot)
             # Explicit maintenance, e.g. `archivebox update --index-only`, may
             # need to run search/index hooks for a paused snapshot. That should
-            # not resume the crawl or make unrelated queued work runnable, so
-            # selected_plugins is required and the paused state is restored in
-            # the finally block below.
+            # not resume the crawl or make unrelated queued work runnable. The
+            # queued ArchiveResult rows are the durable maintenance request, so
+            # run that exact plugin set even when the paused crawl's normal
+            # PLUGINS config names a different extractor surface.
             run_crawl(
                 str(snapshot.crawl_id),
                 snapshot_ids=[str(snapshot.id)],
                 selected_plugins=selected_plugins,
                 process_discovered_snapshots_inline=True,
                 interactive_interrupts=interactive_interrupts,
+                config_overrides=config_overrides_for_queued_plugins(selected_plugins),
                 selected_plugins_are_explicit=False,
             )
         finally:
             # Targeted plugin rows can complete while the Snapshot remains
-            # paused. Put retry_at back at MAX so the orchestrator leaves the
-            # paused lifecycle alone until an explicit resume transition.
-            snapshot.restore_paused_scheduler_marker()
+            # paused. Put retry_at back at MAX only after the queued rows are
+            # gone; if a hook was interrupted before projection, keep the
+            # paused row due so the next runner can retry that targeted work
+            # without a user-visible resume transition.
+            if queued_plugins_for_snapshot(str(snapshot.id)):
+                now = timezone.now()
+                type(snapshot).objects.filter(
+                    pk=snapshot.pk,
+                    status=snapshot.StatusChoices.PAUSED,
+                ).update(
+                    retry_at=now,
+                    modified_at=now,
+                )
+            else:
+                snapshot.restore_paused_scheduler_marker()
         return True
     if snapshot.status == Snapshot.StatusChoices.SEALED:
         if not Snapshot.claim_for_worker(snapshot, lock_seconds=lock_seconds):
@@ -1584,6 +1811,7 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo
                 selected_plugins=selected_plugins,
                 process_discovered_snapshots_inline=True,
                 interactive_interrupts=interactive_interrupts,
+                config_overrides=config_overrides_for_queued_plugins(selected_plugins),
                 selected_plugins_are_explicit=False,
             )
             if search_only_plugins:
@@ -1601,6 +1829,14 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo
                         retry_at=None,
                         modified_at=timezone.now(),
                     )
+                else:
+                    type(snapshot).objects.filter(
+                        pk=snapshot.pk,
+                        status=snapshot.StatusChoices.SEALED,
+                    ).update(
+                        retry_at=timezone.now(),
+                        modified_at=timezone.now(),
+                    )
             return True
         if maintenance_ran:
             return True
@@ -1642,7 +1878,14 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo
     run_crawl(
         str(snapshot.crawl_id),
         snapshot_ids=[str(snapshot.id)],
-        selected_plugins=queued_plugins_for_snapshot(str(snapshot.id)),
+        # Do not pass this snapshot's queued plugin set as a crawl-wide runner
+        # filter. Internal import roots intentionally queue only parser hooks,
+        # but any child snapshots discovered from that root must still run the
+        # normal crawl plugin surface. run_snapshot() narrows the current
+        # snapshot from its queued ArchiveResult rows immediately before
+        # execution; leaving the runner unconstrained keeps that narrowing
+        # local to the one snapshot that owns those rows.
+        selected_plugins=None,
         process_discovered_snapshots_inline=True,
         interactive_interrupts=interactive_interrupts,
         selected_plugins_are_explicit=False,
@@ -1673,6 +1916,7 @@ def run_due_binary(binary, *, lock_seconds: int) -> bool:
 async def _run_install(plugin_names: list[str] | None = None) -> None:
     from archivebox.config.common import get_config
     from archivebox.machine.models import Machine
+    from archivebox.plugins.discovery import get_enabled_plugins
 
     plugins = _discover_archivebox_plugins()
     config = get_config(include_machine=False)
@@ -1693,10 +1937,13 @@ async def _run_install(plugin_names: list[str] | None = None) -> None:
     bus_destroyed = False
 
     try:
-        selected_plugins = filter_plugins(plugins, list(plugin_names), include_providers=True) if plugin_names else plugins
+        if plugin_names:
+            selected_plugins = filter_plugins(plugins, list(plugin_names), include_providers=True)
+        else:
+            selected_plugins = filter_plugins(plugins, get_enabled_plugins(config=config), include_providers=True)
         if not selected_plugins:
             return
-        plugins_label = ", ".join(plugin_names) if plugin_names else f"all ({len(plugins)} available)"
+        plugins_label = ", ".join(plugin_names) if plugin_names else f"enabled ({len(selected_plugins)} of {len(plugins)} available)"
         timeout_seconds = config["TIMEOUT"]
         stdout_is_tty = sys.stdout.isatty()
         stderr_is_tty = sys.stderr.isatty()
@@ -1767,7 +2014,7 @@ async def _run_install(plugin_names: list[str] | None = None) -> None:
                         await bus.destroy(clear=False)
                         bus_destroyed = True
             if live_ui is not None:
-                live_ui.print_summary(output_dir=output_dir)
+                live_ui.print_summary(output_dir=output_dir, archive_results=[])
     finally:
         if not bus_destroyed:
             await bus.destroy(clear=False)
@@ -1844,6 +2091,7 @@ def _run_due_queued_plugin_result(
     lock_seconds: int,
     interactive_interrupts: bool,
     runtime_config,
+    batch_size: int = QUEUED_PLUGIN_RESULT_BATCH_SIZE,
 ) -> bool:
     from archivebox.core.models import ArchiveResult, Snapshot
     from django.db.models import Exists, OuterRef
@@ -1861,7 +2109,7 @@ def _run_due_queued_plugin_result(
             status=ArchiveResult.StatusChoices.QUEUED,
             plugin__in=plugin_names,
             snapshot__retry_at__lte=now,
-            snapshot__status=Snapshot.StatusChoices.SEALED,
+            snapshot__status__in=(Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED),
         )
         .filter(**({"snapshot__crawl_id": crawl_id} if crawl_id else {}))
         .values("snapshot_id", "snapshot__crawl_id")[:1]
@@ -1871,6 +2119,15 @@ def _run_due_queued_plugin_result(
         return False
     root_crawl_id = str(first_due_results[0]["snapshot__crawl_id"])
 
+    first_due_snapshot = Snapshot.objects.filter(pk=first_due_results[0]["snapshot_id"]).first()
+    if first_due_snapshot and first_due_snapshot.status == Snapshot.StatusChoices.PAUSED:
+        return run_due_snapshot(
+            first_due_snapshot,
+            lock_seconds=lock_seconds,
+            interactive_interrupts=interactive_interrupts,
+            runtime_config=runtime_config,
+        )
+
     due_snapshots = Snapshot.objects.filter(
         retry_at__lte=now,
         status=Snapshot.StatusChoices.SEALED,
@@ -1885,7 +2142,7 @@ def _run_due_queued_plugin_result(
         # sibling order is irrelevant; the crawl_id/status index can fetch this
         # small local batch directly while EXISTS proves the enabled queued
         # plugin rows via the existing ArchiveResult unique index.
-        due_snapshots.filter(crawl_id=root_crawl_id).order_by()[:QUEUED_PLUGIN_RESULT_BATCH_SIZE],
+        due_snapshots.filter(crawl_id=root_crawl_id).order_by()[:batch_size],
     )
     if not batch_candidates:
         return False
@@ -1918,20 +2175,13 @@ def _run_due_queued_plugin_result(
     if not claimed_snapshot_ids or selected_plugins is None:
         return True
 
-    config_overrides = {
-        "CRAWL_MAX_CONCURRENT_SNAPSHOTS": QUEUED_PLUGIN_RESULT_BATCH_SIZE,
-    }
-    for plugin_name in selected_plugins:
-        if plugin_name.startswith("search_backend_"):
-            config_overrides[f"{plugin_name.upper()}_ENABLED"] = True
-
     run_crawl(
         root_crawl_id,
         snapshot_ids=claimed_snapshot_ids,
         selected_plugins=selected_plugins,
         process_discovered_snapshots_inline=True,
         interactive_interrupts=interactive_interrupts,
-        config_overrides=config_overrides,
+        config_overrides=config_overrides_for_queued_plugins(selected_plugins, CRAWL_MAX_CONCURRENT_SNAPSHOTS=batch_size),
         selected_plugins_are_explicit=False,
     )
     if all(plugin.startswith("search_backend_") for plugin in selected_plugins):
@@ -2026,6 +2276,7 @@ def run_pending_crawls(
     crawl_id: str | None = None,
     maintenance_only: bool = False,
     interactive_interrupts: bool = False,
+    maintenance_batch_size: int = QUEUED_PLUGIN_RESULT_BATCH_SIZE,
 ) -> int:
     from archivebox.config.common import get_config
     from archivebox.crawls.models import Crawl, CrawlSchedule
@@ -2052,7 +2303,7 @@ def run_pending_crawls(
     while True:
         raise_if_shutdown_requested()
         now_monotonic = time.monotonic()
-        if now_monotonic - last_retention_at >= (60.0 if daemon else 1.0):
+        if crawl_id is None and now_monotonic - last_retention_at >= (60.0 if daemon else 1.0):
             for model in (ArchiveResult, Snapshot, Crawl, Process):
                 # Keep the tight scheduler loop anchored on indexed delete_at
                 # columns only. Backfilling missing delete_at values has to read
@@ -2078,6 +2329,7 @@ def run_pending_crawls(
             lock_seconds=60,
             interactive_interrupts=interactive_interrupts,
             runtime_config=runtime_config,
+            batch_size=maintenance_batch_size,
         ):
             continue
 
@@ -2189,6 +2441,7 @@ def run_pending_crawls(
             lock_seconds=60,
             interactive_interrupts=interactive_interrupts,
             runtime_config=runtime_config,
+            batch_size=maintenance_batch_size,
         ):
             continue
 
@@ -2232,7 +2485,7 @@ def run_pending_crawls(
                 continue
 
         now_monotonic = time.monotonic()
-        if now_monotonic - last_retention_repair_at >= (60.0 if daemon else 0.0):
+        if crawl_id is None and now_monotonic - last_retention_repair_at >= (60.0 if daemon else 0.0):
             for model in (ArchiveResult, Snapshot, Crawl, Process):
                 # No runnable work was found on this scheduler pass. This is
                 # the bounded repair point for missing retention deadlines,

archivebox/templates/admin/snapshot_search_stream.html

@@ -40,6 +40,33 @@
       }
     }
 
+    function setChangelistSearchingState(controller) {
+      if (activeSearch !== controller) return;
+      var changelist = document.getElementById("changelist");
+      if (!changelist) return;
+
+      var results = changelist.querySelector(".results");
+      if (!results) {
+        results = document.createElement("div");
+        var toolbar = changelist.querySelector("#toolbar");
+        if (toolbar && toolbar.nextSibling) {
+          toolbar.parentNode.insertBefore(results, toolbar.nextSibling);
+        } else {
+          changelist.appendChild(results);
+        }
+      }
+      results.className = "results search-empty-state";
+      while (results.firstChild) results.removeChild(results.firstChild);
+      var message = document.createElement("p");
+      message.textContent = "Searching matching snapshots...";
+      results.appendChild(message);
+
+      var paginator = changelist.querySelector(".paginator");
+      if (paginator) paginator.hidden = true;
+      var actions = changelist.querySelector(".actions");
+      if (actions) actions.hidden = true;
+    }
+
     function streamSnapshotSearch(eventOrForm) {
       var form = eventOrForm && eventOrForm.target ? eventOrForm.target : eventOrForm;
       if (!form || form.id !== "changelist-search") return;
@@ -55,6 +82,7 @@
       var controller = new AbortController();
       activeSearch = controller;
       setSearchLoading(controller, "Searching...");
+      setChangelistSearchingState(controller);
       var resultVersion = 0;
       var pendingRender = null;
       var renderInFlight = false;

archivebox/templates/admin/snapshots_grid.html

@@ -229,13 +229,14 @@
           <code class="card-url" title="{{obj.url}}">{{obj.url}}</code>
           <div class="card-media">
             <a href="{% snapshot_base_url obj %}/index.html" class="card-thumbnail">
-              {% snapshot_url obj 'screenshot/screenshot.png' as screenshot_plugin_url %}
-              {% snapshot_url obj 'screenshot.png' as screenshot_root_url %}
-              {% snapshot_url obj 'favicon.ico' as favicon_root_url %}
+              {% snapshot_archiveresult_url obj 'screenshot' 'screenshot.png' as screenshot_url %}
+              {% snapshot_archiveresult_url obj 'chrome_extension_screenshot' 'screenshot-1.png' as extension_screenshot_1_url %}
+              {% snapshot_archiveresult_url obj 'chrome_extension_screenshot' 'screenshot.png' as extension_screenshot_url %}
+              {% snapshot_archiveresult_url obj 'favicon' 'favicon.ico' as favicon_url %}
               <img
-                src="{{ screenshot_plugin_url }}"
+                src="{% firstof screenshot_url extension_screenshot_1_url extension_screenshot_url favicon_url '/static/spinner.gif' %}"
                 alt="{{obj.title|default:'Not yet archived...'}}"
-                data-fallbacks="{{ screenshot_root_url }},{{ favicon_root_url }}"
+                data-fallbacks="{{ extension_screenshot_1_url }},{{ extension_screenshot_url }},{{ favicon_url }}"
                 onerror="const fallbacks=(this.dataset.fallbacks || '').split(',').filter(Boolean); if (fallbacks.length) { this.dataset.fallbacks=fallbacks.slice(1).join(','); this.src=fallbacks[0]; } else { this.onerror=null; this.closest('.card-thumbnail').classList.add('missing'); this.src='/static/spinner.gif'; }"
               />
             </a>

archivebox/templates/core/add.html

@@ -48,7 +48,9 @@
                 <div class="crawl-explanation">
                     <p class="crawl-tip" style="user-select: none;">
                         <span style="float: right">
-                            <a href="/admin/agent">💬 Crawl with AI</a> &nbsp; | &nbsp;
+                            {% if user.is_authenticated and user.is_superuser and request.archivebox_config.OPENCODE_ENABLED %}
+                                <a href="/admin/agent">💬 Crawl with AI</a> &nbsp; | &nbsp;
+                            {% endif %}
                             <a href="https://github.com/ArchiveBox/archivebox-browser-extension"><img src="{% static 'chrome_extension_icon.png' %}" alt="Chrome Extension" style="height: 33px; margin-top: -5px;">&nbsp; Get the extension</a>
                         </span>
                         💡 <strong>Tip:</strong> Instantly save a single URL by visiting:

archivebox/templates/core/navigation.html

@@ -6,7 +6,7 @@
     <a href="{% url 'Home' %}">Snapshots</a> |
     <a href="/admin/core/archiveresult/?o=-1">Log</a> |
     <a href="/admin/core/tag/">Tags</a> &nbsp; &nbsp;
-    {% if user.is_authenticated and user.is_superuser %}
+    {% if user.is_authenticated and user.is_superuser and request.archivebox_config.OPENCODE_ENABLED %}
         <a href="/admin/agent">💬 AI</a> |
     {% endif %}
     <a href="{% url 'Docs' %}" target="_blank" rel="noopener noreferrer">Docs</a> |
@@ -27,10 +27,10 @@
     {% elif request.COOKIES.archivebox_admin_logged_in == "1" %}
         {% comment %}
         Authenticated on the admin host but the session cookie is admin-host-
-        scoped (security boundary — public.* must NEVER see the session).
+        scoped (security boundary — web.* must NEVER see the session).
         The hint cookie is the only signal that crosses, so we render the
         logged-out state's `Account` / `Log out` links pointing at admin host
-        so the user can still reach those pages from public.*/web.*.
+        so the user can still reach those pages from web.*.
         {% endcomment %}
         <a href="/admin/password_change/" title="Change your account password">Account</a> /
         <a href="/admin/logout/">{% trans 'Log out' %}</a>

archivebox/templates/core/public_index.html

@@ -656,18 +656,22 @@
                     return url
                 }
 
+                function replaceRegionFromDocument(doc, selector) {
+                    const next = doc.querySelector(selector)
+                    const current = document.querySelector(selector)
+                    if (next && current) current.innerHTML = next.innerHTML
+                }
+
                 async function renderNow() {
                     const url = buildListUrl()
                     const response = await fetch(url, {headers: {'X-Requested-With': 'XMLHttpRequest'}, signal: controller.signal})
                     const html = await response.text()
                     const doc = new DOMParser().parseFromString(html, 'text/html')
-                    const nextList = doc.querySelector('.public-snapshot-list')
-                    const currentList = document.querySelector('.public-snapshot-list')
-                    if (nextList && currentList) {
-                        currentList.replaceWith(nextList)
-                        initPublicSearchMode()
-                        setSearchLoading(controller, resultVersion ? 'Searching... ' + resultVersion + ' found' : 'Searching...')
-                    }
+                    replaceRegionFromDocument(doc, '.public-snapshot-count')
+                    replaceRegionFromDocument(doc, '#table-bookmarks thead th:nth-child(3)')
+                    replaceRegionFromDocument(doc, '#table-bookmarks tbody')
+                    replaceRegionFromDocument(doc, '.public-pagination')
+                    setSearchLoading(controller, resultVersion ? 'Searching... ' + resultVersion + ' found' : 'Searching...')
                 }
 
                 const listUrl = buildListUrl()

archivebox/templates/core/snapshot.html

@@ -1137,9 +1137,9 @@
             <div class="header-top">
                 <div class="header-nav">
                     <div class="header-col header-left" style="line-height: 58px; vertical-align: middle">
-                        {% public_base_url as public_base %}
-                        <a href="{% if public_base %}{{ public_base }}/public/{% else %}/{% endif %}" class="header-archivebox" title="Go to Public Index...">
-                            <img src="{% if public_base %}{{ public_base }}/static/archive.png{% else %}{% static 'archive.png' %}{% endif %}" alt="Archive Icon">
+                        {% web_base_url as web_base %}
+                        <a href="{% if web_base %}{{ web_base }}/public/{% else %}/{% endif %}" class="header-archivebox" title="Go to Public Index...">
+                            <img src="{% if web_base %}{{ web_base }}/static/archive.png{% else %}{% static 'archive.png' %}{% endif %}" alt="Archive Icon">
                             ArchiveBox
                         </a>
                     </div>
@@ -1150,8 +1150,13 @@
                             </a>
                         </div>
                         <div class="header-title-line header-toggle-trigger">
-                            <img src="{% snapshot_url snapshot 'favicon/favicon.ico' %}" onerror="this.onerror=null;this.src=&quot;data:image/svg+xml;utf8,&lt;svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 20 20' fill='none' stroke='rgba(255,255,255,0.65)' stroke-width='1.4'&gt;&lt;circle cx='10' cy='10' r='7.25'/&gt;&lt;ellipse cx='10' cy='10' rx='3.25' ry='7.25'/&gt;&lt;line x1='2.75' y1='10' x2='17.25' y2='10'/&gt;&lt;/svg&gt;&quot;" alt="Favicon" class="favicon"/>
-                            <span class="header-title-text">{{title|truncatechars:120|safe}}</span>
+                            {% snapshot_archiveresult_url snapshot 'favicon' 'favicon.ico' as header_favicon_url %}
+                            {% if header_favicon_url %}
+                                <img src="{{ header_favicon_url }}" onerror="this.onerror=null;this.src=&quot;data:image/svg+xml;utf8,&lt;svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 20 20' fill='none' stroke='rgba(255,255,255,0.65)' stroke-width='1.4'&gt;&lt;circle cx='10' cy='10' r='7.25'/&gt;&lt;ellipse cx='10' cy='10' rx='3.25' ry='7.25'/&gt;&lt;line x1='2.75' y1='10' x2='17.25' y2='10'/&gt;&lt;/svg&gt;&quot;" alt="Favicon" class="favicon"/>
+                            {% else %}
+                                <img src="data:image/svg+xml;utf8,&lt;svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 20 20' fill='none' stroke='rgba(255,255,255,0.65)' stroke-width='1.4'&gt;&lt;circle cx='10' cy='10' r='7.25'/&gt;&lt;ellipse cx='10' cy='10' rx='3.25' ry='7.25'/&gt;&lt;line x1='2.75' y1='10' x2='17.25' y2='10'/&gt;&lt;/svg&gt;" alt="Favicon" class="favicon"/>
+                            {% endif %}
+                            <span class="header-title-text">{{title|truncatechars:120}}</span>
                             {% if title_tags %}
                                 <span class="header-tags">
                                     <span class="tag-pill permission-pill">{{ snapshot_permissions_icon }} {{ snapshot_permissions }}</span>

archivebox/tests/conftest.py

@@ -23,15 +23,23 @@ from collections.abc import Callable
 import psutil
 import pytest
 import requests
-from django.utils import timezone
 
 REPO_ROOT = Path(__file__).resolve().parents[2]
 WORKSPACE_ROOT = REPO_ROOT.parent
 PYTEST_BASETEMP_ROOT = (REPO_ROOT / "tests" / "out").resolve()
-SESSION_DATA_DIR = Path(tempfile.mkdtemp(prefix="archivebox-pytest-session-")).resolve()
+SESSION_DATA_DIR = Path(
+    os.environ.get("ARCHIVEBOX_PYTEST_SESSION_DATA_DIR") or tempfile.mkdtemp(prefix="archivebox-pytest-session-"),
+).resolve()
+
+os.environ["ARCHIVEBOX_PYTEST_SESSION_DATA_DIR"] = str(SESSION_DATA_DIR)
+os.environ["DATA_DIR"] = str(SESSION_DATA_DIR)
 (SESSION_DATA_DIR / "tests").mkdir(parents=True, exist_ok=True)
 os.chdir(SESSION_DATA_DIR)
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "archivebox.core.settings")
+os.environ.pop("ARCHIVE_DIR", None)
+os.environ.pop("USERS_DIR", None)
+os.environ.pop("CRAWL_DIR", None)
+os.environ.pop("SNAP_DIR", None)
 
 
 def _is_repo_path(path: Path) -> bool:
@@ -57,7 +65,7 @@ def _assert_safe_runtime_paths(*, cwd: Path | None = None, env: dict[str, str] |
 
 
 def _test_source_pythonpath() -> str:
-    entries: list[str] = []
+    entries: list[str] = [str(REPO_ROOT.resolve(strict=False))]
     for repo_name in ("abxpkg", "abx-plugins", "abx-dl"):
         for repo_path in (WORKSPACE_ROOT / repo_name, REPO_ROOT / repo_name):
             if repo_path.exists():
@@ -286,6 +294,14 @@ def find_snapshot_dir(data_dir: Path, snapshot_id: str) -> Path | None:
 # =============================================================================
 
 
+def pytest_configure():
+    import django
+    from django.apps import apps
+
+    if not apps.ready:
+        django.setup()
+
+
 @pytest.fixture(autouse=True)
 def isolate_test_runtime(tmp_path, monkeypatch):
     """
@@ -362,19 +378,18 @@ def isolated_data_dir(tmp_path):
 @pytest.fixture
 def hermetic_lib_dir(tmp_path, monkeypatch):
     """
-    Point LIB_DIR at a tmp directory so the test can write fake binaries
+    Point ABXPKG_LIB_DIR at a tmp directory so the test can write fake binaries
     without touching the real ``~/Library/Application Support/abx/lib`` (which
     can contain symlinks to SIP-protected system binaries on macOS).
 
-    Opt-in only: most tests should reuse the cached real LIB_DIR for speed —
+    Opt-in only: most tests should reuse the cached real ABXPKG_LIB_DIR for speed —
     rebuilding from scratch per-test adds ~10× overhead. Use this only when
-    the test synthesizes binary paths or validates LIB_DIR-relative behavior.
+    the test synthesizes binary paths or validates ABXPKG_LIB_DIR-relative behavior.
     """
     import archivebox.machine.models as machine_models
 
     lib_dir = tmp_path / "lib"
     (lib_dir / "bin").mkdir(parents=True, exist_ok=True)
-    monkeypatch.setenv("LIB_DIR", str(lib_dir))
     monkeypatch.setenv("ABXPKG_LIB_DIR", str(lib_dir))
     machine_models._CURRENT_MACHINE = None
     machine_models._CURRENT_PROCESS = None
@@ -489,11 +504,14 @@ def archivebox_daemon_server(initialized_archive, free_tcp_port_factory):
     started: list[tuple[Path, dict[str, str]]] = []
 
     def start(**env_overrides: str):
+        env_config = {
+            "SEARCH_BACKEND_SONIC_HOST_NAME": "127.0.0.1",
+            "SEARCH_BACKEND_SONIC_PORT": str(free_tcp_port_factory()),
+            **{key: str(value) for key, value in env_overrides.items()},
+        }
         env = cli_env(
             live=True,
-            SEARCH_BACKEND_SONIC_HOST_NAME="127.0.0.1",
-            SEARCH_BACKEND_SONIC_PORT=str(free_tcp_port_factory()),
-            **{key: str(value) for key, value in env_overrides.items()},
+            **env_config,
         )
         port = free_tcp_port_factory()
         result = run_archivebox_cmd(
@@ -944,7 +962,6 @@ def run_python_cwd(
 
 API_TEST_HOST = "api.archivebox.localhost:8000"
 ADMIN_TEST_HOST = "admin.archivebox.localhost:8000"
-PUBLIC_TEST_HOST = "public.archivebox.localhost:8000"
 WEB_TEST_HOST = "web.archivebox.localhost:8000"
 
 
@@ -1143,6 +1160,7 @@ def wait_for_http(
 def make_latest_schedule_due(cwd: Path) -> None:
     from archivebox.crawls.models import Crawl, CrawlSchedule
     from archivebox.tests.test_orm_helpers import use_archivebox_db
+    from django.utils import timezone
 
     with use_archivebox_db(cwd):
         schedule = CrawlSchedule.objects.order_by("-created_at").select_related("template").first()

archivebox/tests/test_api_v1_cli_add.py

@@ -1,22 +1,258 @@
 import pytest
+import json
+import time
+from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor
+from threading import Event
 
 from .conftest import (
     api_client_request,
+    cli_env,
+    create_admin_and_token,
+    get_free_port,
     init_archive,
+    live_api_request,
+    start_archivebox_server,
+    stop_server,
 )
+from archivebox.core.models import Snapshot, SnapshotTag
+from archivebox.crawls.models import Crawl
+from archivebox.tests.test_orm_helpers import use_archivebox_db
 
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
+IMPORT_FORMAT_EXPECTATIONS = {
+    "rss": {
+        "url": "https://example.com/",
+        "title": "RSS Example Import",
+        "date": "2024-01-01",
+        "tags": {"rss-tag", "metadata"},
+    },
+    "netscape": {
+        "url": "https://www.iana.org/domains/reserved",
+        "title": "IANA Reserved Domains",
+        "date": "2024-01-02",
+        "tags": {"netscape-tag", "metadata"},
+    },
+    "dom": {
+        "url": "https://www.iana.org/help/example-domains",
+    },
+    "json": {
+        "url": "https://example.com/?archivebox-json-import=1",
+        "title": "JSON Import Example",
+        "date": "2024-01-03",
+        "tags": {"json-tag", "metadata"},
+    },
+    "jsonl": {
+        "url": "https://example.com/?archivebox-jsonl-import=1",
+        "title": "JSONL Import Example",
+        "date": "2024-01-04",
+        "tags": {"jsonl-tag", "metadata"},
+    },
+    "txt": {
+        "url": "https://example.org/",
+    },
+}
+
+
+def write_import_format_files(base_dir: Path) -> dict[str, Path]:
+    files = {
+        "rss": base_dir / "test_rss.xml",
+        "netscape": base_dir / "test_netscape.html",
+        "dom": base_dir / "test_dom.html",
+        "json": base_dir / "test_bookmarks.json",
+        "jsonl": base_dir / "test_bookmarks.jsonl",
+        "txt": base_dir / "test_urls.txt",
+    }
+    files["rss"].write_text(
+        """<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0">
+  <channel>
+    <title>ArchiveBox RSS import fixture</title>
+    <link>https://example.com/</link>
+    <description>ArchiveBox RSS import fixture</description>
+    <item>
+      <title>RSS Example Import</title>
+      <link>https://example.com/</link>
+      <guid>https://example.com/</guid>
+      <pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>
+      <category>rss-tag</category>
+      <category>metadata</category>
+    </item>
+  </channel>
+</rss>
+""",
+        encoding="utf-8",
+    )
+    files["netscape"].write_text(
+        """<!DOCTYPE NETSCAPE-Bookmark-file-1>
+<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=UTF-8">
+<TITLE>Bookmarks</TITLE>
+<H1>Bookmarks</H1>
+<DL><p>
+  <DT><A HREF="https://www.iana.org/domains/reserved" ADD_DATE="1704153600" TAGS="netscape-tag,metadata">IANA Reserved Domains</A>
+</DL><p>
+""",
+        encoding="utf-8",
+    )
+    files["dom"].write_text(
+        """<!doctype html>
+<html>
+  <head><title>DOM import fixture</title></head>
+  <body>
+    <a href="https://www.iana.org/help/example-domains">IANA Example Domains</a>
+  </body>
+</html>
+""",
+        encoding="utf-8",
+    )
+    files["json"].write_text(
+        json.dumps(
+            {
+                "url": "https://example.com/?archivebox-json-import=1",
+                "title": "JSON Import Example",
+                "tags": ["json-tag", "metadata"],
+                "bookmarked_at": "2024-01-03T00:00:00+00:00",
+            },
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+    files["jsonl"].write_text(
+        json.dumps(
+            {
+                "url": "https://example.com/?archivebox-jsonl-import=1",
+                "title": "JSONL Import Example",
+                "tags": "jsonl-tag,metadata",
+                "bookmarked_at": "2024-01-04T00:00:00+00:00",
+            },
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+    files["txt"].write_text(
+        "Plain text import fixture containing https://example.org/ as a real live URL.\n",
+        encoding="utf-8",
+    )
+    return files
+
+
+IMPORT_FORMAT_ENV = {
+    "USE_COLOR": "False",
+    "SHOW_PROGRESS": "False",
+    "PLUGINS": "parse_html_urls,parse_jsonl_urls,parse_netscape_urls,parse_rss_urls,parse_txt_urls,wget,headers",
+    "SAVE_WGET": "True",
+    "SAVE_HEADERS": "True",
+    "USE_CHROME": "False",
+    "URL_ALLOWLIST": r"example\.com|example\.org|iana\.org|www\.iana\.org",
+}
+
+
+def wait_for_import_processing(cwd: Path, expected_urls: set[str], *, timeout: float = 120.0) -> None:
+    import time
+
+    deadline = time.time() + timeout
+    while time.time() < deadline:
+        with use_archivebox_db(cwd):
+            snapshot_started = Snapshot.objects.filter(url__in=expected_urls).exists()
+        if snapshot_started:
+            return
+        time.sleep(1)
+    raise AssertionError("timed out waiting for import crawl processing to start")
+
+
+def wait_for_expected_import_snapshots(
+    cwd: Path,
+    expected_urls: set[str],
+    *,
+    timeout: float = 180.0,
+    expected_tags: set[str] | None = None,
+) -> None:
+    import time
+
+    allowed_statuses = {Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.SEALED}
+    deadline = time.time() + timeout
+    while time.time() < deadline:
+        with use_archivebox_db(cwd):
+            snapshots = list(Snapshot.objects.filter(url__in=expected_urls).values("id", "url", "status"))
+            tag_names_by_snapshot_id = {}
+            if expected_tags and snapshots:
+                for snapshot_id, tag_name in SnapshotTag.objects.filter(
+                    snapshot_id__in=[snapshot["id"] for snapshot in snapshots],
+                ).values_list("snapshot_id", "tag__name"):
+                    tag_names_by_snapshot_id.setdefault(snapshot_id, set()).add(tag_name)
+        counts = {url: 0 for url in expected_urls}
+        bad_statuses = []
+        missing_tags = {}
+        for snapshot in snapshots:
+            counts[snapshot["url"]] += 1
+            if snapshot["status"] not in allowed_statuses:
+                bad_statuses.append((snapshot["url"], snapshot["status"]))
+            if expected_tags:
+                tag_names = tag_names_by_snapshot_id.get(snapshot["id"], set())
+                missing = expected_tags - tag_names
+                if missing:
+                    missing_tags[snapshot["url"]] = missing
+        if all(count == 1 for count in counts.values()) and not bad_statuses and not missing_tags:
+            return
+        time.sleep(1)
+    raise AssertionError(
+        f"timed out waiting for one queued/started/sealed snapshot per URL, got counts={counts}, bad_statuses={bad_statuses}, missing_tags={missing_tags}",
+    )
+
+
+def malicious_add_inputs(tmp_path: Path, *, safe_url: str) -> tuple[list[str], Path]:
+    other_crawl_source = tmp_path / "sources" / "other_crawl_source.txt"
+    other_crawl_source.parent.mkdir(parents=True, exist_ok=True)
+    other_crawl_source.write_text("https://example.com/not-owned-by-this-crawl\n", encoding="utf-8")
+    canary = tmp_path / "archivebox_shell_injection_canary"
+    return (
+        [
+            safe_url,
+            "file:///etc/hosts",
+            "/etc/hosts",
+            "../../../../etc/passwd",
+            f"file://{other_crawl_source}",
+            str(other_crawl_source),
+            f"'; touch {canary}; #",
+            f'" && touch {canary} && echo "',
+            f"$(touch {canary})",
+            f"`touch {canary}`",
+            """<?xml version="1.0"?>
+<!DOCTYPE rss [
+  <!ENTITY localfile SYSTEM "file:///etc/hosts">
+]>
+<rss version="2.0" xmlns:xi="http://www.w3.org/2001/XInclude">
+  <channel>
+    <item><title>&localfile;</title><link>file:///etc/passwd</link></item>
+    <xi:include href="file:///etc/hosts" parse="text"/>
+  </channel>
+</rss>""",
+        ],
+        canary,
+    )
+
+
+def assert_no_file_or_shell_payload_snapshots(cwd: Path, *, canary: Path) -> None:
+    with use_archivebox_db(cwd):
+        snapshots = list(Snapshot.objects.all())
+    assert not canary.exists()
+    assert not [snapshot.url for snapshot in snapshots if str(snapshot.url).startswith("file:")]
+    for forbidden in ("/etc/hosts", "/etc/passwd", "other_crawl_source", "archivebox_shell_injection_canary"):
+        assert not [snapshot.url for snapshot in snapshots if forbidden in str(snapshot.url)]
+
+
 def test_basic_success_case_request(client, tmp_path, api_headers):
     init_archive(tmp_path)
+    submitted_url = "https://example.com/api-cli-add-basic"
 
     response = api_client_request(
         client,
         "post",
         "/api/v1/cli/add",
         payload={
-            "urls": ["https://example.com/api-cli-add-basic"],
+            "urls": [submitted_url],
             "depth": 0,
             "parser": "url_list",
             "plugins": "__archivebox_test_no_plugins__",
@@ -27,3 +263,206 @@ def test_basic_success_case_request(client, tmp_path, api_headers):
 
     assert response.status_code == 200, response.content
     assert response.json()["success"] is True
+    crawl = Crawl.objects.get()
+    assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": submitted_url, "depth": 0}
+    assert Snapshot.objects.count() == 0
+
+
+@pytest.mark.timeout(180)
+def test_api_cli_add_concurrent_first_time_default_persona_creation(tmp_path):
+    """Concurrent live API add requests should share one first-created Default persona."""
+    init_archive(tmp_path)
+    with use_archivebox_db(tmp_path):
+        from archivebox.personas.models import Persona
+
+        Persona.objects.filter(name="Default").delete()
+        assert Persona.objects.filter(name="Default").count() == 0
+
+    port = get_free_port()
+    env = cli_env(port=port, server=True, USE_COLOR="False", SHOW_PROGRESS="False")
+    api_token = create_admin_and_token(tmp_path)
+    submitted_urls = [f"https://example.com/api-cli-add-concurrent-persona-{idx}" for idx in range(4)]
+    start = Event()
+
+    def post_add(url: str):
+        start.wait(timeout=10)
+        return live_api_request(
+            port,
+            "post",
+            "/api/v1/cli/add",
+            api_token=api_token,
+            timeout=60,
+            json={
+                "urls": [url],
+                "depth": 0,
+                "parser": "url_list",
+                "plugins": "__archivebox_test_no_plugins__",
+                "index_only": True,
+            },
+        )
+
+    try:
+        start_archivebox_server(tmp_path, env=env, port=port)
+        with ThreadPoolExecutor(max_workers=len(submitted_urls)) as pool:
+            futures = [pool.submit(post_add, url) for url in submitted_urls]
+            start.set()
+            responses = [future.result(timeout=75) for future in futures]
+    finally:
+        stop_server(tmp_path)
+
+    assert [response.status_code for response in responses] == [200] * len(responses), [response.text[:500] for response in responses]
+    bodies = [response.json() for response in responses]
+    assert all(body["success"] is True for body in bodies)
+    assert {body["result"]["queued_urls"][0] for body in bodies} == set(submitted_urls)
+
+    with use_archivebox_db(tmp_path):
+        from archivebox.personas.models import Persona
+
+        assert Persona.objects.filter(name="Default").count() == 1
+        crawls = list(Crawl.objects.order_by("urls").values_list("urls", flat=True))
+        assert Snapshot.objects.count() == 0
+
+    expected_crawl_sources = sorted(
+        json.dumps({"type": "CrawlSeed", "url": url, "depth": 0}, separators=(",", ":")) for url in submitted_urls
+    )
+    assert crawls == expected_crawl_sources
+
+
+@pytest.mark.timeout(360)
+def test_api_cli_add_import_text_formats_preserve_metadata_and_crawl_inner_urls(tmp_path):
+    """REST API add should accept rich import text and queue real inner URLs with metadata preserved."""
+    init_archive(tmp_path)
+    import_files = write_import_format_files(tmp_path)
+    expected_urls = {case["url"] for case in IMPORT_FORMAT_EXPECTATIONS.values()}
+    port = get_free_port()
+    env = cli_env(port=port, server=True, **IMPORT_FORMAT_ENV)
+    api_token = create_admin_and_token(tmp_path)
+
+    try:
+        start_archivebox_server(tmp_path, env=env, port=port)
+        for import_name, import_path in import_files.items():
+            response = live_api_request(
+                port,
+                "post",
+                "/api/v1/cli/add",
+                api_token=api_token,
+                json={
+                    "urls": [import_path.read_text(encoding="utf-8")],
+                    "depth": 0,
+                    "tag": "api-import",
+                    "plugins": IMPORT_FORMAT_ENV["PLUGINS"],
+                    "index_only": False,
+                },
+            )
+            assert response.status_code == 200, response.text
+            body = response.json()
+            assert body["success"] is True
+            assert body["result"]["crawl_id"]
+            with use_archivebox_db(tmp_path):
+                crawl = Crawl.objects.get(id=body["result"]["crawl_id"])
+            source_text = import_path.read_text(encoding="utf-8")
+            assert crawl.urls == source_text
+
+        deadline = time.time() + 240
+        root_counts = {}
+        while time.time() < deadline:
+            with use_archivebox_db(tmp_path):
+                root_counts = {
+                    str(crawl.id): crawl.snapshot_set.filter(url=Snapshot.INTERNAL_INPUT_URL).count() for crawl in Crawl.objects.all()
+                }
+            if root_counts and all(count == 1 for count in root_counts.values()):
+                break
+            time.sleep(1)
+        assert root_counts and all(count == 1 for count in root_counts.values()), root_counts
+        wait_for_import_processing(tmp_path, expected_urls)
+        with use_archivebox_db(tmp_path):
+            for crawl in Crawl.objects.all():
+                root_snapshot = crawl.snapshot_set.get(url=Snapshot.INTERNAL_INPUT_URL)
+                root_input = (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8")
+                assert root_input == crawl.urls
+        stop_server(tmp_path)
+        start_archivebox_server(tmp_path, env=env, port=port)
+        wait_for_expected_import_snapshots(tmp_path, expected_urls)
+
+        for import_name, expected in IMPORT_FORMAT_EXPECTATIONS.items():
+            with use_archivebox_db(tmp_path):
+                snapshot = Snapshot.objects.filter(url=expected["url"]).order_by("-created_at").first()
+                assert snapshot is not None, f"{import_name} did not create Snapshot for {expected['url']}"
+                snapshot_id = str(snapshot.id)
+
+            snapshot_response = live_api_request(
+                port,
+                "get",
+                f"/api/v1/core/snapshot/{snapshot_id}",
+                api_token=api_token,
+            )
+            assert snapshot_response.status_code == 200, snapshot_response.text
+            assert snapshot_response.json()["url"] == expected["url"]
+    finally:
+        stop_server(tmp_path)
+
+    with use_archivebox_db(tmp_path):
+        crawls = list(Crawl.objects.order_by("created_at"))
+        snapshots_by_url = {snapshot.url: snapshot for snapshot in Snapshot.objects.prefetch_related("tags").filter(url__in=expected_urls)}
+        tags_by_url = {snapshot.url: set(snapshot.tags.values_list("name", flat=True)) for snapshot in snapshots_by_url.values()}
+
+    assert len(crawls) == len(import_files)
+    assert [crawl.urls for crawl in crawls] == [path.read_text(encoding="utf-8") for path in import_files.values()]
+    assert all(crawl.tags_str == "api-import" for crawl in crawls)
+    assert all(crawl.status in {Crawl.StatusChoices.STARTED, Crawl.StatusChoices.SEALED} for crawl in crawls)
+    assert len(snapshots_by_url) == len(expected_urls)
+
+    for import_name, expected in IMPORT_FORMAT_EXPECTATIONS.items():
+        snapshot = snapshots_by_url.get(expected["url"])
+        assert snapshot is not None, f"{import_name} did not create Snapshot for {expected['url']}"
+        assert snapshot.status in {Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.SEALED}
+        if expected.get("title"):
+            assert snapshot.title == expected["title"]
+        if expected.get("date"):
+            assert snapshot.bookmarked_at.date().isoformat() == expected["date"]
+        if expected.get("tags"):
+            assert expected["tags"] | {"api-import"} <= tags_by_url[snapshot.url]
+
+
+@pytest.mark.timeout(240)
+def test_api_cli_add_rejects_file_path_and_shell_injection_payloads(tmp_path):
+    """REST add must not let path, file://, traversal, or shell strings become archiveable URLs."""
+    init_archive(tmp_path)
+    safe_url = "https://example.com/?archivebox-api-security=1"
+    inputs, canary = malicious_add_inputs(tmp_path, safe_url=safe_url)
+    port = get_free_port()
+    env = cli_env(port=port, server=True, **IMPORT_FORMAT_ENV)
+    api_token = create_admin_and_token(tmp_path)
+
+    try:
+        start_archivebox_server(tmp_path, env=env, port=port)
+        response = live_api_request(
+            port,
+            "post",
+            "/api/v1/cli/add",
+            api_token=api_token,
+            json={
+                "urls": inputs,
+                "depth": 0,
+                "tag": "api-security",
+                "plugins": IMPORT_FORMAT_ENV["PLUGINS"],
+                "index_only": False,
+            },
+        )
+        assert response.status_code == 200, response.text
+        assert response.json()["success"] is True
+
+        wait_for_expected_import_snapshots(tmp_path, {safe_url}, timeout=120)
+    finally:
+        stop_server(tmp_path)
+
+    wait_for_expected_import_snapshots(tmp_path, {safe_url}, timeout=30, expected_tags={"api-security"})
+    assert_no_file_or_shell_payload_snapshots(tmp_path, canary=canary)
+    with use_archivebox_db(tmp_path):
+        snapshot = Snapshot.objects.get(url=safe_url)
+        crawl = Crawl.objects.get()
+    assert crawl.status in {Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED, Crawl.StatusChoices.SEALED}
+    assert snapshot.status in {Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.SEALED}
+    with use_archivebox_db(tmp_path):
+        tag_names = set(SnapshotTag.objects.filter(snapshot=snapshot).values_list("tag__name", flat=True))
+    assert "api-security" in tag_names

archivebox/tests/test_api_v1_cli_workflow_add_search_update_remove.py

@@ -42,7 +42,7 @@ def test_cli_api_add_search_update_remove_over_server(tmp_path):
                 "tag": "api-cli",
                 "depth": 0,
                 "parser": "url_list",
-                "plugins": "wget",
+                "plugins": "parse_txt_urls,wget",
                 "update": True,
                 "overwrite": False,
                 "index_only": True,

archivebox/tests/test_api_v1_core_tag_tag_id_snapshots_jsonl.py

@@ -1,3 +1,5 @@
+import json
+
 import pytest
 
 from archivebox.core.models import Snapshot, Tag
@@ -31,6 +33,12 @@ def test_tag_snapshots_export_returns_jsonl(client, api_token, tagged_data):
     assert response.status_code == 200
     assert response["Content-Type"].startswith("application/x-ndjson")
     assert f"tag-{tag.slug}-snapshots.jsonl" in response["Content-Disposition"]
-    body = response.content.decode()
-    assert '"type": "Snapshot"' in body
-    assert '"tags": "Alpha Research"' in body
+    rows = [json.loads(line) for line in response.content.decode().splitlines()]
+    rows_by_url = {row["url"]: row for row in rows}
+    assert set(rows_by_url) == {"https://example.com/one", "https://example.com/two"}
+    assert rows_by_url["https://example.com/one"]["type"] == "Snapshot"
+    assert rows_by_url["https://example.com/one"]["title"] == "Example One"
+    assert rows_by_url["https://example.com/two"]["type"] == "Snapshot"
+    assert rows_by_url["https://example.com/two"]["title"] == "Example Two"
+    for row in rows:
+        assert "Alpha Research" in row["tags"].split(",")

archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py

@@ -475,8 +475,17 @@ def test_update_index_only_runs_paused_search_rows_and_resume_later_runs_crawl(t
 
     if snapshot_finished_before_pause:
         indexed_state = get_crawl_runtime_state(tmp_path, crawl_id)
-        assert indexed_state["crawl_status"] == "paused"
         assert indexed_state["snapshots"][0]["status"] == "sealed"
+        wget_results = [result for result in indexed_state["results"] if result["plugin"] == "wget"]
+        assert any(result["status"] == "succeeded" and result["output_size"] > 0 for result in wget_results)
+        if indexed_state["crawl_status"] == "paused":
+            assert indexed_state["crawl_retry_at"] == indexed_state["retry_at_max"]
+        else:
+            assert indexed_state["crawl_status"] == "sealed"
+            assert all(result["status"] not in {"queued", "started", "paused"} for result in indexed_state["results"])
+        captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=60)
+        assert "Root" in captured_text
+        assert "About" in captured_text
         return
 
     update_env = cli_env(

archivebox/tests/test_archive_result_service.py

@@ -3,7 +3,7 @@ import pytest
 
 
 from abxpkg.binary_service import BinaryRequestEvent
-from abx_dl.events import ArchiveResultEvent, ProcessEvent, ProcessStartedEvent
+from abx_dl.events import ArchiveResultEvent, ProcessCompletedEvent, ProcessEvent, ProcessStartedEvent, SnapshotEvent
 from abx_dl.orchestrator import create_bus
 from abx_dl.output_files import OutputFile
 
@@ -331,6 +331,55 @@ def test_process_completed_projects_noresults_archiveresult():
     assert result.output_str == "No title found"
 
 
+def test_process_completed_without_archive_result_does_not_infer_success_from_output_files(snapshot):
+    from archivebox.core.models import ArchiveResult
+    from archivebox.services.archive_result_service import ArchiveResultService
+    import asyncio
+
+    plugin_dir = Path(snapshot.output_dir) / "wget"
+    plugin_dir.mkdir(parents=True, exist_ok=True)
+    (plugin_dir / "index.html").write_text("<html>downloaded but not reported</html>")
+
+    bus = create_bus(name="test_process_completed_without_archive_result_output_files")
+    ArchiveResultService(bus)
+
+    snapshot_event = SnapshotEvent(
+        url=snapshot.url,
+        snapshot_id=str(snapshot.id),
+        output_dir=str(snapshot.output_dir),
+    )
+    completed_event = ProcessCompletedEvent(
+        plugin_name="wget",
+        hook_name="on_Snapshot__06_wget.finite.bg",
+        hook_path="/usr/bin/env",
+        hook_args=[],
+        env={},
+        timeout=60,
+        stdout="",
+        stderr="",
+        exit_code=0,
+        status="succeeded",
+        output_dir=str(plugin_dir),
+        output_files=[OutputFile(path="index.html", extension="html", mimetype="text/html", size=36)],
+        start_ts="2026-03-22T12:00:00+00:00",
+        end_ts="2026-03-22T12:00:01+00:00",
+        event_parent_id=snapshot_event.event_id,
+    )
+
+    async def emit_events() -> None:
+        await bus.emit(snapshot_event).now()
+        await bus.emit(completed_event).now()
+        await bus.wait_until_idle()
+
+    asyncio.run(emit_events())
+
+    result = ArchiveResult.objects.get(snapshot=snapshot, plugin="wget", hook_name="on_Snapshot__06_wget.finite.bg")
+    assert result.status == ArchiveResult.StatusChoices.NORESULTS
+    assert result.output_str == ""
+    assert result.output_files == {"index.html": {"extension": "html", "mimetype": "text/html", "size": 36}}
+    _cleanup_machine_process_rows()
+
+
 def test_retry_failed_archiveresults_requeues_snapshot_in_queued_state():
     from archivebox.core.models import ArchiveResult, Snapshot
 
@@ -679,15 +728,18 @@ def test_process_started_uses_node_binary_for_js_hooks_without_plugin_binary(tmp
 def test_binary_event_reuses_existing_installed_binary_row():
     from archivebox.machine.models import Binary, Machine
     from archivebox.services.binary_service import ArchiveBoxDBBinaryCacheBackend
+    from abxpkg import PROVIDER_CLASS_BY_NAME
     from abxpkg.binary_service import BinaryCacheService, BinaryService
     import asyncio
 
     machine = Machine.current()
+    wget_path = PROVIDER_CLASS_BY_NAME["env"]().get_abspath("wget", quiet=True, no_cache=True)
+    assert wget_path
 
     binary = Binary.objects.create(
         machine=machine,
         name="wget",
-        abspath="/bin/sh",
+        abspath=str(wget_path),
         version="9.9.9",
         binprovider="env",
         binproviders="env,apt,brew",
@@ -715,7 +767,7 @@ def test_binary_event_reuses_existing_installed_binary_row():
     binary.refresh_from_db()
     assert Binary.objects.filter(machine=machine, name="wget").count() == 1
     assert binary.status == Binary.StatusChoices.INSTALLED
-    assert binary.abspath == "/bin/sh"
+    assert binary.abspath == str(wget_path)
     assert binary.version == "9.9.9"
     assert binary.binprovider == "env"
     assert binary.binproviders == "env,apt,brew"

archivebox/tests/test_binary_service.py

@@ -24,13 +24,12 @@ def _link_real_binary(bin_dir: Path, name: str, *, source: str | None = None) ->
     return link
 
 
-def _runtime_env(data_dir: Path, bin_dir: Path) -> dict[str, str]:
+def _runtime_env(data_dir: Path, bin_dir: Path, *, lib_dir: Path | None = None) -> dict[str, str]:
     archivebox_bin = shutil.which("archivebox")
     assert archivebox_bin, "archivebox console script must be available for CLI tests"
+    lib_dir = lib_dir or data_dir / "lib"
     return {
-        "LIB_DIR": str(data_dir / "lib"),
-        "LIB_BIN_DIR": str(data_dir / "lib" / "bin"),
-        "ABXPKG_LIB_DIR": str(data_dir / "lib"),
+        "ABXPKG_LIB_DIR": str(lib_dir),
         "PATH": os.pathsep.join([str(bin_dir), str(Path(archivebox_bin).parent), "/usr/bin", "/bin", "/usr/sbin", "/sbin"]),
     }
 
@@ -129,7 +128,6 @@ def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized
     assert first_abspath.resolve() == Path(shutil.which("rg") or "").resolve()
     assert first_abspath.is_relative_to(initialized_archive / "lib")
     assert (initialized_archive / "lib" / "env" / "bin" / name).exists()
-    assert (initialized_archive / "lib" / "bin" / name).exists()
     assert (initialized_archive / "machines" / machine_id / "binaries" / name / "index.jsonl").exists()
     assert binary_processes
     assert binary_processes[-1].status == Process.StatusChoices.EXITED
@@ -152,7 +150,6 @@ def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized
     assert binary.version in version_stdout
 
     first_abspath.unlink()
-    (initialized_archive / "lib" / "bin" / name).unlink(missing_ok=True)
     _link_real_binary(bootstrap_bin_dir, name, source="rg")
 
     _cmd_result = run_archivebox_cmd(
@@ -176,6 +173,31 @@ def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized
     assert Path(recovered.abspath).resolve() == Path(shutil.which("rg") or "").resolve()
     assert process_count >= 2
 
+    changed_lib_dir = tmp_path / "changed-lib"
+    changed_provider_bin_dir = changed_lib_dir / "env" / "bin"
+    _link_real_binary(changed_provider_bin_dir, name, source="rg")
+
+    _cmd_result = run_archivebox_cmd(
+        ["run"],
+        cwd=initialized_archive,
+        stdin=json.dumps({"type": "BinaryRequest", "name": name, "binproviders": "env"}) + "\n",
+        timeout=120,
+        env=_runtime_env(initialized_archive, bootstrap_bin_dir, lib_dir=changed_lib_dir),
+        default_cli_env=True,
+        disable_extractors=True,
+    )
+    relib_stdout, relib_stderr, relib_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
+
+    assert relib_code == 0, relib_stdout + relib_stderr
+    with use_archivebox_db(initialized_archive):
+        relibbed = Binary.objects.get(pk=first_binary_id)
+
+    assert relibbed.status == Binary.StatusChoices.INSTALLED
+    assert relibbed.version == binary.version
+    assert Path(relibbed.abspath) == changed_provider_bin_dir / name
+    assert Path(relibbed.abspath).exists()
+    assert Path(relibbed.abspath).resolve() == Path(shutil.which("rg") or "").resolve()
+
 
 def test_missing_binary_request_stays_queued_then_recovers_when_provider_can_resolve(initialized_archive, tmp_path):
     name = f"abx-missing-rg-{uuid.uuid4().hex[:8]}"

archivebox/tests/test_cli_add.py

@@ -6,19 +6,238 @@ Verify add creates snapshots in DB, crawls, source files, and archive directorie
 
 import os
 import json
+import time
+from pathlib import Path
 
 import pytest
+from django.utils import timezone
 
-from archivebox.core.models import ArchiveResult, Snapshot
+from archivebox.core.models import ArchiveResult, Snapshot, SnapshotTag
 from archivebox.crawls.models import Crawl
 from archivebox.machine.models import Process
-from archivebox.tests.conftest import _find_system_browser, find_snapshot_dir, run_archivebox_cmd, run_queued_crawls, cli_env
+from archivebox.tests.conftest import (
+    _find_system_browser,
+    cli_env,
+    find_snapshot_dir,
+    get_free_port,
+    run_archivebox_cmd,
+    run_queued_crawls,
+    start_archivebox_server,
+    stop_server,
+)
 
 from archivebox.tests.test_orm_helpers import use_archivebox_db
 
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
+IMPORT_FORMAT_EXPECTATIONS = {
+    "rss": {
+        "url": "https://example.com/",
+        "title": "RSS Example Import",
+        "date": "2024-01-01",
+        "tags": {"rss-tag", "metadata"},
+    },
+    "netscape": {
+        "url": "https://www.iana.org/domains/reserved",
+        "title": "IANA Reserved Domains",
+        "date": "2024-01-02",
+        "tags": {"netscape-tag", "metadata"},
+    },
+    "dom": {
+        "url": "https://www.iana.org/help/example-domains",
+    },
+    "json": {
+        "url": "https://example.com/?archivebox-json-import=1",
+        "title": "JSON Import Example",
+        "date": "2024-01-03",
+        "tags": {"json-tag", "metadata"},
+    },
+    "jsonl": {
+        "url": "https://example.com/?archivebox-jsonl-import=1",
+        "title": "JSONL Import Example",
+        "date": "2024-01-04",
+        "tags": {"jsonl-tag", "metadata"},
+    },
+    "txt": {
+        "url": "https://example.org/",
+    },
+}
+
+
+def write_import_format_files(base_dir: Path) -> dict[str, Path]:
+    files = {
+        "rss": base_dir / "test_rss.xml",
+        "netscape": base_dir / "test_netscape.html",
+        "dom": base_dir / "test_dom.html",
+        "json": base_dir / "test_bookmarks.json",
+        "jsonl": base_dir / "test_bookmarks.jsonl",
+        "txt": base_dir / "test_urls.txt",
+    }
+    files["rss"].write_text(
+        """<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0">
+  <channel>
+    <title>ArchiveBox RSS import fixture</title>
+    <link>https://example.com/</link>
+    <description>ArchiveBox RSS import fixture</description>
+    <item>
+      <title>RSS Example Import</title>
+      <link>https://example.com/</link>
+      <guid>https://example.com/</guid>
+      <pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>
+      <category>rss-tag</category>
+      <category>metadata</category>
+    </item>
+  </channel>
+</rss>
+""",
+        encoding="utf-8",
+    )
+    files["netscape"].write_text(
+        """<!DOCTYPE NETSCAPE-Bookmark-file-1>
+<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=UTF-8">
+<TITLE>Bookmarks</TITLE>
+<H1>Bookmarks</H1>
+<DL><p>
+  <DT><A HREF="https://www.iana.org/domains/reserved" ADD_DATE="1704153600" TAGS="netscape-tag,metadata">IANA Reserved Domains</A>
+</DL><p>
+""",
+        encoding="utf-8",
+    )
+    files["dom"].write_text(
+        """<!doctype html>
+<html>
+  <head><title>DOM import fixture</title></head>
+  <body>
+    <a href="https://www.iana.org/help/example-domains">IANA Example Domains</a>
+  </body>
+</html>
+""",
+        encoding="utf-8",
+    )
+    files["json"].write_text(
+        json.dumps(
+            {
+                "url": "https://example.com/?archivebox-json-import=1",
+                "title": "JSON Import Example",
+                "tags": ["json-tag", "metadata"],
+                "bookmarked_at": "2024-01-03T00:00:00+00:00",
+            },
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+    files["jsonl"].write_text(
+        json.dumps(
+            {
+                "url": "https://example.com/?archivebox-jsonl-import=1",
+                "title": "JSONL Import Example",
+                "tags": "jsonl-tag,metadata",
+                "bookmarked_at": "2024-01-04T00:00:00+00:00",
+            },
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+    files["txt"].write_text(
+        "Plain text import fixture containing https://example.org/ as a real live URL.\n",
+        encoding="utf-8",
+    )
+    return files
+
+
+IMPORT_FORMAT_ENV = {
+    "USE_COLOR": "False",
+    "SHOW_PROGRESS": "False",
+    "PLUGINS": "parse_html_urls,parse_jsonl_urls,parse_netscape_urls,parse_rss_urls,parse_txt_urls",
+    "SAVE_WGET": "False",
+    "SAVE_HEADERS": "False",
+    "USE_CHROME": "False",
+    "URL_ALLOWLIST": r"example\.com|example\.org|iana\.org|www\.iana\.org",
+}
+
+
+def wait_for_import_processing(cwd: Path, expected_urls: set[str], *, timeout: float = 120.0) -> None:
+    import time
+
+    deadline = time.time() + timeout
+    counts = {url: 0 for url in expected_urls}
+    while time.time() < deadline:
+        with use_archivebox_db(cwd):
+            rows = list(Snapshot.objects.filter(url__in=expected_urls).values_list("url", flat=True))
+        counts = {url: 0 for url in expected_urls}
+        for url in rows:
+            counts[url] += 1
+        if all(count >= 1 for count in counts.values()):
+            return
+        time.sleep(1)
+    raise AssertionError(f"timed out waiting for import crawl processing to start, got counts={counts}")
+
+
+def wait_for_expected_import_snapshots(cwd: Path, expected_urls: set[str], *, timeout: float = 180.0) -> None:
+    import time
+
+    allowed_statuses = {Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.SEALED}
+    deadline = time.time() + timeout
+    while time.time() < deadline:
+        with use_archivebox_db(cwd):
+            rows = list(Snapshot.objects.filter(url__in=expected_urls).values_list("url", "status"))
+        counts = {url: 0 for url in expected_urls}
+        bad_statuses = []
+        for url, status in rows:
+            counts[url] += 1
+            if status not in allowed_statuses:
+                bad_statuses.append((url, status))
+        if all(count == 1 for count in counts.values()) and not bad_statuses:
+            return
+        time.sleep(1)
+    raise AssertionError(
+        f"timed out waiting for one queued/started/sealed snapshot per URL, got counts={counts}, bad_statuses={bad_statuses}",
+    )
+
+
+def malicious_add_inputs(tmp_path: Path, *, safe_url: str) -> tuple[list[str], Path]:
+    other_crawl_source = tmp_path / "sources" / "other_crawl_source.txt"
+    other_crawl_source.parent.mkdir(parents=True, exist_ok=True)
+    other_crawl_source.write_text("https://example.com/not-owned-by-this-crawl\n", encoding="utf-8")
+    canary = tmp_path / "archivebox_shell_injection_canary"
+    return (
+        [
+            safe_url,
+            "file:///etc/hosts",
+            "/etc/hosts",
+            "../../../../etc/passwd",
+            f"file://{other_crawl_source}",
+            str(other_crawl_source),
+            f"'; touch {canary}; #",
+            f'" && touch {canary} && echo "',
+            f"$(touch {canary})",
+            f"`touch {canary}`",
+            """<?xml version="1.0"?>
+<!DOCTYPE rss [
+  <!ENTITY localfile SYSTEM "file:///etc/hosts">
+]>
+<rss version="2.0" xmlns:xi="http://www.w3.org/2001/XInclude">
+  <channel>
+    <item><title>&localfile;</title><link>file:///etc/passwd</link></item>
+    <xi:include href="file:///etc/hosts" parse="text"/>
+  </channel>
+</rss>""",
+        ],
+        canary,
+    )
+
+
+def assert_no_file_or_shell_payload_snapshots(cwd: Path, *, canary: Path) -> None:
+    with use_archivebox_db(cwd):
+        snapshots = list(Snapshot.objects.all())
+    assert not canary.exists()
+    assert not [snapshot.url for snapshot in snapshots if str(snapshot.url).startswith("file:")]
+    for forbidden in ("/etc/hosts", "/etc/passwd", "other_crawl_source", "archivebox_shell_injection_canary"):
+        assert not [snapshot.url for snapshot in snapshots if forbidden in str(snapshot.url)]
+
+
 def test_add_single_url_records_url_in_crawl(initialized_archive):
     """Test that adding a single URL queues a crawl with the submitted URL."""
     env = cli_env(disable_extractors=True)
@@ -32,12 +251,226 @@ def test_add_single_url_records_url_in_crawl(initialized_archive):
 
     with use_archivebox_db(initialized_archive):
         crawl = Crawl.objects.get()
+        snapshots = list(Snapshot.objects.all())
 
+    assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": "https://example.com", "depth": 0}
     assert crawl.get_urls_list() == ["https://example.com"]
+    assert snapshots == []
 
 
-def test_add_bg_queues_crawl_without_creating_snapshots(initialized_archive):
-    """Background add should leave root snapshot creation to the runner."""
+@pytest.mark.timeout(360)
+def test_add_stdin_import_formats_preserve_metadata_and_crawl_inner_urls(initialized_archive):
+    """`archivebox add < import-file` should normalize rich import formats before crawling URLs."""
+    import_files = write_import_format_files(initialized_archive)
+    expected_urls = {case["url"] for case in IMPORT_FORMAT_EXPECTATIONS.values()}
+    port = get_free_port()
+    env = cli_env(port=port, server=True, **IMPORT_FORMAT_ENV)
+
+    for import_path in import_files.values():
+        source_text = import_path.read_text(encoding="utf-8")
+        result = run_archivebox_cmd(
+            ["add", "--bg", "--depth=0", "--tag=cli-stdin-import"],
+            cwd=initialized_archive,
+            env=env,
+            stdin=source_text,
+            timeout=360,
+        )
+        assert result.returncode == 0, result.stderr or result.stdout
+        with use_archivebox_db(initialized_archive):
+            crawl = Crawl.objects.order_by("-created_at").first()
+            assert crawl is not None
+            assert crawl.snapshot_set.count() == 0
+        assert crawl.urls == source_text
+
+    try:
+        start_archivebox_server(initialized_archive, env=env, port=port)
+        deadline = time.time() + 120
+        root_counts = {}
+        while time.time() < deadline:
+            with use_archivebox_db(initialized_archive):
+                root_counts = {
+                    str(crawl.id): crawl.snapshot_set.filter(url=Snapshot.INTERNAL_INPUT_URL).count() for crawl in Crawl.objects.all()
+                }
+            if root_counts and all(count == 1 for count in root_counts.values()):
+                break
+            time.sleep(1)
+        assert root_counts and all(count == 1 for count in root_counts.values()), root_counts
+        with use_archivebox_db(initialized_archive):
+            for crawl in Crawl.objects.all():
+                root_snapshot = crawl.snapshot_set.get(url=Snapshot.INTERNAL_INPUT_URL)
+                root_input = (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8")
+                assert root_input == crawl.urls
+        wait_for_import_processing(initialized_archive, expected_urls)
+        stop_server(initialized_archive)
+        start_archivebox_server(initialized_archive, env=env, port=port)
+        wait_for_expected_import_snapshots(initialized_archive, expected_urls)
+
+        list_result = run_archivebox_cmd(
+            ["list", "--json"],
+            cwd=initialized_archive,
+            env=env,
+            timeout=60,
+        )
+        assert list_result.returncode == 0, list_result.stderr or list_result.stdout
+        for expected_url in expected_urls:
+            assert expected_url in list_result.stdout
+    finally:
+        stop_server(initialized_archive)
+
+    with use_archivebox_db(initialized_archive):
+        crawls = list(Crawl.objects.order_by("created_at"))
+        snapshots_by_url = {snapshot.url: snapshot for snapshot in Snapshot.objects.prefetch_related("tags").filter(url__in=expected_urls)}
+        tags_by_url = {snapshot.url: set(snapshot.tags.values_list("name", flat=True)) for snapshot in snapshots_by_url.values()}
+
+    assert len(crawls) == len(import_files)
+    assert [crawl.urls for crawl in crawls] == [path.read_text(encoding="utf-8") for path in import_files.values()]
+    assert all(crawl.tags_str == "cli-stdin-import" for crawl in crawls)
+    assert all(crawl.status in {Crawl.StatusChoices.STARTED, Crawl.StatusChoices.SEALED} for crawl in crawls)
+    assert len(snapshots_by_url) == len(expected_urls)
+
+    for import_name, expected in IMPORT_FORMAT_EXPECTATIONS.items():
+        snapshot = snapshots_by_url.get(expected["url"])
+        assert snapshot is not None, f"{import_name} did not create Snapshot for {expected['url']}"
+        assert snapshot.status in {Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.SEALED}
+        if expected.get("title"):
+            assert snapshot.title == expected["title"]
+        if expected.get("date"):
+            assert snapshot.bookmarked_at.date().isoformat() == expected["date"]
+        if expected.get("tags"):
+            assert expected["tags"] | {"cli-stdin-import"} <= tags_by_url[snapshot.url]
+
+
+@pytest.mark.timeout(240)
+def test_add_rejects_file_path_and_shell_injection_payloads(initialized_archive):
+    """CLI add must not turn user-supplied local paths or shell payloads into snapshots."""
+    safe_url = "https://example.com/?archivebox-cli-security=1"
+    inputs, canary = malicious_add_inputs(initialized_archive, safe_url=safe_url)
+    port = get_free_port()
+    env = cli_env(port=port, server=True, **IMPORT_FORMAT_ENV)
+
+    result = run_archivebox_cmd(
+        ["add", "--bg", "--depth=0", "--tag=cli-security"],
+        cwd=initialized_archive,
+        env=env,
+        stdin="\n".join(inputs),
+        timeout=120,
+    )
+    assert result.returncode == 0, result.stderr or result.stdout
+
+    try:
+        start_archivebox_server(initialized_archive, env=env, port=port)
+        wait_for_expected_import_snapshots(initialized_archive, {safe_url}, timeout=120)
+    finally:
+        stop_server(initialized_archive)
+
+    assert_no_file_or_shell_payload_snapshots(initialized_archive, canary=canary)
+    with use_archivebox_db(initialized_archive):
+        snapshots = list(Snapshot.objects.filter(url=safe_url).values_list("url", "status", "tags__name"))
+        crawl = Crawl.objects.get()
+    assert crawl.status in {Crawl.StatusChoices.STARTED, Crawl.StatusChoices.SEALED}
+    assert len({url for url, _status, _tag in snapshots}) == 1
+    assert all(
+        status in {Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.SEALED}
+        for _url, status, _tag in snapshots
+    )
+    assert "cli-security" in {tag for _url, _status, tag in snapshots}
+
+
+@pytest.mark.timeout(180)
+def test_run_rejects_file_url_injected_directly_into_crawl_urls_with_db_update(initialized_archive):
+    """Runner must validate Crawl.urls again when DB writes bypass normal add/create paths."""
+    secret_url = "https://example.com/?archivebox-sql-crawl-file-secret=1"
+    local_source = initialized_archive / "not_owned_by_crawl_urls.txt"
+    local_source.write_text(f"{secret_url}\n", encoding="utf-8")
+    file_url = local_source.resolve().as_uri()
+    env = cli_env(**{**IMPORT_FORMAT_ENV, "PLUGINS": "parse_txt_urls", "SAVE_HEADERS": "False", "SAVE_WGET": "False"})
+
+    with use_archivebox_db(initialized_archive):
+        crawl = Crawl.objects.create(
+            urls="https://example.com/?archivebox-sql-crawl-control=1",
+            max_depth=2,
+            tags_str="sql-file-url",
+            status=Crawl.StatusChoices.QUEUED,
+            retry_at=timezone.now(),
+        )
+        bad_jsonl = json.dumps({"type": "CrawlSeed", "url": file_url, "depth": 0, "tags": "sql-file-url"})
+        Crawl.objects.filter(pk=crawl.pk).update(urls=bad_jsonl)
+
+    result = run_archivebox_cmd(
+        ["run", f"--crawl-id={crawl.id}"],
+        cwd=initialized_archive,
+        env=env,
+        timeout=180,
+    )
+    assert result.returncode == 0, result.stderr or result.stdout
+
+    with use_archivebox_db(initialized_archive):
+        crawl.refresh_from_db()
+        snapshot_urls = set(Snapshot.objects.values_list("url", flat=True))
+
+    assert crawl.status in {Crawl.StatusChoices.STARTED, Crawl.StatusChoices.SEALED}
+    assert file_url not in snapshot_urls
+    assert secret_url not in snapshot_urls
+
+
+@pytest.mark.timeout(180)
+def test_run_rejects_depth_two_file_url_snapshot_injected_directly_with_db_update(initialized_archive):
+    """A queued Snapshot row with a file:// URL must not be allowed to run hooks or recurse."""
+    secret_url = "https://example.com/?archivebox-sql-depth2-file-secret=1"
+    local_source = initialized_archive / "not_owned_by_depth_two_snapshot.txt"
+    local_source.write_text(f"{secret_url}\n", encoding="utf-8")
+    file_url = local_source.resolve().as_uri()
+    env = cli_env(**{**IMPORT_FORMAT_ENV, "PLUGINS": "parse_txt_urls", "SAVE_HEADERS": "False", "SAVE_WGET": "False"})
+
+    with use_archivebox_db(initialized_archive):
+        crawl = Crawl.objects.create(
+            urls="https://example.com/?archivebox-sql-depth2-root=1",
+            max_depth=2,
+            tags_str="sql-depth2-file-url",
+            status=Crawl.StatusChoices.QUEUED,
+            retry_at=timezone.now(),
+        )
+        root_snapshot = Snapshot.objects.create(
+            url="https://example.com/?archivebox-sql-depth2-root=1",
+            crawl=crawl,
+            depth=0,
+            status=Snapshot.StatusChoices.SEALED,
+            retry_at=None,
+        )
+        injected_snapshot = Snapshot.objects.create(
+            url="https://example.com/?archivebox-sql-depth2-placeholder=1",
+            crawl=crawl,
+            parent_snapshot=root_snapshot,
+            depth=2,
+            status=Snapshot.StatusChoices.QUEUED,
+            retry_at=timezone.now(),
+        )
+        Snapshot.objects.filter(pk=injected_snapshot.pk).update(url=file_url)
+
+    result = run_archivebox_cmd(
+        ["run", f"--snapshot-id={injected_snapshot.id}"],
+        cwd=initialized_archive,
+        env=env,
+        timeout=180,
+    )
+    assert result.returncode == 0, result.stderr or result.stdout
+
+    with use_archivebox_db(initialized_archive):
+        injected_snapshot.refresh_from_db()
+        snapshot_urls = set(Snapshot.objects.values_list("url", flat=True))
+        file_results = list(
+            ArchiveResult.objects.filter(
+                snapshot=injected_snapshot,
+            ).values_list("plugin", "status"),
+        )
+
+    assert injected_snapshot.url == file_url
+    assert secret_url not in snapshot_urls
+    assert file_results == []
+
+
+def test_add_bg_queues_direct_url_snapshot(initialized_archive):
+    """Background add queues explicit URL arguments as real URL snapshots."""
     env = cli_env(disable_extractors=True)
     result = run_archivebox_cmd(
         ["add", "--bg", "--depth=0", "https://example.com"],
@@ -49,11 +482,126 @@ def test_add_bg_queues_crawl_without_creating_snapshots(initialized_archive):
 
     with use_archivebox_db(initialized_archive):
         crawl = Crawl.objects.get()
-        snapshot_count = Snapshot.objects.count()
+        snapshots = list(Snapshot.objects.all())
 
     assert crawl.status == Crawl.StatusChoices.QUEUED
     assert crawl.retry_at is not None
-    assert snapshot_count == 0
+    assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": "https://example.com", "depth": 0}
+    assert snapshots == []
+
+
+@pytest.mark.timeout(180)
+def test_add_tagged_single_url_seals_without_duplicate_snapshot_tags(initialized_archive):
+    env = os.environ.copy()
+    env.update(
+        {
+            "USE_COLOR": "False",
+            "SHOW_PROGRESS": "False",
+            "PLUGINS": "parse_txt_urls,title",
+            "TIMEOUT": "60",
+            "CRAWL_MAX_CONCURRENT_SNAPSHOTS": "1",
+        },
+    )
+    result = run_archivebox_cmd(
+        [
+            "add",
+            "--bg",
+            "--depth=0",
+            "--tag=tagged-single-url",
+            "https://example.com/?archivebox-tagged-single-url=1",
+        ],
+        cwd=initialized_archive,
+        env=env,
+        timeout=60,
+    )
+    assert result.returncode == 0, result.stderr or result.stdout
+
+    run_queued_crawls(initialized_archive, env, timeout=180)
+
+    with use_archivebox_db(initialized_archive):
+        crawl = Crawl.objects.get()
+        snapshots = list(Snapshot.objects.order_by("depth", "url"))
+        tag_counts = {
+            snapshot.url: SnapshotTag.objects.filter(snapshot=snapshot, tag__name="tagged-single-url").count() for snapshot in snapshots
+        }
+        results = list(
+            ArchiveResult.objects.select_related("snapshot")
+            .order_by("snapshot__depth", "snapshot__url", "plugin")
+            .values_list("snapshot__url", "plugin", "status", "output_str"),
+        )
+
+    assert crawl.status == Crawl.StatusChoices.SEALED
+    assert [(snapshot.url, snapshot.depth, snapshot.status) for snapshot in snapshots] == [
+        ("https://example.com/?archivebox-tagged-single-url=1", 0, Snapshot.StatusChoices.SEALED),
+    ]
+    assert tag_counts == {
+        "https://example.com/?archivebox-tagged-single-url=1": 1,
+    }
+    by_url_plugin = {(url, plugin): status for url, plugin, status, _output in results}
+    assert by_url_plugin[("https://example.com/?archivebox-tagged-single-url=1", "title")] == "succeeded"
+    unexpected_failures = [(url, plugin, status, output) for url, plugin, status, output in results if status == "failed"]
+    assert not unexpected_failures
+
+
+@pytest.mark.timeout(180)
+def test_add_title_hook_env_gets_canonical_runtime_config_without_archivebox_selectors_or_aliases(initialized_archive):
+    env = os.environ.copy()
+    env.update(
+        {
+            "USE_COLOR": "False",
+            "SHOW_PROGRESS": "False",
+            "PLUGINS": "title",
+            "SAVE_TITLE": "True",
+            "SECRET_KEY": "hook-env-secret-must-not-leak",
+            "PUBLIC_ADD_VIEW": "True",
+            "ADMIN_PASSWORD": "hook-env-admin-password-must-not-leak",
+            "URL_BLACKLIST": "$^",
+            "ARCHIVEBOX_TEST_ARBITRARY_ENV": "preserved-user-env",
+            "LD_PRELOAD": "",
+            "TIMEOUT": "60",
+            "CRAWL_MAX_CONCURRENT_SNAPSHOTS": "1",
+        },
+    )
+    result = run_archivebox_cmd(
+        [
+            "add",
+            "--depth=0",
+            "--plugins=title",
+            "https://example.com/?archivebox-title-env=1",
+        ],
+        cwd=initialized_archive,
+        env=env,
+        timeout=180,
+    )
+    assert result.returncode == 0, result.stderr or result.stdout
+
+    with use_archivebox_db(initialized_archive):
+        crawl = Crawl.objects.get()
+        snapshot = Snapshot.objects.get()
+        title_result = ArchiveResult.objects.select_related("process").get(snapshot=snapshot, plugin="title")
+        process_env = title_result.process.env
+
+    assert crawl.config["PLUGINS"] == "title"
+    assert snapshot.url == "https://example.com/?archivebox-title-env=1"
+    assert snapshot.status == Snapshot.StatusChoices.SEALED
+    assert snapshot.title == "Example Domain"
+    assert title_result.status == ArchiveResult.StatusChoices.SUCCEEDED
+    assert title_result.output_str == "Example Domain"
+    assert title_result.process.exit_code == 0
+    assert process_env["TITLE_ENABLED"] == "True"
+    assert process_env["CHROME_ENABLED"] == "True"
+    assert process_env["WGET_ENABLED"] == "False"
+    assert process_env["ARCHIVEBOX_TEST_ARBITRARY_ENV"] == "preserved-user-env"
+    assert process_env["LD_PRELOAD"] == ""
+    assert "PLUGINS" not in process_env
+    assert "SAVE_TITLE" not in process_env
+    assert "SEARCH_BACKEND_ENGINE" not in process_env
+    assert "SECRET_KEY" not in process_env
+    assert "ADMIN_PASSWORD" not in process_env
+    assert "PUBLIC_ADD_VIEW" not in process_env
+    assert "URL_BLACKLIST" not in process_env
+    assert "hook-env-secret-must-not-leak" not in json.dumps(process_env)
+    assert "hook-env-admin-password-must-not-leak" not in json.dumps(process_env)
 
 
 def test_add_index_only_rejected_urls_leave_empty_crawl_for_runner_to_seal(initialized_archive):
@@ -75,21 +623,23 @@ def test_add_index_only_rejected_urls_leave_empty_crawl_for_runner_to_seal(initi
 
     with use_archivebox_db(initialized_archive):
         crawl = Crawl.objects.get()
-        snapshot_count = Snapshot.objects.count()
+        snapshot_urls = set(Snapshot.objects.values_list("url", flat=True))
 
     assert crawl.status == Crawl.StatusChoices.QUEUED
     assert crawl.retry_at is None
-    assert snapshot_count == 0
+    assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": "https://example.com", "depth": 0}
+    assert snapshot_urls == set()
 
     run_queued_crawls(initialized_archive, env)
 
     with use_archivebox_db(initialized_archive):
         crawl = Crawl.objects.get()
-        snapshot_count = Snapshot.objects.count()
+        snapshot_urls = set(Snapshot.objects.values_list("url", flat=True))
 
     assert crawl.status == Crawl.StatusChoices.SEALED
     assert crawl.retry_at is None
-    assert snapshot_count == 0
+    assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": "https://example.com", "depth": 0}
+    assert snapshot_urls == set()
 
 
 def test_add_index_only_rejects_archivebox_internal_urls(initialized_archive):
@@ -111,12 +661,14 @@ def test_add_index_only_rejects_archivebox_internal_urls(initialized_archive):
 
     with use_archivebox_db(initialized_archive):
         crawl = Crawl.objects.get()
-        snapshot_count = Snapshot.objects.count()
+        snapshot_urls = set(Snapshot.objects.values_list("url", flat=True))
 
-    assert crawl.get_urls_list() == []
+    assert [json.loads(line) for line in crawl.urls.splitlines()] == [
+        {"type": "CrawlSeed", "url": url, "depth": 0} for url in internal_urls
+    ]
     assert crawl.status == Crawl.StatusChoices.QUEUED
     assert crawl.retry_at is None
-    assert snapshot_count == 0
+    assert snapshot_urls == set()
 
 
 def test_add_creates_crawl_record(initialized_archive):
@@ -134,23 +686,21 @@ def test_add_creates_crawl_record(initialized_archive):
     assert crawl_count == 1
 
 
-def test_add_creates_source_file(initialized_archive):
-    """Test that add creates a source file with the URL."""
+def test_add_direct_url_creates_snapshot_without_internal_input_file(initialized_archive):
+    """Explicit URL args materialize real snapshots when the runner claims the crawl."""
     env = cli_env(disable_extractors=True)
     run_archivebox_cmd(
         ["add", "--index-only", "--depth=0", "https://example.com"],
         cwd=initialized_archive,
         env=env,
     )
+    run_queued_crawls(initialized_archive, env)
 
-    sources_dir = initialized_archive / "sources"
-    assert sources_dir.exists()
-
-    source_files = list(sources_dir.glob("*cli_add.txt"))
-    assert len(source_files) >= 1
-
-    source_content = source_files[0].read_text()
-    assert "https://example.com" in source_content
+    with use_archivebox_db(initialized_archive):
+        snapshot = Snapshot.objects.get()
+        assert snapshot.url == "https://example.com"
+        assert snapshot.depth == 0
+        assert not (snapshot.output_dir / "staticfile" / "stdin.txt").exists()
 
 
 def test_add_multiple_urls_single_command(initialized_archive):
@@ -166,19 +716,18 @@ def test_add_multiple_urls_single_command(initialized_archive):
 
     with use_archivebox_db(initialized_archive):
         crawl = Crawl.objects.get()
+        snapshots = list(Snapshot.objects.order_by("url").values_list("url", "depth"))
 
-    assert crawl.get_urls_list() == ["https://example.com", "https://example.org"]
+    assert [json.loads(line) for line in crawl.urls.splitlines()] == [
+        {"type": "CrawlSeed", "url": "https://example.com", "depth": 0},
+        {"type": "CrawlSeed", "url": "https://example.org", "depth": 0},
+    ]
+    assert snapshots == []
 
 
-def test_add_from_file(initialized_archive):
-    """Test adding URLs from a file.
-
-    The add command should treat a file argument as URL input and queue
-    a crawl containing each URL.
-    """
-
+def test_add_rejects_file_path_argument(initialized_archive):
+    """Local files must be piped through stdin, not passed as archiveable path arguments."""
     env = cli_env(disable_extractors=True)
-    # Create a file with URLs
     urls_file = initialized_archive / "urls.txt"
     urls_file.write_text("https://example.com\nhttps://example.org\n")
 
@@ -188,15 +737,13 @@ def test_add_from_file(initialized_archive):
         env=env,
     )
 
-    assert result.returncode == 0
+    assert result.returncode != 0
 
     with use_archivebox_db(initialized_archive):
-        crawl_count = Crawl.objects.count()
-        urls = Crawl.objects.get().get_urls_list()
+        assert Crawl.objects.count() == 0
+        assert Snapshot.objects.count() == 0
 
-    # The file is parsed into two input URLs.
-    assert crawl_count == 1
-    assert urls == ["https://example.com", "https://example.org"]
+    assert "No URLs provided" in (result.stderr or result.stdout)
 
 
 def test_add_with_depth_0_flag(initialized_archive):
@@ -325,14 +872,15 @@ def test_add_duplicate_url_creates_separate_crawls(initialized_archive):
         cwd=initialized_archive,
         env=env,
     )
+    run_queued_crawls(initialized_archive, env)
 
     with use_archivebox_db(initialized_archive):
         crawl_count = Crawl.objects.count()
-        crawl_urls = list(Crawl.objects.order_by("created_at").values_list("urls", flat=True))
+        snapshots = list(Snapshot.objects.order_by("created_at").values_list("url", "depth"))
 
     # Each add creates a new crawl with its own queued work.
     assert crawl_count == 2
-    assert crawl_urls == ["https://example.com", "https://example.com"]
+    assert snapshots == [("https://example.com", 0), ("https://example.com", 0)]
 
 
 def test_add_with_overwrite_flag(initialized_archive):
@@ -447,28 +995,31 @@ def test_add_index_only_queues_crawl_without_starting_runner(initialized_archive
 
     with use_archivebox_db(initialized_archive):
         crawl = Crawl.objects.get()
-        snapshot_count = Snapshot.objects.count()
+        snapshots = list(Snapshot.objects.all())
 
     assert crawl.status == Crawl.StatusChoices.QUEUED
     assert crawl.retry_at is None
-    assert snapshot_count == 0
+    assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": "https://example.com", "depth": 0}
+    assert snapshots == []
 
 
-def test_add_index_only_leaves_snapshot_creation_to_runner(initialized_archive):
-    """Test that index-only add does not create snapshots before the runner."""
+def test_add_index_only_creates_direct_url_snapshot(initialized_archive):
+    """Index-only add seeds explicit URL args for the runner to snapshot."""
     env = cli_env(disable_extractors=True)
     run_archivebox_cmd(
         ["add", "--index-only", "--depth=0", "https://example.com"],
         cwd=initialized_archive,
         env=env,
     )
+    run_queued_crawls(initialized_archive, env)
 
     with use_archivebox_db(initialized_archive):
-        crawl_id = Crawl.objects.values_list("id", flat=True).get()
-        snapshot_count = Snapshot.objects.count()
+        crawl = Crawl.objects.get()
+        snapshot = Snapshot.objects.get()
 
-    assert crawl_id
-    assert snapshot_count == 0
+    assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": "https://example.com", "depth": 0}
+    assert snapshot.url == "https://example.com"
+    assert snapshot.depth == 0
 
 
 def test_snapshot_create_sets_snapshot_timestamp(initialized_archive):
@@ -609,7 +1160,7 @@ def test_cli_add_real_urls_with_options_writes_inspectable_outputs(initialized_a
 
     snapshot_urls = {url for _id, url, _depth, _status, _title in snapshots}
     assert snapshot_urls >= {*wget_urls, chrome_url}
-    assert all(depth == 0 for _id, _url, depth, _status, _title in snapshots)
+    assert all(depth == 0 for _id, url, depth, _status, _title in snapshots)
 
     by_url_plugin = {(url, plugin): status for url, plugin, status, _files, _size, _output in archive_results}
     assert by_url_plugin[("https://example.com", "wget")] == "succeeded"

archivebox/tests/test_cli_config.py

@@ -228,7 +228,7 @@ SEARCH_BACKEND_PASSWORD = "SecretPassword"
     )
 
     result = run_archivebox_cmd(
-        ["version"],
+        ["version", "--binaries=curl"],
     )
 
     assert result.returncode == 0, result.stderr

archivebox/tests/test_cli_crawl.py

@@ -417,27 +417,41 @@ def test_crawl_multiple_urls_creates_multiple_snapshots(initialized_archive):
     assert "https://iana.org" in urls
 
 
-def test_crawl_from_file_creates_snapshot(initialized_archive):
-    """Test that crawl can create snapshots from a file of URLs."""
+def test_crawl_path_argument_is_rejected_but_stdin_file_contents_create_snapshot(initialized_archive):
+    """Local file paths are not URL args; users must pipe file contents through stdin."""
     env = cli_env(disable_extractors=True)
 
-    # Write URLs to a file
     urls_file = initialized_archive / "urls.txt"
-    urls_file.write_text("https://example.com\n")
+    urls_file.write_text("https://example.com\nhttps://iana.org\n", encoding="utf-8")
 
-    run_archivebox_cmd(
+    path_result = run_archivebox_cmd(
         ["crawl", "create", str(urls_file)],
         cwd=initialized_archive,
         env=env,
+    )
+    assert path_result.returncode == 1
+    assert "No URLs provided" in path_result.stderr
+
+    with use_archivebox_db(initialized_archive):
+        assert Crawl.objects.count() == 0
+        assert Snapshot.objects.count() == 0
+
+    stdin_result = run_archivebox_cmd(
+        ["crawl", "create"],
+        cwd=initialized_archive,
+        env=env,
+        stdin=urls_file.read_text(encoding="utf-8"),
         check=True,
     )
+    assert stdin_result.returncode == 0
     run_queued_crawls(initialized_archive, env)
 
     with use_archivebox_db(initialized_archive):
-        snapshot = Snapshot.objects.first()
+        urls = set(Snapshot.objects.values_list("url", flat=True))
 
-    # Should create at least one snapshot (the source file or the URL)
-    assert snapshot is not None, "Should create at least one snapshot"
+    assert "https://example.com" in urls
+    assert "https://iana.org" in urls
+    assert str(urls_file) not in urls
 
 
 def test_crawl_persists_input_urls_on_crawl(initialized_archive):

archivebox/tests/test_cli_init.py

@@ -5,6 +5,7 @@ Verify init creates correct database schema, filesystem structure, and config.
 """
 
 import pytest
+import subprocess
 from django.utils import timezone
 from django.db import connections
 from django.db.migrations.recorder import MigrationRecorder
@@ -14,6 +15,7 @@ from archivebox.core.models import ArchiveResult, Snapshot
 from archivebox.crawls.models import Crawl
 from archivebox.machine.models import Machine
 from archivebox.tests.conftest import run_queued_crawls, run_archivebox_cmd, cli_env
+from archivebox.tests.conftest import _set_test_source_pythonpath
 
 from archivebox.tests.test_orm_helpers import use_archivebox_db
 
@@ -129,6 +131,23 @@ def test_init_sets_correct_file_permissions(tmp_path):
     assert oct(archive_dir.stat().st_mode)[-3:] in (get_config().OUTPUT_PERMISSIONS, DIR_PERMISSIONS)
 
 
+def test_init_creates_database_with_archivebox_permissions_despite_permissive_umask(tmp_path):
+    """SQLite must create index.sqlite3 with ArchiveBox's restrictive mode from first open."""
+    env = cli_env(disable_extractors=True)
+    _set_test_source_pythonpath(env)
+    result = subprocess.run(
+        ["bash", "-lc", "umask 000; archivebox init"],
+        cwd=tmp_path,
+        env=env,
+        capture_output=True,
+        text=True,
+        timeout=60,
+    )
+
+    assert result.returncode == 0, result.stderr or result.stdout
+    assert oct((tmp_path / "index.sqlite3").stat().st_mode)[-3:] == get_config().OUTPUT_PERMISSIONS
+
+
 def test_init_is_idempotent(tmp_path):
     """Test that running init multiple times is safe (idempotent)."""
 

archivebox/tests/test_cli_install.py

@@ -11,6 +11,8 @@ from archivebox.tests.conftest import run_archivebox_cmd
 
 import pytest
 
+from archivebox.cli.archivebox_install import ensure_data_dir_lib_symlink
+from archivebox.config.paths import get_machine_type
 from archivebox.core.models import Snapshot
 from archivebox.crawls.models import Crawl
 from archivebox.machine.models import Binary
@@ -114,6 +116,9 @@ def test_install_from_empty_dir_initializes_collection(tmp_path):
     env = os.environ.copy()
     tmp_short = Path("/tmp") / f"abx-install-empty-{tmp_path.name}"
     tmp_short.mkdir(parents=True, exist_ok=True)
+    stale_lib_dir = tmp_path / "lib"
+    stale_lib_dir.mkdir()
+    (stale_lib_dir / "stale.txt").write_text("stale")
     env.update(
         {
             "TMP_DIR": str(tmp_short),
@@ -134,6 +139,8 @@ def test_install_from_empty_dir_initializes_collection(tmp_path):
     assert "Installing specific binaries: git" in output
     assert (tmp_path / "ArchiveBox.conf").is_file()
     assert (tmp_path / "index.sqlite3").is_file()
+    assert (tmp_path / "lib").is_symlink()
+    assert (tmp_path / "lib").resolve().is_dir()
 
     with use_archivebox_db(tmp_path):
         assert Snapshot.objects.count() == 0
@@ -141,6 +148,16 @@ def test_install_from_empty_dir_initializes_collection(tmp_path):
         assert Binary.objects.filter(status="installed", name="git").count() == 1
 
 
+def test_install_lib_symlink_preserves_scoped_data_lib_dir(tmp_path):
+    """DATA_DIR/lib/<machine_type> is a valid ABXPKG_LIB_DIR and must not be replaced by a parent symlink."""
+    scoped_lib_dir = tmp_path / "lib" / get_machine_type()
+
+    assert ensure_data_dir_lib_symlink(tmp_path, scoped_lib_dir) is None
+    assert (tmp_path / "lib").is_dir()
+    assert not (tmp_path / "lib").is_symlink()
+    assert scoped_lib_dir.is_dir()
+
+
 def test_install_updates_binary_table(initialized_archive):
     """Test that install completes and only mutates dependency state."""
     env = os.environ.copy()
@@ -174,3 +191,5 @@ def test_install_updates_binary_table(initialized_archive):
     assert snapshot_count == 0
     assert binary_counts.get("installed", 0) > 0
     assert installed_git == 1
+    assert (initialized_archive / "lib").is_symlink()
+    assert (initialized_archive / "lib").resolve().is_dir()

archivebox/tests/test_cli_piping.py

@@ -55,8 +55,7 @@ def test_parse_line_accepts_supported_piping_inputs():
     plain_url = parse_line("https://example.com")
     assert plain_url == {"type": TYPE_SNAPSHOT, "url": "https://example.com"}
 
-    file_url = parse_line("file:///tmp/example.txt")
-    assert file_url == {"type": TYPE_SNAPSHOT, "url": "file:///tmp/example.txt"}
+    assert parse_line("file:///tmp/example.txt") is None
 
     snapshot_json = parse_line('{"type":"Snapshot","url":"https://example.com","tags":"tag1,tag2"}')
     assert snapshot_json is not None
@@ -78,13 +77,17 @@ def test_parse_line_accepts_supported_piping_inputs():
     assert compact_parsed_id == {"type": TYPE_SNAPSHOT, "id": compact_snapshot_id}
 
 
-def test_read_args_or_stdin_handles_args_stdin_and_mixed_jsonl():
+def test_read_args_or_stdin_handles_args_stdin_and_mixed_jsonl(tmp_path):
     """Piping helpers should consume args, structured JSONL, and pass-through records."""
     from archivebox.misc.jsonl import TYPE_CRAWL, read_args_or_stdin
 
     records = list(read_args_or_stdin(("https://example1.com", "https://example2.com")))
     assert [record["url"] for record in records] == ["https://example1.com", "https://example2.com"]
 
+    local_file = tmp_path / "urls.txt"
+    local_file.write_text("https://from-file-arg.example\n")
+    assert list(read_args_or_stdin((str(local_file),))) == []
+
     stdin_records = list(
         read_args_or_stdin(
             (),

archivebox/tests/test_cli_run.py

@@ -566,7 +566,9 @@ class TestRunDaemonMode:
                 if proc.poll() is not None:
                     stdout = proc.stdout.read()
                     stderr = proc.stderr.read()
-                    pytest.fail(f"daemon exited before starting runner: code={proc.returncode}\nstdout={stdout}\nstderr={stderr}")
+                    raise AssertionError(
+                        f"daemon exited before starting runner: code={proc.returncode}\nstdout={stdout}\nstderr={stderr}",
+                    )
                 with use_archivebox_db(initialized_archive):
                     started = Process.objects.filter(
                         process_type=Process.TypeChoices.ORCHESTRATOR,
@@ -601,6 +603,40 @@ class TestRunDaemonMode:
         from archivebox.tests.test_orm_helpers import use_archivebox_db
 
         env = cli_env()
+
+        def active_runners():
+            with use_archivebox_db(initialized_archive):
+                return [
+                    proc
+                    for proc in Process.objects.filter(
+                        process_type=Process.TypeChoices.ORCHESTRATOR,
+                        worker_type=RUNNER_ACTIVE_WORKER_TYPE,
+                        status=Process.StatusChoices.RUNNING,
+                        pwd=str(initialized_archive),
+                    )
+                    if proc.is_running
+                ]
+
+        def wait_for_stable_single_active(*, timeout: float, stable_seconds: float = 1.0, exclude_pid: int | None = None):
+            deadline = time.monotonic() + timeout
+            stable_pid = None
+            stable_since = None
+            while time.monotonic() < deadline:
+                active = active_runners()
+                assert len(active) <= 1
+                if len(active) == 1 and active[0].pid != exclude_pid:
+                    pid = active[0].pid
+                    if pid != stable_pid:
+                        stable_pid = pid
+                        stable_since = time.monotonic()
+                    elif stable_since is not None and time.monotonic() - stable_since >= stable_seconds:
+                        return pid
+                else:
+                    stable_pid = None
+                    stable_since = None
+                time.sleep(0.25)
+            return None
+
         procs = [
             run_archivebox_cmd(
                 ["run", "--daemon"],
@@ -615,40 +651,8 @@ class TestRunDaemonMode:
             for _ in range(2)
         ]
         try:
-            deadline = time.monotonic() + 30
-            active_pid = None
-            while time.monotonic() < deadline:
-                with use_archivebox_db(initialized_archive):
-                    active = [
-                        proc
-                        for proc in Process.objects.filter(
-                            process_type=Process.TypeChoices.ORCHESTRATOR,
-                            worker_type=RUNNER_ACTIVE_WORKER_TYPE,
-                            status=Process.StatusChoices.RUNNING,
-                            pwd=str(initialized_archive),
-                        )
-                        if proc.is_running
-                    ]
-                    assert len(active) <= 1
-                    if len(active) == 1:
-                        active_pid = active[0].pid
-                        break
-                time.sleep(0.25)
-
+            active_pid = wait_for_stable_single_active(timeout=30)
             assert active_pid is not None
-            time.sleep(1)
-            with use_archivebox_db(initialized_archive):
-                active = [
-                    proc
-                    for proc in Process.objects.filter(
-                        process_type=Process.TypeChoices.ORCHESTRATOR,
-                        worker_type=RUNNER_ACTIVE_WORKER_TYPE,
-                        status=Process.StatusChoices.RUNNING,
-                        pwd=str(initialized_archive),
-                    )
-                    if proc.is_running
-                ]
-                assert len(active) == 1
 
             os.killpg(active_pid, signal.SIGKILL)
             replacement = run_archivebox_cmd(
@@ -662,26 +666,7 @@ class TestRunDaemonMode:
                 wait=False,
             )
             procs.append(replacement)
-            deadline = time.monotonic() + 30
-            recovered_pid = None
-            while time.monotonic() < deadline:
-                with use_archivebox_db(initialized_archive):
-                    active = [
-                        proc
-                        for proc in Process.objects.filter(
-                            process_type=Process.TypeChoices.ORCHESTRATOR,
-                            worker_type=RUNNER_ACTIVE_WORKER_TYPE,
-                            status=Process.StatusChoices.RUNNING,
-                            pwd=str(initialized_archive),
-                        )
-                        if proc.is_running
-                    ]
-                    assert len(active) <= 1
-                    if len(active) == 1 and active[0].pid != active_pid:
-                        recovered_pid = active[0].pid
-                        break
-                time.sleep(0.25)
-
+            recovered_pid = wait_for_stable_single_active(timeout=30, exclude_pid=active_pid)
             assert recovered_pid is not None
         finally:
             for proc in procs:
@@ -1006,6 +991,42 @@ class TestRecoverOrchestratorState:
         assert hook_names
         assert all(not hook_name.endswith((".py", ".js", ".sh")) for hook_name in hook_names)
 
+    def test_snapshot_hooks_for_pending_archiveresults_respects_disabled_plugins_when_plugins_empty(self):
+        from django.utils import timezone
+
+        from archivebox.base_models.models import get_or_create_system_user_pk
+        from archivebox.crawls.models import Crawl
+        from archivebox.core.models import Snapshot
+        from archivebox.services.runner import snapshot_hooks_for_pending_archiveresults
+
+        crawl = Crawl.objects.create(
+            urls="https://example.com",
+            created_by_id=get_or_create_system_user_pk(),
+            config={
+                "CLAUDECHROME_ENABLED": False,
+                "CLAUDECODEEXTRACT_ENABLED": False,
+                "CLAUDECODECLEANUP_ENABLED": False,
+                "SEARCH_BACKEND_SQLITE_ENABLED": False,
+            },
+            status=Crawl.StatusChoices.STARTED,
+            retry_at=timezone.now(),
+        )
+        snapshot = Snapshot.objects.create(
+            url="https://example.com",
+            crawl=crawl,
+            status=Snapshot.StatusChoices.QUEUED,
+            retry_at=timezone.now(),
+        )
+
+        hooks = snapshot_hooks_for_pending_archiveresults(snapshot)
+        queued_plugins = {plugin for plugin, _hook_name in hooks}
+
+        assert "title" in queued_plugins
+        assert "claudechrome" not in queued_plugins
+        assert "claudecodeextract" not in queued_plugins
+        assert "claudecodecleanup" not in queued_plugins
+        assert "search_backend_sqlite" not in queued_plugins
+
     def test_run_due_snapshot_pauses_child_when_parent_is_paused(self):
         from django.utils import timezone
 
@@ -1482,10 +1503,77 @@ class TestRecoverOrchestratorState:
         assert result.status == ArchiveResult.StatusChoices.FAILED
         assert snapshot.retry_at is None
 
+    @pytest.mark.django_db(transaction=True)
+    def test_run_due_snapshot_skips_disabled_queued_plugins_and_seals_started_snapshot(self):
+        from django.utils import timezone
+
+        from archivebox.base_models.models import get_or_create_system_user_pk
+        from archivebox.crawls.models import Crawl
+        from archivebox.core.models import ArchiveResult, Snapshot
+        from archivebox.services.runner import run_due_snapshot
+
+        crawl = Crawl.objects.create(
+            urls="https://example.com",
+            created_by_id=get_or_create_system_user_pk(),
+            config={
+                "CLAUDECHROME_ENABLED": False,
+                "CLAUDECODEEXTRACT_ENABLED": False,
+                "CLAUDECODECLEANUP_ENABLED": False,
+                "SEARCH_BACKEND_SQLITE_ENABLED": False,
+            },
+            status=Crawl.StatusChoices.STARTED,
+            retry_at=timezone.now(),
+        )
+        snapshot = Snapshot.objects.create(
+            url="https://example.com",
+            crawl=crawl,
+            status=Snapshot.StatusChoices.STARTED,
+            retry_at=timezone.now(),
+            downloaded_at=timezone.now(),
+        )
+        ArchiveResult.objects.create(
+            snapshot=snapshot,
+            plugin="title",
+            hook_name="on_Snapshot__01_title",
+            status=ArchiveResult.StatusChoices.SUCCEEDED,
+            output_str="Example Domain",
+        )
+        stale_results = [
+            ArchiveResult.objects.create(
+                snapshot=snapshot,
+                plugin=plugin,
+                hook_name=hook_name,
+                status=ArchiveResult.StatusChoices.QUEUED,
+            )
+            for plugin, hook_name in (
+                ("claudechrome", "on_Snapshot__47_claudechrome"),
+                ("claudecodeextract", "on_Snapshot__58_claudecodeextract"),
+                ("claudecodecleanup", "on_Snapshot__92_claudecodecleanup"),
+                ("search_backend_sqlite", "on_Snapshot__90_index_sqlite"),
+            )
+        ]
+        stale_result_ids = [result.id for result in stale_results]
+        stale_plugins = [result.plugin for result in stale_results]
+
+        assert run_due_snapshot(snapshot, lock_seconds=60) is True
+
+        snapshot.refresh_from_db()
+        assert snapshot.status == Snapshot.StatusChoices.SEALED
+        assert snapshot.retry_at is None
+        assert not snapshot.archiveresult_set.filter(
+            plugin__in=stale_plugins,
+            status=ArchiveResult.StatusChoices.QUEUED,
+        ).exists()
+        for result in ArchiveResult.objects.filter(id__in=stale_result_ids):
+            assert result.status == ArchiveResult.StatusChoices.SKIPPED
+            assert result.start_ts is not None
+            assert result.end_ts is not None
+            assert "disabled by this Snapshot/Crawl config" in result.output_str
+
     @pytest.mark.django_db(transaction=True)
     @pytest.mark.timeout(300)
     @pytest.mark.parametrize("chrome_isolation", ["crawl", "snapshot"])
-    def test_resume_queued_chrome_wait_reruns_full_chrome_plugin_lifecycle(
+    def test_resume_queued_chrome_wait_reruns_background_prerequisites(
         self,
         initialized_archive,
         recursive_test_site,

archivebox/tests/test_cli_server.py

@@ -14,6 +14,7 @@ import subprocess
 import sys
 import time
 from datetime import datetime
+from pathlib import Path
 from types import SimpleNamespace
 
 import pytest
@@ -98,6 +99,43 @@ def test_server_auth_secret_and_cookie_settings_are_restart_stable(tmp_path, mon
     assert first_lines[3:] == ["None", "False", "False"]
 
 
+def test_https_base_url_enables_proxy_ssl_header_and_secure_cookies(tmp_path):
+    (tmp_path / ".archivebox_id").write_text("testcoll")
+    env = os.environ.copy()
+    env["BASE_URL"] = "https://archive.example.com"
+    env["DJANGO_SETTINGS_MODULE"] = "archivebox.core.settings"
+    repo_root = Path(__file__).resolve().parents[2]
+    env["PYTHONPATH"] = f"{repo_root}{os.pathsep}{env.get('PYTHONPATH', '')}"
+
+    result = subprocess.run(
+        [
+            sys.executable,
+            "-c",
+            (
+                "import django, json;"
+                "django.setup();"
+                "from django.conf import settings;"
+                "print(json.dumps({"
+                "'csrf_secure': settings.CSRF_COOKIE_SECURE,"
+                "'session_secure': settings.SESSION_COOKIE_SECURE,"
+                "'proxy_ssl_header': settings.SECURE_PROXY_SSL_HEADER,"
+                "}))"
+            ),
+        ],
+        capture_output=True,
+        text=True,
+        check=True,
+        env=env,
+        cwd=tmp_path,
+    )
+
+    assert json.loads(result.stdout) == {
+        "csrf_secure": True,
+        "session_secure": True,
+        "proxy_ssl_header": ["HTTP_X_FORWARDED_PROTO", "https"],
+    }
+
+
 def test_sqlite_connections_use_explicit_busy_timeout():
     from archivebox.core.settings import SQLITE_CONNECTION_OPTIONS
 
@@ -169,8 +207,7 @@ def test_reload_workers_use_current_interpreter_and_supervisord_managed_runner()
 
 
 def test_server_daemon_starts_real_plugin_owned_sonic_worker(archivebox_daemon_server):
-    if shutil.which("sonic") is None:
-        pytest.skip("sonic server binary is required for Sonic worker integration tests")
+    assert shutil.which("sonic") is not None, "sonic server binary is required for Sonic worker integration tests"
 
     server = archivebox_daemon_server(
         SEARCH_BACKEND_ENGINE="sonic",
@@ -294,8 +331,7 @@ def test_sonic_worker_is_disabled_when_sonic_disabled(tmp_path):
 
 
 def test_sonic_daemon_event_handler_accepts_real_running_worker(archivebox_daemon_server):
-    if shutil.which("sonic") is None:
-        pytest.skip("sonic server binary is required for Sonic worker integration tests")
+    assert shutil.which("sonic") is not None, "sonic server binary is required for Sonic worker integration tests"
 
     from abx_dl.events import ProcessStdoutEvent
     from abx_dl.orchestrator import create_bus

archivebox/tests/test_cli_version.py

@@ -8,7 +8,9 @@ import os
 import re
 import tempfile
 from pathlib import Path
-from archivebox.tests.conftest import run_archivebox_cmd
+from archivebox.config.paths import tmp_dir_socket_path_is_short_enough
+from archivebox.cli.archivebox_version import _binary_row_dedupe_key
+from archivebox.tests.conftest import cli_env, run_archivebox_cmd
 
 
 def _make_deep_collection_dir(tmp_path: Path) -> Path:
@@ -29,6 +31,60 @@ def _extract_location_path(output: str, key: str) -> Path:
     raise AssertionError(f"Did not find a {key} location line in output:\n{output}")
 
 
+def test_binary_row_dedupe_key_keeps_distinct_paths_visible(tmp_path):
+    first_path = tmp_path / "lib" / "env" / "bin" / "node"
+    second_path = tmp_path / "other" / "bin" / "node"
+    first_path.parent.mkdir(parents=True)
+    second_path.parent.mkdir(parents=True)
+
+    first = _binary_row_dedupe_key(
+        display_name="node",
+        valid=True,
+        version="26.0.0",
+        provider="env",
+        abspath=str(first_path),
+    )
+    repeat = _binary_row_dedupe_key(
+        display_name="node",
+        valid=True,
+        version="26.0.0",
+        provider="env",
+        abspath=str(first_path),
+    )
+    different_path = _binary_row_dedupe_key(
+        display_name="node",
+        valid=True,
+        version="26.0.0",
+        provider="env",
+        abspath=str(second_path),
+    )
+
+    assert repeat == first
+    assert different_path != first
+
+
+def test_binary_row_dedupe_key_collapses_enabled_and_disabled_plugin_references(tmp_path):
+    binary_path = tmp_path / "bin" / "node"
+    binary_path.parent.mkdir(parents=True)
+
+    enabled_reference = _binary_row_dedupe_key(
+        display_name="node",
+        valid=True,
+        version="26.0.0",
+        provider="env",
+        abspath=str(binary_path),
+    )
+    disabled_reference = _binary_row_dedupe_key(
+        display_name="node",
+        valid=True,
+        version="26.0.0",
+        provider="env",
+        abspath=str(binary_path),
+    )
+
+    assert disabled_reference == enabled_reference
+
+
 def test_version_quiet_outputs_version_number(tmp_path):
     """Test that version --quiet outputs just the version number."""
     result = run_archivebox_cmd(["version", "--quiet"])
@@ -70,6 +126,58 @@ def test_version_shows_binaries_after_init(tmp_path, initialized_archive):
     assert "Binary" in output or "Dependencies" in output
 
 
+def test_version_skips_disabled_plugin_binary_resolution(tmp_path):
+    """Disabled plugins should not trigger live binary detection during version."""
+    data_dir = tmp_path / "no-plugins"
+    data_dir.mkdir()
+    env = cli_env(PLUGINS="__archivebox_test_no_plugins__")
+
+    init_result = run_archivebox_cmd(["init"], cwd=data_dir, env=env)
+    assert init_result.returncode == 0, init_result.stderr
+
+    version_result = run_archivebox_cmd(["version"], cwd=data_dir, env=env)
+    output = version_result.stdout + version_result.stderr
+
+    assert version_result.returncode == 0, output
+    assert "No required binaries declared for discovered plugins" in output
+    assert "not installed" not in output
+
+
+def test_version_honors_legacy_save_aliases_when_disabling_extractors(tmp_path):
+    """Legacy SAVE_* aliases should disable their canonical plugin flags."""
+    data_dir = tmp_path / "legacy-save-disabled"
+    data_dir.mkdir()
+    env = {"PLUGINS": "wget,title"}
+
+    init_result = run_archivebox_cmd(["init"], cwd=data_dir, env=env, default_cli_env=True, disable_extractors=True, timeout=120)
+    assert init_result.returncode == 0, init_result.stderr or init_result.stdout
+
+    config_result = run_archivebox_cmd(
+        ["config", "--get", "WGET_ENABLED", "TITLE_ENABLED"],
+        cwd=data_dir,
+        env=env,
+        default_cli_env=True,
+        disable_extractors=True,
+    )
+    config_output = config_result.stdout + config_result.stderr
+    assert config_result.returncode == 0, config_output
+    assert "WGET_ENABLED = false" in config_output
+    assert "TITLE_ENABLED = false" in config_output
+
+    version_result = run_archivebox_cmd(
+        ["version"],
+        cwd=data_dir,
+        env=env,
+        default_cli_env=True,
+        disable_extractors=True,
+        timeout=60,
+    )
+    output = version_result.stdout + version_result.stderr
+
+    assert version_result.returncode == 0, output
+    assert "not installed" not in output
+
+
 def test_version_shows_data_locations(tmp_path, initialized_archive):
     """Test that version shows data directory locations."""
     result = run_archivebox_cmd(["version"])
@@ -123,7 +231,7 @@ def test_version_auto_selects_short_tmp_dir_for_deep_collection_path(tmp_path):
 
     assert reported_tmp_dir.exists()
     assert not reported_tmp_dir.is_relative_to(default_tmp_dir)
-    assert len(f"file://{reported_tmp_dir / 'supervisord.sock'}") <= 96
+    assert tmp_dir_socket_path_is_short_enough(reported_tmp_dir)
 
 
 def test_version_help_lists_quiet_flag(tmp_path):

archivebox/tests/test_config_DELETE_AFTER.py

@@ -154,7 +154,7 @@ print(json.dumps({{
     retained = json.loads(stdout.strip().splitlines()[-1])
     assert retained["crawl_id"] == created["crawl_id"]
 
-    _cmd_result = run_archivebox_cmd(["run", "--crawl-id", retained["crawl_id"]], cwd=tmp_path, timeout=120, env=run_env)
+    _cmd_result = run_archivebox_cmd(["run", "--maintenance-only"], cwd=tmp_path, timeout=120, env=run_env)
     stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
     assert returncode == 0, f"archivebox run failed:\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}"
 
@@ -206,7 +206,7 @@ def test_delete_after_real_add_page_and_rest_create_paths(client):
             "depth": "0",
             "max_urls": "1",
             "crawl_max_size": "0",
-            "crawl_timeout": "0",
+            "crawl_timeout": "60",
             "snapshot_max_size": "0",
             "delete_after": "2h",
             "crawl_max_concurrent_snapshots": "1",
@@ -233,6 +233,7 @@ def test_delete_after_real_add_page_and_rest_create_paths(client):
     assert run_due_crawl(ui_crawl, lock_seconds=10)
     ui_snapshot = ui_crawl.snapshot_set.get(url="https://example.com/delete-after-ui")
     assert ui_snapshot.delete_at is not None
+    assert not ui_snapshot.output_dir.joinpath("staticfile", "stdin.txt").exists()
 
     from archivebox.api.auth import get_or_create_api_token
 

archivebox/tests/test_config_MAX_limits.py

@@ -115,7 +115,7 @@ def test_run_snapshot_seals_descendant_when_crawl_max_size_is_reached(tmp_path):
     crawl = Crawl.objects.create(
         urls="https://example.com",
         config={
-            "LIB_DIR": str(tmp_path / "lib"),
+            "ABXPKG_LIB_DIR": str(tmp_path / "lib"),
             "PLUGINS": "__archivebox_test_no_plugins__",
             "CHROME_BINARY": "",
             "CRAWL_MAX_SIZE": 16,
@@ -292,4 +292,5 @@ def test_recursive_crawl_respects_max_urls(tmp_path, initialized_archive, recurs
     assert depth_counts.get(0, 0) == 1
     assert depth_counts.get(1, 0) == 3
     assert depth_counts.get(2, 0) == 0
+    assert depth_counts.get(3, 0) == 0
     assert set(recursive_test_site["child_urls"]).issubset({url for url, depth, _parent in snapshot_rows if depth == 1})

archivebox/tests/test_config_SAVE_TITLE.py

@@ -14,7 +14,7 @@ pytestmark = pytest.mark.django_db(transaction=True)
 
 def _install_chrome(tmp_path, env):
     env["CHROME_ISOLATION"] = "snapshot"
-    env["LIB_DIR"] = str(tmp_path / "lib")
+    env["ABXPKG_LIB_DIR"] = str(tmp_path / "lib")
     install_process = run_archivebox_cmd(
         ["install", "chrome"],
         cwd=tmp_path,
@@ -22,7 +22,7 @@ def _install_chrome(tmp_path, env):
         timeout=600,
     )
     assert install_process.returncode == 0, install_process.stderr or install_process.stdout
-    cached_browser = _find_cached_chrome(Path(env["LIB_DIR"]))
+    cached_browser = _find_cached_chrome(Path(env["ABXPKG_LIB_DIR"]))
     if cached_browser is not None:
         env["CHROME_BINARY"] = str(cached_browser)
         return

archivebox/tests/test_core_forms.py

@@ -0,0 +1,31 @@
+import pytest
+
+from archivebox.core.forms import TagField, TagWidget
+from archivebox.core.models import Tag
+
+
+pytestmark = pytest.mark.django_db
+
+
+def test_tag_field_parses_legacy_tag_input():
+    field = TagField()
+
+    assert field.clean("alpha beta alpha") == ["alpha", "beta"]
+    assert field.clean("alpha, beta, Alpha") == ["Alpha", "alpha", "beta"]
+    assert field.clean('"alpha beta", gamma') == ["alpha beta", "gamma"]
+    assert field.clean('"alpha,beta", gamma') == ["alpha,beta", "gamma"]
+    assert field.clean('alpha "beta gamma"') == ["alpha", "beta gamma"]
+    assert field.clean('"alpha,beta') == ["alpha", "beta"]
+
+
+def test_tag_widget_formats_real_tag_rows():
+    tags = [
+        Tag.objects.create(name="plain"),
+        Tag.objects.create(name="two words"),
+        Tag.objects.create(name="comma,tag"),
+    ]
+
+    rendered_value = TagWidget().format_value(tags)
+
+    assert rendered_value == '"comma,tag", "two words", plain'
+    assert TagField().clean(rendered_value) == ["comma,tag", "plain", "two words"]