Bump release version to 0.9.35rc467

Remove unused Python helpers and simplify equivalent test code without
changing active CLI, REST, model, runner, or UI behavior. This is the
mechanical first part of the simplification split; #1879 contains the
architectural and behavior-sensitive remainder. **Merge this PR first.**

- Delete unused hashing helpers, JSONL file loading, legacy TOML/INI
conversion helpers, and plugin-model discovery. Caller searches found no
active consumers; retained production definitions are AST-identical to
`dev`.
- Remove redundant result aliases, share identical POST payloads, and
parameterize equivalent test pairs while retaining their cases.
- Address Cubic's reviews: parse CSV and assert the exact header, URL
values, and sorted order; qualify the TLSNotary result lookup by its
full snapshot/plugin/hook identity.
- Correct the all-plugins HTTP fixture's TLSNotary expectation: every
snapshot must produce the specific HTTPS-only refusal, exit 1, and no
captured output. Other plugin statuses, output files, crawl topology,
and concurrent execution assertions remain enforced.
- Complete real parser dependency installation before public-add tests
deliberately stop the server. This prevents interrupting virtualenv
creation while preserving the crawl restart and metadata assertions.

Based on `dev` at `1fa2d4dc9`. **550 net lines removed**, excluding
`old/` and `docs/`; neither directory changes here. Runtime routing,
models, schemas, UI, dependencies, and architecture remain in #1879.

Validation:

- All-files `uv run --no-sync prek run --all-files` passes.
- The initial equivalent cleanup passed AST checks and 85 targeted
tests. Review follow-ups additionally pass both exact CSV tests, both
affected public-add runtime tests with a fresh dependency directory and
`--no-cache --no-sources`, and the full real depth-two all-plugins
crawl.
- [Full CI at
`42ed15b73`](https://github.com/ArchiveBox/ArchiveBox/actions/runs/35300660816)
**passed**, including every required lane. The subsequent one-line
TLSNotary lookup correction passed the full real all-plugins crawl
locally (131 seconds). All Cubic review threads are resolved.
- [Fresh CI at
`50a462e78`](https://github.com/ArchiveBox/ArchiveBox/actions/runs/35304078254)
is running.

Dockerfile

@@ -10,10 +10,10 @@
 #       --build-context abx-plugins=../abx-plugins \
 #       -t archivebox/abx-dl:dev
 #   docker buildx build . -f Dockerfile \
-#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:1.12.274 \
+#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:1.12.285 \
 #       -t archivebox:multistage
 
-ARG ABX_DL_IMAGE=archivebox/abx-dl:1.12.274
+ARG ABX_DL_IMAGE=archivebox/abx-dl:1.12.285
 
 FROM archivebox/sonic:1.4.9 AS sonic
 FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base
@@ -96,17 +96,15 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
     --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
     <<'EOF'
 echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..."
-echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache
-echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends
-echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests
+export UV_NO_CACHE=true
 rm -f /etc/apt/apt.conf.d/docker-clean
 apt-get update -qq
-apt-get install -qq -y --no-install-recommends \
-    build-essential gcc libldap2-dev libsasl2-dev libssl-dev x11-utils
+apt-get install -qq -y --no-install-recommends x11-utils
 # Extend abx-dl's existing venv instead of clearing it. Clearing and later
 # copying a complete replacement over the base duplicates every venv byte in
 # the final overlay history even when most packages are unchanged.
-/usr/bin/uv pip install --no-cache setuptools pip wheel
+# Do not rewrite inherited native libraries merely to strip them again.
+/usr/bin/find /venv/lib/python3.*/site-packages -type f -name '*.so' -print > /tmp/archivebox-inherited-libraries
 
 mkdir -p /tmp/archivebox-uv-project
 /venv/bin/python - <<'PY'
@@ -155,25 +153,31 @@ PY
     --inexact \
     --no-install-project \
     --no-install-workspace \
+    --no-install-package abxbus \
+    --no-install-package abxpkg \
+    --no-install-package abx-plugins \
+    --no-install-package abx-dl \
     --no-sources
 builder_abxpkg_lib_dir=/tmp/archivebox-builder-abxpkg
 ABXPKG_NO_CACHE=True abxpkg env --install --binproviders=env,apt --lib="$builder_abxpkg_lib_dir" --overrides='{"apt":{"install_args":["binutils"]}}' strip >/dev/null
 /usr/bin/find /venv/lib/python3.*/site-packages -type f -name '*.so' -print0 > /tmp/archivebox-native-libraries
 while IFS= read -r -d '' native_library; do
+    if grep -Fxq "$native_library" /tmp/archivebox-inherited-libraries; then
+        continue
+    fi
     magic=''
     if IFS= read -r -N 4 magic < "$native_library" && [[ "$magic" == $'\x7fELF' ]]; then
         "$builder_abxpkg_lib_dir/env/bin/strip" --strip-unneeded "$native_library" || exit $?
     fi
 done < /tmp/archivebox-native-libraries
-rm -f /tmp/archivebox-native-libraries
+rm -f /tmp/archivebox-native-libraries /tmp/archivebox-inherited-libraries
 rm -f /venv/bin/uv /venv/bin/uvx
-abxpkg run --binproviders=env --lib="$builder_abxpkg_lib_dir" apt-get purge -y binutils build-essential gcc libldap2-dev libsasl2-dev libssl-dev
+abxpkg run --binproviders=env --lib="$builder_abxpkg_lib_dir" apt-get purge -y binutils
 abxpkg run --binproviders=env --lib="$builder_abxpkg_lib_dir" apt-get autoremove -y
 rm -rf "$builder_abxpkg_lib_dir"
 rm -rf /venv/lib/python3.*/site-packages/pip* \
     /venv/lib/python3.*/site-packages/wheel* \
     /venv/bin/pip /venv/bin/pip3 /venv/bin/pip3.* /venv/bin/wheel
-echo 'Binary::apt::APT::Keep-Downloaded-Packages "0";' > /etc/apt/apt.conf.d/99keep-cache
 rm -rf /var/lib/apt/lists/* /tmp/archivebox-uv-project
 EOF
 
@@ -202,7 +206,7 @@ RUN echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
     && test -f "$(/venv/bin/python -c 'import archivebox; print(archivebox.__cached__)')" \
     && /usr/bin/uv pip show archivebox | tee -a /VERSION.txt
 
-# The builder installs and purges compilers in one layer, so its final
+# The builder installs and purges stripping tools in one layer, so its final
 # filesystem is already runtime-clean. Preserve that ancestry: starting again
 # from archivebox-runtime-base and COPYing /venv would bake a second full venv
 # over the inherited one instead of recording only ArchiveBox's package delta.
@@ -225,7 +229,8 @@ COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
 # The builder invokes abxpkg as root for temporary ELF stripping, which can
 # rewrite derived state ownership. Restore UID 911 before resolving binaries
 # as the runtime user; the old copy-based final stage hid this dependency.
-RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_ARCHIVEBOX_UID}..." \
+RUN --mount=type=cache,target=/opt/archivebox/lib/cache,sharing=locked,mode=1777,id=archivebox-runtime-config-$TARGETARCH \
+    echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_ARCHIVEBOX_UID}..." \
     && printf 'export PATH="/venv/bin:/opt/node/bin:$PATH"\n' > /etc/profile.d/archivebox-path.sh \
     && ln -sf /venv/bin/archivebox /usr/local/bin/archivebox \
     && ln -sf /venv/bin/daphne /usr/local/bin/daphne \
@@ -239,8 +244,8 @@ RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_ARCHIVEBOX_UID}..."
     && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$XDG_CACHE_HOME" "$PLAYWRIGHT_BROWSERS_PATH" \
     && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "/home/$ARCHIVEBOX_USER" \
     && chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" \
-    && chown -R "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "$ABXPKG_LIB_DIR" \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abxpkg load --binproviders=env sonic | tee -a /VERSION.txt \
+    && find "$ABXPKG_LIB_DIR" \( ! -user "$DEFAULT_ARCHIVEBOX_UID" -o ! -group "$DEFAULT_ARCHIVEBOX_GID" \) -exec chown -h "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" {} + \
+    && UV_NO_CACHE=true setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abxpkg load --binproviders=env sonic | tee -a /VERSION.txt \
     && openssl rand -hex 16 > /etc/machine-id \
     && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER ARCHIVEBOX_UID=$(id -u "$ARCHIVEBOX_USER") ARCHIVEBOX_GID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
     && echo -e "TMP_DIR=$TMP_DIR\nABXPKG_LIB_DIR=$ABXPKG_LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt

archivebox/api/v1_auth.py

@@ -4,7 +4,7 @@ from django.http import HttpRequest
 
 from ninja import Router, Schema
 
-from archivebox.api.auth import auth_using_token, auth_using_password, get_or_create_api_token
+from archivebox.api.auth import auth_using_token, auth_using_password, get_or_create_api_token, HeaderTokenAuth, BearerTokenAuth
 
 
 router = Router(tags=["Authentication"], auth=None)
@@ -63,3 +63,44 @@ def check_api_token(request: HttpRequest, token_data: TokenAuthSchema):
         return {"success": True, "user_id": str(user.pk)}
 
     return {"success": False, "user_id": None}
+
+
+@router.post(
+    "/browser_session",
+    auth=[HeaderTokenAuth(), BearerTokenAuth()],
+    summary="Exchange an administrator API key for a normal browser session",
+)
+def browser_session(request: HttpRequest):
+    from django.conf import settings
+    from django.contrib.auth import login
+    from django.http import JsonResponse
+    from django.utils import timezone
+    from ninja.errors import HttpError
+    from archivebox.core.routes_util import get_admin_base_url
+
+    user = request.auth
+    if not user.is_active or not user.is_superuser:
+        raise HttpError(403, "An active administrator is required")
+    login(request, user, backend="django.contrib.auth.backends.ModelBackend")
+    # A browser session must not outlive the key used to establish it.
+    token = request._api_token
+    lifetime = settings.SESSION_COOKIE_AGE
+    if token.expires:
+        lifetime = min(lifetime, max(0, int((token.expires - timezone.now()).total_seconds())))
+    request.session.set_expiry(lifetime)
+    request.session.save()
+    response = JsonResponse(
+        {
+            "admin_url": get_admin_base_url(request=request).rstrip("/") + "/admin/",
+            "cookie": {
+                "name": settings.SESSION_COOKIE_NAME,
+                "value": request.session.session_key,
+                "expires": request.session.get_expiry_date().timestamp(),
+                "secure": settings.SESSION_COOKIE_SECURE,
+            },
+        },
+    )
+    # The native client installs a host-only HttpOnly cookie on the admin origin;
+    # API and admin hosts can differ, so a Set-Cookie on the API host is insufficient.
+    response["Cache-Control"] = "no-store"
+    return response

archivebox/cli/archivebox_add.py

@@ -388,10 +388,6 @@ def main(**kwargs):
             raise click.BadParameter(str(err), param_hint="--snapshot-max-size") from err
         if kwargs.get("crawl_max_concurrent_snapshots") is not None and int(kwargs["crawl_max_concurrent_snapshots"]) < 1:
             raise click.BadParameter("crawl_max_concurrent_snapshots must be at least 1.", param_hint="--crawl-max-concurrent-snapshots")
-        if not kwargs.get("index_only"):
-            from archivebox.workers.supervisord_util import require_crawl_memory
-
-            require_crawl_memory()
 
         # Translate --only-new/--no-only-new into a crawl config override.
         # add() takes config overrides as a dict; no per-flag kwargs.

archivebox/cli/archivebox_server.py

@@ -363,12 +363,10 @@ def server(
     command = current_command(Process.TypeChoices.SERVER, data_dir=CONSTANTS.DATA_DIR, url=bind_url)
 
     def still_owns_runtime_stack() -> bool:
-        from django.db import connections
-
-        try:
-            return command_owns_runtime_stack(command, data_dir=CONSTANTS.DATA_DIR)
-        finally:
-            connections.close_all()
+        # This is polled at the log-tail cadence. Keep the autocommit
+        # connection open so SQLite can reuse its page cache between polls;
+        # reconnecting here repeatedly rereads the process table while idle.
+        return command_owns_runtime_stack(command, data_dir=CONSTANTS.DATA_DIR)
 
     shutdown_state = None
     try:
@@ -416,8 +414,13 @@ def server(
     except KeyboardInterrupt:
         pass
     finally:
-        if not shutdown_state or not shutdown_state.signal_name:
-            command.mark_exited()
+        from django.db import connections
+
+        try:
+            if not shutdown_state or not shutdown_state.signal_name:
+                command.mark_exited()
+        finally:
+            connections.close_all()
     print("\n[i][green][🟩] ArchiveBox server shut down gracefully.[/green][/i]")
 
 

archivebox/core/views.py

@@ -765,6 +765,11 @@ def _plugin_full_preview_response(
     response.headers["X-Content-Type-Options"] = "nosniff"
     response.headers["X-ArchiveBox-Security-Mode"] = request.archivebox_config.SERVER_SECURITY_MODE
     response.headers["Referrer-Policy"] = "no-referrer"
+    # Trusted viewers live on the snapshot origin, but the collection UI lives
+    # on web.* (and can be embedded by admin.*). Permit only those configured
+    # origins, not arbitrary sites or other snapshots. This does not grant the
+    # archived document access to the parent frame or its admin session cookies.
+    viewer_origins = " ".join(dict.fromkeys((build_web_url(request=request), build_admin_url(request=request))))
     response.headers["Content-Security-Policy"] = (
         "default-src 'self' data: blob:; "
         "script-src 'self' 'unsafe-inline' 'unsafe-eval' data: blob:; "
@@ -778,7 +783,7 @@ def _plugin_full_preview_response(
         "object-src 'none'; "
         "base-uri 'none'; "
         "form-action 'none'; "
-        "frame-ancestors 'self';"
+        f"frame-ancestors 'self' {viewer_origins};"
     )
     return response
 

archivebox/crawls/admin.py

@@ -1230,7 +1230,7 @@ class CrawlScheduleAdmin(BaseModelAdmin):
         (
             "Configuration",
             {
-                "fields": ("schedule", "template"),
+                "fields": ("is_enabled", "schedule", "template"),
                 "classes": ("card",),
             },
         ),
@@ -1276,7 +1276,21 @@ class CrawlScheduleAdmin(BaseModelAdmin):
 
     def change_view(self, request, object_id, form_url="", extra_context=None):
         self.request = request
-        return super().change_view(request, object_id, form_url, extra_context)
+        from django.db import connections, router
+
+        connection = connections[router.db_for_write(self.model)]
+        if request.method != "POST" or connection.vendor != "sqlite":
+            return super().change_view(request, object_id, form_url, extra_context)
+        # Reserve the writer before Django's atomic admin form reads. A
+        # deferred read transaction cannot safely upgrade while the runner
+        # writes, even when SQLite's existing busy timeout has time left.
+        connection.ensure_connection()
+        previous_mode = connection.transaction_mode
+        connection.transaction_mode = "IMMEDIATE"
+        try:
+            return super().change_view(request, object_id, form_url, extra_context)
+        finally:
+            connection.transaction_mode = previous_mode
 
     def add_view(self, request, form_url="", extra_context=None):
         return redirect("/add/#schedule")

archivebox/misc/hashing.py

@@ -1,260 +0,0 @@
-# Bootable hashing helpers (file content + mtime/size cache, mime detection).
-# MUST NOT import archivebox.config, archivebox.core, or Django — stdlib only.
-
-import hashlib
-import mimetypes
-from functools import lru_cache
-from pathlib import Path
-from collections.abc import Callable
-from datetime import datetime
-
-
-@lru_cache(maxsize=1024)
-def _cached_file_hash(filepath: str, size: int, mtime: float) -> str:
-    """Internal function to calculate file hash with cache key based on path, size and mtime."""
-    sha256_hash = hashlib.sha256()
-
-    with open(filepath, "rb") as f:
-        for chunk in iter(lambda: f.read(4096), b""):
-            sha256_hash.update(chunk)
-
-    return sha256_hash.hexdigest()
-
-
-@lru_cache(maxsize=10)
-def hash_file(file_path: Path, pwd: Path | None = None) -> str:
-    """Calculate SHA256 hash of a file with caching based on path, size and mtime."""
-    pwd = Path(pwd) if pwd else None
-    file_path = Path(file_path)
-    if not file_path.is_absolute():
-        file_path = pwd / file_path if pwd else file_path.absolute()
-
-    abs_path = file_path.resolve()
-    stat_info = abs_path.stat()
-
-    return _cached_file_hash(
-        str(abs_path),
-        stat_info.st_size,
-        stat_info.st_mtime,
-    )
-
-
-@lru_cache(maxsize=10)
-def get_dir_hashes(dir_path: Path, pwd: Path | None = None, filter_func: Callable | None = None, max_depth: int = -1) -> dict[str, str]:
-    """Calculate SHA256 hashes for all files and directories recursively."""
-    pwd = Path(pwd) if pwd else None
-    dir_path = Path(dir_path)
-    if not dir_path.is_absolute():
-        dir_path = pwd / dir_path if pwd else dir_path.absolute()
-
-    if not dir_path.is_dir():
-        raise ValueError(f"Not a directory: {dir_path}")
-    if max_depth < -1:
-        raise ValueError(f"max_depth must be >= -1, got {max_depth}")
-
-    # Get all files recursively
-    all_files = get_dir_entries(
-        dir_path,
-        pwd=pwd,
-        recursive=True,
-        include_files=True,
-        include_dirs=False,
-        filter_func=filter_func,
-    )
-
-    hashes: dict[str, str] = {}
-    hashable_summary = []
-
-    # Calculate hashes for all files
-    for subfile in all_files:
-        subfile_path = dir_path / subfile
-        sha256_hash = hash_file(subfile_path)
-        hashes[subfile] = sha256_hash
-        hashable_summary.append(f"{sha256_hash}  ./{subfile}")
-
-    # Calculate hashes for all directories
-    subdirs = get_dir_entries(
-        dir_path,
-        pwd=pwd,
-        recursive=True,
-        include_files=False,
-        include_dirs=True,
-        include_hidden=False,
-        filter_func=filter_func,
-        max_depth=max_depth,
-    )
-
-    for subdir in subdirs:
-        subdir_path = dir_path / subdir
-        subdir_hashes = get_dir_hashes(
-            subdir_path,
-            filter_func=filter_func,
-            max_depth=0,
-        )
-        hashes[subdir] = subdir_hashes["."]
-
-    # Filter results by max_depth
-    if max_depth >= 0:
-        hashes = {path: value for path, value in hashes.items() if len(Path(path).parts) <= max_depth + 1}
-
-    # Calculate root directory hash
-    hashable_summary.sort()
-    root_sha256 = hashlib.sha256("\n".join(hashable_summary).encode()).hexdigest()
-    hashes["."] = root_sha256
-
-    return hashes
-
-
-@lru_cache(maxsize=128)
-def get_dir_entries(
-    dir_path: Path,
-    pwd: Path | None = None,
-    recursive: bool = True,
-    include_files: bool = True,
-    include_dirs: bool = True,
-    include_hidden: bool = False,
-    filter_func: Callable | None = None,
-    max_depth: int = -1,
-) -> tuple[str, ...]:
-    """Get filtered list of directory entries."""
-    pwd = Path(pwd) if pwd else None
-    dir_path = Path(dir_path)
-    if not dir_path.is_absolute():
-        dir_path = pwd / dir_path if pwd else dir_path.absolute()
-
-    results = []
-
-    def process_path(path: Path, depth: int):
-        if not include_hidden and path.name.startswith("."):
-            return False
-        if max_depth >= 0 and depth > max_depth:
-            return False
-        if filter_func:
-            info = {
-                "abspath": str(path.absolute()),
-                "relpath": str(path.relative_to(dir_path)),
-            }
-            if not filter_func(info):
-                return False
-        return True
-
-    for path in dir_path.rglob("*") if recursive else dir_path.glob("*"):
-        current_depth = len(path.relative_to(dir_path).parts)
-
-        if path.is_file() and include_files and process_path(path, current_depth):
-            results.append(str(path.relative_to(dir_path)))
-        elif path.is_dir() and include_dirs and process_path(path, current_depth):
-            results.append(str(path.relative_to(dir_path)))
-
-        if not recursive:
-            break
-
-    return tuple(sorted(results))  # Make immutable for caching
-
-
-@lru_cache(maxsize=1024)
-def get_dir_sizes(dir_path: Path, pwd: Path | None = None, **kwargs) -> dict[str, int]:
-    """Calculate sizes for all files and directories recursively."""
-    sizes: dict[str, int] = {}
-    hashes = get_dir_hashes(dir_path, pwd=pwd, **kwargs)
-    dir_path = Path(dir_path)
-
-    for path_key in hashes:
-        full_path = dir_path / path_key
-        if full_path.is_file():
-            sizes[path_key] = full_path.stat().st_size
-        else:
-            total = 0
-            for file_path in full_path.rglob("*"):
-                if file_path.is_file() and not file_path.name.startswith("."):
-                    total += file_path.stat().st_size
-            sizes[path_key + "/"] = total
-
-    return sizes
-
-
-@lru_cache(maxsize=10)
-def get_dir_info(dir_path: Path, pwd: Path | None = None, filter_func: Callable | None = None, max_depth: int = -1) -> dict:
-    """Get detailed information about directory contents including hashes and sizes."""
-    pwd = Path(pwd) if pwd else None
-    dir_path = Path(dir_path)
-    if not dir_path.is_absolute():
-        dir_path = pwd / dir_path if pwd else dir_path.absolute()
-
-    hashes = get_dir_hashes(dir_path, pwd=pwd, filter_func=filter_func, max_depth=max_depth)
-    sizes = get_dir_sizes(str(dir_path), pwd=pwd, filter_func=filter_func, max_depth=max_depth)
-
-    num_total_subpaths = sum(1 for name in hashes if name != ".")
-    details = {}
-
-    for filename, sha256_hash in sorted(hashes.items()):
-        abs_path = (dir_path / filename).resolve()
-        stat_info = abs_path.stat()
-        num_subpaths = sum(1 for p in hashes if p.startswith(filename + "/"))
-        is_dir = abs_path.is_dir()
-        if is_dir:
-            mime_type = "inode/directory"
-            basename = abs_path.name
-            extension = ""
-            num_bytes = sizes[filename + "/"]
-            if filename == ".":
-                num_subpaths = num_total_subpaths
-            else:
-                filename += "/"
-                num_subpaths = num_subpaths
-        else:  # is_file
-            num_subpaths = None
-            mime_type = mimetypes.guess_type(str(abs_path))[0]
-            extension = abs_path.suffix
-            basename = abs_path.name.rsplit(extension, 1)[0]
-            num_bytes = sizes[filename]
-
-        details[filename] = {
-            "basename": basename,
-            "mime_type": mime_type,
-            "extension": extension,
-            "num_subpaths": num_subpaths,
-            "num_bytes": num_bytes,
-            "hash_sha256": sha256_hash,
-            "created_at": datetime.fromtimestamp(stat_info.st_ctime).isoformat(),
-            "modified_at": datetime.fromtimestamp(stat_info.st_mtime).isoformat(),
-        }
-
-        if filter_func and not filter_func(details[filename]):
-            del details[filename]
-
-    return details
-
-
-if __name__ == "__main__":
-    import json
-
-    dir_info = get_dir_info(Path("."), max_depth=6)
-    with open(".hashes.json", "w") as f:
-        json.dump(dir_info, f, indent=4)
-    print("Wrote .hashes.json")
-
-# Example output:
-# {
-#     ".": {
-#         "basename": "misc",
-#         "mime_type": "inode/directory",
-#         "extension": "",
-#         "num_subpaths": 25,
-#         "num_bytes": 214677,
-#         "hash_sha256": "addfacf88b2ff6b564846415fb7b21dcb7e63ee4e911bc0aec255ee354958530",
-#         "created_at": "2024-12-04T00:08:38.537449",
-#         "modified_at": "2024-12-04T00:08:38.537449"
-#     },
-#     "__init__.py": {
-#         "basename": "__init__",
-#         "mime_type": "text/x-python",
-#         "extension": ".py",
-#         "num_subpaths": null,
-#         "num_bytes": 32,
-#         "hash_sha256": "b0e5e7ff17db3b60535cf664282787767c336e3e203a43e21b6326c6fe457551",
-#         "created_at": "2024-10-08T00:51:41.001359",
-#         "modified_at": "2024-10-08T00:51:41.001359"
-#     },
-#     ...
-# }

archivebox/misc/jsonl.py

@@ -32,7 +32,6 @@ import sys
 import json
 from typing import Any, TextIO
 from collections.abc import Iterable, Iterator
-from pathlib import Path
 
 
 # Type constants for JSONL records
@@ -120,19 +119,6 @@ def read_stdin(stream: TextIO | None = None) -> Iterator[dict[str, Any]]:
             yield record
 
 
-def read_file(path: Path) -> Iterator[dict[str, Any]]:
-    """
-    Read JSONL or plain URLs from a file.
-
-    Yields parsed records as dicts.
-    """
-    with open(path) as f:
-        for line in f:
-            record = parse_line(line)
-            if record:
-                yield record
-
-
 def read_args_or_stdin(args: Iterable[str], stream: TextIO | None = None) -> Iterator[dict[str, Any]]:
     """
     Read from CLI arguments if provided, otherwise from stdin.

archivebox/misc/toml_util.py

@@ -2,94 +2,11 @@ from typing import Any, cast
 from collections.abc import Callable
 
 import json
-import ast
-import inspect
 import toml
 import re
-import configparser
 
 from pathlib import Path, PosixPath
 
-from pydantic.json_schema import GenerateJsonSchema
-from pydantic_core import to_jsonable_python
-
-JSONValue = str | bool | int | None | list["JSONValue"]
-
-TOML_HEADER = "# Converted from INI to TOML format: https://toml.io/en/\n\n"
-
-
-def load_ini_value(val: str) -> JSONValue:
-    """Convert lax INI values into strict TOML-compliant (JSON) values"""
-    if val.lower() in ("true", "yes", "1"):
-        return True
-    if val.lower() in ("false", "no", "0"):
-        return False
-    if val.isdigit():
-        return int(val)
-
-    try:
-        return ast.literal_eval(val)
-    except Exception:
-        pass
-
-    try:
-        return json.loads(val)
-    except Exception:
-        pass
-
-    return val
-
-
-def convert(ini_str: str) -> str:
-    """Convert a string of INI config into its TOML equivalent (warning: strips comments)"""
-
-    config = configparser.ConfigParser()
-    setattr(config, "optionxform", str)  # capitalize key names
-    config.read_string(ini_str)
-
-    # Initialize an empty dictionary to store the TOML representation
-    toml_dict = {}
-
-    # Iterate over each section in the INI configuration
-    for section in config.sections():
-        toml_dict[section] = {}
-
-        # Iterate over each key-value pair in the section
-        for key, value in config.items(section):
-            parsed_value = load_ini_value(value)
-
-            # Convert the parsed value to its TOML-compatible JSON representation
-            toml_dict[section.upper()][key.upper()] = json.dumps(parsed_value)
-
-    # Build the TOML string
-    toml_str = TOML_HEADER
-    for section, items in toml_dict.items():
-        toml_str += f"[{section}]\n"
-        for key, value in items.items():
-            toml_str += f"{key} = {value}\n"
-        toml_str += "\n"
-
-    return toml_str.strip()
-
-
-class JSONSchemaWithLambdas(GenerateJsonSchema):
-    """
-    Encode lambda functions in default values properly.
-    Usage:
-    >>> json.dumps(value, encoder=JSONSchemaWithLambdas())
-    """
-
-    def encode_default(self, dft: Any) -> Any:
-        config = self._config
-        if isinstance(dft, Callable):
-            return "{{lambda " + inspect.getsource(dft).split("=lambda ")[-1].strip()[:-1] + "}}"
-        return to_jsonable_python(
-            dft,
-            timedelta_mode=config.ser_json_timedelta,
-            bytes_mode=config.ser_json_bytes,
-            serialize_unknown=True,
-        )
-
 
 def better_toml_dump_str(val: Any) -> str:
     try:

archivebox/plugins/discovery.py

@@ -55,10 +55,6 @@ def get_plugins() -> list[str]:
     return sorted(get_plugin_catalog())
 
 
-def get_plugin_models():
-    return get_plugin_catalog().plugins
-
-
 def get_plugin_name(plugin: str) -> str:
     """
     Get the base plugin name without numeric prefix.

archivebox/progressmonitor/collection.py

@@ -0,0 +1,54 @@
+"""Non-blocking, database-only collection totals for the progress monitor."""
+
+import logging
+from threading import Lock, Thread
+from time import time
+
+from django.core.cache import cache
+from django.db import close_old_connections, connections
+from django.db.models import Count, Sum
+
+
+_refresh_lock = Lock()
+
+
+def collection_summary(user):
+    user_id = None if user.is_superuser else user.pk
+    key = f"progress-collection:{'all' if user_id is None else user_id}"
+    summary = cache.get(key)
+    # Even an indexed SUM scans the index. Never make an HTTP request wait for
+    # that scan on a large collection. Serve stale totals while one worker refreshes;
+    # a single non-blocking lock prevents a queue of scans from concurrent clients.
+    if (summary is None or time() - summary["sampled_at"] >= 60) and _refresh_lock.acquire(blocking=False):
+        # Throttle failures too: a broken/locked database must not trigger work
+        # and log output on every sidebar poll. This cache is local to each worker.
+        if not cache.add(f"{key}:refresh", True, timeout=60):
+            _refresh_lock.release()
+            return summary
+
+        def refresh():
+            try:
+                close_old_connections()
+                from archivebox.core.models import Snapshot
+
+                snapshots = Snapshot.objects.all()
+                if user_id is not None:
+                    snapshots = snapshots.filter(crawl__created_by_id=user_id)
+                # COUNT(*) + SUM(output_size) can use Snapshot's covering size index.
+                # Snapshot.output_size is maintained when ArchiveResult sizes change.
+                totals = snapshots.aggregate(snapshots=Count("*"), bytes=Sum("output_size"))
+                totals["bytes"] = totals["bytes"] or 0
+                totals["sampled_at"] = time()
+                cache.set(key, totals, timeout=3600)
+            except Exception:
+                logging.getLogger(__name__).exception("Could not refresh collection totals")
+            finally:
+                connections.close_all()
+                _refresh_lock.release()
+
+        try:
+            Thread(target=refresh, name="collection-summary", daemon=True).start()
+        except RuntimeError:
+            _refresh_lock.release()
+            logging.getLogger(__name__).exception("Could not start collection totals refresh")
+    return summary

archivebox/progressmonitor/views.py

@@ -942,6 +942,10 @@ def live_progress_view(request):
             "queued_crawls_hidden": queued_crawls_hidden,
             "server_time": timezone.now().isoformat(),
         }
+        if is_admin and not snapshot_id_filter and not crawl_id_filter and request.GET.get("collection") == "1":
+            from archivebox.progressmonitor.collection import collection_summary
+
+            payload["collection"] = collection_summary(request.user)
         try:
             import ujson
 

archivebox/search/query.py

@@ -221,6 +221,7 @@ def iter_query_search_ids(
     search_mode: str | None = None,
     config: dict[str, Any] | None = None,
     max_results: int | None = None,
+    stop_event=None,
     **config_kwargs: Any,
 ):
     """Yield snapshot IDs from configured search backend modules."""
@@ -267,6 +268,8 @@ def iter_query_search_ids(
     seen: set[str] = set()
     try:
         for backend_name in backend_names:
+            if stop_event is not None and stop_event.is_set():
+                return
             backend = backends[backend_name]
             try:
                 if backend_name == "sonic":
@@ -292,14 +295,18 @@ def iter_query_search_ids(
                     env=search_backend_command_env(config=config),
                     cwd=CONSTANTS.DATA_DIR,
                     timeout=max(1, int(config.get("TIMEOUT", 60))) * 4,
+                    stop_event=stop_event,
                 )
-                for snapshot_id in ids:
-                    if snapshot_id in seen:
-                        continue
-                    seen.add(snapshot_id)
-                    yield snapshot_id
-                    if max_results and len(seen) >= max_results:
-                        return
+                try:
+                    for snapshot_id in ids:
+                        if snapshot_id in seen:
+                            continue
+                        seen.add(snapshot_id)
+                        yield snapshot_id
+                        if max_results and len(seen) >= max_results:
+                            return
+                finally:
+                    ids.close()
                 successful_backends += 1
             except Exception as err:
                 errors.append(err)

archivebox/search/views.py

@@ -6,7 +6,7 @@ import json
 import threading
 import time
 from copy import copy
-from queue import Full, Queue
+from queue import Empty, Full, Queue
 from urllib.parse import urlsplit
 from uuid import UUID
 
@@ -254,15 +254,18 @@ def iter_filtered_search_result_ids(iterator, queryset, *, flush_max_delay=0.05)
         yield from flush_batch()
 
 
-def iter_search_result_ids(query, base_queryset, *, search_mode, config):
+def iter_search_result_ids(query, base_queryset, *, search_mode, config, stop_event=None):
     """Yield filtered Snapshot IDs from the selected search provider."""
     search_mode_base = get_search_mode_base(search_mode, config=config)
     provider = (
         iter_meta_search_ids(query, base_queryset)
         if search_mode_base == "meta"
-        else iter_query_search_ids(query, search_mode=search_mode, config=config)
+        else iter_query_search_ids(query, search_mode=search_mode, config=config, stop_event=stop_event)
     )
-    yield from iter_filtered_search_result_ids(provider, base_queryset)
+    try:
+        yield from iter_filtered_search_result_ids(provider, base_queryset)
+    finally:
+        provider.close()
 
 
 def snapshot_search_stream_response(query, base_queryset, *, search_mode, config, cache_key, thread_name):
@@ -300,7 +303,7 @@ def snapshot_search_stream_response(query, base_queryset, *, search_mode, config
             iterator = None
             try:
                 close_old_connections()
-                iterator = iter_search_result_ids(query, base_queryset, search_mode=search_mode, config=config)
+                iterator = iter_search_result_ids(query, base_queryset, search_mode=search_mode, config=config, stop_event=stop_event)
                 for snapshot_id in iterator:
                     if stop_event.is_set():
                         break
@@ -326,10 +329,13 @@ def snapshot_search_stream_response(query, base_queryset, *, search_mode, config
                 emit(None)
 
         threading.Thread(target=run_search, name=thread_name, daemon=True).start()
-        yield f"0{stream_padding}\n"
         try:
+            yield f"0{stream_padding}\n"
             while True:
-                item = await asyncio.to_thread(queue.get)
+                try:
+                    item = await asyncio.to_thread(queue.get, True, 0.1)
+                except Empty:
+                    continue
                 if item is None:
                     break
                 if isinstance(item, BaseException):

archivebox/services/resource_admission.py

@@ -0,0 +1,91 @@
+"""Admission signals, not estimates of how much RAM an extractor will need."""
+
+import math
+import os
+import time
+from pathlib import Path
+
+
+RESOURCE_RECHECK_SECONDS = 2.0
+# Scheduling policy, not a minimum-RAM claim: stop increasing extraction
+# load when all non-idle work loses at least one second out of ten to memory
+# stalls. Require both recent and rolling pressure so transient reclaim does
+# not indefinitely starve otherwise viable work on small hosts.
+MEMORY_FULL_STALL_PERCENT = 10.0
+
+
+class ResourceAdmission:
+    def __init__(self):
+        self._memory_stalls: dict[str, tuple[int, float]] = {}
+        self._checked_at = float("-inf")
+        self._reason: str | None = None
+
+    @staticmethod
+    def cgroup_paths() -> list[Path]:
+        root = Path("/sys/fs/cgroup")
+        paths = [root]
+        try:
+            for line in Path("/proc/self/cgroup").read_text().splitlines():
+                if line.startswith("0::"):
+                    current = root / line[3:].lstrip("/")
+                    if current.is_dir() and current.is_relative_to(root):
+                        paths = [current, *[parent for parent in current.parents if parent.is_relative_to(root)]]
+                    break
+        except OSError:
+            pass
+        return paths
+
+    def snapshot_limit(self, configured: int) -> int:
+        cpus = len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else (os.cpu_count() or 1)
+        for path in self.cgroup_paths():
+            try:
+                quota, period = (path / "cpu.max").read_text().split()
+                if quota != "max":
+                    cpus = min(cpus, max(1, math.ceil(int(quota) / int(period))))
+            except (OSError, ValueError, ZeroDivisionError):
+                continue
+        # CPU quota limits parallel admission, not forward progress. A host
+        # with a fractional CPU must still be able to run one snapshot.
+        return max(1, min(configured, cpus))
+
+    def observe_memory_stalls(self, source: str, total: int, avg10: float, sampled_at: float) -> bool:
+        previous = self._memory_stalls.get(source)
+        self._memory_stalls[source] = (total, sampled_at)
+        if previous is None:
+            return avg10 >= MEMORY_FULL_STALL_PERCENT
+        previous_total, previous_time = previous
+        elapsed = sampled_at - previous_time
+        if elapsed <= 0 or total < previous_total:
+            return False
+        stalled_percent = (total - previous_total) / (elapsed * 1_000_000) * 100
+        return avg10 >= MEMORY_FULL_STALL_PERCENT and stalled_percent >= MEMORY_FULL_STALL_PERCENT
+
+    def memory_pressure(self) -> str | None:
+        now = time.monotonic()
+        if now - self._checked_at < RESOURCE_RECHECK_SECONDS:
+            return self._reason
+        self._checked_at = now
+        reasons = []
+        cgroups = self.cgroup_paths()
+        for path in cgroups:
+            try:
+                current = int((path / "memory.current").read_text())
+                for name in ("memory.high", "memory.max"):
+                    limit = (path / name).read_text().strip()
+                    if limit != "max" and current >= int(limit):
+                        reasons.append(f"{name} reached")
+            except (OSError, ValueError):
+                continue
+        for path in [Path("/proc/pressure/memory"), *[p / "memory.pressure" for p in cgroups]]:
+            try:
+                full = next(line for line in path.read_text().splitlines() if line.startswith("full "))
+                fields = dict(field.split("=", 1) for field in full.split()[1:])
+                if self.observe_memory_stalls(str(path), int(fields["total"]), float(fields["avg10"]), now):
+                    reasons.append("memory stalls")
+            except (OSError, ValueError, KeyError, StopIteration):
+                continue
+        self._reason = ", ".join(dict.fromkeys(reasons)) or None
+        return self._reason
+
+
+resource_admission = ResourceAdmission()

archivebox/services/runner.py

@@ -58,6 +58,8 @@ from abxbus import BaseEvent
 from abxbus.event_bus import EventBus, get_current_event, in_handler_context
 from abxbus.event_handler import EventHandlerAbortedError, EventHandlerCancelledError
 
+from archivebox.services.resource_admission import RESOURCE_RECHECK_SECONDS, resource_admission
+
 from archivebox.config.common import (
     ArchiveBoxBaseConfig,
     normalize_runtime_config,
@@ -221,6 +223,8 @@ class CrawlRunner:
         self.snapshot_tasks: dict[str, asyncio.Task[None]] = {}
         self.snapshot_semaphore = asyncio.Semaphore(1)
         self.max_concurrent_snapshots = 1
+        self._resource_deferred = False
+        self._initial_snapshot_admitted = False
         self.persona = None
         self.base_config: ArchiveBoxBaseConfig | dict[str, Any] = {}
         self.derived_config: dict[str, Any] = {}
@@ -294,6 +298,7 @@ class CrawlRunner:
     async def run(self) -> None:
         root_snapshot_id: str | None = None
         bus_destroyed = False
+        run_state_loaded = False
         try:
             first_signal_message = (
                 "\n[🛑] Got {signal_name}, aborting the active hook...\n"
@@ -313,8 +318,15 @@ class CrawlRunner:
                 on_signal=self._request_abort_from_signal,
                 raise_on_first_signal=False,
             ):
+                if await sync_to_async(defer_crawl_for_resources, thread_sensitive=True)(self.crawl):
+                    return
+                # Startup and its first snapshot are one admission. Browser
+                # startup can itself cause reclaim; checking again immediately
+                # would tear it down and repeat startup without doing any work.
+                self._initial_snapshot_admitted = True
+                run_state_loaded = True
                 snapshot_ids = await sync_to_async(self.load_run_state, thread_sensitive=True)()
-                max_concurrent_snapshots = max(1, int(self.base_config.get("CRAWL_MAX_CONCURRENT_SNAPSHOTS", 1)))
+                max_concurrent_snapshots = resource_admission.snapshot_limit(int(self.base_config.get("CRAWL_MAX_CONCURRENT_SNAPSHOTS", 1)))
                 self.max_concurrent_snapshots = max_concurrent_snapshots
                 self.snapshot_semaphore = asyncio.Semaphore(max_concurrent_snapshots)
                 live_ui = self._create_live_ui()
@@ -344,10 +356,19 @@ class CrawlRunner:
                 except Exception:
                     pass
                 self._live_stream = None
-            await sync_to_async(project_abxpkg_derived_cache_to_db, thread_sensitive=True)(self.base_config.get("ABXPKG_LIB_DIR"))
-            await sync_to_async(self.finalize_run_state, thread_sensitive=True)()
+            if run_state_loaded:
+                await sync_to_async(project_abxpkg_derived_cache_to_db, thread_sensitive=True)(self.base_config.get("ABXPKG_LIB_DIR"))
+                await sync_to_async(self.finalize_run_state, thread_sensitive=True)()
 
     async def enqueue_snapshot(self, snapshot_id: str, crawl_start_event: CrawlStartEvent | None = None) -> None:
+        if self._resource_deferred:
+            return
+        if not self._initial_snapshot_admitted and await sync_to_async(defer_crawl_for_resources, thread_sensitive=True)(
+            self.crawl,
+            excluded_snapshot_ids=list(self.snapshot_tasks),
+        ):
+            self._resource_deferred = True
+            return
         if await self.crawl_is_cancelled():
             return
         if await self.crawl_is_paused() and not self.allow_maintenance_on_inactive_crawl:
@@ -362,6 +383,7 @@ class CrawlRunner:
             return
         else:
             task = asyncio.create_task(self.run_snapshot(snapshot_id), context=_runner_task_context())
+        self._initial_snapshot_admitted = False
         self.snapshot_tasks[snapshot_id] = task
 
     async def stop_snapshot_tasks(self) -> None:
@@ -488,6 +510,8 @@ class CrawlRunner:
         from archivebox.core.models import Snapshot
         from archivebox.config.common import get_config
 
+        if self._resource_deferred:
+            return
         if not isinstance(get_current_event(), CrawlStartEvent):
             return
         if await self.crawl_is_cancelled():
@@ -497,7 +521,7 @@ class CrawlRunner:
 
         await sync_to_async(self.crawl.refresh_from_db, thread_sensitive=True)()
         config = await sync_to_async(lambda: get_config(crawl=self.crawl), thread_sensitive=True)()
-        self.max_concurrent_snapshots = max(1, int(config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"]))
+        self.max_concurrent_snapshots = resource_admission.snapshot_limit(int(config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"]))
 
         active_snapshot_ids = [snapshot_id for snapshot_id, task in self.snapshot_tasks.items() if not task.done()]
         available_slots = max(0, self.max_concurrent_snapshots - len(active_snapshot_ids))
@@ -1266,6 +1290,29 @@ def run_snapshot_maintenance(snapshot_id: str, *, output_dir: Path | None = None
     return True
 
 
+def defer_crawl_for_resources(crawl, *, excluded_snapshot_ids=()) -> bool:
+    """Leave extraction durable; never delay cancellation or final cleanup."""
+    from archivebox.core.models import Snapshot
+
+    if crawl.status not in (crawl.StatusChoices.QUEUED, crawl.StatusChoices.STARTED):
+        return False
+    reason = resource_admission.memory_pressure()
+    if reason is None:
+        return False
+    pending = crawl.snapshot_set.filter(status__in=Snapshot.RUNNABLE_STATES).exclude(id__in=excluded_snapshot_ids)
+    if crawl.snapshot_set.exists() and not pending.exists():
+        return False
+    now = timezone.now()
+    retry_at = now + timedelta(seconds=RESOURCE_RECHECK_SECONDS)
+    # Conditional short updates cannot resurrect a concurrent pause/cancel.
+    type(crawl).objects.filter(id=crawl.id, status__in=[crawl.StatusChoices.QUEUED, crawl.StatusChoices.STARTED]).update(
+        retry_at=retry_at,
+        modified_at=now,
+    )
+    pending.filter(retry_at__lte=now).update(retry_at=retry_at, modified_at=now)
+    return True
+
+
 def run_due_crawl(crawl, *, lock_seconds: int, interactive_interrupts: bool = False) -> bool:
     with crawl_lifecycle_lock(str(crawl.id)):
         return _run_due_crawl_locked(

archivebox/templates/core/snapshot.html

@@ -1890,14 +1890,13 @@
 
             function attachPreviewFrameHandlers(frame) {
                 resetPreviewFrameSize(frame)
-                if (frame.src.endsWith('.pdf')) {
+                if (new URL(frame.src || 'about:blank', window.location.href).pathname.endsWith('.pdf')) {
                     frame.removeAttribute('sandbox')
                     frame.src = frame.src + '#toolbar=0'
                 }
                 frame.onload = function() {
-                    if (this.src.includes('.pdf')) {
+                    if (new URL(this.src || 'about:blank', window.location.href).pathname.endsWith('.pdf')) {
                         this.removeAttribute('sandbox')
-                        this.src = this.src.split('?autoplay=')[0] + '#toolbar=0'
                     }
                     tryCenterImageFrame(this)
                     try {
@@ -2083,7 +2082,10 @@
                     nextSrcAbs = new URL(nextSrc, window.location.href).href
                 } catch (err) {}
                 const iframe_elem = ensureMainFrame(currentSrc !== nextSrcAbs)
-                if (target.endsWith('.pdf')) {
+                // Preview wrappers retain the .pdf path but add ?preview=1.
+                // The native PDF renderer must be allowed before navigation,
+                // not after a failed sandboxed load has already completed.
+                if (new URL(target, window.location.href).pathname.endsWith('.pdf')) {
                     iframe_elem.removeAttribute('sandbox')
                 } else {
                     iframe_elem.sandbox = "allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms"

archivebox/templates/core/system_warnings_banner.html

@@ -34,7 +34,7 @@
     {% elif mode == "high_load" %}
       15-min loadavg <code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">{{ load_15 }}</code>
       exceeds <code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">{{ load_threshold }}</code>
-      ({{ cpu_count }} cores × 3) — the host is saturated; crawls are queuing faster than the runner can finish them.
+      ({{ cpu_count }} cores × 3) — recent CPU or I/O contention may slow crawls. This average can stay elevated after load subsides; it does not measure queue growth.
     {% elif mode == "unsafe" %}
       ArchiveBox single-domain mode — archived pages share an origin with this site
       <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Modes" target="_blank" rel="noopener noreferrer" style="color:#fff;text-decoration:underline;margin-left:6px;">why?</a>

archivebox/tests/test_api_v1_auth_browser_session.py

@@ -0,0 +1,56 @@
+from datetime import timedelta
+from urllib.parse import urlsplit
+
+import pytest
+from django.contrib.sessions.models import Session
+from django.utils import timezone
+
+from archivebox.api.models import APIToken
+from archivebox.tests.conftest import API_TEST_HOST, api_client_request
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+@pytest.mark.parametrize("header", ["HTTP_AUTHORIZATION", "HTTP_X_ARCHIVEBOX_API_KEY"])
+def test_browser_session_authenticates_admin_and_expires_with_key(client, api_admin_user, header):
+    token = APIToken.objects.create(created_by=api_admin_user, expires=timezone.now() + timedelta(minutes=5))
+    response = api_client_request(
+        client,
+        "post",
+        "/api/v1/auth/browser_session",
+        payload={},
+        headers={
+            "HTTP_HOST": API_TEST_HOST,
+            header: f"Bearer {token.token}" if header == "HTTP_AUTHORIZATION" else token.token,
+        },
+    )
+    assert response.status_code == 200, response.content
+    assert response["Cache-Control"] == "no-store"
+    data = response.json()
+    session = Session.objects.get(session_key=data["cookie"]["value"])
+    assert session.get_decoded()["_auth_user_id"] == str(api_admin_user.pk)
+    assert timezone.now() < session.expire_date <= token.expires
+    client.cookies.clear()
+    client.cookies[data["cookie"]["name"]] = data["cookie"]["value"]
+    response = client.get("/admin/", HTTP_HOST=urlsplit(data["admin_url"]).netloc)
+    assert response.status_code == 200, response.content
+    assert response.wsgi_request.user.pk == api_admin_user.pk
+
+
+@pytest.mark.parametrize("kind", ["missing", "invalid", "expired", "inactive", "nonadmin", "query"])
+def test_browser_session_rejects_invalid_credentials(client, api_admin_user, kind):
+    token = APIToken.objects.create(created_by=api_admin_user, expires=timezone.now() + timedelta(minutes=5))
+    if kind == "expired":
+        token.expires = timezone.now() - timedelta(seconds=1)
+        token.save()
+    if kind in ("inactive", "nonadmin"):
+        setattr(api_admin_user, "is_active" if kind == "inactive" else "is_superuser", False)
+        api_admin_user.save()
+    headers = {"HTTP_HOST": API_TEST_HOST}
+    if kind not in ("missing", "query"):
+        headers["HTTP_AUTHORIZATION"] = "Bearer " + ("invalid" if kind == "invalid" else token.token)
+    path = "/api/v1/auth/browser_session" + (f"?api_key={token.token}" if kind == "query" else "")
+    before = Session.objects.count()
+    response = api_client_request(client, "post", path, payload={}, headers=headers)
+    assert response.status_code in (401, 403), response.content
+    assert Session.objects.count() == before

archivebox/tests/test_auth_ldap.py

@@ -153,7 +153,7 @@ class TestArchiveBoxWithLDAP:
 
     def test_archivebox_init_without_ldap(self, tmp_path):
         """Test that archivebox init works without LDAP enabled."""
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["init"],
             cwd=tmp_path,
             timeout=45,
@@ -161,14 +161,13 @@ class TestArchiveBoxWithLDAP:
             default_cli_env=True,
             disable_extractors=True,
         )
-        _, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         # Should succeed
-        assert code == 0, f"archivebox init failed: {stderr}"
+        assert result.returncode == 0, f"archivebox init failed: {result.stderr}"
 
     def test_archivebox_version_with_ldap_config(self, tmp_path):
         """Test that archivebox version works with LDAP config set."""
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["version"],
             cwd=tmp_path,
             timeout=10,
@@ -179,10 +178,9 @@ class TestArchiveBoxWithLDAP:
             default_cli_env=True,
             disable_extractors=True,
         )
-        _, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         # Should succeed
-        assert code == 0, f"archivebox version failed: {stderr}"
+        assert result.returncode == 0, f"archivebox version failed: {result.stderr}"
 
 
 class TestLDAPConfigValidationInArchiveBox:
@@ -190,7 +188,7 @@ class TestLDAPConfigValidationInArchiveBox:
 
     def test_archivebox_init_with_incomplete_ldap_config(self, tmp_path):
         """Test that archivebox init fails with helpful error when LDAP config is incomplete."""
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["init"],
             cwd=tmp_path,
             timeout=45,
@@ -201,10 +199,9 @@ class TestLDAPConfigValidationInArchiveBox:
             default_cli_env=True,
             disable_extractors=True,
         )
-        _, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         # Should fail with validation error
-        assert code != 0, "Should fail with incomplete LDAP config"
+        assert result.returncode != 0, "Should fail with incomplete LDAP config"
 
         # Check error message
-        assert "LDAP_* config options must all be set" in stderr, f"Expected validation error message in: {stderr}"
+        assert "LDAP_* config options must all be set" in result.stderr, f"Expected validation error message in: {result.stderr}"

archivebox/tests/test_cli_add.py

@@ -230,27 +230,23 @@ def test_add_single_url_records_url_in_crawl(initialized_archive):
     assert snapshots == []
 
 
-def test_add_memory_preflight_fails_before_creating_crawl(initialized_archive, capsys):
-    from archivebox.workers.supervisord_util import MIN_CRAWL_AVAILABLE_MEMORY_BYTES, require_crawl_memory
-
-    assert MIN_CRAWL_AVAILABLE_MEMORY_BYTES == 512 * 1024 * 1024
-    require_crawl_memory(MIN_CRAWL_AVAILABLE_MEMORY_BYTES)
-
+def test_add_bg_persists_queue_without_starting_extraction_workers(initialized_archive):
+    result = run_archivebox_cmd(
+        ["add", "--bg", "--plugins=wget", "--crawl-max-concurrent-snapshots=4", "https://example.com", "https://example.org"],
+        cwd=initialized_archive,
+        env=cli_env(),
+    )
+    assert result.returncode == 0, result.stderr
     with use_archivebox_db(initialized_archive):
-        before_crawls = Crawl.objects.count()
-        before_processes = Process.objects.count()
-
-        with pytest.raises(SystemExit) as err:
-            require_crawl_memory(MIN_CRAWL_AVAILABLE_MEMORY_BYTES - 1)
-
-        assert Crawl.objects.count() == before_crawls
-        assert Process.objects.count() == before_processes
-
-    assert err.value.code == 1
-    output = capsys.readouterr().err
-    assert "Not enough available memory to archive a crawl" in output
-    assert "No crawl, runner, or Sonic workers were started" in output
-    assert "configure swap" in output
+        crawl = Crawl.objects.get()
+        assert crawl.status == Crawl.StatusChoices.QUEUED
+        assert crawl.retry_at is not None
+        assert crawl.get_urls_list() == ["https://example.com", "https://example.org"]
+        assert crawl.config["PLUGINS"] == "wget"
+        assert crawl.config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] == 4
+        assert not Snapshot.objects.exists()
+        assert not ArchiveResult.objects.exists()
+        assert not Process.objects.filter(process_type__in=[Process.TypeChoices.ORCHESTRATOR, Process.TypeChoices.HOOK]).exists()
 
 
 def test_initial_crawl_creation_does_not_warn_as_an_outside_runner_update(initialized_archive, caplog):
@@ -1208,7 +1204,7 @@ def test_cli_recursive_crawl_processes_discovered_html_urls(initialized_archive,
     root_url = recursive_test_site["root_url"]
     child_url = recursive_test_site["child_urls"][0]
 
-    _cmd_result = run_archivebox_cmd(
+    result = run_archivebox_cmd(
         [
             "add",
             "--depth=2",
@@ -1223,8 +1219,7 @@ def test_cli_recursive_crawl_processes_discovered_html_urls(initialized_archive,
         env=env,
         timeout=180,
     )
-    stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
-    assert returncode == 0, stderr or stdout
+    assert result.returncode == 0, result.stderr or result.stdout
 
     with use_archivebox_db(initialized_archive):
         crawl = Crawl.objects.order_by("-created_at").values_list("max_depth", "tags_str", "config").first()

archivebox/tests/test_cli_archiveresult.py

@@ -142,18 +142,17 @@ class TestArchiveResultCreate:
         assert crawl_result.returncode == 0, crawl_result.stderr
         crawl_record = parse_jsonl_output(crawl_result.stdout)[0]
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["archiveresult", "create"],
             stdin=crawl_result.stdout,
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        assert "Passed through" in stderr
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        assert "Passed through" in result.stderr
+        records = parse_jsonl_output(result.stdout)
         assert len(records) == 1
         assert records[0]["id"] == crawl_record["id"]
 
@@ -163,16 +162,15 @@ class TestArchiveResultList:
 
     def test_list_empty(self, initialized_archive):
         """List with no archive results returns empty."""
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["archiveresult", "list"],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        _stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        assert "Listed 0 archive results" in stderr
+        assert result.returncode == 0
+        assert "Listed 0 archive results" in result.stderr
 
     def test_list_filter_by_status(self, initialized_archive):
         """Filter archive results by status."""

archivebox/tests/test_cli_crawl.py

@@ -33,19 +33,18 @@ class TestCrawlCreate:
         """Create crawl from URL arguments."""
         url = create_test_url()
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["crawl", "create", url],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0, f"Command failed: {stderr}"
-        assert "Created crawl" in stderr
+        assert result.returncode == 0, f"Command failed: {result.stderr}"
+        assert "Created crawl" in result.stderr
 
         # Check JSONL output
-        records = parse_jsonl_output(stdout)
+        records = parse_jsonl_output(result.stdout)
         assert len(records) == 1
         assert records[0]["type"] == "Crawl"
         assert url in records[0]["urls"]
@@ -55,18 +54,17 @@ class TestCrawlCreate:
         urls = [create_test_url() for _ in range(3)]
         stdin = "\n".join(urls)
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["crawl", "create"],
             stdin=stdin,
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0, f"Command failed: {stderr}"
+        assert result.returncode == 0, f"Command failed: {result.stderr}"
 
-        records = parse_jsonl_output(stdout)
+        records = parse_jsonl_output(result.stdout)
         assert len(records) == 1
         crawl = records[0]
         assert crawl["type"] == "Crawl"
@@ -78,32 +76,30 @@ class TestCrawlCreate:
         """Create crawl with --depth flag."""
         url = create_test_url()
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["crawl", "create", "--depth=2", url],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
         assert records[0]["max_depth"] == 2
 
     def test_create_with_tag(self, initialized_archive):
         """Create crawl with --tag flag."""
         url = create_test_url()
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["crawl", "create", "--tag=test-tag", url],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
         assert "test-tag" in records[0].get("tags_str", "")
 
     def test_create_passes_through_tag_emitted_by_cli(self, initialized_archive):
@@ -119,17 +115,16 @@ class TestCrawlCreate:
         url = create_test_url()
         stdin = tag_result.stdout + json.dumps({"url": url}) + "\n"
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["crawl", "create"],
             stdin=stdin,
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
 
         assert any(record.get("type") == "Tag" and record["id"] == tag_record["id"] for record in records)
         assert any(record.get("type") == "Crawl" and url in record["urls"] for record in records)
@@ -163,32 +158,30 @@ class TestCrawlList:
 
     def test_list_empty(self, initialized_archive):
         """List with no crawls returns empty."""
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["crawl", "list"],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        _stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        assert "Listed 0 crawls" in stderr
+        assert result.returncode == 0
+        assert "Listed 0 crawls" in result.stderr
 
     def test_list_returns_created(self, initialized_archive):
         """List returns previously created crawls."""
         url = create_test_url()
         run_archivebox_cmd(["crawl", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["crawl", "list"],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
         assert len(records) >= 1
         assert any(url in r.get("urls", "") for r in records)
 
@@ -197,16 +190,15 @@ class TestCrawlList:
         url = create_test_url()
         run_archivebox_cmd(["crawl", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["crawl", "list", "--status=queued"],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
         for r in records:
             assert r["status"] == "queued"
 
@@ -221,16 +213,15 @@ class TestCrawlList:
                 disable_extractors=True,
             )
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["crawl", "list", "--limit=2"],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
         assert len(records) == 2
 
 

archivebox/tests/test_cli_list.py

@@ -3,6 +3,8 @@ Tests for archivebox list command.
 Verify list emits snapshot JSONL and applies the documented filters.
 """
 
+import csv
+import io
 import json
 from pathlib import Path
 
@@ -427,16 +429,15 @@ def test_snapshot_list_search_meta(initialized_archive):
     url = create_test_url(domain="meta-search-example.com")
     run_archivebox_cmd(["snapshot", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
 
-    _cmd_result = run_archivebox_cmd(
+    result = run_archivebox_cmd(
         ["snapshot", "list", "--search=meta", "meta-search-example.com"],
         cwd=initialized_archive,
         default_cli_env=True,
         disable_extractors=True,
     )
-    stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-    assert code == 0, f"Command failed: {stderr}"
-    records = parse_jsonl_output(stdout)
+    assert result.returncode == 0, f"Command failed: {result.stderr}"
+    records = parse_jsonl_output(result.stdout)
     assert len(records) == 1
     assert "meta-search-example.com" in records[0]["url"]
 
@@ -446,16 +447,15 @@ def test_list_search_meta_matches_metadata(initialized_archive):
     url = create_test_url(domain="top-level-meta-search-example.com")
     run_archivebox_cmd(["snapshot", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
 
-    _cmd_result = run_archivebox_cmd(
+    result = run_archivebox_cmd(
         ["list", "--search=meta", "top-level-meta-search-example.com"],
         cwd=initialized_archive,
         default_cli_env=True,
         disable_extractors=True,
     )
-    stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-    assert code == 0, f"Command failed: {stderr}"
-    records = parse_jsonl_output(stdout)
+    assert result.returncode == 0, f"Command failed: {result.stderr}"
+    records = parse_jsonl_output(result.stdout)
     assert len(records) == 1
     assert "top-level-meta-search-example.com" in records[0]["url"]
 
@@ -468,11 +468,10 @@ def test_search_command_finds_snapshots(initialized_archive):
         disable_extractors=True,
     )
 
-    _cmd_result = run_archivebox_cmd(["search", "example"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
-    stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
+    result = run_archivebox_cmd(["search", "example"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
 
-    assert code == 0, stderr
-    assert "example" in stdout
+    assert result.returncode == 0, result.stderr
+    assert "example" in result.stdout
 
 
 def test_search_command_returns_no_results_for_missing_term(initialized_archive):
@@ -483,22 +482,20 @@ def test_search_command_returns_no_results_for_missing_term(initialized_archive)
         disable_extractors=True,
     )
 
-    _cmd_result = run_archivebox_cmd(
+    result = run_archivebox_cmd(
         ["search", "nonexistentterm12345"],
         cwd=initialized_archive,
         default_cli_env=True,
         disable_extractors=True,
     )
-    _stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-    assert code in [0, 1]
+    assert result.returncode in [0, 1]
 
 
 def test_search_command_on_empty_archive(initialized_archive):
-    _cmd_result = run_archivebox_cmd(["search", "anything"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
-    _stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
+    result = run_archivebox_cmd(["search", "anything"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
 
-    assert code in [0, 1]
+    assert result.returncode in [0, 1]
 
 
 def test_search_command_outputs_matching_snapshots_as_jsonl(initialized_archive):
@@ -509,11 +506,10 @@ def test_search_command_outputs_matching_snapshots_as_jsonl(initialized_archive)
         disable_extractors=True,
     )
 
-    _cmd_result = run_archivebox_cmd(["search"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
-    stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
+    result = run_archivebox_cmd(["search"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
 
-    assert code == 0, stderr
-    records = parse_jsonl_output(stdout)
+    assert result.returncode == 0, result.stderr
+    records = parse_jsonl_output(result.stdout)
     assert any("example.com" in row.get("url", "") for row in records)
 
 
@@ -576,52 +572,47 @@ def test_search_command_csv_outputs_requested_column(initialized_archive):
         disable_extractors=True,
     )
 
-    _cmd_result = run_archivebox_cmd(
+    result = run_archivebox_cmd(
         ["search", "--csv", "url", "--with-headers"],
         cwd=initialized_archive,
         default_cli_env=True,
         disable_extractors=True,
     )
-    stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-    assert code == 0, stderr
-    assert "url" in stdout
-    assert "example.com" in stdout
+    assert result.returncode == 0, result.stderr
+    assert list(csv.reader(io.StringIO(result.stdout))) == [["url"], ["https://example.com"]]
 
 
 def test_search_command_with_headers_requires_structured_output_format(initialized_archive):
-    _cmd_result = run_archivebox_cmd(["search", "--with-headers"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
-    _stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
+    result = run_archivebox_cmd(["search", "--with-headers"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
 
-    assert code != 0
-    assert "requires" in stderr.lower()
-    assert "json" in stderr.lower()
+    assert result.returncode != 0
+    assert "requires" in result.stderr.lower()
+    assert "json" in result.stderr.lower()
 
 
 def test_search_command_sort_option_runs_successfully(initialized_archive):
     for url in ["https://iana.org", "https://example.com"]:
         run_archivebox_cmd(["snapshot", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
 
-    _cmd_result = run_archivebox_cmd(
+    result = run_archivebox_cmd(
         ["search", "--csv", "url", "--sort=url"],
         cwd=initialized_archive,
         default_cli_env=True,
         disable_extractors=True,
     )
-    stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-    assert code == 0, stderr
-    assert "example.com" in stdout or "iana.org" in stdout
+    assert result.returncode == 0, result.stderr
+    assert list(csv.reader(io.StringIO(result.stdout))) == [["https://example.com"], ["https://iana.org"]]
 
 
 def test_search_command_help_lists_supported_filters(initialized_archive):
-    _cmd_result = run_archivebox_cmd(["search", "--help"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
-    stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
+    result = run_archivebox_cmd(["search", "--help"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
 
-    assert code == 0
-    assert "--url__icontains" in stdout
-    assert "--crawl-id" in stdout
-    assert "--status" in stdout
-    assert "--sort" in stdout
-    assert "--json" in stdout
-    assert "--html" in stdout
+    assert result.returncode == 0
+    assert "--url__icontains" in result.stdout
+    assert "--crawl-id" in result.stdout
+    assert "--status" in result.stdout
+    assert "--sort" in result.stdout
+    assert "--json" in result.stdout
+    assert "--html" in result.stdout

archivebox/tests/test_cli_run.py

@@ -53,15 +53,14 @@ def test_cli_run_signal_cleans_real_chrome_hook_process_group(initialized_archiv
     env = cli_env(live=True, PLUGINS="chrome", CHROME_ISOLATION="crawl", CHROME_HEADLESS="true", CHROME_SANDBOX="false")
     _install_real_chrome_for_test(initialized_archive, env, isolation="crawl")
 
-    _cmd_result = run_archivebox_cmd(
+    result = run_archivebox_cmd(
         ["snapshot", "create", recursive_test_site["root_url"]],
         cwd=initialized_archive,
         env=env,
         timeout=60,
     )
-    stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
-    assert returncode == 0, stderr or stdout
-    records = parse_jsonl_output(stdout)
+    assert result.returncode == 0, result.stderr or result.stdout
+    records = parse_jsonl_output(result.stdout)
     snapshot_id = next(record["id"] for record in records if record.get("type") == "Snapshot")
     with use_archivebox_db(initialized_archive):
         browser_state = Snapshot.objects.get(id=snapshot_id).output_dir / "chrome" / "browser.json"
@@ -107,7 +106,7 @@ class TestRunWithCrawl:
         )
         assert create_result.returncode == 0, create_result.stderr
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["run"],
             stdin=create_result.stdout,
             cwd=initialized_archive,
@@ -116,12 +115,11 @@ class TestRunWithCrawl:
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0, f"Command failed: {stderr}"
+        assert result.returncode == 0, f"Command failed: {result.stderr}"
 
         # Should output the created Crawl
-        records = parse_jsonl_output(stdout)
+        records = parse_jsonl_output(result.stdout)
         crawl_records = [r for r in records if r.get("type") == "Crawl"]
         assert len(crawl_records) >= 1
         assert crawl_records[0].get("id")  # Should have an id now
@@ -170,7 +168,7 @@ class TestRunWithSnapshot:
         )
         assert create_result.returncode == 0, create_result.stderr
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["run"],
             stdin=create_result.stdout,
             cwd=initialized_archive,
@@ -179,11 +177,10 @@ class TestRunWithSnapshot:
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0, f"Command failed: {stderr}"
+        assert result.returncode == 0, f"Command failed: {result.stderr}"
 
-        records = parse_jsonl_output(stdout)
+        records = parse_jsonl_output(result.stdout)
         snapshot_records = [r for r in records if r.get("type") == "Snapshot"]
         assert len(snapshot_records) >= 1
         assert snapshot_records[0].get("id")
@@ -220,7 +217,7 @@ class TestRunWithSnapshot:
     def test_run_with_plain_url(self, initialized_archive):
         """Run accepts plain URL records (no type field)."""
         url = create_test_url()
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["run"],
             stdin=url + "\n",
             cwd=initialized_archive,
@@ -229,10 +226,9 @@ class TestRunWithSnapshot:
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
         assert len(records) >= 1
 
 
@@ -405,7 +401,7 @@ class TestRunRecovery:
             crawl_id = crawl.id
             snapshot_id = snapshot.id
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["run", "--maintenance-only"],
             cwd=initialized_archive,
             timeout=90,
@@ -413,13 +409,12 @@ class TestRunRecovery:
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0, stdout + stderr
-        assert "Repairing" in stderr
-        assert "Resuming 1 Crawl(s) with pending URLs ready to archive" in stderr
-        assert "interrupted before" in stderr
-        assert "remaining URLs" in stderr
+        assert result.returncode == 0, result.stdout + result.stderr
+        assert "Repairing" in result.stderr
+        assert "Resuming 1 Crawl(s) with pending URLs ready to archive" in result.stderr
+        assert "interrupted before" in result.stderr
+        assert "remaining URLs" in result.stderr
 
         with use_archivebox_db(initialized_archive):
             crawl = Crawl.objects.get(id=crawl_id)
@@ -444,17 +439,16 @@ class TestRunPassThrough:
         assert tag_result.returncode == 0, tag_result.stderr
         tag_record = parse_jsonl_output(tag_result.stdout)[0]
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["run"],
             stdin=tag_result.stdout,
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
         tag_records = [record for record in records if record.get("type") == "Tag"]
         assert len(tag_records) == 1
         assert tag_records[0]["id"] == tag_record["id"]
@@ -471,7 +465,7 @@ class TestRunPassThrough:
         )
         assert create_result.returncode == 0, create_result.stderr
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["run"],
             stdin=create_result.stdout,
             cwd=initialized_archive,
@@ -480,10 +474,9 @@ class TestRunPassThrough:
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
         # Should have at least the Crawl in output
         assert len(records) >= 1
 
@@ -516,7 +509,7 @@ class TestRunMixedInput:
         )
         assert snapshot_result.returncode == 0, snapshot_result.stderr
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["run"],
             stdin=tag_result.stdout + snapshot_result.stdout,
             cwd=initialized_archive,
@@ -525,10 +518,9 @@ class TestRunMixedInput:
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
 
         types = {record.get("type") for record in records}
         assert {"Crawl", "Snapshot", "Tag"}.issubset(types)
@@ -539,16 +531,15 @@ class TestRunEmpty:
 
     def test_run_empty_stdin(self, initialized_archive):
         """Run with empty stdin returns success."""
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["run"],
             stdin="",
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        _stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
+        assert result.returncode == 0
 
     def test_run_no_runnable_records_to_process(self, initialized_archive):
         """Run with only a real non-runnable Tag reports no work."""
@@ -560,17 +551,16 @@ class TestRunEmpty:
         )
         assert tag_result.returncode == 0, tag_result.stderr
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["run"],
             stdin=tag_result.stdout,
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        _stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        assert "No records to process" in stderr
+        assert result.returncode == 0
+        assert "No records to process" in result.stderr
 
 
 class TestRunDaemonMode:

archivebox/tests/test_cli_server.py

@@ -124,10 +124,17 @@ def test_server_auth_secret_and_cookie_settings_are_restart_stable(tmp_path):
     assert first_lines[3:] == ["None", "False", "False"]
 
 
-def test_https_base_url_enables_proxy_ssl_header_and_secure_cookies(tmp_path):
+@pytest.mark.parametrize(
+    "base_url, secure",
+    [
+        ("https://archive.example.com", True),
+        ("", False),
+    ],
+)
+def test_base_url_controls_secure_cookies(tmp_path, base_url, secure):
     (tmp_path / ".archivebox_id").write_text("testcoll")
     env = os.environ.copy()
-    env["BASE_URL"] = "https://archive.example.com"
+    env["BASE_URL"] = base_url
     env["DJANGO_SETTINGS_MODULE"] = "archivebox.core.settings"
     repo_root = Path(__file__).resolve().parents[2]
     env["PYTHONPATH"] = f"{repo_root}{os.pathsep}{env.get('PYTHONPATH', '')}"
@@ -155,45 +162,8 @@ def test_https_base_url_enables_proxy_ssl_header_and_secure_cookies(tmp_path):
     )
 
     assert json.loads(result.stdout) == {
-        "csrf_secure": True,
-        "session_secure": True,
-        "proxy_ssl_header": ["HTTP_X_FORWARDED_PROTO", "https"],
-    }
-
-
-def test_unconfigured_base_url_enables_proxy_ssl_header_without_secure_cookies(tmp_path):
-    (tmp_path / ".archivebox_id").write_text("testcoll")
-    env = os.environ.copy()
-    env["BASE_URL"] = ""
-    env["DJANGO_SETTINGS_MODULE"] = "archivebox.core.settings"
-    repo_root = Path(__file__).resolve().parents[2]
-    env["PYTHONPATH"] = f"{repo_root}{os.pathsep}{env.get('PYTHONPATH', '')}"
-
-    result = subprocess.run(
-        [
-            sys.executable,
-            "-c",
-            (
-                "import django, json;"
-                "django.setup();"
-                "from django.conf import settings;"
-                "print(json.dumps({"
-                "'csrf_secure': settings.CSRF_COOKIE_SECURE,"
-                "'session_secure': settings.SESSION_COOKIE_SECURE,"
-                "'proxy_ssl_header': settings.SECURE_PROXY_SSL_HEADER,"
-                "}))"
-            ),
-        ],
-        capture_output=True,
-        text=True,
-        check=True,
-        env=env,
-        cwd=tmp_path,
-    )
-
-    assert json.loads(result.stdout) == {
-        "csrf_secure": False,
-        "session_secure": False,
+        "csrf_secure": secure,
+        "session_secure": secure,
         "proxy_ssl_header": ["HTTP_X_FORWARDED_PROTO", "https"],
     }
 
@@ -759,9 +729,8 @@ def test_live_server_signal_exit_and_resume_uses_existing_supervisor_state(initi
 
         resumed = start_archivebox_server(initialized_archive, port=port, log_name=f"server-{stop_signal.name}-resumed.log", env=env)
         resumed_log = resumed.log_path
-        _cmd_result = run_archivebox_cmd(["status"], cwd=initialized_archive, env=env, timeout=60)
-        stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
-        assert returncode == 0, stderr or stdout
+        result = run_archivebox_cmd(["status"], cwd=initialized_archive, env=env, timeout=60)
+        assert result.returncode == 0, result.stderr or result.stdout
 
         os.kill(resumed.pid, signal.SIGTERM)
         resumed.wait(timeout=20)
@@ -783,14 +752,13 @@ def test_live_daemonized_server_keeps_supervisord_owned_by_archivebox_parent(ini
     port = get_free_port()
     bind_url = f"http://127.0.0.1:{port}"
     try:
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["server", "--daemonize", f"127.0.0.1:{port}"],
             cwd=initialized_archive,
             env=env,
             timeout=90,
         )
-        stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
-        assert returncode == 0, stderr or stdout
+        assert result.returncode == 0, result.stderr or result.stdout
         _wait_for_archivebox_workers(initialized_archive, env, ("worker_daphne", "worker_runner"), timeout=30)
         assert_port_open("127.0.0.1", port, timeout=30)
 

archivebox/tests/test_cli_snapshot.py

@@ -33,18 +33,17 @@ class TestSnapshotCreate:
         """Create snapshot from URL arguments."""
         url = create_test_url()
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["snapshot", "create", url],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0, f"Command failed: {stderr}"
-        assert "Created" in stderr
+        assert result.returncode == 0, f"Command failed: {result.stderr}"
+        assert "Created" in result.stderr
 
-        records = parse_jsonl_output(stdout)
+        records = parse_jsonl_output(result.stdout)
         assert len(records) == 1
         assert records[0]["type"] == "Snapshot"
         assert records[0]["url"] == url
@@ -83,16 +82,15 @@ class TestSnapshotCreate:
         """Create snapshot with --tag flag."""
         url = create_test_url()
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["snapshot", "create", "--tag=test-tag", url],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
         assert "test-tag" in records[0].get("tags", "")
 
     def test_create_passes_through_tag_emitted_by_cli(self, initialized_archive):
@@ -108,17 +106,16 @@ class TestSnapshotCreate:
         url = create_test_url()
         stdin = tag_result.stdout + url + "\n"
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["snapshot", "create"],
             stdin=stdin,
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
 
         assert any(record.get("type") == "Tag" and record["id"] == tag_record["id"] for record in records)
         assert any(record.get("type") == "Snapshot" and record["url"] == url for record in records)
@@ -127,16 +124,15 @@ class TestSnapshotCreate:
         """Create snapshots from multiple URLs."""
         urls = [create_test_url() for _ in range(3)]
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["snapshot", "create"] + urls,
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
         assert len(records) == 3
 
         created_urls = {r["url"] for r in records}
@@ -149,32 +145,30 @@ class TestSnapshotList:
 
     def test_list_empty(self, initialized_archive):
         """List with no snapshots returns empty."""
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["snapshot", "list"],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        _stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        assert "Listed 0 snapshots" in stderr
+        assert result.returncode == 0
+        assert "Listed 0 snapshots" in result.stderr
 
     def test_list_returns_created(self, initialized_archive):
         """List returns previously created snapshots."""
         url = create_test_url()
         run_archivebox_cmd(["snapshot", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["snapshot", "list"],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
         assert len(records) >= 1
         assert any(r.get("url") == url for r in records)
 
@@ -183,16 +177,15 @@ class TestSnapshotList:
         url = create_test_url()
         run_archivebox_cmd(["snapshot", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["snapshot", "list", "--status=queued"],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
         for r in records:
             assert r["status"] == "queued"
 
@@ -201,16 +194,15 @@ class TestSnapshotList:
         url = create_test_url(domain="unique-domain-12345.com")
         run_archivebox_cmd(["snapshot", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["snapshot", "list", "--url__icontains=unique-domain-12345"],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
         assert len(records) == 1
         assert "unique-domain-12345" in records[0]["url"]
 
@@ -224,16 +216,15 @@ class TestSnapshotList:
                 disable_extractors=True,
             )
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["snapshot", "list", "--limit=2"],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
         assert len(records) == 2
 
     def test_list_with_sort_and_limit(self, initialized_archive):
@@ -246,16 +237,15 @@ class TestSnapshotList:
                 disable_extractors=True,
             )
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["snapshot", "list", "--limit=2", "--sort=-created_at"],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0, f"Command failed: {stderr}"
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0, f"Command failed: {result.stderr}"
+        records = parse_jsonl_output(result.stdout)
         assert len(records) == 2
 
 

archivebox/tests/test_recursive_crawl.py

@@ -635,7 +635,22 @@ def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(
         for _snapshot_id, url, depth, plugin, hook_name, status, _files, _size, output_str in archive_results
         if not (status in allowed_statuses or (plugin == "archivedotorg" and status == ArchiveResult.StatusChoices.FAILED))
     ]
-    assert not unexpected_results
+    # This fixture serves HTTP. TLSNotary must reject every plaintext document;
+    # a setup, timeout, or verifier error must not count as that expected refusal.
+    tlsnotary_results = [row for row in archive_results if row[3] == "tlsnotary"]
+    assert {row[1] for row in tlsnotary_results} == expected_urls
+    assert len(tlsnotary_results) == len(expected_urls)
+    with use_archivebox_db(initialized_archive):
+        for snapshot_id, _url, _depth, _plugin, hook_name, status, files, size, _output in tlsnotary_results:
+            assert status == ArchiveResult.StatusChoices.FAILED
+            assert not files
+            assert size == 0
+            process = ArchiveResult.objects.get(snapshot_id=snapshot_id, plugin="tlsnotary", hook_name=hook_name).process
+            assert process is not None
+            assert process.status == Process.StatusChoices.EXITED
+            assert process.exit_code == 1
+            assert process.stderr.strip() == "[tlsnotary] TLSNotary requires an HTTPS document"
+    assert not [result for result in unexpected_results if result["plugin"] != "tlsnotary"]
     ytdlp_results = [
         (url, status, output_str)
         for _snapshot_id, url, _depth, plugin, _hook_name, status, _files, _size, output_str in archive_results
@@ -682,19 +697,6 @@ def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(
     assert set(recursive_test_site["deep_urls"]).issubset(parsed_urls)
 
     assert processes
-    failed_hook_results = [
-        {
-            "url": url,
-            "depth": depth,
-            "plugin": plugin,
-            "hook_name": hook_name,
-            "status": status,
-            "output_str": output_str,
-        }
-        for _snapshot_id, url, depth, plugin, hook_name, status, _files, _size, output_str in archive_results
-        if status == ArchiveResult.StatusChoices.FAILED and plugin != "archivedotorg"
-    ]
-    assert not failed_hook_results
     assert all(status == Process.StatusChoices.EXITED for _id, _pwd, _cmd, status, _exit_code, _started_at, _ended_at in processes)
 
     intervals = []

archivebox/tests/test_resource_admission.py

@@ -0,0 +1,30 @@
+import os
+
+from archivebox.services.resource_admission import ResourceAdmission
+
+
+def test_cpu_admission_respects_process_affinity():
+    admission = ResourceAdmission()
+    available = len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else os.cpu_count()
+    assert 1 <= admission.snapshot_limit(100000) <= available
+    assert admission.snapshot_limit(1) == 1
+
+
+def test_memory_stall_counter_recovers_without_resetting_monitor():
+    # Real kernel PSI units: cumulative microseconds and ten-second average.
+    # Decision samples exercise the policy, not simulated hooks/processes.
+    admission = ResourceAdmission()
+    assert admission.observe_memory_stalls("host", 100, 0.0, 0.0) is False
+    assert admission.observe_memory_stalls("host", 400100, 20.0, 2.0) is True
+    assert admission.observe_memory_stalls("host", 400100, 20.0, 4.0) is False
+    assert admission.observe_memory_stalls("new-cgroup", 900, 20.0, 0.0) is True
+    assert admission.observe_memory_stalls("new-cgroup", 900, 20.0, 2.0) is False
+
+
+def test_transient_and_low_memory_stalls_do_not_starve_admission():
+    admission = ResourceAdmission()
+    for sample in range(20):
+        assert admission.observe_memory_stalls("host", sample * 2600, 0.13, sample * 2.0) is False
+    # One large interval alone is not sustained pressure; the rolling average
+    # must agree before deferring work.
+    assert admission.observe_memory_stalls("host", 1_000_000, 0.13, 40.0) is False

archivebox/tests/test_search.py

@@ -1289,3 +1289,75 @@ class TestSearchBackendsE2E:
             fixture_server.shutdown()
             fixture_server.server_close()
             fixture_thread.join(timeout=5)
+
+
+@pytest.mark.timeout(90)
+def test_live_public_search_disconnect_reaps_backend(initialized_archive):
+    """A real HTTP disconnect cancels a silent ripgrep process and its wrapper."""
+    import signal
+    from uuid import uuid4
+
+    import psutil
+
+    from archivebox.core.models import Snapshot
+    from archivebox.tests.test_orm_helpers import use_archivebox_db
+
+    rg = shutil.which("rg")
+    assert rg
+    query = "disconnect" + uuid4().hex
+    port = get_free_port()
+    env = cli_env(
+        live=True,
+        PUBLIC_INDEX="True",
+        PERMISSIONS="public",
+        SEARCH_BACKEND_ENGINE="ripgrep",
+        SEARCH_BACKEND_RIPGREP_ENABLED="True",
+        RIPGREP_BINARY=rg,
+        RIPGREP_ARGS_EXTRA='["--no-mmap", "--text", "--sort", "path"]',
+        RIPGREP_TIMEOUT="30",
+        ALLOWED_HOSTS="*",
+    )
+    created = run_archivebox_cmd(
+        ["snapshot", "create", "--status=sealed", "https://example.com/disconnect-test"],
+        cwd=initialized_archive,
+        env=env,
+    )
+    assert created.returncode == 0, created.stderr
+    with use_archivebox_db(initialized_archive):
+        snapshot = Snapshot.objects.get(url="https://example.com/disconnect-test")
+        directory = snapshot.output_dir / "dom"
+        directory.mkdir(parents=True, exist_ok=True)
+        (directory / "a.txt").write_text(query + "\n")
+        with (directory / "z.txt").open("wb") as large:
+            large.truncate(8 * 1024**3)
+            for offset in range(0, 8 * 1024**3, 1024**2):
+                large.seek(offset)
+                large.write(b"\n")
+    server = start_archivebox_server(initialized_archive, port=port, env=env, daemonize=False, log_name="disconnect-server.log")
+    children = []
+    try:
+        with requests.get(
+            f"http://127.0.0.1:{port}/public/search-stream/",
+            params={"q": query, "search_mode": "deep:ripgrep"},
+            headers={"Host": f"web.archivebox.localhost:{port}"},
+            stream=True,
+            timeout=(5, 10),
+        ) as response:
+            assert response.status_code == 200, response.text
+            lines = response.iter_lines()
+            assert int(next(lines).strip()) == 0
+            assert int(next(lines).strip()) == 1
+            children = [child for child in psutil.Process(server.pid).children(recursive=True) if query in " ".join(child.cmdline())]
+            assert len(children) == 2
+            for child in children:
+                if Path(child.exe()).name == Path(rg).name:
+                    child.send_signal(signal.SIGSTOP)
+        _, alive = psutil.wait_procs(children, timeout=5)
+        assert not alive, [(child.pid, child.status()) for child in alive]
+    finally:
+        for child in children:
+            try:
+                child.kill()
+            except psutil.NoSuchProcess:
+                pass
+        stop_archivebox_process(server)

archivebox/tests/test_server_security_browser.py

@@ -963,14 +963,13 @@ def test_archivewebpage_wacz_preview_serves_real_capture_frame(initialized_archi
             log_name="archivewebpage_server.log",
         )
         get_http_response(port, host=f"archivebox.localhost:{port}", path="/")
-        _cmd_result = run_archivebox_cmd(
+        command_result = run_archivebox_cmd(
             ["add", "--depth=0", "--max-urls=1", "--plugins=archivewebpage", url],
             cwd=initialized_archive,
             env=env,
             timeout=120,
         )
-        stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
-        assert returncode == 0, f"archivebox add failed:\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}"
+        assert command_result.returncode == 0, f"archivebox add failed:\nSTDOUT:\n{command_result.stdout}\nSTDERR:\n{command_result.stderr}"
 
         capture = _get_archivewebpage_capture(initialized_archive, url)
         snapshot_host = f"{get_snapshot_subdomain(capture['snapshot_id'])}.archivebox.localhost:{port}"

archivebox/tests/test_snapshot_service.py

@@ -163,14 +163,15 @@ def test_snapshot_service_cli_add_seals_snapshot_and_writes_indexes(tmp_path, re
 
     port = get_free_port()
     env = cli_env(port=port, server=True, PLUGINS="wget", SAVE_WGET="True")
-    _cmd_result = run_archivebox_cmd(
+    command_result = run_archivebox_cmd(
         ["add", "--depth=0", "--plugins=wget", recursive_test_site["root_url"]],
         cwd=tmp_path,
         env=env,
         timeout=180,
     )
-    stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
-    assert code == 0, f"archivebox add failed with code {code}\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}"
+    assert command_result.returncode == 0, (
+        f"archivebox add failed with code {command_result.returncode}\nSTDOUT:\n{command_result.stdout}\nSTDERR:\n{command_result.stderr}"
+    )
 
     state = _snapshot_state(tmp_path, recursive_test_site["root_url"])
     snapshot_dir = state["snapshot_dir"]

archivebox/tests/test_takeover_util.py

@@ -387,14 +387,13 @@ def test_live_second_server_takes_over_existing_server_process(tmp_path, initial
         assert "A newer archivebox process took over the orchestrator, server" in first_text
         assert "Starting orchestrator, server" in second_text
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["status"],
             cwd=tmp_path,
             env=env,
             timeout=60,
         )
-        stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
-        assert returncode == 0, stderr or stdout
+        assert result.returncode == 0, result.stderr or result.stdout
 
         first_resumes = first_log.read_text(encoding="utf-8", errors="replace").count("Other newer archivebox process")
         stop_archivebox_process(second, signal.SIGTERM)
@@ -422,15 +421,14 @@ def test_live_update_index_only_does_not_take_over_server_runtime(tmp_path, init
         supervisor_pid_before = supervisor_pid_from_log(server_log)
         daphne_pid_before = worker_pid_from_log(server_log, "worker_daphne")
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["update", "--index-only", "--before=0"],
             cwd=tmp_path,
             env=env,
             timeout=90,
         )
-        update_stdout, update_stderr, update_returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert update_returncode == 0, update_stderr or update_stdout
+        assert result.returncode == 0, result.stderr or result.stdout
         assert pid_is_alive(server.pid)
         assert pid_is_alive(supervisor_pid_before)
         assert pid_is_alive(daphne_pid_before)
@@ -471,15 +469,14 @@ def test_live_server_keeps_http_runtime_while_update_runs_real_sqlite_indexer(tm
         runner_spawn_text = "spawned: 'worker_runner' with pid"
         runner_spawn_count = supervisord_log.read_text(encoding="utf-8", errors="replace").count(runner_spawn_text)
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["update", "--index-only", "--batch-size=1"],
             cwd=tmp_path,
             env=env,
             timeout=180,
         )
-        update_stdout, update_stderr, update_returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert update_returncode == 0, update_stderr or update_stdout
+        assert result.returncode == 0, result.stderr or result.stdout
         assert pid_is_alive(server.pid)
         assert supervisor_pid_from_log(server_log) == supervisor_pid_before
         assert pid_is_alive(daphne_pid_before)
@@ -628,14 +625,13 @@ def test_live_repeated_server_startups_take_over_cleanly(tmp_path, initialized_a
                 wait_for_pid_to_disappear(daphne_pids[index - 1], timeout=15)
                 wait_for_pid_to_disappear(runner_pids[index - 1], timeout=15)
 
-            _cmd_result = run_archivebox_cmd(
+            result = run_archivebox_cmd(
                 ["status"],
                 cwd=tmp_path,
                 env=env,
                 timeout=60,
             )
-            stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
-            assert returncode == 0, stderr or stdout
+            assert result.returncode == 0, result.stderr or result.stdout
 
         assert pid_is_alive(servers[-1].pid)
         assert all(pid_is_alive(server.pid) for server in servers)
@@ -673,14 +669,13 @@ def test_live_background_add_survives_server_exit_and_foreground_run_reclaims(tm
         server_log = server.log_path
         supervisor_pid_before = supervisor_pid_from_log(server_log)
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["update", "--index-only", "--batch-size=10"],
             cwd=tmp_path,
             env=env,
             timeout=90,
         )
-        update_stdout, update_stderr, update_returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
-        assert update_returncode == 0, update_stderr or update_stdout
+        assert result.returncode == 0, result.stderr or result.stdout
         assert pid_is_alive(server.pid)
         assert pid_is_alive(supervisor_pid_before)
         assert supervisor_pid_from_log(server_log) == supervisor_pid_before

archivebox/tests/test_ui_add_view.py

@@ -20,6 +20,23 @@ WEB_HOST = "web.archivebox.localhost:8000"
 ADMIN_HOST = "admin.archivebox.localhost:8000"
 
 
+ADD_FORM_DEFAULTS = {
+    "tag": "",
+    "depth": "0",
+    "max_urls": "0",
+    "crawl_max_size": "0",
+    "snapshot_max_size": "0",
+    "url_filters_allowlist": "",
+    "url_filters_denylist": "",
+    "notes": "",
+    "schedule": "",
+    "persona": "Default",
+    "permissions": "public",
+    "start_paused": "",
+    "config": "{}",
+}
+
+
 @pytest.fixture
 def admin_user(db):
     return User.objects.create_superuser(
@@ -140,22 +157,13 @@ def test_add_view_staff_user_cannot_override_raw_or_plugin_config(client):
     response = client.post(
         reverse("add"),
         data={
+            **ADD_FORM_DEFAULTS,
             "url": "https://example.com/staff-config",
-            "tag": "",
-            "depth": "0",
-            "max_urls": "0",
-            "crawl_max_size": "0",
             "crawl_timeout": "0",
             "timeout": "60",
-            "snapshot_max_size": "0",
             "delete_after": "0",
             "crawl_max_concurrent_snapshots": "1",
-            "url_filters_allowlist": "",
-            "url_filters_denylist": "",
-            "notes": "",
             "schedule": "daily",
-            "persona": "Default",
-            "permissions": "public",
             "start_paused": "on",
             "main_plugins": ["wget"],
             "plugin_config__wget__WGET_TIMEOUT": "77",
@@ -246,6 +254,7 @@ def test_add_view_creates_crawl_with_tag_and_url_filter_overrides(client, admin_
     response = client.post(
         reverse("add"),
         data={
+            **ADD_FORM_DEFAULTS,
             "url": "https://example.com\nhttps://cdn.example.com/asset.js",
             "tag": "alpha,beta",
             "depth": "1",
@@ -260,11 +269,6 @@ def test_add_view_creates_crawl_with_tag_and_url_filter_overrides(client, admin_
             "url_filters_denylist": "cdn.example.com",
             "url_filters_only_new": "1",
             "notes": "Created from /add/",
-            "schedule": "",
-            "persona": "Default",
-            "permissions": "public",
-            "start_paused": "",
-            "config": "{}",
         },
         HTTP_HOST=ADMIN_HOST,
     )
@@ -290,24 +294,12 @@ def test_add_view_creates_crawl_with_tag_and_url_filter_overrides(client, admin_
 
 def _csrf_test_add_data(url: str) -> dict[str, str]:
     return {
+        **ADD_FORM_DEFAULTS,
         "url": url,
-        "tag": "",
-        "depth": "0",
-        "max_urls": "0",
-        "crawl_max_size": "0",
         "crawl_timeout": "0",
         "timeout": "",
-        "snapshot_max_size": "0",
         "delete_after": "0",
         "crawl_max_concurrent_snapshots": "1",
-        "url_filters_allowlist": "",
-        "url_filters_denylist": "",
-        "notes": "",
-        "schedule": "",
-        "persona": "Default",
-        "permissions": "public",
-        "start_paused": "",
-        "config": "{}",
     }
 
 
@@ -337,24 +329,14 @@ def test_add_view_sanitizes_crawl_notes_before_safe_update(client, admin_user):
     response = client.post(
         reverse("add"),
         data={
+            **ADD_FORM_DEFAULTS,
             "url": "https://example.com/notes-xss",
-            "tag": "",
-            "depth": "0",
             "max_urls": "1",
-            "crawl_max_size": "0",
             "crawl_timeout": "0",
             "timeout": "",
-            "snapshot_max_size": "0",
             "delete_after": "0",
             "crawl_max_concurrent_snapshots": "1",
-            "url_filters_allowlist": "",
-            "url_filters_denylist": "",
             "notes": malicious_notes,
-            "schedule": "",
-            "persona": "Default",
-            "permissions": "public",
-            "start_paused": "",
-            "config": "{}",
         },
         HTTP_HOST=ADMIN_HOST,
     )
@@ -373,20 +355,8 @@ def test_add_view_unchecked_only_new_sets_crawl_override(client, admin_user):
     response = client.post(
         reverse("add"),
         data={
+            **ADD_FORM_DEFAULTS,
             "url": "https://example.com/rearchive",
-            "tag": "",
-            "depth": "0",
-            "max_urls": "0",
-            "crawl_max_size": "0",
-            "snapshot_max_size": "0",
-            "url_filters_allowlist": "",
-            "url_filters_denylist": "",
-            "notes": "",
-            "schedule": "",
-            "persona": "Default",
-            "permissions": "public",
-            "start_paused": "",
-            "config": "{}",
         },
         HTTP_HOST=ADMIN_HOST,
     )
@@ -407,20 +377,10 @@ def test_add_view_selected_persona_wins_over_stale_config_override(client, admin
     response = client.post(
         reverse("add"),
         data={
+            **ADD_FORM_DEFAULTS,
             "url": "https://example.com/private",
-            "tag": "",
-            "depth": "0",
-            "max_urls": "0",
-            "crawl_max_size": "0",
-            "snapshot_max_size": "0",
-            "url_filters_allowlist": "",
-            "url_filters_denylist": "",
             "url_filters_only_new": "1",
-            "notes": "",
-            "schedule": "",
             "persona": "Private",
-            "permissions": "public",
-            "start_paused": "",
             "config": '{"DEFAULT_PERSONA": "Default"}',
         },
         HTTP_HOST=ADMIN_HOST,
@@ -447,20 +407,9 @@ def test_add_view_applies_plugin_config_overrides(client, admin_user):
     response = client.post(
         reverse("add"),
         data={
+            **ADD_FORM_DEFAULTS,
             "url": "https://example.com/plugin-config",
-            "tag": "",
-            "depth": "0",
-            "max_urls": "0",
-            "crawl_max_size": "0",
-            "snapshot_max_size": "0",
-            "url_filters_allowlist": "",
-            "url_filters_denylist": "",
             "url_filters_only_new": "1",
-            "notes": "",
-            "schedule": "",
-            "persona": "Default",
-            "permissions": "public",
-            "start_paused": "",
             "main_plugins": ["wget"],
             "plugin_config__wget__WGET_ENABLED": "false",
             "plugin_config__wget__WGET_TIMEOUT": "77",
@@ -488,9 +437,8 @@ def test_add_view_public_submission_ignores_plugin_and_custom_config(client, adm
     response = client.post(
         reverse("add"),
         data={
+            **ADD_FORM_DEFAULTS,
             "url": "https://example.com/public-safe",
-            "tag": "",
-            "depth": "0",
             "max_urls": "10",
             "crawl_max_size": "45mb",
             "crawl_timeout": "120",
@@ -503,8 +451,6 @@ def test_add_view_public_submission_ignores_plugin_and_custom_config(client, adm
             "url_filters_only_new": "1",
             "notes": "public add",
             "schedule": "daily",
-            "persona": "Default",
-            "permissions": "public",
             "start_paused": "on",
             "main_plugins": ["wget"],
             "plugin_config__twocaptcha__TWOCAPTCHA_API_KEY": "posted-token",
@@ -541,21 +487,9 @@ def test_add_view_queues_crawl_for_background_runner(client, admin_user):
     response = client.post(
         reverse("add"),
         data={
+            **ADD_FORM_DEFAULTS,
             "url": "https://example.com",
-            "tag": "",
-            "depth": "0",
-            "max_urls": "0",
-            "crawl_max_size": "0",
-            "snapshot_max_size": "0",
-            "url_filters_allowlist": "",
-            "url_filters_denylist": "",
             "url_filters_only_new": "1",
-            "notes": "",
-            "schedule": "",
-            "persona": "Default",
-            "permissions": "public",
-            "start_paused": "",
-            "config": "{}",
         },
         HTTP_HOST=ADMIN_HOST,
     )
@@ -575,21 +509,11 @@ def test_add_view_start_paused_creates_paused_crawl_without_snapshots(client, ad
     response = client.post(
         reverse("add"),
         data={
+            **ADD_FORM_DEFAULTS,
             "url": "https://example.com/paused",
-            "tag": "",
             "depth": "1",
-            "max_urls": "0",
-            "crawl_max_size": "0",
-            "snapshot_max_size": "0",
-            "url_filters_allowlist": "",
-            "url_filters_denylist": "",
             "url_filters_only_new": "1",
-            "notes": "",
-            "schedule": "",
-            "persona": "Default",
-            "permissions": "public",
             "start_paused": "on",
-            "config": "{}",
         },
         HTTP_HOST=ADMIN_HOST,
     )
@@ -610,6 +534,7 @@ def test_add_view_extracts_urls_from_mixed_text_input(client, admin_user):
     response = client.post(
         reverse("add"),
         data={
+            **ADD_FORM_DEFAULTS,
             "url": (
                 "https://sweeting.me,https://google.com\n"
                 "Notes: [ArchiveBox](https://github.com/ArchiveBox/ArchiveBox), https://news.ycombinator.com\n"
@@ -617,20 +542,7 @@ def test_add_view_extracts_urls_from_mixed_text_input(client, admin_user):
                 '{"items":["https://example.com/three"]}\n'
                 "csv,https://example.com/four"
             ),
-            "tag": "",
-            "depth": "0",
-            "max_urls": "0",
-            "crawl_max_size": "0",
-            "snapshot_max_size": "0",
-            "url_filters_allowlist": "",
-            "url_filters_denylist": "",
             "url_filters_only_new": "1",
-            "notes": "",
-            "schedule": "",
-            "persona": "Default",
-            "permissions": "public",
-            "start_paused": "",
-            "config": "{}",
         },
         HTTP_HOST=ADMIN_HOST,
     )
@@ -656,21 +568,9 @@ def test_add_view_trims_trailing_punctuation_from_markdown_urls(client, admin_us
     response = client.post(
         reverse("add"),
         data={
+            **ADD_FORM_DEFAULTS,
             "url": ("Docs: https://github.com/ArchiveBox/ArchiveBox.\nIssue: https://github.com/abc?abc#234234?."),
-            "tag": "",
-            "depth": "0",
-            "max_urls": "0",
-            "crawl_max_size": "0",
-            "snapshot_max_size": "0",
-            "url_filters_allowlist": "",
-            "url_filters_denylist": "",
             "url_filters_only_new": "1",
-            "notes": "",
-            "schedule": "",
-            "persona": "Default",
-            "permissions": "public",
-            "start_paused": "",
-            "config": "{}",
         },
         HTTP_HOST=ADMIN_HOST,
     )

archivebox/tests/test_ui_add_view_runtime.py

@@ -431,6 +431,10 @@ def test_public_add_view_import_text_formats_preserve_metadata_and_resume_withou
         URL_ALLOWLIST=r"127\.0\.0\.1[:/].*",
     )
     create_admin_and_token(tmp_path)
+    # These tests deliberately stop the server while crawls are queued. Finish
+    # dependency installation first so shutdown cannot leave a half-built venv.
+    install_result = run_archivebox_cmd(["install", *env["PLUGINS"].split(",")], cwd=tmp_path, env=env, timeout=180)
+    assert install_result.returncode == 0, install_result.stderr or install_result.stdout
 
     try:
         start_archivebox_server(tmp_path, env=env, port=port)
@@ -536,6 +540,10 @@ def test_public_add_view_rejects_file_path_and_shell_injection_payloads(tmp_path
         URL_ALLOWLIST=r"example\.com|example\.org|iana\.org|www\.iana\.org",
     )
     create_admin_and_token(tmp_path)
+    # These tests deliberately stop the server while crawls are queued. Finish
+    # dependency installation first so shutdown cannot leave a half-built venv.
+    install_result = run_archivebox_cmd(["install", *env["PLUGINS"].split(",")], cwd=tmp_path, env=env, timeout=180)
+    assert install_result.returncode == 0, install_result.stderr or install_result.stdout
 
     try:
         start_archivebox_server(tmp_path, env=env, port=port)

archivebox/tests/test_ui_admin_crawl.py

@@ -14,6 +14,7 @@ pytestmark = pytest.mark.django_db
 
 
 class TestCrawlScheduleAdmin:
+    @pytest.mark.django_db(transaction=True)
     def test_crawlschedule_change_view_renders_and_saves(self, client, admin_user, crawl):
         from archivebox.crawls.models import CrawlSchedule
 
@@ -35,24 +36,33 @@ class TestCrawlScheduleAdmin:
         assert b"No Crawls yet..." not in get_response.content
         assert b"No Snapshots yet..." not in get_response.content
 
-        post_response = client.post(
-            change_url,
-            {
-                "label": "Morning crawl",
-                "notes": "updated",
-                "schedule": "0 8 * * *",
-                "template": str(crawl.pk),
-                "created_by": str(admin_user.pk),
-                "_save": "Save",
-            },
-            HTTP_HOST=ADMIN_TEST_HOST,
-        )
+        from django.db import connection
+        from django.test.utils import CaptureQueriesContext
+
+        previous_mode = getattr(connection, "transaction_mode", None)
+        with CaptureQueriesContext(connection) as queries:
+            post_response = client.post(
+                change_url,
+                {
+                    "label": "Morning crawl",
+                    "notes": "updated",
+                    "schedule": "0 8 * * *",
+                    "template": str(crawl.pk),
+                    "created_by": str(admin_user.pk),
+                    "_save": "Save",
+                },
+                HTTP_HOST=ADMIN_TEST_HOST,
+            )
+        if connection.vendor == "sqlite":
+            assert any(query["sql"] == "BEGIN IMMEDIATE" for query in queries)
+            assert connection.transaction_mode == previous_mode
 
         assert post_response.status_code == 302
         schedule.refresh_from_db()
         assert schedule.label == "Morning crawl"
         assert schedule.notes == "updated"
         assert schedule.schedule == "0 8 * * *"
+        assert schedule.is_enabled is False
         assert schedule.template_id == crawl.pk
         assert schedule.created_by_id == admin_user.pk
 

archivebox/tests/test_ui_admin_snapshot.py

@@ -607,7 +607,14 @@ class TestSnapshotProgressStats:
 
         assert result.embed_path_db() is None
 
-    def test_embed_path_db_prefers_valid_output_str_over_first_output_file(self, snapshot, real_hash_projection):
+    @pytest.mark.parametrize(
+        "output_str",
+        [
+            ("wget/example.com/index.html"),
+            (""),
+        ],
+    )
+    def test_embed_path_db_selects_html_output(self, snapshot, real_hash_projection, output_str):
         from archivebox.core.models import ArchiveResult
 
         output_dir = Path(snapshot.output_dir) / "wget" / "example.com" / "assets" / "css"
@@ -619,32 +626,7 @@ class TestSnapshotProgressStats:
         _process, result = real_hash_projection
         ArchiveResult.objects.filter(pk=result.pk).update(
             plugin="wget",
-            output_str="wget/example.com/index.html",
-            output_files={
-                "example.com/assets/css/mobile.css": {"size": (output_dir / "mobile.css").stat().st_size, "mimetype": "text/css"},
-                "example.com/index.html": {
-                    "size": (Path(snapshot.output_dir) / "wget" / "example.com" / "index.html").stat().st_size,
-                    "mimetype": "text/html",
-                },
-            },
-        )
-        result.refresh_from_db()
-
-        assert result.embed_path_db() == "wget/example.com/index.html"
-
-    def test_embed_path_db_scores_output_files_instead_of_using_first_entry(self, snapshot, real_hash_projection):
-        from archivebox.core.models import ArchiveResult
-
-        output_dir = Path(snapshot.output_dir) / "wget" / "example.com" / "assets" / "css"
-        output_dir.mkdir(parents=True, exist_ok=True)
-        (Path(snapshot.output_dir) / "wget" / "example.com" / "index.html").parent.mkdir(parents=True, exist_ok=True)
-        shutil.copyfile(REPO_ROOT / "README.md", Path(snapshot.output_dir) / "wget" / "example.com" / "index.html")
-        shutil.copyfile(REPO_ROOT / "archivebox" / "templates" / "static" / "bootstrap.min.css", output_dir / "mobile.css")
-
-        _process, result = real_hash_projection
-        ArchiveResult.objects.filter(pk=result.pk).update(
-            plugin="wget",
-            output_str="",
+            output_str=output_str,
             output_files={
                 "example.com/assets/css/mobile.css": {"size": (output_dir / "mobile.css").stat().st_size, "mimetype": "text/css"},
                 "example.com/index.html": {

archivebox/tests/test_ui_live_progress.py

@@ -104,6 +104,74 @@ def real_crawl_setup_process(snapshot, hermetic_lib_dir):
 
 
 class TestLiveProgressView:
+    def test_collection_summary_reports_stored_sizes(self, client, admin_user, snapshot):
+        import time
+        from django.core.cache import cache
+        from django.db.models import Count, Sum
+        from archivebox.core.models import Snapshot
+
+        Snapshot.objects.filter(pk=snapshot.pk).update(output_size=12345)
+        expected = Snapshot.objects.aggregate(snapshots=Count("*"), bytes=Sum("output_size"))
+        cache.delete_many(["progress-collection:all", "progress-collection:all:refresh"])
+        client.force_login(admin_user)
+        deadline = time.monotonic() + 5
+        while True:
+            response = client.get(reverse("live_progress"), {"collection": "1"}, HTTP_HOST=ADMIN_TEST_HOST)
+            assert response.status_code == 200
+            summary = response.json().get("collection")
+            if summary is not None or time.monotonic() >= deadline:
+                break
+            time.sleep(0.05)
+        assert summary is not None
+        assert summary["snapshots"] == expected["snapshots"]
+        assert summary["bytes"] == expected["bytes"]
+        from django.db import connection
+        from django.test.utils import CaptureQueriesContext
+
+        if connection.vendor == "sqlite":
+            with connection.cursor() as cursor:
+                cursor.execute("EXPLAIN QUERY PLAN SELECT COUNT(*), SUM(output_size) FROM core_snapshot")
+                plan = " ".join(str(row) for row in cursor.fetchall())
+            assert "COVERING INDEX" in plan, plan
+
+        timings = []
+        with CaptureQueriesContext(connection) as queries:
+            for _ in range(20):
+                start = time.perf_counter()
+                response = client.get(reverse("live_progress"), {"collection": "1"}, HTTP_HOST=ADMIN_TEST_HOST)
+                timings.append((time.perf_counter() - start) * 1000)
+                assert response.status_code == 200
+                assert response.json()["collection"] == summary
+        assert not any('AS "bytes"' in query["sql"] for query in queries)
+        print(f"cached progress response: median={sorted(timings)[10]:.1f}ms max={max(timings):.1f}ms")
+
+    def test_collection_summary_is_scoped_to_staff_owner(self, client, admin_user, snapshot):
+        import time
+        from django.core.cache import cache
+        from archivebox.core.models import Snapshot
+
+        admin_user.is_superuser = False
+        admin_user.save(update_fields=["is_superuser"])
+        from django.contrib.auth import get_user_model
+
+        # Give the snapshot another owner; staff must not learn its size/count.
+        snapshot.crawl.created_by = get_user_model().objects.create_user(username="other-summary-owner")
+        snapshot.crawl.save(update_fields=["created_by"])
+        Snapshot.objects.filter(pk=snapshot.pk).update(output_size=12345)
+        cache.delete_many([f"progress-collection:{admin_user.pk}", f"progress-collection:{admin_user.pk}:refresh"])
+        client.force_login(admin_user)
+        deadline = time.monotonic() + 5
+        while True:
+            response = client.get(reverse("live_progress"), {"collection": "1"}, HTTP_HOST=ADMIN_TEST_HOST)
+            assert response.status_code == 200
+            summary = response.json().get("collection")
+            if summary is not None or time.monotonic() >= deadline:
+                break
+            time.sleep(0.05)
+        assert summary is not None
+        assert summary["snapshots"] == 0
+        assert summary["bytes"] == 0
+
     def test_live_progress_rejects_unauthenticated_unscoped_request(self, client):
         response = client.get(reverse("live_progress"), HTTP_HOST=ADMIN_TEST_HOST)
 

archivebox/tests/test_urls.py

@@ -1431,7 +1431,7 @@ class TestUrlRouting:
             [
                 "add",
                 "--depth=0",
-                "--plugins=consolelog,screenshot,chrome_mhtml",
+                "--plugins=consolelog,screenshot,pdf,chrome_mhtml",
                 console_source_url,
             ],
             cwd=self.data_dir,
@@ -1475,6 +1475,14 @@ class TestUrlRouting:
             resp = client.get(f"/{screenshot_rel}?preview=1", HTTP_HOST=snapshot_host)
             assert resp.status_code == 200
             assert resp["Content-Type"].startswith("text/html")
+            assert f"frame-ancestors 'self' http://{web_host}" in resp["Content-Security-Policy"]
+
+            resp = client.get("/pdf/output.pdf?preview=1", HTTP_HOST=snapshot_host)
+            assert resp.status_code == 200
+            pdf_preview = response_body(resp).decode("utf-8")
+            assert '<iframe ' in pdf_preview
+            assert '<embed ' not in pdf_preview
+            assert "object-src 'none'" in resp["Content-Security-Policy"]
 
             root_screenshot = screenshot_file.read_bytes()
             import shutil
@@ -1507,6 +1515,58 @@ class TestUrlRouting:
             """.replace("__CONSOLE_MARKER__", console_marker),
         )
 
+        # Exercise the same real captured files with each request-time posture,
+        # including both auto branches and an HTTPS canonical reverse-proxy URL.
+        for mode, base in (
+            ("auto", "http://archivebox.localhost:8000"),
+            ("auto", "http://archivebox.example:8000"),
+            ("safe-subdomains-fullreplay", "https://archivebox.example"),
+            ("safe-onedomain-nojsreplay", "http://archivebox.example:8000"),
+            ("unsafe-onedomain-noadmin", "http://archivebox.example:8000"),
+            ("danger-onedomain-fullreplay", "http://archivebox.example:8000"),
+        ):
+            self._run(
+                """
+                from urllib.parse import urlsplit
+                from archivebox.config.common import get_request_config
+                snapshot = get_snapshot()
+                client = Client()
+                for path in ('screenshot/screenshot.png', 'pdf/output.pdf', 'chrome_mhtml/snapshot.mhtml'):
+                    target = urlsplit(build_snapshot_url(str(snapshot.id), path))
+                    response = client.get(target.path + '?preview=1', HTTP_HOST=target.netloc)
+                    assert response.status_code == 200, (target, response.status_code)
+                    csp = response['Content-Security-Policy']
+                    config = get_request_config(response.wsgi_request)
+                    origins = tuple(dict.fromkeys((get_web_base_url(config=config), get_admin_base_url(config=config))))
+                    ancestors = next(d.strip() for d in csp.split(';') if d.strip().startswith('frame-ancestors'))
+                    assert ancestors == "frame-ancestors 'self' " + ' '.join(origins), ancestors
+                    assert '*' not in ancestors  # no unrelated snapshots/sites
+                    assert "object-src 'none'" in csp
+                    assert "form-action 'none'" in csp
+                    assert "base-uri 'none'" in csp
+                    assert "script-src 'self'" in csp  # trusted viewer can run
+                    if path.endswith('.mhtml'):
+                        assert 'sandbox="allow-popups allow-popups-to-escape-sandbox allow-same-origin"' in response_body(response).decode()
+
+                dangerous, _, _ = write_replay_fixtures(snapshot)
+                target = urlsplit(build_snapshot_url(str(snapshot.id), dangerous))
+                response = client.get(target.path, HTTP_HOST=target.netloc)
+                assert response.status_code == 200
+                config = get_request_config(response.wsgi_request)
+                csp = response.headers.get('Content-Security-Policy', '')
+                assert ("script-src 'none'" in csp) == config.SHOULD_NEUTER_RISKY_REPLAY
+                assert ('sandbox;' in csp) == config.SHOULD_NEUTER_RISKY_REPLAY
+                admin = client.get('/admin/login/', HTTP_HOST=get_admin_host())
+                assert admin.status_code == (200 if config.CONTROL_PLANE_ENABLED else 403)
+                if not config.CONTROL_PLANE_ENABLED:
+                    assert client.get('/api/v1/docs', HTTP_HOST=get_api_host()).status_code == 403
+                    assert client.post('/public/', HTTP_HOST=get_web_host()).status_code == 403
+                print('OK')
+                """,
+                mode=mode,
+                env_overrides={"BASE_URL": base},
+            )
+
     def test_api_available_on_admin_and_api_hosts(self) -> None:
         self._run(
             """

archivebox/workers/supervisord_util.py

@@ -47,7 +47,6 @@ _SUPERVISORD_ERRORS = (XmlRpcError, OSError, RuntimeError, TimeoutError)
 _PROCESS_STATE_ERRORS = (DatabaseError, OSError, RuntimeError, ValueError, psutil.Error)
 MIN_SERVER_WORKER_AVAILABLE_MEMORY_BYTES = 256 * 1024 * 1024
 MIN_DEPENDENCY_INSTALL_AVAILABLE_MEMORY_BYTES = MIN_SERVER_WORKER_AVAILABLE_MEMORY_BYTES
-MIN_CRAWL_AVAILABLE_MEMORY_BYTES = 512 * 1024 * 1024
 
 
 def _shell_join(args: list[str]) -> str:
@@ -118,16 +117,6 @@ def require_dependency_install_memory(available_bytes: int | None = None) -> Non
     )
 
 
-def require_crawl_memory(available_bytes: int | None = None) -> None:
-    available_bytes = effective_available_memory_bytes() if available_bytes is None else available_bytes
-    _require_available_memory(
-        "archive a crawl",
-        "    No crawl, runner, or Sonic workers were started.",
-        available_bytes,
-        MIN_CRAWL_AVAILABLE_MEMORY_BYTES,
-    )
-
-
 def archivebox_cmd(*args: str) -> list[str]:
     return [str(resolve_env_binary("archivebox")), *args]
 
@@ -1280,11 +1269,18 @@ def tail_multiple_worker_logs(log_files: list[str], follow=True, proc=None, keep
 
     print()
 
+    # Log display needs a short refresh interval, but each ownership check
+    # queries the database and validates OS processes. Check leadership once
+    # per second even when workers continuously produce log output.
+    next_ownership_check = 0.0
     try:
         while follow:
-            if keep_running is not None and not keep_running():
-                print("\n[newer ArchiveBox process is now running the orchestrator and server]")
-                return "transferred"
+            now = time.monotonic()
+            if keep_running is not None and now >= next_ownership_check:
+                if not keep_running():
+                    print("\n[newer ArchiveBox process is now running the orchestrator and server]")
+                    return "transferred"
+                next_ownership_check = time.monotonic() + 1.0
 
             # Check if the monitored process has exited
             if proc is not None and proc.poll() is not None:

bin/collect_ui_screenshots.sh

@@ -193,6 +193,22 @@ echo "[*] Creating a temporary screenshot admin"
 )
 CREATED_TEMP_USER=1
 
+# Configure the optional UI before the server snapshots its startup config.
+OPENCODE_PORT="$(uv run --no-cache --project "$REPO_DIR" python - <<'PYPORT'
+import socket
+
+with socket.socket() as sock:
+    sock.bind(("127.0.0.1", 0))
+    print(sock.getsockname()[1])
+PYPORT
+)"
+(
+    cd "$DATA_DIR"
+    uv run --no-cache --project "$REPO_DIR" archivebox config --set \
+        OPENCODE_ENABLED=True "OPENCODE_PORT=$OPENCODE_PORT" TLSNOTARY_ENABLED=True
+    uv run --no-cache --project "$REPO_DIR" archivebox install opencode --binproviders=env,pnpm
+)
+
 echo "[*] Starting ArchiveBox on port $PORT"
 (
     cd "$DATA_DIR"
@@ -248,6 +264,10 @@ while [[ "$capture_index" -lt "${#VIEWS[@]}" ]]; do
     capture_index=$((capture_index + 1))
     slug="$(printf '%s' "$name" | tr '[:upper:]' '[:lower:]' | tr -cs '[:alnum:]' '-' | sed 's/^-//; s/-$//')"
     expected_plugin=""
+    expected_frame_text=""
+    if [[ "$capture_mode" == wait-frame-text:* ]]; then
+        expected_frame_text="${capture_mode#wait-frame-text:}"
+    fi
     if [[ "$name" == "Snapshot View ("*")" && "$capture_mode" != "live-progress" && "$capture_mode" != "snapshot-collapsed" ]]; then
         expected_plugin="${name#Snapshot View (}"
         expected_plugin="${expected_plugin%)}"
@@ -407,6 +427,7 @@ PY
                     SCREENSHOT_VARIANTS_JSON="$output_variants" \
                     SCREENSHOT_COLLAPSE_FILTERS=1 \
                     SCREENSHOT_EXPECT_PLUGIN="$expected_plugin" \
+                    SCREENSHOT_EXPECT_FRAME_TEXT="$expected_frame_text" \
                     node "$REPO_DIR/bin/take_screenshot.js" "$url" "$screenshot_path" >"$capture_dir/report.json"
                 view_timing_report="$capture_dir/report.json"
                 timing_report_path="$view_timing_report"
@@ -494,21 +515,6 @@ PY
             CREATE_WEBHOOK="$CREATE_WEBHOOK" \
             node "$REPO_DIR/bin/setup_ui_screenshot_data.js" "$ADMIN_BASE_URL" "$USERNAME"
 
-        OPENCODE_PORT="$(uv run --no-cache --project "$REPO_DIR" python - <<'PY'
-import socket
-
-with socket.socket() as sock:
-    sock.bind(("127.0.0.1", 0))
-    print(sock.getsockname()[1])
-PY
-)"
-        (
-            cd "$DATA_DIR"
-            OPENCODE_PORT="$OPENCODE_PORT" uv run --no-cache --project "$REPO_DIR" archivebox manage shell --no-imports -c \
-                'import os; from archivebox.machine.models import Machine; Machine.from_json({"config": {"OPENCODE_ENABLED": True, "OPENCODE_PORT": int(os.environ["OPENCODE_PORT"])}})'
-            uv run --no-cache --project "$REPO_DIR" archivebox install opencode --binproviders=env,pnpm
-        )
-
         SWEETING_CAPTURE_STARTED_AT="$( (
             cd "$DATA_DIR"
             uv run --no-cache --project "$REPO_DIR" archivebox manage shell --no-imports -c \
@@ -561,7 +567,7 @@ PY
         VIEWS+=(
             "Add URLs|$ADMIN_BASE_URL/add/|/add/|archivebox/core/views.py"
             "Admin dashboard|$ADMIN_BASE_URL/admin/|/admin/|archivebox/core/admin_site.py"
-            "AI agent|$ADMIN_BASE_URL/admin/agent/|/admin/agent/|abx_plugins/plugins/opencode/views.py|wait-text:ArchiveBox AI Agent"
+            "AI agent|$ADMIN_BASE_URL/admin/agent/|/admin/agent/|archivebox/opencode/views.py|wait-text:ArchiveBox AI Agent"
             "Snapshots table|$ADMIN_BASE_URL/admin/core/snapshot/|/admin/core/snapshot/|archivebox/core/admin_snapshots.py"
             "Snapshots grid|$ADMIN_BASE_URL/admin/core/snapshot/grid/|/admin/core/snapshot/grid/|archivebox/templates/admin/snapshots_grid.html"
             "Snapshot admin detail|$ADMIN_BASE_URL/admin/core/snapshot/$SNAPSHOT_ID/change/|/admin/core/snapshot/$SNAPSHOT_ID/change/|archivebox/core/admin_snapshots.py"
@@ -625,11 +631,17 @@ PY
             SCREENSHOT_HEIGHT=1000 \
             node "$REPO_DIR/bin/take_screenshot.js" "$LIVE_SNAPSHOT_VIEW_URL" "$CAPTURE_ROOT/snapshot-output-discovery.png" >"$SNAPSHOT_DISCOVERY_REPORT"
         SNAPSHOT_OUTPUT_PLUGINS="$(UI_SCREENSHOT_DISCOVERY_REPORT="$SNAPSHOT_DISCOVERY_REPORT" uv run --no-cache --project "$REPO_DIR" python -c \
-            'import json, os; from urllib.parse import urlsplit; report=json.load(open(os.environ["UI_SCREENSHOT_DISCOVERY_REPORT"])); print("\n".join("{}\t{}".format(output["plugin"], "wait-replay:Nick Sweeting" if urlsplit(output["previewUrl"]).path.endswith(".wacz") else "") for output in report["checks"]["snapshotOutputs"]))')"
+            'import json, os; from urllib.parse import urlsplit; report=json.load(open(os.environ["UI_SCREENSHOT_DISCOVERY_REPORT"])); print("\n".join("{}\t{}".format(output["plugin"], "wait-frame-text:Verified · signature and archived response match" if output["plugin"] == "tlsnotary" else "wait-replay:Nick Sweeting" if urlsplit(output["previewUrl"]).path.endswith(".wacz") else "") for output in report["checks"]["snapshotOutputs"]))')"
         if [[ -z "$SNAPSHOT_OUTPUT_PLUGINS" ]]; then
             echo "[!] The Sweeting.me snapshot detail page exposed no selectable outputs" >&2
             exit 1
         fi
+        if ! printf '%s\n' "$SNAPSHOT_OUTPUT_PLUGINS" | cut -f1 | grep -qx tlsnotary; then
+            echo "[!] Live capture did not expose the required TLSNotary verifier output" >&2
+            tail -200 "$CAPTURE_ROOT/sweeting-live-capture.log" >&2
+            cat "$SNAPSHOT_DISCOVERY_REPORT" >&2
+            exit 1
+        fi
         while IFS=$'\t' read -r plugin_name output_capture_mode; do
             [[ -z "$plugin_name" ]] && continue
             VIEWS+=("Snapshot View ($plugin_name)|$LIVE_SNAPSHOT_VIEW_URL#$plugin_name|/|archivebox/templates/core/snapshot.html|$output_capture_mode")

bin/take_screenshot.js

@@ -32,6 +32,7 @@ Environment:
   SCREENSHOT_SNAPSHOT_VIEW   Set to list or grid before loading the page
   SCREENSHOT_SNAPSHOT_HEADER Set to expanded or collapsed before loading a snapshot detail page
   SCREENSHOT_EXPECT_PLUGIN   Require this snapshot output plugin to be selected
+  SCREENSHOT_EXPECT_FRAME_TEXT  Require this text in the selected output frame
   SCREENSHOT_EXPECT_LIVE_PROGRESS  Require real progress bars and a loaded screencast frame
   SCREENSHOT_COLLAPSE_FILTERS Set to 1 to keep admin filters out of screenshots
   SCREENSHOT_RESET_FILTERS   Set to 1 to clear the admin filter collapsed preference
@@ -231,6 +232,12 @@ async function main() {
           }
         });
       }, { timeout: 45000, polling: 250 }).catch(() => {});
+      if (process.env.SCREENSHOT_EXPECT_FRAME_TEXT) {
+        const frameElement = await page.$('#main-frame');
+        const frame = await frameElement.contentFrame();
+        await frame.waitForFunction((text) => document.body.innerText.includes(text),
+          { timeout: 45000 }, process.env.SCREENSHOT_EXPECT_FRAME_TEXT);
+      }
     }
 
     if (process.env.SCREENSHOT_EXPECT_LIVE_PROGRESS === '1') {

docs/Configuration.md

@@ -411,6 +411,8 @@ ArchiveBox splits its surfaces across three logical hosts: `admin.*` (Django adm
 
 SingleFile output is served as ordinary HTML in every mode; it remains usable in no-JS modes because SingleFile removes the page's scripts during capture. ArchiveWeb.page/ReplayWeb.page and MHTML use their existing trusted preview templates. ArchiveBox always attempts to load these viewers, including when the incoming request is plain HTTP, because HTTPS may be terminated by an upstream proxy. Browser service-worker rules still require ReplayWeb.page to be reached through HTTPS or localhost for replay to initialize.
 
+Trusted preview wrappers allow embedding by their own origin and the configured web/admin origins, so the collection UI can display snapshot-host previews without opening them to arbitrary embedding sites. PDF previews use a document frame rather than an object/embed, retaining `object-src 'none'` on the wrapper. Allowing a trusted viewer's scripts does not exempt the archived documents inside it from their replay policy; the MHTML viewer keeps its archived child document sandboxed without scripts.
+
 > [!WARNING]
 > Switching to any mode whose name starts with `unsafe-` or `danger-` is logged at startup and surfaces a banner in the UI. **Don't use these modes on a public hostname** — archived JavaScript will run on the same origin as your admin session.
 

etc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "archivebox",
-  "version": "0.9.35rc449",
+  "version": "0.9.35rc467",
   "repository": "github:ArchiveBox/ArchiveBox",
   "license": "MIT",
   "dependencies": {

publicsite/index.html

@@ -54,11 +54,9 @@
       <a href="#use-cases"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M16 21v-2a4 4 0 0 0-4-4H6a4 4 0 0 0-4 4v2"/><circle cx="9" cy="7" r="4"/><path d="M22 21v-2a4 4 0 0 0-3-3.87"/><path d="M16 3.13a4 4 0 0 1 0 7.75"/></svg><span>Use cases</span></a>
       <a href="#configuration"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M4 21v-7"/><path d="M4 10V3"/><path d="M12 21v-9"/><path d="M12 8V3"/><path d="M20 21v-5"/><path d="M20 12V3"/><path d="M2 14h4"/><path d="M10 8h4"/><path d="M18 16h4"/></svg><span>Configuration</span></a>
       <a href="#layout-security"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 22s8-4 8-10V5l-8-3-8 3v7c0 6 8 10 8 10Z"/><path d="m9 12 2 2 4-4"/></svg><span>Layout &amp; security</span></a>
-      <a href="#background"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 8v4l3 3"/><circle cx="12" cy="12" r="9"/></svg><span>Backstory</span></a>
-      <a href="#ecosystem"><svg viewBox="0 0 24 24" aria-hidden="true"><rect x="3" y="3" width="7" height="7" rx="1.5"/><rect x="14" y="3" width="7" height="7" rx="1.5"/><rect x="14" y="14" width="7" height="7" rx="1.5"/><path d="M10 6.5h4"/><path d="M17.5 10v4"/><path d="M10 17.5h4"/></svg><span>Ecosystem</span></a>
+      <details class="project-menu"><summary><svg viewBox="0 0 24 24" aria-hidden="true"><rect x="3" y="3" width="7" height="7" rx="1.5"/><rect x="14" y="3" width="7" height="7" rx="1.5"/><rect x="14" y="14" width="7" height="7" rx="1.5"/><path d="M10 6.5h4"/><path d="M17.5 10v4"/><path d="M10 17.5h4"/></svg><span>Apps</span></summary><div class="project-menu__links"><a href="https://archivebox.io/" aria-current="page">ArchiveBox</a><a href="https://archivebox.github.io/ios-archivebox/">Apple apps</a><a href="https://archivebox.github.io/archivebox-browser-extension/">Browser extension</a><a href="https://archivebox.github.io/abx-dl/">abx-dl</a><a href="https://archivebox.github.io/abx-plugins/">abx-plugins</a></div></details>
       <a href="https://github.com/ArchiveBox/ArchiveBox/wiki"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M4 19.5A2.5 2.5 0 0 1 6.5 17H20"/><path d="M6.5 2H20v20H6.5A2.5 2.5 0 0 1 4 19.5v-15A2.5 2.5 0 0 1 6.5 2Z"/></svg><span>Docs</span></a>
       <a href="https://github.com/ArchiveBox/ArchiveBox"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M15 22v-4a4.8 4.8 0 0 0-.09-.93"/><path d="M9 22v-4a4.8 4.8 0 0 1 .09-.93"/><path d="M17.8 20.2c1.4-.5 2.2-1.7 2.2-3.7V12.7c0-.7-.2-1.4-.7-2 .1-.3.3-1.7-.1-3.2 0 0-1.1-.4-3.4.9a11.5 11.5 0 0 0-6 0C7.5 7.1 6.4 7.5 6.4 7.5c-.4 1.5-.2 2.9-.1 3.2-.5.6-.7 1.3-.7 2v3.8c0 2 1 3.2 2.4 3.7"/><path d="M9 18c-3 1.3-5 0-6-2"/></svg><span>GitHub</span></a>
-      <a href="https://pypi.org/project/archivebox/"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M16.5 9.4 7.5 4.2"/><path d="M21 16V8a2 2 0 0 0-1-1.73l-7-4a2 2 0 0 0-2 0l-7 4A2 2 0 0 0 3 8v8a2 2 0 0 0 1 1.73l7 4a2 2 0 0 0 2 0l7-4A2 2 0 0 0 21 16Z"/><path d="M3.3 7 12 12l8.7-5"/><path d="M12 22V12"/></svg><span>PyPI</span></a>
     </nav>
   </header>
 

publicsite/styles.css

@@ -125,7 +125,8 @@ code {
   gap: 8px 18px;
 }
 
-.site-header nav a {
+.site-header nav a,
+.site-header nav .project-menu > summary {
   display: inline-flex;
   align-items: center;
   gap: 8px;
@@ -140,7 +141,8 @@ code {
   transition: border-color 160ms ease, color 160ms ease;
 }
 
-.site-header nav a:hover {
+.site-header nav a:hover,
+.site-header nav .project-menu > summary:hover {
   color: var(--text);
   text-decoration: none;
   border-color: hsl(var(--primary) / 0.42);
@@ -806,7 +808,7 @@ h3 {
   }
 
   .site-header {
-    position: static;
+    position: relative;
     width: 100%;
     margin: 0;
     padding: 14px 16px;
@@ -816,7 +818,8 @@ h3 {
     width: 100%;
   }
 
-  .site-header nav a {
+  .site-header nav a,
+  .site-header nav .project-menu > summary {
     font-size: 0.74rem;
     letter-spacing: 0.1em;
   }
@@ -881,3 +884,16 @@ h3 {
     font-size: 0.82rem;
   }
 }
+
+/* Shared project navigation stays usable without JavaScript. */
+.project-menu { position: relative; flex: 0 0 auto; }
+.project-menu > summary { cursor: pointer; list-style: none; }
+.project-menu > summary::-webkit-details-marker { display: none; }
+.project-menu > summary::after { content: "▾"; }
+.project-menu > summary:focus-visible, .project-menu__links a:focus-visible { outline: 3px solid #9b2854; outline-offset: 3px; }
+.project-menu__links { position: absolute; right: 0; top: 100%; z-index: 30; width: 220px; max-width: calc(100vw - 32px); padding: 8px; border: 1px solid #e9e2e3; border-radius: 12px; background: #fff; box-shadow: 0 12px 32px #28252a20; }
+header nav .project-menu__links a { display: flex; align-items: center; min-height: 44px; padding: 8px 12px; color: #28252a; font-size: 13px; letter-spacing: normal; text-transform: none; text-decoration: none; white-space: nowrap; }
+header nav .project-menu__links a:hover, header nav .project-menu__links a[aria-current="page"] { background: #f8edf2; color: #9b2854; border-radius: 6px; }
+@media (max-width: 700px) { header.site-header { flex-wrap: wrap; } header.site-header nav { flex-wrap: wrap; gap: 8px 16px; } }
+
+@media (max-width: 1100px) { .project-menu__links { position: fixed; left: 16px; right: 16px; top: auto; width: auto; } }

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "archivebox"
-version = "0.9.35rc449"
+version = "0.9.35rc467"
 requires-python = ">=3.13"
 description = "Self-hosted internet archiving solution."
 authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@@ -79,8 +79,8 @@ dependencies = [
     ### Binary/Package Management
     "abxbus==2.5.56",                 # direct imports only; version constrained by abx-dl -> abx-plugins
     "abxpkg==1.12.117",                # direct imports only; version constrained by abx-dl -> abx-plugins
-    "abx-plugins[opencode]==1.12.236", # optional AI runtime clients are owned by the plugin
-    "abx-dl==1.12.274",       # shared ArchiveBox downloader package
+    "abx-plugins[opencode]==1.12.249", # optional AI runtime clients are owned by the plugin
+    "abx-dl==1.12.285",       # shared ArchiveBox downloader package
     ### UUID7 backport for Python <3.14
     "uuid7>=0.1.0; python_version < '3.14'",  # provides the uuid_extensions module on Python 3.13
     "browser-cookie3>=0.20.1",
@@ -342,7 +342,7 @@ Donate = "https://github.com/ArchiveBox/ArchiveBox/wiki/Donations"
 
 
 [tool.bumpver]
-current_version = "v0.9.35rc449"
+current_version = "v0.9.35rc467"
 version_pattern = "vMAJOR.MINOR.PATCH[PYTAGNUM]"
 commit_message = "bump version {old_version} -> {new_version}"
 tag_message = "{new_version}"

uv.lock

@@ -13,18 +13,18 @@ supported-markers = [
 ]
 
 [options]
-exclude-newer = "2026-09-11T22:03:43.68620639Z"
+exclude-newer = "2026-09-13T04:51:40.198736069Z"
 exclude-newer-span = "P5D"
 
 [options.exclude-newer-package]
-abxbus = { timestamp = "2026-09-16T22:03:42.686219334Z", span = "PT1S" }
+abxbus = { timestamp = "2026-09-18T04:51:39.198748378Z", span = "PT1S" }
 abx-plugins = "2100-01-01T00:00:00Z"
 abx-dl = "2100-01-01T00:00:00Z"
 abxpkg = "2100-01-01T00:00:00Z"
 
 [[package]]
 name = "abx-dl"
-version = "1.12.274"
+version = "1.12.285"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "abx-plugins", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -38,14 +38,14 @@ dependencies = [
     { name = "rich", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/c6/69/223fb0cfd106eda015a1a73daa58fc9aa87bb7b974094c5eadc65c15b693/abx_dl-1.12.274.tar.gz", hash = "sha256:58b7f85aec60db8233d4ea33a7683edf12be63fe32b8eccbd45d57edcab1a35b", size = 87257 }
+sdist = { url = "https://files.pythonhosted.org/packages/78/7f/c02c8e95a17427e6258d789ee943d029195188fede7fc420df263a1f6b66/abx_dl-1.12.285.tar.gz", hash = "sha256:84f9dd4ae3b58fb2e7923c67905583f39d163e0e95991ea9d2da9f71cd31b783", size = 88234 }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/ab/36/b12628d4ce600cbcd1c3ba87c6f76721a1c21e074e1f30782baf2eb2eb06/abx_dl-1.12.274-py3-none-any.whl", hash = "sha256:5481d25ab82da2449a3a0a47079fd37a8768f1175ed2068bef45392beb4ee684", size = 92246 },
+    { url = "https://files.pythonhosted.org/packages/a6/e0/70774861b35ddd4b52c99253d83c8d2a7be039e8856ee1cfcaa4b662b05a/abx_dl-1.12.285-py3-none-any.whl", hash = "sha256:e729ebe51dc4f23c546d0ab7743100de821c03d56ecb4fba7ff3d6c50e7bf4f6", size = 93263 },
 ]
 
 [[package]]
 name = "abx-plugins"
-version = "1.12.236"
+version = "1.12.249"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "abxbus", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -54,9 +54,9 @@ dependencies = [
     { name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "uv", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/11/90/617fe56f24af0ed1a6bcc255233703c9378f49d381d2192eb76a0a1c9ad7/abx_plugins-1.12.236.tar.gz", hash = "sha256:1c0a23a4878787a64109c69160c71df74885c90e5fca9def7c13b8ea731b1add", size = 269546, upload-time = "2026-09-16T21:51:20.39Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/56/ec/92145ef9893b5e2e83905997d1e824fc521f304333d895f0b9d81f349d5b/abx_plugins-1.12.249.tar.gz", hash = "sha256:25438e5f4cc3942b9cc0dc6d857c0243309d444a7e97aaf62818e040cf124c02", size = 309461, upload-time = "2026-09-18T04:30:46.963Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/6c/ed/ac9682624eea2604cb3d35ac6f952c92c95e52c064401044d43c3df5864a/abx_plugins-1.12.236-py3-none-any.whl", hash = "sha256:0bf8738c64d27dd88d2ac4e91c4c85504352cc1a47bdc9aa863a12a9b7f8a5da", size = 424591, upload-time = "2026-09-16T21:51:21.816Z" },
+    { url = "https://files.pythonhosted.org/packages/24/eb/8ea746a247bb195bba545e678133774830a4e14ec36f97fb353f0345d44e/abx_plugins-1.12.249-py3-none-any.whl", hash = "sha256:0445c8cba4263845a04d7b30c8c2e58da33feb52c4dba477c19e5f219c8e3f39", size = 473839, upload-time = "2026-09-18T04:30:48.418Z" },
 ]
 
 [package.optional-dependencies]
@@ -127,7 +127,7 @@ wheels = [
 
 [[package]]
 name = "archivebox"
-version = "0.9.35rc449"
+version = "0.9.35rc467"
 source = { editable = "." }
 dependencies = [
     { name = "abx-dl", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -226,8 +226,8 @@ dev = [
 
 [package.metadata]
 requires-dist = [
-    { name = "abx-dl", specifier = "==1.12.274" },
-    { name = "abx-plugins", extras = ["opencode"], specifier = "==1.12.236" },
+    { name = "abx-dl", specifier = "==1.12.285" },
+    { name = "abx-plugins", extras = ["opencode"], specifier = "==1.12.249" },
     { name = "abxbus", specifier = "==2.5.56" },
     { name = "abxpkg", specifier = "==1.12.117" },
     { name = "archivebox", extras = ["sonic", "ldap", "debug"], marker = "extra == 'all'" },