Bump release version to 0.9.71

This reverts commit 56b57df407.

.github/STAGING.md

@@ -33,3 +33,10 @@ screenshots are retained as Actions artifacts. Other failures remain
 failures; do not publish a success deployment record to bypass them. No code
 mounts are allowed. The Cabbage job must pass before its record authorizes a
 stable upload. Queued capture progress is polled; failed tests are not retried.
+
+Browser acceptance follows the default WACZ viewer into the captured document,
+matching its replay base and original URL and checking that its body is visible
+and contains substantial text. An attached viewer iframe can still be
+`about:blank`, so its presence alone cannot prove replay works. Extracted display
+titles and the captured document's `<title>` may legitimately differ; neither
+title equality nor output thumbnails substitute for checking the actual page.

.github/scripts/discover_test_matrix.py

@@ -52,13 +52,15 @@ def main() -> None:
     # Ordinary Linux tests can use either pool. Only genuine hosted-only
     # requirements need a file header; filenames never enter runner policy.
     # Keep a bounded NAS share so hosted capacity continues working in parallel.
-    capacity = int(os.environ.get("UGNAS_CI_MAX_JOBS", "3"))
+    # This is queued work, not a worker count: several waves keep the capped
+    # runners occupied while hosted jobs run in parallel.
+    nas_job_limit = int(os.environ.get("UGNAS_CI_MAX_JOBS", "3"))
     # Keep long jobs early in the hosted queue. Giving the longest jobs to the
     # CPU-capped NAS instead created a tail after hosted work had finished; use
     # its extra slots for a bounded share of shorter work without filename rules.
     matrix.sort(key=lambda item: sum(durations.get(path, 60) for path in item["paths"]), reverse=True)
     eligible = [item for item in reversed(matrix) if not any(root / path in hosted for path in item["paths"])]
-    for item in eligible[:capacity]:
+    for item in eligible[:nas_job_limit]:
         item["ugnas"] = True
 
     discovered_paths = [path for entry in matrix for path in entry["paths"]]
@@ -67,7 +69,10 @@ def main() -> None:
     if sorted(discovered_paths) != [path.relative_to(root).as_posix() for path in archivebox_tests]:
         raise SystemExit("Discovered test shard coverage does not match test files")
 
-    print(f"Discovered {len(discovered_paths)} test files exactly once across {len(matrix)} jobs")
+    print(
+        f"Discovered {len(discovered_paths)} test files exactly once across {len(matrix)} jobs; "
+        f"{sum(item['ugnas'] for item in matrix)} selected for ugNAS, subject to the availability check",
+    )
     print(json.dumps(matrix, separators=(",", ":")))
 
 

.github/test-durations.json

@@ -1,9 +1,10 @@
 {
   "source_runs": [
     "https://github.com/ArchiveBox/ArchiveBox/actions/runs/36302511923",
-    "https://github.com/ArchiveBox/ArchiveBox/actions/runs/36353196552"
+    "https://github.com/ArchiveBox/ArchiveBox/actions/runs/36353196552",
+    "https://github.com/ArchiveBox/ArchiveBox/actions/runs/36454992009"
   ],
-  "measurement": "Whole job seconds, including setup and excluding queue time. Unbatched files use the latest complete successful run; batched files retain their individual baseline measurement. Unknown files stay independent.",
+  "measurement": "Whole job seconds, including setup and excluding queue time. Hosted unbatched files use the latest complete successful run; other files retain their individual baseline measurement. Unknown files stay independent.",
   "seconds": {
     "archivebox/tests/test_api_v1_auth_browser_session.py": 29,
     "archivebox/tests/test_api_v1_auth_check_api_token.py": 27,
@@ -12,12 +13,12 @@
     "archivebox/tests/test_api_v1_cli_remove.py": 50,
     "archivebox/tests/test_api_v1_cli_schedule.py": 41,
     "archivebox/tests/test_api_v1_cli_search.py": 26,
-    "archivebox/tests/test_api_v1_cli_update.py": 100,
+    "archivebox/tests/test_api_v1_cli_update.py": 74,
     "archivebox/tests/test_api_v1_cli_workflow_add_search_update_remove.py": 64,
     "archivebox/tests/test_api_v1_core_any_id.py": 21,
     "archivebox/tests/test_api_v1_core_archiveresult_archiveresult_id.py": 55,
     "archivebox/tests/test_api_v1_core_archiveresults.py": 32,
-    "archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py": 203,
+    "archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py": 199,
     "archivebox/tests/test_api_v1_core_snapshots_rss.py": 27,
     "archivebox/tests/test_api_v1_core_snapshots.py": 28,
     "archivebox/tests/test_api_v1_core_tag_tag_id_rename.py": 23,
@@ -45,23 +46,23 @@
     "archivebox/tests/test_api_v1_personas_sync.py": 29,
     "archivebox/tests/test_api_v1_workflow_core_token_auth_side_effects.py": 36,
     "archivebox/tests/test_api_v1_workflow_frozen_crawl_config_sources.py": 33,
-    "archivebox/tests/test_archive_result_service.py": 122,
+    "archivebox/tests/test_archive_result_service.py": 137,
     "archivebox/tests/test_auth_ldap.py": 44,
     "archivebox/tests/test_binary_service.py": 83,
-    "archivebox/tests/test_cli_add_1.py": 179,
-    "archivebox/tests/test_cli_add_2.py": 167,
-    "archivebox/tests/test_cli_add_3.py": 194,
-    "archivebox/tests/test_cli_add_4.py": 168,
-    "archivebox/tests/test_cli_archiveresult.py": 145,
+    "archivebox/tests/test_cli_add_1.py": 168,
+    "archivebox/tests/test_cli_add_2.py": 203,
+    "archivebox/tests/test_cli_add_3.py": 215,
+    "archivebox/tests/test_cli_add_4.py": 201,
+    "archivebox/tests/test_cli_archiveresult.py": 199,
     "archivebox/tests/test_cli_binary.py": 20,
-    "archivebox/tests/test_cli_config.py": 141,
-    "archivebox/tests/test_cli_crawl.py": 127,
-    "archivebox/tests/test_cli_extract_input.py": 205,
-    "archivebox/tests/test_cli_extract.py": 113,
+    "archivebox/tests/test_cli_config.py": 148,
+    "archivebox/tests/test_cli_crawl.py": 177,
+    "archivebox/tests/test_cli_extract_input.py": 155,
+    "archivebox/tests/test_cli_extract.py": 106,
     "archivebox/tests/test_cli_help.py": 30,
-    "archivebox/tests/test_cli_init.py": 133,
+    "archivebox/tests/test_cli_init.py": 135,
     "archivebox/tests/test_cli_install.py": 58,
-    "archivebox/tests/test_cli_list.py": 177,
+    "archivebox/tests/test_cli_list.py": 260,
     "archivebox/tests/test_cli_machine.py": 50,
     "archivebox/tests/test_cli_manage.py": 48,
     "archivebox/tests/test_cli_mcp.py": 51,
@@ -70,22 +71,22 @@
     "archivebox/tests/test_cli_piping.py": 97,
     "archivebox/tests/test_cli_pluginmap.py": 21,
     "archivebox/tests/test_cli_process.py": 20,
-    "archivebox/tests/test_cli_remove.py": 150,
-    "archivebox/tests/test_cli_run_1.py": 165,
-    "archivebox/tests/test_cli_run_2.py": 241,
-    "archivebox/tests/test_cli_run_3.py": 162,
-    "archivebox/tests/test_cli_run_4.py": 200,
+    "archivebox/tests/test_cli_remove.py": 162,
+    "archivebox/tests/test_cli_run_1.py": 195,
+    "archivebox/tests/test_cli_run_2.py": 255,
+    "archivebox/tests/test_cli_run_3.py": 167,
+    "archivebox/tests/test_cli_run_4.py": 178,
     "archivebox/tests/test_cli_schedule.py": 87,
     "archivebox/tests/test_cli_search.py": 35,
-    "archivebox/tests/test_cli_server.py": 251,
+    "archivebox/tests/test_cli_server.py": 259,
     "archivebox/tests/test_cli_shell.py": 32,
-    "archivebox/tests/test_cli_snapshot.py": 138,
-    "archivebox/tests/test_cli_status.py": 105,
+    "archivebox/tests/test_cli_snapshot.py": 181,
+    "archivebox/tests/test_cli_status.py": 102,
     "archivebox/tests/test_cli_tag.py": 21,
-    "archivebox/tests/test_cli_update_reindex_snapshots.py": 144,
-    "archivebox/tests/test_cli_update.py": 148,
-    "archivebox/tests/test_cli_version.py": 181,
-    "archivebox/tests/test_config_DELETE_AFTER.py": 192,
+    "archivebox/tests/test_cli_update_reindex_snapshots.py": 163,
+    "archivebox/tests/test_cli_update.py": 142,
+    "archivebox/tests/test_cli_version.py": 143,
+    "archivebox/tests/test_config_DELETE_AFTER.py": 169,
     "archivebox/tests/test_config_MAX_limits.py": 34,
     "archivebox/tests/test_config_ONLY_NEW.py": 24,
     "archivebox/tests/test_config_SAVE_TITLE.py": 99,
@@ -99,19 +100,19 @@
     "archivebox/tests/test_hook_result_lifecycle.py": 78,
     "archivebox/tests/test_hooks.py": 90,
     "archivebox/tests/test_machine_models.py": 52,
-    "archivebox/tests/test_machine_service.py": 104,
+    "archivebox/tests/test_machine_service.py": 89,
     "archivebox/tests/test_migrations_04_to_09.py": 36,
-    "archivebox/tests/test_migrations_07_to_09.py": 140,
-    "archivebox/tests/test_migrations_08_to_09.py": 257,
+    "archivebox/tests/test_migrations_07_to_09.py": 183,
+    "archivebox/tests/test_migrations_08_to_09.py": 244,
     "archivebox/tests/test_migrations_fresh.py": 99,
     "archivebox/tests/test_misc_checks.py": 22,
     "archivebox/tests/test_misc_monkey_patches.py": 20,
-    "archivebox/tests/test_opencode_agent_1.py": 135,
-    "archivebox/tests/test_opencode_agent_2.py": 138,
-    "archivebox/tests/test_opencode_agent_3.py": 140,
-    "archivebox/tests/test_opencode_browser_1.py": 290,
-    "archivebox/tests/test_opencode_browser_2.py": 157,
-    "archivebox/tests/test_opencode_browser_3.py": 129,
+    "archivebox/tests/test_opencode_agent_1.py": 158,
+    "archivebox/tests/test_opencode_agent_2.py": 145,
+    "archivebox/tests/test_opencode_agent_3.py": 154,
+    "archivebox/tests/test_opencode_browser_1.py": 262,
+    "archivebox/tests/test_opencode_browser_2.py": 146,
+    "archivebox/tests/test_opencode_browser_3.py": 148,
     "archivebox/tests/test_orm_helpers.py": 22,
     "archivebox/tests/test_permissions.py": 21,
     "archivebox/tests/test_persona_runtime.py": 56,
@@ -119,18 +120,18 @@
     "archivebox/tests/test_process_runtime_paths.py": 28,
     "archivebox/tests/test_process_service.py": 49,
     "archivebox/tests/test_proof_previews.py": 54,
-    "archivebox/tests/test_recursive_crawl_1.py": 396,
-    "archivebox/tests/test_recursive_crawl_2.py": 193,
-    "archivebox/tests/test_recursive_crawl_3.py": 144,
+    "archivebox/tests/test_recursive_crawl_1.py": 339,
+    "archivebox/tests/test_recursive_crawl_2.py": 171,
+    "archivebox/tests/test_recursive_crawl_3.py": 176,
     "archivebox/tests/test_release_notes.py": 14,
     "archivebox/tests/test_release_source_fingerprint.py": 22,
     "archivebox/tests/test_release_workflow.py": 21,
     "archivebox/tests/test_resource_admission.py": 21,
-    "archivebox/tests/test_runner_watch.py": 182,
-    "archivebox/tests/test_search.py": 203,
+    "archivebox/tests/test_runner_watch.py": 177,
+    "archivebox/tests/test_search.py": 153,
     "archivebox/tests/test_serve_static.py": 31,
     "archivebox/tests/test_server_automatic_url.py": 36,
-    "archivebox/tests/test_server_security_browser.py": 241,
+    "archivebox/tests/test_server_security_browser.py": 278,
     "archivebox/tests/test_settings_signal_webhooks.py": 21,
     "archivebox/tests/test_setup_script.py": 21,
     "archivebox/tests/test_shutdown_util.py": 22,
@@ -140,12 +141,12 @@
     "archivebox/tests/test_snapshot_service.py": 41,
     "archivebox/tests/test_snapshot_url_length.py": 26,
     "archivebox/tests/test_tag_service.py": 24,
-    "archivebox/tests/test_takeover_util_1.py": 122,
-    "archivebox/tests/test_takeover_util_2.py": 171,
-    "archivebox/tests/test_takeover_util_3.py": 165,
-    "archivebox/tests/test_takeover_util_4.py": 160,
+    "archivebox/tests/test_takeover_util_1.py": 143,
+    "archivebox/tests/test_takeover_util_2.py": 201,
+    "archivebox/tests/test_takeover_util_3.py": 138,
+    "archivebox/tests/test_takeover_util_4.py": 156,
     "archivebox/tests/test_test_harness.py": 22,
-    "archivebox/tests/test_ui_add_view_runtime.py": 239,
+    "archivebox/tests/test_ui_add_view_runtime.py": 221,
     "archivebox/tests/test_ui_add_view.py": 46,
     "archivebox/tests/test_ui_admin_apitoken.py": 23,
     "archivebox/tests/test_ui_admin_archiveresult.py": 30,
@@ -154,14 +155,14 @@
     "archivebox/tests/test_ui_admin_links.py": 35,
     "archivebox/tests/test_ui_admin_machine.py": 29,
     "archivebox/tests/test_ui_admin_persona.py": 28,
-    "archivebox/tests/test_ui_admin_snapshot.py": 288,
+    "archivebox/tests/test_ui_admin_snapshot.py": 273,
     "archivebox/tests/test_ui_admin_tag.py": 26,
     "archivebox/tests/test_ui_admin_user.py": 28,
     "archivebox/tests/test_ui_config_views.py": 43,
-    "archivebox/tests/test_ui_live_progress.py": 303,
-    "archivebox/tests/test_ui_public_snapshot.py": 138,
+    "archivebox/tests/test_ui_live_progress.py": 297,
+    "archivebox/tests/test_ui_public_snapshot.py": 179,
     "archivebox/tests/test_ui_savepagenow.py": 48,
-    "archivebox/tests/test_urls.py": 179,
+    "archivebox/tests/test_urls.py": 220,
     "archivebox/tests/test_util.py": 20,
     "archivebox/tests/test_version_metadata.py": 24
   }

Dockerfile

@@ -125,6 +125,7 @@ FROM archivebox-runtime-base AS archivebox-builder
 WORKDIR "$CODE_DIR"
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
     --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
+    --mount=type=bind,source=uv.lock,target=/app/uv.lock \
     <<'EOF'
 echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..."
 export UV_NO_CACHE=true
@@ -137,48 +138,14 @@ apt-get install -qq -y --no-install-recommends x11-utils
 # Do not rewrite inherited native libraries merely to strip them again.
 /usr/bin/find /venv/lib/python3.*/site-packages -type f -name '*.so' -print > /tmp/archivebox-inherited-libraries
 
-mkdir -p /tmp/archivebox-uv-project
-/venv/bin/python - <<'PY'
-from pathlib import Path
-import json
-import re
-import urllib.request
-
-source = Path("/app/pyproject.toml")
-target = Path("/tmp/archivebox-uv-project/pyproject.toml")
-text = source.read_text()
-text = text.replace(
-    'environments = ["sys_platform == \'darwin\'", "sys_platform == \'linux\'"]',
-    'environments = ["sys_platform == \'linux\'"]',
-)
-
-# Docker builds need the just-published internal abx wheels immediately, but
-# PyPI simple can lag the version JSON endpoints by tens of minutes. Generate a
-# Docker-only dependency view from the version JSON so the published package
-# metadata stays normal while image builds remain resumable after a release.
-for package in ("abxbus", "abxpkg", "abx-plugins", "abx-dl"):
-    match = re.search(
-        rf'"{re.escape(package)}(?P<operator>==|>=)(?P<version>[^"]+)"',
-        text,
-    )
-    if not match:
-        continue
-    version = match.group("version")
-    with urllib.request.urlopen(f"https://pypi.org/pypi/{package}/{version}/json", timeout=20) as response:
-        data = json.load(response)
-    wheel_url = next(url["url"] for url in data["urls"] if url["filename"].endswith(".whl"))
-    text = re.sub(
-        rf'"{re.escape(package)}(?:==|>=)[^"]+"',
-        f'"{package} @ {wheel_url}"',
-        text,
-        count=1,
-    )
-
-target.write_text(text)
-PY
-
+# The old Docker-only dependency rewrite predated this checkout's lockfile:
+# it fetched version JSON to avoid stale PyPI Simple listings. That extra API
+# can itself lag or return 503s. The verified lock already records exact public
+# artifact URLs and hashes, including extras and platform markers. Consume it
+# directly, retaining the freshness check instead of resolving a second graph.
 /usr/bin/uv sync \
-    --project /tmp/archivebox-uv-project \
+    --project /app \
+    --locked \
     --no-cache \
     --no-dev \
     --inexact \
@@ -209,7 +176,7 @@ rm -rf "$builder_abxpkg_lib_dir"
 rm -rf /venv/lib/python3.*/site-packages/pip* \
     /venv/lib/python3.*/site-packages/wheel* \
     /venv/bin/pip /venv/bin/pip3 /venv/bin/pip3.* /venv/bin/wheel
-rm -rf /var/lib/apt/lists/* /tmp/archivebox-uv-project
+rm -rf /var/lib/apt/lists/*
 EOF
 
 COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
@@ -291,7 +258,13 @@ RUN --mount=type=cache,target=/var/tmp/abxpkg-cache,sharing=locked,mode=1777,id=
         setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abx-dl install $ABX_DOCKER_PLUGINS \
     # pnpm includes musl variants that Debian's glibc runtime cannot use.
     && find "$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules" -type l -name 'opencode-linux-*-musl' -delete \
-    && find "$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.pnpm" -maxdepth 1 -type d -name 'opencode-linux-*-musl@*' -exec rm -rf {} +
+    && find "$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.pnpm" -maxdepth 1 -type d -name 'opencode-linux-*-musl@*' -exec rm -rf {} + \
+    # pnpm's explicit store lives under ABXPKG_LIB_DIR, regardless of XDG_CACHE_HOME.
+    # Retaining its download copies made image size depend on whether the build
+    # preserved hardlinks to installed files. Remove the store in this layer;
+    # installed files retain their contents, and the offline check below proves
+    # the complete catalog still works without the disposable package payloads.
+    && rm -rf "$ABXPKG_LIB_DIR/cache/pnpm"
 RUN --network=none export ABX_DOCKER_PLUGINS="$(/venv/bin/python3 -c 'from abx_dl.catalog import PluginCatalog; print(" ".join(PluginCatalog.discover(runtime="archivebox")))')" \
     && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups \
     bash -c '"$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" --version && abx-dl install $ABX_DOCKER_PLUGINS'

archivebox/core/models.py

@@ -895,7 +895,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             return False
         if self.status == self.StatusChoices.QUEUED:
             return bool(self.url) and self.start_processing()
-        # abx-dl emits SnapshotCompletedEvent after the complete hook sequence;
+        # SnapshotCompletedEvent seals completed or explicitly aborted attempts;
+        # runner takeover preserves the active capture for its replacement.
         # ArchiveResult projection state never drives Snapshot completion.
         return False
 

archivebox/core/views.py

@@ -1212,6 +1212,18 @@ class OriginalDomainReplayView(View):
 
 
 class PublicIndexView(ListView):
+    """Keep browsing responsive while the same small server is archiving.
+
+    Performance takes priority over perfect initial totals: on a 1 vCPU / 1 GB
+    host with hundreds of thousands of snapshots, even an indexed count can
+    consume the entire roughly 0.5-0.7 second page-load budget. Benchmark with
+    the runner active, since it shares the CPU, memory, and SQLite database.
+
+    The unfiltered list therefore renders rows first and accepts missing/stale
+    display totals. A later ordinary reload can show the background-cached
+    count; do not restore a synchronous COUNT or add polling just to fill it in.
+    """
+
     template_name = "public_index.html"
     model = Snapshot
     ordering: ClassVar[list[str]] = ["-bookmarked_at", "-created_at"]
@@ -1251,6 +1263,11 @@ class PublicIndexView(ListView):
         public_snapshots: list[Snapshot] = []
         scanned = 0
         chunk_size = max(self.public_page_scan_chunk_size, page_size + 1)
+        # Read in snapshot_public_order_idx order and filter visibility in small
+        # batches. Combining the permissions predicate with ORDER BY can make
+        # SQLite select the permissions index and sort the whole public archive
+        # before LIMIT. This favors the common early pages without a new index;
+        # deep pages or mostly private collections can still require more reads.
         ordered_snapshots = (
             Snapshot.objects.select_related("crawl__created_by").order_by(*self.ordering).only(*self._base_public_snapshot_fields())
         )
@@ -1283,6 +1300,10 @@ class PublicIndexView(ListView):
         snapshots = self._ordered_public_page_from_order_index(page_number=page_number, page_size=page_size)
         if not snapshots and page_number > 1:
             raise Http404(f"Invalid page ({page_number}): No snapshots")
+        # Exhausting the rows gives an exact total for this read at no extra
+        # cost. Otherwise the lookahead is only a lower bound: an absent count
+        # is preferable to an on-request scan or a cached total smaller than
+        # the rows we just observed. Cached overestimates are acceptable here.
         observed_count = (page_number - 1) * page_size + len(snapshots)
         self.public_snapshot_count = observed_count
         if len(snapshots) > page_size:
@@ -1290,6 +1311,10 @@ class PublicIndexView(ListView):
             cached_count = summary["snapshots"] if summary is not None else None
             self.public_snapshot_count = cached_count if cached_count is not None and cached_count >= observed_count else None
 
+        # Give Django an in-memory lower bound, never the collection QuerySet
+        # whose Paginator.count would issue COUNT(*). Navigation uses this read's
+        # lookahead even when a stale total would hide a newly added page.
+        # The template must use the separate display totals, not num_pages here.
         paginator = self.get_paginator(range(observed_count), page_size)
         page = paginator.page(page_number)
         page.object_list = snapshots[:page_size]

archivebox/progressmonitor/collection.py

@@ -1,4 +1,15 @@
-"""Non-blocking, database-only collection totals for the progress monitor."""
+"""Cached display totals for progress polling and the public snapshot list.
+
+Missing/stale statistics are intentional: returning useful rows and progress
+quickly matters more than exact initial counts. Each viewer may poll once per
+second; repeating aggregates per request would consume the small server's CPU
+and hold SQLite read locks needed by the archiving runner. Callers must accept
+None on a cold cache and must never wait for the refresh or count synchronously.
+
+Background reads still take normal SQLite read locks. Existing indexes, short
+batches where needed, and throttled refreshes limit their cost; moving work to
+a thread does not make a scan free or require switching the database to WAL.
+"""
 
 import logging
 from threading import Lock, Thread
@@ -9,6 +20,10 @@ from django.db import close_old_connections, connections
 from django.db.models import Count, Sum
 
 
+# This is only a process-local gate for background work, not a DB/file lock.
+# Every request tries it without waiting; a busy refresh means returning the
+# cached value (or None). Serializing refreshes also avoids competing scans for
+# different crawls/users when many viewer tabs open at once.
 _refresh_lock = Lock()
 
 
@@ -66,10 +81,16 @@ def crawl_summary(crawl_id, *, snapshot_id=None):
 
 
 def collection_summary(user, *, public=False):
+    """Return cached totals immediately and, when due, start one refresh.
+
+    Public pages need no extra AJAX/polling for these totals: an ordinary later
+    page load reads the populated cache. Keep this best-effort display cache
+    separate from pagination/authorization decisions, which need actual rows.
+    """
     user_id = None if user.is_superuser else user.pk
     key = "progress-collection:public" if public else f"progress-collection:{'all' if user_id is None else user_id}"
     summary = cache.get(key)
-    # Even an indexed SUM scans the index. Never make an HTTP request wait for
+    # Even indexed COUNT/SUM aggregates scan the index. Never make HTTP wait for
     # that scan on a large collection. Serve stale totals while one worker refreshes;
     # a single non-blocking lock prevents a queue of scans from concurrent clients.
     if (summary is None or time() - summary["sampled_at"] >= 60) and _refresh_lock.acquire(blocking=False):
@@ -92,6 +113,8 @@ def collection_summary(user, *, public=False):
                     # on an all-public archive. Equal range bounds keep the count
                     # on the existing permissions index. It is still O(n), so
                     # public pages share this background cache, never await it.
+                    # Count the indexed, non-null permissions column so this
+                    # needs no Snapshot table reads or new covering-index migration.
                     totals = snapshots.filter(permissions__gte=PERMISSIONS_PUBLIC, permissions__lte=PERMISSIONS_PUBLIC).aggregate(
                         snapshots=Count("permissions"),
                     )
@@ -103,6 +126,9 @@ def collection_summary(user, *, public=False):
                     totals = snapshots.aggregate(snapshots=Count("*"), bytes=Sum("output_size"))
                 totals["bytes"] = totals.get("bytes") or 0
                 totals["sampled_at"] = time()
+                # Retain old totals longer than the 60s refresh interval so a
+                # slow/failed refresh does not force every tab back to a cold
+                # count. Staleness is acceptable; blocking page loads is not.
                 cache.set(key, totals, timeout=3600)
             except Exception:
                 logging.getLogger(__name__).exception("Could not refresh collection totals")

archivebox/services/runner.py

@@ -224,6 +224,7 @@ class CrawlRunner:
         self.snapshot_service = SnapshotService(
             self.bus,
             crawl_id=str(crawl.id),
+            interrupted=self.execution_interrupted,
         )
         HookArchiveResultService(self.bus, emit_jsonl=False)
         ArchiveResultService(self.bus)
@@ -258,6 +259,10 @@ class CrawlRunner:
         self._signal_abort_requested = False
         self._last_lease_heartbeat_at = 0.0
 
+    def execution_interrupted(self) -> bool:
+        """Owner loss preserves work; explicit user abort keeps its sealing policy."""
+        return self._signal_abort_requested and not self._user_aborted
+
     async def on_CrawlAbortEvent(self, event: CrawlAbortEvent) -> None:
         self._signal_abort_requested = True
         # An abort can arrive while the prompt transport is being established,
@@ -1131,18 +1136,17 @@ class CrawlRunner:
             finally:
                 cancel_watcher.cancel()
                 await asyncio.gather(cancel_watcher, return_exceptions=True)
-            completed_event = event.emit(
-                CrawlCompletedEvent(
-                    url=snapshot["url"],
-                    snapshot_id=snapshot["id"],
-                    output_dir=str(output_dir),
-                ),
-            )
-            # Same signal lifecycle as CrawlCleanupEvent above: completion is a
-            # normal bus event unless the interpreter is already unwinding from
-            # SIGINT/SIGTERM/SIGHUP, where synchronous bus delivery is no
-            # longer a dependable shutdown primitive.
-            if not self._signal_abort_requested:
+            # emit() schedules handlers immediately; guarding only the await
+            # still lets completion projectors run during takeover. An aborted
+            # execution owner must leave durable work for its replacement.
+            if not self.execution_interrupted():
+                completed_event = event.emit(
+                    CrawlCompletedEvent(
+                        url=snapshot["url"],
+                        snapshot_id=snapshot["id"],
+                        output_dir=str(output_dir),
+                    ),
+                )
                 await _run_event_now(completed_event, CrawlCompletedEvent.model_fields["event_timeout"].default)
 
         on_archivebox_CrawlStartEvent.__name__ = "on_archivebox_CrawlStartEvent__run_snapshots"
@@ -1279,6 +1283,8 @@ class CrawlRunner:
                     raise RuntimeError(f"Snapshot {snapshot_id} did not complete")
                 await completed_snapshot.wait(timeout=snapshot_phase_timeout)
                 await completed_snapshot.event_results_list()
+                if self.execution_interrupted():
+                    return
                 if snapshot["status"] == "sealed":
                     await sync_to_async(run_snapshot_maintenance, thread_sensitive=True)(snapshot_id, output_dir=output_dir)
                     return

archivebox/services/snapshot_service.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import asyncio
+from collections.abc import Callable
 from pathlib import Path
 
 from asgiref.sync import sync_to_async
@@ -73,10 +74,11 @@ def finalize_completed_snapshot(
             modified_at=timezone.now(),
         )
 
-    # SnapshotCompletedEvent is abx-dl's authoritative signal that the complete
-    # snapshot hook sequence (including cleanup) finished. ArchiveResult rows
-    # are projections of that work, never prerequisites used to decide whether
-    # the Snapshot may seal.
+    # SnapshotCompletedEvent finishes an attempt, including abort cleanup. An
+    # explicit user abort deliberately seals that point-in-time capture; the
+    # runner's interrupted-owner guard below keeps takeover work resumable.
+    # ArchiveResult rows are projections of that work, never prerequisites used
+    # to decide whether the Snapshot may seal.
     if not was_sealed and snapshot.status == Snapshot.StatusChoices.STARTED and snapshot.retry_at == owned_retry_at:
         snapshot.seal()
         snapshot.refresh_from_db()
@@ -120,8 +122,9 @@ class SnapshotService(BaseService):
     LISTENS_TO = [SnapshotEvent, SnapshotCompletedEvent]
     EMITS = []
 
-    def __init__(self, bus, *, crawl_id: str):
+    def __init__(self, bus, *, crawl_id: str, interrupted: Callable[[], bool] | None = None):
         self.crawl_id = crawl_id
+        self.interrupted = interrupted
         self._run_ownership: dict[str, tuple[str, object, bool, list[str]]] = {}
         self._ownership_lock = asyncio.Lock()
         super().__init__(bus)
@@ -175,6 +178,15 @@ class SnapshotService(BaseService):
             if ownership is None or ownership[0] != str(event.event_parent_id):
                 return
             self._run_ownership.pop(snapshot_id, None)
+        # abx-dl also emits completion after abort cleanup, even before the
+        # first hook. Explicit user aborts seal a point-in-time capture so later
+        # work cannot silently mix observations taken days apart; reopening it
+        # is a separate user action. Takeover only replaces its execution owner.
+        # Consult the runner's synchronous signal state: an exiting owner cannot
+        # rely on another bus event arriving before this completion projector runs.
+        # Leave its durable lease for the next elected runner's startup recovery.
+        if self.interrupted is not None and self.interrupted():
+            return
         _, owned_retry_at, was_sealed, retry_plugins = ownership
         await sync_to_async(finalize_completed_snapshot, thread_sensitive=True)(
             event.snapshot_id,

archivebox/templates/core/public_index.html

@@ -654,6 +654,13 @@
                 </span>
                 <input type="submit" value="Search">
             </form>
+            {% comment %}
+                Missing/stale totals are intentional on large collections: show rows
+                immediately and let a later normal reload pick up the cached count.
+                Do not fill this in with an extra count request or paginator.count:
+                the unfiltered paginator only knows the rows needed for navigation.
+                Use the optional public_* totals here and in the footer below.
+            {% endcomment %}
             <div class="public-snapshot-count">
                 {% if public_search_stream_pending %}
                     <span class="archivebox-search-stream-status" role="status" aria-live="polite">Searching...</span>

archivebox/tests/test_archive_result_service.py

@@ -296,9 +296,11 @@ def test_archiveresult_event_create_uses_one_result_lookup():
     assert len(result_lookups) == 1
 
 
-def test_process_completed_projects_failed_archiveresult_from_shipped_hook(tmp_path, hermetic_lib_dir):
+def test_failed_shipped_title_hook_projects_failure_without_overwriting_title(tmp_path, hermetic_lib_dir):
     from archivebox.core.models import ArchiveResult
 
+    # Result linkage and the unchanged title are consequences of the same failed
+    # hook. Check them together so this isolated runtime is installed only once.
     snapshot = _create_snapshot()
     process, result = _run_shipped_snapshot_hook(
         snapshot,
@@ -310,22 +312,6 @@ def test_process_completed_projects_failed_archiveresult_from_shipped_hook(tmp_p
     assert result.status == ArchiveResult.StatusChoices.FAILED
     assert result.process_id == process.id
     assert "Chrome session" in result.output_str
-    _cleanup_machine_process_rows()
-
-
-def test_failed_title_archiveresult_does_not_overwrite_snapshot_title(tmp_path, hermetic_lib_dir):
-    from archivebox.core.models import ArchiveResult
-
-    snapshot = _create_snapshot()
-    _, result = _run_shipped_snapshot_hook(
-        snapshot,
-        plugin="title",
-        hook_name="on_Snapshot__54_title.js",
-        lib_dir=hermetic_lib_dir,
-        expected_exit_codes=(1,),
-    )
-    assert result.status == ArchiveResult.StatusChoices.FAILED
-    assert "Chrome session" in result.output_str
     snapshot.refresh_from_db()
     assert snapshot.title in (None, "")
     assert snapshot.resolved_title == ""

archivebox/tests/test_cli_add_2.py

@@ -36,15 +36,26 @@ from .test_cli_add_1 import (
 )
 
 
+@pytest.fixture
+def progress_page(httpserver):
+    # Exercise the real HTTP fetch and parser with fixed content. example.com's
+    # outgoing IANA link has changed independently of ArchiveBox releases.
+    httpserver.expect_request("/").respond_with_data(
+        '<html><body><a href="https://iana.org/domains/example">Example domains</a></body></html>',
+        content_type="text/html",
+    )
+    return httpserver.url_for("/")
+
+
 @pytest.mark.parametrize("terminal_stream", ["stdout", "stderr"])
-def test_add_renders_live_progress_in_foreground_terminal(initialized_archive, terminal_stream):
+def test_add_renders_live_progress_in_foreground_terminal(initialized_archive, terminal_stream, progress_page):
     master, slave = pty.openpty()
     termios.tcsetwinsize(slave, (40, 160))
     output = bytearray()
     result = None
     try:
         result = run_archivebox_cmd(
-            ["add", "--plugins=parse_txt_urls", "https://example.com"],
+            ["add", "--plugins=parse_txt_urls", progress_page],
             cwd=initialized_archive,
             env=cli_env(USE_COLOR="True", SHOW_PROGRESS="True"),
             **{terminal_stream: slave},
@@ -65,7 +76,7 @@ def test_add_renders_live_progress_in_foreground_terminal(initialized_archive, t
         with use_archivebox_db(initialized_archive):
             assert Crawl.objects.get().status == Crawl.StatusChoices.SEALED
             snapshot = Snapshot.objects.get()
-            assert snapshot.url == "https://example.com"
+            assert snapshot.url == progress_page
             parsed_urls = Path(snapshot.output_dir) / "parse_txt_urls" / "urls.jsonl"
             assert json.loads(parsed_urls.read_text())["url"] == "https://iana.org/domains/example"
     finally:
@@ -76,9 +87,9 @@ def test_add_renders_live_progress_in_foreground_terminal(initialized_archive, t
         os.close(master)
 
 
-def test_add_redirected_progress_remains_plain_text(initialized_archive):
+def test_add_redirected_progress_remains_plain_text(initialized_archive, progress_page):
     result = run_archivebox_cmd(
-        ["add", "--plugins=parse_txt_urls", "https://example.com"],
+        ["add", "--plugins=parse_txt_urls", progress_page],
         cwd=initialized_archive,
         env=cli_env(),
     )

archivebox/tests/test_crawl_runner_2.py

@@ -0,0 +1,61 @@
+"""Takeover preserves a capture; an explicit user abort ends its attempt."""
+
+import asyncio
+
+import pytest
+
+
+@pytest.mark.django_db(transaction=True)
+@pytest.mark.parametrize("user_initiated", [False, True], ids=["takeover", "user-abort"])
+def test_abort_before_first_snapshot_hook_preserves_capture_intent(recursive_test_site, user_initiated):
+    from abx_dl.events import CrawlAbortEvent, CrawlCompletedEvent, ProcessStartedEvent, SnapshotCompletedEvent, SnapshotEvent
+    from archivebox.base_models.models import get_or_create_system_user_pk
+    from archivebox.core.models import Snapshot
+    from archivebox.crawls.models import Crawl
+    from archivebox.services.runner import CrawlRunner
+
+    crawl = Crawl.objects.create(
+        urls=recursive_test_site["root_url"],
+        config={"PLUGINS": "wget"},
+        created_by_id=get_or_create_system_user_pk(),
+    )
+    runner = CrawlRunner(crawl)
+    completed = []
+    started = []
+    completed_crawls = []
+
+    async def abort_on_snapshot(event: SnapshotEvent) -> None:
+        # Observe the real snapshot command after its DB owner is recorded but
+        # before the hook service consumes it. This is the takeover window that
+        # used to turn shutdown cleanup into a successful, empty capture.
+        await event.emit(CrawlAbortEvent(user_initiated=user_initiated)).now()
+
+    runner.bus.on(SnapshotEvent, abort_on_snapshot)
+    runner.bus.on(SnapshotCompletedEvent, lambda event: completed.append(event))
+    runner.bus.on(ProcessStartedEvent, lambda event: started.append(event))
+    runner.bus.on(CrawlCompletedEvent, lambda event: completed_crawls.append(event))
+    if user_initiated:
+        with pytest.raises(KeyboardInterrupt):
+            asyncio.run(runner.run())
+    else:
+        asyncio.run(runner.run())
+
+    assert len(completed) == 1, "Aborting must still finish the real snapshot cleanup phase"
+    assert not [event for event in started if event.hook_name.startswith("on_Snapshot")]
+    snapshot = Snapshot.objects.get(crawl=crawl)
+    crawl.refresh_from_db()
+    assert len(completed_crawls) == int(user_initiated)
+    assert snapshot.archiveresult_set.count() == 0
+    if user_initiated:
+        # Preserve the historical point-in-time boundary for an explicit abort.
+        assert snapshot.status == Snapshot.StatusChoices.SEALED
+        assert snapshot.retry_at is None
+        assert snapshot.downloaded_at is not None
+        assert crawl.status == Crawl.StatusChoices.SEALED
+        assert crawl.retry_at is None
+    else:
+        assert snapshot.status in Snapshot.RUNNABLE_STATES, "Takeover must leave the capture resumable"
+        assert snapshot.retry_at is not None
+        assert snapshot.downloaded_at is None
+        assert crawl.status in Crawl.RUNNABLE_STATES
+        assert crawl.retry_at is not None

archivebox/workers/supervisord_util.py

@@ -1677,15 +1677,19 @@ def start_server_workers(
 ):
     from archivebox.config.common import get_config
 
-    require_server_worker_memory()
-    config = get_config()
     shutdown_state = None
     tail_result = "stopped"
-    # Enter cooperative handling only once startup finishes, but retain it
-    # through our cleanup finally: Popen.wait() has the same interruption risk
-    # as poll(). ExitStack preserves the existing startup and daemon policies.
+    # A real takeover regression sent SIGTERM after a standby server resumed,
+    # while it was still syncing workers. Immediate exceptions can interrupt
+    # subprocess polling or waits during startup and cleanup, so keep the
+    # existing cooperative policy active for the whole foreground lifecycle.
+    # Daemon startup retains its existing signal policy.
     with ExitStack() as shutdown_context:
+        if not daemonize:
+            shutdown_state = shutdown_context.enter_context(foreground_shutdown_signals(raise_on_first_signal=False))
         try:
+            require_server_worker_memory()
+            config = get_config()
             supervisor = get_or_create_supervisord_process(daemonize=daemonize)
             workers, log_files, components = build_server_worker_plan(
                 config=config,
@@ -1714,13 +1718,13 @@ def start_server_workers(
 
             from django.db import connections
 
-            connections.close_all()
-            # Immediate exceptions can strand Popen's wait lock inside poll().
-            # Use the shared cooperative contract in core.shutdown_util; the
-            # tail checks intent between operations, including while draining
-            # busy logs. No interruption checks run inside child cleanup.
-            shutdown_state = shutdown_context.enter_context(foreground_shutdown_signals(raise_on_first_signal=False))
             try:
+                # Finish the current startup operation before honoring a
+                # shutdown received during supervisor/worker synchronization.
+                # The tail also checks intent between operations, including
+                # while draining busy logs. Cleanup remains synchronous.
+                raise_if_shutdown_requested()
+                connections.close_all()
                 # Tail worker logs while supervisord runs.
                 sys.stdout.write("Tailing worker logs (Ctrl+C to stop)...\n\n")
                 sys.stdout.flush()

bin/check-staging-deployment.py

@@ -10,8 +10,9 @@ import sys
 import time
 import tomllib
 from pathlib import Path
+from urllib.parse import urljoin
 
-from playwright.sync_api import sync_playwright
+from playwright.sync_api import expect, sync_playwright
 
 
 def main():
@@ -96,8 +97,49 @@ def main():
             response = page.goto(snapshot["replay_url"], wait_until="domcontentloaded")
             assert response and response.ok, snapshot["replay_url"]
             page.locator('.thumb-card[data-plugin-name="screenshot"]').wait_for(state="attached")
-            page.locator("#main-frame").wait_for(state="attached")
-            page.screenshot(path=str(evidence / f"{snapshot['id']}.png"), full_page=True)
+            try:
+                # The shell creates an about:blank iframe before replay starts.
+                # Its presence previously accepted blank screenshots as proof of
+                # replay. Exercise the default viewer and its real captured page,
+                # using the same readiness contract as the WACZ browser tests.
+                expect(page.locator(".header-title-text")).to_contain_text(re.compile(r"\S"))
+                preview = page.locator("#main-frame")
+                expect(preview).to_have_attribute("src", re.compile(r"/archivewebpage/archivewebpage\.wacz\?preview=1"))
+                replay_root = preview.content_frame.locator("#replay-root")
+                expect(replay_root).to_have_attribute("data-url", snapshot["url"])
+                preview_frame = preview.element_handle().content_frame()
+                assert preview_frame is not None
+                preview_frame.wait_for_function(
+                    """() => {
+                        if (window.archiveboxReplayError) throw new Error(window.archiveboxReplayError);
+                        return Boolean(window.archiveboxReplayReady);
+                    }""",
+                )
+                # Extracted display titles need not match the captured document's
+                # <title>: HedgeDoc's note heading and HTML title differ. Identify
+                # the archived document by its replay origin/path and original URL,
+                # as the ArchiveWebpage browser tests do. Wrapper frames and other
+                # output thumbnails cannot satisfy this check.
+                replay_base = replay_root.get_attribute("data-replaybase")
+                assert replay_base
+                replay_prefix = urljoin(preview_frame.url, f"{replay_base}w/")
+                frames = list(preview_frame.child_frames)
+                frame_titles = {}
+                while frames:
+                    replay = frames.pop()
+                    frame_titles[replay.url] = replay.title().strip()
+                    if replay.url.startswith(replay_prefix) and replay.url.endswith(f"/{snapshot['url']}"):
+                        body = replay.locator("body")
+                        expect(body).to_be_visible()
+                        # Both acceptance URLs contain substantial article/profile
+                        # text. Reject a blank frame or a short viewer error message.
+                        assert len(" ".join(body.inner_text().split())) > 1000, f"Missing captured page content: {replay.url}"
+                        break
+                    frames.extend(replay.child_frames)
+                else:
+                    raise AssertionError(f"Captured URL {snapshot['url']!r} not found in replay frames: {frame_titles}")
+            finally:
+                page.screenshot(path=str(evidence / f"{snapshot['id']}.png"), full_page=True)
         browser.close()
     # Recheck identity and host OOM history after the browser flow too.
     final = remote("capture")

docs/test_codeblocks_manifest.py

@@ -373,6 +373,16 @@ def run_snippets(snippet_ids: tuple[str, ...]) -> None:
                 "PATH": f"{Path(sys.executable).parent}:{env['PATH']}",
             },
         )
+        # A runnable Python alias can still select an interpreter without the
+        # project's packages. Keep its first hop visible when a real hook fails;
+        # resolving every symlink here would hide the venv path we need to check.
+        if env.get("PYTHON_BINARY"):
+            python_override = Path(env["PYTHON_BINARY"])
+            print(
+                f"Documentation Python: harness={sys.executable}, override={python_override}, "
+                f"target={python_override.readlink() if python_override.is_symlink() else python_override}",
+                flush=True,
+            )
         Path(env["HOME"]).mkdir()
         scenarios = {records[snippet_id]["scenario"] for snippet_id in snippet_ids}
         workdirs = {

etc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "archivebox",
-  "version": "0.9.71rc43",
+  "version": "0.9.71",
   "repository": "github:ArchiveBox/ArchiveBox",
   "license": "MIT",
   "dependencies": {

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "archivebox"
-version = "0.9.71rc43"
+version = "0.9.71"
 requires-python = ">=3.13"
 description = "Self-hosted internet archiving solution."
 authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@@ -77,10 +77,10 @@ dependencies = [
     "w3lib>=2.2.1",          # used for parsing content-type encoding from http response headers & html tags
     ### Extractor dependencies (runtime binaries resolved through abxpkg)
     ### Binary/Package Management
-    "abxbus==2.5.68",                 # direct imports only; version constrained by abx-dl -> abx-plugins
-    "abxpkg==1.13.15",                # direct imports only; version constrained by abx-dl -> abx-plugins
-    "abx-plugins[opencode]==1.13.103", # optional AI runtime clients are owned by the plugin
-    "abx-dl==1.13.71",       # shared ArchiveBox downloader package
+    "abxbus==2.5.70",                 # direct imports only; version constrained by abx-dl -> abx-plugins
+    "abxpkg==1.13.21",                # direct imports only; version constrained by abx-dl -> abx-plugins
+    "abx-plugins[opencode]==1.13.109", # optional AI runtime clients are owned by the plugin
+    "abx-dl==1.13.79",       # shared ArchiveBox downloader package
     ### UUID7 backport for Python <3.14
     "uuid7>=0.1.0; python_version < '3.14'",  # provides the uuid_extensions module on Python 3.13
     "browser-cookie3>=0.20.1",
@@ -343,7 +343,7 @@ Donate = "https://github.com/ArchiveBox/ArchiveBox/wiki/Donations"
 
 
 [tool.bumpver]
-current_version = "v0.9.71rc43"
+current_version = "v0.9.71"
 version_pattern = "vMAJOR.MINOR.PATCH[PYTAGNUM]"
 commit_message = "bump version {old_version} -> {new_version}"
 tag_message = "{new_version}"

uv.lock

@@ -13,18 +13,18 @@ supported-markers = [
 ]
 
 [options]
-exclude-newer = "2026-09-23T08:33:14.714060607Z"
+exclude-newer = "2026-09-23T19:26:02.6709672Z"
 exclude-newer-span = "P5D"
 
 [options.exclude-newer-package]
-abxbus = { timestamp = "2026-09-28T08:33:13.714073511Z", span = "PT1S" }
+abxbus = { timestamp = "2026-09-28T19:26:01.670984315Z", span = "PT1S" }
 abx-plugins = "2100-01-01T00:00:00Z"
 abx-dl = "2100-01-01T00:00:00Z"
 abxpkg = "2100-01-01T00:00:00Z"
 
 [[package]]
 name = "abx-dl"
-version = "1.13.71"
+version = "1.13.79"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "abx-plugins", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -38,14 +38,14 @@ dependencies = [
     { name = "rich", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/79/67/9eaf667a429db72f0609b493aca107ff42f29a31dfd2333fcea02ece900a/abx_dl-1.13.71.tar.gz", hash = "sha256:1fdb6e584e2d4e7d7c04048cbd5c37bef3578352d6c2fe7ab76fc706ddadf09b", size = 97387 }
+sdist = { url = "https://files.pythonhosted.org/packages/f5/59/e178d330d244351f01026a2c261c7c0d8067c0efcbef9cf095635fa97579/abx_dl-1.13.79.tar.gz", hash = "sha256:8825b06570df48f3ecdd525e8fca09a2d8dac77b8ba059efaac7d0df845b578a", size = 97588, upload-time = "2026-09-28T19:23:47.984Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/31/23/d12ed879d5253ae6bbbf0e2ced77b0b59f3b4d4c3b381cc84518fee0443e/abx_dl-1.13.71-py3-none-any.whl", hash = "sha256:0585182839d8e9d3c93ee87d0ca6e2fa2d9d8ac936515f2700046324d210ce29", size = 102837 },
+    { url = "https://files.pythonhosted.org/packages/60/6f/f7de1fd42bbcdf56769a7277ca261007525d2a482544fe03d509b3cc6805/abx_dl-1.13.79-py3-none-any.whl", hash = "sha256:3b9b8426da6c0fcd55583d0f26c037561b40982866c4a12f481c8f481fbe56b2", size = 102934, upload-time = "2026-09-28T19:23:49.287Z" },
 ]
 
 [[package]]
 name = "abx-plugins"
-version = "1.13.103"
+version = "1.13.109"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "abxbus", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -54,9 +54,9 @@ dependencies = [
     { name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "uv", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/5c/ba/9b9f6d72c95026fbaf8721adc36705a5f3f6916387d2c7d15d29f095e4e0/abx_plugins-1.13.103.tar.gz", hash = "sha256:ff49efbee93a54203ecf3680251ef60299f29d4c77b1d20bfbfebfabdccd93b2", size = 460937, upload-time = "2026-09-28T07:58:11.436Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/6a/22/e236ab70ff3c24c3be7f511c202e9943b4acf7988ca180f6593a959cf190/abx_plugins-1.13.109.tar.gz", hash = "sha256:056556249fbe5e1aa5eca3cca84923fab9a4a1fcfd0d3f82c4911de97f2e49f5", size = 461059, upload-time = "2026-09-28T19:08:05.06Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/69/ec/1c2db942f2ebd8945160084aaed8e4cac29a3364937c64771c8e76700cef/abx_plugins-1.13.103-py3-none-any.whl", hash = "sha256:54ee14dbad090f180b7d35075c175460be0a783745ed86a732bb960157778dff", size = 673577, upload-time = "2026-09-28T07:58:09.949Z" },
+    { url = "https://files.pythonhosted.org/packages/a4/c9/6331394242f8c5afa1fb03abfd805e509add8a7d03c14b9e126e6724af20/abx_plugins-1.13.109-py3-none-any.whl", hash = "sha256:5bdecde68a9bc2265a26b95205c9214a07e39328c981c8700234f577a57d0ba7", size = 673893, upload-time = "2026-09-28T19:08:06.808Z" },
 ]
 
 [package.optional-dependencies]
@@ -68,21 +68,21 @@ opencode = [
 
 [[package]]
 name = "abxbus"
-version = "2.5.68"
+version = "2.5.70"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "pydantic", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "uuid7", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/8e/d7/0c92e436230b40ee4c21c83e4081ab67a6930538e3ff759857c3d4cebe51/abxbus-2.5.68.tar.gz", hash = "sha256:e53c68467749f17ba14b982184b02aed8ba106e36412217f1c1cd61d8ce5a7b0", size = 134319, upload-time = "2026-09-28T07:13:42.183Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/52/80/43ab38e5a88a6c8864c2460e0685f946f9fcd6cb4776033611107d70ecd6/abxbus-2.5.70.tar.gz", hash = "sha256:e2a6593d67ed619833a6a2b2a282c58312404f166436a0991f5017d51f45fbe5", size = 134824, upload-time = "2026-09-28T18:36:05.082Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/bc/ee/e2b567d576b13bf32aaddcacf376a89c4a4df0de1081f95a56a0d03bdc23/abxbus-2.5.68-py3-none-any.whl", hash = "sha256:61f3ba656f1010b6801d8a625f65f124521e6ec2e03f454baf7491e5ed4fb345", size = 127969, upload-time = "2026-09-28T07:13:40.627Z" },
+    { url = "https://files.pythonhosted.org/packages/4d/0b/67ebd55f417b641f05b9ba07d738b75b33924026cda220c9d4f771d6e16e/abxbus-2.5.70-py3-none-any.whl", hash = "sha256:a33727a432ec6729d3bd0674d36bb82d87a1faac66480a3eeddaee23e6960a25", size = 128460, upload-time = "2026-09-28T18:36:06.186Z" },
 ]
 
 [[package]]
 name = "abxpkg"
-version = "1.13.15"
+version = "1.13.21"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "platformdirs", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -90,9 +90,9 @@ dependencies = [
     { name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/62/34/16a0501195c9a29c5f532a23a07e50237d0e492b78b5d9d8f43c13a7517c/abxpkg-1.13.15.tar.gz", hash = "sha256:7f29ca4432399ab15180bd58a60f39a3ef7fedcd7940affe2eda6b57e914c98c", size = 242799, upload-time = "2026-09-28T07:26:27.62Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/0f/bb/56f863aa00e8e9311c0c4af059e7f0ebed2242fd40be9f3b4bbb0e70d646/abxpkg-1.13.21.tar.gz", hash = "sha256:9d5e05a0534836ea637e1c94f4555558c5d6a09368e3ee24e5a056812afc2342", size = 243339, upload-time = "2026-09-28T18:51:45.417Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/7d/85/988013c6e773bed15f1671e017317edc3b047effa594cbf826af49abce11/abxpkg-1.13.15-py3-none-any.whl", hash = "sha256:8b1924225c4f79408d2d1729c59614014779fbae2800d1e7e2ce3ef1389be198", size = 254220, upload-time = "2026-09-28T07:26:26.22Z" },
+    { url = "https://files.pythonhosted.org/packages/37/9c/e60dc8ef5ec6dbe9b39c6bd06165f9d018660cba891295ff625d7cf577b1/abxpkg-1.13.21-py3-none-any.whl", hash = "sha256:b796b99dbbdfdafa3ab05f881ad505539deabbe14eb7b358811f12c47c5a47ad", size = 254735, upload-time = "2026-09-28T18:51:43.415Z" },
 ]
 
 [[package]]
@@ -127,7 +127,7 @@ wheels = [
 
 [[package]]
 name = "archivebox"
-version = "0.9.71rc43"
+version = "0.9.71"
 source = { editable = "." }
 dependencies = [
     { name = "abx-dl", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -227,10 +227,10 @@ dev = [
 
 [package.metadata]
 requires-dist = [
-    { name = "abx-dl", specifier = "==1.13.71" },
-    { name = "abx-plugins", extras = ["opencode"], specifier = "==1.13.103" },
-    { name = "abxbus", specifier = "==2.5.68" },
-    { name = "abxpkg", specifier = "==1.13.15" },
+    { name = "abx-dl", specifier = "==1.13.79" },
+    { name = "abx-plugins", extras = ["opencode"], specifier = "==1.13.109" },
+    { name = "abxbus", specifier = "==2.5.70" },
+    { name = "abxpkg", specifier = "==1.13.21" },
     { name = "archivebox", extras = ["sonic", "ldap", "debug"], marker = "extra == 'all'" },
     { name = "atomicwrites", specifier = "==1.4.1" },
     { name = "base32-crockford", specifier = ">=0.3.0" },