Bump release version to 0.9.72rc105

This reverts commit da1665acb3.

.github/CONTRIBUTING.md

@@ -12,7 +12,7 @@
 - https://github.com/ArchiveBox/ArchiveBox/issues
 - https://github.com/ArchiveBox/ArchiveBox/pulls
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Install#manual-setup
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-c-bare-metal-setup
 
 ### Development Setup
 

.github/STAGING.md

@@ -33,3 +33,10 @@ screenshots are retained as Actions artifacts. Other failures remain
 failures; do not publish a success deployment record to bypass them. No code
 mounts are allowed. The Cabbage job must pass before its record authorizes a
 stable upload. Queued capture progress is polled; failed tests are not retried.
+
+Browser acceptance follows the default WACZ viewer into the captured document,
+matching its replay base and original URL and checking that its body is visible
+and contains substantial text. An attached viewer iframe can still be
+`about:blank`, so its presence alone cannot prove replay works. Extracted display
+titles and the captured document's `<title>` may legitimately differ; neither
+title equality nor output thumbnails substitute for checking the actual page.

.github/dependabot.yml

@@ -15,7 +15,7 @@ updates:
         patterns:
           - "*"
   - package-ecosystem: "npm"
-    directory: "/"
+    directory: "/etc"
     target-branch: "dev"
     schedule:
       interval: "monthly"

.github/lychee.toml

@@ -0,0 +1,38 @@
+# Documentation destinations only: do not crawl archived URLs, app routes, or image assets.
+# Manual run from the repository root with Lychee 0.24.2:
+# git ls-files -- '*.md' '*.html' '*.py' '*.rst' '*.yml' '*.yaml' '*.sh' '*.toml' > /tmp/archivebox-link-inputs.txt
+# lychee --config .github/lychee.toml --root-dir "$PWD" --files-from /tmp/archivebox-link-inputs.txt
+# Explicit tracked inputs include README symlinks and let errors in one file leave others checkable.
+include_fragments = "anchor-only"
+include_verbatim = true
+max_retries = 0
+max_concurrency = 4
+timeout = 20
+no_progress = true
+hidden = true
+extensions = ["md", "html", "py", "rst", "yml", "yaml", "sh", "toml"]
+exclude_path = ['(^|/)(tests|data|apidocs|node_modules|\.git)(/|$)']
+# Includes override this exclude; this is a documentation-link allowlist.
+exclude = ['.*']
+include = [
+ '^https://github\.com/ArchiveBox/ArchiveBox/blob/[^/]+/.*\.md(?:#|$)',
+ '^https://raw\.githubusercontent\.com/wiki/ArchiveBox/ArchiveBox/',
+ '^https?://github\.com/(?:ArchiveBox|pirate)/ArchiveBox(?:/wiki(?:/|#|$)|/?#)',
+ '^https?://docs\.archivebox\.io(?:/|$)',
+ '^file:.*\.md(?:#|$)',
+ '^file:.*/docs/[^/.#]+(?:#|$)',
+]
+fallback_extensions = ["md"]
+remap = [
+ # Lychee's Markdown slugger drops variation selectors. Check these against rendered HTML.
+
+ '^file:.*/README\.md(#%EF%B8%8F-cli-usage)$ https://github.com/ArchiveBox/ArchiveBox${1}',
+ '^file:.*/docs/Install\.md(#option-a-docker--docker-compose-setup-%EF%B8%8F)$ https://github.com/ArchiveBox/ArchiveBox/wiki/Install${1}',
+ # The wiki accepts this case alias, but raw file paths are case-sensitive.
+ '^https?://github\.com/(?:ArchiveBox|pirate)/ArchiveBox/wiki/Setting-up-Storage$ https://raw.githubusercontent.com/wiki/ArchiveBox/ArchiveBox/Setting-Up-Storage.md',
+
+ # Bare missing wiki pages silently redirect to Home; raw pages return an actual 404.
+ # Keep fragment URLs on the rendered wiki to catch published heading drift.
+ '^https?://github\.com/(?:ArchiveBox|pirate)/ArchiveBox/wiki/([^#/?]+)$ https://raw.githubusercontent.com/wiki/ArchiveBox/ArchiveBox/${1}.md',
+]
+preprocess = { command = "bin/lychee_preprocess.py" }

.github/pages/README.md

@@ -65,10 +65,15 @@ Build: `uv run --no-project python .github/pages/site.py render _site --source p
 Check the built site with real Chromium (also run by Pages CI):
 
 ```sh
-uv run --no-config --no-project --with playwright==1.63.0 playwright install chromium
+export ABXPKG_LIB_DIR=/tmp/archivebox-site-browser
+CHROME_SCHEMA="$(uv run python -c 'from abx_plugins import get_plugins_dir; print(get_plugins_dir() / "chrome" / "config.json")')"
+uv run abxpkg env --install --deps-from="$CHROME_SCHEMA:required_binaries"
+export CHROME_BINARY="$ABXPKG_LIB_DIR/env/bin/chromium"
 uv run --no-config --no-project --with playwright==1.63.0 python .github/pages/verify.py SITE_OUTPUT --evidence /tmp/site-evidence
 ```
 
+The browser comes from the normal abxpkg provider/cache path; Playwright drives
+that resolved executable instead of installing a second Chromium build.
 Use this workflow's `SITE_OUTPUT` directory. The check covers desktop/mobile
 layout, keyboard dismissal, no-JavaScript links, local resources and visible
 images, and uploads its screenshots as `site-verification` for 14 days.

.github/pages/artifacts.py

@@ -14,7 +14,7 @@ def trusted(run, repo, workflow, branch, successful=True):
     return (
         (run.get("head_repository") or {}).get("full_name", "").lower() == repo.lower()
         and run["head_branch"] == branch
-        and run["event"] in ("push", "workflow_dispatch")
+        and run["event"] in ("push", "workflow_dispatch", "schedule")
         and run["path"] == f".github/workflows/{workflow}"
         and (not successful or run["conclusion"] == "success")
     )

.github/pages/verify.py

@@ -2,6 +2,7 @@
 
 import argparse
 import json
+import os
 import re
 from functools import partial
 from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
@@ -75,7 +76,7 @@ def verify(output, evidence):
     origin = f"http://127.0.0.1:{server.server_port}"
     try:
         with sync_playwright() as playwright:
-            browser = playwright.chromium.launch(ignore_default_args=["--hide-scrollbars"])
+            browser = playwright.chromium.launch(executable_path=os.environ.get("CHROME_BINARY"), ignore_default_args=["--hide-scrollbars"])
             for name in config["pages"]:
                 route = name.removesuffix("index.html")
                 page = browser.new_page(reduced_motion="reduce")

.github/scripts/discover_test_matrix.py

@@ -13,8 +13,9 @@ def main() -> None:
     if not archivebox_tests:
         raise SystemExit("No ArchiveBox tests discovered")
 
-    # These are observed whole-job seconds, including setup. Unknown/long files,
-    # optional dependencies and hosted-only requirements keep independent jobs.
+    # These are observed per-file pytest seconds, including fixture/startup
+    # time. Shared job setup is paid once per batch. Unknown/long files, optional
+    # dependencies and hosted-only requirements keep independent jobs.
     durations = json.loads((root / ".github/test-durations.json").read_text())["seconds"]
     hosted = {path for path in archivebox_tests if "# ci-runner: hosted" in path.read_text().splitlines()[:5]}
     requested = os.environ.get("TEST_FILE", "")

.github/test-durations.json

@@ -1,169 +1,176 @@
 {
   "source_runs": [
-    "https://github.com/ArchiveBox/ArchiveBox/actions/runs/36302511923",
-    "https://github.com/ArchiveBox/ArchiveBox/actions/runs/36353196552",
-    "https://github.com/ArchiveBox/ArchiveBox/actions/runs/36454992009"
+    "https://github.com/ArchiveBox/ArchiveBox/actions/runs/37190995942",
+    "https://github.com/ArchiveBox/ArchiveBox/actions/runs/37237793977"
   ],
-  "measurement": "Whole job seconds, including setup and excluding queue time. Hosted unbatched files use the latest complete successful run; other files retain their individual baseline measurement. Unknown files stay independent.",
+  "measurement": "Unbatched files use successful per-file pytest step wall seconds from run 37237793977, including process and fixture startup, on their actual hosted or ugNAS runner. Already-batched files retain prior JUnit measurements plus 9 seconds startup. Queue and shared job setup are excluded; unknown files remain independent.",
   "seconds": {
-    "archivebox/tests/test_api_v1_auth_browser_session.py": 29,
-    "archivebox/tests/test_api_v1_auth_check_api_token.py": 27,
-    "archivebox/tests/test_api_v1_auth_get_api_token.py": 24,
-    "archivebox/tests/test_api_v1_cli_add.py": 96,
-    "archivebox/tests/test_api_v1_cli_remove.py": 50,
-    "archivebox/tests/test_api_v1_cli_schedule.py": 41,
-    "archivebox/tests/test_api_v1_cli_search.py": 26,
-    "archivebox/tests/test_api_v1_cli_update.py": 74,
-    "archivebox/tests/test_api_v1_cli_workflow_add_search_update_remove.py": 64,
-    "archivebox/tests/test_api_v1_core_any_id.py": 21,
-    "archivebox/tests/test_api_v1_core_archiveresult_archiveresult_id.py": 55,
-    "archivebox/tests/test_api_v1_core_archiveresults.py": 32,
-    "archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py": 199,
-    "archivebox/tests/test_api_v1_core_snapshots_rss.py": 27,
-    "archivebox/tests/test_api_v1_core_snapshots.py": 28,
-    "archivebox/tests/test_api_v1_core_tag_tag_id_rename.py": 23,
-    "archivebox/tests/test_api_v1_core_tag_tag_id_snapshots_jsonl.py": 27,
-    "archivebox/tests/test_api_v1_core_tag_tag_id_urls_txt.py": 28,
-    "archivebox/tests/test_api_v1_core_tag_tag_id.py": 27,
-    "archivebox/tests/test_api_v1_core_tags_add_to_snapshot.py": 27,
-    "archivebox/tests/test_api_v1_core_tags_autocomplete.py": 26,
-    "archivebox/tests/test_api_v1_core_tags_create.py": 21,
-    "archivebox/tests/test_api_v1_core_tags_remove_from_snapshot.py": 25,
-    "archivebox/tests/test_api_v1_core_tags_search.py": 26,
-    "archivebox/tests/test_api_v1_core_tags.py": 25,
-    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_filename.py": 22,
-    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_filename.py": 25,
-    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_subfolder_filename.py": 24,
-    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py": 86,
-    "archivebox/tests/test_api_v1_crawls_crawls.py": 21,
-    "archivebox/tests/test_api_v1_machine_binaries.py": 23,
-    "archivebox/tests/test_api_v1_machine_binary_binary_id.py": 23,
-    "archivebox/tests/test_api_v1_machine_binary_by_name_name.py": 26,
-    "archivebox/tests/test_api_v1_machine_machine_current.py": 23,
-    "archivebox/tests/test_api_v1_machine_machine_machine_id.py": 23,
-    "archivebox/tests/test_api_v1_machine_machines.py": 27,
-    "archivebox/tests/test_api_v1_personas_personas.py": 23,
-    "archivebox/tests/test_api_v1_personas_sync.py": 29,
-    "archivebox/tests/test_api_v1_workflow_core_token_auth_side_effects.py": 36,
-    "archivebox/tests/test_api_v1_workflow_frozen_crawl_config_sources.py": 33,
-    "archivebox/tests/test_archive_result_service.py": 137,
-    "archivebox/tests/test_auth_ldap.py": 44,
-    "archivebox/tests/test_binary_service.py": 83,
-    "archivebox/tests/test_cli_add_1.py": 168,
-    "archivebox/tests/test_cli_add_2.py": 203,
-    "archivebox/tests/test_cli_add_3.py": 215,
-    "archivebox/tests/test_cli_add_4.py": 201,
-    "archivebox/tests/test_cli_archiveresult.py": 199,
-    "archivebox/tests/test_cli_binary.py": 20,
-    "archivebox/tests/test_cli_config.py": 148,
-    "archivebox/tests/test_cli_crawl.py": 177,
-    "archivebox/tests/test_cli_extract_input.py": 155,
-    "archivebox/tests/test_cli_extract.py": 106,
-    "archivebox/tests/test_cli_help.py": 30,
-    "archivebox/tests/test_cli_init.py": 135,
-    "archivebox/tests/test_cli_install.py": 58,
-    "archivebox/tests/test_cli_list.py": 260,
-    "archivebox/tests/test_cli_machine.py": 50,
-    "archivebox/tests/test_cli_manage.py": 48,
-    "archivebox/tests/test_cli_mcp.py": 51,
-    "archivebox/tests/test_cli_oneshot.py": 23,
-    "archivebox/tests/test_cli_persona.py": 82,
-    "archivebox/tests/test_cli_piping.py": 97,
-    "archivebox/tests/test_cli_pluginmap.py": 21,
-    "archivebox/tests/test_cli_process.py": 20,
-    "archivebox/tests/test_cli_remove.py": 162,
-    "archivebox/tests/test_cli_run_1.py": 195,
-    "archivebox/tests/test_cli_run_2.py": 255,
-    "archivebox/tests/test_cli_run_3.py": 167,
-    "archivebox/tests/test_cli_run_4.py": 178,
-    "archivebox/tests/test_cli_schedule.py": 87,
-    "archivebox/tests/test_cli_search.py": 35,
-    "archivebox/tests/test_cli_server.py": 259,
-    "archivebox/tests/test_cli_shell.py": 32,
-    "archivebox/tests/test_cli_snapshot.py": 181,
-    "archivebox/tests/test_cli_status.py": 102,
-    "archivebox/tests/test_cli_tag.py": 21,
-    "archivebox/tests/test_cli_update_reindex_snapshots.py": 163,
-    "archivebox/tests/test_cli_update.py": 142,
-    "archivebox/tests/test_cli_version.py": 143,
-    "archivebox/tests/test_config_DELETE_AFTER.py": 169,
-    "archivebox/tests/test_config_MAX_limits.py": 34,
-    "archivebox/tests/test_config_ONLY_NEW.py": 24,
-    "archivebox/tests/test_config_SAVE_TITLE.py": 99,
-    "archivebox/tests/test_config_URL_filters.py": 26,
-    "archivebox/tests/test_core_config.py": 20,
-    "archivebox/tests/test_core_forms.py": 28,
-    "archivebox/tests/test_crawl_pause.py": 20,
-    "archivebox/tests/test_crawl_runner.py": 45,
-    "archivebox/tests/test_crawl_service.py": 34,
-    "archivebox/tests/test_frozen_crawl_config.py": 47,
-    "archivebox/tests/test_hook_result_lifecycle.py": 78,
-    "archivebox/tests/test_hooks.py": 90,
-    "archivebox/tests/test_machine_models.py": 52,
-    "archivebox/tests/test_machine_service.py": 89,
-    "archivebox/tests/test_migrations_04_to_09.py": 36,
-    "archivebox/tests/test_migrations_07_to_09.py": 183,
-    "archivebox/tests/test_migrations_08_to_09.py": 244,
-    "archivebox/tests/test_migrations_fresh.py": 99,
-    "archivebox/tests/test_misc_checks.py": 22,
-    "archivebox/tests/test_misc_monkey_patches.py": 20,
-    "archivebox/tests/test_opencode_agent_1.py": 158,
-    "archivebox/tests/test_opencode_agent_2.py": 145,
-    "archivebox/tests/test_opencode_agent_3.py": 154,
-    "archivebox/tests/test_opencode_browser_1.py": 262,
-    "archivebox/tests/test_opencode_browser_2.py": 146,
-    "archivebox/tests/test_opencode_browser_3.py": 148,
-    "archivebox/tests/test_orm_helpers.py": 22,
-    "archivebox/tests/test_permissions.py": 21,
-    "archivebox/tests/test_persona_runtime.py": 56,
-    "archivebox/tests/test_postgres_backend.py": 85,
-    "archivebox/tests/test_process_runtime_paths.py": 28,
-    "archivebox/tests/test_process_service.py": 49,
-    "archivebox/tests/test_proof_previews.py": 54,
-    "archivebox/tests/test_recursive_crawl_1.py": 339,
-    "archivebox/tests/test_recursive_crawl_2.py": 171,
-    "archivebox/tests/test_recursive_crawl_3.py": 176,
-    "archivebox/tests/test_release_notes.py": 14,
-    "archivebox/tests/test_release_source_fingerprint.py": 22,
-    "archivebox/tests/test_release_workflow.py": 21,
-    "archivebox/tests/test_resource_admission.py": 21,
-    "archivebox/tests/test_runner_watch.py": 177,
-    "archivebox/tests/test_search.py": 153,
-    "archivebox/tests/test_serve_static.py": 31,
-    "archivebox/tests/test_server_automatic_url.py": 36,
-    "archivebox/tests/test_server_security_browser.py": 278,
-    "archivebox/tests/test_settings_signal_webhooks.py": 21,
-    "archivebox/tests/test_setup_script.py": 21,
-    "archivebox/tests/test_shutdown_util.py": 22,
-    "archivebox/tests/test_snapshot_filesystem_migration.py": 29,
-    "archivebox/tests/test_snapshot_output_stacks.py": 62,
-    "archivebox/tests/test_snapshot_ref_lookup.py": 23,
-    "archivebox/tests/test_snapshot_service.py": 41,
-    "archivebox/tests/test_snapshot_url_length.py": 26,
-    "archivebox/tests/test_tag_service.py": 24,
-    "archivebox/tests/test_takeover_util_1.py": 143,
-    "archivebox/tests/test_takeover_util_2.py": 201,
-    "archivebox/tests/test_takeover_util_3.py": 138,
-    "archivebox/tests/test_takeover_util_4.py": 156,
-    "archivebox/tests/test_test_harness.py": 22,
-    "archivebox/tests/test_ui_add_view_runtime.py": 221,
-    "archivebox/tests/test_ui_add_view.py": 46,
-    "archivebox/tests/test_ui_admin_apitoken.py": 23,
-    "archivebox/tests/test_ui_admin_archiveresult.py": 30,
-    "archivebox/tests/test_ui_admin_config_widget.py": 25,
-    "archivebox/tests/test_ui_admin_crawl.py": 44,
-    "archivebox/tests/test_ui_admin_links.py": 35,
-    "archivebox/tests/test_ui_admin_machine.py": 29,
-    "archivebox/tests/test_ui_admin_persona.py": 28,
-    "archivebox/tests/test_ui_admin_snapshot.py": 273,
-    "archivebox/tests/test_ui_admin_tag.py": 26,
-    "archivebox/tests/test_ui_admin_user.py": 28,
-    "archivebox/tests/test_ui_config_views.py": 43,
-    "archivebox/tests/test_ui_live_progress.py": 297,
-    "archivebox/tests/test_ui_public_snapshot.py": 179,
-    "archivebox/tests/test_ui_savepagenow.py": 48,
-    "archivebox/tests/test_urls.py": 220,
-    "archivebox/tests/test_util.py": 20,
-    "archivebox/tests/test_version_metadata.py": 24
+    "archivebox/tests/test_api_discovery_performance.py": 28,
+    "archivebox/tests/test_api_v1_auth_browser_session.py": 17,
+    "archivebox/tests/test_api_v1_auth_check_api_token.py": 15,
+    "archivebox/tests/test_api_v1_auth_get_api_token.py": 16,
+    "archivebox/tests/test_api_v1_cli_add.py": 52,
+    "archivebox/tests/test_api_v1_cli_remove.py": 33,
+    "archivebox/tests/test_api_v1_cli_schedule.py": 29,
+    "archivebox/tests/test_api_v1_cli_search.py": 15,
+    "archivebox/tests/test_api_v1_cli_update.py": 60,
+    "archivebox/tests/test_api_v1_cli_workflow_add_search_update_remove.py": 59,
+    "archivebox/tests/test_api_v1_core_any_id.py": 13,
+    "archivebox/tests/test_api_v1_core_archiveresult_archiveresult_id.py": 24,
+    "archivebox/tests/test_api_v1_core_archiveresults.py": 19,
+    "archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py": 106,
+    "archivebox/tests/test_api_v1_core_snapshots.py": 17,
+    "archivebox/tests/test_api_v1_core_snapshots_rss.py": 16,
+    "archivebox/tests/test_api_v1_core_tag_tag_id.py": 15,
+    "archivebox/tests/test_api_v1_core_tag_tag_id_rename.py": 16,
+    "archivebox/tests/test_api_v1_core_tag_tag_id_snapshots_jsonl.py": 16,
+    "archivebox/tests/test_api_v1_core_tag_tag_id_urls_txt.py": 16,
+    "archivebox/tests/test_api_v1_core_tags.py": 15,
+    "archivebox/tests/test_api_v1_core_tags_add_to_snapshot.py": 16,
+    "archivebox/tests/test_api_v1_core_tags_autocomplete.py": 15,
+    "archivebox/tests/test_api_v1_core_tags_create.py": 15,
+    "archivebox/tests/test_api_v1_core_tags_remove_from_snapshot.py": 15,
+    "archivebox/tests/test_api_v1_core_tags_search.py": 17,
+    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py": 69,
+    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_filename.py": 15,
+    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_filename.py": 15,
+    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_subfolder_filename.py": 16,
+    "archivebox/tests/test_api_v1_crawls_crawls.py": 16,
+    "archivebox/tests/test_api_v1_machine_binaries.py": 16,
+    "archivebox/tests/test_api_v1_machine_binary_binary_id.py": 16,
+    "archivebox/tests/test_api_v1_machine_binary_by_name_name.py": 16,
+    "archivebox/tests/test_api_v1_machine_machine_current.py": 15,
+    "archivebox/tests/test_api_v1_machine_machine_machine_id.py": 16,
+    "archivebox/tests/test_api_v1_machine_machines.py": 15,
+    "archivebox/tests/test_api_v1_personas_personas.py": 15,
+    "archivebox/tests/test_api_v1_personas_sync.py": 16,
+    "archivebox/tests/test_api_v1_workflow_core_token_auth_side_effects.py": 27,
+    "archivebox/tests/test_api_v1_workflow_frozen_crawl_config_sources.py": 19,
+    "archivebox/tests/test_archive_result_service.py": 49,
+    "archivebox/tests/test_auth_ldap.py": 37,
+    "archivebox/tests/test_binary_service.py": 46,
+    "archivebox/tests/test_cli_add_1.py": 75,
+    "archivebox/tests/test_cli_add_2.py": 98,
+    "archivebox/tests/test_cli_add_3.py": 93,
+    "archivebox/tests/test_cli_add_4.py": 94,
+    "archivebox/tests/test_cli_archiveresult.py": 48,
+    "archivebox/tests/test_cli_background_screenshot.py": 49,
+    "archivebox/tests/test_cli_binary.py": 12,
+    "archivebox/tests/test_cli_config.py": 88,
+    "archivebox/tests/test_cli_crawl.py": 81,
+    "archivebox/tests/test_cli_extract.py": 97,
+    "archivebox/tests/test_cli_extract_input.py": 84,
+    "archivebox/tests/test_cli_help.py": 16,
+    "archivebox/tests/test_cli_init.py": 111,
+    "archivebox/tests/test_cli_install.py": 38,
+    "archivebox/tests/test_cli_list.py": 116,
+    "archivebox/tests/test_cli_machine.py": 19,
+    "archivebox/tests/test_cli_manage.py": 32,
+    "archivebox/tests/test_cli_mcp.py": 30,
+    "archivebox/tests/test_cli_oneshot.py": 15,
+    "archivebox/tests/test_cli_persona.py": 56,
+    "archivebox/tests/test_cli_piping.py": 118,
+    "archivebox/tests/test_cli_pluginmap.py": 12,
+    "archivebox/tests/test_cli_process.py": 12,
+    "archivebox/tests/test_cli_remove.py": 46,
+    "archivebox/tests/test_cli_run_1.py": 93,
+    "archivebox/tests/test_cli_run_2.py": 111,
+    "archivebox/tests/test_cli_run_3.py": 51,
+    "archivebox/tests/test_cli_run_4.py": 78,
+    "archivebox/tests/test_cli_schedule.py": 59,
+    "archivebox/tests/test_cli_search.py": 25,
+    "archivebox/tests/test_cli_server.py": 126,
+    "archivebox/tests/test_cli_shell.py": 17,
+    "archivebox/tests/test_cli_snapshot.py": 55,
+    "archivebox/tests/test_cli_status.py": 46,
+    "archivebox/tests/test_cli_tag.py": 12,
+    "archivebox/tests/test_cli_update.py": 62,
+    "archivebox/tests/test_cli_update_reindex_snapshots.py": 84,
+    "archivebox/tests/test_cli_version.py": 81,
+    "archivebox/tests/test_config_DELETE_AFTER.py": 145,
+    "archivebox/tests/test_config_MAX_limits.py": 25,
+    "archivebox/tests/test_config_ONLY_NEW.py": 15,
+    "archivebox/tests/test_config_SAVE_TITLE.py": 59,
+    "archivebox/tests/test_config_URL_filters.py": 15,
+    "archivebox/tests/test_core_config.py": 11,
+    "archivebox/tests/test_core_forms.py": 14,
+    "archivebox/tests/test_crawl_history.py": 60,
+    "archivebox/tests/test_crawl_pause.py": 11,
+    "archivebox/tests/test_crawl_runner.py": 22,
+    "archivebox/tests/test_crawl_runner_2.py": 27,
+    "archivebox/tests/test_crawl_service.py": 19,
+    "archivebox/tests/test_frozen_crawl_config.py": 26,
+    "archivebox/tests/test_hook_result_lifecycle.py": 48,
+    "archivebox/tests/test_hooks.py": 54,
+    "archivebox/tests/test_issue_1896.py": 115,
+    "archivebox/tests/test_machine_models.py": 26,
+    "archivebox/tests/test_machine_service.py": 77,
+    "archivebox/tests/test_migrations_04_to_09.py": 25,
+    "archivebox/tests/test_migrations_07_to_09.py": 86,
+    "archivebox/tests/test_migrations_08_to_09.py": 85,
+    "archivebox/tests/test_migrations_fresh.py": 55,
+    "archivebox/tests/test_misc_checks.py": 12,
+    "archivebox/tests/test_misc_monkey_patches.py": 12,
+    "archivebox/tests/test_opencode_agent_1.py": 73,
+    "archivebox/tests/test_opencode_agent_2.py": 141,
+    "archivebox/tests/test_opencode_agent_3.py": 61,
+    "archivebox/tests/test_opencode_browser_1.py": 139,
+    "archivebox/tests/test_opencode_browser_2.py": 164,
+    "archivebox/tests/test_opencode_browser_3.py": 125,
+    "archivebox/tests/test_orm_helpers.py": 14,
+    "archivebox/tests/test_permissions.py": 12,
+    "archivebox/tests/test_persona_casing.py": 45,
+    "archivebox/tests/test_persona_runtime.py": 30,
+    "archivebox/tests/test_postgres_backend.py": 79,
+    "archivebox/tests/test_process_runtime_paths.py": 20,
+    "archivebox/tests/test_process_service.py": 40,
+    "archivebox/tests/test_proof_previews.py": 24,
+    "archivebox/tests/test_recursive_crawl_1.py": 361,
+    "archivebox/tests/test_recursive_crawl_2.py": 55,
+    "archivebox/tests/test_recursive_crawl_3.py": 67,
+    "archivebox/tests/test_release_notes.py": 12,
+    "archivebox/tests/test_release_source_fingerprint.py": 13,
+    "archivebox/tests/test_release_workflow.py": 15,
+    "archivebox/tests/test_resource_admission.py": 13,
+    "archivebox/tests/test_runner_deletion.py": 91,
+    "archivebox/tests/test_runner_watch.py": 75,
+    "archivebox/tests/test_search.py": 142,
+    "archivebox/tests/test_serve_static.py": 19,
+    "archivebox/tests/test_server_automatic_url.py": 22,
+    "archivebox/tests/test_server_security_browser.py": 121,
+    "archivebox/tests/test_settings_signal_webhooks.py": 13,
+    "archivebox/tests/test_setup_script.py": 12,
+    "archivebox/tests/test_shutdown_util.py": 13,
+    "archivebox/tests/test_snapshot_filesystem_migration.py": 16,
+    "archivebox/tests/test_snapshot_output_stacks.py": 63,
+    "archivebox/tests/test_snapshot_ref_lookup.py": 15,
+    "archivebox/tests/test_snapshot_service.py": 48,
+    "archivebox/tests/test_snapshot_url_length.py": 16,
+    "archivebox/tests/test_tag_service.py": 15,
+    "archivebox/tests/test_takeover_util_1.py": 60,
+    "archivebox/tests/test_takeover_util_2.py": 62,
+    "archivebox/tests/test_takeover_util_3.py": 76,
+    "archivebox/tests/test_takeover_util_4.py": 84,
+    "archivebox/tests/test_test_harness.py": 12,
+    "archivebox/tests/test_ui_add_view.py": 26,
+    "archivebox/tests/test_ui_add_view_runtime.py": 120,
+    "archivebox/tests/test_ui_admin_apitoken.py": 16,
+    "archivebox/tests/test_ui_admin_archiveresult.py": 18,
+    "archivebox/tests/test_ui_admin_config_widget.py": 13,
+    "archivebox/tests/test_ui_admin_crawl.py": 25,
+    "archivebox/tests/test_ui_admin_links.py": 23,
+    "archivebox/tests/test_ui_admin_machine.py": 18,
+    "archivebox/tests/test_ui_admin_persona.py": 19,
+    "archivebox/tests/test_ui_admin_snapshot.py": 138,
+    "archivebox/tests/test_ui_admin_tag.py": 16,
+    "archivebox/tests/test_ui_admin_user.py": 16,
+    "archivebox/tests/test_ui_config_views.py": 20,
+    "archivebox/tests/test_ui_live_progress.py": 121,
+    "archivebox/tests/test_ui_public_snapshot.py": 72,
+    "archivebox/tests/test_ui_savepagenow.py": 31,
+    "archivebox/tests/test_urls.py": 105,
+    "archivebox/tests/test_util.py": 13,
+    "archivebox/tests/test_version_metadata.py": 18,
+    "archivebox/tests/test_zip_preview.py": 29
   }
 }

.github/workflows/ci.yml

@@ -45,13 +45,6 @@ jobs:
     uses: ./.github/workflows/test-parallel.yml
     secrets: inherit
 
-  documentation:
-    needs: prepare
-    if: needs.prepare.outputs.run_jobs == 'true'
-    name: Documentation and root tests
-    uses: ./.github/workflows/docs.yml
-    secrets: inherit
-
   codeql:
     needs: prepare
     if: needs.prepare.outputs.run_jobs == 'true'
@@ -83,7 +76,6 @@ jobs:
       - lint
       - install-cli-platform
       - tests
-      - documentation
       - codeql
       - python-artifacts
       - docker-digests
@@ -94,7 +86,6 @@ jobs:
           LINT_RESULT: ${{ needs.lint.result }}
           INSTALL_CLI_PLATFORM_RESULT: ${{ needs.install-cli-platform.result }}
           TESTS_RESULT: ${{ needs.tests.result }}
-          DOCUMENTATION_RESULT: ${{ needs.documentation.result }}
           CODEQL_RESULT: ${{ needs.codeql.result }}
           PYTHON_ARTIFACTS_RESULT: ${{ needs.python-artifacts.result }}
           DOCKER_DIGESTS_RESULT: ${{ needs.docker-digests.result }}
@@ -103,7 +94,6 @@ jobs:
             "$LINT_RESULT" \
             "$INSTALL_CLI_PLATFORM_RESULT" \
             "$TESTS_RESULT" \
-            "$DOCUMENTATION_RESULT" \
             "$CODEQL_RESULT" \
             "$PYTHON_ARTIFACTS_RESULT" \
             "$DOCKER_DIGESTS_RESULT"

.github/workflows/deploy-publicsite.yml

@@ -52,8 +52,17 @@ jobs:
       # END REPOSITORY-SPECIFIC BUILD
 
       - name: Verify website in Chromium
+        env:
+          ABXPKG_LIB_DIR: ${{ runner.tool_cache }}/abxpkg/site-browser
         run: |
-          uv run --no-config --no-project --with playwright==1.63.0 playwright install --with-deps chromium
+          # Resolve the same supported browser as captures; Playwright is the
+          # driver only and must not provision a separate browser/runtime.
+          ABX_DL_VERSION="$(uv run --no-project python -c 'import tomllib; print(next(p["version"] for p in tomllib.load(open("uv.lock", "rb"))["package"] if p["name"] == "abx-dl"))')"
+          CHROME_SCHEMA="$(uv run --no-project --with "abx-dl==$ABX_DL_VERSION" python -c 'from abx_plugins import get_plugins_dir; print(get_plugins_dir() / "chrome" / "config.json")')"
+          uv run --no-project --with "abx-dl==$ABX_DL_VERSION" abxpkg env --install --json --deps-from="$CHROME_SCHEMA:required_binaries" > "$RUNNER_TEMP/browser-env.json"
+          CHROME_BINARY="$(uv run --no-project python -c 'import json, sys; print(json.load(open(sys.argv[1]))["CHROME_BINARY"])' "$RUNNER_TEMP/browser-env.json")"
+          export CHROME_BINARY
+          test -x "$CHROME_BINARY"
           uv run --no-config --no-project --with playwright==1.63.0 python .github/pages/verify.py "$SITE_OUTPUT" --evidence "$RUNNER_TEMP/site-evidence"
       - name: Save website verification evidence
         if: always()

.github/workflows/docker.yml

@@ -274,24 +274,21 @@ jobs:
         if: inputs.push_digests
         run: echo ${{ steps.docker_build.outputs.digest }}
 
-      - name: Verify OpenCode install with a read-only browser profile
+      - name: Verify optional agent install with a read-only browser profile
         shell: bash
         env:
           IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
         run: |
           set -Eeuo pipefail
           PROFILE_DIR="$(mktemp -d)"
+          # shellcheck disable=SC2016
           "$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' \
             --volume "$PROFILE_DIR:/home/archivebox/.config/chromium:ro" \
             "$IMAGE" bash -c '
               set -Eeuo pipefail
               test -w "$XDG_CONFIG_HOME"
               test ! -w "$XDG_CONFIG_HOME/chromium"
-              # Check the shipped executable before the installer can repair
-              # an image that accidentally omitted this opt-in dependency.
-              "$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" --version
-              archivebox install opencode
-              "$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" --version
+              uv run --no-project python -c "from abx_plugins.plugins.opencode.image import verify_archivebox_install; verify_archivebox_install()"
             '
 
       - name: Validate exact built image version and commit
@@ -350,6 +347,7 @@ jobs:
           (( COMPRESSED_SIZE < LIMIT ))
 
       - name: Resolve real Docker CLI test dependencies through abxpkg
+        id: mount_dependencies
         if: inputs.full_tests && matrix.platform == 'linux/amd64'
         shell: bash
         run: |
@@ -371,100 +369,142 @@ jobs:
             test -x "$binary"
           done
 
-      - name: Validate Docker UID and GID behavior on real NFS and SMB mounts
-        if: inputs.full_tests && matrix.platform == 'linux/amd64'
+      - name: Diagnose failed mount dependency installation
+        if: failure() && steps.mount_dependencies.conclusion == 'failure'
         shell: bash
         run: |
-          set -Eeuo pipefail
-          NFS_SOURCE="/srv/archivebox-ci-nfs"
-          SMB_SOURCE="/srv/archivebox-ci-smb"
-          NFS_MOUNT="$RUNNER_TEMP/archivebox-nfs-mount"
-          SMB_MOUNT="$RUNNER_TEMP/archivebox-smb-mount"
-          mounted_nfs=0
-          mounted_smb=0
+          sudo tail -n 120 /var/log/apt/term.log /var/log/dpkg.log
+          sudo journalctl --since '-10 minutes' --no-pager \
+            -u smbd -u nmbd -u samba-ad-dc -u nfs-server
+          ps -eo pid,ppid,stat,etimes,wchan:32,comm
 
-          cleanup() {
-            local original_status=$? cleanup_status=0
-            trap - EXIT
-            if [[ "$mounted_smb" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$SMB_MOUNT"; then
-              cleanup_status=1
-            fi
-            if [[ "$mounted_nfs" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$NFS_MOUNT"; then
-              cleanup_status=1
-            fi
-            if ! "$SUDO_BINARY" "$SYSTEMCTL_BINARY" stop smbd.service nfs-kernel-server.service; then
-              cleanup_status=1
-            fi
-            if [[ "$original_status" != "0" ]]; then
-              exit "$original_status"
-            fi
-            exit "$cleanup_status"
-          }
-          trap cleanup EXIT
-
-          "$SUDO_BINARY" "$MKDIR_BINARY" -p "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
-          "$SUDO_BINARY" "$MKDIR_BINARY" -p /etc/exports.d
-          "$SUDO_BINARY" "$CHMOD_BINARY" 0777 "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
-          "$SUDO_BINARY" "$BASH_BINARY" -c \
-            "printf '%s *(rw,sync,no_subtree_check,root_squash,insecure)\\n' '$NFS_SOURCE' > /etc/exports.d/archivebox-ci.exports"
-          "$SUDO_BINARY" "$BASH_BINARY" -c \
-            "printf '\\n[archivebox-ci]\\npath = %s\\nbrowseable = yes\\nread only = no\\nguest ok = yes\\nguest only = yes\\nforce user = nobody\\ncreate mask = 0777\\ndirectory mask = 0777\\n' '$SMB_SOURCE' >> /etc/samba/smb.conf"
-
-          "$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart nfs-kernel-server.service
-          "$SUDO_BINARY" "$EXPORTFS_BINARY" -ra
-          "$SYSTEMCTL_BINARY" is-active --quiet nfs-kernel-server.service
-          "$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart smbd.service
-          "$SYSTEMCTL_BINARY" is-active --quiet smbd.service
-
-          "$SUDO_BINARY" "$MOUNT_NFS_BINARY" -o vers=3,nolock "127.0.0.1:$NFS_SOURCE" "$NFS_MOUNT"
-          mounted_nfs=1
-          "$SUDO_BINARY" "$MOUNT_CIFS_BINARY" //127.0.0.1/archivebox-ci "$SMB_MOUNT" \
-            -o "guest,vers=3.0,uid=0,gid=0,forceuid,forcegid,file_mode=0777,dir_mode=0777,nounix"
-          mounted_smb=1
-
-          NFS_TEST_DIR="$NFS_MOUNT" \
-          SMB_TEST_DIR="$SMB_MOUNT" \
-          IMAGE="${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}" \
-            "$BASH_BINARY" bin/validate_docker_uid_gid.sh --local-only
-
-      - name: Validate Docker port compatibility
-        shell: bash
-        env:
-          IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
-        run: '"$BASH_BINARY" bin/validate_docker_ports.sh'
-
-      - name: Validate Docker CLI add/list and compose behavior
+      - name: Validate Docker mounts, ports, CLI and compose concurrently
         if: inputs.full_tests && matrix.platform == 'linux/amd64'
         shell: bash
         env:
           ARCHIVEBOX_IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
         run: |
           set -Eeuo pipefail
-          "$MKDIR_BINARY" "${{ github.workspace }}/data"
-          "$DOCKER_BINARY" run -v "${{ github.workspace }}/data":/data "$ARCHIVEBOX_IMAGE" init
+          # These checks own separate collections, Compose projects and ports.
+          # Subshells keep mount cleanup traps and environment changes isolated.
+          validate_mounts() (
+            set -Eeuo pipefail
+            NFS_SOURCE="/srv/archivebox-ci-nfs"
+            SMB_SOURCE="/srv/archivebox-ci-smb"
+            NFS_MOUNT="$RUNNER_TEMP/archivebox-nfs-mount"
+            SMB_MOUNT="$RUNNER_TEMP/archivebox-smb-mount"
+            mounted_nfs=0
+            mounted_smb=0
 
-          "$SUDO_BINARY" "$BASH_BINARY" -c 'echo "127.0.0.1  www.test-nginx-1.local www.test-nginx-2.local" >> /etc/hosts'
-          "$DOCKER_BINARY" run --name www-nginx -p 80:80 -d nginx:1.29.5@sha256:0236ee02dcbce00b9bd83e0f5fbc51069e7e1161bd59d99885b3ae1734f3392e
+            # Invoked by the subshell EXIT trap.
+            # shellcheck disable=SC2329
+            cleanup() {
+              local original_status=$? cleanup_status=0
+              trap - EXIT
+              if [[ "$mounted_smb" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$SMB_MOUNT"; then
+                cleanup_status=1
+              fi
+              if [[ "$mounted_nfs" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$NFS_MOUNT"; then
+                cleanup_status=1
+              fi
+              if ! "$SUDO_BINARY" "$SYSTEMCTL_BINARY" stop smbd.service nfs-kernel-server.service; then
+                cleanup_status=1
+              fi
+              if [[ "$original_status" != "0" ]]; then
+                exit "$original_status"
+              fi
+              exit "$cleanup_status"
+            }
+            trap cleanup EXIT
 
-          "$DOCKER_BINARY" run -v "$PWD"/data:/data --network host "$ARCHIVEBOX_IMAGE" add http://www.test-nginx-1.local
-          echo "http://www.test-nginx-2.local" | "$DOCKER_BINARY" run -i --network host -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" add
+            "$SUDO_BINARY" "$MKDIR_BINARY" -p "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
+            "$SUDO_BINARY" "$MKDIR_BINARY" -p /etc/exports.d
+            "$SUDO_BINARY" "$CHMOD_BINARY" 0777 "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
+            "$SUDO_BINARY" "$BASH_BINARY" -c \
+              "printf '%s *(rw,sync,no_subtree_check,root_squash,insecure)\\n' '$NFS_SOURCE' > /etc/exports.d/archivebox-ci.exports"
+            "$SUDO_BINARY" "$BASH_BINARY" -c \
+              "printf '\\n[archivebox-ci]\\npath = %s\\nbrowseable = yes\\nread only = no\\nguest ok = yes\\nguest only = yes\\nforce user = nobody\\ncreate mask = 0777\\ndirectory mask = 0777\\n' '$SMB_SOURCE' >> /etc/samba/smb.conf"
 
-          archive_list="$("$DOCKER_BINARY" run -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" list)"
-          [[ "$archive_list" == *"www.test-nginx-1.local"* ]] || { echo "The site 1 isn't in the list"; exit 1; }
-          [[ "$archive_list" == *"www.test-nginx-2.local"* ]] || { echo "The site 2 isn't in the list"; exit 1; }
+            "$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart nfs-kernel-server.service
+            "$SUDO_BINARY" "$EXPORTFS_BINARY" -ra
+            "$SYSTEMCTL_BINARY" is-active --quiet nfs-kernel-server.service
+            "$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart smbd.service
+            "$SYSTEMCTL_BINARY" is-active --quiet smbd.service
 
-          "$DOCKER_BINARY" compose run --rm archivebox init
-          "$DOCKER_BINARY" compose up -d --wait
-          "$DOCKER_BINARY" network connect archivebox_default www-nginx
-          index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/public/')"
-          admin_js="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/static/admin/js/jquery.init.js')"
-          [[ "$index_html" == *ArchiveBox* ]]
-          [[ "$admin_js" == *window.django* ]]
+            "$SUDO_BINARY" "$MOUNT_NFS_BINARY" -o vers=3,nolock "127.0.0.1:$NFS_SOURCE" "$NFS_MOUNT"
+            mounted_nfs=1
+            "$SUDO_BINARY" "$MOUNT_CIFS_BINARY" //127.0.0.1/archivebox-ci "$SMB_MOUNT" \
+              -o "guest,vers=3.0,uid=0,gid=0,forceuid,forcegid,file_mode=0777,dir_mode=0777,nounix"
+            mounted_smb=1
 
-          "$DOCKER_BINARY" compose run --rm archivebox add --plugins=parse_txt_urls 'http://www-nginx/#test_docker'
-          index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/public/')"
-          [[ "$index_html" == *'http://www-nginx/#test_docker'* ]]
-          "$DOCKER_BINARY" compose down
+            NFS_TEST_DIR="$NFS_MOUNT" \
+            SMB_TEST_DIR="$SMB_MOUNT" \
+            IMAGE="${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}" \
+              "$BASH_BINARY" bin/validate_docker_uid_gid.sh --local-only
+          )
+          validate_cli() (
+            set -Eeuo pipefail
+            "$MKDIR_BINARY" "${{ github.workspace }}/data"
+            "$DOCKER_BINARY" run -v "${{ github.workspace }}/data":/data "$ARCHIVEBOX_IMAGE" init
+
+            "$SUDO_BINARY" "$BASH_BINARY" -c 'echo "127.0.0.1  www.test-nginx-1.local www.test-nginx-2.local" >> /etc/hosts'
+            "$DOCKER_BINARY" run --name www-nginx -p 80:80 -d nginx:1.29.5@sha256:0236ee02dcbce00b9bd83e0f5fbc51069e7e1161bd59d99885b3ae1734f3392e
+
+            "$DOCKER_BINARY" run -v "$PWD"/data:/data --network host "$ARCHIVEBOX_IMAGE" add http://www.test-nginx-1.local
+            # The argument form above exercises the complete default pipeline.
+            # Keep a real capture for stdin parsing without repeating every plugin.
+            echo "http://www.test-nginx-2.local" | "$DOCKER_BINARY" run -i --network host -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" add --plugins=wget,hashes
+
+            archive_list="$("$DOCKER_BINARY" run -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" list)"
+            [[ "$archive_list" == *"www.test-nginx-1.local"* ]] || { echo "The site 1 isn't in the list"; exit 1; }
+            [[ "$archive_list" == *"www.test-nginx-2.local"* ]] || { echo "The site 2 isn't in the list"; exit 1; }
+
+            "$DOCKER_BINARY" compose run --rm archivebox init
+            "$DOCKER_BINARY" compose up -d --wait
+            "$DOCKER_BINARY" network connect archivebox_default www-nginx
+            index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/public/')"
+            admin_js="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/static/admin/js/jquery.init.js')"
+            [[ "$index_html" == *ArchiveBox* ]]
+            [[ "$admin_js" == *window.django* ]]
+
+            "$DOCKER_BINARY" compose run --rm archivebox add --plugins=parse_txt_urls 'http://www-nginx/#test_docker'
+            index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/public/')"
+            [[ "$index_html" == *'http://www-nginx/#test_docker'* ]]
+            "$DOCKER_BINARY" compose down
+          )
+          validate_ports() (
+            IMAGE="$ARCHIVEBOX_IMAGE" "$BASH_BINARY" bin/validate_docker_ports.sh
+          )
+
+          checks=(mounts ports cli)
+          pids=()
+          validate_mounts > "$RUNNER_TEMP/docker-mounts.log" 2>&1 &
+          pids+=("$!")
+          validate_ports > "$RUNNER_TEMP/docker-ports.log" 2>&1 &
+          pids+=("$!")
+          validate_cli > "$RUNNER_TEMP/docker-cli.log" 2>&1 &
+          pids+=("$!")
+          status=0
+          for index in "${!checks[@]}"; do
+            check="${checks[$index]}"
+            echo "::group::Docker $check validation"
+            if wait "${pids[$index]}"; then
+              echo "PASS: Docker $check validation"
+            else
+              echo "::error::Docker $check validation failed"
+              status=1
+            fi
+            cat "$RUNNER_TEMP/docker-$check.log"
+            echo "::endgroup::"
+          done
+          exit "$status"
+
+      - name: Validate Docker port compatibility
+        if: ${{ !(inputs.full_tests && matrix.platform == 'linux/amd64') }}
+        shell: bash
+        env:
+          IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
+        run: '"$BASH_BINARY" bin/validate_docker_ports.sh'
 
       - name: Validate compressed image size
         if: inputs.full_tests && !inputs.push_digests

.github/workflows/docs-links.yml

@@ -0,0 +1,38 @@
+name: Documentation link checks
+
+on:
+  schedule:
+    - cron: '17 10 * * *' # Daily at 10:17 UTC, away from the busy top of the hour.
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+env:
+  PYTHONIOENCODING: utf-8
+
+jobs:
+  links:
+    name: Documentation links and anchors
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+        with:
+          version: "0.11.3"
+          enable-cache: false
+      - name: List tracked documentation and source inputs
+        run: |
+          git ls-files -- '*.md' '*.html' '*.py' '*.rst' '*.yml' '*.yaml' '*.sh' '*.toml' > "$RUNNER_TEMP/link-inputs.txt"
+      - uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2
+        with:
+          lycheeVersion: v0.24.2
+          args: >-
+            --config .github/lychee.toml
+            --root-dir "${{ github.workspace }}"
+            --files-from "${{ runner.temp }}/link-inputs.txt"
+          fail: true
+          failIfEmpty: true

.github/workflows/docs.yml

@@ -1,7 +1,17 @@
 name: Documentation tests
 
 on:
-  workflow_call:
+  schedule:
+    - cron: '7 5,17 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  actions: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
 
 env:
   PYTHONIOENCODING: utf-8
@@ -9,7 +19,18 @@ env:
   SHOW_PROGRESS: "False"
 
 jobs:
+  changes:
+    runs-on: ubuntu-24.04
+    outputs:
+      changed: ${{ steps.check.outputs.changed }}
+      key: ${{ steps.check.outputs.key }}
+    steps:
+      - id: check
+        uses: ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main
+
   discover:
+    needs: changes
+    if: needs.changes.outputs.changed == 'true'
     name: Discover documentation examples
     runs-on: ubuntu-24.04
     timeout-minutes: 5
@@ -158,3 +179,17 @@ jobs:
           "$SUDO_BINARY" \
             --preserve-env=PATH,ABXPKG_LIB_DIR,BASH_BINARY,PYTHON_BINARY,SUDO_BINARY,UV_BINARY \
             "$UV_BINARY" run --no-cache --no-sync python docs/test_codeblocks_manifest.py run-group '${{ matrix.snippet_ids }}'
+      - name: Diagnose failed Linux package installation
+        if: failure() && runner.os == 'Linux'
+        run: |
+          sudo tail -n 120 /var/log/apt/term.log /var/log/dpkg.log
+          apt-config dump Acquire::Retries Acquire::http::Timeout Acquire::https::Timeout
+          ps -eo pid,ppid,stat,etimes,wchan:32,comm
+
+  complete:
+    name: 'Scheduled inputs: ${{ needs.changes.outputs.key }}'
+    needs: [changes, discover, examples]
+    if: needs.changes.outputs.changed == 'true'
+    runs-on: ubuntu-24.04
+    steps:
+      - run: echo 'Scheduled work completed successfully.'

.github/workflows/macos.yml

@@ -0,0 +1,48 @@
+name: macOS compatibility
+
+on:
+  schedule:
+    - cron: '51 2,10,18 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  actions: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  changes:
+    runs-on: ubuntu-24.04
+    outputs:
+      changed: ${{ steps.check.outputs.changed }}
+      key: ${{ steps.check.outputs.key }}
+    steps:
+      - id: check
+        uses: ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main
+
+  install-cli:
+    needs: changes
+    if: needs.changes.outputs.changed == 'true'
+    uses: ./.github/workflows/test.yml
+    with:
+      platform: macos
+    secrets: inherit
+
+  python-artifacts:
+    needs: changes
+    if: needs.changes.outputs.changed == 'true'
+    uses: ./.github/workflows/pip.yml
+    with:
+      platforms: '["macos-15"]'
+    secrets: inherit
+
+  complete:
+    name: 'Scheduled inputs: ${{ needs.changes.outputs.key }}'
+    needs: [changes, install-cli, python-artifacts]
+    if: needs.changes.outputs.changed == 'true'
+    runs-on: ubuntu-24.04
+    steps:
+      - run: echo 'Scheduled macOS coverage completed successfully.'

.github/workflows/pip.yml

@@ -3,6 +3,11 @@ name: Build Pip package
 on:
   workflow_call:
     inputs:
+      platforms:
+        description: Runner matrix for package compatibility
+        required: false
+        default: '["ubuntu-24.04"]'
+        type: string
       full_tests:
         description: Run the full package compatibility matrix instead of one release smoke check
         required: false
@@ -95,7 +100,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-24.04, macos-15]
+        os: ${{ fromJSON(inputs.platforms) }}
         python: ["3.13", "3.14"]
         artifact: [wheel, sdist]
 

.github/workflows/release.yml

@@ -392,7 +392,8 @@ jobs:
 
       - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
         with:
-          install: true
+          # These images were already built and tested; only publish manifests.
+          driver: docker
 
       - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
         with:
@@ -525,17 +526,27 @@ jobs:
           for digest in "${DIGESTS[@]}"; do REFS+=("${GHCR_IMAGE}@sha256:${digest}"); done
           $DOCKER_BINARY buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
 
-      - name: Verify published Docker images run
+      - name: Verify published tags contain the exact tested images
+        env:
+          DOCKERHUB_TAGS: ${{ steps.docker_meta.outputs.dockerhub_tags }}
+          GHCR_TAGS: ${{ steps.docker_meta.outputs.ghcr_tags }}
         shell: bash
         run: |
           set -Eeuo pipefail
-          VERSION='${{ needs.python-release.outputs.version }}'
-          SHORT_SHA="${RELEASE_SHA::12}"
-          for image in "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}" "${GHCR_IMAGE}:sha-${SHORT_SHA}"; do
-            "$DOCKER_BINARY" buildx imagetools inspect "$image" >/dev/null
-            INSTALLED_VERSION="$("$DOCKER_BINARY" run --rm "$image" --version)"
-            test "$INSTALLED_VERSION" = "$VERSION"
-          done
+          # Both architectures already passed CLI/version/capture acceptance by
+          # digest. Verify promotion preserves those exact images and attestations
+          # instead of downloading and running the same image again on one arch.
+          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'import re; from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file() and re.fullmatch(r"[0-9a-f]{64}", path.name))))')
+          REFS=()
+          for digest in "${DIGESTS[@]}"; do REFS+=("${DOCKERHUB_IMAGE}@sha256:${digest}"); done
+          EXPECTED="$("$DOCKER_BINARY" buildx imagetools create --dry-run "${REFS[@]}" | "$JQ_BINARY" -ceS '.manifests | sort_by(.digest)')"
+          "$JQ_BINARY" -e 'map(select(.platform.os == "linux") | .platform.architecture) | sort == ["amd64", "arm64"]' <<< "$EXPECTED"
+          while IFS= read -r image; do
+            [[ -n "$image" ]] || continue
+            ACTUAL="$("$DOCKER_BINARY" buildx imagetools inspect --raw "$image" | "$JQ_BINARY" -ceS 'select(.schemaVersion == 2) | .manifests | sort_by(.digest)')"
+            [[ "$ACTUAL" == "$EXPECTED" ]] || { echo "Published tag differs from tested images: $image" >&2; exit 1; }
+            echo "Verified exact tested images: $image"
+          done < <(printf '%s\n' "$DOCKERHUB_TAGS" "$GHCR_TAGS")
 
       - name: Record published image source
         shell: bash

.github/workflows/screenshots.yml

@@ -1,38 +1,31 @@
 name: Capture publicsite screenshots
 
 on:
-  push:
-    branches:
-      - dev
-    paths:
-      - archivebox/**
-      - bin/collect_ui_screenshots.*
-      - bin/generate_ui_screenshot_gallery.py
-      - bin/take_screenshot.js
-      - docs/Screenshots.md
-      - pyproject.toml
-      - uv.lock
-      - .github/workflows/screenshots.yml
-      - .github/workflows/prepare-release.yml
+  schedule:
+    - cron: '37 5,17 * * *'
   workflow_dispatch:
 
 permissions:
   contents: read
+  actions: read
 
 concurrency:
-  group: ${{ github.event_name == 'workflow_dispatch' && format('publicsite-captures-manual-{0}', github.run_id) || 'publicsite-captures' }}
+  group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: false
 
 jobs:
-  prepare:
-    permissions:
-      contents: write
-    uses: ./.github/workflows/prepare-release.yml
-    secrets: inherit
+  changes:
+    runs-on: ubuntu-24.04
+    outputs:
+      changed: ${{ steps.check.outputs.changed }}
+      key: ${{ steps.check.outputs.key }}
+    steps:
+      - id: check
+        uses: ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main
 
   deploy:
-    needs: prepare
-    if: needs.prepare.outputs.run_jobs == 'true'
+    needs: changes
+    if: needs.changes.outputs.changed == 'true'
     runs-on: ubuntu-24.04
     steps:
       - name: Checkout
@@ -78,3 +71,11 @@ jobs:
           path: publicsite/screenshots
           if-no-files-found: error
           retention-days: 90
+
+  complete:
+    name: 'Scheduled inputs: ${{ needs.changes.outputs.key }}'
+    needs: [changes, deploy]
+    if: needs.changes.outputs.changed == 'true'
+    runs-on: ubuntu-24.04
+    steps:
+      - run: echo 'Scheduled work completed successfully.'

.github/workflows/staging-acceptance.yml

@@ -84,12 +84,20 @@ jobs:
         env:
           STAGING_SSH_KEY: ${{ secrets.STAGING_SSH_KEY }}
           STAGING_KNOWN_HOSTS: ${{ secrets.STAGING_KNOWN_HOSTS }}
+          ABXPKG_LIB_DIR: ${{ runner.tool_cache }}/abxpkg/staging-browser
         run: |
           set -Eeuo pipefail
           umask 077
           printf '%s\n' "$STAGING_SSH_KEY" > "$RUNNER_TEMP/staging-key"
           printf '%s\n' "$STAGING_KNOWN_HOSTS" > "$RUNNER_TEMP/staging-known-hosts"
-          uv run --no-project --with playwright==1.63.0 playwright install chromium
+          # Use the candidate's normal provider/cache path, not a second
+          # Playwright-managed browser that bypasses abxpkg validation.
+          ABX_DL_VERSION="$(uv run --no-project python -c 'import tomllib; print(next(p["version"] for p in tomllib.load(open("uv.lock", "rb"))["package"] if p["name"] == "abx-dl"))')"
+          CHROME_SCHEMA="$(uv run --no-project --with "abx-dl==$ABX_DL_VERSION" python -c 'from abx_plugins import get_plugins_dir; print(get_plugins_dir() / "chrome" / "config.json")')"
+          uv run --no-project --with "abx-dl==$ABX_DL_VERSION" abxpkg env --install --json --deps-from="$CHROME_SCHEMA:required_binaries" > "$RUNNER_TEMP/browser-env.json"
+          CHROME_BINARY="$(uv run --no-project python -c 'import json, sys; print(json.load(open(sys.argv[1]))["CHROME_BINARY"])' "$RUNNER_TEMP/browser-env.json")"
+          test -x "$CHROME_BINARY"
+          echo "CHROME_BINARY=$CHROME_BINARY" >> "$GITHUB_ENV"
       - name: Verify actual deployment, full captures and rendered snapshots
         env:
           GH_TOKEN: ${{ github.token }}

.github/workflows/test-parallel.yml

@@ -36,16 +36,9 @@ jobs:
       - name: Select available CI capacity
         id: runner
         if: vars.UGNAS_CI_ENABLED == 'true' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main')
-        uses: mikehardy/runner-fallback-action@dc7732f9e532c451b2527b288ba8d58fd4b0a4ca # v1.1.0
+        uses: ArchiveBox/monorepo/.github/actions/select-ci-runner@main
         with:
           github-token: ${{ secrets.RUNNER_STATUS_TOKEN }}
-          organization: ArchiveBox
-          primary-runner: self-hosted,ugnas-linux
-          fallback-runner: ubuntu-24.04
-          # Jobs queued for this pool must not be mistaken for idle runner slots:
-          # requiring MAX_JOBS idle runners prevents assigning a second wave.
-          primaries-required: ${{ vars.UGNAS_CI_MIN_IDLE || 1 }}
-          fallback-on-error: true
 
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
@@ -106,7 +99,7 @@ jobs:
     needs: discover-tests
     # Opt in trusted branches only; PRs always use GitHub-hosted runners.
     # Ordinary Linux tests are eligible; # ci-runner: hosted declares exceptions.
-    # The hosted discovery job falls back when workers are offline/busy or the
+    # The hosted discovery job falls back when workers are offline or the
     # availability API fails. UGNAS_CI_ENABLED=false disables the pool entirely.
     runs-on: ${{ fromJSON(matrix.test.ugnas && needs.discover-tests.outputs.runner || '["ubuntu-24.04"]') }}
     env:
@@ -119,8 +112,21 @@ jobs:
 
     steps:
       - name: Set job-local runtime paths
+        env:
+          CI_RUNNER_ENVIRONMENT: ${{ runner.environment }}
         run: |
+          workers=2
+          cache_suffix=''
+          if [[ "$CI_RUNNER_ENVIRONMENT" == github-hosted ]]; then
+            workers="$(nproc)"
+            cache_suffix="-workers-$workers"
+            # Avoid the hosted image's cold Chromium executable page faults.
+            # abxpkg owns installation and reuse in the existing runtime cache.
+            echo "CHROME_BINPROVIDERS=playwright,puppeteer" >> "$GITHUB_ENV"
+          fi
           {
+            echo "CI_TEST_WORKERS=$workers"
+            echo "ABXPKG_CACHE_WORKERS_SUFFIX=$cache_suffix"
             echo "ABXPKG_LIB_DIR=$RUNNER_TEMP/archivebox-lib"
             echo "DATA_DIR=$RUNNER_TEMP/archivebox-test-data"
             echo "PERSONAS_DIR=$RUNNER_TEMP/abx-personas"
@@ -136,6 +142,7 @@ jobs:
         with:
           python-version: ${{ env.CI_PYTHON_VERSION }}
           architecture: x64
+        id: python
 
       - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
         with:
@@ -143,6 +150,41 @@ jobs:
           enable-cache: false
           cache-dependency-glob: uv.lock
 
+      # Self-hosted tool storage survives jobs. Normal abxpkg validation and
+      # install locks allow reuse across files; cold-install tests use tmp_path.
+      - name: Reuse persistent runner tools
+        if: runner.environment == 'self-hosted'
+        env:
+          PERSISTENT_ABXPKG_LIB: ${{ runner.tool_cache }}/abxpkg/${{ github.repository }}/${{ steps.python.outputs.python-version }}
+        run: |
+          mkdir -p "$PERSISTENT_ABXPKG_LIB"
+          echo "ABXPKG_LIB_DIR=$PERSISTENT_ABXPKG_LIB" >> "$GITHUB_ENV"
+
+      # Restored binaries still pass normal abxpkg resolution/version checks.
+      # Installation and isolation tests explicitly choose fresh temporary libs.
+      - name: Identify runtime dependency cache
+        shell: bash
+        run: |
+          uv run --no-project python - <<'PYTHON' >> "$GITHUB_ENV"
+          import hashlib
+          import json
+          import tomllib
+          from pathlib import Path
+
+          packages = tomllib.loads(Path("uv.lock").read_text())["package"]
+          versions = {p["name"]: p["version"] for p in packages if p["name"] in {"abxpkg", "abx-plugins"} and "editable" not in p["source"]}
+          print("ABXPKG_CACHE_DEPS=" + hashlib.sha256(json.dumps(versions, sort_keys=True).encode()).hexdigest())
+          PYTHON
+
+      - name: Reuse validated runtime dependencies
+        if: runner.environment != 'self-hosted'
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+        with:
+          path: ${{ runner.temp }}/archivebox-lib
+          key: abxpkg-v1-${{ runner.os }}-${{ runner.arch }}-${{ runner.environment }}-${{ steps.python.outputs.python-version }}-${{ matrix.test.name }}-${{ env.ABXPKG_CACHE_DEPS }}${{ env.ABXPKG_CACHE_WORKERS_SUFFIX }}
+          restore-keys: |
+            abxpkg-v1-${{ runner.os }}-${{ runner.arch }}-${{ runner.environment }}-${{ steps.python.outputs.python-version }}-${{ matrix.test.name }}-
+
       - name: Resolve optional build dependencies through abxpkg
         if: matrix.test.extra != '' || contains(matrix.test.paths_arg, 'test_postgres_backend')
         env:
@@ -176,35 +218,25 @@ jobs:
             extra_args=(--extra "$TEST_EXTRA")
           fi
           uv sync --no-cache --locked --dev "${extra_args[@]}"
-      - name: Install Playwright Chromium for browser tests
-        if: contains(matrix.test.paths_arg, 'test_proof_previews') || contains(matrix.test.paths_arg, 'test_snapshot_output_stacks') || contains(matrix.test.paths_arg, 'test_ui_admin_snapshot') || contains(matrix.test.paths_arg, 'test_cli_extract')
-        run: uv run --no-cache --no-sync --no-sources playwright install chromium
-
       - name: Run ${{ matrix.test.name }}
         env:
           TEST_PATHS: ${{ matrix.test.paths_arg }}
+          CI_RUNNER_ENVIRONMENT: ${{ runner.environment }}
         run: |
           set -Eeuo pipefail
+          echo "Running pytest with $CI_TEST_WORKERS workers on $CI_RUNNER_ENVIRONMENT"
           read -r -a test_paths <<< "$TEST_PATHS"
+          test_args=()
           for test_path in "${test_paths[@]}"; do
             test -f "$test_path"
+            test_args+=("$GITHUB_WORKSPACE/$test_path")
           done
-          test_status=0
-          for test_path in "${test_paths[@]}"; do
-            test_name="$(basename "$test_path" .py)"
-            test_slug="${test_name//[^A-Za-z0-9_.-]/_}"
-            export DATA_DIR="$RUNNER_TEMP/archivebox-test-data/$test_slug"
-            export PERSONAS_DIR="$RUNNER_TEMP/abx-personas/$test_slug"
-            export CHROME_USER_DATA_DIR="$PERSONAS_DIR/Default/chrome_profile"
-            if (( ${#test_paths[@]} > 1 )); then
-              export ABXPKG_LIB_DIR="$RUNNER_TEMP/archivebox-lib/$test_slug"
-            fi
-            mkdir -p "$DATA_DIR" "$PERSONAS_DIR" "$RUNNER_TEMP/pytest/$test_slug"
-            uv run --no-cache --no-sync --no-sources pytest -vs "$test_path" --durations=20 \
-              --basetemp="$RUNNER_TEMP/pytest/$test_slug" \
-              --junitxml="$RUNNER_TEMP/test-results/$test_slug.xml" || test_status=1
-          done
-          exit "$test_status"
+          # Each test still gets isolated DB/filesystem state from conftest.
+          # Keep one worker pool per batch instead of starting it for every file.
+          mkdir -p "$DATA_DIR" "$PERSONAS_DIR" "$RUNNER_TEMP/pytest"
+          uv run --no-cache --no-sync --no-sources pytest -vs -n "$CI_TEST_WORKERS" "${test_args[@]}" --durations=20 \
+            --basetemp="$RUNNER_TEMP/pytest" \
+            --junitxml="$RUNNER_TEMP/test-results/results.xml"
       - name: Save test results
         if: always()
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4

.github/workflows/test.yml

@@ -2,6 +2,12 @@ name: Install and CLI Compatibility
 
 on:
   workflow_call:
+    inputs:
+      platform:
+        description: Platform for install and CLI compatibility
+        required: false
+        default: linux
+        type: string
 
 env:
   DOCKER_IMAGE: archivebox-ci
@@ -19,16 +25,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - os: ubuntu-24.04
-            os_name: linux
-            python: "3.14"
-          - os: macos-15
-            os_name: macOS
-            python: "3.13"
-          - os: macos-15
-            os_name: macOS
-            python: "3.14"
+        include: ${{ fromJSON(inputs.platform == 'macos' && '[{"os":"macos-15","os_name":"macOS","python":"3.13"},{"os":"macos-15","os_name":"macOS","python":"3.14"}]' || '[{"os":"ubuntu-24.04","os_name":"linux","python":"3.14"}]') }}
 
     steps:
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4

Dockerfile

@@ -3,14 +3,14 @@
 # Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image.
 # WHY: abx-dl owns Python, Node, Chromium, and extraction dependencies so the
 # standalone downloader stays independent of ArchiveBox's server features.
-# ArchiveBox-only dependencies (Sonic, OpenCode, supervisor, Django) belong in
+# ArchiveBox-only dependencies (server plugins, supervisor, Django) belong in
 # the layers added by this Dockerfile, never in the shared abx-dl base image.
 # REQUIRED INSTALLATION CONTRACT — do not work around failures:
 # 1. IMAGE CONSTRUCTION: preinstall ALL plugins supported by runtime="archivebox"
 #    and ALL their required_binaries, regardless of default enabled state.
 #    Inherit the complete downloader dependencies from the tested abx-dl image;
 #    install every remaining ArchiveBox-only dependency in this image's layers,
-#    including OpenCode and Sonic. Never put server dependencies in abx-dl.
+#    including optional server plugins. Never put server dependencies in abx-dl.
 #    Use the normal plugin dependency pipeline: install command -> config.json
 #    required_binaries -> abxpkg. Never add manual apt/pip/npm installs to hide
 #    failures in that pipeline. OS/interpreter bootstrap is a separate concern,
@@ -156,6 +156,22 @@ apt-get install -qq -y --no-install-recommends x11-utils
     --no-install-package abx-plugins \
     --no-install-package abx-dl \
     --no-sources
+# Sync may replace inherited dependencies (e.g. pydantic/rich-click), removing
+# their bytecode. Runtime writes are disabled, so missing caches would make
+# every hook compile them again. Compile only missing entries: compileall's
+# timestamp fast path rewrites even valid checked-hash caches, copying the
+# entire inherited venv into this OCI layer for no runtime benefit.
+/usr/bin/uv run --no-project --no-sync python - <<'PY'
+import importlib.util
+from pathlib import Path
+import py_compile
+import sysconfig
+
+for source in Path(sysconfig.get_path("purelib")).rglob("*.py"):
+    cache = Path(importlib.util.cache_from_source(str(source)))
+    if not cache.is_file():
+        py_compile.compile(str(source), doraise=True, invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH)
+PY
 builder_abxpkg_lib_dir=/tmp/archivebox-builder-abxpkg
 ABXPKG_NO_CACHE=True abxpkg env --install --binproviders=env,apt --lib="$builder_abxpkg_lib_dir" --overrides='{"apt":{"install_args":["binutils"]}}' strip >/dev/null
 /usr/bin/find /venv/lib/python3.*/site-packages -type f -name '*.so' -print0 > /tmp/archivebox-native-libraries
@@ -250,15 +266,13 @@ RUN --mount=type=cache,target=/opt/archivebox/lib/cache,sharing=locked,mode=1777
 
 # Install the complete ArchiveBox catalog, including disabled server plugins.
 # Explicit plugin names make installation independent of enabled defaults.
-RUN --mount=type=cache,target=/var/tmp/abxpkg-cache,sharing=locked,mode=1777,id=archivebox-opencode-$TARGETARCH \
+RUN --mount=type=cache,target=/var/tmp/abxpkg-cache,sharing=locked,mode=1777,id=archivebox-server-plugins-$TARGETARCH \
     chown -R "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" /var/tmp/abxpkg-cache \
     && chmod 1777 /var/tmp/abxpkg-cache \
     && export ABX_DOCKER_PLUGINS="$(/venv/bin/python3 -c 'from abx_dl.catalog import PluginCatalog; print(" ".join(PluginCatalog.discover(runtime="archivebox")))')" \
     && env ABXPKG_TMP_CACHE_DIR=/var/tmp/abxpkg-cache XDG_CACHE_HOME=/var/tmp/abxpkg-cache \
         setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abx-dl install $ABX_DOCKER_PLUGINS \
-    # pnpm includes musl variants that Debian's glibc runtime cannot use.
-    && find "$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules" -type l -name 'opencode-linux-*-musl' -delete \
-    && find "$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.pnpm" -maxdepth 1 -type d -name 'opencode-linux-*-musl@*' -exec rm -rf {} + \
+    && UV_NO_CACHE=true /usr/bin/uv run --no-project python -c 'from abx_plugins.plugins.opencode.image import prune_incompatible_image_files; prune_incompatible_image_files()' \
     # pnpm's explicit store lives under ABXPKG_LIB_DIR, regardless of XDG_CACHE_HOME.
     # Retaining its download copies made image size depend on whether the build
     # preserved hardlinks to installed files. Remove the store in this layer;
@@ -267,7 +281,7 @@ RUN --mount=type=cache,target=/var/tmp/abxpkg-cache,sharing=locked,mode=1777,id=
     && rm -rf "$ABXPKG_LIB_DIR/cache/pnpm"
 RUN --network=none export ABX_DOCKER_PLUGINS="$(/venv/bin/python3 -c 'from abx_dl.catalog import PluginCatalog; print(" ".join(PluginCatalog.discover(runtime="archivebox")))')" \
     && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups \
-    bash -c '"$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" --version && abx-dl install $ABX_DOCKER_PLUGINS'
+    bash -c 'uv run --no-project python -c "from abx_plugins.plugins.opencode.image import verify_installed; verify_installed()" && abx-dl install $ABX_DOCKER_PLUGINS'
 
 WORKDIR "$DATA_DIR"
 RUN echo "[+] Initializing image collection..." \
@@ -287,6 +301,9 @@ RUN chmod +x "$CODE_DIR"/bin/*.sh \
     && chmod g+w "$TMP_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"
 
 RUN for forbidden_bin in gcc g++ make; do ! abxpkg load --binproviders=env "$forbidden_bin" >/dev/null 2>&1 || (echo "Unexpected build tool in runtime: $forbidden_bin" >&2 && exit 1); done \
+    && test -f "$(/venv/bin/python -c 'import pydantic; print(pydantic.__cached__)')" \
+    && test -f "$(/venv/bin/python -c 'import rich_click; print(rich_click.__cached__)')" \
+    && test -f "$(/venv/bin/python -c 'import platformdirs; print(platformdirs.__cached__)')" \
     && stat -c "%U:%G %a %n" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
     && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$CONFIG_DIR" \
     && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$ABXPKG_LIB_DIR" \

README.md

@@ -123,13 +123,13 @@ curl -fsSL 'https://get.archivebox.io' | bash
 ## Key Features
 
 - [**Free & open source**](https://github.com/ArchiveBox/ArchiveBox/blob/dev/LICENSE), own your own data & maintain your privacy by self-hosting
-- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) with [modular dependencies](#dependencies) and [support for Google Drive/NFS/SMB/S3/B2/etc.](https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage)
+- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage) with [modular dependencies](#dependencies) and [support for Google Drive/NFS/SMB/S3/B2/etc.](https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage)
 - [**Comprehensive documentation**](https://github.com/ArchiveBox/ArchiveBox/wiki), [active development](https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap), and [rich community](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community)
 - [**Extracts a wide variety of content out-of-the-box**](https://github.com/ArchiveBox/ArchiveBox/issues/51): [media (yt-dlp), articles (readability), code (git), etc.](#output-formats)
 - [**Supports scheduled/realtime importing**](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving) from [many types of sources](#input-formats)
 - [**Uses standard, durable, long-term formats**](#output-formats) like HTML, JSON, PDF, PNG, MP4, TXT, and WARC
-- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [Python API](https://docs.archivebox.io/en/dev/apidocs/archivebox/archivebox.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://electron.archivebox.io/)
-- [**Saves all pages to archive.org as well**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#save_archive_dot_org) by default for redundancy (can be [disabled](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#stealth-mode) for local-only mode)
+- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#ui-usage), [Python API](https://docs.archivebox.io/dev/apidocs/archivebox/archivebox.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://electron.archivebox.io/)
+- [**Saves all pages to archive.org as well**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#archivedotorg_enabled) by default for redundancy (can be [disabled](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#archivedotorg_enabled) for local-only mode)
 - Support for archiving [content requiring login/paywall/cookies](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir) (see wiki security caveats!)
 - Support for running [JS during archiving](https://github.com/ArchiveBox/ArchiveBox/issues/51) to adblock, [infiniscroll](https://github.com/ArchiveBox/ArchiveBox/issues/80), [modal-hide](https://github.com/ArchiveBox/ArchiveBox/issues/175), [thread-expand](https://github.com/ArchiveBox/ArchiveBox/issues/345)
 
@@ -220,7 +220,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <ol>
 <li>Run the shortcut for the <code>uv</code> install method. It installs <code>uv</code> first when needed.
 <pre lang="bash"><code style="white-space: pre-line">curl -fsSL 'https://get.archivebox.io' | bash</code></pre>
-<i>For more info, see <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-b-automatic-setup-script">Install: Bare Metal</a> in the Wiki. ➡️</i>
+<i>For more info, see <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-b-uv-install-shortcut">Install: Bare Metal</a> in the Wiki. ➡️</i>
 </li>
 </ol>
 
@@ -431,7 +431,7 @@ For more discussion on third-party hosting options see here: <a href="https://gi
 #### ➡️&nbsp; Next Steps
 
 - Import URLs from some of the supported [Input Formats](#input-formats) or view the supported [Output Formats](#output-formats)...
-- (Optional) [Import browser cookies and settings into a persona](docs/Chromium-Install.md#import-an-existing-browser-profile).
+- (Optional) [Import browser cookies and settings into a persona](https://github.com/ArchiveBox/ArchiveBox/blob/dev/docs/Chromium-Install.md#import-an-existing-browser-profile).
 - Tweak your UI or archiving behavior [Configuration](#configuration), read about some of the [Caveats](#caveats), or [Troubleshoot](https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting)
 - Read about the [Dependencies](#dependencies) used for archiving, the [Upgrading Process](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives), or the [Archive Layout](#archive-layout) on disk...
 - Or check out our full [Documentation](#documentation) or [Community Wiki](#internet-archiving-ecosystem)...
@@ -613,7 +613,7 @@ docker run -it -v $PWD:/data archivebox/archivebox:dev add --depth=1 'https://ex
 <sub>. . . . . . . . . . . . . . . . . . . . . . . . . . . .</sub>
 <br/><br/>
 <a href="https://demo.archivebox.io">DEMO: <code>https://demo.archivebox.io</code></a><br/>
-<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration">Configuration</a> | <a href="#Caveats">Caveats</a>
+<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration">Configuration</a> | <a href="#caveats">Caveats</a>
 <br/>
 </div>
 
@@ -641,7 +641,7 @@ docker run -it -v $PWD:/data archivebox/archivebox:dev add --depth=1 'https://ex
   <i>Save directly from the Share menu or Siri/Shortcuts to your server.</i>
 
 - <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/64078483-21d7-4eb1-aa6e-9ad55afe45b8" height="22px"/> From manual imports of URLs from RSS, JSON, CSV, TXT, SQL, HTML, Markdown, etc. files  
-  <i>ArchiveBox supports injesting URLs in [any text-based format](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Import-a-list-of-URLs-from-a-text-file).</i>
+  <i>ArchiveBox supports injesting URLs in [any text-based format](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#import-a-list-of-urls-from-a-text-file).</i>
 
 - <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/32b494e6-4de1-4984-8d88-dc02f18e5c34" height="22px"/> From manually exported [browser history](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) or [browser bookmarks](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) (in Netscape format)  
   <i>Instructions: <a href="https://support.google.com/chrome/answer/96816?hl=en">Chrome</a>, <a href="https://support.mozilla.org/en-US/kb/export-firefox-bookmarks-to-backup-or-transfer">Firefox</a>, <a href="https://github.com/ArchiveBox/ArchiveBox/assets/511499/24ad068e-0fa6-41f4-a7ff-4c26fc91f71a">Safari</a>, <a href="https://support.microsoft.com/en-us/help/211089/how-to-import-and-export-the-internet-explorer-favorites-folder-to-a-32-bit-version-of-windows">IE</a>, <a href="https://help.opera.com/en/latest/features/#bookmarks:~:text=Click%20the%20import/-,export%20button,-on%20the%20bottom">Opera</a>, <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive">and more...</a></i>
@@ -670,7 +670,7 @@ echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox a
 # echo 'https://example.com' | docker compose run --rm -T archivebox add
 ```
 
-See the [Usage: CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) page for documentation and examples.
+See the [Usage: CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage) page for documentation and examples.
 
 It also includes a built-in scheduled import feature with `archivebox schedule`, handled by the same orchestrator that powers `archivebox server`, so you can pull in URLs from RSS feeds and websites regularly without a separate cron container.
 
@@ -811,7 +811,7 @@ Installing directly on <strong>Windows without Docker or WSL/WSL2/Cygwin is not
 
 <h4>Learn More</h4>
 <ul>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#dependencies">Wiki: Install (Dependencies)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#3-install-runtime-dependencies-using-archivebox-install">Wiki: Install (Dependencies)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install">Wiki: Chromium Install</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives">Wiki: Upgrading or Merging Archives</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#installing">Wiki: Troubleshooting (Installing)</a></li>
@@ -865,7 +865,7 @@ Each snapshot subfolder includes static metadata and plain extractor output file
 <h4>Learn More</h4>
 <ul>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage">Wiki: Setting Up Storage (SMB, NFS, S3, B2, Google Drive, etc.)</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">Wiki: Usage (Disk Layout)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout">Wiki: Usage (Disk Layout)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#large-archives">Wiki: Usage (Large Archives)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#output-folder">Wiki: Security Overview (Output Folder)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive">Wiki: Publishing Your Archive</a></li>
@@ -951,7 +951,7 @@ archivebox manage createsuperuser
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive">Wiki: Publishing Your Archive</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview">Wiki: Security Overview</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile">Wiki: Chromium Install (Setting Up a User Profile)</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#import-an-existing-browser-profile">Wiki: Chrome profiles and cookies</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/docs/Chromium-Install.md#import-an-existing-browser-profile">Wiki: Chrome profiles and cookies</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#default_persona">Wiki: Configuration (<code>DEFAULT_PERSONA</code>)</a></li>
 </ul>
 
@@ -1004,7 +1004,7 @@ For various reasons, many large sites (Reddit, Twitter, Cloudflare, etc.) active
 
 
 <ul>
-<li>Set up a logged-in browser session for archiving by <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#import-an-existing-browser-profile">importing a chrome profile into a persona</a></li>
+<li>Set up a logged-in browser session for archiving by <a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/docs/Chromium-Install.md#import-an-existing-browser-profile">importing a chrome profile into a persona</a></li>
 <li>Rewrite your URLs before archiving to swap in alternative frontends that are more bot-friendly e.g.<br>
 <code>reddit.com/some/url</code> -&gt; <code>teddit.net/some/url</code>: <a href="https://github.com/mendel5/alternative-front-ends">https://github.com/mendel5/alternative-front-ends</a></li>
 </ul>
@@ -1065,7 +1065,7 @@ Because ArchiveBox is designed to ingest a large volume of URLs with multiple co
 <h4>Learn More</h4>
 
 <ul>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">Wiki: Usage (Disk Layout)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout">Wiki: Usage (Disk Layout)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#output-folder">Wiki: Security Overview (Output-Folder)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#large-archives">Wiki: Usage (Large Archives)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#run-archivebox-as-an-unprivileged-user">Wiki: Security Overview (Privilege Dropping)</a></li>
@@ -1190,7 +1190,7 @@ By having users store their content locally or within their organizations, we ca
 
 <h3>Comparison With Other Self-Hosted Archiving Options</h3>
 
-ArchiveBox differentiates itself from [similar self-hosted projects](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#Web-Archiving-Projects) by providing both a comprehensive CLI interface for managing your archive, a Web UI that can be used either independently or together with the CLI, and a simple on-disk data format that can be used without either.
+ArchiveBox differentiates itself from [similar self-hosted projects](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#web-archiving-projects) by providing both a comprehensive CLI interface for managing your archive, a Web UI that can be used either independently or together with the CLI, and a simple on-disk data format that can be used without either.
 
 
 *If you want better fidelity for very complex interactive pages with heavy JS/streams/API requests, check out [ArchiveWeb.page](https://archiveweb.page) and [ReplayWeb.page](https://replayweb.page).*
@@ -1199,7 +1199,7 @@ ArchiveBox differentiates itself from [similar self-hosted projects](https://git
 
 *If you need more advanced recursive spider/crawling ability beyond `--depth=1`, check out [Browsertrix](https://github.com/webrecorder/browsertrix-crawler), [Photon](https://github.com/s0md3v/Photon), or [Scrapy](https://scrapy.org/) and pipe the outputted URLs into ArchiveBox.*
 
-For more alternatives, see our [list here](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#Web-Archiving-Projects)...
+For more alternatives, see our [list here](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#web-archiving-projects)...
 
 ArchiveBox is neither the highest fidelity nor the simplest tool available for self-hosted archiving, rather it's a jack-of-all-trades that tries to do most things well by default. We encourage you to try these other tools made by our friends if ArchiveBox isn't suited to your needs.
 
@@ -1254,7 +1254,7 @@ We use the [ArchiveBox GitHub Wiki](https://github.com/ArchiveBox/ArchiveBox/wik
 - [Usage](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage)
 - [Configuration](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration)
 - [Supported Sources](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive)
-- [Supported Outputs](https://github.com/ArchiveBox/ArchiveBox/wiki#can-save-these-things-for-each-site)
+- [Supported Outputs](https://github.com/ArchiveBox/ArchiveBox#output-formats)
 - [Scheduled Archiving](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving)
 
 ## Advanced

archivebox/api/v1_api.py

@@ -2,6 +2,8 @@ __package__ = "archivebox.api"
 
 
 from io import StringIO
+from functools import lru_cache
+from typing import Any
 from traceback import format_exception
 from contextlib import redirect_stdout, redirect_stderr
 
@@ -10,6 +12,7 @@ from django.core.exceptions import ObjectDoesNotExist, EmptyResultSet, Permissio
 from django.contrib.auth.models import User
 
 from ninja import NinjaAPI, Swagger
+from ninja.openapi.schema import OpenAPISchema
 
 # TODO: explore adding https://eadwincode.github.io/django-ninja-extra/
 
@@ -49,6 +52,16 @@ def register_urls(api: NinjaAPI) -> NinjaAPI:
 
 
 class NinjaAPIWithIOCapture(NinjaAPI):
+    def get_openapi_schema(self, *, path_prefix: str | None = None, path_params: dict[str, Any] | None = None) -> OpenAPISchema:
+        prefix = path_prefix if path_prefix is not None else self.get_root_path(path_params or {})
+        return self._schema_for_prefix(prefix)
+
+    @lru_cache(maxsize=8)
+    def _schema_for_prefix(self, path_prefix: str) -> OpenAPISchema:
+        # Routes are registered at startup. Server discovery can reuse their
+        # schema while preserving distinct URL prefixes for mounted instances.
+        return super().get_openapi_schema(path_prefix=path_prefix)
+
     def create_temporal_response(self, request: HttpRequest) -> HttpResponse:
         stdout, stderr = StringIO(), StringIO()
 

archivebox/api/v1_core.py

@@ -211,11 +211,22 @@ class ArchiveResultFilterSchema(FilterSchema):
     created_at__lt: Annotated[datetime | None, FilterLookup("created_at__lt")] = None
 
 
+def _with_archiveresult_processes(queryset):
+    # cmd/pwd/version delegate to Process and Binary. Without this join a full
+    # capture's metadata response performs two extra indexed lookups per hook
+    # (roughly 100 queries), despite the initial Snapshot UUID lookup being fast.
+    # Logs and the hydrated hook environment are not response fields; loading
+    # those potentially large values would replace round trips with excess I/O.
+    return queryset.select_related("process__binary").defer("process__stdout", "process__stderr", "process__env")
+
+
 @router.get("/archiveresults", response=list[ArchiveResultSchema], url_name="get_archiveresult")
 @paginate(CustomPagination)
 def get_archiveresults(request: HttpRequest, filters: Query[ArchiveResultFilterSchema]):
     """List all ArchiveResult entries matching these filters."""
-    queryset = filters.filter(ArchiveResult.objects.all())
+    queryset = filters.filter(
+        _with_archiveresult_processes(ArchiveResult.objects.all()).prefetch_related("snapshot__crawl__created_by", "snapshot__tags"),
+    )
     if filters.search or filters.snapshot_tag:
         return queryset.distinct()
     return queryset
@@ -223,26 +234,31 @@ def get_archiveresults(request: HttpRequest, filters: Query[ArchiveResultFilterS
 
 def _uuid_ref_query(field_name: str, ref: str) -> Q:
     raw_ref = str(ref or "").strip()
-    query = Q(**{f"{field_name}__startswith": raw_ref})
-    if raw_ref:
-        query |= Q(**{f"{field_name}__icontains": raw_ref})
     try:
         parsed_uuid = UUID(raw_ref)
     except (TypeError, ValueError):
+        query = Q(**{f"{field_name}__startswith": raw_ref})
+        if raw_ref:
+            query |= Q(**{f"{field_name}__icontains": raw_ref})
         normalized_ref = raw_ref.replace("-", "")
         if normalized_ref and normalized_ref != raw_ref:
             query |= Q(**{f"{field_name}__startswith": normalized_ref})
             query |= Q(**{f"{field_name}__icontains": normalized_ref})
-    else:
-        query |= Q(**{field_name: parsed_uuid})
-        query |= Q(**{f"{field_name}__startswith": parsed_uuid.hex})
-    return query
+        return query
+    # An exact UUID cannot need a substring fallback. OR-ing it with LIKE
+    # prevents an indexed lookup and made Cabbage metadata reads take seconds.
+    return Q(**{field_name: parsed_uuid})
 
 
 @router.get("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema, url_name="get_archiveresult")
 def get_archiveresult(request: HttpRequest, archiveresult_id: str):
     """Get a specific ArchiveResult by id."""
-    return ArchiveResult.objects.get(_uuid_ref_query("id", archiveresult_id))
+    return (
+        _with_archiveresult_processes(ArchiveResult.objects.all())
+        .select_related("snapshot__crawl__created_by")
+        .prefetch_related("snapshot__tags")
+        .get(_uuid_ref_query("id", archiveresult_id))
+    )
 
 
 def _normalize_uploaded_archiveresult_plugin(plugin: str) -> str:
@@ -350,6 +366,12 @@ def _summarize_archiveresult_output_files(output_files: dict[str, dict[str, Any]
 
 def _get_snapshot_by_ref(snapshot_id: str):
     queryset = Snapshot.objects.select_related("crawl__created_by")
+    try:
+        full_id = UUID(snapshot_id)
+    except (TypeError, ValueError):
+        pass
+    else:
+        return queryset.get(pk=full_id)
     try:
         return queryset.get(_uuid_ref_query("id", snapshot_id) | Q(timestamp__startswith=snapshot_id))
     except Snapshot.DoesNotExist:
@@ -392,13 +414,13 @@ def _write_archiveresult_files(
     mime_types = _get_archiveresult_upload_form_values(request, "mime_types", "mime_type")
     chunk_output_path = _get_archiveresult_upload_form_value(request, "chunk_output_path")
 
+    output_files = dict(existing_output_files or {})
+    if not files:
+        return output_files
+
     snapshot_dir = snapshot.output_dir
     plugin_dir = snapshot_dir / plugin_name
     storage = FileSystemStorage(location=str(plugin_dir))
-    output_files = dict(existing_output_files or {})
-
-    if not files:
-        return output_files
 
     if chunk_output_path:
         if len(files) != 1:
@@ -630,6 +652,8 @@ def patch_archiveresult(
 class SnapshotSchema(Schema):
     TYPE: str = "core.models.Snapshot"
     id: UUID
+    crawl_id: UUID
+    persona: str | None
     created_by_id: str
     created_by_username: str
     created_at: datetime
@@ -648,6 +672,17 @@ class SnapshotSchema(Schema):
     num_archiveresults: int
     archiveresults: list[MinimalArchiveResultSchema]
 
+    @staticmethod
+    def resolve_archive_path(obj):
+        return obj.archive_path_from_db
+
+    @staticmethod
+    def resolve_persona(obj):
+        # A deduplicated submission can reuse a capture from a different
+        # persona. Report its actual DB-backed owner, not the new request's choice.
+        persona = obj.crawl.resolve_persona()
+        return persona.name if persona else None
+
     @staticmethod
     def resolve_created_by_id(obj):
         return str(obj.created_by.pk)
@@ -670,12 +705,14 @@ class SnapshotSchema(Schema):
 
     @staticmethod
     def resolve_num_archiveresults(obj, context):
-        return obj.archiveresult_set.all().distinct().count()
+        # This reverse FK contains each result once. DISTINCT would materialize
+        # wide JSON output columns just to count rows through snapshot_id's index.
+        return obj.archiveresult_set.count()
 
     @staticmethod
     def resolve_archiveresults(obj, context):
         if bool(context["request"].__dict__.get("with_archiveresults", False)):
-            return obj.archiveresult_set.all().distinct()
+            return _with_archiveresult_processes(obj.archiveresult_set.all())
         return ArchiveResult.objects.none()
 
 
@@ -700,6 +737,7 @@ class SnapshotDeleteResponseSchema(Schema):
     snapshot_id: str
     crawl_id: str
     deleted_count: int
+    queued_count: int
 
 
 def normalize_tag_list(tags: list[str] | None = None) -> list[str]:
@@ -938,6 +976,15 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
     # lock. The unique insert recovery and CAS update below are the request-side
     # coordination boundary for this idempotent metadata sync.
     snapshot = Snapshot.objects.filter(url=data.url, crawl=crawl).first()
+    if snapshot is None and bool(crawl.get_current_config().get("ONLY_NEW", True)):
+        # Uploading browser metadata is part of the same submission as /add.
+        # It must honor the runner's deduplication policy, even if it arrives
+        # before the runner has processed that crawl. Report the actual owner
+        # so clients never mistake a reused capture for one they just created.
+        snapshot = Snapshot.objects.filter(url=data.url).exclude(status=Snapshot.DELETING_STATE).order_by("-created_at").first()
+        if snapshot is not None:
+            setattr(request, "with_archiveresults", False)
+            return snapshot
     if snapshot is None:
         try:
             snapshot = Snapshot.objects.create(
@@ -971,11 +1018,6 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
             created_by=request.user if isinstance(request.user, User) else None,
         )
 
-    try:
-        snapshot.ensure_crawl_symlink()
-    except Exception:
-        pass
-
     setattr(request, "with_archiveresults", False)
     return snapshot
 
@@ -1037,22 +1079,23 @@ def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateS
 
 @router.delete("/snapshot/{snapshot_id}", response=SnapshotDeleteResponseSchema, url_name="delete_snapshot")
 def delete_snapshot(request: HttpRequest, snapshot_id: str):
+    """Queue deletion; the normal server runner stops writers and removes files.
+
+    The snapshot remains readable with status=deleting until cleanup succeeds.
+    queued_count reports acceptance; deleted_count is zero in this response.
+    """
     snapshot = get_snapshot(request, snapshot_id, with_archiveresults=False)
     snapshot_id_str = str(snapshot.id)
-    crawl_id_str = str(snapshot.crawl.pk)
-    snapshot.cancel()
-
-    from archivebox.services.runner import run_pending_crawls
-
-    with crawl_lifecycle_lock(crawl_id_str):
-        run_pending_crawls(crawl_id=crawl_id_str, daemon=False)
-        snapshot = get_snapshot(request, snapshot_id_str, with_archiveresults=False)
-        deleted_count, _ = snapshot.delete()
+    crawl_id_str = str(snapshot.crawl_id)
+    # Use the same durable queue as the admin delete action. Stopping writers
+    # and removing archive files belongs to the runner, never the HTTP request.
+    queued_count = Snapshot.objects.filter(pk=snapshot.pk).request_delete()
     return {
         "success": True,
         "snapshot_id": snapshot_id_str,
         "crawl_id": crawl_id_str,
-        "deleted_count": deleted_count,
+        "deleted_count": 0,
+        "queued_count": queued_count,
     }
 
 

archivebox/api/v1_personas.py

@@ -5,9 +5,11 @@ from datetime import datetime
 from typing import Any, Literal
 from uuid import UUID
 
+from django.core.exceptions import ValidationError
 from django.db.models import Q
 from django.http import HttpRequest
 from ninja import Router, Schema
+from ninja.errors import HttpError
 from ninja.pagination import paginate
 from pydantic import Field
 
@@ -108,10 +110,9 @@ def browser_settings_to_config(extension_persona_id: str, settings: PersonaBrows
 
 
 def find_persona(extension_persona_id: str, name: str) -> Persona | None:
+    named = Persona.find_named(name)
     return (
-        Persona.objects.filter(
-            Q(config__BROWSER_EXTENSION_PERSONA_ID=extension_persona_id) | Q(name=name),
-        )
+        Persona.objects.filter(Q(config__BROWSER_EXTENSION_PERSONA_ID=extension_persona_id) | Q(pk=named.pk if named else None))
         .order_by("created_at")
         .first()
     )
@@ -138,7 +139,11 @@ def sync_persona(request: HttpRequest, payload: PersonaSyncSchema):
     if not is_valid:
         raise ValueError(error_message)
 
-    persona = find_persona(payload.extension_persona_id, name)
+    try:
+        persona = find_persona(payload.extension_persona_id, name)
+        (persona or Persona(name=name)).validate_name(persona.name if persona else name)
+    except ValidationError as err:
+        raise HttpError(409, "; ".join(err.messages)) from err
     created = persona is None
     if persona is None:
         persona = Persona(name=name)

archivebox/base_models/models.py

@@ -9,6 +9,7 @@ from typing import Any
 from archivebox.uuid_compat import CompactUUIDField, uuid7
 from pathlib import Path
 
+from django.core.exceptions import FieldDoesNotExist
 from django.db import models
 from django.db.models import F
 from django.db import transaction
@@ -23,6 +24,110 @@ from django_stubs_ext.db.models import TypedModelMeta
 from archivebox.config import CONSTANTS
 
 
+class PagedQuerySet(models.QuerySet):
+    def paged_iterator(self, chunk_size: int = 500):
+        """
+        Iterate rows using bounded keyset pages instead of one streaming cursor.
+
+        Django's iterator(chunk_size=...) still keeps a single SQLite SELECT
+        cursor open until the full queryset is exhausted. That is fine for
+        read-only exports, but update/migration code does filesystem work and
+        writes while iterating; a long-lived read cursor there can stretch lock
+        waits across thousands of rows. This respects the queryset's existing
+        filters, order_by(), select_related(), and prefetch_related() state; if
+        no ordering is defined, it falls back to primary-key order.
+        """
+        pk_field = self.model._meta.pk.name
+        raw_ordering = tuple(self.query.order_by or self.model._meta.ordering or (pk_field,))
+
+        if any(not isinstance(term, str) or term == "?" for term in raw_ordering):
+            offset = 0
+            while True:
+                batch = list(self[offset : offset + chunk_size])
+                if not batch:
+                    break
+                yield from batch
+                offset += chunk_size
+            return
+
+        ordering = []
+        for term in raw_ordering:
+            descending = term.startswith("-")
+            field_name = term[1:] if descending else term
+            if field_name == "pk":
+                field_name = pk_field
+            ordering.append(f"-{field_name}" if descending else field_name)
+
+        ordered_field_names = [term.removeprefix("-") for term in ordering]
+        try:
+            if any(self.model._meta.get_field(field_name).null for field_name in ordered_field_names):
+                offset = 0
+                while True:
+                    batch = list(self[offset : offset + chunk_size])
+                    if not batch:
+                        break
+                    yield from batch
+                    offset += chunk_size
+                return
+        except (AttributeError, FieldDoesNotExist):
+            offset = 0
+            while True:
+                batch = list(self[offset : offset + chunk_size])
+                if not batch:
+                    break
+                yield from batch
+                offset += chunk_size
+            return
+
+        unique_field_names = {pk_field, *(field.name for field in self.model._meta.fields if field.unique)}
+        if not any(field_name in unique_field_names for field_name in ordered_field_names):
+            offset = 0
+            while True:
+                batch = list(self[offset : offset + chunk_size])
+                if not batch:
+                    break
+                yield from batch
+                offset += chunk_size
+            return
+
+        last_values = None
+        value_field_names = tuple(dict.fromkeys([*ordered_field_names, pk_field]))
+        while True:
+            batch_qs = self.order_by(*ordering)
+            if last_values is not None:
+                page_filter = models.Q()
+                for idx, term in enumerate(ordering):
+                    descending = term.startswith("-")
+                    field_name = term[1:] if descending else term
+                    prefix = {ordered_field_names[i]: last_values[i] for i in range(idx)}
+                    comparison = "lt" if descending else "gt"
+                    page_filter |= models.Q(**prefix, **{f"{field_name}__{comparison}": last_values[idx]})
+                # Anchor the OR tie-breaks with an explicit leading range so
+                # SQLite/PG can seek the existing order index instead of
+                # rescanning its prefix for every keyset page.
+                first_comparison = "lte" if ordering[0].startswith("-") else "gte"
+                batch_qs = batch_qs.filter(
+                    page_filter,
+                    **{f"{ordered_field_names[0]}__{first_comparison}": last_values[0]},
+                )
+
+            batch_rows = list(batch_qs.values_list(*value_field_names)[:chunk_size])
+            if not batch_rows:
+                break
+
+            pk_idx = value_field_names.index(pk_field)
+            row_ids = [row[pk_idx] for row in batch_rows]
+            rows_by_id = {obj.pk: obj for obj in self.filter(pk__in=row_ids).order_by()}
+
+            for row in batch_rows:
+                row_id = row[pk_idx]
+                obj = rows_by_id.get(row_id)
+                if obj is not None:
+                    yield obj
+
+            last_values = batch_rows[-1][: len(ordered_field_names)]
+
+
 def normalize_config_json_values(config: Any) -> Any:
     if not isinstance(config, dict):
         return config
@@ -191,9 +296,21 @@ class ModelWithDeleteAfter(models.Model):
         return cls.objects.none()
 
     @classmethod
-    def delete_expired(cls, *, batch_size: int = 100, backfill_missing: bool = True) -> int:
+    def delete_expired(cls, *, batch_size: int = 100, backfill_missing: bool = True, backfill_cursors: dict | None = None) -> int:
         if backfill_missing:
-            missing_delete_at = list(cls.missing_delete_at_candidates().order_by("created_at", "pk")[:batch_size])
+            # Bound the scanned rows BEFORE evaluating JSON policy/parent
+            # joins. LIMIT on the policy query only bounded its output: finding
+            # no matches could scan the entire history with a SQLite read lock.
+            # The runner keeps a keyset cursor between idle passes, so disabled
+            # policies cannot starve older rows. Restarting safely repeats work.
+            page = cls.objects.order_by("-pk")
+            before = backfill_cursors.get(cls) if backfill_cursors is not None else None
+            if before is not None:
+                page = page.filter(pk__lt=before)
+            page_ids = list(page.values_list("pk", flat=True)[:batch_size])
+            if backfill_cursors is not None:
+                backfill_cursors[cls] = page_ids[-1] if len(page_ids) == batch_size else None
+            missing_delete_at = list(cls.missing_delete_at_candidates().filter(pk__in=page_ids).order_by())
             for obj in missing_delete_at:
                 if obj.set_delete_at_from_config():
                     cls.objects.filter(pk=obj.pk, delete_at__isnull=True).update(
@@ -231,12 +348,9 @@ class ModelWithOutputDir(ModelWithUUID):
 
     _delete_signal_registered = False
 
-    def save(self, *args, **kwargs):
-        super().save(*args, **kwargs)
-        output_dir = Path(self.output_dir)
-        # Avoid holding SQLite write transactions open across slow filesystem work.
-        transaction.on_commit(lambda: output_dir.mkdir(parents=True, exist_ok=True))
-        # Note: index.json is deprecated, models should use write_index_jsonl() for full data
+    # Saving metadata must not resolve or create output paths. The runner and
+    # explicit artifact writers create their own directories when writing files;
+    # on_commit would still make HTTP requests wait on remote archive storage.
 
     @property
     def output_dir_parent(self) -> str:
@@ -282,7 +396,14 @@ class ModelWithOutputDir(ModelWithUUID):
             if path.is_symlink() or path.is_file():
                 path.unlink(missing_ok=True)
             elif path.is_dir():
-                shutil.rmtree(path, ignore_errors=True)
+                shutil.rmtree(path)
+            # lstat also catches dangling symlinks and propagates mount and
+            # permission errors rather than mistaking them for absence.
+            try:
+                path.lstat()
+            except FileNotFoundError:
+                continue
+            raise OSError(f"Output path still exists after deletion: {path}")
 
     def schedule_delete_cleanup(self, *, using: str | None = None) -> None:
         """Capture output paths before DB deletion and remove them after commit."""
@@ -294,7 +415,9 @@ class ModelWithOutputDir(ModelWithUUID):
         if cls._delete_signal_registered:
             return
 
-        def schedule_output_dir_cleanup(sender, instance, using, **kwargs):
+        def schedule_output_dir_cleanup(sender, instance, using, origin=None, **kwargs):
+            if getattr(origin, "_output_files_deleted", False):
+                return
             if not isinstance(instance, ModelWithOutputDir):
                 return
             instance.schedule_delete_cleanup(using=using)

archivebox/cli/__init__.py

@@ -2,6 +2,7 @@ __package__ = "archivebox.cli"
 __command__ = "archivebox"
 import os
 import sys
+from contextlib import redirect_stdout
 from importlib import import_module
 
 import rich_click as click
@@ -219,6 +220,12 @@ def main(args=None, prog_name=None):
         raise SystemExit(130) from None
     finally:
         sys.unraisablehook = previous_unraisablehook
+        supervisor_util = sys.modules.get("archivebox.workers.supervisord_util")
+        if supervisor_util is not None:
+            # Short commands can start a foreground-owned search daemon too.
+            # Finish its cleanup before another CLI can borrow a dying stack.
+            with redirect_stdout(sys.stderr):
+                supervisor_util.stop_own_supervisord_process()
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_add.py

@@ -144,7 +144,6 @@ def add(
     persona_name = (persona or "Default").strip() or "Default"
     plugins = plugins or ""
     persona_obj = Persona.get_or_create_named(persona_name)
-    persona_obj.ensure_dirs()
     effective_persona_config = get_config(persona=persona_obj)
 
     crawl_config = {
@@ -217,9 +216,6 @@ def add(
         print(
             "[yellow]\\[*] URLs queued. The background runner will process them (run `archivebox server` or `archivebox run --daemon` if not already running).[/yellow]",
         )
-        from archivebox.services.runner import ensure_background_runner
-
-        ensure_background_runner()
     else:
         # Foreground mode: run full crawl runner until all work is done
         print("[green]\\[*] Starting crawl runner to process crawl...[/green]")
@@ -427,7 +423,23 @@ def main(**kwargs):
             existing = [token.strip() for token in (kwargs.get("plugins") or "").split(",") if token.strip()]
             kwargs["plugins"] = ",".join(dict.fromkeys([*existing, *selected]))
 
-        add(urls=urls, **kwargs)
+        from archivebox.machine.models import Process
+
+        # CLI commands own their process record and logs. Queue model saves
+        # must not register a process implicitly: HTTP add() shares that path.
+        process = Process.current()
+        exit_code = 0
+        try:
+            add(urls=urls, **kwargs)
+        except BaseException:
+            exit_code = 1
+            raise
+        finally:
+            # --bg/--index-only return without the foreground runner's cleanup.
+            # Leaving them RUNNING makes the active-process index accumulate
+            # historical CLI commands forever, defeating bounded recovery.
+            if process.status == Process.StatusChoices.RUNNING:
+                process.mark_exited(exit_code=exit_code)
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_init.py

@@ -116,6 +116,11 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     # Registration requires the final schema and reconciles legacy file config.
     Machine.current()
 
+    from archivebox.personas.models import Persona
+
+    # Initialize here, not when /add/ is rendered: a GET must not create profiles.
+    Persona.get_or_create_default().ensure_dirs()
+
     from archivebox.misc.db import database_display_location, is_postgres
 
     assert database_exists()

archivebox/cli/archivebox_install.py

@@ -58,7 +58,6 @@ def _install_raw_binary_names(binary_names: list[str], binproviders: str) -> Non
     from django.utils import timezone
 
     from archivebox.machine.models import Binary, Machine, _canonical_binary_name
-    from archivebox.services.runner import run_due_binary
 
     machine = Machine.current()
     for requested_name in binary_names:
@@ -80,7 +79,13 @@ def _install_raw_binary_names(binary_names: list[str], binproviders: str) -> Non
             binary.status = Binary.StatusChoices.QUEUED
             binary.retry_at = timezone.now()
             binary.save(update_fields=["binproviders", "overrides", "status", "retry_at", "modified_at"])
-        run_due_binary(binary, lock_seconds=60)
+        if binary.status == Binary.StatusChoices.INSTALLED:
+            continue
+        # The synchronous CLI needs the lifecycle's verified installation result,
+        # not the scheduler's "claimed work" result. Reuse its failure/requeue and
+        # health-stat handling so a failed install cannot exit successfully.
+        if not binary.install_claimed(lock_seconds=60):
+            raise click.ClickException(f"Binary {binary_name} is already being installed")
 
 
 def ensure_data_dir_lib_symlink(data_dir: Path, abxpkg_lib_dir: Path) -> Path | None:

archivebox/cli/archivebox_persona.py

@@ -163,8 +163,9 @@ def create_personas(
             rprint(f'[red]Invalid persona name "{name}": {error_msg}[/red]', file=sys.stderr)
             continue
 
-        persona, created = Persona.objects.get_or_create(
-            name=name,
+        created = Persona.find_named(name) is None
+        persona = Persona.get_or_create_named(
+            name,
             defaults={"config": {"PERMISSIONS": permissions}} if permissions else {},
         )
 
@@ -313,7 +314,9 @@ def update_personas(name: str | None = None) -> int:
             if persona_id:
                 persona = Persona.objects.get(id=persona_id)
             else:
-                persona = Persona.objects.get(name=old_name)
+                persona = Persona.find_named(old_name)
+                if persona is None:
+                    raise Persona.DoesNotExist
 
             # Apply updates from CLI flags
             if name:
@@ -325,7 +328,7 @@ def update_personas(name: str | None = None) -> int:
 
                 # Rename the persona directory too
                 old_path = persona.path
-                persona.name = name
+                persona.name = persona.validate_name(name)
                 new_path = persona.path
 
                 if old_path.exists() and old_path != new_path:

archivebox/cli/archivebox_remove.py

@@ -87,9 +87,8 @@ def remove(
     # Delete one snapshot at a time. Each ``.delete()`` is its own short
     # Django-atomic block, so the writer lock is released between rows and
     # an in-flight daemon transaction can interleave instead of deadlocking.
-    # Filesystem cleanup for each row is scheduled via ``transaction.on_commit``
-    # in ``base_models/models.py`` and runs AFTER its row's tx commits — so
-    # rmtree doesn't hold the lock either.
+    # Snapshot.delete() removes and verifies filesystem outputs first, outside
+    # the short database delete transaction. Failed cleanup retains the row.
     #
     deleted_snapshot_pks = []
     timed_out = False

archivebox/cli/archivebox_server.py

@@ -260,6 +260,12 @@ def server(
     bind_spec = next((arg for arg in runserver_args if arg), "")
     host, port = _parse_and_validate_bind_spec(bind_spec)
 
+    from archivebox.personas.models import Persona
+
+    # Older collections deferred this until /add/. Reconcile before serving any
+    # requests, and report ambiguous profile names before Chrome can use them.
+    Persona.get_or_create_default().ensure_dirs()
+
     if not User.objects.filter(is_superuser=True).exclude(username="system").exists():
         from archivebox.core.routes_util import build_admin_url
 

archivebox/cli/archivebox_snapshot.py

@@ -298,7 +298,7 @@ def list_snapshots(
     Exit codes:
         0: Success (even if no results)
     """
-    from archivebox.misc.jsonl import write_record
+    from archivebox.misc.jsonl import write_records
 
     output_formats = sum(bool(output_format) for output_format in (csv, as_json, as_html))
     if output_formats > 1:
@@ -351,9 +351,7 @@ def list_snapshots(
         return 0
 
     if not is_tty:
-        for snapshot_json in iter_snapshot_json(queryset):
-            write_record(snapshot_json)
-            count += 1
+        count = write_records(iter_snapshot_json(queryset))
         rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
         return 0
 

archivebox/cli/archivebox_version.py

@@ -101,6 +101,7 @@ def _binary_row_dedupe_key(
 def version(
     quiet: bool = False,
     binaries: Iterable[str] = (),
+    no_cache: bool = False,
 ) -> list[str]:
     """Print the ArchiveBox version, debug metadata, and installed dependency versions"""
 
@@ -275,6 +276,7 @@ def version(
             prnt("", f"[yellow]Warning: Could not query collection machine config; resolving through abxpkg: {e}[/yellow]")
 
     declared_binary_specs: dict[str, dict[str, object]] = {}
+    binary_records_by_plugin: dict[str, list[dict[str, object]]] = {}
     for plugin_name, plugin in plugins.items():
         if not plugin_may_have_requested_binary(plugin):
             continue
@@ -286,6 +288,10 @@ def version(
             derived_overrides=derived_config,
             run_output_dir=CONSTANTS.DATA_DIR,
         )
+        # Render the exact requests resolved below. Rehydrating all schemas for
+        # display doubled config work, and could describe different requests if
+        # a config file changed while dependency resolution was running.
+        binary_records_by_plugin[plugin_name] = binary_records
         for binary_record in binary_records:
             actual_name = str(binary_record["name"])
             logical_name = (
@@ -296,9 +302,13 @@ def version(
             display_name = logical_name
             if not plugin_requested and not binary_is_requested(logical_name, actual_name, display_name):
                 continue
-            # This is a diagnostic: recheck the provider instead of trusting an
-            # installation record or a cached version, including disabled plugins.
-            binary_record["no_cache"] = True
+            # Resolve through abxpkg, never the DB's last-known installation row.
+            # Its cache validates current paths, permissions, package fingerprints
+            # and runtime options. Forcing cold resolution here rehashed every
+            # large binary and reran every version subprocess on each diagnostic,
+            # even in an image that had already validated all of them at build.
+            # Keep an explicit full recheck for diagnosing a suspect installation.
+            binary_record["no_cache"] = no_cache
             signature = json.dumps(binary_record, sort_keys=True, default=str)
             declared_binary_specs.setdefault(signature, binary_record)
 
@@ -330,15 +340,9 @@ def version(
             for dependency in plugins.select([plugin_name]).values()
             if not config.get(dependency.enabled_key, True)
         )
-        binary_records = get_required_binary_requests(
-            plugin,
-            plugin.config.required_binaries,
-            overrides=runtime_config,
-            derived_overrides=derived_config,
-            run_output_dir=CONSTANTS.DATA_DIR,
-        )
+        binary_records = binary_records_by_plugin[plugin_name]
         for binary_record in binary_records:
-            binary_record["no_cache"] = True
+            binary_record["no_cache"] = no_cache
             actual_name = str(binary_record["name"])
             logical_name = (
                 Path(actual_name).expanduser().name
@@ -485,6 +489,11 @@ def version(
     is_flag=True,
     help="Only print ArchiveBox version number and nothing else. (equivalent to archivebox --version)",
 )
+@click.option(
+    "--no-cache",
+    is_flag=True,
+    help="Reprobe binary versions and hashes instead of reusing validated provider caches.",
+)
 @click.option(
     "--binaries",
     "-b",

archivebox/cli/rescan.py

@@ -102,9 +102,19 @@ def _reconcile_directory(directory: Path) -> str:
     results = {(item.get("plugin"), item.get("hook_name", "")): item for item in records if item.get("type") == "ArchiveResult"}.values()
     pending = []
     for result in results:
-        result_id = uuid.UUID(result["id"])
         if uuid.UUID(result["snapshot_id"]) != snapshot_id:
             raise ValueError("ArchiveResult belongs to another snapshot")
+        # Legacy JSON conversion produced records without IDs. Recover these
+        # by hook identity while preserving strict collision checks for IDs.
+        if not result.get("id"):
+            if not ArchiveResult.objects.filter(
+                snapshot_id=snapshot_id,
+                plugin=result["plugin"],
+                hook_name=result.get("hook_name", ""),
+            ).exists():
+                pending.append(result)
+            continue
+        result_id = uuid.UUID(result["id"])
         existing = ArchiveResult.objects.filter(pk=result_id).first()
         identity = (snapshot_id, result["plugin"], result.get("hook_name", ""))
         if existing:
@@ -154,7 +164,10 @@ def _reconcile_directory(directory: Path) -> str:
     # Each insert is durable. An interruption after the Snapshot insert is not
     # completion: the next pass still imports missing result IDs and tags.
     for result in pending:
-        snapshot._create_archive_result_if_missing(result, {}, preserve_identity=True)
+        snapshot._create_archive_result_if_missing(result, {}, preserve_identity=bool(result.get("id")))
+    if any(not result.get("id") for result in pending):
+        snapshot.hydrate_archiveresult_output_metadata(snapshot_dir=directory)
+        snapshot.write_index_jsonl(output_dir=directory)
     tags_before = set(snapshot.tags.values_list("name", flat=True))
     snapshot._merge_tags_from_index(record)
     tags_changed = tags_before != set(snapshot.tags.values_list("name", flat=True))
@@ -182,6 +195,10 @@ def reconcile_known_snapshots(snapshots, wait_for_turn=None):
             snapshot.migrate_filesystem_to_current_version()
         directory = Path(snapshot.output_dir)
         if not directory.is_dir():
+            # Canceling queued metadata can seal a row before the runner ever
+            # creates storage. There is nothing on disk to reconcile for it.
+            if snapshot.downloaded_at is None and not snapshot.archiveresult_set.exists():
+                continue
             raise ValueError(f"Missing snapshot directory: {directory}")
         _reconcile_directory(directory)
         for result in snapshot.archiveresult_set.all():

archivebox/config/common.py

@@ -34,6 +34,25 @@ ConfigPayload = dict[str, object]
 PluginSchemaDocuments = dict[str, dict[str, Any]]
 LIVE_CONFIG_BASE_URL = "/admin/environment/config/"
 
+
+def _is_cookie_file_config_key(key: str) -> bool:
+    return key in {"COOKIES_FILE", "AUTH_STORAGE_FILE"} or key.endswith("_COOKIES_FILE")
+
+
+def _resolve_cookie_file_paths(config: dict[str, Any]) -> dict[str, Any]:
+    """Keep collection-relative auth paths valid after hooks change cwd."""
+    for key, value in config.items():
+        if _is_cookie_file_config_key(key):
+            if value is None or (isinstance(value, str) and not value.strip()):
+                config[key] = ""
+                continue
+            file_path = Path(value).expanduser()
+            if not file_path.is_absolute():
+                file_path = CONSTANTS.DATA_DIR / file_path
+            config[key] = str(file_path.absolute())
+    return config
+
+
 ###################### Config ##########################
 
 _STDOUT_CONSOLE = Console()
@@ -395,6 +414,36 @@ class ServerConfig(BaseConfigSet):
         )
 
 
+@lru_cache(maxsize=16)
+def _default_sqlite_journal_mode(database_name: str) -> str:
+    if not IN_DOCKER:
+        return "WAL"
+
+    # Native Linux bind mounts share the kernel's locks and mmap; Docker alone
+    # does not make WAL unsafe. DELETE there lets any slow reader block tiny
+    # writes. Desktop VM shares (virtiofs/9p/FUSE), network filesystems and
+    # unknown mounts keep rollback journaling: their host-side readers may not
+    # share SQLite's -shm locking domain. Inspect the DB mount, not archive/.
+    database_path = Path(database_name).expanduser().resolve()
+    try:
+        mounts = Path("/proc/self/mountinfo").read_text().splitlines()
+    except OSError:
+        return "DELETE"
+    filesystem = ""
+    mount_depth = -1
+    for line in mounts:
+        fields, separator, details = line.partition(" - ")
+        if not separator:
+            continue
+        mount_path = fields.split()[4]
+        mount_path = re.sub(r"\\([0-7]{3})", lambda match: chr(int(match[1], 8)), mount_path)
+        mount = Path(mount_path)
+        if database_path.is_relative_to(mount) and len(mount.parts) > mount_depth:
+            filesystem = details.split()[0]
+            mount_depth = len(mount.parts)
+    return "WAL" if filesystem in {"ext2", "ext3", "ext4", "xfs", "btrfs", "zfs", "overlay", "tmpfs", "f2fs"} else "DELETE"
+
+
 class DatabaseConfig(BaseConfigSet):
     toml_section_header: str = "DATABASE_CONFIG"
     _scope: str = PrivateAttr(default=_SCOPE_SERVER)
@@ -410,11 +459,7 @@ class DatabaseConfig(BaseConfigSet):
     DATABASE_USER: str = Field(default="archivebox", alias="ARCHIVEBOX_DATABASE_USER")
     DATABASE_PASSWORD: str = Field(default="", alias="ARCHIVEBOX_DATABASE_PASSWORD")
     SQLITE_JOURNAL_MODE: str = Field(
-        # Docker collections commonly live on a host bind mount. WAL's -shm
-        # locking is only safe when every SQLite process is on the same host;
-        # Docker Desktop/OrbStack place the container and host in different
-        # locking domains and a host-side reader can corrupt the live DB.
-        default="DELETE" if IN_DOCKER else "WAL",
+        default_factory=lambda data: _default_sqlite_journal_mode(str(data["DATABASE_NAME"])),
         alias="ARCHIVEBOX_SQLITE_JOURNAL_MODE",
         pattern=r"(?i)^(DELETE|TRUNCATE|PERSIST|MEMORY|WAL|OFF)$",
     )
@@ -428,11 +473,16 @@ class DatabaseConfig(BaseConfigSet):
     SQLITE_LOCK_RETRY_INTERVAL: float = Field(default=5.0, alias="ARCHIVEBOX_SQLITE_LOCK_RETRY_INTERVAL", gt=0)
 
     @model_validator(mode="after")
-    def reject_docker_sqlite_wal(self):
-        if IN_DOCKER and self.DATABASE_ENGINE.lower() == "sqlite" and self.SQLITE_JOURNAL_MODE.upper() == "WAL":
+    def reject_shared_filesystem_sqlite_wal(self):
+        if (
+            IN_DOCKER
+            and self.DATABASE_ENGINE.lower() == "sqlite"
+            and self.SQLITE_JOURNAL_MODE.upper() == "WAL"
+            and _default_sqlite_journal_mode(self.DATABASE_NAME) != "WAL"
+        ):
             raise ValueError(
-                "SQLITE_JOURNAL_MODE=WAL is unsafe for Docker collections because host bind mounts cross SQLite "
-                "locking domains; use DELETE (the Docker default) or PostgreSQL",
+                "SQLITE_JOURNAL_MODE=WAL requires a local filesystem with shared SQLite locking; "
+                "keep the database on a local Docker volume, use DELETE for host/VM shared mounts, or use PostgreSQL",
             )
         return self
 
@@ -468,6 +518,11 @@ class ArchivingConfig(BaseConfigSet):
     COOKIES_FILE: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
     AUTH_STORAGE_FILE: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
 
+    @field_validator("COOKIES_FILE", "AUTH_STORAGE_FILE", mode="before")
+    @classmethod
+    def empty_cookie_file_is_unset(cls, value):
+        return None if isinstance(value, str) and not value.strip() else value
+
     URL_DENYLIST: str = Field(
         default=(
             r"\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$"
@@ -498,7 +553,7 @@ class ArchivingConfig(BaseConfigSet):
             rprint("    (Setting it to somewhere between 30 and 3000 seconds is recommended)", file=sys.stderr)
             rprint(file=sys.stderr)
             rprint("    If you want to make ArchiveBox run faster, disable specific archive methods instead:", file=sys.stderr)
-            rprint("        https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#archive-method-toggles", file=sys.stderr)
+            rprint("        https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#plugins", file=sys.stderr)
             rprint(file=sys.stderr)
 
     @field_validator("CHECK_SSL_VALIDITY", mode="after")
@@ -825,6 +880,8 @@ class ArchiveBoxBaseConfig(
         if runtime_overrides:
             config.update(normalize_runtime_config(runtime_overrides, json_safe=False))
 
+        _resolve_cookie_file_paths(config)
+
         # Hooks should only see concrete plugin-local flags, never the
         # ArchiveBox selectors used to derive them.
         config.pop("PLUGINS", None)
@@ -849,6 +906,13 @@ class ArchiveBoxBaseConfig(
             lib_dir = CONSTANTS.DATA_DIR / lib_dir
         self.ABXPKG_LIB_DIR = lib_dir.resolve()
 
+        cookie_paths = {key: getattr(self, key) for key in type(self).model_fields if _is_cookie_file_config_key(key)}
+        for key, value in _resolve_cookie_file_paths(cookie_paths).items():
+            if key in {"COOKIES_FILE", "AUTH_STORAGE_FILE"}:
+                setattr(self, key, Path(value) if value else None)
+            else:
+                setattr(self, key, value)
+
         return self
 
     @model_validator(mode="after")
@@ -1156,11 +1220,15 @@ def get_config(
     scope_overrides: ConfigPayload = {}
 
     if include_machine and machine is not None and machine.config:
-        from archivebox.machine.models import _sanitize_machine_config
+        machine_config = machine.config
+        if resolve_plugins:
+            from archivebox.machine.models import _sanitize_machine_config
+
+            machine_config = _sanitize_machine_config(machine_config, lib_dir=config_data.get("ABXPKG_LIB_DIR"))
 
         scope_overrides.update(
             normalize_runtime_config(
-                _sanitize_machine_config(machine.config, lib_dir=config_data.get("ABXPKG_LIB_DIR")),
+                machine_config,
                 only_crawl_execution=crawl_config_base,
                 exclude_runtime_derived=True,
                 json_safe=False,

archivebox/config/views.py

@@ -150,9 +150,9 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
 
     rows = {
         "Binary Name": [],
-        "Found Version": [],
+        "Recorded Version": [],
         "Provided By": [],
-        "Found Abspath": [],
+        "Recorded Abspath": [],
     }
 
     db_binaries = get_db_binaries_by_name()
@@ -160,18 +160,10 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
 
     for name in all_binary_names:
         binary = db_binaries.get(name)
-        binary_is_valid = bool(binary and binary.is_valid)
-
         rows["Binary Name"].append(ItemLink(name, key=name))
-
-        if binary_is_valid:
-            rows["Found Version"].append(f"✅ {binary.version}" if binary.version else "✅ found")
-            rows["Provided By"].append(binary.binprovider or "-")
-            rows["Found Abspath"].append(binary.abspath or "-")
-        else:
-            rows["Found Version"].append("❌ missing")
-            rows["Provided By"].append("-")
-            rows["Found Abspath"].append("-")
+        rows["Recorded Version"].append(binary.version or "-")
+        rows["Provided By"].append(binary.binprovider or "-")
+        rows["Recorded Abspath"].append(binary.abspath or "-")
 
     return TableContext(
         title="Binaries",

archivebox/core/admin_snapshots.py

@@ -131,11 +131,11 @@ class SnapshotStatusListFilter(admin.SimpleListFilter):
     parameter_name = "snapshot_status"
 
     def lookups(self, request, model_admin):
-        return Snapshot.StatusChoices.choices
+        return Snapshot._meta.get_field("status").choices
 
     def queryset(self, request, queryset):
         value = self.value()
-        if value in Snapshot.StatusChoices.values:
+        if value in dict(Snapshot._meta.get_field("status").choices):
             return queryset.filter(status=value)
         return queryset
 
@@ -992,10 +992,14 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         return format_html(
             '<a class="snapshot-title-detail-hitbox" href="{}" aria-label="Open snapshot details"></a>'
             "{}"
+            "{}"
             '<div class="snapshot-title-url">'
             '<a class="snapshot-original-url" href="{}">{}<code style="user-select: all;">{}</code></a>'
             "</div>",
             detail_url,
+            format_html('<strong class="snapshot-delete-status">{}</strong>', "Deleting — verifying file removal")
+            if obj.status == Snapshot.DELETING_STATE
+            else "",
             title_html,
             url_raw or obj.url,
             favicon_html,
@@ -1535,25 +1539,14 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     @admin.action(
         description="🗑️ Delete",
+        permissions=["delete"],
     )
     def delete_snapshots(self, request, queryset):
-        """Delete snapshots in a single transaction to avoid SQLite concurrency issues."""
-        from django.db import transaction
-
-        total = queryset.count()
-
-        # Get list of IDs to delete first (outside transaction)
-        ids_to_delete = list(queryset.values_list("pk", flat=True))
-
-        # Delete everything in a single atomic transaction
-        with transaction.atomic():
-            deleted_count, _ = Snapshot.objects.filter(pk__in=ids_to_delete).delete()
-
+        """Keep the rows until the runner removes and verifies their files."""
+        total = queryset.request_delete()
         messages.success(
             request,
-            mark_safe(
-                f"Successfully deleted {total} Snapshots ({deleted_count} total objects including related records). Don't forget to scrub URLs from import logs (data/sources) and error logs (data/logs) if needed.",
-            ),
+            f"Queued {total} snapshots for deletion. They remain visible as Deleting until ArchiveBox verifies their files are removed.",
         )
 
     @admin.action(

archivebox/core/asgi.py

@@ -31,7 +31,7 @@ django_application = ArchiveBoxASGIHandler()
 
 async def application(scope, receive, send):
     if scope["type"] == "websocket":
-        from archivebox.opencode.views import websocket_view
+        from abx_plugins.plugins.opencode.archivebox.views import websocket_view
 
         return await websocket_view(scope, receive, send)
     return await django_application(scope, receive, send)

archivebox/core/forms.py

@@ -304,13 +304,13 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             self.can_override_crawl_config = bool(user.is_authenticated and user.is_active and user.is_superuser)
         super().__init__(*args, **kwargs)
 
-        default_persona = Persona.get_or_create_default()
+        # Persona creation belongs to init/server startup, never a GET of /add/.
         persona_queryset = Persona.objects.order_by("name")
         if not self.can_override_crawl_config:
             persona_queryset = filter_personas_by_permissions(persona_queryset, {PERMISSIONS_PUBLIC})
         self.fields["persona"].queryset = persona_queryset
 
-        selected_persona = persona_queryset.filter(id=default_persona.id).first() or persona_queryset.first()
+        selected_persona = persona_queryset.filter(name__iexact="Default").first() or persona_queryset.first()
         default_config = get_config(persona=selected_persona) if selected_persona else get_config()
         if selected_persona:
             self.fields["persona"].initial = selected_persona.name
@@ -338,6 +338,12 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
                 get_choice_field(self, field_name).choices = [(plugin.name, plugin.name) for plugin in grouped_plugins.get(category, [])]
             self.plugin_groups = []
 
+    def clean_persona(self):
+        persona = self.cleaned_data.get("persona")
+        if persona:
+            persona.validate_name(persona.name)
+        return persona
+
     def clean(self):
         cleaned_data = super().clean() or {}
 

archivebox/core/migrations/0058_snapshot_deleting_status.py

@@ -0,0 +1,28 @@
+# Generated by Django 6.1 on 2026-09-30 03:32
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0057_repair_migrated_process_links"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="snapshot",
+            name="status",
+            field=models.CharField(
+                choices=[
+                    ("queued", "Queued"),
+                    ("started", "Started"),
+                    ("paused", "Paused"),
+                    ("sealed", "Sealed"),
+                    ("deleting", "Deleting"),
+                ],
+                db_index=True,
+                default="queued",
+                max_length=15,
+            ),
+        ),
+    ]

archivebox/core/models.py

@@ -1,6 +1,7 @@
 __package__ = "archivebox.core"
 
 import json
+import logging
 import os
 import uuid
 from collections.abc import Iterable, Mapping, Sequence
@@ -11,7 +12,7 @@ from urllib.parse import urlparse
 
 from django.conf import settings
 from django.contrib import admin
-from django.core.exceptions import FieldDoesNotExist, ObjectDoesNotExist, ValidationError
+from django.core.exceptions import ObjectDoesNotExist, ValidationError
 from django.db import IntegrityError, models, transaction
 from django.db.models import F, Q, QuerySet, Sum, Value
 from django.db.models.fields.json import KT
@@ -23,6 +24,7 @@ from django.utils.safestring import mark_safe
 from django.utils.text import slugify
 
 from archivebox.base_models.models import (
+    PagedQuerySet,
     ModelWithConfig,
     ModelWithDeleteAfter,
     ModelWithHealthStats,
@@ -172,9 +174,28 @@ class SnapshotTag(models.Model):
         unique_together: ClassVar[list[tuple[str, str]]] = [("snapshot", "tag")]
 
 
-class SnapshotQuerySet(models.QuerySet):
+class SnapshotQuerySet(PagedQuerySet):
     """Custom QuerySet for Snapshot model with export methods that persist through .filter() etc."""
 
+    def request_delete(self) -> int:
+        # Admin and API deletion share this durable command. Keep the row until
+        # the runner stops its writers and verifies removal from remote storage;
+        # RETRY_AT_MAX prevents capture scheduling while cleanup is pending.
+        return self.update(status=self.model.DELETING_STATE, retry_at=RETRY_AT_MAX, modified_at=timezone.now())
+
+    def delete(self):
+        # Finish the SELECT before filesystem work, so no read cursor or SQLite
+        # transaction is held while deleting remote archive payloads.
+        snapshots = list(self.select_related("crawl__created_by"))
+        total = 0
+        counts = {}
+        for snapshot in snapshots:
+            deleted, details = snapshot.delete(using=self.db)
+            total += deleted
+            for label, count in details.items():
+                counts[label] = counts.get(label, 0) + count
+        return total, counts
+
     def bulk_create(self, objs, *args, **kwargs):
         objs = list(objs)
         missing_crawl_ids = set()
@@ -206,101 +227,6 @@ class SnapshotQuerySet(models.QuerySet):
                 truncate_overlong_charfields(obj)
         return super().bulk_create(objs, *args, **kwargs)
 
-    def paged_iterator(self, chunk_size: int = 500):
-        """
-        Iterate snapshots using bounded keyset pages instead of one streaming cursor.
-
-        Django's iterator(chunk_size=...) still keeps a single SQLite SELECT
-        cursor open until the full queryset is exhausted. That is fine for
-        read-only exports, but update/migration code does filesystem work and
-        writes while iterating; a long-lived read cursor there can stretch lock
-        waits across thousands of rows. This respects the queryset's existing
-        filters, order_by(), select_related(), and prefetch_related() state; if
-        no ordering is defined, it falls back to primary-key order.
-        """
-        pk_field = self.model._meta.pk.name
-        raw_ordering = tuple(self.query.order_by or self.model._meta.ordering or (pk_field,))
-
-        if any(not isinstance(term, str) or term == "?" for term in raw_ordering):
-            offset = 0
-            while True:
-                batch = list(self[offset : offset + chunk_size])
-                if not batch:
-                    break
-                yield from batch
-                offset += chunk_size
-            return
-
-        ordering = []
-        for term in raw_ordering:
-            descending = term.startswith("-")
-            field_name = term[1:] if descending else term
-            if field_name == "pk":
-                field_name = pk_field
-            ordering.append(f"-{field_name}" if descending else field_name)
-
-        ordered_field_names = [term.removeprefix("-") for term in ordering]
-        try:
-            if any(self.model._meta.get_field(field_name).null for field_name in ordered_field_names):
-                offset = 0
-                while True:
-                    batch = list(self[offset : offset + chunk_size])
-                    if not batch:
-                        break
-                    yield from batch
-                    offset += chunk_size
-                return
-        except (AttributeError, FieldDoesNotExist):
-            offset = 0
-            while True:
-                batch = list(self[offset : offset + chunk_size])
-                if not batch:
-                    break
-                yield from batch
-                offset += chunk_size
-            return
-
-        unique_field_names = {pk_field, *(field.name for field in self.model._meta.fields if field.unique)}
-        if not any(field_name in unique_field_names for field_name in ordered_field_names):
-            offset = 0
-            while True:
-                batch = list(self[offset : offset + chunk_size])
-                if not batch:
-                    break
-                yield from batch
-                offset += chunk_size
-            return
-
-        last_values = None
-        value_field_names = tuple(dict.fromkeys([*ordered_field_names, pk_field]))
-        while True:
-            batch_qs = self.order_by(*ordering)
-            if last_values is not None:
-                page_filter = models.Q()
-                for idx, term in enumerate(ordering):
-                    descending = term.startswith("-")
-                    field_name = term[1:] if descending else term
-                    prefix = {ordered_field_names[i]: last_values[i] for i in range(idx)}
-                    comparison = "lt" if descending else "gt"
-                    page_filter |= models.Q(**prefix, **{f"{field_name}__{comparison}": last_values[idx]})
-                batch_qs = batch_qs.filter(page_filter)
-
-            batch_rows = list(batch_qs.values_list(*value_field_names)[:chunk_size])
-            if not batch_rows:
-                break
-
-            pk_idx = value_field_names.index(pk_field)
-            snapshot_ids = [row[pk_idx] for row in batch_rows]
-            snapshots_by_id = {snapshot.pk: snapshot for snapshot in self.filter(pk__in=snapshot_ids).order_by()}
-
-            for row in batch_rows:
-                snapshot_id = row[pk_idx]
-                snapshot = snapshots_by_id.get(snapshot_id)
-                if snapshot is not None:
-                    yield snapshot
-
-            last_values = batch_rows[-1][: len(ordered_field_names)]
-
     # =========================================================================
     # Filtering Methods
     # =========================================================================
@@ -541,6 +467,7 @@ class SnapshotManager(models.Manager.from_queryset(SnapshotQuerySet)):  # ty: ig
 
 class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHealthStats, ModelWithQueue):
     BROWSER_EXTENSION_UPLOAD_HOOK_NAME = "on_Snapshot__archivebox_browser_extension_upload"
+    DELETING_STATE = "deleting"
 
     id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
@@ -574,7 +501,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
     )
     retry_at = ModelWithQueue.RetryAtField(default=timezone.now)
     status = ModelWithQueue.StatusField(
-        choices=ModelWithQueue.StatusChoices,
+        choices=[*ModelWithQueue.StatusChoices.choices, (DELETING_STATE, "Deleting")],
         default=ModelWithQueue.StatusChoices.QUEUED,
     )
     config = models.JSONField(default=dict, null=False, blank=False, editable=True)
@@ -661,6 +588,49 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
     def __str__(self):
         return f"[{self.id}] {self.url[:64]}"
 
+    def output_paths_for_delete(self) -> tuple[Path, ...]:
+        current = self.get_storage_path_for_version(self.fs_version)
+        paths = [Path(self.output_dir), current, CONSTANTS.ARCHIVE_DIR / self.timestamp]
+        if self.fs_version in ("0.9.0", "0.9.1", "0.9.2", "0.9.3", "0.9.4", "1.0.0"):
+            paths.append(current.with_name(str(uuid.UUID(hex=self.id.hex))))
+        if self.crawl_id:
+            domain_dir = self.crawl.output_dir / CONSTANTS.SNAPSHOTS_DIR_NAME / self.extract_domain_from_url(self.url)
+            paths.extend((domain_dir / str(self.id), domain_dir / str(uuid.UUID(hex=self.id.hex))))
+        return tuple(dict.fromkeys(paths))
+
+    def delete(self, using=None, keep_parents=False):
+        from django.db import connections
+
+        using = using or self._state.db or "default"
+        if connections[using].in_atomic_block:
+            raise RuntimeError("Snapshot filesystem deletion must run outside a database transaction")
+        type(self).objects.using(using).filter(pk=self.pk).request_delete()
+        self.status = self.DELETING_STATE
+        from archivebox.machine.models import Process
+
+        process_ids = self.archiveresult_set.filter(process__status=Process.StatusChoices.RUNNING).values_list("process_id", flat=True)
+        for process in list(Process.objects.using(using).filter(pk__in=process_ids)):
+            process.kill_tree()
+            if process.is_running:
+                raise OSError(f"Snapshot output writer is still running: {process.pk}")
+        paths = self.output_paths_for_delete()
+        self.delete_output_paths(paths)
+        # Cascaded ArchiveResults share this directory. Their normal post-commit
+        # cleanup would repeat filesystem work and update an already-deleted row.
+        self._output_files_deleted = True
+        return super().delete(using=using, keep_parents=keep_parents)
+
+    @classmethod
+    def delete_requested(cls, *, batch_size=100):
+        # Finish the bounded read before touching files: even an open SELECT
+        # cursor can hold a SQLite read lock for the duration of slow cleanup.
+        pending = list(cls.objects.filter(status=cls.DELETING_STATE).select_related("crawl__created_by")[:batch_size])
+        for snapshot in pending:
+            try:
+                snapshot.delete()
+            except OSError:
+                logging.getLogger(__name__).exception("Snapshot %s file deletion failed; keeping its database row", snapshot.pk)
+
     @classmethod
     def crawl_count_subquery(cls, *, status: str | None = None, outer_ref: str = "pk") -> QuerySet:
         """Return a scalar subquery counting Snapshots for one outer Crawl."""
@@ -911,8 +881,11 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
     @classmethod
     def missing_delete_at_candidates(cls):
-        return cls.objects.filter(delete_at__isnull=True).filter(
-            Q(config__has_key="DELETE_AFTER") | Q(crawl__config__has_key="DELETE_AFTER"),
+        return (
+            cls.objects.filter(delete_at__isnull=True)
+            .filter(Q(config__has_key="DELETE_AFTER") | Q(crawl__config__has_key="DELETE_AFTER"))
+            .select_related("crawl")
+            .only("id", "created_at", "delete_at", "config", "crawl__config")
         )
 
     @classmethod
@@ -1073,7 +1046,6 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         from django.db import transaction
 
         def finish_snapshot_save():
-            self.reconcile_filesystem_links()
             crawl = Crawl.objects.filter(pk=self.crawl_id).first()
             if crawl is None:
                 return
@@ -1093,8 +1065,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             # Crawl.urls remains the original submitted source. Snapshot rows
             # are the normalized work queue and discovery projection.
 
-        # get_or_create/update_or_create wrap save() in atomic(); defer filesystem
-        # work and crawl maintenance so SQLite commits before touching the disk.
+        # get_or_create/update_or_create wrap save() in atomic(); attach inherited
+        # tags after that write commits. Filesystem projections belong to the runner.
         transaction.on_commit(finish_snapshot_save)
 
     # =========================================================================
@@ -1198,6 +1170,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
         target_dir = self.get_storage_path_for_version(target)
         if target_dir.exists():
+            self.reconcile_with_index(output_dir=target_dir, update_existing_archive_results=False)
             self.hydrate_archiveresult_output_metadata(snapshot_dir=target_dir)
         if cleanup:
             old_dir, new_dir = cleanup
@@ -1230,7 +1203,9 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
     def hydrate_archiveresult_output_metadata(self, snapshot_dir: Path | None = None) -> int:
         """Populate missing ArchiveResult file metadata from existing outputs."""
         hydrated = 0
-        for result in self.archiveresult_set.filter(output_files={}).iterator():
+        # Files may live on slow remote storage: finish the bounded read before
+        # inspecting outputs or writing their metadata back to the database.
+        for result in self.archiveresult_set.filter(output_files={}).order_by("pk").paged_iterator():
             hydrated += int(result.update_output_metadata_from_filesystem(snapshot_dir=snapshot_dir))
         return hydrated
 
@@ -3975,6 +3950,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
 
 class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
+    objects = PagedQuerySet.as_manager()
+
     class StatusChoices(models.TextChoices):
         QUEUED = "queued", "Queued"
         STARTED = "started", "Started"
@@ -4196,8 +4173,11 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
 
     @classmethod
     def missing_delete_at_candidates(cls):
-        return cls.objects.filter(delete_at__isnull=True).filter(
-            Q(snapshot__config__has_key="DELETE_AFTER") | Q(snapshot__crawl__config__has_key="DELETE_AFTER"),
+        return (
+            cls.objects.filter(delete_at__isnull=True)
+            .filter(Q(snapshot__config__has_key="DELETE_AFTER") | Q(snapshot__crawl__config__has_key="DELETE_AFTER"))
+            .select_related("snapshot__crawl")
+            .only("id", "created_at", "delete_at", "snapshot__config", "snapshot__crawl__config")
         )
 
     @property
@@ -4773,9 +4753,13 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
 
     @property
     def pwd(self) -> str:
-        """Working directory, derived from the snapshot/plugin path if the Process row is gone."""
+        """Working directory metadata without probing archive storage."""
         process = self.process_record
-        return process.pwd if process and process.pwd else str(self.output_dir)
+        return (
+            process.pwd
+            if process and process.pwd
+            else str(self.snapshot.get_storage_path_for_version(self.snapshot.fs_version) / self.plugin)
+        )
 
     @property
     def cmd(self) -> list:

archivebox/core/snapshot_status.py

@@ -6,7 +6,7 @@ from django.db.models import QuerySet
 def snapshot_status_values() -> tuple[str, ...]:
     from archivebox.core.models import Snapshot
 
-    return tuple(Snapshot.StatusChoices.values)
+    return tuple(value for value, _label in Snapshot._meta.get_field("status").choices)
 
 
 def normalize_snapshot_status(status: str | None) -> str | None:

archivebox/core/takeover_util.py

@@ -62,7 +62,7 @@ def runtime_stack_owner(*, data_dir: str | Path, exclude_id=None):
         # not keep stealing leadership from the next foreground command.
         base_qs.filter(process_type=Process.TypeChoices.ORCHESTRATOR).exclude(parent__process_type=Process.TypeChoices.SUPERVISORD),
     ):
-        for proc in qs.order_by("-created_at", "-modified_at").iterator(chunk_size=50):
+        for proc in qs.order_by("-created_at", "-modified_at").paged_iterator(chunk_size=50):
             if proc.is_running:
                 return proc
             proc.mark_exited(exit_code=proc.exit_code if proc.exit_code is not None else 0)
@@ -95,7 +95,7 @@ def foreground_runner_owner(*, data_dir: str | Path, exclude_id=None):
     )
     if exclude_id is not None:
         qs = qs.exclude(id=exclude_id)
-    for proc in qs.order_by("-created_at", "-modified_at").iterator(chunk_size=50):
+    for proc in qs.order_by("-created_at", "-modified_at").paged_iterator(chunk_size=50):
         if proc.is_running:
             return proc
         proc.mark_exited(exit_code=proc.exit_code if proc.exit_code is not None else 0)
@@ -178,7 +178,7 @@ def live_runner_processes(*, data_dir: str | Path):
     qs = qs.filter(machine=machine)
     live = []
     foreign_namespace_ids = []
-    for process in qs.order_by("started_at", "created_at").iterator(chunk_size=20):
+    for process in qs.order_by("started_at", "created_at").paged_iterator(chunk_size=20):
         if not process.shares_pid_namespace:
             foreign_namespace_ids.append(process.id)
             continue

archivebox/core/urls.py

@@ -26,15 +26,14 @@ from archivebox.core.views import (
 )
 from archivebox.progressmonitor.views import live_progress_view
 from archivebox.search.views import public_snapshot_search_stream_view
-from archivebox.opencode.views import opencode_proxy_view
+from abx_plugins.plugins.opencode.archivebox.urls import urlpatterns as agent_urlpatterns
 
 urlpatterns = [
     re_path(r"^static/(?P<path>.*)$", serve_static),
     path("robots.txt", static.serve, {"document_root": CONSTANTS.STATIC_DIR, "path": "robots.txt"}),
     path("favicon.ico", static.serve, {"document_root": CONSTANTS.STATIC_DIR, "path": "favicon.ico"}),
     path("docs/", RedirectView.as_view(url="https://github.com/ArchiveBox/ArchiveBox/wiki"), name="Docs"),
-    re_path(r"^admin/agent/?(?=$|opencode)", include("archivebox.opencode.urls")),
-    re_path(r"^(?P<path>assets/.*)$", opencode_proxy_view, name="opencode-assets"),
+    *agent_urlpatterns,
     path("public/search-stream/", public_snapshot_search_stream_view, name="public-search-stream"),
     path("public/", PublicIndexView.as_view(), name="public-index"),
     path("public.html", RedirectView.as_view(url="/public/"), name="public-index-html"),

archivebox/core/views.py

@@ -122,7 +122,7 @@ def _find_snapshot_by_ref(snapshot_ref: str) -> Snapshot | None:
     if len(lookup) == 12 and "-" not in lookup:
         # Resolve suffixes using the covering primary-key index before fetching
         # wide snapshot rows and joining their crawl/user data.
-        matches = list(Snapshot.objects.filter(id__endswith=lookup).order_by().values_list("id", flat=True))
+        matches = Snapshot.objects.filter(id__endswith=lookup).order_by().values_list("id", flat=True)
         return snapshots.filter(pk__in=matches).order_by("-created_at", "-downloaded_at").first()
 
     try:
@@ -506,7 +506,7 @@ class SnapshotView(View):
         def _resolve_snapshots_for_slug(slug: str):
             # full URLs go straight to the url-only path (fast, indexed)
             if "://" in slug:
-                return SnapshotView.find_snapshots_for_url(slug)
+                return SnapshotView.find_snapshots_for_url(slug, allow_fallback=False)
             # short uuid-shaped slugs (>=8 hex chars after stripping non-hex) try id matching first
             id_qs = SnapshotView.find_snapshots_for_id(slug)
             if id_qs.exists():
@@ -517,6 +517,12 @@ class SnapshotView(View):
         try:
             if "://" in path:
                 snapshot = snapshots.order_by("-bookmarked_at").first()
+                if snapshot is None:
+                    # Fetch the exact match once before trying legacy URL
+                    # fallbacks; an exists() probe followed by the same lookup
+                    # needlessly doubled reads on every normal redirect.
+                    snapshots = direct_snapshots_queryset(request, self.find_snapshots_for_url(path)).select_related("crawl__created_by")
+                    snapshot = snapshots.order_by("-bookmarked_at").first()
                 if snapshot is None:
                     raise Snapshot.DoesNotExist
             else:
@@ -1228,7 +1234,6 @@ class PublicIndexView(ListView):
     model = Snapshot
     ordering: ClassVar[list[str]] = ["-bookmarked_at", "-created_at"]
     paginator_class = AcceleratedPaginator
-    public_page_scan_chunk_size = 50
 
     def get_paginate_by(self, queryset):
         runtime_config = self.__dict__.get("runtime_config")
@@ -1256,31 +1261,31 @@ class PublicIndexView(ListView):
         )
 
     def _ordered_public_page_from_order_index(self, *, page_number: int, page_size: int) -> list[Snapshot]:
-        # One extra row determines Next without a full collection count. Cached
-        # totals can be stale while the runner adds/removes snapshots, so they
-        # must never decide whether the requested page exists.
-        target_count = page_number * page_size + 1
-        public_snapshots: list[Snapshot] = []
-        scanned = 0
-        chunk_size = max(self.public_page_scan_chunk_size, page_size + 1)
-        # Read in snapshot_public_order_idx order and filter visibility in small
-        # batches. Combining the permissions predicate with ORDER BY can make
-        # SQLite select the permissions index and sort the whole public archive
-        # before LIMIT. This favors the common early pages without a new index;
-        # deep pages or mostly private collections can still require more reads.
-        ordered_snapshots = (
-            Snapshot.objects.select_related("crawl__created_by").order_by(*self.ordering).only(*self._base_public_snapshot_fields())
-        )
-
-        while len(public_snapshots) < target_count:
-            chunk = list(ordered_snapshots[scanned : scanned + chunk_size])
-            if not chunk:
-                break
-            scanned += len(chunk)
-            public_snapshots.extend(snapshot for snapshot in chunk if snapshot.permissions == PERMISSIONS_PUBLIC)
-
+        # Walk the existing chronological index in bounded keyset pages,
+        # reading only identity/visibility until reaching the requested page.
+        # Repeated OFFSET batches used to revisit every earlier row AND join
+        # owners each time, making deep pages quadratic. Filtering visibility
+        # in SQL instead made SQLite sort the entire permissions index first.
+        # Each helper page closes its cursor before yielding; only the final
+        # displayed IDs load wide rows/owners. No new index or SQLite hint.
         start = (page_number - 1) * page_size
-        return public_snapshots[start:target_count]
+        visible_count = 0
+        page_ids = []
+        candidates = Snapshot.objects.only("id", "permissions").order_by(*self.ordering, "id")
+        for snapshot in candidates.paged_iterator(chunk_size=max(100, page_size + 1)):
+            if snapshot.permissions != PERMISSIONS_PUBLIC:
+                continue
+            if visible_count >= start:
+                page_ids.append(snapshot.pk)
+                if len(page_ids) > page_size:
+                    break
+            visible_count += 1
+        return list(
+            Snapshot.objects.filter(pk__in=page_ids, permissions=PERMISSIONS_PUBLIC)
+            .select_related("crawl__created_by")
+            .order_by(*self.ordering, "id")
+            .only(*self._base_public_snapshot_fields()),
+        )
 
     def paginate_queryset(self, queryset, page_size):
         if self.request.GET.get("q", default="").strip():

archivebox/crawls/admin.py

@@ -947,7 +947,7 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         updated = 0
         batch = []
         crawls_to_update = []
-        for crawl in queryset.only("id", "config", "permissions").iterator(chunk_size=500):
+        for crawl in queryset.only("id", "config", "permissions").order_by("pk").paged_iterator(chunk_size=500):
             old_permissions = crawl.permissions
             config = dict(crawl.config or {})
             config["PERMISSIONS"] = permissions

archivebox/crawls/models.py

@@ -23,6 +23,7 @@ from archivebox.config.common import rprint as print
 from archivebox.core.permissions import PERMISSIONS_VALUES, normalize_permissions
 
 from archivebox.base_models.models import (
+    PagedQuerySet,
     ModelWithUUID,
     ModelWithDeleteAfter,
     ModelWithOutputDir,
@@ -148,6 +149,8 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes):
 
 
 class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWithQueue):
+    objects = PagedQuerySet.as_manager()
+
     id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
@@ -229,7 +232,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         if paused and save and self.pk:
             from archivebox.core.models import Snapshot
 
-            for snapshot in self.snapshot_set.exclude(status__in=Snapshot.FINAL_STATES).iterator():
+            for snapshot in self.snapshot_set.exclude(status__in=Snapshot.FINAL_STATES).order_by("pk").paged_iterator():
                 snapshot.pause()
         return paused
 
@@ -301,7 +304,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
 
     @classmethod
     def missing_delete_at_candidates(cls):
-        return cls.objects.filter(delete_at__isnull=True, config__has_key="DELETE_AFTER")
+        return cls.objects.filter(delete_at__isnull=True, config__has_key="DELETE_AFTER").only("id", "created_at", "delete_at", "config")
 
     def save(self, *args, **kwargs):
         update_fields = kwargs.get("update_fields")
@@ -368,7 +371,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         batch = []
         updated = 0
         queryset = self.snapshot_set.filter(Q(permissions=old_permissions) | Q(permissions__isnull=True)).only("id", "config")
-        for snapshot in queryset.iterator(chunk_size=500):
+        for snapshot in queryset.order_by("pk").paged_iterator(chunk_size=500):
             config = dict(snapshot.config or {})
             config["PERMISSIONS"] = normalized_new_permissions
             snapshot.config = config
@@ -427,7 +430,11 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
                 tags_by_name = {tag.name: tag for tag in Tag.objects.filter(name__in=added_names)}
 
             tag_ids = [tag.pk for tag_name in added_names if (tag := tags_by_name.get(tag_name))]
-            snapshot_ids = Snapshot.objects.filter(crawl=self).values_list("id", flat=True).iterator(chunk_size=5000)
+            # Fetch each page before writing M2M rows. iterator(chunk_size=...)
+            # would keep one read cursor open across every bulk write.
+            snapshot_ids = (
+                snapshot.pk for snapshot in Snapshot.objects.filter(crawl=self).only("id").order_by("pk").paged_iterator(chunk_size=500)
+            )
             while True:
                 batch_snapshot_ids = list(islice(snapshot_ids, 5000))
                 if not batch_snapshot_ids:

archivebox/machine/migrations/0023_repair_binary_field_indexes.py

@@ -0,0 +1,27 @@
+from django.db import migrations, models
+
+
+def repair_binary_field_indexes(apps, schema_editor):
+    Binary = apps.get_model("machine", "Binary")
+    connection = schema_editor.connection
+    with connection.cursor() as cursor:
+        constraints = connection.introspection.get_constraints(cursor, Binary._meta.db_table)
+    indexed_columns = {
+        tuple(constraint["columns"])
+        for constraint in constraints.values()
+        if constraint.get("index") or constraint.get("unique") or constraint.get("primary_key")
+    }
+    # 0005's legacy raw-SQL table creation omitted field indexes even though
+    # migration state declares them. Editing 0005 alone cannot repair already
+    # upgraded collections. Restore only missing single-field indexes, checking
+    # columns rather than generated names so fresh SQLite/PostgreSQL installs
+    # retain their existing indexes without duplicates or a schema-state change.
+    for field in Binary._meta.local_fields:
+        if field.db_index and not field.unique and (field.column,) not in indexed_columns:
+            schema_editor.add_index(Binary, models.Index(fields=[field.name], name=f"mach_binary_{field.column}_idx"))
+
+
+class Migration(migrations.Migration):
+    dependencies = [("machine", "0022_networkinterface_identity_without_mac")]
+
+    operations = [migrations.RunPython(repair_binary_field_indexes, migrations.RunPython.noop)]

archivebox/machine/models.py

@@ -20,7 +20,7 @@ from django.utils.functional import cached_property
 
 from archivebox.config import CONSTANTS
 from archivebox.config.common import rprint
-from archivebox.base_models.models import ModelWithDeleteAfter, ModelWithHealthStats, normalize_config_json_values
+from archivebox.base_models.models import PagedQuerySet, ModelWithDeleteAfter, ModelWithHealthStats, normalize_config_json_values
 from archivebox.workers.models import ModelWithQueue
 from .detect import get_host_guid, get_os_info, get_vm_info, get_host_network, get_host_stats
 
@@ -48,7 +48,6 @@ _CURRENT_PROCESS: Process | None = None
 MACHINE_RECHECK_INTERVAL = 7 * 24 * 60 * 60
 NETWORK_INTERFACE_RECHECK_INTERVAL = 1 * 60 * 60
 BINARY_RECHECK_INTERVAL = 1 * 30 * 60
-PROCESS_RECHECK_INTERVAL = 60  # Re-validate every 60 seconds
 PID_REUSE_WINDOW = timedelta(hours=24)  # Max age for considering a PID match valid
 PROCESS_TIMEOUT_GRACE = timedelta(seconds=30)  # Extra margin before force-cleaning timed-out RUNNING rows
 START_TIME_TOLERANCE = 5.0  # Seconds tolerance for start time matching
@@ -255,40 +254,40 @@ class Machine(ModelWithHealthStats):
                 **get_vm_info(),
                 stats=get_host_stats(),
             )
-        else:
-            if timezone.now() >= machine.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL):
-                for key, value in {
-                    "hostname": socket.gethostname(),
-                    **get_os_info(),
-                    **get_vm_info(),
-                    "stats": get_host_stats(),
-                }.items():
-                    setattr(machine, key, value)
-                machine.save(
-                    update_fields=[
-                        "hostname",
-                        "hw_in_docker",
-                        "hw_in_vm",
-                        "hw_manufacturer",
-                        "hw_product",
-                        "hw_uuid",
-                        "os_arch",
-                        "os_family",
-                        "os_platform",
-                        "os_release",
-                        "os_kernel",
-                        "stats",
-                        "modified_at",
-                    ],
-                )
-        machine = cls._sanitize_config(machine)
-        # Reconcile once, on the first Machine.current() call in this process.
+        needs_recheck = timezone.now() >= machine.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL)
+        # Reconcile before any save can mirror stale DB config over file edits.
         try:
             from archivebox.config.collection import sync_machine_and_file
 
             sync_machine_and_file(machine)
         except Exception:
             pass
+        if needs_recheck:
+            for key, value in {
+                "hostname": socket.gethostname(),
+                **get_os_info(),
+                **get_vm_info(),
+                "stats": get_host_stats(),
+            }.items():
+                setattr(machine, key, value)
+            machine.save(
+                update_fields=[
+                    "hostname",
+                    "hw_in_docker",
+                    "hw_in_vm",
+                    "hw_manufacturer",
+                    "hw_product",
+                    "hw_uuid",
+                    "os_arch",
+                    "os_family",
+                    "os_platform",
+                    "os_release",
+                    "os_kernel",
+                    "stats",
+                    "modified_at",
+                ],
+            )
+        machine = cls._sanitize_config(machine)
         # Publish only after initialization is complete.
         _CURRENT_MACHINE = machine
         return machine
@@ -473,7 +472,7 @@ class NetworkInterface(ModelWithHealthStats):
         return _CURRENT_INTERFACE
 
 
-class BinaryManager(models.Manager):
+class BinaryManager(models.Manager.from_queryset(PagedQuerySet)):
     def get_from_db_or_cache(self, name: str, abspath: str = "", version: str = "", sha256: str = "", binprovider: str = "env") -> Binary:
         """Get or create an Binary record from the database or cache."""
         cached = _CURRENT_BINARIES.get(name)
@@ -890,7 +889,7 @@ class Binary(ModelWithHealthStats, ModelWithQueue):
 # =============================================================================
 
 
-class ProcessManager(models.Manager):
+class ProcessManager(models.Manager.from_queryset(PagedQuerySet)):
     """Manager for Process model."""
 
     def current(self) -> Process:
@@ -1209,8 +1208,11 @@ class Process(ModelWithDeleteAfter, models.Model):
 
     @classmethod
     def missing_delete_at_candidates(cls):
-        return cls.objects.filter(delete_at__isnull=True).filter(
-            Q(env__has_key="DELETE_AFTER") | Q(machine__config__has_key="DELETE_AFTER"),
+        return (
+            cls.objects.filter(delete_at__isnull=True)
+            .filter(Q(env__has_key="DELETE_AFTER") | Q(machine__config__has_key="DELETE_AFTER"))
+            .select_related("machine")
+            .only("id", "created_at", "delete_at", "env", "machine__config")
         )
 
     # Properties that delegate to related objects
@@ -1410,6 +1412,13 @@ class Process(ModelWithDeleteAfter, models.Model):
     # Process.current() and hierarchy methods
     # =========================================================================
 
+    @classmethod
+    def current_readonly(cls) -> Process | None:
+        """Return this process's registered identity without registration or probes."""
+        if _CURRENT_PROCESS is not None and _CURRENT_PROCESS.pid == os.getpid():
+            return _CURRENT_PROCESS
+        return None
+
     @classmethod
     def current(cls) -> Process:
         """
@@ -1427,40 +1436,19 @@ class Process(ModelWithDeleteAfter, models.Model):
 
         current_pid = os.getpid()
 
-        # Fast path used by model save diagnostics and hot runner loops. A
-        # cached Process object is valid when the immutable identity we wrote
-        # at creation time still describes this Python process. PID reuse cannot
-        # happen while this process is alive, so pid + present started_at/cmd is
-        # enough here; the slower psutil validation below remains the fallback
-        # for missing/stale cache.
-        if (
-            _CURRENT_PROCESS
-            and _CURRENT_PROCESS.pid == current_pid
-            and _CURRENT_PROCESS.status == cls.StatusChoices.RUNNING
-            and timezone.now() < _CURRENT_PROCESS.modified_at + timedelta(seconds=PROCESS_RECHECK_INTERVAL)
-            and _CURRENT_PROCESS.started_at is not None
-            and bool(_CURRENT_PROCESS.cmd)
-        ):
-            return _CURRENT_PROCESS
+        # This is process identity, not a health check. A live Python process
+        # cannot reuse its own PID, and a fork fails current_readonly's PID
+        # check. Expiring this cache by modified_at made every call after 60s
+        # query the database and touch logs until something else saved the row.
+        # Explicit lifecycle/recovery code owns liveness and interface refresh.
+        cached = cls.current_readonly()
+        if cached is not None:
+            return cached
+        _CURRENT_PROCESS = None
 
         machine = Machine.current()
         iface = NetworkInterface.current()
 
-        # Check cache validity
-        if _CURRENT_PROCESS:
-            # Verify: same PID, same machine, cache not expired
-            if (
-                _CURRENT_PROCESS.pid == current_pid
-                and _CURRENT_PROCESS.machine_id == machine.id
-                and timezone.now() < _CURRENT_PROCESS.modified_at + timedelta(seconds=PROCESS_RECHECK_INTERVAL)
-            ):
-                if _CURRENT_PROCESS.iface_id != iface.id:
-                    _CURRENT_PROCESS.iface = iface
-                    _CURRENT_PROCESS.save(update_fields=["iface", "modified_at"])
-                _CURRENT_PROCESS.ensure_log_files()
-                return _CURRENT_PROCESS
-            _CURRENT_PROCESS = None
-
         # Get actual process start time from OS for validation
         os_start_time = None
         if PSUTIL_AVAILABLE:
@@ -1640,9 +1628,8 @@ class Process(ModelWithDeleteAfter, models.Model):
         if machine is not None:
             stale = stale.filter(machine=machine)
 
-        # Recovery can run against damaged DB state; stream rows so a large
-        # stale Process backlog cannot be materialized in memory at once.
-        for proc in stale.iterator(chunk_size=100):
+        # Close each bounded read before inspecting processes or writing their state.
+        for proc in stale.order_by("pk").paged_iterator(chunk_size=100):
             shares_pid_namespace = proc.shares_pid_namespace
             if shares_pid_namespace and proc.poll() is not None:
                 cleaned += 1
@@ -2582,14 +2569,17 @@ class Process(ModelWithDeleteAfter, models.Model):
         """
         cleaned = 0
 
+        # Select active IDs through the existing status index first. Combining
+        # type/status directly let SQLite prefer the historical type index and
+        # scan 350k processes to return zero workers (blocking writes for 12s).
+        active_ids = cls.objects.filter(status=cls.StatusChoices.RUNNING).order_by().values("pk")
         running_children = cls.objects.filter(
+            pk__in=active_ids,
             process_type__in=[cls.TypeChoices.WORKER, cls.TypeChoices.HOOK],
-            status=cls.StatusChoices.RUNNING,
         )
 
-        # Recovery can run against damaged DB state; stream rows so a large
-        # orphaned Process backlog cannot be materialized in memory at once.
-        for proc in running_children.iterator(chunk_size=100):
+        # Close each bounded read before inspecting processes or writing their state.
+        for proc in running_children.order_by("pk").paged_iterator(chunk_size=100):
             if not proc.is_running:
                 proc.mark_exited(
                     exit_code=proc.exit_code if proc.exit_code is not None else _default_exit_code_for_unowned_process(proc.process_type),

archivebox/misc/checks.py

@@ -194,7 +194,10 @@ def check_not_root():
     if os.geteuid() == 0 and not (is_getting_help or is_getting_version):
         print("[yellow][!] Running ArchiveBox as root is not recommended.[/yellow]", file=sys.stderr)
         print("    Root-owned DATA_DIR files may be inaccessible to non-root users later.", file=sys.stderr)
-        print("        https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#do-not-run-as-root", file=sys.stderr)
+        print(
+            "        https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#run-archivebox-as-an-unprivileged-user",
+            file=sys.stderr,
+        )
 
 
 def is_archivebox_source_root(path: Path | str | None = None) -> bool:
@@ -255,7 +258,7 @@ def check_data_dir_permissions(config=None, **config_kwargs):
             "    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#permissions]https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#permissions[/link]",
         )
         STDERR.print(
-            "    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#puid--pgid]https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#puid--pgid[/link]",
+            "    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#docker-permissions-issues]https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#docker-permissions-issues[/link]",
         )
         STDERR.print(
             "    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#filesystem-doesnt-support-fsync-eg-network-mounts]https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#filesystem-doesnt-support-fsync-eg-network-mounts[/link]",

archivebox/misc/db.py

@@ -240,89 +240,38 @@ def drop_models_on_postgres(apps, schema_editor, app_label: str, model_names: li
             schema_editor.delete_model(model)
 
 
-def run_db_analyze_batch(
-    remaining: list[str] | None,
-    *,
-    max_seconds_per_table: float = 120.0,
-) -> list[str]:
-    """Advance one step of a batched SQLite ``ANALYZE`` sweep.
+def run_db_analyze_batch(remaining: list[str] | None) -> list[str]:
+    """Refresh approximate SQLite planner stats for one table per idle tick.
 
-    Without periodic ANALYZE the optimizer's table stats go stale as
-    snapshot/archiveresult tables grow, causing it to start large joins from
-    ``auth_user`` instead of using the indexed url column and blowing snapshot
-    detail page render time from ~50ms to ~500ms+.
-
-    The whole sweep is spread across many calls instead of running as one
-    blocking ``ANALYZE``: pass ``None`` to start a fresh sweep (this call
-    enumerates user tables and runs ``ANALYZE`` on the first one); pass the
-    returned list to advance one more table on each subsequent call. An
-    empty return value means the sweep is complete (or has been aborted) and
-    the next caller should pass ``None`` again. Caller is responsible for
-    throttling new sweeps (orchestrator starts at most one per 24hr while
-    idle) and enforcing a hard upper bound on total sweep wall time.
-
-    Safety guarantees:
-
-    - **Never raises**: every database call is wrapped; on any failure the
-      function returns ``[]`` (abandoning the rest of the sweep) so the
-      orchestrator never crashes on maintenance errors.
-    - **Bounded per-call wall time**: a SQLite progress handler aborts the
-      current ``ANALYZE`` statement once ``max_seconds_per_table`` is
-      exceeded, so a single pathological table cannot wedge the call.
-    - **Never leaves the db locked**: each ``ANALYZE`` runs as a single
-      statement transaction that auto-commits (or rolls back on
-      abort/error). The cursor and progress handler are always cleaned up
-      in ``finally`` blocks even if Python raises mid-call.
-    - Silent no-op on non-SQLite backends.
-
-    WAL journal mode (set in Django settings) keeps readers fully unblocked
-    throughout; the writer lock is only held for the brief ``sqlite_stat*``
-    flush after each table completes.
+    A full ANALYZE scans every index while holding a write transaction; moving
+    it into the runner did not prevent it from blocking submissions. Sampling
+    at most about 1000 entries per index keeps maintenance independent of
+    collection size. Approximate stats are sufficient for choosing indexes.
+    PostgreSQL owns its statistics through autovacuum/autoanalyze.
     """
     from django.db import connection
 
     if connection.vendor != "sqlite":
         return []
-
-    if remaining is None:
-        try:
-            with connection.cursor() as cursor:
-                cursor.execute(
-                    "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name",
-                )
-                remaining = [row[0] for row in cursor.fetchall()]
-        except Exception:
-            return []
-
-    if not remaining:
-        return []
-
-    next_table, *rest = remaining
-    raw_conn = connection.connection
-    progress_handler_set = False
-    if raw_conn is not None and max_seconds_per_table > 0:
-        deadline = time.monotonic() + max_seconds_per_table
-        try:
-            raw_conn.set_progress_handler(lambda: 1 if time.monotonic() > deadline else 0, 10000)
-            progress_handler_set = True
-        except Exception:
-            progress_handler_set = False
-
     try:
         with connection.cursor() as cursor:
-            cursor.execute(f'ANALYZE "{next_table}"')
-    except Exception:
-        # Aborted by progress handler, locked db, or any other failure — skip
-        # this table and continue the sweep. ANALYZE is idempotent so we can
-        # retry on the next 24hr sweep.
-        pass
-    finally:
-        if progress_handler_set and raw_conn is not None:
+            if remaining is None:
+                cursor.execute("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name")
+                remaining = [row[0] for row in cursor.fetchall()]
+            if not remaining:
+                return []
+            next_table, *rest = remaining
+            cursor.execute("PRAGMA analysis_limit")
+            previous_limit = int(cursor.fetchone()[0])
             try:
-                raw_conn.set_progress_handler(None, 0)
-            except Exception:
-                pass
-    return rest
+                cursor.execute("PRAGMA analysis_limit = 1000")
+                cursor.execute(f"ANALYZE {connection.ops.quote_name(next_table)}")
+            finally:
+                cursor.execute(f"PRAGMA analysis_limit = {previous_limit}")
+            return rest
+    except Exception:
+        # Optional planner maintenance must not prevent queued captures.
+        return []
 
 
 def compact_command(cmdline: list[str] | None, fallback: str = "") -> str:

archivebox/misc/jsonl.py

@@ -142,3 +142,27 @@ def write_record(record: dict[str, Any], stream: TextIO | None = None) -> None:
     active_stream: TextIO = sys.stdout if stream is None else stream
     active_stream.write(json.dumps(record) + "\n")
     active_stream.flush()
+
+
+def write_records(records: Iterable[dict[str, Any]], stream: TextIO | None = None) -> int:
+    """Stream bulk JSONL output with bounded buffering, returning its row count."""
+    active_stream: TextIO = sys.stdout if stream is None else stream
+    lines: list[str] = []
+    pending_size = 0
+    count = 0
+    for record in records:
+        line = json.dumps(record) + "\n"
+        lines.append(line)
+        pending_size += len(line)
+        count += 1
+        # CLI stdout can be unbuffered. A syscall per snapshot makes large
+        # exports IO-bound; small batches keep memory and pipe latency bounded.
+        if pending_size >= 64 * 1024:
+            active_stream.write("".join(lines))
+            active_stream.flush()
+            lines.clear()
+            pending_size = 0
+    if lines:
+        active_stream.write("".join(lines))
+        active_stream.flush()
+    return count

archivebox/misc/serve_static.py

@@ -32,6 +32,7 @@ from archivebox.misc.logging_util import printable_filesize
 from archivebox.plugins.discovery import render_plugin_full_response
 
 _HASHES_CACHE: dict[Path, tuple[float, dict[str, str]]] = {}
+DIRECTORY_PREVIEW_MAX_BYTES = 100 * 1024
 FAVICON_CACHE_CONTROL = "public, max-age=31536000, s-maxage=31536000, immutable"
 IMG_SRC_ATTR_RE = re.compile(r'(<img\b[^>]*?\s(?:src|data-src)=["\'])([^"\']+)(["\'])', re.IGNORECASE)
 TRANSFORMED_HTML_PREVIEW_STYLE = """<style id="archivebox-static-html-preview-style">
@@ -69,6 +70,192 @@ a > img:not([width]):not([height]) {
 </style>"""
 
 
+def add_replay_image_fallback(response: HttpResponse) -> HttpResponse:
+    """Add a best-effort responsive-image fallback to the locally served replay worker.
+
+    WHY: a recorder saves the image Chrome selected, not every srcset candidate.
+    Replaying at another viewport/DPR can select an uncaptured URL (the motivating
+    case was Drive's missing 2x logo with its 1x logo already in the same WACZ).
+    WACZ requests are answered inside Webrecorder's service worker, so a Django
+    404 handler cannot repair them. Append this adapter when serving that worker;
+    do not change captures, request more live assets, or resize the recorded page.
+
+    Only an image GET that already returned 404 is eligible. The worker reads the
+    referring HTML from the SAME archive and tries src/srcset candidates declared
+    on that img or its enclosing picture. It never guesses filename substitutions
+    or borrows an unrelated nearby image. The original response wins on absent
+    metadata, unsupported worker APIs, parse errors, timeout, or failed candidates.
+    This deliberately does not promise to fix JS-only srcsets, CSS backgrounds,
+    corrupt images returning 200, or pages whose archived HTML is too large.
+
+    Keep this self-contained in the static-file server: it is a replay repair,
+    independent of the capturing plugin/site. Feature detection makes upstream
+    worker changes disable the repair rather than break ordinary replay. The
+    archive's HTML/ZIP bytes and successful HTTP responses remain untouched.
+    """
+    if (
+        response.status_code != 200
+        or response.streaming
+        or not response.has_header("Service-Worker-Allowed")
+        or "javascript" not in response.get("Content-Type", "")
+    ):
+        return response
+    script = rb"""
+;(() => {
+  try {
+    const replay = self.sw;
+    if (!replay || typeof replay.handleFetch !== "function" ||
+        typeof replay.getResponseFor !== "function" || !replay.collections ||
+        replay.proxyOriginMode || replay.topFramePassthrough || replay.__abxImageFallback) return;
+    const original = replay.handleFetch;
+    const documents = new Map();
+    const decode = value => value.replace(/&(?:amp|quot|apos|lt|gt|#\d+|#x[\da-f]+);/gi, entity => {
+      const named = {"&amp;":"&", "&quot;":'"', "&apos;":"'", "&lt;":"<", "&gt;":">"};
+      return named[entity.toLowerCase()] || String.fromCodePoint(
+        entity[2].toLowerCase() === "x" ? parseInt(entity.slice(3), 16) : parseInt(entity.slice(2), 10));
+    });
+    const attributes = tag => {
+      const attrs = {};
+      for (const m of tag.matchAll(/([^\s=/>]+)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+))/g)) {
+        attrs[m[1].toLowerCase()] = decode(m[2] ?? m[3] ?? m[4]);
+      }
+      return attrs;
+    };
+    const sourceSet = value => {
+      // URLs may contain commas (e.g. image-CDN transforms). Like HTML's
+      // srcset tokenizer, consume a URL through whitespace, then descriptors.
+      const urls = [];
+      let rest = value || "";
+      while (rest && urls.length < 64) {
+        rest = rest.replace(/^[\s,]+/, "");
+        const token = rest.match(/^\S+/)?.[0];
+        if (!token) break;
+        urls.push(token.replace(/,+$/, ""));
+        rest = rest.slice(token.length);
+        if (!token.endsWith(",")) rest = rest.replace(/^[^,]*(?:,|$)/, "");
+      }
+      return urls;
+    };
+    const splitReplay = (url, prefix) => {
+      if (!url.startsWith(prefix)) return null;
+      return url.slice(prefix.length).match(/^((?::[^/]+\/)?\d{0,14})(?:[a-z]+_)?\/((?:https?:)?\/\/.+)$/);
+    };
+    replay.handleFetch = async function(event) {
+      // Never catch/retry the original operation: preserve its normal behavior.
+      const response = await original.call(this, event);
+      const request = event.request;
+      if (response.status !== 404 || request.method !== "GET" || request.destination !== "image") return response;
+      let expired = false, timer;
+      try {
+        const repair = async () => {
+          if (!request.url.startsWith(this.replayPrefix)) return response;
+          const name = this.collections.root || request.url.slice(this.replayPrefix.length).split("/", 1)[0];
+          const collection = await this.collections.getColl(name);
+          // Do not operate on live/proxy collections or remote source archives.
+          // lookupUrl is the archive index API, absent on the live proxy store.
+          if (!collection || typeof collection.store?.lookupUrl !== "function" || collection.liveRedirectOnNotFound ||
+              new URL(collection.config.sourceUrl).origin !== self.location.origin) return response;
+          const missing = splitReplay(request.url, collection.prefix);
+          const referring = splitReplay(request.referrer, collection.prefix);
+          if (!missing || !referring) return response;
+          const archiveURL = url => collection.prefix + referring[1] + "id_/" + url;
+          const read = url => this.getResponseFor(new Request(archiveURL(url), {
+            credentials: "same-origin", redirect: "manual", signal: request.signal,
+          }), event);
+          const key = archiveURL(referring[2]);
+          if (!documents.has(key)) {
+            // Cache only candidate metadata, never whole HTML or image bodies.
+            // Both entry count and HTML reads are bounded; WACZ range loading
+            // remains the upstream replay engine's responsibility.
+            if (documents.size >= 16) documents.delete(documents.keys().next().value);
+            documents.set(key, (async () => {
+              const html = await read(referring[2]);
+              if (html.status !== 200 || !html.headers.get("content-type")?.includes("text/html")) return [];
+              const reader = html.body.getReader(), decoder = new TextDecoder();
+              let text = "", size = 0;
+              try {
+                while (true) {
+                  const chunk = await reader.read();
+                  if (chunk.done) break;
+                  size += chunk.value.byteLength;
+                  if (expired || size > 2 * 1024 * 1024) return [];
+                  text += decoder.decode(chunk.value, {stream:true});
+                }
+                text += decoder.decode();
+              } finally {await reader.cancel().catch(() => {});}
+              // Ignore tag-shaped strings in scripts/comments. Parsing is
+              // intentionally conservative; an odd/malformed page stays broken
+              // instead of acquiring an unrelated "similar looking" image.
+              text = text.replace(/<!--[\s\S]*?-->|<(script|style)\b[^>]*>[\s\S]*?<\/\1\s*>/gi, "");
+              const groups = [];
+              let base = referring[2], picture = null;
+              const absolute = value => {
+                try {
+                  const url = new URL(value, base);
+                  if (!/^https?:$/.test(url.protocol) || url.username || url.password) return null;
+                  url.hash = "";
+                  return url.href;
+                } catch {return null;}
+              };
+              for (const token of text.matchAll(/<\/?(?:base|picture|source|img)\b(?:"[^"]*"|'[^']*'|[^'">])*>/gi)) {
+                const tag = token[0], attrs = attributes(tag);
+                if (/^<base\b/i.test(tag)) {if (attrs.href) base = absolute(attrs.href) || base; continue;}
+                if (/^<picture\b/i.test(tag)) {picture = []; continue;}
+                if (/^<\/picture\b/i.test(tag)) {picture = null; continue;}
+                const urls = [...new Set([attrs.src, ...sourceSet(attrs.srcset)].filter(Boolean).map(absolute).filter(Boolean))];
+                if (/^<source\b/i.test(tag)) {if (picture) picture.push(...urls); continue;}
+                if (!attrs.srcset && !picture?.length) continue;
+                groups.push([...new Set([...urls, ...(picture || [])])].slice(0, 64));
+                if (groups.length >= 512) break;
+              }
+              return groups;
+            })());
+          }
+          const groups = await documents.get(key);
+          const failed = new URL(missing[2], referring[2]); failed.hash = "";
+          const candidates = groups.find(group => group.includes(failed.href));
+          if (!candidates) return response;
+          // Prefer the img's explicit src, then alternatives in declaration
+          // order. Limit extra archive lookups, and never follow redirects.
+          for (const url of candidates.filter(url => url !== failed.href).slice(0, 6)) {
+            if (expired) break;
+            const alternative = await read(url);
+            if (alternative.status !== 200 || !alternative.headers.get("content-type")?.startsWith("image/")) continue;
+            if (expired) break;
+            const headers = new Headers(alternative.headers);
+            headers.set("X-ArchiveBox-Image-Fallback", url);
+            headers.set("Cache-Control", "no-store");
+            return new Response(alternative.body, {status:200, headers});
+          }
+          return response;
+        };
+        return await Promise.race([
+          repair().catch(() => response),
+          new Promise(resolve => {timer = setTimeout(() => {expired = true; resolve(response);}, 1500);}),
+        ]);
+      } catch {return response;}
+      finally {expired = true; clearTimeout(timer);}
+    };
+    replay.__abxImageFallback = true;
+  } catch { /* Optional repair must never prevent the worker from starting. */ }
+})();
+"""
+    try:
+        body = response.content
+        if b"self.sw=" not in body or b"__abxImageFallback" in body:
+            return response
+        response.content = body + script
+        # Upstream headers describe the unmodified worker. Revalidate this
+        # generated asset so existing captures can receive the repair too.
+        for header in ("ETag", "Last-Modified", "Content-Length"):
+            if response.has_header(header):
+                del response[header]
+        response["Cache-Control"] = "no-cache"
+    except Exception:
+        pass
+    return response
+
+
 def _load_hash_map(snapshot_dir: Path) -> dict[str, str] | None:
     hashes_path = snapshot_dir / "hashes" / "hashes.json"
     if not hashes_path.exists():
@@ -126,7 +313,8 @@ def _resolve_archive_path(document_root: str | Path, rel_path: str) -> tuple[Pat
         current = match
         resolved_parts.append(match.name)
 
-    return current, posixpath.join(*resolved_parts) if resolved_parts else ""
+    resolved_path = posixpath.join(*resolved_parts) if resolved_parts else ""
+    return Path(safe_join(document_root, resolved_path)), resolved_path
 
 
 def _cache_policy(config=None, **config_kwargs) -> str:
@@ -308,10 +496,24 @@ def _render_directory_index(request, path: str, fullpath: Path) -> HttpResponse:
         file_list.append(url)
 
         stat_result = entry.stat()
+        preview_kind = ""
+        mime = mimetypes.guess_type(entry.name)[0] or "application/octet-stream"
+        if entry.is_file() and stat_result.st_size < DIRECTORY_PREVIEW_MAX_BYTES:
+            if mime.startswith("image/"):
+                preview_kind = "image"
+            elif (
+                mime.startswith("text/")
+                or re.search(r"(?:^|[.+/-])(?:json|xml|javascript|ecmascript|yaml|toml|graphql|sql)(?:[.+/-]|$)", mime)
+                or entry.suffix.lower() in {".log", ".jsonl", ".toml"}
+            ):
+                preview_kind = "text"
         entries.append(
             {
                 "name": url,
                 "url": url,
+                "preview_kind": preview_kind,
+                "mimetype": "Directory" if entry.is_dir() else mime,
+                "size_bytes": 0 if entry.is_dir() else stat_result.st_size,
                 "is_dir": entry.is_dir(),
                 "size": "—" if entry.is_dir() else printable_filesize(stat_result.st_size),
                 "timestamp": _format_direntry_timestamp(stat_result),
@@ -851,7 +1053,8 @@ def _is_risky_replay_document(fullpath: Path, content_type: str) -> bool:
     # Unknown archived response paths often have no extension. Sniff a small prefix
     # so one-domain no-JS mode still catches HTML/SVG documents.
     try:
-        head = fullpath.read_bytes()[:4096].decode("utf-8", errors="ignore").lower()
+        with fullpath.open("rb") as source:
+            head = source.read(4096).decode("utf-8", errors="ignore").lower()
     except (OSError, UnicodeDecodeError):
         return False
 
@@ -1020,6 +1223,29 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
             config=config,
         )
 
+    # A shared archive explorer for ZIP outputs from any plugin. Raw/range
+    # requests retain the original bytes for downloads and seekable reads.
+    if fullpath.suffix.lower() == ".zip" and request.GET.get("preview") and not request.META.get("HTTP_RANGE"):
+        raw_query = request.GET.copy()
+        raw_query.pop("preview", None)
+        raw_query["raw"] = "1"
+        wrapped = loader.get_template("static/zip_index.html").render(
+            {
+                "title": fullpath.name,
+                "output_path": f"{request.path}?{raw_query.urlencode()}",
+                "archive_size": statobj.st_size,
+            },
+        )
+        response = HttpResponse(wrapped, content_type="text/html; charset=utf-8")
+        _set_transformed_response_headers(response, fullpath, statobj, None, cache_policy)
+        return _apply_archive_replay_headers(
+            response,
+            fullpath=fullpath,
+            content_type="text/html; charset=utf-8",
+            is_archive_replay=is_archive_replay,
+            config=config,
+        )
+
     # Wrap text-like outputs in HTML when explicitly requested for iframe previewing.
     if preview_as_text_html:
         try:

archivebox/misc/shell_welcome_message.py

@@ -48,10 +48,10 @@ if __name__ == "__main__":
 
     prnt("[i] :heavy_dollar_sign: Welcome to the ArchiveBox Shell!")
     prnt(
-        "    [deep_sky_blue4]Docs:[/deep_sky_blue4] [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Shell-Usage]https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Shell-Usage[/link]",
+        "    [deep_sky_blue4]Docs:[/deep_sky_blue4] [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage]https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage[/link]",
     )
     prnt(
-        "          [link=https://docs.archivebox.io/en/dev/apidocs/archivebox/archivebox.html]https://docs.archivebox.io/en/dev/apidocs/archivebox/archivebox.html[/link]",
+        "          [link=https://docs.archivebox.io/dev/apidocs/archivebox/archivebox.html]https://docs.archivebox.io/dev/apidocs/archivebox/archivebox.html[/link]",
     )
     prnt()
     prnt(" :grey_question: [violet]Hint[/] [i]Here are some examples to get started:[/]")

archivebox/opencode/__init__.py

@@ -1 +0,0 @@
-"""ArchiveBox OpenCode admin integration."""

archivebox/opencode/urls.py

@@ -1,13 +0,0 @@
-from django.urls import path, re_path
-
-from archivebox.opencode.views import agent_view, opencode_proxy_view
-
-
-urlpatterns = [
-    path("", agent_view, name="opencode-agent"),
-    re_path(
-        r"^opencode(?:/(?P<path>.*))?$",
-        opencode_proxy_view,
-        name="opencode-proxy",
-    ),
-]

archivebox/opencode/views.py

@@ -1,140 +0,0 @@
-"""Django adapter for the optional plugin; never import its runtime at startup."""
-
-import logging
-from io import BytesIO
-
-from asgiref.sync import sync_to_async
-
-from django.contrib.auth.middleware import AuthenticationMiddleware
-from django.contrib.auth.views import redirect_to_login
-from django.contrib.sessions.middleware import SessionMiddleware
-from django.core.exceptions import PermissionDenied
-from django.core.handlers.asgi import ASGIRequest
-from django.db import close_old_connections
-from django.http import Http404, HttpResponse, HttpResponseForbidden, StreamingHttpResponse
-from django.template import engines
-from django.urls import Resolver404, resolve
-from django.views.decorators.csrf import csrf_exempt
-
-from archivebox.config import CONSTANTS
-from archivebox.config.common import get_request_config
-from archivebox.core.middleware import ReverseProxyAuthMiddleware
-from archivebox.core.routes_util import build_admin_url, get_admin_host, get_api_base_url, get_base_url, host_matches
-from archivebox.plugins.discovery import get_plugin_template
-
-_LOGGER = logging.getLogger(__name__)
-
-
-def _runtime_settings(request, config):
-    from abx_plugins.plugins.opencode import runtime
-
-    settings = runtime._settings(config, CONSTANTS.DATA_DIR)
-    route_config = request.__dict__.get("archivebox_config")
-    settings.update(
-        archivebox_base_url=get_base_url(request=request, config=route_config).rstrip("/"),
-        archivebox_admin_url=build_admin_url("/admin/", request=request, config=route_config).rstrip("/"),
-        archivebox_api_url=f"{get_api_base_url(request=request, config=route_config).rstrip('/')}/api/",
-    )
-    return runtime, settings
-
-
-def _dispatch(request, path=None):
-    try:
-        # Middleware already merged server defaults and live Machine overrides.
-        # Re-resolving every extractor's config here stalls the UI request burst.
-        config = get_request_config(request).model_dump(mode="json")
-        if not config.get("OPENCODE_ENABLED", False):
-            raise Http404
-        if not request.user.is_authenticated:
-            return redirect_to_login(request.get_full_path(), login_url="/admin/login/")
-        if not request.user.is_active or not request.user.is_superuser:
-            return HttpResponseForbidden("Agent access requires a superuser account.")
-
-        runtime, settings = _runtime_settings(request, config)
-        if path is None:
-            from archivebox.core.admin_site import archivebox_admin
-
-            context = {**archivebox_admin.each_context(request), **runtime.agent_context(settings)}
-            source = get_plugin_template("opencode", "agent", fallback=False)
-            if source is None:
-                raise RuntimeError("Agent template unavailable")
-            return HttpResponse(engines["django"].from_string(source).render(context, request))
-
-        if not runtime._origin_allowed(request.method, request.get_host(), request.headers):
-            return HttpResponseForbidden("Cross-origin agent requests are blocked.")
-        status, headers, body = runtime.proxy(
-            settings,
-            request.method,
-            path,
-            tuple((key, value) for key, values in request.GET.lists() for value in values),
-            request.headers,
-            request.body,
-        )
-        response_type = HttpResponse if isinstance(body, bytes) else StreamingHttpResponse
-        response = response_type(body, status=status, headers=headers)
-        response.xframe_options_exempt = True
-        response.headers["X-Frame-Options"] = "SAMEORIGIN"
-        response.headers["Content-Security-Policy"] = "frame-ancestors 'self'"
-        return response
-    except Http404:
-        raise
-    except Exception:
-        # Optional-service boundary, including imports and template rendering.
-        _LOGGER.exception("Optional AI service failed")
-        return HttpResponse("AI service unavailable. See server logs.", status=503, content_type="text/plain")
-
-
-def agent_view(request):
-    return _dispatch(request)
-
-
-@csrf_exempt
-def opencode_proxy_view(request, path=None):
-    return _dispatch(request, path=path or "")
-
-
-def _websocket_context(scope):
-    close_old_connections()
-    try:
-        match = resolve(scope["path"])
-        if match.url_name != "opencode-proxy":
-            raise PermissionDenied
-        request = ASGIRequest(
-            {**scope, "type": "http", "method": "GET", "scheme": "https" if scope.get("scheme") == "wss" else "http"},
-            BytesIO(),
-        )
-        route_config = get_request_config(request, resolve_plugins=False)
-        request.archivebox_config = route_config
-        if not route_config.CONTROL_PLANE_ENABLED:
-            raise PermissionDenied
-        if (
-            route_config.USES_SUBDOMAIN_ROUTING
-            and route_config.BASE_URL
-            and not host_matches(request.get_host(), get_admin_host(config=route_config, request=request))
-        ):
-            raise PermissionDenied
-        SessionMiddleware(_dispatch).process_request(request)
-        AuthenticationMiddleware(_dispatch).process_request(request)
-        ReverseProxyAuthMiddleware(_dispatch).process_request(request)
-        config = route_config.model_dump(mode="json")
-        if not config.get("OPENCODE_ENABLED") or not request.user.is_active or not request.user.is_superuser:
-            raise PermissionDenied
-        runtime, settings = _runtime_settings(request, config)
-        if not request.headers.get("Origin") or not runtime._origin_allowed("POST", request.get_host(), request.headers):
-            raise PermissionDenied
-        return runtime, settings, match.kwargs.get("path", "")
-    finally:
-        close_old_connections()
-
-
-async def websocket_view(scope, receive, send):
-    if (await receive())["type"] != "websocket.connect":
-        return
-    try:
-        runtime, settings, path = await sync_to_async(_websocket_context)(scope)
-        await runtime.websocket_proxy(settings, path, scope.get("query_string", b""), scope.get("subprotocols", []), receive, send)
-    except (PermissionDenied, Resolver404, Http404):
-        await send({"type": "websocket.close", "code": 1008})
-    except Exception:
-        _LOGGER.exception("Optional AI WebSocket failed")
-        await send({"type": "websocket.close", "code": 1011})

archivebox/personas/forms.py

@@ -122,14 +122,14 @@ class PersonaAdminForm(PluginConfigFormMixin, forms.ModelForm):
                 "Use the custom path/CDP option if the browser data lives elsewhere."
             )
 
-        self.build_plugin_groups(get_config(persona=self.instance) if self.instance and self.instance.pk else get_config())
+        self.build_plugin_groups(get_config(persona=self.instance) if not self.instance._state.adding else get_config())
 
     def clean_name(self) -> str:
         name = str(self.cleaned_data.get("name") or "").strip()
         is_valid, error_message = validate_persona_name(name)
         if not is_valid:
             raise forms.ValidationError(error_message)
-        return name
+        return self.instance.validate_name(name)
 
     def clean(self) -> dict[str, Any]:
         cleaned_data = super().clean()

archivebox/personas/models.py

@@ -18,6 +18,7 @@ from typing import TYPE_CHECKING, Any
 from collections.abc import Mapping
 
 from django.db import IntegrityError, models
+from django.core.exceptions import ValidationError
 from django.db.models.fields.json import KT
 from django.conf import settings
 from django.utils import timezone
@@ -74,8 +75,12 @@ def derive_persona_config(*, name: str, config: Mapping[str, Any] | None, person
         derived["COOKIES_FILE"] = str(cookies_path)
 
     auth_path = persona_dir / "auth.json"
-    if "AUTH_STORAGE_FILE" not in derived and auth_path.exists():
-        derived["AUTH_STORAGE_FILE"] = str(auth_path)
+    if "AUTH_STORAGE_FILE" not in derived:
+        try:
+            if auth_path.stat().st_size > 0:
+                derived["AUTH_STORAGE_FILE"] = str(auth_path)
+        except FileNotFoundError:
+            pass
 
     derived["ACTIVE_PERSONA"] = name
     return derived
@@ -117,6 +122,13 @@ class Persona(ModelWithConfig):
         app_label = "personas"
 
     def save(self, *args, **kwargs):
+        # Adopt the spelling on disk, never rename a Chrome profile just to fix case.
+        self.__dict__.pop("_persona_path", None)
+        canonical_name = self.validate_name(self.name)
+        if canonical_name != self.name:
+            self.name = canonical_name
+            if kwargs.get("update_fields") is not None:
+                kwargs["update_fields"] = tuple(dict.fromkeys([*kwargs["update_fields"], "name"]))
         config = dict(self.config or {})
         if str(config.get("PERMISSIONS") or "").strip().lower() not in PERMISSIONS_VALUES:
             from archivebox.config.common import get_config
@@ -133,10 +145,59 @@ class Persona(ModelWithConfig):
 
     @property
     def path(self) -> Path:
-        """Path to persona directory under PERSONAS_DIR."""
+        """Reuse the directory's spelling without renaming a Chrome profile."""
         from archivebox.config.constants import CONSTANTS
 
-        return CONSTANTS.PERSONAS_DIR / self.name
+        cached = self.__dict__.get("_persona_path")
+        if cached is None or cached[0] != self.name:
+            directory_name = self.directory_name(self.name)
+            cached = (self.name, CONSTANTS.PERSONAS_DIR / directory_name)
+            self.__dict__["_persona_path"] = cached
+        return cached[1]
+
+    @staticmethod
+    def directory_name(name: str) -> str:
+        from archivebox.config.constants import CONSTANTS
+        from archivebox.personas.importers import validate_persona_name
+
+        valid, error = validate_persona_name(name)
+        if not valid:
+            raise ValidationError(error)
+        # Linux can hold both Default/default; macOS usually aliases them. Reuse a
+        # single match on either filesystem, but never guess which conflicting
+        # profile contains the user's login state. Only list names, not profile trees.
+        try:
+            with os.scandir(CONSTANTS.PERSONAS_DIR) as entries:
+                matches = sorted(entry.name for entry in entries if entry.name.casefold() == name.casefold())
+        except FileNotFoundError:
+            matches = []
+        if len(matches) > 1:
+            raise ValidationError(
+                f"Conflicting persona directories: {'; '.join(str(CONSTANTS.PERSONAS_DIR / match) for match in matches)}. "
+                "Stop Chrome and rename the conflicting directories manually to distinct names; no profile directories were renamed.",
+            )
+        return matches[0] if matches else name
+
+    @classmethod
+    def find_named(cls, name: str) -> "Persona | None":
+        # SQLite's iexact is ASCII-only; Python casefold gives every entry point
+        # the same Unicode matching rules as the filesystem-name lookup above.
+        matches = [(pk, existing) for pk, existing in cls.objects.values_list("pk", "name") if existing.casefold() == name.casefold()]
+        if len(matches) > 1:
+            raise ValidationError(
+                f"Conflicting persona records: {'; '.join(f'{existing!r} (ID {pk})' for pk, existing in matches)}. "
+                "Rename the conflicting personas manually to distinct names; profile files have not been changed.",
+            )
+        return cls.objects.get(pk=matches[0][0]) if matches else None
+
+    def validate_name(self, name: str) -> str:
+        existing = self.find_named(name)
+        if existing is not None and existing.pk != self.pk:
+            raise ValidationError(
+                f"Name conflicts with persona {existing.name!r} (ID {existing.pk}, directory {existing.path}). "
+                "Choose a distinct name or rename that persona record manually.",
+            )
+        return self.directory_name(name)
 
     @property
     def CHROME_USER_DATA_DIR(self) -> str:
@@ -156,9 +217,12 @@ class Persona(ModelWithConfig):
 
     @property
     def AUTH_STORAGE_FILE(self) -> str:
-        """Derived path to auth.json for this persona (if it exists)."""
+        """Derived path to a nonempty auth.json export for this persona."""
         auth_path = self.path / "auth.json"
-        return str(auth_path) if auth_path.exists() else ""
+        try:
+            return str(auth_path) if auth_path.stat().st_size > 0 else ""
+        except FileNotFoundError:
+            return ""
 
     def get_derived_config(self) -> dict:
         """
@@ -171,10 +235,13 @@ class Persona(ModelWithConfig):
         - AUTH_STORAGE_FILE (derived from persona path, if file exists)
         - ACTIVE_PERSONA (set to this persona's name)
         """
-        return derive_persona_config(name=self.name, config=self.config, persona_dir=self.path)
+        return derive_persona_config(name=self.path.name, config=self.config, persona_dir=self.path)
 
     def ensure_dirs(self) -> None:
         """Create persona directories if they don't exist."""
+        # Recheck at profile preparation, including changes made outside this process.
+        self.__dict__.pop("_persona_path", None)
+        self.validate_name(self.name)
         self.path.mkdir(parents=True, exist_ok=True)
         (self.path / "chrome_profile").mkdir(parents=True, exist_ok=True)
         (self.path / "chrome_downloads").mkdir(parents=True, exist_ok=True)
@@ -246,16 +313,26 @@ class Persona(ModelWithConfig):
                     fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
 
     @classmethod
-    def get_or_create_named(cls, name: str) -> "Persona":
+    def get_or_create_named(cls, name: str, *, defaults: dict | None = None) -> "Persona":
+        # Init/startup must reuse a user-created 'default', not create a second
+        # DB row pointing at a different (or shared) profile (issue #1896).
         persona_name = (name or "Default").strip() or "Default"
-        persona = cls.objects.filter(name=persona_name).first()
+        persona = cls.find_named(persona_name)
         if persona is not None:
+            canonical_name = persona.validate_name(persona.name)
+            if canonical_name != persona.name:
+                persona.name = canonical_name
+                persona.save(update_fields=["name"])
             return persona
 
         try:
-            return cls.objects.create(name=persona_name)
-        except IntegrityError:
-            return cls.objects.get(name=persona_name)
+            return cls.objects.create(name=persona_name, **(defaults or {}))
+        except (IntegrityError, ValidationError):
+            persona = cls.find_named(persona_name)
+            if persona is None:
+                raise
+            persona.validate_name(persona.name)
+            return persona
 
     def runtime_root_for_crawl(self, crawl) -> Path:
         return Path(crawl.output_dir) / ".persona" / self.name
@@ -315,20 +392,9 @@ class Persona(ModelWithConfig):
             if chrome_binary:
                 (runtime_root / "chrome_binary.txt").write_text(chrome_binary)
 
-        return {
-            # Hooks derive CHROME_USER_DATA_DIR/CHROME_DOWNLOADS_DIR from
-            # PERSONAS_DIR + ACTIVE_PERSONA. Point PERSONAS_DIR at the
-            # per-crawl runtime root here so CHROME_ISOLATION=crawl never
-            # leaks or reuses the template profile while keeping Chrome path
-            # derivation centralized in the Chrome plugin helpers.
-            "PERSONAS_DIR": str(runtime_root.parent),
-            "ACTIVE_PERSONA": self.name,
-            **{
-                key: str(runtime_root / filename)
-                for key, filename in (("COOKIES_FILE", "cookies.txt"), ("AUTH_STORAGE_FILE", "auth.json"))
-                if (runtime_root / filename).is_file()
-            },
-        }
+        # Hooks derive Chrome profile paths from the runtime persona root.
+        # Use the same optional auth discovery as the template persona.
+        return derive_persona_config(name=self.name, config={}, persona_dir=runtime_root)
 
     def prepare_runtime_for_snapshot(self, snapshot, chrome_binary: str = "") -> dict[str, str]:
         crawl_runtime_profile_dir = self.runtime_profile_dir_for_crawl(snapshot.crawl)
@@ -358,18 +424,7 @@ class Persona(ModelWithConfig):
         if chrome_binary:
             (runtime_root / "chrome_binary.txt").write_text(chrome_binary)
 
-        return {
-            # See prepare_runtime_for_crawl(): snapshot isolation changes the
-            # persona root, not individual CHROME_* config keys, so standalone
-            # Chrome hooks and ArchiveBox-driven hooks resolve paths the same way.
-            "PERSONAS_DIR": str(runtime_root.parent),
-            "ACTIVE_PERSONA": self.name,
-            **{
-                key: str(runtime_root / filename)
-                for key, filename in (("COOKIES_FILE", "cookies.txt"), ("AUTH_STORAGE_FILE", "auth.json"))
-                if (runtime_root / filename).is_file()
-            },
-        }
+        return derive_persona_config(name=self.name, config={}, persona_dir=runtime_root)
 
     def cleanup_runtime_for_crawl(self, crawl) -> None:
         shutil.rmtree(Path(crawl.output_dir) / ".persona", ignore_errors=True)

archivebox/plugins/discovery.py

@@ -192,7 +192,9 @@ def serve_plugin_replay_asset(path: str, config, response_class):
     for module in get_plugin_presentation_modules():
         hook = getattr(module, "serve_replay_asset_response", None)
         if hook is not None and (response := hook(path, config, response_class)) is not None:
-            return response
+            from archivebox.misc.serve_static import add_replay_image_fallback
+
+            return add_replay_image_fallback(response)
     return None
 
 

archivebox/services/archive_result_service.py

@@ -210,6 +210,12 @@ def _save_archiveresult_event_to_db(
 
     with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.diff_fields"):
         update_fields = []
+        # This runner already loaded the Snapshot/Crawl for the event. Resolve
+        # retention here without extra parent reads so new results do not wait
+        # for the bounded legacy-repair sweep to traverse historical rows.
+        result.snapshot = snapshot
+        if result.set_delete_at_from_config():
+            update_fields.append("delete_at")
         for field, value in defaults.items():
             if result.__dict__[field] != value:
                 setattr(result, field, value)

archivebox/services/resource_admission.py

@@ -170,6 +170,18 @@ class ResourceAdmission:
             used_bytes = 0
             for child in processes:
                 try:
+                    if sys.platform.startswith("linux"):
+                        try:
+                            memory = child.memory_full_info()
+                            # Chrome's processes share substantial resident
+                            # memory. RSS charges those pages once per process
+                            # and can falsely serialize captures on an idle host.
+                            used_bytes += memory.pss + memory.swap
+                            continue
+                        except psutil.AccessDenied:
+                            # Restricted /proc access still gets conservative
+                            # accounting through the available RSS measurement.
+                            pass
                     used_bytes += child.memory_info().rss
                 except (psutil.Error, OSError):
                     continue

archivebox/services/runner.py

@@ -12,6 +12,7 @@ import threading
 import time
 from contextlib import nullcontext
 from datetime import timedelta
+from itertools import batched
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from typing import Any, Literal
@@ -65,6 +66,7 @@ from archivebox.services.resource_admission import RESOURCE_RECHECK_SECONDS, res
 from archivebox.config.common import (
     ArchiveBoxBaseConfig,
     normalize_runtime_config,
+    _is_cookie_file_config_key,
     _plugin_enabled_config_keys,
 )
 from archivebox.misc.db import run_db_analyze_batch
@@ -738,8 +740,16 @@ class CrawlRunner:
             current_process.save(update_fields=["iface", "machine", "modified_at"])
         self.persona = self.crawl.resolve_persona()
         self.base_config = get_config(crawl=self.crawl, overrides=self.config_overrides)
-        self.derived_config = dict(machine.config or {})
+        # The merged user payload owns auth paths and binary selections. The
+        # install phase publishes resolved binaries; stale machine paths must
+        # not override explicit runtime settings before that phase runs.
+        self.derived_config = {
+            key: value
+            for key, value in (machine.config or {}).items()
+            if not _is_cookie_file_config_key(key) and (not key.endswith("_BINARY") or value == self.base_config.get(key))
+        }
         self.crawl_output_dir = str(self.crawl.output_dir)
+        Path(self.crawl_output_dir).mkdir(parents=True, exist_ok=True)
         if self.persona:
             self.base_config.update(
                 self.persona.prepare_runtime_for_crawl(
@@ -948,6 +958,7 @@ class CrawlRunner:
                     },
                 )
         snapshot_output_dir = str(snapshot.output_dir)
+        Path(snapshot_output_dir).mkdir(parents=True, exist_ok=True)
         tags = snapshot.tags_str()
         config = self.base_config.for_crawl_runtime(
             crawl=snapshot.crawl,
@@ -1245,6 +1256,7 @@ class CrawlRunner:
             user_config_event = MachineEvent(config=config, config_type="user")
             user_config_event.event_parent_id = crawl_start_event.event_id
             await self.bus.emit(user_config_event).now()
+            derived_config_event = None
             if derived_config:
                 derived_config_event = MachineEvent(config=derived_config, config_type="derived")
                 derived_config_event.event_parent_id = crawl_start_event.event_id
@@ -1260,6 +1272,7 @@ class CrawlRunner:
                 snapshot_cleanup_phase_timeout=snapshot_phase_timeout,
                 abort_requested=self.crawl_is_cancelled,
             )
+            projected = False
             try:
                 snapshot_event = SnapshotEvent(
                     url=snapshot["url"],
@@ -1283,6 +1296,7 @@ class CrawlRunner:
                     raise RuntimeError(f"Snapshot {snapshot_id} did not complete")
                 await completed_snapshot.wait(timeout=snapshot_phase_timeout)
                 await completed_snapshot.event_results_list()
+                projected = True
                 if self.execution_interrupted():
                     return
                 if snapshot["status"] == "sealed":
@@ -1306,6 +1320,16 @@ class CrawlRunner:
                 await sync_to_async(_seal_when_last_snapshot_finished, thread_sensitive=True)()
             finally:
                 snapshot_service.close()
+                # Projection and snapshot cleanup have finished. Historical
+                # capture results now live in the database; only active work
+                # needs its event ancestry available to the downloader.
+                expired = [snapshot_event, user_config_event] if projected else []
+                if projected and derived_config_event is not None:
+                    expired.append(derived_config_event)
+                if projected:
+                    expired.extend(await self.bus.filter("*", child_of=snapshot_event, past=True, future=False))
+                for event in expired:
+                    event.event_ttl = 0
 
     def seal_snapshot_due_to_limit(self, snapshot_id: str) -> None:
         from archivebox.core.models import Snapshot
@@ -1670,7 +1694,9 @@ def _run_due_snapshot_locked(snapshot, *, lock_seconds: int, interactive_interru
         # every URL, even though crawl isolation owns one browser for the whole
         # crawl. The crawl lifecycle lock already serializes all of its child
         # work, so let one runner drain the due siblings and discoveries.
-        if any(process.is_running for process in snapshot.process_set.filter(status="running").iterator()):
+        if any(
+            process.is_running for process in snapshot.process_set.filter(status="running").order_by("pk").paged_iterator(chunk_size=100)
+        ):
             snapshot.update_and_requeue(retry_at=timezone.now() + timedelta(seconds=lock_seconds))
             return True
         if snapshot.fs_migration_needed:
@@ -1704,7 +1730,7 @@ def _run_due_snapshot_locked(snapshot, *, lock_seconds: int, interactive_interru
     if not snapshot.claim_processing_lock(lock_seconds=lock_seconds):
         return False
     snapshot.refresh_from_db()
-    if any(process.is_running for process in snapshot.process_set.filter(status="running").iterator()):
+    if any(process.is_running for process in snapshot.process_set.filter(status="running").order_by("pk").paged_iterator(chunk_size=100)):
         # The Snapshot lease may have expired while an abx-dl hook process is
         # still alive. Preserve the snapshot-level ownership boundary and do
         # not launch a second sequence; ArchiveResult status is irrelevant.
@@ -1878,7 +1904,24 @@ def run_install(*, plugin_names: list[str] | None = None) -> None:
 
 def _first_due_id(queryset, *, newest_first: bool = False):
     ordering = ("-retry_at", "-created_at") if newest_first else ("retry_at", "created_at")
-    return queryset.order_by(*ordering).values_list("id", flat=True).first()
+    # Queue membership comes from retry_at, not status: almost the entire
+    # archive can be sealed while only a handful of rows need maintenance.
+    # Combining both predicates lets SQLite choose the broad status index and
+    # scan/sort archival history on every idle pass (567ms on Cabbage vs 3ms
+    # through retry_at). First materialize bounded due IDs using the existing
+    # queue index, then apply status/parent filters to those IDs. This also
+    # bounds PostgreSQL reads and keeps every cursor closed before a CAS claim.
+    due_ids = (
+        queryset.model.objects.filter(retry_at__lte=timezone.now())
+        .order_by(*ordering)
+        .values_list("id", flat=True)
+        .paged_iterator(chunk_size=100)
+    )
+    for candidate_ids in batched(due_ids, 100):
+        match = queryset.filter(id__in=candidate_ids).order_by(*ordering).values_list("id", flat=True).first()
+        if match is not None:
+            return match
+    return None
 
 
 def _run_due_crawl_status(
@@ -1949,12 +1992,8 @@ def _run_due_snapshot_id(snapshot_id, *, lock_seconds: int, interactive_interrup
 def _run_due_binary() -> bool:
     from archivebox.machine.models import Binary
 
-    due_binary_id = (
-        Binary.objects.filter(retry_at__lte=timezone.now())
-        .exclude(status=Binary.StatusChoices.INSTALLED)
-        .order_by("retry_at", "created_at")
-        .values_list("id", flat=True)
-        .first()
+    due_binary_id = _first_due_id(
+        Binary.objects.filter(retry_at__lte=timezone.now()).exclude(status=Binary.StatusChoices.INSTALLED),
     )
     if due_binary_id is None:
         return False
@@ -1981,14 +2020,36 @@ def run_pending_crawls(
     last_recovery_at = 0.0
     last_retention_at = 0.0
     last_retention_repair_at = 0.0
+    retention_backfill_cursors: dict = {}
+    deletion_thread: threading.Thread | None = None
     last_analyze_at = 0.0
     analyze_queue: list[str] | None = None
     analyze_sweep_started_at = 0.0
     orchestrator_started_at = time.monotonic()
+
+    def delete_in_background() -> None:
+        from django.db import connections
+
+        try:
+            Snapshot.delete_requested(batch_size=100)
+        finally:
+            connections.close_all()
+
     while True:
         raise_if_shutdown_requested()
         now_monotonic = time.monotonic()
         if crawl_id is None and now_monotonic - last_retention_at >= (60.0 if daemon else 1.0):
+            if daemon:
+                # At most one batch can be doing filesystem I/O. A stuck mount
+                # must not block capture scheduling or runner shutdown; rows
+                # remain pending until cleanup succeeds, including on restart.
+                if (deletion_thread is None or not deletion_thread.is_alive()) and Snapshot.objects.filter(
+                    status=Snapshot.DELETING_STATE,
+                ).exists():
+                    deletion_thread = threading.Thread(target=delete_in_background, name="snapshot-delete", daemon=True)
+                    deletion_thread.start()
+            else:
+                Snapshot.delete_requested(batch_size=100)
             for model in (ArchiveResult, Snapshot, Crawl, Process):
                 # Keep the tight scheduler loop anchored on indexed delete_at
                 # columns only. Backfilling missing delete_at values has to read
@@ -2156,7 +2217,7 @@ def run_pending_crawls(
                 # delete_at in the plugin-result hot path. Running it here keeps
                 # DELETE_AFTER resolution fresh without making every hook event
                 # load parent Snapshot/Crawl config.
-                model.delete_expired(batch_size=100, backfill_missing=True)
+                model.delete_expired(batch_size=100, backfill_missing=True, backfill_cursors=retention_backfill_cursors)
             last_retention_repair_at = now_monotonic
 
         if daemon:
@@ -2172,12 +2233,10 @@ def run_pending_crawls(
             # snapshot detail page out to ~500ms. Refresh stats at most once per
             # 24hr while the queue is idle, and only after the orchestrator has
             # been alive for at least an hour so short server boots / one-off work
-            # never pay the cost. The sweep is batched one table per idle tick;
-            # individual table ANALYZE statements abort after 2min (progress
-            # handler) and the whole sweep is hard-capped at 5min so a
-            # pathological table cannot wedge maintenance forever. Any failure
-            # inside the maintenance hook is swallowed — orchestrator must never
-            # be taken down by stats refresh.
+            # never pay the cost. Each idle tick samples a bounded number of
+            # index entries for one table; never scan the full history while
+            # holding ANALYZE's write transaction. Optional stats refresh must
+            # not prevent queued captures.
             try:
                 if (
                     analyze_queue is None

archivebox/services/snapshot_service.py

@@ -146,7 +146,7 @@ class SnapshotService(BaseService):
             elif snapshot.status not in (Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.SEALED):
                 return
             if snapshot.status == Snapshot.StatusChoices.STARTED:
-                await sync_to_async(snapshot.ensure_crawl_symlink, thread_sensitive=True)()
+                await sync_to_async(snapshot.reconcile_filesystem_links, thread_sensitive=True)()
             retry_plugins = [str(name).strip() for name in (snapshot.config or {}).get("RETRY_PLUGINS", []) if str(name).strip()]
             async with self._ownership_lock:
                 self._run_ownership[str(event.snapshot_id)] = (str(event.event_id), snapshot.retry_at, was_sealed, retry_plugins)

archivebox/templates/core/setup_wizard.html

@@ -35,7 +35,7 @@
       <div class="abx-question-options" id="archivebox-setup-dns-options">
         <label class="abx-question-option"><input type="radio" name="archivebox-dns-mode" value="localhost"> <span><b>⭐ Localhost</b><small>No setup needed. <code id="archivebox-setup-localhost-dns-example">*.archivebox.localhost</code> works directly in modern browsers with all features.</small></span></label>
         <label class="abx-question-option"><input type="radio" name="archivebox-dns-mode" value="single"> <span><b>⚠️ Single-domain DNS</b><small>Register a DNS record like <code>A archivebox.example.com</code> → <code>your server's IP</code>, or add an equivalent <code>/etc/hosts</code> entry pointing to your server. Some archived content that depends on JavaScript may not display properly in this mode. JavaScript still runs during capture, but viewing HTML captured by <code>wget</code>, <code>dom</code>, <code>responses</code>, <code>staticfile</code>, and similar plugins will not replay JavaScript unless wildcard DNS is used.</small></span></label>
-        <label class="abx-question-option"><input type="radio" name="archivebox-dns-mode" value="wildcard"> <span><b>⭐ Wildcard DNS</b><small>Point both the bare <code>BASE_URL</code> hostname and <code>*.BASE_URL</code> to this server using A/AAAA records or CNAME records. Ideal: safest isolation with all replay features. <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Modes" target="_blank" rel="noopener noreferrer">Setup instructions</a>.</small></span></label>
+        <label class="abx-question-option"><input type="radio" name="archivebox-dns-mode" value="wildcard"> <span><b>⭐ Wildcard DNS</b><small>Point both the bare <code>BASE_URL</code> hostname and <code>*.BASE_URL</code> to this server using A/AAAA records or CNAME records. Ideal: safest isolation with all replay features. <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#server_security_mode" target="_blank" rel="noopener noreferrer">Setup instructions</a>.</small></span></label>
       </div>
       <div class="abx-question-status" id="archivebox-setup-dns-status">Choose a DNS mode to begin testing.</div>
     </fieldset>

archivebox/templates/core/system_warnings_banner.html

@@ -27,7 +27,7 @@
       Only <code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">{{ free_gb }} GiB</code>
       free on <code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">DATA_DIR</code>
       — new archive jobs will fail until space is reclaimed.
-      <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Disk-Usage" target="_blank" rel="noopener noreferrer" style="color:#fff;text-decoration:underline;margin-left:6px;">docs</a>
+      <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#purging-entries" target="_blank" rel="noopener noreferrer" style="color:#fff;text-decoration:underline;margin-left:6px;">docs</a>
     {% elif mode == "high_memory" %}
       Virtual memory at <code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">{{ mem_pct }}%</code>
       — the host is close to OOM. Consider stopping crawls or scaling up.
@@ -35,7 +35,7 @@
       15-min loadavg <code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">{{ load_15 }}</code>
     {% elif mode == "unsafe" %}
       ArchiveBox single-domain mode — archived pages share an origin with this site
-      <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Modes" target="_blank" rel="noopener noreferrer" style="color:#fff;text-decoration:underline;margin-left:6px;">why?</a>
+      <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#server_security_mode" target="_blank" rel="noopener noreferrer" style="color:#fff;text-decoration:underline;margin-left:6px;">why?</a>
     {% elif mode == "base_url_mismatch" %}
       Editing disabled from unrecognized URL. Visit
       <a href="{{ configured_base_url }}" style="color:#fff;text-decoration:underline;text-transform:none;">{{ configured_base_url }}</a>

archivebox/templates/static/directory_index.html

@@ -1,357 +1,177 @@
 {% load i18n %}
+{% get_current_language as language_code %}
 <!DOCTYPE html>
-<html lang="en">
+<html lang="{{ language_code }}">
   <head>
     <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
-    <meta http-equiv="Content-Language" content="en-us">
+    <meta http-equiv="Content-Language" content="{{ language_code }}">
     <meta name="robots" content="NONE,NOARCHIVE">
     <meta name="viewport" content="width=device-width, initial-scale=1">
     <title>{% blocktranslate %}Index of {{ directory }}{% endblocktranslate %}</title>
     <style>
-      :root {
-        color-scheme: light;
-        --bg: #f3f5f8;
-        --panel: #ffffff;
-        --panel-border: #d9e0e8;
-        --text: #16202a;
-        --muted: #5e6b78;
-        --accent: #0f5cc0;
-        --accent-soft: #e7f0ff;
-        --folder-bg: #f6f9ff;
-        --folder-border: #d6e5ff;
-        --shadow: 0 10px 30px rgba(20, 32, 52, 0.08);
-      }
-
-      * {
-        box-sizing: border-box;
-      }
-
-      html,
-      body {
-        margin: 0;
-        padding: 0;
-        background: linear-gradient(180deg, #eef3f9 0%, var(--bg) 100%);
-        color: var(--text);
-        font-family:
-          ui-sans-serif,
-          system-ui,
-          -apple-system,
-          BlinkMacSystemFont,
-          "Segoe UI",
-          sans-serif;
-      }
-
-      body {
-        min-height: 100vh;
-        padding: 24px;
-      }
-
-      .directory-shell {
-        width: min(980px, 100%);
-        margin: 0 auto;
-      }
-
-      .directory-toolbar {
-        position: sticky;
-        top: 0;
-        z-index: 10;
-        display: flex;
-        flex-wrap: wrap;
-        gap: 10px;
-        margin-bottom: 14px;
-        padding: 12px;
-        border: 1px solid var(--panel-border);
-        border-radius: 16px;
-        background: rgba(255, 255, 255, 0.9);
-        box-shadow: var(--shadow);
-        backdrop-filter: blur(10px);
-      }
-
-      .toolbar-link {
-        display: inline-flex;
-        align-items: center;
-        gap: 8px;
-        min-height: 40px;
-        padding: 0 14px;
-        border: 1px solid var(--panel-border);
-        border-radius: 999px;
-        background: #fff;
-        color: var(--text);
-        font-size: 14px;
-        font-weight: 600;
-        text-decoration: none;
-        transition:
-          background-color 0.15s ease,
-          border-color 0.15s ease,
-          transform 0.15s ease;
-      }
-
-      .toolbar-link:hover {
-        background: #f8fbff;
-        border-color: #c5d2e0;
-        transform: translateY(-1px);
-      }
-
-      .toolbar-link.primary {
-        background: var(--accent-soft);
-        border-color: var(--folder-border);
-        color: var(--accent);
-      }
-
-      .archivebox-zip-spinner {
-        display: none;
-        width: 14px;
-        height: 14px;
-        border: 2px solid #2563eb;
-        border-right-color: transparent;
-        border-radius: 999px;
-        animation: archivebox-zip-spin 0.75s linear infinite;
-        flex: 0 0 auto;
-        box-shadow: 0 0 0 1px rgba(255, 255, 255, 0.45);
-      }
-
-      .archivebox-zip-button.is-loading {
-        pointer-events: none;
-        opacity: 0.96;
-      }
-
-      .archivebox-zip-button.is-loading .archivebox-zip-spinner {
-        display: inline-block;
-      }
-
-      .archivebox-zip-button.is-loading .archivebox-zip-label {
-        opacity: 0.72;
-      }
-
-      @keyframes archivebox-zip-spin {
-        to {
-          transform: rotate(360deg);
-        }
-      }
-
-      .directory-card {
-        background: var(--panel);
-        border: 1px solid var(--panel-border);
-        border-radius: 18px;
-        box-shadow: var(--shadow);
-        overflow: hidden;
-      }
-
-      .directory-header {
-        padding: 22px 24px 18px;
-        border-bottom: 1px solid var(--panel-border);
-        background:
-          radial-gradient(circle at top right, rgba(15, 92, 192, 0.08), transparent 28%),
-          linear-gradient(180deg, #ffffff, #f8fbff);
-      }
-
-      .directory-kicker {
-        margin: 0 0 8px;
-        color: var(--muted);
-        font-size: 12px;
-        font-weight: 700;
-        letter-spacing: 0.08em;
-        text-transform: uppercase;
-      }
-
-      .directory-title {
-        margin: 0;
-        font-size: clamp(22px, 4vw, 34px);
-        line-height: 1.1;
-        word-break: break-word;
-      }
-
-      .directory-meta {
-        margin-top: 12px;
-        color: var(--muted);
-        font-size: 14px;
-      }
-
-      .directory-path {
-        display: inline-block;
-        margin-top: 12px;
-        padding: 8px 12px;
-        border-radius: 999px;
-        background: #f4f7fb;
-        border: 1px solid var(--panel-border);
-        color: #304152;
-        font-family:
-          ui-monospace,
-          "SFMono-Regular",
-          Menlo,
-          Consolas,
-          monospace;
-        font-size: 13px;
-      }
-
-      .directory-list {
-        list-style: none;
-        margin: 0;
-        padding: 12px;
-      }
-
-      .directory-entry + .directory-entry {
-        margin-top: 6px;
-      }
-
-      .directory-link {
-        display: flex;
-        align-items: center;
-        gap: 12px;
-        width: 100%;
-        padding: 12px 14px;
-        border: 1px solid transparent;
-        border-radius: 12px;
-        color: inherit;
-        text-decoration: none;
-        transition:
-          background-color 0.15s ease,
-          border-color 0.15s ease,
-          transform 0.15s ease,
-          box-shadow 0.15s ease;
-      }
-
-      .directory-link:hover {
-        background: #f8fbff;
-        border-color: var(--panel-border);
-        transform: translateY(-1px);
-      }
-
-      .directory-link:focus-visible {
-        outline: 3px solid rgba(15, 92, 192, 0.2);
-        outline-offset: 1px;
-      }
-
-      .directory-entry.folder .directory-link {
-        background: var(--folder-bg);
-        border-color: var(--folder-border);
-      }
-
-      .entry-icon {
-        flex: 0 0 auto;
-        width: 34px;
-        height: 34px;
-        display: inline-flex;
-        align-items: center;
-        justify-content: center;
-        border-radius: 10px;
-        background: #edf2f8;
-        font-size: 18px;
-      }
-
-      .directory-entry.folder .entry-icon {
-        background: var(--accent-soft);
-      }
-
-      .entry-label {
-        min-width: 0;
-        display: flex;
-        flex-direction: column;
-        gap: 3px;
-      }
-
-      .entry-name {
-        font-size: 15px;
-        font-weight: 600;
-        line-height: 1.35;
-        word-break: break-word;
-      }
-
-      .entry-kind {
-        color: var(--muted);
-        font-size: 12px;
-        line-height: 1.2;
-      }
-
-      .directory-entry.parent .entry-kind {
-        color: var(--accent);
-      }
-
-      .empty-state {
-        padding: 28px 24px 32px;
-        text-align: center;
-        color: var(--muted);
-      }
-
-      @media (max-width: 640px) {
-        body {
-          padding: 14px;
-        }
-
-        .directory-header {
-          padding: 18px 18px 16px;
-        }
-
-        .directory-list {
-          padding: 10px;
-        }
-
-        .directory-link {
-          padding: 11px 12px;
-        }
-      }
+      :root { color-scheme: light; --line: #dfe4eb; --text: #253247; --muted: #667085; --accent: #245dc1; }
+      * { box-sizing: border-box; }
+      html, body { margin: 0; padding: 0; min-height: 100%; background: white; color: var(--text); font: 13px/1.4 system-ui, sans-serif; }
+      main { width: 100%; }
+      a { color: inherit; text-decoration: none; }
+      .directory-toolbar { display: flex; flex-wrap: wrap; gap: 6px; padding: 8px 12px; border-bottom: 1px solid var(--line); }
+      .toolbar-link, .entry-download { display: inline-flex; align-items: center; justify-content: center; gap: 5px; height: 30px; padding: 4px 9px; border: 1px solid var(--line); border-radius: 4px; background: white; }
+      .toolbar-link:hover, .entry-download:hover { background: #f0f5ff; }
+      .directory-header { display: flex; align-items: center; flex-wrap: wrap; gap: 8px; padding: 8px 12px; border-bottom: 1px solid var(--line); background: #f8fafc; }
+      .directory-title { margin: 0; font-size: 13px; font-weight: 600; overflow-wrap: anywhere; }
+      .directory-meta { margin-left: auto; color: var(--muted); white-space: nowrap; }
+      #filter-files { min-width: 0; width: 180px; padding: 5px 8px; border: 1px solid var(--line); border-radius: 4px; font: inherit; }
+      .directory-list { width: 100%; border-collapse: collapse; table-layout: fixed; }
+      th { height: 30px; padding: 0; text-align: left; background: #f8fafc; border-bottom: 1px solid var(--line); }
+      th:first-child button { padding-left: 72px; }
+      th button { width: 100%; height: 30px; padding: 0 8px; border: 0; background: none; color: var(--muted); text-align: left; font: inherit; cursor: pointer; }
+      th[aria-sort="ascending"] button::after { content: " ↑"; }
+      th[aria-sort="descending"] button::after { content: " ↓"; }
+      .size-column { width: 90px; }
+      .type-column { width: 220px; }
+      .download-column { width: 50px; }
+      td { padding: 0; }
+      .entry-size, .entry-type { padding: 0 8px; color: var(--muted); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
+      .entry-size { font-variant-numeric: tabular-nums; }
+      .directory-entry { height: 50px; border-bottom: 1px solid var(--line); }
+      .directory-entry[hidden] { display: none; }
+      .directory-entry:hover { background: #f8fafc; }
+      .directory-link { display: grid; grid-template-columns: 48px minmax(0, 1fr); align-items: center; gap: 12px; min-width: 0; height: 49px; padding: 6px 12px; }
+      a:focus-visible { outline: 2px solid var(--accent); outline-offset: -2px; }
+      .entry-icon, .entry-preview { width: 48px; height: 36px; margin: 0; overflow: hidden; border-radius: 3px; }
+      .entry-icon { display: grid; place-items: center; font-size: 20px; }
+      .entry-preview { border: 1px solid var(--line); background: white; }
+      img.entry-preview { object-fit: contain; }
+      pre.entry-preview { padding: 3px; white-space: pre-wrap; overflow-wrap: anywhere; font: 7px/1.2 ui-monospace, monospace; color: var(--muted); }
+      .entry-label { min-width: 0; display: flex; flex-direction: column; gap: 1px; }
+      .entry-name { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; font-weight: 500; }
+      .entry-kind { color: var(--muted); font-size: 11px; }
+      .entry-download { flex-shrink: 0; color: var(--accent); }
+      .empty-state { padding: 24px; text-align: center; color: var(--muted); }
+      .archivebox-zip-spinner { display: none; width: 12px; height: 12px; border: 2px solid var(--accent); border-right-color: transparent; border-radius: 50%; animation: archivebox-zip-spin .75s linear infinite; }
+      .archivebox-zip-button.is-loading { pointer-events: none; opacity: .7; }
+      .archivebox-zip-button.is-loading .archivebox-zip-spinner { display: inline-block; }
+      @keyframes archivebox-zip-spin { to { transform: rotate(360deg); } }
+      @media (max-width: 640px) { .size-column { width: 65px; } .type-column { width: 100px; } .download-column { width: 40px; } #filter-files { width: 130px; } }
     </style>
   </head>
   <body>
-    <main class="directory-shell">
-      <nav class="directory-toolbar" aria-label="Directory navigation">
+    <main>
+      <nav class="directory-toolbar" aria-label="{% translate "Directory navigation" %}">
         {% if directory != "/" %}
-          <a class="toolbar-link" href="../{{ directory_query }}">↩ Up One Level</a>
+          <a class="toolbar-link" href="../{{ directory_query }}">↩ {% translate "Up One Level" %}</a>
         {% endif %}
-        <a class="toolbar-link primary" href="{{ snapshot_page_url }}">⌂ Snapshot Page</a>
+        <a class="toolbar-link" href="{{ snapshot_page_url }}">⌂ {% translate "Snapshot Page" %}</a>
         <a
           class="toolbar-link archivebox-zip-button"
           href="{{ zip_url }}"
-          data-loading-label="Preparing..."
+          data-loading-label="{% translate "Preparing..." %}"
           onclick="return window.archiveboxHandleZipClick(this, event);"
         >
           <span class="archivebox-zip-spinner" aria-hidden="true"></span>
-          <span class="archivebox-zip-label">⬇ Download Zip</span>
+          <span class="archivebox-zip-label">⬇ {% translate "Download Zip" %}</span>
         </a>
       </nav>
 
-      <section class="directory-card">
+      <section>
         <header class="directory-header">
-          <p class="directory-kicker">ArchiveBox File Index</p>
-          <h1 class="directory-title">{% blocktranslate %}Index of {{ directory }}{% endblocktranslate %}</h1>
-          <div class="directory-meta">
-            {{ file_list|length }} item{{ file_list|length|pluralize }}
+          <h1 class="directory-title">{{ directory }}</h1>
+          <div class="directory-meta" id="file-count" aria-live="polite">
+            {% translate "Items" %}: <span id="visible-file-count">{{ file_list|length }}</span>
           </div>
-          <div class="directory-path">{{ directory }}</div>
+          <input id="filter-files" type="search" aria-label="{% translate "Filter files" %}" placeholder="{% translate "Filter files" %}">
         </header>
 
         {% if file_list or directory != "/" %}
-          <ul class="directory-list">
+          <table class="directory-list">
+            <colgroup><col><col class="size-column"><col class="type-column"><col class="download-column"></colgroup>
+            <thead><tr>
+              <th scope="col" aria-sort="ascending"><button type="button" data-sort="name" aria-label="{% translate "Name" %}">{% translate "Name" %}</button></th>
+              <th scope="col" aria-sort="none"><button type="button" data-sort="size" aria-label="{% translate "Size" %}">{% translate "Size" %}</button></th>
+              <th scope="col" aria-sort="none"><button type="button" data-sort="type" aria-label="{% translate "Type" %}">{% translate "Type" %}</button></th>
+              <th scope="col" aria-label="{% translate "Download" %}"></th>
+            </tr></thead>
+            <tbody>
             {% if directory != "/" %}
-              <li class="directory-entry parent">
+              <tr class="directory-entry parent"><td>
                 <a class="directory-link" href="../{{ directory_query }}">
                   <span class="entry-icon" aria-hidden="true">↩</span>
                   <span class="entry-label">
                     <span class="entry-name">../</span>
-                    <span class="entry-kind">Parent directory</span>
                   </span>
                 </a>
-              </li>
+              </td><td class="entry-size">—</td><td class="entry-type">{% translate "Directory" %}</td><td></td></tr>
             {% endif %}
 
-            {% for f in file_list %}
-              <li class="directory-entry{% if f|slice:'-1:' == '/' %} folder{% endif %}">
-                <a class="directory-link" href="{{ f|urlencode }}{% if f|slice:'-1:' == '/' %}{{ directory_query }}{% endif %}">
-                  <span class="entry-icon" aria-hidden="true">{% if f|slice:'-1:' == '/' %}📁{% else %}📄{% endif %}</span>
+            {% for entry in entries %}
+              {% with f=entry.name %}
+              <tr class="directory-entry{% if entry.is_dir %} folder{% endif %}" data-name="{{ f }}" data-size="{{ entry.size_bytes }}" data-type="{{ entry.mimetype }}"><td>
+                <a class="directory-link" href="{{ f|urlencode }}{% if f|slice:'-1:' == '/' %}{{ directory_query }}{% elif f|lower|slice:'-4:' == '.zip' %}?preview=1{% endif %}">
+                  {% if entry.preview_kind == "image" %}
+                    <img class="entry-preview" src="{{ f|urlencode }}?raw=1" alt="" loading="lazy" decoding="async">
+                  {% elif entry.preview_kind == "text" %}
+                    <pre class="entry-preview" data-url="{{ f|urlencode }}?raw=1" aria-hidden="true">…</pre>
+                  {% else %}
+                    <span class="entry-icon" aria-hidden="true">{% if entry.is_dir %}📁{% else %}📄{% endif %}</span>
+                  {% endif %}
                   <span class="entry-label">
-                    <span class="entry-name">{{ f }}</span>
-                    <span class="entry-kind">{% if f|slice:'-1:' == '/' %}Directory{% else %}File{% endif %}</span>
+                    <span class="entry-name" title="{{ f }}">{{ f }}</span>
                   </span>
                 </a>
-              </li>
+                </td><td class="entry-size">{{ entry.size }}</td><td class="entry-type" title="{{ entry.mimetype }}">{{ entry.mimetype }}</td><td>
+                <a class="entry-download" aria-label="{% blocktranslate %}Download {{ f }}{% endblocktranslate %}" title="{% blocktranslate %}Download {{ f }}{% endblocktranslate %}" href="{{ f|urlencode }}{% if entry.is_dir %}?files=1&amp;download=zip{% else %}?raw=1{% endif %}" download>⬇</a>
+              </td></tr>
+              {% endwith %}
             {% endfor %}
-          </ul>
+            </tbody>
+          </table>
         {% else %}
-          <div class="empty-state">This directory is empty.</div>
+          <div class="empty-state">{% translate "This directory is empty." %}</div>
         {% endif %}
       </section>
     </main>
     <script>
+      document.getElementById("filter-files").addEventListener("input", (event) => {
+        const query = event.target.value.toLowerCase();
+        let count = 0;
+        document.querySelectorAll(".directory-entry:not(.parent)").forEach((row) => {
+          row.hidden = !row.querySelector(".entry-name").textContent.toLowerCase().includes(query);
+          if (!row.hidden) count++;
+        });
+        document.getElementById("visible-file-count").textContent = count;
+      });
+      let sortKey = null, sortDirection = 1;
+      document.querySelectorAll("[data-sort]").forEach((button) => {
+        button.addEventListener("click", () => {
+          sortDirection = sortKey === button.dataset.sort ? -sortDirection : 1;
+          sortKey = button.dataset.sort;
+          const rows = [...document.querySelectorAll(".directory-entry:not(.parent)")];
+          rows.sort((a, b) => {
+            const folders = Number(b.classList.contains("folder")) - Number(a.classList.contains("folder"));
+            if (folders) return folders;
+            const comparison = sortKey === "size"
+              ? Number(a.dataset.size) - Number(b.dataset.size)
+              : a.dataset[sortKey].localeCompare(b.dataset[sortKey], undefined, {numeric: true, sensitivity: "base"});
+            return sortDirection * comparison || a.dataset.name.localeCompare(b.dataset.name, undefined, {numeric: true});
+          });
+          rows.forEach((row) => row.parentElement.appendChild(row));
+          document.querySelectorAll("th[aria-sort]").forEach((header) => header.setAttribute("aria-sort", "none"));
+          button.parentElement.setAttribute("aria-sort", sortDirection === 1 ? "ascending" : "descending");
+        });
+      });
+      document.querySelector('[data-sort="name"]')?.click();
+      const previews = new IntersectionObserver((items) => {
+        for (const item of items) {
+          if (!item.isIntersecting) continue;
+          const preview = item.target;
+          previews.unobserve(preview);
+          fetch(preview.dataset.url)
+            .then((response) => { if (!response.ok) throw new Error("Unavailable"); return response.text(); })
+            .then((text) => { preview.textContent = text.slice(0, 12000); })
+            .catch(() => { preview.textContent = "📄"; });
+        }
+      }, {rootMargin: "100px"});
+      document.querySelectorAll("pre.entry-preview").forEach((preview) => previews.observe(preview));
       window.archiveboxHandleZipClick = function(link, event) {
         if (!link || link.dataset.loading === "1") {
           if (event) event.preventDefault();
@@ -365,7 +185,7 @@
         const spinner = link.querySelector(".archivebox-zip-spinner");
         if (label) {
           link.dataset.originalLabel = label.textContent || "";
-          label.textContent = link.dataset.loadingLabel || "Preparing...";
+          label.textContent = link.dataset.loadingLabel;
           label.style.opacity = "0.92";
         }
         if (spinner) {

archivebox/templates/static/zip_index.html

@@ -0,0 +1,792 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width,initial-scale=1" />
+    <title>Saved files</title>
+    <style>
+      :root {
+        color-scheme: light;
+        --line: #dfe4eb;
+        --muted: #667085;
+        --blue: #245dc1;
+      }
+      * {
+        box-sizing: border-box;
+      }
+      html,
+      body {
+        height: 100%;
+        margin: 0;
+      }
+      body {
+        font: 13px/1.5 system-ui, sans-serif;
+        color: #253247;
+        background: white;
+        display: flex;
+        flex-direction: column;
+      }
+      a {
+        color: var(--blue);
+        text-decoration: none;
+      }
+      button,
+      input {
+        font: inherit;
+      }
+      button,
+      a.action {
+        cursor: pointer;
+        border: 1px solid var(--line);
+        border-radius: 5px;
+        background: white;
+        padding: 5px 10px;
+        color: #344054;
+      }
+      button:hover,
+      a.action:hover {
+        background: #f0f5ff;
+      }
+      header {
+        display: flex;
+        align-items: center;
+        gap: 12px;
+        padding: 10px 16px;
+        border-bottom: 1px solid var(--line);
+      }
+      h1 {
+        font-size: 14px;
+        font-weight: 600;
+        margin: 0;
+        overflow: hidden;
+        text-overflow: ellipsis;
+        white-space: nowrap;
+      }
+      nav {
+        display: flex;
+        gap: 6px;
+        margin-left: auto;
+        flex-shrink: 0;
+      }
+      .toolbar {
+        display: flex;
+        align-items: center;
+        gap: 12px;
+        padding: 9px 16px;
+        background: #f8fafc;
+        border-bottom: 1px solid var(--line);
+      }
+      #breadcrumbs {
+        display: flex;
+        gap: 5px;
+        align-items: center;
+        flex-wrap: wrap;
+        min-width: 0;
+      }
+      #breadcrumbs button {
+        border: 0;
+        background: none;
+        padding: 0;
+        color: var(--blue);
+      }
+      #count {
+        color: var(--muted);
+        white-space: nowrap;
+        margin-left: auto;
+      }
+      #search {
+        width: 160px;
+        border: 1px solid var(--line);
+        border-radius: 5px;
+        padding: 4px 8px;
+      }
+      main {
+        display: flex;
+        min-height: 0;
+        flex: 1;
+      }
+      .listing {
+        overflow: auto;
+        flex: 1;
+        min-width: 260px;
+      }
+      table {
+        width: 100%;
+        border-collapse: collapse;
+        text-align: left;
+      }
+      th {
+        position: sticky;
+        top: 0;
+        background: #fff;
+        color: var(--muted);
+        font-weight: 500;
+        border-bottom: 1px solid var(--line);
+        padding: 8px 16px;
+      }
+      td {
+        border-bottom: 1px solid #eef1f5;
+        padding: 8px 16px;
+      }
+      td.name {
+        width: 100%;
+      }
+      .entry {
+        border: 0;
+        background: none;
+        padding: 0;
+        display: flex;
+        align-items: center;
+        gap: 10px;
+        color: #253247;
+        text-align: left;
+        overflow-wrap: anywhere;
+      }
+      .entry:hover {
+        background: none;
+        color: var(--blue);
+      }
+      .size,
+      .date {
+        color: var(--muted);
+        white-space: nowrap;
+        font-size: 12px;
+      }
+      tr.selected {
+        background: #eaf2ff;
+      }
+      tr:hover {
+        background: #f6f9fc;
+      }
+      #preview {
+        width: 55%;
+        min-width: 280px;
+        border-left: 1px solid var(--line);
+        display: flex;
+        flex-direction: column;
+        min-height: 0;
+      }
+      #preview[hidden] {
+        display: none;
+      }
+      .preview-head {
+        padding: 9px 12px;
+        display: flex;
+        align-items: center;
+        gap: 8px;
+        border-bottom: 1px solid var(--line);
+        background: #f8fafc;
+      }
+      #filename {
+        overflow-wrap: anywhere;
+        font-weight: 600;
+        margin-right: auto;
+      }
+      #content {
+        overflow: auto;
+        padding: 16px;
+        flex: 1;
+        min-height: 0;
+        display: flex;
+        align-items: flex-start;
+        justify-content: center;
+      }
+      #content img {
+        max-width: 100%;
+        max-height: 100%;
+        object-fit: contain;
+      }
+      #content pre {
+        white-space: pre-wrap;
+        overflow-wrap: anywhere;
+        align-self: stretch;
+        margin: 0;
+        width: 100%;
+        font: 12px/1.6 ui-monospace, monospace;
+      }
+      #content iframe {
+        width: 100%;
+        height: 100%;
+        border: 0;
+      }
+      #content audio,
+      #content video {
+        max-width: 100%;
+      }
+      .empty {
+        padding: 22px;
+        color: var(--muted);
+      }
+      #error {
+        color: #b42318;
+        padding: 8px 16px;
+        margin: 0;
+      }
+      #error:empty {
+        display: none;
+      }
+      @media (max-width: 640px) {
+        header {
+          flex-wrap: wrap;
+          padding: 9px 12px;
+        }
+        nav {
+          margin-left: 0;
+        }
+        .toolbar {
+          padding: 8px 12px;
+        }
+        #search {
+          width: 100px;
+        }
+        .date {
+          display: none;
+        }
+        main {
+          flex-direction: column;
+        }
+        .listing {
+          min-height: 140px;
+        }
+        #preview {
+          width: 100%;
+          border-left: 0;
+          border-top: 1px solid var(--line);
+          min-height: 300px;
+          flex: 1;
+        }
+        td,
+        th {
+          padding: 8px 12px;
+        }
+      }
+    </style>
+  </head>
+  <body data-output="{{ output_path }}" data-size="{{ archive_size }}">
+    <header>
+      <h1 id="title">{{ title }}</h1>
+      <nav id="actions" aria-label="Output actions">
+        <a class="action" id="files">View all files</a>
+      </nav>
+    </header>
+    <div class="toolbar">
+      <div id="breadcrumbs"></div>
+      <span id="count"></span
+      ><input
+        id="search"
+        aria-label="Filter files"
+        placeholder="Filter files"
+      />
+    </div>
+    <p id="error" role="alert"></p>
+    <main>
+      <section class="listing" aria-label="Files">
+        <table>
+          <thead>
+            <tr>
+              <th>Name</th>
+              <th>Size</th>
+              <th class="date">Modified</th>
+            </tr>
+          </thead>
+          <tbody id="entries"></tbody>
+        </table>
+      </section>
+      <section id="preview" aria-label="File preview" hidden>
+        <div class="preview-head">
+          <span id="filename"></span><button id="download">Download</button
+          ><button id="close" aria-label="Close preview">×</button>
+        </div>
+        <div id="content"></div>
+      </section>
+    </main>
+    <script>
+      (() => {
+        const byId = (id) => document.getElementById(id),
+          el = (tag, text, cls) => {
+            const n = document.createElement(tag);
+            if (text != null) n.textContent = text;
+            if (cls) n.className = cls;
+            return n;
+          };
+        const output = new URL(document.body.dataset.output, document.baseURI);
+        output.search = "";
+        output.hash = "";
+        const folder = new URL("./", output),
+          raw = new URL(output);
+        raw.searchParams.set("raw", "1");
+        byId("files").href = folder.href + "?files=1";
+        const MAX_PREVIEW = 16 * 1024 * 1024,
+          CHUNK = 256 * 1024,
+          decoder = new TextDecoder();
+        const bytes = (n) =>
+          n < 1024
+            ? n + " B"
+            : n < 1048576
+            ? (n / 1024).toFixed(1) + " KB"
+            : n < 1073741824
+            ? (n / 1048576).toFixed(1) + " MB"
+            : (n / 1073741824).toFixed(2) + " GB";
+        const safePath = (name) => {
+          const p = name.replace(/\\/g, "/").replace(/\/$/, "");
+          return p &&
+            !p.startsWith("/") &&
+            !p.split("/").some((x) => !x || x === "." || x === "..")
+            ? p
+            : null;
+        };
+        const view = (b) => new DataView(b.buffer, b.byteOffset, b.byteLength);
+        const u64 = (v, p) => {
+          const n = v.getBigUint64(p, true);
+          if (n > BigInt(Number.MAX_SAFE_INTEGER))
+            throw Error("ZIP offsets exceed browser precision");
+          return Number(n);
+        };
+        async function rangeResponse(archive, start, length, signal) {
+          if (
+            !Number.isSafeInteger(start) ||
+            !Number.isSafeInteger(length) ||
+            start < 0 ||
+            length <= 0 ||
+            start + length > archive.size
+          )
+            throw Error("Invalid ZIP byte range");
+          const response = await fetch(archive.url, {
+            headers: { Range: `bytes=${start}-${start + length - 1}` },
+            signal,
+          });
+          const expected = `bytes ${start}-${start + length - 1}/${
+            archive.size
+          }`;
+          if (
+            response.status !== 206 ||
+            response.headers.get("Content-Range") !== expected
+          ) {
+            await response.body?.cancel();
+            throw Error("ZIP browsing requires HTTP byte-range support");
+          }
+          return response;
+        }
+        async function range(archive, start, length, signal) {
+          const response = await rangeResponse(archive, start, length, signal),
+            reader = response.body.getReader(),
+            result = new Uint8Array(length);
+          let offset = 0;
+          try {
+            while (true) {
+              const { done, value } = await reader.read();
+              if (done) break;
+              if (offset + value.length > length)
+                throw Error("ZIP range exceeds requested size");
+              result.set(value, offset);
+              offset += value.length;
+            }
+            if (offset !== length) throw Error("Incomplete ZIP byte range");
+            return result;
+          } finally {
+            await reader.cancel();
+          }
+        }
+        async function indexZip(archive) {
+          const tail = await range(
+            archive,
+            Math.max(0, archive.size - 65557),
+            Math.min(archive.size, 65557)
+          );
+          const v = view(tail);
+          let end = -1;
+          for (let i = tail.length - 22; i >= 0; i--)
+            if (
+              v.getUint32(i, true) === 0x06054b50 &&
+              i + 22 + v.getUint16(i + 20, true) === tail.length
+            ) {
+              end = i;
+              break;
+            }
+          if (end < 0) throw Error("Missing ZIP directory");
+          if (v.getUint16(end + 4, true) || v.getUint16(end + 6, true))
+            throw Error("Multi-volume ZIPs are not supported");
+          let count = v.getUint16(end + 10, true),
+            size = v.getUint32(end + 12, true),
+            offset = v.getUint32(end + 16, true);
+          if (count === 65535 || size === 0xffffffff || offset === 0xffffffff) {
+            const locatorOffset = archive.size - tail.length + end - 20,
+              locator = view(await range(archive, locatorOffset, 20));
+            if (
+              locator.getUint32(0, true) !== 0x07064b50 ||
+              locator.getUint32(4, true) !== 0 ||
+              locator.getUint32(16, true) !== 1
+            )
+              throw Error("Invalid ZIP64 locator");
+            const z = view(await range(archive, u64(locator, 8), 56));
+            if (
+              z.getUint32(0, true) !== 0x06064b50 ||
+              z.getUint32(16, true) ||
+              z.getUint32(20, true)
+            )
+              throw Error("Invalid ZIP64 directory");
+            count = u64(z, 32);
+            size = u64(z, 40);
+            offset = u64(z, 48);
+          }
+          if (count > 100000)
+            throw Error(
+              "ZIP has more than 100,000 entries; use the ZIP download"
+            );
+          if (offset + size > archive.size || offset < 0)
+            throw Error("Invalid ZIP directory bounds");
+          let buffer = new Uint8Array(),
+            base = -1;
+          const read = async (position, length) => {
+            if (position < offset || position + length > offset + size)
+              throw Error("Invalid ZIP entry bounds");
+            if (position < base || position + length > base + buffer.length) {
+              base = position;
+              buffer = await range(
+                archive,
+                position,
+                Math.min(Math.max(CHUNK, length), offset + size - position)
+              );
+            }
+            return buffer.subarray(position - base, position - base + length);
+          };
+          const entries = [];
+          let cursor = offset;
+          for (let i = 0; i < count; i++) {
+            const h = view(await read(cursor, 46));
+            if (h.getUint32(0, true) !== 0x02014b50)
+              throw Error("Invalid ZIP directory entry");
+            const nameLength = h.getUint16(28, true),
+              extraLength = h.getUint16(30, true),
+              commentLength = h.getUint16(32, true),
+              flags = h.getUint16(8, true),
+              method = h.getUint16(10, true),
+              crc = h.getUint32(16, true),
+              date = h.getUint16(14, true),
+              time = h.getUint16(12, true);
+            let compressed = h.getUint32(20, true),
+              size = h.getUint32(24, true),
+              local = h.getUint32(42, true);
+            const fields = await read(
+                cursor + 46,
+                nameLength + extraLength + commentLength
+              ),
+              name = decoder.decode(fields.subarray(0, nameLength)),
+              extra = view(
+                fields.subarray(nameLength, nameLength + extraLength)
+              );
+            for (let p = 0; p + 4 <= extra.byteLength; ) {
+              const id = extra.getUint16(p, true),
+                length = extra.getUint16(p + 2, true);
+              if (p + 4 + length > extra.byteLength)
+                throw Error("Invalid ZIP extra field");
+              if (id === 1) {
+                let q = p + 4;
+                const next = () => {
+                  if (q + 8 > p + 4 + length)
+                    throw Error("Invalid ZIP64 entry");
+                  const n = u64(extra, q);
+                  q += 8;
+                  return n;
+                };
+                if (size === 0xffffffff) size = next();
+                if (compressed === 0xffffffff) compressed = next();
+                if (local === 0xffffffff) local = next();
+              }
+              p += 4 + length;
+            }
+            const path = safePath(name);
+            if (path)
+              entries.push({
+                path,
+                directory: name.endsWith("/"),
+                size,
+                compressed,
+                local,
+                flags,
+                method,
+                crc,
+                archive,
+                date: date
+                  ? new Date(
+                      1980 + (date >> 9),
+                      ((date >> 5) & 15) - 1,
+                      date & 31,
+                      time >> 11,
+                      (time >> 5) & 63,
+                      (time & 31) * 2
+                    )
+                  : null,
+              });
+            cursor += 46 + nameLength + extraLength + commentLength;
+          }
+          return entries;
+        }
+        const crcTable = Uint32Array.from({ length: 256 }, (_, n) => {
+          for (let i = 0; i < 8; i++)
+            n = n & 1 ? 0xedb88320 ^ (n >>> 1) : n >>> 1;
+          return n >>> 0;
+        });
+        async function entryStream(file, signal) {
+          if (file.flags & 1)
+            throw Error("Encrypted ZIP entries cannot be previewed");
+          if (![0, 8].includes(file.method))
+            throw Error("This ZIP compression method cannot be previewed");
+          const h = view(await range(file.archive, file.local, 30, signal));
+          if (h.getUint32(0, true) !== 0x04034b50)
+            throw Error("Invalid ZIP file header");
+          const start =
+            file.local + 30 + h.getUint16(26, true) + h.getUint16(28, true);
+          let stream = file.compressed
+            ? (
+                await rangeResponse(
+                  file.archive,
+                  start,
+                  file.compressed,
+                  signal
+                )
+              ).body
+            : new Blob([]).stream();
+          if (file.method === 8)
+            stream = stream.pipeThrough(new DecompressionStream("deflate-raw"));
+          let size = 0,
+            crc = 0xffffffff;
+          return stream.pipeThrough(
+            new TransformStream({
+              transform(chunk, controller) {
+                size += chunk.length;
+                if (size > file.size)
+                  throw Error("ZIP entry exceeds its declared size");
+                for (const byte of chunk)
+                  crc = crcTable[(crc ^ byte) & 255] ^ (crc >>> 8);
+                controller.enqueue(chunk);
+              },
+              flush() {
+                if (size !== file.size || (crc ^ 0xffffffff) >>> 0 !== file.crc)
+                  throw Error("ZIP file size or checksum mismatch");
+              },
+            })
+          );
+        }
+        let files = [],
+          prefix = "",
+          selected = null,
+          controller = null,
+          blobURL = null;
+        const clearPreview = () => {
+          controller?.abort();
+          if (blobURL) URL.revokeObjectURL(blobURL);
+          blobURL = null;
+          byId("content").replaceChildren();
+        };
+        const report = (error) => {
+          if (error.name !== "AbortError")
+            byId("error").textContent = error.message;
+        };
+        function browse(next) {
+          prefix = next;
+          const filter = byId("search").value.toLowerCase(),
+            crumbs = byId("breadcrumbs");
+          crumbs.replaceChildren();
+          const crumb = (label, path) => {
+            const b = el("button", label);
+            b.onclick = () => {
+              byId("search").value = "";
+              browse(path);
+            };
+            crumbs.append(b);
+          };
+          crumb("Files", "");
+          let part = "";
+          for (const segment of prefix.split("/").filter(Boolean)) {
+            part += segment + "/";
+            crumbs.append(el("span", "/"));
+            crumb(segment, part);
+          }
+          const items = new Map();
+          for (const f of files) {
+            if (!f.path.startsWith(prefix)) continue;
+            const tail = f.path.slice(prefix.length),
+              name = tail.split("/")[0];
+            if (!name || !name.toLowerCase().includes(filter)) continue;
+            const directory = f.directory || tail.includes("/");
+            if (!items.has(name) || directory)
+              items.set(
+                name,
+                directory ? { path: prefix + name, directory: true } : f
+              );
+          }
+          const table = byId("entries");
+          table.replaceChildren();
+          for (const file of [...items.values()].sort(
+            (a, b) =>
+              Number(b.directory) - Number(a.directory) ||
+              a.path.localeCompare(b.path)
+          )) {
+            const row = el("tr");
+            if (file === selected) row.className = "selected";
+            const name = el("td", null, "name"),
+              button = el("button", null, "entry");
+            button.append(
+              el("span", file.directory ? "📁" : "📄"),
+              el("span", file.path.slice(prefix.length))
+            );
+            button.onclick = () =>
+              file.directory ? browse(file.path + "/") : openFile(file);
+            name.append(button);
+            row.append(
+              name,
+              el("td", file.directory ? "—" : bytes(file.size), "size"),
+              el("td", file.date?.toLocaleDateString() || "", "date")
+            );
+            table.append(row);
+          }
+          byId("count").textContent = items.size + " items";
+          if (!items.size) {
+            const row = el("tr"),
+              cell = el("td", "No files", "empty");
+            cell.colSpan = 3;
+            row.append(cell);
+            table.append(row);
+          }
+        }
+        const mime = (path) =>
+          ({
+            png: "image/png",
+            jpg: "image/jpeg",
+            jpeg: "image/jpeg",
+            gif: "image/gif",
+            webp: "image/webp",
+            avif: "image/avif",
+            pdf: "application/pdf",
+            mp3: "audio/mpeg",
+            ogg: "audio/ogg",
+            wav: "audio/wav",
+            mp4: "video/mp4",
+            webm: "video/webm",
+          }[path.split(".").pop().toLowerCase()] || "application/octet-stream");
+        async function openFile(file) {
+          clearPreview();
+          selected = file;
+          controller = new AbortController();
+          const signal = controller.signal;
+          byId("error").textContent = "";
+          byId("preview").hidden = false;
+          byId("filename").textContent = file.path.split("/").pop();
+          browse(prefix);
+          const content = byId("content");
+          content.append(el("span", "Loading…", "empty"));
+          if (file.size > MAX_PREVIEW) {
+            content.replaceChildren(
+              el("span", "File is too large to preview. Use Download.", "empty")
+            );
+            return;
+          }
+          try {
+            const stream = file.archive
+              ? await entryStream(file, signal)
+              : (await fetch(file.url, { signal })).body;
+            const blob = await new Response(stream).blob();
+            if (signal.aborted) return;
+            const type = mime(file.path);
+            content.replaceChildren();
+            if (type !== "application/octet-stream") {
+              blobURL = URL.createObjectURL(new Blob([blob], { type }));
+              let node;
+              if (type.startsWith("image/")) {
+                node = el("img");
+                node.alt = file.path;
+              } else if (type === "application/pdf") {
+                node = el("iframe");
+                node.title = file.path;
+                node.setAttribute("sandbox", "");
+              } else {
+                node = el(type.startsWith("audio/") ? "audio" : "video");
+                node.controls = true;
+              }
+              node.src = blobURL;
+              content.append(node);
+            } else if (
+              /\.(txt|md|csv|tsv|json|xml|html?|css|js|py|log|ya?ml|svg|rtf)$/i.test(
+                file.path
+              )
+            )
+              content.append(el("pre", await blob.text()));
+            else
+              content.append(
+                el(
+                  "span",
+                  "Preview unavailable for this file type. Use Download.",
+                  "empty"
+                )
+              );
+          } catch (error) {
+            if (!signal.aborted) {
+              content.replaceChildren();
+              report(error);
+            }
+          }
+        }
+        byId("search").oninput = () => browse(prefix);
+        byId("close").onclick = () => {
+          clearPreview();
+          selected = null;
+          byId("preview").hidden = true;
+          browse(prefix);
+        };
+        byId("download").onclick = async () => {
+          if (!selected) return;
+          const file = selected;
+          byId("error").textContent = "";
+          if (!file.archive) {
+            const a = el("a");
+            a.href = file.url;
+            a.download = file.path;
+            a.click();
+            return;
+          }
+          try {
+            // Stream large entries to disk where supported; never buffer an entire ZIP.
+            if ("showSaveFilePicker" in window) {
+              const handle = await window.showSaveFilePicker({
+                suggestedName: file.path.split("/").pop(),
+              });
+              const writable = await handle.createWritable();
+              try {
+                await (await entryStream(file)).pipeTo(writable);
+              } catch (e) {
+                await writable.abort().catch(() => {});
+                throw e;
+              }
+            } else {
+              if (file.size > MAX_PREVIEW)
+                throw Error(
+                  "Large individual downloads require a browser with streaming file saves. Use ZIP to download the archive."
+                );
+              const blob = await new Response(await entryStream(file)).blob(),
+                url = URL.createObjectURL(blob),
+                a = el("a");
+              a.href = url;
+              a.download = file.path.split("/").pop();
+              a.click();
+              setTimeout(() => URL.revokeObjectURL(url), 60000);
+            }
+          } catch (error) {
+            report(error);
+          }
+        };
+        const download = el("a", "Download ZIP", "action");
+        download.href = raw.href;
+        download.download = "";
+        byId("actions").insertBefore(download, byId("files"));
+        document.title = byId("title").textContent;
+        indexZip({ url: raw.href, size: Number(document.body.dataset.size) })
+          .then((entries) => {
+            files = entries;
+            browse("");
+          })
+          .catch(report);
+      })();
+    </script>
+  </body>
+</html>

archivebox/tests/conftest.py

@@ -14,6 +14,7 @@ import textwrap
 import time
 import shutil
 import ctypes
+from urllib.parse import urlsplit
 from datetime import timedelta
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from pathlib import Path
@@ -22,6 +23,13 @@ from types import SimpleNamespace
 from typing import Any
 from collections.abc import Callable
 
+from abx_plugins.plugins.opencode.archivebox.conftest import (
+    opencode_archive_config as opencode_archive_config,
+    installed_opencode as installed_opencode,
+    live_opencode as live_opencode,
+    agent_server as agent_server,
+)
+
 import psutil
 import pytest
 import requests
@@ -32,6 +40,14 @@ PYTEST_BASETEMP_ROOT = (REPO_ROOT / "tests" / "out").resolve()
 SESSION_DATA_DIR = Path(
     os.environ.get("ARCHIVEBOX_PYTEST_SESSION_DATA_DIR") or tempfile.mkdtemp(prefix="archivebox-pytest-session-"),
 ).resolve()
+PORT_ALLOCATION_DIR = Path(os.environ.setdefault("ARCHIVEBOX_PYTEST_PORTS_DIR", str(SESSION_DATA_DIR / "ports")))
+if worker_id := os.environ.get("PYTEST_XDIST_WORKER"):
+    # Workers inherit the coordinator's environment. Give each one its own
+    # collection and cleanup root so a finishing worker cannot delete another's.
+    SESSION_DATA_DIR /= worker_id
+    for runtime_key in ("ABXPKG_LIB_DIR", "PERSONAS_DIR", "CHROME_USER_DATA_DIR"):
+        if runtime_path := os.environ.get(runtime_key):
+            os.environ[runtime_key] = str(Path(runtime_path) / worker_id)
 
 os.environ["ARCHIVEBOX_PYTEST_SESSION_DATA_DIR"] = str(SESSION_DATA_DIR)
 os.environ["DATA_DIR"] = str(SESSION_DATA_DIR)
@@ -323,6 +339,18 @@ def pytest_configure():
     if not apps.ready:
         django.setup()
 
+    if os.environ.get("PYTEST_XDIST_WORKER"):
+        from django.conf import settings
+        from archivebox.config import django as django_config
+        from archivebox.config.common import get_config
+
+        # pytest-django may import startup config before this conftest isolates
+        # the worker's paths. Refresh that real startup config once so HTTP
+        # requests resolve the same installed binaries as worker subprocesses;
+        # otherwise a service restart searches the coordinator's empty lib dir.
+        django_config.CONFIG = get_config(include_machine=False)
+        settings.CONFIG = django_config.CONFIG
+
 
 @pytest.fixture(scope="session")
 def django_db_modify_db_settings(
@@ -437,7 +465,7 @@ def hermetic_lib_dir(tmp_path):
 @pytest.fixture
 def cached_abxpkg_lib_dir():
     """Reuse one real abxpkg installation cache for the current pytest session."""
-    lib_dir = SESSION_DATA_DIR / "lib"
+    lib_dir = Path(os.environ["ABXPKG_LIB_DIR"]) if os.environ.get("ABXPKG_LIB_DIR") else SESSION_DATA_DIR / "lib"
     lib_dir.mkdir(parents=True, exist_ok=True)
     original_lib_dir = os.environ.get("ABXPKG_LIB_DIR")
     os.environ["ABXPKG_LIB_DIR"] = str(lib_dir)
@@ -1076,6 +1104,8 @@ def start_archivebox_server(
     else:
         output = log_path.read_text(encoding="utf-8", errors="replace") if log_path else ""
         raise AssertionError(f"ArchiveBox server did not open port {port} within 30 seconds:\n{output}")
+    response = get_http_response(port, host=f"127.0.0.1:{port}", path="/health/", timeout=5)
+    assert response.status_code == 200 and response.headers.get("X-ArchiveBox-Health") == "OK", response.text
     return proc
 
 
@@ -1294,6 +1324,43 @@ def api_auth_headers(api_token: str, *, django_client: bool = False, port: int |
     }
 
 
+class LocalhostHTTPAdapter(requests.adapters.HTTPAdapter):
+    """Connect real HTTP requests to loopback while preserving URL authorities."""
+
+    def get_connection_with_tls_context(self, request, verify, proxies=None, cert=None):
+        url = urlsplit(request.url)
+        if url.scheme != "http" or not (url.hostname == "localhost" or (url.hostname or "").endswith(".localhost")):
+            raise ValueError(f"Expected an HTTP localhost test URL, got {request.url}")
+        return self.poolmanager.connection_from_host("127.0.0.1", port=url.port, scheme="http")
+
+    def send(self, request, **kwargs):
+        # Send Host only on the wire. Requests copies prepared headers on
+        # redirects and uses Host for cookie matching; leaving it attached
+        # would select the previous origin's cookies after a cross-host redirect.
+        original_host = request.headers.get("Host")
+        request.headers["Host"] = urlsplit(request.url).netloc
+        try:
+            return super().send(request, **kwargs)
+        finally:
+            if original_host is None:
+                request.headers.pop("Host", None)
+            else:
+                request.headers["Host"] = original_host
+
+
+def localhost_session() -> requests.Session:
+    # Browsers resolve *.localhost themselves; Linux libc in stock containers
+    # often does not. This is curl --connect-to for real server tests: only the
+    # socket destination changes. Keep the original URL so Requests exercises
+    # real redirect and cookie-domain rules, including dynamic snap-* hosts.
+    # Never replace URLs with 127.0.0.1 or stub responses: that hides isolation
+    # bugs. This also avoids global /etc/hosts edits and runner-specific DNS.
+    session = requests.Session()
+    session.trust_env = False
+    session.mount("http://", LocalhostHTTPAdapter())
+    return session
+
+
 def wait_for_live_api(port: int, *, path: str = "/api/v1/docs"):
     return get_http_response(port, host=f"api.archivebox.localhost:{port}", path=path)
 
@@ -1339,9 +1406,20 @@ def init_archive(cwd: Path) -> None:
 
 
 def get_free_port() -> int:
-    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
-        sock.bind(("127.0.0.1", 0))
-        return sock.getsockname()[1]
+    # Binding port zero only proves availability now. Another pytest worker can
+    # receive that same port before either service starts. Claim it atomically
+    # for this session while the socket is still bound, across all workers and
+    # helper subprocesses; the session cleanup removes these small markers.
+    PORT_ALLOCATION_DIR.mkdir(parents=True, exist_ok=True)
+    while True:
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+            sock.bind(("127.0.0.1", 0))
+            port = sock.getsockname()[1]
+            try:
+                (PORT_ALLOCATION_DIR / str(port)).touch(exist_ok=False)
+            except FileExistsError:
+                continue
+            return port
 
 
 def stop_server(cwd: Path) -> None:
@@ -1937,3 +2015,60 @@ def create_test_snapshot_json(url: str | None = None, **kwargs) -> dict[str, Any
         "status": kwargs.get("status", "queued"),
         **{k: v for k, v in kwargs.items() if k not in ("tags_str", "status")},
     }
+
+
+@pytest.fixture
+def browser_runtime(cached_abxpkg_lib_dir):
+    from abx_plugins import get_plugins_dir
+    from abx_plugins.plugins.base.utils import get_config
+
+    # UI tests need isolated collections, but share the browser installation.
+    # Resolve it through the same abxpkg schema/cache as capture hooks rather
+    # than downloading Playwright's separate Chromium in CI. This keeps local
+    # and CI runs on the supported install path without per-test cold installs.
+    shared_lib = cached_abxpkg_lib_dir
+    env = cli_env(
+        ABXPKG_LIB_DIR=str(shared_lib),
+        CHROME_HEADLESS="True",
+        CHROME_SANDBOX="False",
+        CHROME_ISOLATION="snapshot",
+    )
+    env.pop("CHROME_BINARY", None)
+    resolved_env = resolve_abxpkg_chrome_env(shared_lib, env)
+    chrome_config = get_config(
+        Path(get_plugins_dir()) / "chrome" / "config.json",
+        environ={**env, **resolved_env},
+        hydrate_binaries=False,
+    )
+
+    return {
+        "lib_dir": shared_lib,
+        "node_modules_dir": Path(resolved_env["NODE_MODULES_DIR"]),
+        "node_path": resolved_env["NODE_PATH"],
+        "node_binary": Path(resolved_env["NODE_BINARY"]),
+        "chrome_binary": Path(resolved_env["CHROME_BINARY"]),
+        "chrome_args": [*chrome_config.CHROME_ARGS, *chrome_config.CHROME_ARGS_EXTRA],
+    }
+
+
+def _reset_runtime_config() -> None:
+    from archivebox.config import common
+    from archivebox.machine.models import Machine
+
+    for value in vars(common).values():
+        cache_clear = getattr(value, "cache_clear", None)
+        if cache_clear is not None:
+            cache_clear()
+    Machine.current()
+
+
+def _set_archivebox_config(data_dir: Path, *values: str, env: dict[str, str] | None = None) -> None:
+    os.chdir(data_dir)
+    result = run_archivebox_cmd(
+        ["config", "--set", *values],
+        cwd=data_dir,
+        env=env,
+        timeout=120,
+    )
+    assert result.returncode == 0, result.stderr or result.stdout
+    _reset_runtime_config()

archivebox/tests/fixtures/responsive-images/README.md

@@ -0,0 +1,13 @@
+# Responsive-image replay regression
+
+`google-drive.wacz` contains two unmodified, gzip-compressed WARC response
+records from a real ArchiveWeb.page capture on 2026-10-04:
+
+- https://drive.google.com/drive/folders/1KpLl_1tcK0eeehzN980zbG-3M2nhbVks
+- https://www.gstatic.com/images/branding/productlogos/drive_2026/v1/web-48dp/logo_drive_2026_color_1x_web_48dp.png
+
+The public folder's HTML declares both 1x and 2x logo URLs, but the recording
+only contains the 1x image. These records were copied byte-for-byte from the
+capture's `archive/data.warc.gz` using its CDX offsets; the subset CDX offsets and datapackage hashes were recomputed. No HTTP
+responses were synthesized or edited. Other resources were omitted to keep the fixture small.
+The test replays this actual archive at DPR 1 and 2, entirely from local storage.

archivebox/tests/test_api_discovery_performance.py

@@ -0,0 +1,54 @@
+"""Server discovery must not repeat schema generation or probe installed tools."""
+
+import sys
+import threading
+import os
+from pathlib import Path
+
+import pytest
+from django.test.utils import override_script_prefix
+
+from archivebox.config import VERSION
+from archivebox.machine.models import Machine
+from archivebox.tests.conftest import install_real_binary, resolve_abxpkg_binary_env
+
+
+@pytest.mark.django_db(transaction=True)
+@pytest.mark.parametrize("script_name", ["", "/mounted"])
+def test_repeated_discovery_reuses_schema_without_binary_filesystem_probes(client, hermetic_lib_dir, script_name):
+    install_real_binary("node", binproviders="env,apt,brew")
+    resolve_abxpkg_binary_env(hermetic_lib_dir, "node")
+    machine = Machine.current()
+    machine.config = {**machine.config, "NODE_BINARY": str(hermetic_lib_dir / "env" / "bin" / "node")}
+    machine.save(update_fields=["config"])
+    request = {"HTTP_HOST": "api.archivebox.localhost:5797", "SCRIPT_NAME": script_name}
+    with override_script_prefix(script_name):
+        first = client.get("/api/v1/openapi.json", **request)
+    assert first.status_code == 200, first.content
+    schema_calls = []
+    filesystem_calls = []
+
+    def observe(frame, event, arg):
+        if event == "call" and frame.f_code.co_name == "model_json_schema":
+            schema_calls.append(frame.f_code.co_filename)
+        if (event == "call" and frame.f_code.co_name in {"stat", "lstat", "open", "iterdir", "glob", "rglob"}) or (
+            event == "c_call" and arg in (os.stat, os.lstat)
+        ):
+            path = frame.f_locals.get("self", frame.f_locals.get("path"))
+            if isinstance(path, (str, Path)) and Path(path).is_relative_to(hermetic_lib_dir):
+                filesystem_calls.append(str(path))
+
+    previous = sys.getprofile()
+    previous_thread = threading.getprofile()
+    threading.setprofile_all_threads(observe)
+    try:
+        with override_script_prefix(script_name):
+            second = client.get("/api/v1/openapi.json", **request)
+    finally:
+        threading.setprofile_all_threads(previous_thread)
+        sys.setprofile(previous)
+    assert second.status_code == 200, second.content
+    assert second.json() == first.json()
+    assert second.json()["info"]["version"] == VERSION
+    assert f"{script_name}/api/v1/core/snapshots" in second.json()["paths"]
+    assert (len(schema_calls), filesystem_calls) == (0, [])

archivebox/tests/test_api_v1_cli_add.py

@@ -478,3 +478,40 @@ def test_api_cli_add_rejects_file_path_and_shell_injection_payloads(tmp_path):
     with use_archivebox_db(tmp_path):
         tag_names = set(SnapshotTag.objects.filter(snapshot=snapshot).values_list("tag__name", flat=True))
     assert "api-security" in tag_names
+
+
+def test_api_add_only_queues_work_without_storage_or_worker_control(client, api_headers):
+    import sys
+    from archivebox.config import CONSTANTS
+
+    calls = []
+
+    def observe(frame, event, arg):
+        if event != "call":
+            return
+        name = frame.f_code.co_name
+        if name == "ensure_background_runner":
+            calls.append(name)
+        if name in {"stat", "lstat", "exists", "is_dir", "is_file", "is_symlink", "mkdir", "iterdir", "symlink_to"}:
+            path = frame.f_locals.get("self")
+            if isinstance(path, Path) and path.is_relative_to(CONSTANTS.ARCHIVE_DIR):
+                calls.append((name, str(path)))
+
+    previous = sys.getprofile()
+    sys.setprofile(observe)
+    try:
+        response = client.post(
+            "/api/v1/cli/add",
+            data={"urls": ["https://example.com/queue-only"], "tag": "queue-only"},
+            content_type="application/json",
+            **api_headers,
+        )
+    finally:
+        sys.setprofile(previous)
+    assert response.status_code == 200, response.content
+    crawl = Crawl.objects.get(pk=response.json()["result"]["crawl_id"])
+    assert crawl.urls == "https://example.com/queue-only"
+    assert crawl.status == Crawl.StatusChoices.QUEUED
+    assert crawl.retry_at is not None
+    assert not crawl.snapshot_set.exists()
+    assert calls == []

archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py

@@ -477,25 +477,38 @@ def test_paused_snapshot_survives_server_restart_and_resumes_via_api(client, tmp
         stop_server(tmp_path)
 
 
-def test_rest_snapshot_delete_removes_output_dir(client, api_headers):
-    url = "https://example.com/delete-path-snapshot"
+@pytest.mark.timeout(180)
+def test_rest_snapshot_delete_removes_output_dir(tmp_path, recursive_test_site):
+    import time
+    from .test_snapshot_service import _snapshot_state
+    from .conftest import create_admin_and_token, live_api_request, start_archivebox_server, stop_archivebox_process
 
-    response = api_client_request(
-        client,
-        "post",
-        "/api/v1/core/snapshots",
-        payload={"url": url, "depth": 0, "status": Snapshot.StatusChoices.QUEUED},
-        headers=api_headers,
-    )
-    assert response.status_code == 200, response.content.decode()
-
-    snapshot = Snapshot.objects.get(url=url)
-    snapshot_dir = Path(snapshot.output_dir)
-    snapshot_dir.mkdir(parents=True, exist_ok=True)
-    (snapshot_dir / "delete-path-test.txt").write_text("snapshot output")
-    assert snapshot_dir.exists()
-
-    response = client.delete(f"/api/v1/core/snapshot/{snapshot.id}", **api_headers)
-    assert response.status_code == 200, response.content.decode()
-    assert not Snapshot.objects.filter(pk=snapshot.pk).exists()
-    assert not snapshot_dir.exists()
+    init_archive(tmp_path)
+    port = get_free_port()
+    env = cli_env(port=port, server=True, PLUGINS="wget")
+    url = recursive_test_site["root_url"]
+    captured = run_archivebox_cmd(["add", "--plugins=wget", url], cwd=tmp_path, env=env)
+    assert captured.returncode == 0, captured.stdout + captured.stderr
+    state = _snapshot_state(tmp_path, url)
+    assert any("Root" in path.read_text(errors="ignore") for path in (state["snapshot_dir"] / "wget").rglob("*.html"))
+    with use_archivebox_db(tmp_path):
+        snapshot_id = str(Snapshot.objects.get(url=url).pk)
+    token = create_admin_and_token(tmp_path)
+    server = start_archivebox_server(tmp_path, port=port, env=env, log_name="api-delete-handoff.log")
+    try:
+        response = live_api_request(port, "delete", f"/api/v1/core/snapshot/{snapshot_id}", api_token=token)
+        assert response.status_code == 200, response.text
+        assert response.json()["queued_count"] == 1
+        assert response.json()["deleted_count"] == 0
+        deadline = time.monotonic() + 75
+        with use_archivebox_db(tmp_path):
+            while Snapshot.objects.filter(pk=snapshot_id).exists() and time.monotonic() < deadline:
+                assert server.poll() is None
+                time.sleep(0.1)
+            assert not Snapshot.objects.filter(pk=snapshot_id).exists()
+        assert not state["snapshot_dir"].exists()
+        assert not state["crawl_link"].is_symlink()
+        missing = live_api_request(port, "get", f"/api/v1/core/snapshot/{snapshot_id}", api_token=token)
+        assert missing.status_code == 404
+    finally:
+        stop_archivebox_process(server)

archivebox/tests/test_api_v1_core_snapshots.py

@@ -13,6 +13,65 @@ from archivebox.crawls.models import Crawl
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
+def test_browser_metadata_respects_only_new_and_reports_capture_owner(client, api_headers):
+    url = "https://example.com/browser-deduplication"
+    for name in ("Original", "Replacement"):
+        persona = client.post(
+            "/api/v1/personas/sync",
+            data={"extension_persona_id": f"dedupe-{name}", "name": name},
+            content_type="application/json",
+            **api_headers,
+        )
+        assert persona.status_code == 200, persona.content
+
+    def submit(only_new, persona):
+        queued = client.post(
+            "/api/v1/cli/add",
+            data={"urls": [url], "only_new": only_new, "persona": persona},
+            content_type="application/json",
+            **api_headers,
+        )
+        assert queued.status_code == 200, queued.content
+        crawl_id = queued.json()["result"]["crawl_id"]
+        metadata = client.post(
+            "/api/v1/core/snapshots",
+            data={"url": url, "crawl_id": crawl_id},
+            content_type="application/json",
+            **api_headers,
+        )
+        assert metadata.status_code == 200, metadata.content
+        return crawl_id, metadata.json()
+
+    first_crawl, first = submit(True, "Original")
+    second_crawl, reused = submit(True, "Replacement")
+    assert second_crawl != first_crawl
+    assert reused["id"] == first["id"]
+    assert reused["crawl_id"] == first_crawl
+    assert reused["persona"] == "Original"
+    assert Snapshot.objects.filter(url=url).count() == 1
+    forced_crawl, forced = submit(False, "Replacement")
+    assert forced["id"] != first["id"]
+    assert forced["crawl_id"] == forced_crawl
+    assert forced["persona"] == "Replacement"
+    assert Snapshot.objects.filter(url=url).count() == 2
+
+
+def test_snapshot_delete_only_queues_cleanup(client, api_headers):
+    created = client.post(
+        "/api/v1/core/snapshots",
+        data={"url": "https://example.com/queued-delete"},
+        content_type="application/json",
+        **api_headers,
+    )
+    assert created.status_code == 200, created.content
+    snapshot_id = created.json()["id"]
+    response = client.delete(f"/api/v1/core/snapshot/{snapshot_id}", **api_headers)
+    assert response.status_code == 200, response.content
+    assert response.json()["queued_count"] == 1
+    assert response.json()["deleted_count"] == 0
+    assert Snapshot.objects.get(pk=snapshot_id).status == Snapshot.DELETING_STATE
+
+
 def test_snapshots_api_filters_status_column(client, api_admin_user, api_headers):
     crawl = Crawl.objects.create(
         urls="https://example.com",
@@ -150,3 +209,81 @@ def test_new_snapshot_creation_does_not_wait_for_active_crawl(client, api_admin_
     assert response.status_code == 200, response.content
     assert Snapshot.objects.filter(url=url, crawl=crawl).count() == 1
     assert elapsed < 1
+
+
+def test_snapshot_metadata_requests_do_not_touch_archive_storage(client, api_admin_user, api_headers):
+    """Metadata writes and serialization must work without replay storage access."""
+    import sys
+    import threading
+    from pathlib import Path
+    from archivebox.config import CONSTANTS
+
+    url = "https://example.com/metadata-without-storage"
+    crawl = Crawl.objects.create(urls=url, created_by=api_admin_user)
+    storage_calls = []
+
+    def observe(frame, event, arg):
+        if event == "call" and frame.f_code.co_name in {
+            "stat",
+            "lstat",
+            "exists",
+            "is_dir",
+            "is_file",
+            "is_symlink",
+            "open",
+            "mkdir",
+            "iterdir",
+            "glob",
+            "rglob",
+            "readlink",
+            "symlink_to",
+            "unlink",
+        }:
+            path = frame.f_locals.get("self", frame.f_locals.get("path"))
+            if isinstance(path, (str, Path)) and Path(path).is_relative_to(CONSTANTS.ARCHIVE_DIR):
+                storage_calls.append((frame.f_code.co_name, str(path)))
+
+    previous = sys.getprofile()
+    thread_profile = threading.getprofile()
+    threading.setprofile_all_threads(observe)
+    try:
+        response = client.post(
+            "/api/v1/core/snapshots",
+            data={"url": url, "crawl_id": str(crawl.id), "title": "Browser title", "tags": ["browser"]},
+            content_type="application/json",
+            **api_headers,
+        )
+        assert response.status_code == 200, response.content
+        snapshot_id = response.json()["id"]
+        updated = client.post(
+            "/api/v1/core/snapshots",
+            data={"url": url, "crawl_id": str(crawl.id), "title": "Updated title", "tags": ["browser"]},
+            content_type="application/json",
+            **api_headers,
+        )
+        assert updated.status_code == 200, updated.content
+        result = client.post(
+            "/api/v1/core/archiveresults",
+            data={"snapshot_id": snapshot_id, "plugin": "chrome_extension_viewport", "status": "started"},
+            **api_headers,
+        )
+        assert result.status_code == 200, result.content
+        result_id = result.json()["id"]
+        result_get = client.get(f"/api/v1/core/archiveresult/{result_id}", **api_headers)
+        assert result_get.status_code == 200, result_get.content
+        assert result_get.json()["output_files"] == {}
+        fetched = client.get(f"/api/v1/core/snapshot/{snapshot_id}", **api_headers)
+        assert fetched.status_code == 200, fetched.content
+        assert fetched.json()["archiveresults"][0]["id"] == result_id
+        deleted = client.delete(f"/api/v1/core/snapshot/{snapshot_id}", **api_headers)
+        assert deleted.status_code == 200, deleted.content
+        assert deleted.json()["queued_count"] == 1
+    finally:
+        threading.setprofile_all_threads(thread_profile)
+        sys.setprofile(previous)
+    snapshot = Snapshot.objects.get(pk=snapshot_id)
+    assert snapshot.title == "Updated title"
+    assert snapshot.status == Snapshot.DELETING_STATE
+    assert list(snapshot.tags.values_list("name", flat=True)) == ["browser"]
+    assert fetched.json()["title"] == "Updated title"
+    assert storage_calls == []

archivebox/tests/test_archive_result_service.py

@@ -51,7 +51,7 @@ def _run_shipped_snapshot_hook(
 
     from abx_dl.services.process_service import ProcessService as HookProcessService
     from abx_dl.services.archive_result_service import ArchiveResultService as HookArchiveResultService
-    from abx_plugins.plugins.base.utils import _collect_required_schema_paths, get_hydrated_required_binaries
+    from abx_plugins.plugins.base.utils import _collect_required_schema_paths
     from abxpkg import prepare_script_exec_plan
     from archivebox.core.models import ArchiveResult
     from archivebox.machine.models import Process
@@ -66,15 +66,9 @@ def _run_shipped_snapshot_hook(
     hook_path = matching_hooks[0].path
     projected_hook_name = event_hook_name or hook_name
     hook_config = hook_path.parent / "config.json"
-    for required_binary in get_hydrated_required_binaries(
-        hook_config,
-        environ={**os.environ, "ABXPKG_LIB_DIR": str(lib_dir)},
-    ):
-        install_real_binary(
-            required_binary["name"],
-            binproviders=required_binary["binproviders"],
-            overrides=required_binary.get("overrides"),
-        )
+    # This helper runs standalone hooks. Resolve their dependencies through the
+    # public abxpkg CLI below; creating a new ArchiveBox Binary row on every
+    # invocation makes a second run collide with the already-installed identity.
     binary_env = resolve_abxpkg_binary_env(lib_dir, deps_from=_collect_required_schema_paths(hook_config))
     hook_env = {
         **binary_env,

archivebox/tests/test_binary_service.py

@@ -198,21 +198,26 @@ def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized
     assert Path(relibbed.abspath).resolve() == Path(host_binary).resolve()
 
 
-def test_missing_binary_request_stays_queued_then_recovers_when_provider_can_resolve(initialized_archive, tmp_path):
-    name = "http"
+def test_missing_binary_request_stays_queued_then_recovers_when_provider_can_resolve(initialized_archive):
+    name = "httpie"
     provider_bin_dir = initialized_archive / "lib" / "pip" / "venv" / "bin"
     runtime_env = _runtime_env(initialized_archive)
 
-    _cmd_result = _run_real_binary_lifecycle(initialized_archive, name=name, binproviders="env", env=runtime_env)
-    stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
-
-    assert returncode != 0, stdout + stderr
-    assert "Binary http installation failed" in stderr
+    failed = run_archivebox_cmd(
+        ["install", "--binproviders=env", name],
+        cwd=initialized_archive,
+        timeout=120,
+        env=runtime_env,
+        default_cli_env=True,
+        disable_extractors=True,
+    )
+    assert failed.returncode != 0, failed.stdout + failed.stderr
+    assert f"Binary {name} installation failed" in failed.stderr
 
     with use_archivebox_db(initialized_archive):
         queued = Binary.objects.get(name=name)
         queued_id = str(queued.id)
-        failed_process = Process.objects.filter(process_type=Process.TypeChoices.BINARY).latest("created_at")
+        failed_process = Process.objects.filter(binary_id=queued_id).latest("created_at")
         machine_config = Machine.objects.get(pk=queued.machine_id).config or {}
 
     assert queued.status == Binary.StatusChoices.QUEUED
@@ -223,28 +228,42 @@ def test_missing_binary_request_stays_queued_then_recovers_when_provider_can_res
     assert f"{name.upper().replace('-', '_')}_BINARY" not in machine_config
     assert not (provider_bin_dir / name).exists()
 
-    with use_archivebox_db(initialized_archive):
-        queued = Binary.objects.get(pk=queued_id)
-        queued.binproviders = "pip"
-        queued.overrides = {"pip": {"install_args": ["httpie>=3.2.4"]}}
-        queued.save(update_fields=["binproviders", "overrides", "modified_at"])
-    recovered_runtime_env = _runtime_env(initialized_archive)
-
-    _cmd_result = run_archivebox_cmd(
-        ["run", f"--binary-id={queued_id}"],
+    recovered_install = run_archivebox_cmd(
+        ["install", "--binproviders=pip", name],
         cwd=initialized_archive,
         timeout=120,
-        env=recovered_runtime_env,
+        env=runtime_env,
         default_cli_env=True,
         disable_extractors=True,
     )
-    recover_stdout, recover_stderr, recover_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
-
-    assert recover_code == 0, recover_stdout + recover_stderr
+    assert recovered_install.returncode == 0, recovered_install.stdout + recovered_install.stderr
+    repeated_install = run_archivebox_cmd(
+        ["install", "--binproviders=pip", name],
+        cwd=initialized_archive,
+        timeout=120,
+        env=runtime_env,
+        default_cli_env=True,
+        disable_extractors=True,
+    )
+    assert repeated_install.returncode == 0, repeated_install.stdout + repeated_install.stderr
+    # Changing the provider policy runs a real installation request against the
+    # now-warm cache. Its fast result must still finalize the DB projection.
+    cached_install = run_archivebox_cmd(
+        ["install", "--binproviders=pip,env", name],
+        cwd=initialized_archive,
+        timeout=120,
+        env=runtime_env,
+        default_cli_env=True,
+        disable_extractors=True,
+    )
+    assert cached_install.returncode == 0, cached_install.stdout + cached_install.stderr
     with use_archivebox_db(initialized_archive):
         recovered = Binary.objects.get(pk=queued_id)
+        # Providers may resolve their own dependencies during installation. Those
+        # Process rows are not attempts to install this binary; track its exact
+        # identity instead of assuming the last two global rows belong to it.
         process_exit_codes = list(
-            Process.objects.filter(process_type=Process.TypeChoices.BINARY).order_by("created_at").values_list("exit_code", flat=True),
+            Process.objects.filter(binary_id=queued_id).order_by("created_at").values_list("exit_code", flat=True),
         )
 
     assert recovered.status == Binary.StatusChoices.INSTALLED
@@ -253,4 +272,4 @@ def test_missing_binary_request_stays_queued_then_recovers_when_provider_can_res
     assert Path(recovered.abspath) == provider_bin_dir / name
     assert recovered.binprovider == "pip"
     assert Path(recovered.abspath).is_file()
-    assert process_exit_codes[-2:] == [1, 0]
+    assert process_exit_codes == [1, 0, 0]

archivebox/tests/test_cli_add_4.py

@@ -325,11 +325,15 @@ def test_add_duplicate_url_creates_separate_crawls(initialized_archive):
     assert snapshots == [("https://example.com", 0), ("https://example.com", 0)]
 
 
-def test_snapshot_create_creates_current_output_directory(initialized_archive):
-    """Test the user-facing snapshot creation path creates an output directory."""
-    env = cli_env(disable_extractors=True)
+def test_snapshot_create_server_creates_current_output_directory(initialized_archive, recursive_test_site):
+    """Queued metadata gets its output directory from the normal server worker."""
+    from .conftest import start_archivebox_server, stop_archivebox_process, get_free_port, wait_for_snapshot_capture
+
+    port = get_free_port()
+    env = cli_env(port=port, server=True, PLUGINS="wget")
+    url = recursive_test_site["root_url"]
     run_archivebox_cmd(
-        ["snapshot", "create", "https://example.com"],
+        ["snapshot", "create", url],
         cwd=initialized_archive,
         env=env,
         check=True,
@@ -338,6 +342,12 @@ def test_snapshot_create_creates_current_output_directory(initialized_archive):
     with use_archivebox_db(initialized_archive):
         snapshot_id = str(Snapshot.objects.values_list("id", flat=True).get())
 
+    assert find_snapshot_dir(initialized_archive, snapshot_id) is None
+    server = start_archivebox_server(initialized_archive, env=env, port=port, log_name="snapshot-output-server.log")
+    try:
+        assert "Root" in wait_for_snapshot_capture(initialized_archive, url, timeout=120)
+    finally:
+        stop_archivebox_process(server)
     snapshot_dir = find_snapshot_dir(initialized_archive, snapshot_id)
     assert snapshot_dir is not None, f"Snapshot output directory not found for {snapshot_id}"
     assert snapshot_dir.is_dir()

archivebox/tests/test_cli_background_screenshot.py

@@ -0,0 +1,39 @@
+"""Exercise the browser target lifecycle through a real multi-page CLI crawl."""
+
+import os
+import sqlite3
+import subprocess
+
+import pytest
+
+
+@pytest.mark.timeout(180)
+def test_cli_crawl_captures_background_screenshots(tmp_path):
+    env = {**os.environ, "SCREENSHOT_TIMEOUT": "5"}
+    initialized = subprocess.run(
+        ["archivebox", "init", "--quick"],
+        cwd=tmp_path,
+        env=env,
+        capture_output=True,
+        text=True,
+        timeout=60,
+    )
+    assert initialized.returncode == 0, initialized.stderr
+    captured = subprocess.run(
+        ["archivebox", "add", "--plugins=title,wget,screenshot", "https://example.com", "https://example.org"],
+        cwd=tmp_path,
+        env=env,
+        capture_output=True,
+        text=True,
+        timeout=120,
+    )
+    assert captured.returncode == 0, captured.stderr
+    with sqlite3.connect(tmp_path / "index.sqlite3") as db:
+        results = db.execute("SELECT status, output_str FROM core_archiveresult WHERE plugin='screenshot'").fetchall()
+    assert len(results) == 2, captured.stdout
+    if any(status != "succeeded" for status, _ in results):
+        print(captured.stdout, captured.stderr)
+    assert all(status == "succeeded" for status, _ in results), (results, captured.stdout, captured.stderr)
+    images = list((tmp_path / "archive" / "users").glob("*/snapshots/**/screenshot/screenshot.png"))
+    assert len(images) == 2
+    assert all(image.read_bytes().startswith(b"\x89PNG\r\n\x1a\n") and image.stat().st_size > 1000 for image in images)

archivebox/tests/test_cli_config.py

@@ -8,37 +8,20 @@ from archivebox.tests.conftest import run_archivebox_cmd
 from archivebox.config.configset import read_ini_config
 
 
-def test_config_displays_all_config(initialized_archive):
-    """Test that config without args displays all configuration."""
-    result = run_archivebox_cmd(["config"])
-
-    assert result.returncode == 0
-    output = result.stdout
-    # Should show config sections
-    assert len(output) > 100
-    # Should show at least some standard config keys
-    assert "TIMEOUT" in output or "OUTPUT_PERMISSIONS" in output
-
-
-def test_config_shows_derived_collection_paths_but_not_runtime_dirs(initialized_archive):
-    """The CLI should expose collection paths, not per-crawl/per-snapshot runtime dirs."""
-    run_archivebox_cmd(["init"], cwd=initialized_archive, check=True)
-
+def test_config_read_get_and_search_cover_core_plugin_alias_and_path_options(initialized_archive):
+    """Exercise every read-only CLI branch against one unchanged collection."""
     result = run_archivebox_cmd(["config"], cwd=initialized_archive)
-
     assert result.returncode == 0, result.stderr
-    unwrapped_output = result.stdout.replace("\n", "")
-    assert "DATA_DIR" in result.stdout
-    assert result.stdout.count("\nDATA_DIR =") == 1
-    assert str(initialized_archive) in unwrapped_output
-    assert "PERSONAS_DIR" in result.stdout
-    assert result.stdout.count("\nPERSONAS_DIR =") == 1
-    assert "SNAP_DIR" not in result.stdout
-    assert "CRAWL_DIR" not in result.stdout
-
-
-def test_config_get_derived_path_but_rejects_runtime_dir(initialized_archive):
-    run_archivebox_cmd(["init"], cwd=initialized_archive, check=True)
+    output = result.stdout
+    assert len(output) > 100
+    assert "TIMEOUT" in output or "OUTPUT_PERMISSIONS" in output
+    assert "DATA_DIR" in output
+    assert output.count("\nDATA_DIR =") == 1
+    assert str(initialized_archive) in output.replace("\n", "")
+    assert "PERSONAS_DIR" in output
+    assert output.count("\nPERSONAS_DIR =") == 1
+    assert "SNAP_DIR" not in output
+    assert "CRAWL_DIR" not in output
 
     data_dir = run_archivebox_cmd(["config", "--get", "DATA_DIR"], cwd=initialized_archive)
     snap_dir = run_archivebox_cmd(["config", "--get", "SNAP_DIR"], cwd=initialized_archive)
@@ -49,10 +32,21 @@ def test_config_get_derived_path_but_rejects_runtime_dir(initialized_archive):
     assert snap_dir.returncode != 0
     assert "SNAP_DIR =" not in snap_dir.stdout
 
+    result = run_archivebox_cmd(
+        ["config", "--get", "TIMEOUT"],
+    )
 
-def test_config_set_rejects_readonly_and_runtime_dirs(initialized_archive):
-    run_archivebox_cmd(["init"], cwd=initialized_archive, check=True)
+    assert result.returncode == 0
+    assert "TIMEOUT" in result.stdout
 
+    for query, expected in (("TIMEOUT", "TIMEOUT"), ("wget", "WGET_BINARY"), ("URL_BLACK", "URL_DENYLIST")):
+        result = run_archivebox_cmd(["config", "--search", query])
+        assert result.returncode == 0, result.stderr
+        assert expected in result.stdout, query
+
+
+def test_config_rejects_readonly_runtime_unknown_and_malformed_keys(initialized_archive):
+    """Each rejected input must fail through its own CLI validation branch."""
     data_dir = run_archivebox_cmd(
         ["config", "--set", f"DATA_DIR={initialized_archive / 'other'}"],
         cwd=initialized_archive,
@@ -68,87 +62,12 @@ def test_config_set_rejects_readonly_and_runtime_dirs(initialized_archive):
     assert "DATA_DIR" not in content
     assert "CRAWL_DIR" not in content
 
-
-def test_config_get_specific_key(initialized_archive):
-    """Test that config --get KEY retrieves specific value."""
-    result = run_archivebox_cmd(
-        ["config", "--get", "TIMEOUT"],
-    )
-
-    assert result.returncode == 0
-    assert "TIMEOUT" in result.stdout
-
-
-def test_config_set_writes_to_file(initialized_archive):
-    """Test that config --set KEY=VALUE writes to ArchiveBox.conf."""
-
-    result = run_archivebox_cmd(
-        ["config", "--set", "TIMEOUT=120"],
-    )
-
-    assert result.returncode == 0
-    assert "TIMEOUT=120" in result.stdout
-
-    # Verify config file was updated
-    config_file = initialized_archive / "ArchiveBox.conf"
-    assert config_file.exists()
-
-    content = config_file.read_text()
-    assert "TIMEOUT" in content or "120" in content
-
-
-def test_config_set_and_get_roundtrip(initialized_archive):
-    """Test that set value can be retrieved with get."""
-
-    # Set a unique value
-    run_archivebox_cmd(
-        ["config", "--set", "TIMEOUT=987"],
-    )
-
-    updated = run_archivebox_cmd(
-        ["config", "--set", "TIMEOUT=654"],
-    )
-
-    assert updated.returncode == 0
-    assert "TIMEOUT=654" in updated.stdout
-
-    # Get the value back
-    result = run_archivebox_cmd(
-        ["config", "--get", "TIMEOUT"],
-    )
-
-    assert "654" in result.stdout
-
-
-def test_config_set_multiple_values(initialized_archive):
-    """Test setting multiple config values at once."""
-
-    result = run_archivebox_cmd(
-        ["config", "--set", "TIMEOUT=111", "YTDLP_TIMEOUT=222"],
-    )
-
-    assert result.returncode == 0
-
-    # Verify both were written
-    config_file = initialized_archive / "ArchiveBox.conf"
-    content = config_file.read_text()
-    assert "111" in content
-    assert "222" in content
-
-
-def test_config_set_invalid_key_fails(initialized_archive):
-    """Test that setting invalid config key fails."""
-
     result = run_archivebox_cmd(
         ["config", "--set", "TOTALLY_INVALID_KEY_XYZ=value"],
     )
 
     assert result.returncode != 0
 
-
-def test_config_set_requires_equals_sign(initialized_archive):
-    """Test that set requires KEY=VALUE format."""
-
     result = run_archivebox_cmd(
         ["config", "--set", "TIMEOUT"],
     )
@@ -156,93 +75,46 @@ def test_config_set_requires_equals_sign(initialized_archive):
     assert result.returncode != 0
 
 
-def test_config_search_finds_keys(initialized_archive):
-    """Test that config --search finds matching keys."""
-
-    result = run_archivebox_cmd(
-        ["config", "--search", "TIMEOUT"],
-    )
-
-    # Should find timeout-related config
-    assert "TIMEOUT" in result.stdout
-
-
-def test_config_search_finds_plugin_options(initialized_archive):
-    """Test that config --search finds plugin keys and descriptions."""
-
-    result = run_archivebox_cmd(
-        ["config", "--search", "wget"],
-    )
-
-    assert result.returncode == 0
-    assert "WGET_BINARY" in result.stdout
-
-
-def test_config_search_finds_core_aliases(initialized_archive):
-    """Test that config --search finds core options by partial alias."""
-
-    result = run_archivebox_cmd(
-        ["config", "--search", "URL_BLACK"],
-    )
-
-    assert result.returncode == 0
-    assert "URL_DENYLIST" in result.stdout
-
-
-def test_config_preserves_existing_values(initialized_archive):
-    """Test that setting new values preserves existing ones."""
-
-    # Set first value
-    run_archivebox_cmd(
-        ["config", "--set", "TIMEOUT=100"],
-    )
-
-    # Set second value
-    run_archivebox_cmd(
-        ["config", "--set", "YTDLP_TIMEOUT=200"],
-    )
-
-    # Verify both are in config file
+def test_config_set_update_get_preserves_other_keys(initialized_archive):
+    """Exercise each write path against one real config, checking each transition."""
+    result = run_archivebox_cmd(["config", "--set", "TIMEOUT=120"])
+    assert result.returncode == 0, result.stderr
+    assert "TIMEOUT=120" in result.stdout
     config_file = initialized_archive / "ArchiveBox.conf"
+    assert config_file.is_file()
+    content = config_file.read_text()
+    assert "TIMEOUT" in content and "120" in content
+    assert "[" in content or "=" in content
+    assert read_ini_config(config_file)["TIMEOUT"] == "120"
+
+    result = run_archivebox_cmd(["config", "--set", "YTDLP_TIMEOUT=200"])
+    assert result.returncode == 0, result.stderr
     content = config_file.read_text()
     assert "TIMEOUT" in content
     assert "YTDLP_TIMEOUT" in content
+    assert read_ini_config(config_file)["TIMEOUT"] == "120"
+    assert read_ini_config(config_file)["YTDLP_TIMEOUT"] == "200"
+
+    updated = run_archivebox_cmd(["config", "--set", "TIMEOUT=654"])
+    assert updated.returncode == 0, updated.stderr
+    assert "TIMEOUT=654" in updated.stdout
+    result = run_archivebox_cmd(["config", "--get", "TIMEOUT"])
+    assert result.returncode == 0, result.stderr
+    assert "654" in result.stdout
+    assert read_ini_config(config_file)["TIMEOUT"] == "654"
+    assert read_ini_config(config_file)["YTDLP_TIMEOUT"] == "200"
 
 
-def test_config_file_is_valid_toml(initialized_archive):
-    """Test that config file remains valid TOML after set."""
-
-    run_archivebox_cmd(
-        ["config", "--set", "TIMEOUT=150"],
-    )
-
+def test_config_set_multiple_values_in_fresh_sections(initialized_archive):
+    """Keep first writes into new sections distinct from updating existing keys."""
+    result = run_archivebox_cmd(["config", "--set", "TIMEOUT=111", "YTDLP_TIMEOUT=222"])
+    assert result.returncode == 0, result.stderr
     config_file = initialized_archive / "ArchiveBox.conf"
     content = config_file.read_text()
-
-    # Basic TOML validation - should have sections and key=value pairs
-    assert "[" in content or "=" in content
-
-
-def test_config_updates_existing_value(initialized_archive):
-    """Test that setting same key twice updates the value."""
-
-    # Set initial value
-    run_archivebox_cmd(
-        ["config", "--set", "TIMEOUT=100"],
-    )
-
-    # Update to new value
-    run_archivebox_cmd(
-        ["config", "--set", "TIMEOUT=200"],
-    )
-
-    # Get current value
-    result = run_archivebox_cmd(
-        ["config", "--get", "TIMEOUT"],
-    )
-
-    # Should show updated value
-    assert "200" in result.stdout
+    assert "111" in content
+    assert "222" in content
+    assert read_ini_config(config_file)["TIMEOUT"] == "111"
+    assert read_ini_config(config_file)["YTDLP_TIMEOUT"] == "222"
 
 
 def test_config_ignores_legacy_unknown_keys(tmp_path, initialized_archive):

archivebox/tests/test_cli_extract.py

@@ -8,10 +8,12 @@ from urllib.parse import urlsplit
 import pytest
 from abx_plugins import get_plugins_dir
 from django.test import Client
+from django.db import connection
+from django.test.utils import CaptureQueriesContext
 
 from archivebox.core.models import ArchiveResult, Snapshot
 from archivebox.core.routes_util import build_snapshot_url
-from archivebox.tests.conftest import cli_env, find_snapshot_dir, parse_jsonl_output, run_archivebox_cmd
+from archivebox.tests.conftest import cli_env, create_admin_and_token, find_snapshot_dir, parse_jsonl_output, run_archivebox_cmd
 
 from archivebox.tests.test_orm_helpers import use_archivebox_db
 
@@ -80,8 +82,29 @@ def test_extract_runs_on_existing_snapshots(archive_with_extractors):
     assert archiveresults["wget"].output_str == "wget/example.com/index.html"
     assert archiveresults["wget"].output_files["example.com/index.html"]["size"] == wget_path.stat().st_size
 
+    token = create_admin_and_token(initialized_archive)
+    with use_archivebox_db(initialized_archive):
+        with CaptureQueriesContext(connection) as queries:
+            response = Client().get(
+                f"/api/v1/core/snapshot/{snapshot_id}",
+                HTTP_HOST="api.archivebox.localhost",
+                HTTP_X_ARCHIVEBOX_API_KEY=token,
+            )
+        assert response.status_code == 200, response.content
+        returned = {row["plugin"]: row for row in response.json()["archiveresults"]}
+        for plugin, result in archiveresults.items():
+            assert returned[plugin]["cmd"] == result.cmd
+            assert returned[plugin]["cmd_version"] == result.cmd_version
+            assert returned[plugin]["pwd"] == result.pwd
+        # Each real hook owns a Process and Binary. Metadata reads must fetch
+        # them with the result rows, not run two extra queries per plugin.
+        related_lookups = [
+            query["sql"] for query in queries if 'FROM "machine_process"' in query["sql"] or 'FROM "machine_binary"' in query["sql"]
+        ]
+        assert related_lookups == []
 
-def test_wget_literal_percent_output_url_serves_captured_file(initialized_archive, snapshot, live_server):
+
+def test_wget_literal_percent_output_url_serves_captured_file(initialized_archive, snapshot, live_server, browser_runtime):
     """Replay the real HedgeDoc link-interstitial file saved by wget."""
     import shutil
     from urllib.parse import quote
@@ -155,7 +178,10 @@ def test_wget_literal_percent_output_url_serves_captured_file(initialized_archiv
     )
 
     with sync_playwright() as playwright:
-        browser = playwright.chromium.launch(args=["--host-resolver-rules=MAP *.archivebox.localhost 127.0.0.1"])
+        browser = playwright.chromium.launch(
+            executable_path=str(browser_runtime["chrome_binary"]),
+            args=["--host-resolver-rules=MAP *.archivebox.localhost 127.0.0.1"],
+        )
         page = browser.new_page()
         page.goto(f"http://web.archivebox.localhost:{port}{ui_snapshot.get_absolute_url()}/index.html", wait_until="domcontentloaded")
         expected_path = quote(str(output_path), safe="/@=")

archivebox/tests/test_cli_init.py

@@ -95,110 +95,33 @@ def test_cli_refuses_source_root_without_side_effects_and_allows_separate_data_d
     assert (data_dir / "logs").is_dir()
 
 
-def test_init_creates_database_file(tmp_path):
-    """Test that init creates index.sqlite3 database file."""
+def test_init_creates_collection_schema_directories_config_and_permissions(tmp_path):
+    """Inspect all outputs of one real fresh init; upgrade cases stay separate."""
     result = run_archivebox_cmd(["init"])
+    assert result.returncode == 0, result.stderr
 
-    assert result.returncode == 0
     db_path = tmp_path / "index.sqlite3"
-    assert db_path.exists()
     assert db_path.is_file()
-
-
-def test_init_creates_archive_directory(tmp_path):
-    """Test that init creates archive directory."""
-    run_archivebox_cmd(["init"])
-
-    archive_dir = tmp_path / "archive"
-    assert archive_dir.exists()
-    assert archive_dir.is_dir()
-
-
-def test_init_uses_cwd_archive_and_users_dirs(tmp_path):
-    """Test that init creates archive/users storage roots under cwd."""
-
-    result = run_archivebox_cmd(["init"])
-
-    assert result.returncode == 0
-    assert (tmp_path / "archive").is_dir()
-    assert (tmp_path / "archive" / "users").is_dir()
-
-
-def test_init_creates_sources_directory(tmp_path):
-    """Test that init creates sources directory."""
-    run_archivebox_cmd(["init"])
-
-    sources_dir = tmp_path / "sources"
-    assert sources_dir.exists()
-    assert sources_dir.is_dir()
-
-
-def test_init_creates_logs_directory(tmp_path):
-    """Test that init creates logs directory."""
-    run_archivebox_cmd(["init"])
-
-    logs_dir = tmp_path / "logs"
-    assert logs_dir.exists()
-    assert logs_dir.is_dir()
-
-
-def test_init_creates_config_file(tmp_path):
-    """Test that init creates ArchiveBox.conf config file."""
-    run_archivebox_cmd(["init"])
-
     config_file = tmp_path / "ArchiveBox.conf"
-    assert config_file.exists()
     assert config_file.is_file()
-
-
-def test_init_runs_migrations(tmp_path):
-    """Test that init runs Django migrations and creates core tables."""
-    run_archivebox_cmd(["init"])
+    for relative_path in ("archive", "archive/users", "sources", "logs"):
+        assert (tmp_path / relative_path).is_dir(), relative_path
 
     with use_archivebox_db(tmp_path):
-        migration_count = MigrationRecorder.Migration.objects.count()
-
-    assert migration_count > 0
-
-
-def test_init_creates_core_snapshot_table(tmp_path):
-    """Test that init creates core_snapshot table."""
-    run_archivebox_cmd(["init"])
-
-    assert Snapshot._meta.db_table == "core_snapshot"
-    with use_archivebox_db(tmp_path):
+        assert MigrationRecorder.Migration.objects.count() > 0
+        assert Snapshot._meta.db_table == "core_snapshot"
         assert Snapshot.objects.count() == 0
-
-
-def test_init_creates_crawls_crawl_table(tmp_path):
-    """Test that init creates crawls_crawl table."""
-    run_archivebox_cmd(["init"])
-
-    assert Crawl._meta.db_table == "crawls_crawl"
-    with use_archivebox_db(tmp_path):
+        assert Crawl._meta.db_table == "crawls_crawl"
         assert Crawl.objects.count() == 0
-
-
-def test_init_creates_core_archiveresult_table(tmp_path):
-    """Test that init creates core_archiveresult table."""
-    run_archivebox_cmd(["init"])
-
-    assert ArchiveResult._meta.db_table == "core_archiveresult"
-    with use_archivebox_db(tmp_path):
+        assert ArchiveResult._meta.db_table == "core_archiveresult"
         assert ArchiveResult.objects.count() == 0
+        assert Machine._meta.db_table == "machine_machine"
+        Machine.objects.count()
 
-
-def test_init_sets_correct_file_permissions(tmp_path):
-    """Test that init sets correct permissions on created files."""
-    run_archivebox_cmd(["init"])
-
-    # Check database permissions
-    db_path = tmp_path / "index.sqlite3"
     assert oct(db_path.stat().st_mode)[-3:] in (get_config().OUTPUT_PERMISSIONS, DIR_PERMISSIONS)
-
-    # Check directory permissions
-    archive_dir = tmp_path / "archive"
-    assert oct(archive_dir.stat().st_mode)[-3:] in (get_config().OUTPUT_PERMISSIONS, DIR_PERMISSIONS)
+    assert oct((tmp_path / "archive").stat().st_mode)[-3:] in (get_config().OUTPUT_PERMISSIONS, DIR_PERMISSIONS)
+    output = result.stdout
+    assert "ArchiveBox" in output or "collection" in output.lower() or "Initializing" in output
 
 
 def test_init_creates_database_with_archivebox_permissions_despite_permissive_umask(tmp_path):
@@ -325,24 +248,6 @@ def test_init_quick_flag_skips_checks(tmp_path):
     assert db_path.exists()
 
 
-def test_init_creates_machine_table(tmp_path):
-    """Test that init creates the machine_machine table."""
-    run_archivebox_cmd(["init"])
-
-    assert Machine._meta.db_table == "machine_machine"
-    with use_archivebox_db(tmp_path):
-        Machine.objects.count()
-
-
-def test_init_output_shows_collection_info(tmp_path):
-    """Test that init output shows helpful collection information."""
-    result = run_archivebox_cmd(["init"])
-
-    output = result.stdout
-    # Should show some helpful info about the collection
-    assert "ArchiveBox" in output or "collection" in output.lower() or "Initializing" in output
-
-
 def test_init_ignores_unrecognized_archive_directories(initialized_archive):
     """Test that init upgrades existing dirs without choking on extra folders."""
     env = cli_env(disable_extractors=True)

archivebox/tests/test_cli_install.py

@@ -22,52 +22,14 @@ from archivebox.tests.test_orm_helpers import use_archivebox_db
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
-def test_install_runs_successfully(initialized_archive):
-    """Test that install command runs without error."""
-    result = run_archivebox_cmd(
-        ["install", "--dry-run"],
-        timeout=60,
-    )
-
-    assert result.returncode == 0, result.stderr or result.stdout
-    assert "Dry run - would detect ArchiveBox dependencies" in result.stdout
-
-
-def test_install_creates_binary_records_in_db(initialized_archive):
-    """Test that install --dry-run does not create Binary records in database."""
-
-    result = run_archivebox_cmd(
-        ["install", "--dry-run"],
-        timeout=60,
-    )
-    assert result.returncode == 0, result.stderr or result.stdout
-
-    with use_archivebox_db(initialized_archive):
-        assert Binary.objects.count() == 0
-
-
-def test_install_dry_run_does_not_install(initialized_archive):
-    """Test that --dry-run doesn't actually install anything."""
-
-    result = run_archivebox_cmd(
-        ["install", "--dry-run"],
-        timeout=60,
-    )
-
+def test_install_dry_run_reports_plan_without_creating_binary_records(initialized_archive):
+    """Inspect one dry run; real installs and empty-dir bootstrap stay separate."""
+    result = run_archivebox_cmd(["install", "--dry-run"], timeout=60)
     assert result.returncode == 0, result.stderr or result.stdout
     assert result.stdout.strip() == "Dry run - would detect ArchiveBox dependencies and run the abx-dl install flow"
-
-
-def test_install_detects_system_binaries(initialized_archive):
-    """Test that install detects existing system binaries."""
-
-    result = run_archivebox_cmd(
-        ["install", "--dry-run"],
-        timeout=60,
-    )
-
-    assert result.returncode == 0, result.stderr or result.stdout
-    assert "ArchiveBox dependencies" in result.stdout
+    assert "dry run" in result.stdout.lower()
+    with use_archivebox_db(initialized_archive):
+        assert Binary.objects.count() == 0
 
 
 def test_install_target_resolver_uses_declared_binary_aliases():
@@ -95,29 +57,6 @@ def test_default_dependency_install_memory_preflight_fails_before_installers(cap
     assert "No plugin dependency installers were started" in output
 
 
-def test_install_shows_binary_status(initialized_archive):
-    """Test that install shows status of binaries."""
-
-    result = run_archivebox_cmd(
-        ["install", "--dry-run"],
-        timeout=60,
-    )
-
-    assert result.returncode == 0, result.stderr or result.stdout
-    assert result.stdout.strip() == "Dry run - would detect ArchiveBox dependencies and run the abx-dl install flow"
-
-
-def test_install_dry_run_prints_dry_run_message(initialized_archive):
-    """Test that install --dry-run clearly reports that no changes will be made."""
-    result = run_archivebox_cmd(
-        ["install", "--dry-run"],
-        timeout=60,
-    )
-
-    assert result.returncode == 0
-    assert "dry run" in result.stdout.lower()
-
-
 def test_install_help_lists_dry_run_flag(tmp_path):
     """Test that install --help documents the dry-run option."""
     result = run_archivebox_cmd(

archivebox/tests/test_cli_list.py

@@ -26,9 +26,7 @@ def test_static_export_creates_detail_page_for_unarchived_snapshot(snapshot):
     from archivebox.config import CONSTANTS
 
     snapshot_dir = Path(snapshot.output_dir)
-    assert snapshot_dir.is_dir()
-    assert not any(snapshot_dir.iterdir())
-    snapshot_dir.rmdir()
+    # Queued metadata has no output directory until an explicit writer runs.
     assert not snapshot_dir.exists()
 
     html = Snapshot.objects.filter(pk=snapshot.pk).to_html(with_headers=True)

archivebox/tests/test_cli_persona.py

@@ -50,12 +50,15 @@ def test_persona_import_real_browser_session(initialized_archive, tmp_path, http
     """Import an actual Chromium-created session and archive its private page."""
     import hashlib
     import json
+    import shlex
     import subprocess
+    from pathlib import Path
 
     from werkzeug.wrappers import Response
     from archivebox.tests.conftest import resolve_abxpkg_chrome_env
 
     browser_env = resolve_abxpkg_chrome_env(tmp_path / "lib")
+    browser_env["ABXPKG_LIB_DIR"] = str(tmp_path / "lib")
     source = tmp_path / "browser"
     httpserver.expect_request("/login").respond_with_data(
         '<html><title>Signed in</title><script>localStorage.setItem("persona-preference", "forest-green")</script>Signed in</html>',
@@ -90,6 +93,7 @@ const puppeteer = require(process.argv[1]);
         const page = await browser.newPage();
         await page.goto(process.argv[4]);
         console.log(await page.title());
+        console.log(await page.evaluate(() => localStorage.getItem('persona-preference')));
     } finally { await browser.close(); }
 })().catch(error => { console.error(error); process.exit(1); });
 """,
@@ -104,6 +108,7 @@ const puppeteer = require(process.argv[1]);
     )
     assert seeded.returncode == 0, seeded.stderr
     assert "Signed in" in seeded.stdout
+    assert "forest-green" in seeded.stdout
     preferences = source / "Profile 2" / "Preferences"
     original_preferences = preferences.read_bytes()
 
@@ -161,6 +166,11 @@ const puppeteer = require(process.argv[1]);
         env={**browser_env, "CHROME_SANDBOX": "false"},
     )
     assert archived.returncode == 0, archived.stderr
+    launch_commands = list((initialized_archive / "archive" / "users").glob("*/crawls/**/chrome/cmd.sh"))
+    assert len(launch_commands) == 1
+    launched_browser = Path(shlex.split(launch_commands[0].read_text().splitlines()[1])[0])
+    # abxpkg's environment and provider projections can name the same binary.
+    assert launched_browser.samefile(browser_env["CHROME_BINARY"])
     outputs = list((initialized_archive / "archive" / "users").glob("*/snapshots/**/dom/output.html"))
     assert len(outputs) == 1
     html = outputs[0].read_text()

archivebox/tests/test_cli_remove.py

@@ -29,8 +29,8 @@ print(json.dumps([
     return json.loads(result.stdout.strip().splitlines()[-1])
 
 
-def test_remove_deletes_snapshot_from_db(initialized_archive):
-    """Test that remove command deletes snapshot from database."""
+def test_remove_yes_deletes_database_row_and_directory_without_confirmation(initialized_archive):
+    """Verify the DB, filesystem, exit code and summary of one confirmed removal."""
     env = cli_env(disable_extractors=True)
 
     # Add a snapshot
@@ -46,62 +46,17 @@ def test_remove_deletes_snapshot_from_db(initialized_archive):
     snapshot_dir = find_snapshot_dir(initialized_archive, snapshot_id)
     assert snapshot_dir is not None, f"Snapshot output directory not found for {snapshot_id}"
 
-    # Remove it
-    run_archivebox_cmd(
-        ["remove", "https://example.com", "--yes"],
-        env=env,
-    )
-
-    assert len(_snapshot_rows(initialized_archive, env)) == 0
-    assert not snapshot_dir.exists()
-
-
-def test_remove_deletes_archive_directory(initialized_archive):
-    """Test that remove --yes removes the current snapshot output directory."""
-    env = cli_env(disable_extractors=True)
-
-    # Add a snapshot
-    run_archivebox_cmd(
-        ["add", "--index-only", "--depth=0", "https://example.com"],
-        env=env,
-    )
-    run_queued_crawls(initialized_archive, env)
-
-    rows = _snapshot_rows(initialized_archive, env)
-    assert len(rows) == 1
-    snapshot_id = rows[0]["id"]
-
-    snapshot_dir = find_snapshot_dir(initialized_archive, snapshot_id)
-    assert snapshot_dir is not None, f"Snapshot output directory not found for {snapshot_id}"
-
-    run_archivebox_cmd(
-        ["remove", "https://example.com", "--yes"],
-        env=env,
-    )
-
-    assert not snapshot_dir.exists()
-
-
-def test_remove_yes_flag_skips_confirmation(initialized_archive):
-    """Test that --yes flag skips confirmation prompt."""
-    env = cli_env(disable_extractors=True)
-
-    run_archivebox_cmd(
-        ["add", "--index-only", "--depth=0", "https://example.com"],
-        env=env,
-    )
-    run_queued_crawls(initialized_archive, env)
-
-    # Remove with --yes should complete without interaction
+    # Remove with --yes must finish without interaction.
     result = run_archivebox_cmd(
         ["remove", "https://example.com", "--yes"],
         env=env,
         timeout=30,
     )
-
     assert result.returncode == 0
-    output = result.stdout + result.stderr
-    assert "Index now contains 0 links." in output
+    assert "Index now contains 0 links." in result.stdout + result.stderr
+
+    assert len(_snapshot_rows(initialized_archive, env)) == 0
+    assert not snapshot_dir.exists()
 
 
 def test_remove_without_yes_prompts_and_keeps_snapshot(initialized_archive):

archivebox/tests/test_cli_server.py

@@ -177,70 +177,41 @@ def test_sqlite_connections_use_explicit_busy_timeout():
     assert "PRAGMA journal_mode = WAL;" in SQLITE_CONNECTION_OPTIONS["OPTIONS"]["init_command"]
 
 
-def test_docker_sqlite_never_uses_wal_on_host_shared_collection(tmp_path):
-    """Docker collections must not expose a WAL database through /data.
+def test_sqlite_add_commits_while_read_cursor_is_open(tmp_path):
+    """A real read snapshot must not hold up a new public CLI submission."""
+    import sqlite3
 
-    Docker Desktop/OrbStack bind mounts cross the Linux VM/host locking
-    boundary. A container WAL writer plus a host-side sqlite reader can make
-    the writer SIGBUS and leave index.sqlite3 malformed; the reader does not
-    need to write. Run a fresh real settings process with Docker identity so a
-    host test cannot accidentally pass by reusing this process's non-Docker
-    config imports.
-    """
-    env = os.environ.copy()
-    env["IN_DOCKER"] = "True"
-    env["DJANGO_SETTINGS_MODULE"] = "archivebox.core.settings"
-    result = subprocess.run(
-        [
-            sys.executable,
-            "-c",
-            (
-                "import django;"
-                "django.setup();"
-                "from django.db import connection;"
-                "cursor=connection.cursor();"
-                "cursor.execute('CREATE TABLE journal_probe (value INTEGER)');"
-                "cursor.execute('INSERT INTO journal_probe VALUES (1)');"
-                "cursor.execute('PRAGMA journal_mode');"
-                "print(cursor.fetchone()[0]);"
-                "cursor.close();"
-                "connection.close()"
-            ),
-        ],
-        cwd=tmp_path,
-        env=env,
-        capture_output=True,
-        text=True,
-        check=True,
-    )
+    env = {"SQLITE_JOURNAL_MODE": "WAL"}
+    initialized = run_archivebox_cmd(["init", "--quick"], cwd=tmp_path, env=env, timeout=90)
+    assert initialized.returncode == 0, initialized.stderr or initialized.stdout
+    for url in ("https://example.com/read-one", "https://example.com/read-two"):
+        initial = run_archivebox_cmd(["add", "--index-only", url], cwd=tmp_path, env=env, timeout=90)
+        assert initial.returncode == 0, initial.stderr or initial.stdout
 
-    assert result.stdout.strip().lower() == "delete"
-    assert not tmp_path.joinpath("index.sqlite3-wal").exists()
-    assert not tmp_path.joinpath("index.sqlite3-shm").exists()
-
-
-def test_docker_rejects_explicit_wal_override(tmp_path):
-    """An old config or environment override must not bypass the invariant."""
-    env = os.environ.copy()
-    env["IN_DOCKER"] = "True"
-    result = subprocess.run(
-        [
-            sys.executable,
-            "-c",
-            ("from archivebox.config.common import DatabaseConfig;DatabaseConfig(SQLITE_JOURNAL_MODE='WAL')"),
-        ],
-        cwd=tmp_path,
-        env=env,
-        capture_output=True,
-        text=True,
-    )
-
-    assert result.returncode != 0
-    assert "WAL is unsafe for Docker collections" in result.stderr
+    reader = sqlite3.connect(f"file:{tmp_path / 'index.sqlite3'}?mode=ro", uri=True)
+    try:
+        assert reader.execute("PRAGMA journal_mode").fetchone()[0] == "wal"
+        cursor = reader.execute("SELECT urls FROM crawls_crawl ORDER BY urls")
+        first = cursor.fetchone()
+        assert first == ("https://example.com/read-one",)
+        # Do not exhaust/close the actual cursor until add commits. This is the
+        # contention that rollback journaling turns into seconds of waiting.
+        added = run_archivebox_cmd(
+            ["add", "--index-only", "https://example.com/write-during-read"],
+            cwd=tmp_path,
+            env=env,
+            timeout=30,
+        )
+        assert added.returncode == 0, added.stderr or added.stdout
+        assert cursor.fetchall() == [("https://example.com/read-two",)]
+        cursor.close()
+        assert reader.execute("SELECT urls FROM crawls_crawl WHERE urls = ?", ("https://example.com/write-during-read",)).fetchone()
+    finally:
+        reader.close()
 
 
 def test_docker_postgres_ignores_irrelevant_sqlite_wal_override(tmp_path):
-    """The Docker SQLite safety invariant must not reject PostgreSQL.
+    """SQLite filesystem constraints must not reject PostgreSQL.
 
     Operators can switch an existing deployment to PostgreSQL while an old
     SQLITE_JOURNAL_MODE setting remains in ArchiveBox.conf or the environment.
@@ -448,7 +419,10 @@ def test_port_compatibility_does_not_add_listeners_outside_docker_defaults(tmp_p
         check=True,
     )
     workers = json.loads(result.stdout)
-    assert [worker["name"] for worker in workers] == ["worker_runserver" if debug else "worker_daphne", "worker_runner"]
+    assert [worker["name"] for worker in workers] == [
+        "worker_runserver" if debug else "worker_daphne",
+        "worker_runner",
+    ]
     command = shlex.split(workers[0]["command"])
     assert "--endpoint" not in command
     assert (f"127.0.0.1:{port}" if debug else f"--port={port}") in command

archivebox/tests/test_cli_snapshot.py

@@ -20,6 +20,10 @@ from archivebox.tests.conftest import (
     create_test_url,
     parse_jsonl_output,
     run_archivebox_cmd,
+    start_archivebox_server,
+    stop_archivebox_process,
+    get_free_port,
+    wait_for_snapshot_capture,
 )
 from archivebox.tests.test_orm_helpers import use_archivebox_db
 
@@ -380,19 +384,30 @@ class TestSnapshotDelete:
         assert "Would delete" in stderr
 
 
-def test_snapshot_creates_snapshot_with_correct_url(tmp_path, initialized_archive):
+def test_snapshot_creates_snapshot_with_correct_url(tmp_path, initialized_archive, recursive_test_site):
     """Test that snapshot stores the exact URL in the database."""
-    env = cli_env(disable_extractors=True)
+    port = get_free_port()
+    env = cli_env(port=port, server=True, PLUGINS="wget")
+    url = recursive_test_site["root_url"]
 
-    run_archivebox_cmd(
-        ["snapshot", "create", "https://example.com"],
+    result = run_archivebox_cmd(
+        ["snapshot", "create", url],
         cwd=tmp_path,
         env=env,
     )
+    assert result.returncode == 0, result.stderr
 
     with use_archivebox_db(tmp_path):
-        snapshot = Snapshot.objects.select_related("crawl__created_by").get(url="https://example.com")
+        snapshot = Snapshot.objects.select_related("crawl__created_by").get(url=url)
         username = snapshot.crawl.created_by.username
+        assert snapshot.status == Snapshot.StatusChoices.QUEUED
+        assert not snapshot.output_dir.exists()
+
+    server = start_archivebox_server(tmp_path, env=env, port=port, log_name="snapshot-create-server.log")
+    try:
+        assert "Root" in wait_for_snapshot_capture(tmp_path, url, timeout=120)
+    finally:
+        stop_archivebox_process(server)
 
     # Verify the crawl tree contains a relative symlink to the user-scoped snapshot output.
     snapshots_root = tmp_path / "archive" / "users" / username / "snapshots"

archivebox/tests/test_cli_status.py

@@ -24,21 +24,17 @@ def _create_snapshot_rows(initialized_archive, env, *urls):
     return result
 
 
-def test_status_runs_successfully(initialized_archive):
-    """Test that status command runs without error."""
+def test_status_reports_empty_collection_size_user_index_and_path(initialized_archive):
+    """All empty-collection fields come from the same real status response."""
     result = run_archivebox_cmd(["status"], cwd=initialized_archive)
-
     assert result.returncode == 0
-    assert len(result.stdout) > 100
-
-
-def test_status_shows_zero_snapshots_in_empty_archive(initialized_archive):
-    """Test status shows 0 snapshots in empty archive."""
-    result = run_archivebox_cmd(["status"], cwd=initialized_archive)
-
     output = result.stdout
-    # Should indicate empty/zero state
+    assert len(output) > 100
     assert "0" in output
+    assert "Size" in output or "size" in output
+    assert "user" in output.lower() or "login" in output.lower()
+    assert "index" in output.lower() or "Index" in output
+    assert "archive" in output.lower() or str(initialized_archive) in output
 
 
 def test_status_shows_correct_snapshot_count(initialized_archive):
@@ -58,36 +54,16 @@ def test_status_shows_correct_snapshot_count(initialized_archive):
     assert "3" in result.stdout
 
 
-def test_status_shows_archived_count(initialized_archive):
-    """Test status distinguishes archived vs unarchived snapshots."""
+def test_status_reports_database_snapshot_count_and_archive_categories(initialized_archive):
+    """One uncaptured snapshot exercises the DB count and filesystem categories."""
     env = cli_env(disable_extractors=True)
-
     _create_snapshot_rows(initialized_archive, env, "https://example.com")
-
+    with use_archivebox_db(initialized_archive):
+        assert Snapshot.objects.count() == 1
     result = run_archivebox_cmd(["status"], cwd=initialized_archive)
-
-    # Should show archived/unarchived categories
+    assert result.returncode == 0, result.stderr
+    assert "1" in result.stdout
     assert "archived" in result.stdout.lower() or "queued" in result.stdout.lower()
-
-
-def test_status_shows_archive_directory_size(initialized_archive):
-    """Test status reports archive directory size."""
-    result = run_archivebox_cmd(["status"], cwd=initialized_archive)
-
-    output = result.stdout
-    # Should show size info
-    assert "Size" in output or "size" in output
-
-
-def test_status_counts_archive_directories(initialized_archive):
-    """Test status counts directories in archive/ folder."""
-    env = cli_env(disable_extractors=True)
-
-    _create_snapshot_rows(initialized_archive, env, "https://example.com")
-
-    result = run_archivebox_cmd(["status"], cwd=initialized_archive)
-
-    # Should show directory count
     assert "present" in result.stdout.lower() or "directories" in result.stdout
 
 
@@ -107,22 +83,22 @@ def test_status_detects_orphaned_directories(initialized_archive):
     assert "archivebox update" in result.stdout
 
 
-def test_status_counts_new_snapshot_output_dirs_as_archived(initialized_archive):
+def test_status_counts_new_snapshot_output_dirs_as_archived(initialized_archive, recursive_test_site):
     """Test status reads archived/present counts from the current snapshot output layout."""
     env = cli_env(disable_extractors=True)
     env = env.copy()
     env["ARCHIVEBOX_ALLOW_NO_UNIX_SOCKETS"] = "true"
 
-    _create_snapshot_rows(initialized_archive, env, "https://example.com")
+    url = recursive_test_site["root_url"]
+    captured = run_archivebox_cmd(["add", "--plugins=wget", url], cwd=initialized_archive, env=env, timeout=120)
+    assert captured.returncode == 0, captured.stdout + captured.stderr
 
     with use_archivebox_db(initialized_archive):
-        snapshot_id = Snapshot.objects.values_list("id", flat=True).get(url="https://example.com")
+        snapshot_id = Snapshot.objects.values_list("id", flat=True).get(url=url)
 
     snapshot_dir = find_snapshot_dir(initialized_archive, str(snapshot_id))
     assert snapshot_dir is not None, f"Snapshot output directory not found for {snapshot_id}"
-    title_dir = snapshot_dir / "title"
-    title_dir.mkdir(parents=True, exist_ok=True)
-    (title_dir / "title.txt").write_text("Example Domain")
+    assert any("Root" in path.read_text(errors="ignore") for path in (snapshot_dir / "wget").rglob("*.html"))
 
     result = run_archivebox_cmd(["status"], cwd=initialized_archive, env=env)
 
@@ -131,40 +107,6 @@ def test_status_counts_new_snapshot_output_dirs_as_archived(initialized_archive)
     assert "present: 1" in result.stdout
 
 
-def test_status_shows_user_info(initialized_archive):
-    """Test status shows user/login information."""
-    result = run_archivebox_cmd(["status"], cwd=initialized_archive)
-
-    output = result.stdout
-    # Should show user section
-    assert "user" in output.lower() or "login" in output.lower()
-
-
-def test_status_reads_from_db_not_filesystem(initialized_archive):
-    """Test that status uses DB as source of truth, not filesystem."""
-    env = cli_env(disable_extractors=True)
-
-    _create_snapshot_rows(initialized_archive, env, "https://example.com")
-
-    # Verify DB has snapshot
-    with use_archivebox_db(initialized_archive):
-        db_count = Snapshot.objects.count()
-
-    assert db_count == 1
-
-    # Status should reflect DB count
-    result = run_archivebox_cmd(["status"], cwd=initialized_archive)
-    assert "1" in result.stdout
-
-
-def test_status_shows_index_file_info(initialized_archive):
-    """Test status shows index file information."""
-    result = run_archivebox_cmd(["status"], cwd=initialized_archive)
-
-    # Should mention index
-    assert "index" in result.stdout.lower() or "Index" in result.stdout
-
-
 def test_status_help_lists_available_options(initialized_archive):
     """Test that status --help works and documents the command."""
     result = run_archivebox_cmd(
@@ -174,10 +116,3 @@ def test_status_help_lists_available_options(initialized_archive):
 
     assert result.returncode == 0
     assert "status" in result.stdout.lower() or "statistic" in result.stdout.lower()
-
-
-def test_status_shows_data_directory_path(initialized_archive):
-    """Test that status reports which collection directory it is inspecting."""
-    result = run_archivebox_cmd(["status"], cwd=initialized_archive)
-
-    assert "archive" in result.stdout.lower() or str(initialized_archive) in result.stdout

archivebox/tests/test_cli_update.py

@@ -19,6 +19,50 @@ from archivebox.tests.test_orm_helpers import use_archivebox_db
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
+def test_rescan_recovers_already_migrated_idless_result(initialized_archive, tmp_path):
+    """Repair collections already migrated by versions affected by #1899."""
+    env = cli_env(disable_extractors=True)
+    run_archivebox_cmd(["add", "--index-only", "https://example.com/legacy-result"], env=env, check=True)
+    run_queued_crawls(initialized_archive, env)
+    with use_archivebox_db(initialized_archive):
+        snapshot = Snapshot.objects.get(url="https://example.com/legacy-result")
+        snapshot.write_index_jsonl()
+        output_dir = snapshot.output_dir
+        snapshot_id = snapshot.pk
+    payload = b"<html>Recovered after a previous migration</html>"
+    (output_dir / "singlefile.html").write_bytes(payload)
+    with (output_dir / "index.jsonl").open("a") as index:
+        index.write(
+            json.dumps(
+                {
+                    "type": "ArchiveResult",
+                    "snapshot_id": str(snapshot_id),
+                    "plugin": "singlefile",
+                    "hook_name": "",
+                    "status": "succeeded",
+                    "output_str": "singlefile.html",
+                    "start_ts": "2021-03-07T23:56:16-08:00",
+                },
+            )
+            + "\n",
+        )
+    result_id = None
+    for expected in ("Repaired: 1", "Unchanged: 1"):
+        result = run_archivebox_cmd(["update", "--rescan", "--migrate-only"], env=env, timeout=60)
+        assert result.returncode == 0, result.stdout + result.stderr
+        assert expected in result.stdout
+        with use_archivebox_db(initialized_archive):
+            archived = ArchiveResult.objects.get(snapshot_id=snapshot_id, plugin="singlefile")
+            assert archived.status == "succeeded"
+            assert archived.output_size == len(payload)
+            assert archived.output_files
+            assert archived.snapshot.output_size == len(payload)
+            if result_id is not None:
+                assert archived.pk == result_id
+            result_id = archived.pk
+        assert (output_dir / "singlefile.html").read_bytes() == payload
+
+
 def test_rescan_imports_current_layout_and_is_idempotent(initialized_archive, tmp_path):
     env = cli_env(disable_extractors=True)
     run_archivebox_cmd(["add", "--index-only", "https://example.com/rescan"], cwd=initialized_archive, env=env, check=True)
@@ -284,3 +328,39 @@ def test_update_seals_migrated_snapshots(initialized_archive):
         status = Snapshot.objects.values_list("status", flat=True).get()
 
     assert status == "sealed"
+
+
+@pytest.mark.parametrize("captured", [False, True])
+def test_index_only_distinguishes_uncaptured_metadata_from_missing_archive(initialized_archive, recursive_test_site, captured):
+    url = recursive_test_site["root_url"]
+    env = cli_env(disable_extractors=True)
+    if captured:
+        result = run_archivebox_cmd(["add", "--plugins=wget", url], cwd=initialized_archive, env=env, timeout=120)
+        assert result.returncode == 0, result.stdout + result.stderr
+    else:
+        created = run_archivebox_cmd(["snapshot", "create", url], cwd=initialized_archive, env=env, check=True)
+        sealed = run_archivebox_cmd(
+            ["snapshot", "update", "--status=sealed"],
+            cwd=initialized_archive,
+            env=env,
+            input=created.stdout,
+        )
+        assert sealed.returncode == 0, sealed.stdout + sealed.stderr
+    with use_archivebox_db(initialized_archive):
+        snapshot = Snapshot.objects.get(url=url)
+        directory = snapshot.output_dir
+        assert snapshot.status == Snapshot.StatusChoices.SEALED
+        if captured:
+            assert snapshot.downloaded_at is not None
+            assert snapshot.archiveresult_set.filter(plugin="wget", status="succeeded").exists()
+            directory.rename(directory.with_name(directory.name + ".removed"))
+        else:
+            assert snapshot.downloaded_at is None
+            assert not snapshot.archiveresult_set.exists()
+        assert not directory.exists()
+    result = run_archivebox_cmd(["update", "--index-only"], cwd=initialized_archive, env=env, timeout=120)
+    if captured:
+        assert result.returncode != 0
+        assert "Missing snapshot directory" in result.stdout + result.stderr
+    else:
+        assert result.returncode == 0, result.stdout + result.stderr

archivebox/tests/test_cli_update_reindex_snapshots.py

@@ -15,6 +15,49 @@ from archivebox.tests.test_orm_helpers import use_archivebox_db
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
+@pytest.mark.parametrize("index_format", ["history", "archive_results"])
+def test_rescan_imports_legacy_result_rows_and_saved_content(tmp_path, initialized_archive, index_format):
+    """Issue #1899: orphan metadata must become visible results on the first import."""
+    env = cli_env(disable_extractors=True)
+    legacy_dir = tmp_path / "archive" / "1615190176"
+    legacy_dir.mkdir(parents=True)
+    payload = b"<html><title>Recovered capture</title><body>saved content</body></html>"
+    (legacy_dir / "singlefile.html").write_bytes(payload)
+    entry = {
+        "plugin": "singlefile",
+        "status": "succeeded",
+        "output": "singlefile.html",
+        "start_ts": "2021-03-07T23:56:16-08:00",
+        "end_ts": "2021-03-07T23:56:16-08:00",
+    }
+    metadata = {
+        "url": "https://example.com/recovered",
+        "timestamp": legacy_dir.name,
+        "title": "Recovered capture",
+        index_format: {"singlefile": [entry]} if index_format == "history" else [entry],
+    }
+    original_json = json.dumps(metadata)
+    (legacy_dir / "index.json").write_text(original_json)
+    result_id = None
+    for _ in range(2):
+        command = run_archivebox_cmd(["update", "--rescan", "--migrate-only"], env=env, timeout=60)
+        assert command.returncode == 0, command.stdout + command.stderr
+        with use_archivebox_db(tmp_path):
+            snapshot = Snapshot.objects.get(url=metadata["url"])
+            result = snapshot.archiveresult_set.get(plugin="singlefile")
+            assert result.status == "succeeded"
+            assert result.output_size == len(payload)
+            assert result.output_files
+            assert snapshot.status == "sealed"
+            assert snapshot.archiveresult_set.count() == 1
+            assert snapshot.output_size == len(payload)
+            assert (snapshot.output_dir / "singlefile.html").read_bytes() == payload
+            assert (snapshot.output_dir / "index.json").read_text() == original_json
+            if result_id is not None:
+                assert result.pk == result_id
+            result_id = result.pk
+
+
 def test_update_imports_orphaned_snapshots(tmp_path, initialized_archive):
     """Test that archivebox update imports real legacy archive directories."""
     env = cli_env(disable_extractors=True)

archivebox/tests/test_cli_version.py

@@ -18,6 +18,50 @@ from pathlib import Path
 from archivebox.config.paths import tmp_dir_socket_path_is_short_enough
 from archivebox.cli.archivebox_version import _binary_row_dedupe_key
 from archivebox.tests.conftest import cli_env, run_archivebox_cmd
+from abx_plugins.plugins.opencode.archivebox.host_cases import (
+    test_version_names_disabling_config_in_path_column as test_version_names_disabling_config_in_path_column,
+    test_version_shared_git_stays_installed_when_opencode_disabled as test_version_shared_git_stays_installed_when_opencode_disabled,
+)
+
+
+@pytest.mark.parametrize("no_cache", [False, True])
+def test_version_reuses_validated_provider_cache(tmp_path, no_cache):
+    """A warm diagnostic must validate the install without hashing it again."""
+    env = cli_env(ABXPKG_LIB_DIR=str(tmp_path / "lib"), COLUMNS="500")
+    first = run_archivebox_cmd(["version", "--binaries=git"], cwd=tmp_path, env=env)
+    assert first.returncode == 0, first.stdout + first.stderr
+    assert "✅" in first.stdout
+
+    # Observe the real CLI and its resolver threads; no replacement providers or
+    # timing thresholds. This distinguishes validated cache reads from a full
+    # rehash even on a fast machine with a small git executable.
+    script = """
+import runpy, sys, threading
+calls = []
+def observe(frame, event, arg):
+    if event == 'call' and frame.f_code.co_name == 'get_sha256':
+        calls.append(frame.f_code.co_filename)
+sys.setprofile(observe)
+threading.setprofile(observe)
+sys.argv = ['archivebox', 'version', '--binaries=git', *sys.argv[1:]]
+try:
+    runpy.run_module('archivebox', run_name='__main__')
+finally:
+    sys.setprofile(None)
+    threading.setprofile(None)
+    print('BINARY_HASH_CALLS=' + str(len(calls)))
+"""
+    second = subprocess.run(
+        ["uv", "run", "--active", "python", "-c", script, *(["--no-cache"] if no_cache else [])],
+        cwd=tmp_path,
+        env={**os.environ, **env},
+        capture_output=True,
+        text=True,
+        timeout=60,
+    )
+    assert second.returncode == 0, second.stdout + second.stderr
+    assert "✅" in second.stdout
+    assert f"BINARY_HASH_CALLS={int(no_cache)}" in second.stdout
 
 
 @pytest.mark.parametrize("width", [80, 160])
@@ -243,30 +287,6 @@ def test_version_includes_disabled_plugin_dependencies(tmp_path):
     assert "Failed to detect the following binaries" not in output
 
 
-def test_version_names_disabling_config_in_path_column(tmp_path):
-    env = cli_env(OPENCODE_ENABLED="False", COLUMNS="200")
-    result = run_archivebox_cmd(["version", "--binaries=opencode"], cwd=tmp_path, env=env)
-    assert result.returncode == 0, result.stdout + result.stderr
-    rows = [re.split(r"\s{2,}", line.strip()) for line in result.stdout.splitlines()]
-    opencode_rows = [row for row in rows if len(row) == 6 and row[1] == "opencode"]
-    assert len(opencode_rows) == 4, result.stdout
-    assert all(row[2] == "disabled" and row[5] == "disabled by OPENCODE_ENABLED=False" for row in opencode_rows)
-    assert "Disabled plugins are dimmed" not in result.stdout
-
-
-def test_version_shared_git_stays_installed_when_opencode_disabled(tmp_path):
-    env = cli_env(PLUGINS="git", GIT_ENABLED="True", OPENCODE_ENABLED="False", COLUMNS="300")
-    result = run_archivebox_cmd(["version", "--binaries=git"], cwd=tmp_path, env=env)
-    assert result.returncode == 0, result.stdout + result.stderr
-    rows = [re.split(r"\s{2,}", line.strip()) for line in result.stdout.splitlines()]
-    git_rows = [row for row in rows if len(row) == 6 and row[0] == "git"]
-    assert len(git_rows) == 1, result.stdout
-    assert set(git_rows[0][1].split(", ")) == {"git", "opencode"}
-    assert git_rows[0][2] == "✅"
-    assert Path(git_rows[0][5]).is_file()
-    assert "disabled by OPENCODE_ENABLED=False" not in result.stdout
-
-
 def test_version_honors_legacy_save_aliases_when_disabling_extractors(tmp_path):
     """Legacy SAVE_* aliases should disable their canonical plugin flags."""
     data_dir = tmp_path / "legacy-save-disabled"

archivebox/tests/test_config_DELETE_AFTER.py

@@ -159,7 +159,11 @@ print(json.dumps({{
     assert not Path(retained["archiveresult_dir"]).exists()
 
 
-def test_delete_after_real_add_page_and_rest_create_paths(client):
+def test_delete_after_real_add_page_and_rest_create_paths(client, recursive_test_site):
+    from archivebox.personas.models import Persona
+
+    # Match init/server startup; visiting /add/ must not create persona records.
+    Persona.get_or_create_default()
     User = get_user_model()
     admin_user = User.objects.create_superuser(
         username="retentionadmin",
@@ -168,6 +172,7 @@ def test_delete_after_real_add_page_and_rest_create_paths(client):
     )
 
     client.force_login(admin_user)
+    ui_url, rest_url = recursive_test_site["child_urls"][:2]
     response = client.get(reverse("add"), HTTP_HOST=ADMIN_HOST)
     assert response.status_code == 200
     assert 'name="delete_after"' in response.content.decode()
@@ -175,7 +180,7 @@ def test_delete_after_real_add_page_and_rest_create_paths(client):
     response = client.post(
         reverse("add"),
         data={
-            "url": "https://example.com/delete-after-ui",
+            "url": ui_url,
             "tag": "retention-ui",
             "depth": "0",
             "max_urls": "1",
@@ -199,13 +204,13 @@ def test_delete_after_real_add_page_and_rest_create_paths(client):
 
     from archivebox.crawls.models import Crawl
 
-    ui_crawl = Crawl.objects.get(urls__contains="https://example.com/delete-after-ui")
+    ui_crawl = Crawl.objects.get(urls=ui_url)
     assert ui_crawl.config["DELETE_AFTER"] == "2h"
     assert ui_crawl.delete_at is not None
     from archivebox.services.runner import run_due_crawl
 
     assert run_due_crawl(ui_crawl, lock_seconds=10)
-    ui_snapshot = ui_crawl.snapshot_set.get(url="https://example.com/delete-after-ui")
+    ui_snapshot = ui_crawl.snapshot_set.get(url=ui_url)
     assert ui_snapshot.delete_at is not None
     assert not ui_snapshot.output_dir.joinpath("staticfile", "stdin.txt").exists()
 
@@ -217,10 +222,12 @@ def test_delete_after_real_add_page_and_rest_create_paths(client):
         "/api/v1/crawls/crawls",
         data=json.dumps(
             {
-                "urls": ["https://example.com/delete-after-rest"],
+                "urls": [rest_url],
                 "max_depth": 0,
                 "max_urls": 1,
-                "config": {"DELETE_AFTER": "3h"},
+                # Exercise retention through a real capture. The full plugin
+                # pipeline has its own recursive-crawl integration coverage.
+                "config": {"DELETE_AFTER": "3h", "PLUGINS": "wget,hashes"},
             },
         ),
         content_type="application/json",
@@ -228,9 +235,16 @@ def test_delete_after_real_add_page_and_rest_create_paths(client):
         HTTP_X_ARCHIVEBOX_API_KEY=api_token.token,
     )
     assert response.status_code == 200
-    rest_crawl = Crawl.objects.get(urls__contains="https://example.com/delete-after-rest")
+    rest_crawl = Crawl.objects.get(urls=rest_url)
     assert rest_crawl.config["DELETE_AFTER"] == "3h"
     assert rest_crawl.delete_at is not None
     assert run_due_crawl(rest_crawl, lock_seconds=10)
-    rest_snapshot = rest_crawl.snapshot_set.get(url="https://example.com/delete-after-rest")
+    rest_snapshot = rest_crawl.snapshot_set.get(url=rest_url)
     assert rest_snapshot.delete_at is not None
+    from archivebox.core.models import ArchiveResult
+
+    for plugin in ("wget", "hashes"):
+        result = rest_snapshot.archiveresult_set.get(plugin=plugin)
+        assert result.status == ArchiveResult.StatusChoices.SUCCEEDED
+        assert result.delete_at is not None
+        assert result.output_size > 0

archivebox/tests/test_config_MAX_limits.py

@@ -203,6 +203,7 @@ def test_seal_snapshot_cancels_queued_descendants_after_crawl_max_size():
 
     Snapshot.objects.filter(pk=root.pk).update(output_size=32)
 
+    root_output_dir = str(root.output_dir)
     bus = create_bus(name=f"test_snapshot_limit_cancel_{str(crawl.id).replace('-', '_')}")
     SnapshotService(bus, crawl_id=str(crawl.id))
     try:
@@ -212,14 +213,14 @@ def test_seal_snapshot_cancels_queued_descendants_after_crawl_max_size():
                 SnapshotEvent(
                     url=root.url,
                     snapshot_id=str(root.id),
-                    output_dir=str(root.output_dir),
+                    output_dir=root_output_dir,
                 ),
             )
             await snapshot_event.now()
             completed_event = SnapshotCompletedEvent(
                 url=root.url,
                 snapshot_id=str(root.id),
-                output_dir=str(root.output_dir),
+                output_dir=root_output_dir,
             )
             completed_event.event_parent_id = snapshot_event.event_id
             await bus.emit(completed_event).now()

archivebox/tests/test_config_ONLY_NEW.py

@@ -93,4 +93,6 @@ def test_create_discovered_snapshots_respects_only_new_exact_url_matches(admin_u
         crawl_link = (
             Path(crawl.output_dir) / CONSTANTS.SNAPSHOTS_DIR_NAME / snapshot.extract_domain_from_url(snapshot.url) / str(snapshot.id)
         )
-        assert crawl_link.is_symlink()
+        # Queued metadata stays in the DB; the normal runner creates links.
+        assert not crawl_link.is_symlink()
+        assert not snapshot.output_dir.exists()

archivebox/tests/test_core_config.py

@@ -1,4 +1,5 @@
 import os
+from pathlib import Path
 
 import pytest
 from pydantic import ValidationError
@@ -95,3 +96,40 @@ def test_resolving_scoped_config_does_not_mutate_process_environment(tmp_path):
             os.environ.pop("ABXPKG_LIB_DIR", None)
         else:
             os.environ["ABXPKG_LIB_DIR"] = previous_lib_dir
+
+
+@pytest.mark.parametrize("key", ["COOKIES_FILE", "AUTH_STORAGE_FILE", "WGET_COOKIES_FILE", "YTDLP_COOKIES_FILE", "GALLERYDL_COOKIES_FILE"])
+def test_cookie_paths_remain_collection_relative_after_runtime_overlays(key, tmp_path):
+    from archivebox.config.common import ArchiveBoxConfig
+
+    config = ArchiveBoxConfig(**{key: "auth/cookies.txt"})
+    assert str(config[key]) == str(CONSTANTS.DATA_DIR / "auth/cookies.txt")
+    runtime = config.for_crawl_runtime(runtime_overrides={key: "other/cookies.txt"})
+    assert runtime[key] == str(CONSTANTS.DATA_DIR / "other/cookies.txt")
+    absolute = tmp_path / "absolute-cookies.txt"
+    config = ArchiveBoxConfig(**{key: str(absolute)})
+    assert str(config[key]) == str(absolute)
+
+
+@pytest.mark.parametrize("key", ["COOKIES_FILE", "AUTH_STORAGE_FILE"])
+@pytest.mark.parametrize("value", ["", "  ", None])
+def test_empty_cookie_path_disables_import(key, value):
+    from archivebox.config.common import ArchiveBoxConfig
+
+    config = ArchiveBoxConfig(**{key: value})
+    assert config[key] is None
+    assert key not in config.for_crawl_runtime()
+
+
+def test_cookie_runtime_path_preserves_configured_symlink(tmp_path):
+    from archivebox.config.common import ArchiveBoxConfig
+
+    export = tmp_path / "export.txt"
+    export.write_text("# Netscape HTTP Cookie File\n")
+    configured = tmp_path / "cookies.txt"
+    configured.symlink_to(export)
+    config = ArchiveBoxConfig(COOKIES_FILE=configured)
+    assert str(config.COOKIES_FILE) == str(configured)
+    runtime = config.for_crawl_runtime()
+    assert runtime["COOKIES_FILE"] == str(configured)
+    assert Path(runtime["COOKIES_FILE"]).read_bytes() == export.read_bytes()

archivebox/tests/test_crawl_history.py

@@ -0,0 +1,111 @@
+"""Completed captures belong in the database, not a crawl-long event history."""
+
+import asyncio
+import gc
+import json
+import threading
+import weakref
+from collections import Counter
+
+import pytest
+
+from abx_dl.events import ProcessCompletedEvent, SnapshotCompletedEvent, SnapshotEvent
+from werkzeug import Response
+from archivebox.base_models.models import get_or_create_system_user_pk
+from archivebox.core.models import ArchiveResult, Snapshot
+from archivebox.crawls.models import Crawl
+from archivebox.services.runner import CrawlRunner, run_install
+
+
+@pytest.mark.django_db(transaction=True)
+@pytest.mark.parametrize("timeout_page", [False, True])
+def test_completed_snapshots_expire_after_database_projection(tmp_path, httpserver, hermetic_lib_dir, timeout_page):
+    urls = []
+    response_started = threading.Event()
+    release_response = threading.Event()
+
+    def stalled_response(_request):
+        def body():
+            response_started.set()
+            yield b"<html><head><title>Saved page 3</title></head><body>"
+            release_response.wait()
+            yield b"Completed response after the browser timed out.</body></html>"
+
+        return Response(body(), content_type="text/html")
+
+    httpserver.expect_request("/favicon.ico").respond_with_data("", status=204)
+    for index in range(8):
+        path = f"/page/{index}"
+        if timeout_page and index == 3:
+            httpserver.expect_request(path).respond_with_handler(stalled_response)
+            urls.append(httpserver.url_for(path))
+            continue
+        httpserver.expect_request(path).respond_with_data(
+            f"<html><head><title>Saved page {index}</title></head><body>"
+            "This real page is part of a multi-snapshot crawl whose completed captures "
+            "must remain available in the database after their events expire.</body></html>",
+            content_type="text/html",
+        )
+        urls.append(httpserver.url_for(path))
+    run_install(plugin_names=["title"])
+    crawl = Crawl.objects.create(
+        urls="\n".join(urls),
+        config={
+            "ABXPKG_LIB_DIR": str(hermetic_lib_dir),
+            "PLUGINS": "title",
+            "CRAWL_MAX_CONCURRENT_SNAPSHOTS": 1,
+            "CHROME_PAGELOAD_TIMEOUT": 5,
+        },
+        created_by_id=get_or_create_system_user_pk(),
+    )
+    runner = CrawlRunner(crawl, selected_plugins=["title"], show_progress=False)
+    runner.bus.event_history.max_history_size = 120
+    sizes = []
+    references = []
+    counts = Counter()
+    retired_ids = set()
+    retained_completed = []
+
+    def count_event(event):
+        counts[event.event_type] += 1
+
+    def observe_snapshot(event: SnapshotEvent):
+        sizes.append(sum(isinstance(previous, SnapshotEvent) for previous in runner.bus.event_history.values()))
+        retained_completed.append(len(retired_ids.intersection(runner.bus.event_history)))
+        references.append(weakref.ref(event))
+
+    async def observe_completed(event: SnapshotCompletedEvent):
+        root = await runner.bus.find(SnapshotEvent, snapshot_id=event.snapshot_id, past=True, future=False)
+        assert root is not None
+        retired_ids.add(root.event_id)
+        retired_ids.update(item.event_id for item in await runner.bus.filter("*", child_of=root, past=True, future=False))
+
+    def release_timed_out_response(event: ProcessCompletedEvent):
+        if response_started.is_set() and event.hook_name == "on_Snapshot__30_chrome_navigate" and event.status == "failed":
+            release_response.set()
+
+    runner.bus.on("*", count_event)
+    runner.bus.on(SnapshotEvent, observe_snapshot)
+    runner.bus.on(SnapshotCompletedEvent, observe_completed)
+    runner.bus.on(ProcessCompletedEvent, release_timed_out_response)
+    try:
+        asyncio.run(runner.run())
+    finally:
+        release_response.set()
+    (tmp_path / "event-counts.json").write_text(json.dumps({"events": counts, "snapshots_in_history": sizes}))
+    snapshots = list(Snapshot.objects.filter(crawl=crawl).order_by("url"))
+    assert len(snapshots) == len(urls)
+    assert all(snapshot.status == "sealed" for snapshot in snapshots)
+    for index, snapshot in enumerate(snapshots):
+        if timeout_page and index == 3:
+            assert response_started.is_set()
+            result = ArchiveResult.objects.get(snapshot=snapshot, hook_name="on_Snapshot__30_chrome_navigate", status="failed")
+            assert "timeout" in result.output_str.lower()
+            continue
+        result = ArchiveResult.objects.get(snapshot=snapshot, plugin="title", status="succeeded")
+        assert result.output_str == f"Saved page {index}"
+        assert (snapshot.output_dir / "title" / "title.txt").read_text().strip() == f"Saved page {index}"
+    assert sizes == [1] * len(urls), sizes
+    assert retained_completed == [0] * len(urls), retained_completed
+    gc.collect()
+    assert all(reference() is None for reference in references)