Bump release version to 0.9.72rc68

Resolve relative cookie files before hooks change cwd, reconcile file edits before machine startup saves, and prevent stale cached binaries overriding runtime selections. Cover issue #1896 and real persona localStorage capture through CLI regressions.

.github/CONTRIBUTING.md

@@ -12,7 +12,7 @@
 - https://github.com/ArchiveBox/ArchiveBox/issues
 - https://github.com/ArchiveBox/ArchiveBox/pulls
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Install#manual-setup
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-c-bare-metal-setup
 
 ### Development Setup
 

.github/STAGING.md

@@ -33,3 +33,10 @@ screenshots are retained as Actions artifacts. Other failures remain
 failures; do not publish a success deployment record to bypass them. No code
 mounts are allowed. The Cabbage job must pass before its record authorizes a
 stable upload. Queued capture progress is polled; failed tests are not retried.
+
+Browser acceptance follows the default WACZ viewer into the captured document,
+matching its replay base and original URL and checking that its body is visible
+and contains substantial text. An attached viewer iframe can still be
+`about:blank`, so its presence alone cannot prove replay works. Extracted display
+titles and the captured document's `<title>` may legitimately differ; neither
+title equality nor output thumbnails substitute for checking the actual page.

.github/dependabot.yml

@@ -15,7 +15,7 @@ updates:
         patterns:
           - "*"
   - package-ecosystem: "npm"
-    directory: "/"
+    directory: "/etc"
     target-branch: "dev"
     schedule:
       interval: "monthly"

.github/lychee.toml

@@ -0,0 +1,38 @@
+# Documentation destinations only: do not crawl archived URLs, app routes, or image assets.
+# Manual run from the repository root with Lychee 0.24.2:
+# git ls-files -- '*.md' '*.html' '*.py' '*.rst' '*.yml' '*.yaml' '*.sh' '*.toml' > /tmp/archivebox-link-inputs.txt
+# lychee --config .github/lychee.toml --root-dir "$PWD" --files-from /tmp/archivebox-link-inputs.txt
+# Explicit tracked inputs include README symlinks and let errors in one file leave others checkable.
+include_fragments = "anchor-only"
+include_verbatim = true
+max_retries = 0
+max_concurrency = 4
+timeout = 20
+no_progress = true
+hidden = true
+extensions = ["md", "html", "py", "rst", "yml", "yaml", "sh", "toml"]
+exclude_path = ['(^|/)(tests|data|apidocs|node_modules|\.git)(/|$)']
+# Includes override this exclude; this is a documentation-link allowlist.
+exclude = ['.*']
+include = [
+ '^https://github\.com/ArchiveBox/ArchiveBox/blob/[^/]+/.*\.md(?:#|$)',
+ '^https://raw\.githubusercontent\.com/wiki/ArchiveBox/ArchiveBox/',
+ '^https?://github\.com/(?:ArchiveBox|pirate)/ArchiveBox(?:/wiki(?:/|#|$)|/?#)',
+ '^https?://docs\.archivebox\.io(?:/|$)',
+ '^file:.*\.md(?:#|$)',
+ '^file:.*/docs/[^/.#]+(?:#|$)',
+]
+fallback_extensions = ["md"]
+remap = [
+ # Lychee's Markdown slugger drops variation selectors. Check these against rendered HTML.
+
+ '^file:.*/README\.md(#%EF%B8%8F-cli-usage)$ https://github.com/ArchiveBox/ArchiveBox${1}',
+ '^file:.*/docs/Install\.md(#option-a-docker--docker-compose-setup-%EF%B8%8F)$ https://github.com/ArchiveBox/ArchiveBox/wiki/Install${1}',
+ # The wiki accepts this case alias, but raw file paths are case-sensitive.
+ '^https?://github\.com/(?:ArchiveBox|pirate)/ArchiveBox/wiki/Setting-up-Storage$ https://raw.githubusercontent.com/wiki/ArchiveBox/ArchiveBox/Setting-Up-Storage.md',
+
+ # Bare missing wiki pages silently redirect to Home; raw pages return an actual 404.
+ # Keep fragment URLs on the rendered wiki to catch published heading drift.
+ '^https?://github\.com/(?:ArchiveBox|pirate)/ArchiveBox/wiki/([^#/?]+)$ https://raw.githubusercontent.com/wiki/ArchiveBox/ArchiveBox/${1}.md',
+]
+preprocess = { command = "bin/lychee_preprocess.py" }

.github/pages/artifacts.py

@@ -14,7 +14,7 @@ def trusted(run, repo, workflow, branch, successful=True):
     return (
         (run.get("head_repository") or {}).get("full_name", "").lower() == repo.lower()
         and run["head_branch"] == branch
-        and run["event"] in ("push", "workflow_dispatch")
+        and run["event"] in ("push", "workflow_dispatch", "schedule")
         and run["path"] == f".github/workflows/{workflow}"
         and (not successful or run["conclusion"] == "success")
     )

.github/scripts/discover_test_matrix.py

@@ -13,8 +13,9 @@ def main() -> None:
     if not archivebox_tests:
         raise SystemExit("No ArchiveBox tests discovered")
 
-    # These are observed whole-job seconds, including setup. Unknown/long files,
-    # optional dependencies and hosted-only requirements keep independent jobs.
+    # These are observed per-file pytest seconds, including fixture/startup
+    # time. Shared job setup is paid once per batch. Unknown/long files, optional
+    # dependencies and hosted-only requirements keep independent jobs.
     durations = json.loads((root / ".github/test-durations.json").read_text())["seconds"]
     hosted = {path for path in archivebox_tests if "# ci-runner: hosted" in path.read_text().splitlines()[:5]}
     requested = os.environ.get("TEST_FILE", "")

.github/test-durations.json

@@ -1,168 +1,174 @@
 {
   "source_runs": [
-    "https://github.com/ArchiveBox/ArchiveBox/actions/runs/36302511923",
-    "https://github.com/ArchiveBox/ArchiveBox/actions/runs/36353196552"
+    "https://github.com/ArchiveBox/ArchiveBox/actions/runs/37190995942"
   ],
-  "measurement": "Whole job seconds, including setup and excluding queue time. Unbatched files use the latest complete successful run; batched files retain their individual baseline measurement. Unknown files stay independent.",
+  "measurement": "Per-file pytest wall seconds from JUnit suites, including fixture setup, plus 9 seconds for process startup outside JUnit (observed median 8.1 seconds across 69 successful unbatched jobs). Failed files retain the larger prior successful estimate. Unknown files stay independent. Queue and shared job setup are excluded.",
   "seconds": {
-    "archivebox/tests/test_api_v1_auth_browser_session.py": 29,
-    "archivebox/tests/test_api_v1_auth_check_api_token.py": 27,
-    "archivebox/tests/test_api_v1_auth_get_api_token.py": 24,
-    "archivebox/tests/test_api_v1_cli_add.py": 96,
-    "archivebox/tests/test_api_v1_cli_remove.py": 50,
-    "archivebox/tests/test_api_v1_cli_schedule.py": 41,
-    "archivebox/tests/test_api_v1_cli_search.py": 26,
-    "archivebox/tests/test_api_v1_cli_update.py": 100,
-    "archivebox/tests/test_api_v1_cli_workflow_add_search_update_remove.py": 64,
-    "archivebox/tests/test_api_v1_core_any_id.py": 21,
-    "archivebox/tests/test_api_v1_core_archiveresult_archiveresult_id.py": 55,
-    "archivebox/tests/test_api_v1_core_archiveresults.py": 32,
-    "archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py": 203,
-    "archivebox/tests/test_api_v1_core_snapshots_rss.py": 27,
-    "archivebox/tests/test_api_v1_core_snapshots.py": 28,
-    "archivebox/tests/test_api_v1_core_tag_tag_id_rename.py": 23,
-    "archivebox/tests/test_api_v1_core_tag_tag_id_snapshots_jsonl.py": 27,
-    "archivebox/tests/test_api_v1_core_tag_tag_id_urls_txt.py": 28,
-    "archivebox/tests/test_api_v1_core_tag_tag_id.py": 27,
-    "archivebox/tests/test_api_v1_core_tags_add_to_snapshot.py": 27,
-    "archivebox/tests/test_api_v1_core_tags_autocomplete.py": 26,
-    "archivebox/tests/test_api_v1_core_tags_create.py": 21,
-    "archivebox/tests/test_api_v1_core_tags_remove_from_snapshot.py": 25,
-    "archivebox/tests/test_api_v1_core_tags_search.py": 26,
-    "archivebox/tests/test_api_v1_core_tags.py": 25,
-    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_filename.py": 22,
-    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_filename.py": 25,
-    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_subfolder_filename.py": 24,
-    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py": 86,
-    "archivebox/tests/test_api_v1_crawls_crawls.py": 21,
-    "archivebox/tests/test_api_v1_machine_binaries.py": 23,
-    "archivebox/tests/test_api_v1_machine_binary_binary_id.py": 23,
-    "archivebox/tests/test_api_v1_machine_binary_by_name_name.py": 26,
-    "archivebox/tests/test_api_v1_machine_machine_current.py": 23,
-    "archivebox/tests/test_api_v1_machine_machine_machine_id.py": 23,
-    "archivebox/tests/test_api_v1_machine_machines.py": 27,
-    "archivebox/tests/test_api_v1_personas_personas.py": 23,
-    "archivebox/tests/test_api_v1_personas_sync.py": 29,
-    "archivebox/tests/test_api_v1_workflow_core_token_auth_side_effects.py": 36,
-    "archivebox/tests/test_api_v1_workflow_frozen_crawl_config_sources.py": 33,
-    "archivebox/tests/test_archive_result_service.py": 122,
-    "archivebox/tests/test_auth_ldap.py": 44,
-    "archivebox/tests/test_binary_service.py": 83,
-    "archivebox/tests/test_cli_add_1.py": 179,
-    "archivebox/tests/test_cli_add_2.py": 167,
-    "archivebox/tests/test_cli_add_3.py": 194,
-    "archivebox/tests/test_cli_add_4.py": 168,
-    "archivebox/tests/test_cli_archiveresult.py": 145,
-    "archivebox/tests/test_cli_binary.py": 20,
-    "archivebox/tests/test_cli_config.py": 141,
-    "archivebox/tests/test_cli_crawl.py": 127,
-    "archivebox/tests/test_cli_extract_input.py": 205,
-    "archivebox/tests/test_cli_extract.py": 113,
-    "archivebox/tests/test_cli_help.py": 30,
-    "archivebox/tests/test_cli_init.py": 133,
-    "archivebox/tests/test_cli_install.py": 58,
-    "archivebox/tests/test_cli_list.py": 177,
-    "archivebox/tests/test_cli_machine.py": 50,
-    "archivebox/tests/test_cli_manage.py": 48,
-    "archivebox/tests/test_cli_mcp.py": 51,
-    "archivebox/tests/test_cli_oneshot.py": 23,
-    "archivebox/tests/test_cli_persona.py": 82,
-    "archivebox/tests/test_cli_piping.py": 97,
-    "archivebox/tests/test_cli_pluginmap.py": 21,
-    "archivebox/tests/test_cli_process.py": 20,
-    "archivebox/tests/test_cli_remove.py": 150,
-    "archivebox/tests/test_cli_run_1.py": 165,
-    "archivebox/tests/test_cli_run_2.py": 241,
-    "archivebox/tests/test_cli_run_3.py": 162,
-    "archivebox/tests/test_cli_run_4.py": 200,
-    "archivebox/tests/test_cli_schedule.py": 87,
-    "archivebox/tests/test_cli_search.py": 35,
-    "archivebox/tests/test_cli_server.py": 251,
-    "archivebox/tests/test_cli_shell.py": 32,
-    "archivebox/tests/test_cli_snapshot.py": 138,
-    "archivebox/tests/test_cli_status.py": 105,
-    "archivebox/tests/test_cli_tag.py": 21,
-    "archivebox/tests/test_cli_update_reindex_snapshots.py": 144,
-    "archivebox/tests/test_cli_update.py": 148,
-    "archivebox/tests/test_cli_version.py": 181,
-    "archivebox/tests/test_config_DELETE_AFTER.py": 192,
-    "archivebox/tests/test_config_MAX_limits.py": 34,
-    "archivebox/tests/test_config_ONLY_NEW.py": 24,
-    "archivebox/tests/test_config_SAVE_TITLE.py": 99,
-    "archivebox/tests/test_config_URL_filters.py": 26,
-    "archivebox/tests/test_core_config.py": 20,
-    "archivebox/tests/test_core_forms.py": 28,
-    "archivebox/tests/test_crawl_pause.py": 20,
-    "archivebox/tests/test_crawl_runner.py": 45,
-    "archivebox/tests/test_crawl_service.py": 34,
-    "archivebox/tests/test_frozen_crawl_config.py": 47,
-    "archivebox/tests/test_hook_result_lifecycle.py": 78,
-    "archivebox/tests/test_hooks.py": 90,
-    "archivebox/tests/test_machine_models.py": 52,
-    "archivebox/tests/test_machine_service.py": 104,
-    "archivebox/tests/test_migrations_04_to_09.py": 36,
-    "archivebox/tests/test_migrations_07_to_09.py": 140,
-    "archivebox/tests/test_migrations_08_to_09.py": 257,
-    "archivebox/tests/test_migrations_fresh.py": 99,
-    "archivebox/tests/test_misc_checks.py": 22,
-    "archivebox/tests/test_misc_monkey_patches.py": 20,
-    "archivebox/tests/test_opencode_agent_1.py": 135,
-    "archivebox/tests/test_opencode_agent_2.py": 138,
-    "archivebox/tests/test_opencode_agent_3.py": 140,
-    "archivebox/tests/test_opencode_browser_1.py": 290,
-    "archivebox/tests/test_opencode_browser_2.py": 157,
-    "archivebox/tests/test_opencode_browser_3.py": 129,
-    "archivebox/tests/test_orm_helpers.py": 22,
-    "archivebox/tests/test_permissions.py": 21,
-    "archivebox/tests/test_persona_runtime.py": 56,
-    "archivebox/tests/test_postgres_backend.py": 85,
-    "archivebox/tests/test_process_runtime_paths.py": 28,
-    "archivebox/tests/test_process_service.py": 49,
-    "archivebox/tests/test_proof_previews.py": 54,
-    "archivebox/tests/test_recursive_crawl_1.py": 396,
-    "archivebox/tests/test_recursive_crawl_2.py": 193,
-    "archivebox/tests/test_recursive_crawl_3.py": 144,
-    "archivebox/tests/test_release_notes.py": 14,
-    "archivebox/tests/test_release_source_fingerprint.py": 22,
-    "archivebox/tests/test_release_workflow.py": 21,
-    "archivebox/tests/test_resource_admission.py": 21,
-    "archivebox/tests/test_runner_watch.py": 182,
-    "archivebox/tests/test_search.py": 203,
-    "archivebox/tests/test_serve_static.py": 31,
-    "archivebox/tests/test_server_automatic_url.py": 36,
-    "archivebox/tests/test_server_security_browser.py": 241,
-    "archivebox/tests/test_settings_signal_webhooks.py": 21,
-    "archivebox/tests/test_setup_script.py": 21,
-    "archivebox/tests/test_shutdown_util.py": 22,
-    "archivebox/tests/test_snapshot_filesystem_migration.py": 29,
-    "archivebox/tests/test_snapshot_output_stacks.py": 62,
-    "archivebox/tests/test_snapshot_ref_lookup.py": 23,
-    "archivebox/tests/test_snapshot_service.py": 41,
-    "archivebox/tests/test_snapshot_url_length.py": 26,
-    "archivebox/tests/test_tag_service.py": 24,
-    "archivebox/tests/test_takeover_util_1.py": 122,
-    "archivebox/tests/test_takeover_util_2.py": 171,
-    "archivebox/tests/test_takeover_util_3.py": 165,
-    "archivebox/tests/test_takeover_util_4.py": 160,
-    "archivebox/tests/test_test_harness.py": 22,
-    "archivebox/tests/test_ui_add_view_runtime.py": 239,
-    "archivebox/tests/test_ui_add_view.py": 46,
-    "archivebox/tests/test_ui_admin_apitoken.py": 23,
-    "archivebox/tests/test_ui_admin_archiveresult.py": 30,
-    "archivebox/tests/test_ui_admin_config_widget.py": 25,
-    "archivebox/tests/test_ui_admin_crawl.py": 44,
-    "archivebox/tests/test_ui_admin_links.py": 35,
-    "archivebox/tests/test_ui_admin_machine.py": 29,
-    "archivebox/tests/test_ui_admin_persona.py": 28,
-    "archivebox/tests/test_ui_admin_snapshot.py": 288,
-    "archivebox/tests/test_ui_admin_tag.py": 26,
-    "archivebox/tests/test_ui_admin_user.py": 28,
-    "archivebox/tests/test_ui_config_views.py": 43,
-    "archivebox/tests/test_ui_live_progress.py": 303,
-    "archivebox/tests/test_ui_public_snapshot.py": 138,
-    "archivebox/tests/test_ui_savepagenow.py": 48,
-    "archivebox/tests/test_urls.py": 179,
-    "archivebox/tests/test_util.py": 20,
-    "archivebox/tests/test_version_metadata.py": 24
+    "archivebox/tests/test_api_discovery_performance.py": 28,
+    "archivebox/tests/test_api_v1_auth_browser_session.py": 17,
+    "archivebox/tests/test_api_v1_auth_check_api_token.py": 15,
+    "archivebox/tests/test_api_v1_auth_get_api_token.py": 16,
+    "archivebox/tests/test_api_v1_cli_add.py": 84,
+    "archivebox/tests/test_api_v1_cli_remove.py": 33,
+    "archivebox/tests/test_api_v1_cli_schedule.py": 29,
+    "archivebox/tests/test_api_v1_cli_search.py": 15,
+    "archivebox/tests/test_api_v1_cli_update.py": 175,
+    "archivebox/tests/test_api_v1_cli_workflow_add_search_update_remove.py": 81,
+    "archivebox/tests/test_api_v1_core_any_id.py": 13,
+    "archivebox/tests/test_api_v1_core_archiveresult_archiveresult_id.py": 24,
+    "archivebox/tests/test_api_v1_core_archiveresults.py": 19,
+    "archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py": 106,
+    "archivebox/tests/test_api_v1_core_snapshots.py": 17,
+    "archivebox/tests/test_api_v1_core_snapshots_rss.py": 16,
+    "archivebox/tests/test_api_v1_core_tag_tag_id.py": 15,
+    "archivebox/tests/test_api_v1_core_tag_tag_id_rename.py": 16,
+    "archivebox/tests/test_api_v1_core_tag_tag_id_snapshots_jsonl.py": 16,
+    "archivebox/tests/test_api_v1_core_tag_tag_id_urls_txt.py": 16,
+    "archivebox/tests/test_api_v1_core_tags.py": 15,
+    "archivebox/tests/test_api_v1_core_tags_add_to_snapshot.py": 16,
+    "archivebox/tests/test_api_v1_core_tags_autocomplete.py": 15,
+    "archivebox/tests/test_api_v1_core_tags_create.py": 15,
+    "archivebox/tests/test_api_v1_core_tags_remove_from_snapshot.py": 15,
+    "archivebox/tests/test_api_v1_core_tags_search.py": 17,
+    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py": 131,
+    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_filename.py": 15,
+    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_filename.py": 15,
+    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_subfolder_filename.py": 16,
+    "archivebox/tests/test_api_v1_crawls_crawls.py": 16,
+    "archivebox/tests/test_api_v1_machine_binaries.py": 16,
+    "archivebox/tests/test_api_v1_machine_binary_binary_id.py": 16,
+    "archivebox/tests/test_api_v1_machine_binary_by_name_name.py": 16,
+    "archivebox/tests/test_api_v1_machine_machine_current.py": 15,
+    "archivebox/tests/test_api_v1_machine_machine_machine_id.py": 16,
+    "archivebox/tests/test_api_v1_machine_machines.py": 15,
+    "archivebox/tests/test_api_v1_personas_personas.py": 15,
+    "archivebox/tests/test_api_v1_personas_sync.py": 16,
+    "archivebox/tests/test_api_v1_workflow_core_token_auth_side_effects.py": 27,
+    "archivebox/tests/test_api_v1_workflow_frozen_crawl_config_sources.py": 19,
+    "archivebox/tests/test_archive_result_service.py": 49,
+    "archivebox/tests/test_auth_ldap.py": 32,
+    "archivebox/tests/test_binary_service.py": 109,
+    "archivebox/tests/test_cli_add_1.py": 107,
+    "archivebox/tests/test_cli_add_2.py": 86,
+    "archivebox/tests/test_cli_add_3.py": 108,
+    "archivebox/tests/test_cli_add_4.py": 105,
+    "archivebox/tests/test_cli_archiveresult.py": 86,
+    "archivebox/tests/test_cli_background_screenshot.py": 49,
+    "archivebox/tests/test_cli_binary.py": 12,
+    "archivebox/tests/test_cli_config.py": 75,
+    "archivebox/tests/test_cli_crawl.py": 98,
+    "archivebox/tests/test_cli_extract.py": 63,
+    "archivebox/tests/test_cli_extract_input.py": 124,
+    "archivebox/tests/test_cli_help.py": 16,
+    "archivebox/tests/test_cli_init.py": 72,
+    "archivebox/tests/test_cli_install.py": 38,
+    "archivebox/tests/test_cli_list.py": 141,
+    "archivebox/tests/test_cli_machine.py": 19,
+    "archivebox/tests/test_cli_manage.py": 32,
+    "archivebox/tests/test_cli_mcp.py": 30,
+    "archivebox/tests/test_cli_oneshot.py": 15,
+    "archivebox/tests/test_cli_persona.py": 147,
+    "archivebox/tests/test_cli_piping.py": 60,
+    "archivebox/tests/test_cli_pluginmap.py": 12,
+    "archivebox/tests/test_cli_process.py": 12,
+    "archivebox/tests/test_cli_remove.py": 102,
+    "archivebox/tests/test_cli_run_1.py": 117,
+    "archivebox/tests/test_cli_run_2.py": 112,
+    "archivebox/tests/test_cli_run_3.py": 80,
+    "archivebox/tests/test_cli_run_4.py": 111,
+    "archivebox/tests/test_cli_schedule.py": 147,
+    "archivebox/tests/test_cli_search.py": 25,
+    "archivebox/tests/test_cli_server.py": 143,
+    "archivebox/tests/test_cli_shell.py": 17,
+    "archivebox/tests/test_cli_snapshot.py": 109,
+    "archivebox/tests/test_cli_status.py": 46,
+    "archivebox/tests/test_cli_tag.py": 12,
+    "archivebox/tests/test_cli_update.py": 114,
+    "archivebox/tests/test_cli_update_reindex_snapshots.py": 100,
+    "archivebox/tests/test_cli_version.py": 119,
+    "archivebox/tests/test_config_DELETE_AFTER.py": 187,
+    "archivebox/tests/test_config_MAX_limits.py": 25,
+    "archivebox/tests/test_config_ONLY_NEW.py": 15,
+    "archivebox/tests/test_config_SAVE_TITLE.py": 59,
+    "archivebox/tests/test_config_URL_filters.py": 15,
+    "archivebox/tests/test_core_config.py": 11,
+    "archivebox/tests/test_core_forms.py": 14,
+    "archivebox/tests/test_crawl_history.py": 162,
+    "archivebox/tests/test_crawl_pause.py": 11,
+    "archivebox/tests/test_crawl_runner.py": 22,
+    "archivebox/tests/test_crawl_runner_2.py": 27,
+    "archivebox/tests/test_crawl_service.py": 19,
+    "archivebox/tests/test_frozen_crawl_config.py": 26,
+    "archivebox/tests/test_hook_result_lifecycle.py": 83,
+    "archivebox/tests/test_hooks.py": 54,
+    "archivebox/tests/test_issue_1896.py": 107,
+    "archivebox/tests/test_machine_models.py": 26,
+    "archivebox/tests/test_machine_service.py": 87,
+    "archivebox/tests/test_migrations_04_to_09.py": 25,
+    "archivebox/tests/test_migrations_07_to_09.py": 107,
+    "archivebox/tests/test_migrations_08_to_09.py": 141,
+    "archivebox/tests/test_migrations_fresh.py": 55,
+    "archivebox/tests/test_misc_checks.py": 12,
+    "archivebox/tests/test_misc_monkey_patches.py": 12,
+    "archivebox/tests/test_opencode_agent_1.py": 92,
+    "archivebox/tests/test_opencode_agent_2.py": 60,
+    "archivebox/tests/test_opencode_agent_3.py": 154,
+    "archivebox/tests/test_opencode_browser_1.py": 268,
+    "archivebox/tests/test_opencode_browser_2.py": 77,
+    "archivebox/tests/test_opencode_browser_3.py": 74,
+    "archivebox/tests/test_orm_helpers.py": 14,
+    "archivebox/tests/test_permissions.py": 12,
+    "archivebox/tests/test_persona_casing.py": 96,
+    "archivebox/tests/test_persona_runtime.py": 30,
+    "archivebox/tests/test_postgres_backend.py": 76,
+    "archivebox/tests/test_process_runtime_paths.py": 20,
+    "archivebox/tests/test_process_service.py": 40,
+    "archivebox/tests/test_proof_previews.py": 24,
+    "archivebox/tests/test_recursive_crawl_1.py": 374,
+    "archivebox/tests/test_recursive_crawl_2.py": 55,
+    "archivebox/tests/test_recursive_crawl_3.py": 82,
+    "archivebox/tests/test_release_notes.py": 12,
+    "archivebox/tests/test_release_source_fingerprint.py": 13,
+    "archivebox/tests/test_release_workflow.py": 15,
+    "archivebox/tests/test_resource_admission.py": 13,
+    "archivebox/tests/test_runner_deletion.py": 135,
+    "archivebox/tests/test_runner_watch.py": 104,
+    "archivebox/tests/test_search.py": 151,
+    "archivebox/tests/test_serve_static.py": 19,
+    "archivebox/tests/test_server_automatic_url.py": 22,
+    "archivebox/tests/test_server_security_browser.py": 137,
+    "archivebox/tests/test_settings_signal_webhooks.py": 13,
+    "archivebox/tests/test_setup_script.py": 12,
+    "archivebox/tests/test_shutdown_util.py": 13,
+    "archivebox/tests/test_snapshot_filesystem_migration.py": 16,
+    "archivebox/tests/test_snapshot_output_stacks.py": 76,
+    "archivebox/tests/test_snapshot_ref_lookup.py": 15,
+    "archivebox/tests/test_snapshot_service.py": 48,
+    "archivebox/tests/test_snapshot_url_length.py": 16,
+    "archivebox/tests/test_tag_service.py": 15,
+    "archivebox/tests/test_takeover_util_1.py": 100,
+    "archivebox/tests/test_takeover_util_2.py": 86,
+    "archivebox/tests/test_takeover_util_3.py": 98,
+    "archivebox/tests/test_takeover_util_4.py": 96,
+    "archivebox/tests/test_test_harness.py": 12,
+    "archivebox/tests/test_ui_add_view.py": 26,
+    "archivebox/tests/test_ui_add_view_runtime.py": 121,
+    "archivebox/tests/test_ui_admin_apitoken.py": 16,
+    "archivebox/tests/test_ui_admin_archiveresult.py": 18,
+    "archivebox/tests/test_ui_admin_config_widget.py": 13,
+    "archivebox/tests/test_ui_admin_crawl.py": 25,
+    "archivebox/tests/test_ui_admin_links.py": 23,
+    "archivebox/tests/test_ui_admin_machine.py": 18,
+    "archivebox/tests/test_ui_admin_persona.py": 19,
+    "archivebox/tests/test_ui_admin_snapshot.py": 116,
+    "archivebox/tests/test_ui_admin_tag.py": 16,
+    "archivebox/tests/test_ui_admin_user.py": 16,
+    "archivebox/tests/test_ui_config_views.py": 20,
+    "archivebox/tests/test_ui_live_progress.py": 140,
+    "archivebox/tests/test_ui_public_snapshot.py": 95,
+    "archivebox/tests/test_ui_savepagenow.py": 31,
+    "archivebox/tests/test_urls.py": 132,
+    "archivebox/tests/test_util.py": 13,
+    "archivebox/tests/test_version_metadata.py": 13
   }
 }

.github/workflows/ci.yml

@@ -45,13 +45,6 @@ jobs:
     uses: ./.github/workflows/test-parallel.yml
     secrets: inherit
 
-  documentation:
-    needs: prepare
-    if: needs.prepare.outputs.run_jobs == 'true'
-    name: Documentation and root tests
-    uses: ./.github/workflows/docs.yml
-    secrets: inherit
-
   codeql:
     needs: prepare
     if: needs.prepare.outputs.run_jobs == 'true'
@@ -83,7 +76,6 @@ jobs:
       - lint
       - install-cli-platform
       - tests
-      - documentation
       - codeql
       - python-artifacts
       - docker-digests
@@ -94,7 +86,6 @@ jobs:
           LINT_RESULT: ${{ needs.lint.result }}
           INSTALL_CLI_PLATFORM_RESULT: ${{ needs.install-cli-platform.result }}
           TESTS_RESULT: ${{ needs.tests.result }}
-          DOCUMENTATION_RESULT: ${{ needs.documentation.result }}
           CODEQL_RESULT: ${{ needs.codeql.result }}
           PYTHON_ARTIFACTS_RESULT: ${{ needs.python-artifacts.result }}
           DOCKER_DIGESTS_RESULT: ${{ needs.docker-digests.result }}
@@ -103,7 +94,6 @@ jobs:
             "$LINT_RESULT" \
             "$INSTALL_CLI_PLATFORM_RESULT" \
             "$TESTS_RESULT" \
-            "$DOCUMENTATION_RESULT" \
             "$CODEQL_RESULT" \
             "$PYTHON_ARTIFACTS_RESULT" \
             "$DOCKER_DIGESTS_RESULT"

.github/workflows/docker.yml

@@ -350,6 +350,7 @@ jobs:
           (( COMPRESSED_SIZE < LIMIT ))
 
       - name: Resolve real Docker CLI test dependencies through abxpkg
+        id: mount_dependencies
         if: inputs.full_tests && matrix.platform == 'linux/amd64'
         shell: bash
         run: |
@@ -371,6 +372,15 @@ jobs:
             test -x "$binary"
           done
 
+      - name: Diagnose failed mount dependency installation
+        if: failure() && steps.mount_dependencies.conclusion == 'failure'
+        shell: bash
+        run: |
+          sudo tail -n 120 /var/log/apt/term.log /var/log/dpkg.log
+          sudo journalctl --since '-10 minutes' --no-pager \
+            -u smbd -u nmbd -u samba-ad-dc -u nfs-server
+          ps -eo pid,ppid,stat,etimes,wchan:32,comm
+
       - name: Validate Docker UID and GID behavior on real NFS and SMB mounts
         if: inputs.full_tests && matrix.platform == 'linux/amd64'
         shell: bash

.github/workflows/docs-links.yml

@@ -0,0 +1,38 @@
+name: Documentation link checks
+
+on:
+  schedule:
+    - cron: '17 10 * * *' # Daily at 10:17 UTC, away from the busy top of the hour.
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+env:
+  PYTHONIOENCODING: utf-8
+
+jobs:
+  links:
+    name: Documentation links and anchors
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+        with:
+          version: "0.11.3"
+          enable-cache: false
+      - name: List tracked documentation and source inputs
+        run: |
+          git ls-files -- '*.md' '*.html' '*.py' '*.rst' '*.yml' '*.yaml' '*.sh' '*.toml' > "$RUNNER_TEMP/link-inputs.txt"
+      - uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2
+        with:
+          lycheeVersion: v0.24.2
+          args: >-
+            --config .github/lychee.toml
+            --root-dir "${{ github.workspace }}"
+            --files-from "${{ runner.temp }}/link-inputs.txt"
+          fail: true
+          failIfEmpty: true

.github/workflows/docs.yml

@@ -1,7 +1,17 @@
 name: Documentation tests
 
 on:
-  workflow_call:
+  schedule:
+    - cron: '7 5,17 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  actions: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
 
 env:
   PYTHONIOENCODING: utf-8
@@ -9,7 +19,18 @@ env:
   SHOW_PROGRESS: "False"
 
 jobs:
+  changes:
+    runs-on: ubuntu-24.04
+    outputs:
+      changed: ${{ steps.check.outputs.changed }}
+      key: ${{ steps.check.outputs.key }}
+    steps:
+      - id: check
+        uses: ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main
+
   discover:
+    needs: changes
+    if: needs.changes.outputs.changed == 'true'
     name: Discover documentation examples
     runs-on: ubuntu-24.04
     timeout-minutes: 5
@@ -158,3 +179,17 @@ jobs:
           "$SUDO_BINARY" \
             --preserve-env=PATH,ABXPKG_LIB_DIR,BASH_BINARY,PYTHON_BINARY,SUDO_BINARY,UV_BINARY \
             "$UV_BINARY" run --no-cache --no-sync python docs/test_codeblocks_manifest.py run-group '${{ matrix.snippet_ids }}'
+      - name: Diagnose failed Linux package installation
+        if: failure() && runner.os == 'Linux'
+        run: |
+          sudo tail -n 120 /var/log/apt/term.log /var/log/dpkg.log
+          apt-config dump Acquire::Retries Acquire::http::Timeout Acquire::https::Timeout
+          ps -eo pid,ppid,stat,etimes,wchan:32,comm
+
+  complete:
+    name: 'Scheduled inputs: ${{ needs.changes.outputs.key }}'
+    needs: [changes, discover, examples]
+    if: needs.changes.outputs.changed == 'true'
+    runs-on: ubuntu-24.04
+    steps:
+      - run: echo 'Scheduled work completed successfully.'

.github/workflows/macos.yml

@@ -0,0 +1,48 @@
+name: macOS compatibility
+
+on:
+  schedule:
+    - cron: '51 2,10,18 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  actions: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  changes:
+    runs-on: ubuntu-24.04
+    outputs:
+      changed: ${{ steps.check.outputs.changed }}
+      key: ${{ steps.check.outputs.key }}
+    steps:
+      - id: check
+        uses: ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main
+
+  install-cli:
+    needs: changes
+    if: needs.changes.outputs.changed == 'true'
+    uses: ./.github/workflows/test.yml
+    with:
+      platform: macos
+    secrets: inherit
+
+  python-artifacts:
+    needs: changes
+    if: needs.changes.outputs.changed == 'true'
+    uses: ./.github/workflows/pip.yml
+    with:
+      platforms: '["macos-15"]'
+    secrets: inherit
+
+  complete:
+    name: 'Scheduled inputs: ${{ needs.changes.outputs.key }}'
+    needs: [changes, install-cli, python-artifacts]
+    if: needs.changes.outputs.changed == 'true'
+    runs-on: ubuntu-24.04
+    steps:
+      - run: echo 'Scheduled macOS coverage completed successfully.'

.github/workflows/pip.yml

@@ -3,6 +3,11 @@ name: Build Pip package
 on:
   workflow_call:
     inputs:
+      platforms:
+        description: Runner matrix for package compatibility
+        required: false
+        default: '["ubuntu-24.04"]'
+        type: string
       full_tests:
         description: Run the full package compatibility matrix instead of one release smoke check
         required: false
@@ -95,7 +100,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-24.04, macos-15]
+        os: ${{ fromJSON(inputs.platforms) }}
         python: ["3.13", "3.14"]
         artifact: [wheel, sdist]
 

.github/workflows/screenshots.yml

@@ -1,38 +1,31 @@
 name: Capture publicsite screenshots
 
 on:
-  push:
-    branches:
-      - dev
-    paths:
-      - archivebox/**
-      - bin/collect_ui_screenshots.*
-      - bin/generate_ui_screenshot_gallery.py
-      - bin/take_screenshot.js
-      - docs/Screenshots.md
-      - pyproject.toml
-      - uv.lock
-      - .github/workflows/screenshots.yml
-      - .github/workflows/prepare-release.yml
+  schedule:
+    - cron: '37 5,17 * * *'
   workflow_dispatch:
 
 permissions:
   contents: read
+  actions: read
 
 concurrency:
-  group: ${{ github.event_name == 'workflow_dispatch' && format('publicsite-captures-manual-{0}', github.run_id) || 'publicsite-captures' }}
+  group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: false
 
 jobs:
-  prepare:
-    permissions:
-      contents: write
-    uses: ./.github/workflows/prepare-release.yml
-    secrets: inherit
+  changes:
+    runs-on: ubuntu-24.04
+    outputs:
+      changed: ${{ steps.check.outputs.changed }}
+      key: ${{ steps.check.outputs.key }}
+    steps:
+      - id: check
+        uses: ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main
 
   deploy:
-    needs: prepare
-    if: needs.prepare.outputs.run_jobs == 'true'
+    needs: changes
+    if: needs.changes.outputs.changed == 'true'
     runs-on: ubuntu-24.04
     steps:
       - name: Checkout
@@ -78,3 +71,11 @@ jobs:
           path: publicsite/screenshots
           if-no-files-found: error
           retention-days: 90
+
+  complete:
+    name: 'Scheduled inputs: ${{ needs.changes.outputs.key }}'
+    needs: [changes, deploy]
+    if: needs.changes.outputs.changed == 'true'
+    runs-on: ubuntu-24.04
+    steps:
+      - run: echo 'Scheduled work completed successfully.'

.github/workflows/test-parallel.yml

@@ -36,16 +36,9 @@ jobs:
       - name: Select available CI capacity
         id: runner
         if: vars.UGNAS_CI_ENABLED == 'true' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main')
-        uses: mikehardy/runner-fallback-action@dc7732f9e532c451b2527b288ba8d58fd4b0a4ca # v1.1.0
+        uses: ArchiveBox/monorepo/.github/actions/select-ci-runner@main
         with:
           github-token: ${{ secrets.RUNNER_STATUS_TOKEN }}
-          organization: ArchiveBox
-          primary-runner: self-hosted,ugnas-linux
-          fallback-runner: ubuntu-24.04
-          # Jobs queued for this pool must not be mistaken for idle runner slots:
-          # requiring MAX_JOBS idle runners prevents assigning a second wave.
-          primaries-required: ${{ vars.UGNAS_CI_MIN_IDLE || 1 }}
-          fallback-on-error: true
 
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
@@ -106,7 +99,7 @@ jobs:
     needs: discover-tests
     # Opt in trusted branches only; PRs always use GitHub-hosted runners.
     # Ordinary Linux tests are eligible; # ci-runner: hosted declares exceptions.
-    # The hosted discovery job falls back when workers are offline/busy or the
+    # The hosted discovery job falls back when workers are offline or the
     # availability API fails. UGNAS_CI_ENABLED=false disables the pool entirely.
     runs-on: ${{ fromJSON(matrix.test.ugnas && needs.discover-tests.outputs.runner || '["ubuntu-24.04"]') }}
     env:
@@ -119,8 +112,18 @@ jobs:
 
     steps:
       - name: Set job-local runtime paths
+        env:
+          CI_RUNNER_ENVIRONMENT: ${{ runner.environment }}
         run: |
+          workers=2
+          cache_suffix=''
+          if [[ "$CI_RUNNER_ENVIRONMENT" == github-hosted ]]; then
+            workers="$(nproc)"
+            cache_suffix="-workers-$workers"
+          fi
           {
+            echo "CI_TEST_WORKERS=$workers"
+            echo "ABXPKG_CACHE_WORKERS_SUFFIX=$cache_suffix"
             echo "ABXPKG_LIB_DIR=$RUNNER_TEMP/archivebox-lib"
             echo "DATA_DIR=$RUNNER_TEMP/archivebox-test-data"
             echo "PERSONAS_DIR=$RUNNER_TEMP/abx-personas"
@@ -136,6 +139,7 @@ jobs:
         with:
           python-version: ${{ env.CI_PYTHON_VERSION }}
           architecture: x64
+        id: python
 
       - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
         with:
@@ -143,6 +147,41 @@ jobs:
           enable-cache: false
           cache-dependency-glob: uv.lock
 
+      # Self-hosted tool storage survives jobs. Normal abxpkg validation and
+      # install locks allow reuse across files; cold-install tests use tmp_path.
+      - name: Reuse persistent runner tools
+        if: runner.environment == 'self-hosted'
+        env:
+          PERSISTENT_ABXPKG_LIB: ${{ runner.tool_cache }}/abxpkg/${{ github.repository }}/${{ steps.python.outputs.python-version }}
+        run: |
+          mkdir -p "$PERSISTENT_ABXPKG_LIB"
+          echo "ABXPKG_LIB_DIR=$PERSISTENT_ABXPKG_LIB" >> "$GITHUB_ENV"
+
+      # Restored binaries still pass normal abxpkg resolution/version checks.
+      # Installation and isolation tests explicitly choose fresh temporary libs.
+      - name: Identify runtime dependency cache
+        shell: bash
+        run: |
+          uv run --no-project python - <<'PYTHON' >> "$GITHUB_ENV"
+          import hashlib
+          import json
+          import tomllib
+          from pathlib import Path
+
+          packages = tomllib.loads(Path("uv.lock").read_text())["package"]
+          versions = {p["name"]: p["version"] for p in packages if p["name"] in {"abxpkg", "abx-plugins"} and "editable" not in p["source"]}
+          print("ABXPKG_CACHE_DEPS=" + hashlib.sha256(json.dumps(versions, sort_keys=True).encode()).hexdigest())
+          PYTHON
+
+      - name: Reuse validated runtime dependencies
+        if: runner.environment != 'self-hosted'
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+        with:
+          path: ${{ runner.temp }}/archivebox-lib
+          key: abxpkg-v1-${{ runner.os }}-${{ runner.arch }}-${{ runner.environment }}-${{ steps.python.outputs.python-version }}-${{ matrix.test.name }}-${{ env.ABXPKG_CACHE_DEPS }}${{ env.ABXPKG_CACHE_WORKERS_SUFFIX }}
+          restore-keys: |
+            abxpkg-v1-${{ runner.os }}-${{ runner.arch }}-${{ runner.environment }}-${{ steps.python.outputs.python-version }}-${{ matrix.test.name }}-
+
       - name: Resolve optional build dependencies through abxpkg
         if: matrix.test.extra != '' || contains(matrix.test.paths_arg, 'test_postgres_backend')
         env:
@@ -183,8 +222,10 @@ jobs:
       - name: Run ${{ matrix.test.name }}
         env:
           TEST_PATHS: ${{ matrix.test.paths_arg }}
+          CI_RUNNER_ENVIRONMENT: ${{ runner.environment }}
         run: |
           set -Eeuo pipefail
+          echo "Running pytest with $CI_TEST_WORKERS workers on $CI_RUNNER_ENVIRONMENT"
           read -r -a test_paths <<< "$TEST_PATHS"
           for test_path in "${test_paths[@]}"; do
             test -f "$test_path"
@@ -196,11 +237,8 @@ jobs:
             export DATA_DIR="$RUNNER_TEMP/archivebox-test-data/$test_slug"
             export PERSONAS_DIR="$RUNNER_TEMP/abx-personas/$test_slug"
             export CHROME_USER_DATA_DIR="$PERSONAS_DIR/Default/chrome_profile"
-            if (( ${#test_paths[@]} > 1 )); then
-              export ABXPKG_LIB_DIR="$RUNNER_TEMP/archivebox-lib/$test_slug"
-            fi
             mkdir -p "$DATA_DIR" "$PERSONAS_DIR" "$RUNNER_TEMP/pytest/$test_slug"
-            uv run --no-cache --no-sync --no-sources pytest -vs "$test_path" --durations=20 \
+            uv run --no-cache --no-sync --no-sources pytest -vs -n "$CI_TEST_WORKERS" "$GITHUB_WORKSPACE/$test_path" --durations=20 \
               --basetemp="$RUNNER_TEMP/pytest/$test_slug" \
               --junitxml="$RUNNER_TEMP/test-results/$test_slug.xml" || test_status=1
           done

.github/workflows/test.yml

@@ -2,6 +2,12 @@ name: Install and CLI Compatibility
 
 on:
   workflow_call:
+    inputs:
+      platform:
+        description: Platform for install and CLI compatibility
+        required: false
+        default: linux
+        type: string
 
 env:
   DOCKER_IMAGE: archivebox-ci
@@ -19,16 +25,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - os: ubuntu-24.04
-            os_name: linux
-            python: "3.14"
-          - os: macos-15
-            os_name: macOS
-            python: "3.13"
-          - os: macos-15
-            os_name: macOS
-            python: "3.14"
+        include: ${{ fromJSON(inputs.platform == 'macos' && '[{"os":"macos-15","os_name":"macOS","python":"3.13"},{"os":"macos-15","os_name":"macOS","python":"3.14"}]' || '[{"os":"ubuntu-24.04","os_name":"linux","python":"3.14"}]') }}
 
     steps:
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4

README.md

@@ -123,13 +123,13 @@ curl -fsSL 'https://get.archivebox.io' | bash
 ## Key Features
 
 - [**Free & open source**](https://github.com/ArchiveBox/ArchiveBox/blob/dev/LICENSE), own your own data & maintain your privacy by self-hosting
-- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) with [modular dependencies](#dependencies) and [support for Google Drive/NFS/SMB/S3/B2/etc.](https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage)
+- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage) with [modular dependencies](#dependencies) and [support for Google Drive/NFS/SMB/S3/B2/etc.](https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage)
 - [**Comprehensive documentation**](https://github.com/ArchiveBox/ArchiveBox/wiki), [active development](https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap), and [rich community](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community)
 - [**Extracts a wide variety of content out-of-the-box**](https://github.com/ArchiveBox/ArchiveBox/issues/51): [media (yt-dlp), articles (readability), code (git), etc.](#output-formats)
 - [**Supports scheduled/realtime importing**](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving) from [many types of sources](#input-formats)
 - [**Uses standard, durable, long-term formats**](#output-formats) like HTML, JSON, PDF, PNG, MP4, TXT, and WARC
-- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [Python API](https://docs.archivebox.io/en/dev/apidocs/archivebox/archivebox.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://electron.archivebox.io/)
-- [**Saves all pages to archive.org as well**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#save_archive_dot_org) by default for redundancy (can be [disabled](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#stealth-mode) for local-only mode)
+- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#ui-usage), [Python API](https://docs.archivebox.io/dev/apidocs/archivebox/archivebox.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://electron.archivebox.io/)
+- [**Saves all pages to archive.org as well**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#archivedotorg_enabled) by default for redundancy (can be [disabled](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#archivedotorg_enabled) for local-only mode)
 - Support for archiving [content requiring login/paywall/cookies](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir) (see wiki security caveats!)
 - Support for running [JS during archiving](https://github.com/ArchiveBox/ArchiveBox/issues/51) to adblock, [infiniscroll](https://github.com/ArchiveBox/ArchiveBox/issues/80), [modal-hide](https://github.com/ArchiveBox/ArchiveBox/issues/175), [thread-expand](https://github.com/ArchiveBox/ArchiveBox/issues/345)
 
@@ -220,7 +220,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <ol>
 <li>Run the shortcut for the <code>uv</code> install method. It installs <code>uv</code> first when needed.
 <pre lang="bash"><code style="white-space: pre-line">curl -fsSL 'https://get.archivebox.io' | bash</code></pre>
-<i>For more info, see <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-b-automatic-setup-script">Install: Bare Metal</a> in the Wiki. ➡️</i>
+<i>For more info, see <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-b-uv-install-shortcut">Install: Bare Metal</a> in the Wiki. ➡️</i>
 </li>
 </ol>
 
@@ -431,7 +431,7 @@ For more discussion on third-party hosting options see here: <a href="https://gi
 #### ➡️&nbsp; Next Steps
 
 - Import URLs from some of the supported [Input Formats](#input-formats) or view the supported [Output Formats](#output-formats)...
-- (Optional) [Import browser cookies and settings into a persona](docs/Chromium-Install.md#import-an-existing-browser-profile).
+- (Optional) [Import browser cookies and settings into a persona](https://github.com/ArchiveBox/ArchiveBox/blob/dev/docs/Chromium-Install.md#import-an-existing-browser-profile).
 - Tweak your UI or archiving behavior [Configuration](#configuration), read about some of the [Caveats](#caveats), or [Troubleshoot](https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting)
 - Read about the [Dependencies](#dependencies) used for archiving, the [Upgrading Process](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives), or the [Archive Layout](#archive-layout) on disk...
 - Or check out our full [Documentation](#documentation) or [Community Wiki](#internet-archiving-ecosystem)...
@@ -613,7 +613,7 @@ docker run -it -v $PWD:/data archivebox/archivebox:dev add --depth=1 'https://ex
 <sub>. . . . . . . . . . . . . . . . . . . . . . . . . . . .</sub>
 <br/><br/>
 <a href="https://demo.archivebox.io">DEMO: <code>https://demo.archivebox.io</code></a><br/>
-<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration">Configuration</a> | <a href="#Caveats">Caveats</a>
+<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration">Configuration</a> | <a href="#caveats">Caveats</a>
 <br/>
 </div>
 
@@ -641,7 +641,7 @@ docker run -it -v $PWD:/data archivebox/archivebox:dev add --depth=1 'https://ex
   <i>Save directly from the Share menu or Siri/Shortcuts to your server.</i>
 
 - <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/64078483-21d7-4eb1-aa6e-9ad55afe45b8" height="22px"/> From manual imports of URLs from RSS, JSON, CSV, TXT, SQL, HTML, Markdown, etc. files  
-  <i>ArchiveBox supports injesting URLs in [any text-based format](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Import-a-list-of-URLs-from-a-text-file).</i>
+  <i>ArchiveBox supports injesting URLs in [any text-based format](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#import-a-list-of-urls-from-a-text-file).</i>
 
 - <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/32b494e6-4de1-4984-8d88-dc02f18e5c34" height="22px"/> From manually exported [browser history](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) or [browser bookmarks](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) (in Netscape format)  
   <i>Instructions: <a href="https://support.google.com/chrome/answer/96816?hl=en">Chrome</a>, <a href="https://support.mozilla.org/en-US/kb/export-firefox-bookmarks-to-backup-or-transfer">Firefox</a>, <a href="https://github.com/ArchiveBox/ArchiveBox/assets/511499/24ad068e-0fa6-41f4-a7ff-4c26fc91f71a">Safari</a>, <a href="https://support.microsoft.com/en-us/help/211089/how-to-import-and-export-the-internet-explorer-favorites-folder-to-a-32-bit-version-of-windows">IE</a>, <a href="https://help.opera.com/en/latest/features/#bookmarks:~:text=Click%20the%20import/-,export%20button,-on%20the%20bottom">Opera</a>, <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive">and more...</a></i>
@@ -670,7 +670,7 @@ echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox a
 # echo 'https://example.com' | docker compose run --rm -T archivebox add
 ```
 
-See the [Usage: CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) page for documentation and examples.
+See the [Usage: CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage) page for documentation and examples.
 
 It also includes a built-in scheduled import feature with `archivebox schedule`, handled by the same orchestrator that powers `archivebox server`, so you can pull in URLs from RSS feeds and websites regularly without a separate cron container.
 
@@ -811,7 +811,7 @@ Installing directly on <strong>Windows without Docker or WSL/WSL2/Cygwin is not
 
 <h4>Learn More</h4>
 <ul>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#dependencies">Wiki: Install (Dependencies)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#3-install-runtime-dependencies-using-archivebox-install">Wiki: Install (Dependencies)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install">Wiki: Chromium Install</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives">Wiki: Upgrading or Merging Archives</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#installing">Wiki: Troubleshooting (Installing)</a></li>
@@ -865,7 +865,7 @@ Each snapshot subfolder includes static metadata and plain extractor output file
 <h4>Learn More</h4>
 <ul>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage">Wiki: Setting Up Storage (SMB, NFS, S3, B2, Google Drive, etc.)</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">Wiki: Usage (Disk Layout)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout">Wiki: Usage (Disk Layout)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#large-archives">Wiki: Usage (Large Archives)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#output-folder">Wiki: Security Overview (Output Folder)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive">Wiki: Publishing Your Archive</a></li>
@@ -951,7 +951,7 @@ archivebox manage createsuperuser
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive">Wiki: Publishing Your Archive</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview">Wiki: Security Overview</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile">Wiki: Chromium Install (Setting Up a User Profile)</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#import-an-existing-browser-profile">Wiki: Chrome profiles and cookies</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/docs/Chromium-Install.md#import-an-existing-browser-profile">Wiki: Chrome profiles and cookies</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#default_persona">Wiki: Configuration (<code>DEFAULT_PERSONA</code>)</a></li>
 </ul>
 
@@ -1004,7 +1004,7 @@ For various reasons, many large sites (Reddit, Twitter, Cloudflare, etc.) active
 
 
 <ul>
-<li>Set up a logged-in browser session for archiving by <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#import-an-existing-browser-profile">importing a chrome profile into a persona</a></li>
+<li>Set up a logged-in browser session for archiving by <a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/docs/Chromium-Install.md#import-an-existing-browser-profile">importing a chrome profile into a persona</a></li>
 <li>Rewrite your URLs before archiving to swap in alternative frontends that are more bot-friendly e.g.<br>
 <code>reddit.com/some/url</code> -&gt; <code>teddit.net/some/url</code>: <a href="https://github.com/mendel5/alternative-front-ends">https://github.com/mendel5/alternative-front-ends</a></li>
 </ul>
@@ -1065,7 +1065,7 @@ Because ArchiveBox is designed to ingest a large volume of URLs with multiple co
 <h4>Learn More</h4>
 
 <ul>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">Wiki: Usage (Disk Layout)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout">Wiki: Usage (Disk Layout)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#output-folder">Wiki: Security Overview (Output-Folder)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#large-archives">Wiki: Usage (Large Archives)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#run-archivebox-as-an-unprivileged-user">Wiki: Security Overview (Privilege Dropping)</a></li>
@@ -1190,7 +1190,7 @@ By having users store their content locally or within their organizations, we ca
 
 <h3>Comparison With Other Self-Hosted Archiving Options</h3>
 
-ArchiveBox differentiates itself from [similar self-hosted projects](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#Web-Archiving-Projects) by providing both a comprehensive CLI interface for managing your archive, a Web UI that can be used either independently or together with the CLI, and a simple on-disk data format that can be used without either.
+ArchiveBox differentiates itself from [similar self-hosted projects](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#web-archiving-projects) by providing both a comprehensive CLI interface for managing your archive, a Web UI that can be used either independently or together with the CLI, and a simple on-disk data format that can be used without either.
 
 
 *If you want better fidelity for very complex interactive pages with heavy JS/streams/API requests, check out [ArchiveWeb.page](https://archiveweb.page) and [ReplayWeb.page](https://replayweb.page).*
@@ -1199,7 +1199,7 @@ ArchiveBox differentiates itself from [similar self-hosted projects](https://git
 
 *If you need more advanced recursive spider/crawling ability beyond `--depth=1`, check out [Browsertrix](https://github.com/webrecorder/browsertrix-crawler), [Photon](https://github.com/s0md3v/Photon), or [Scrapy](https://scrapy.org/) and pipe the outputted URLs into ArchiveBox.*
 
-For more alternatives, see our [list here](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#Web-Archiving-Projects)...
+For more alternatives, see our [list here](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#web-archiving-projects)...
 
 ArchiveBox is neither the highest fidelity nor the simplest tool available for self-hosted archiving, rather it's a jack-of-all-trades that tries to do most things well by default. We encourage you to try these other tools made by our friends if ArchiveBox isn't suited to your needs.
 
@@ -1254,7 +1254,7 @@ We use the [ArchiveBox GitHub Wiki](https://github.com/ArchiveBox/ArchiveBox/wik
 - [Usage](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage)
 - [Configuration](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration)
 - [Supported Sources](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive)
-- [Supported Outputs](https://github.com/ArchiveBox/ArchiveBox/wiki#can-save-these-things-for-each-site)
+- [Supported Outputs](https://github.com/ArchiveBox/ArchiveBox#output-formats)
 - [Scheduled Archiving](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving)
 
 ## Advanced

archivebox/api/v1_api.py

@@ -2,6 +2,8 @@ __package__ = "archivebox.api"
 
 
 from io import StringIO
+from functools import lru_cache
+from typing import Any
 from traceback import format_exception
 from contextlib import redirect_stdout, redirect_stderr
 
@@ -10,6 +12,7 @@ from django.core.exceptions import ObjectDoesNotExist, EmptyResultSet, Permissio
 from django.contrib.auth.models import User
 
 from ninja import NinjaAPI, Swagger
+from ninja.openapi.schema import OpenAPISchema
 
 # TODO: explore adding https://eadwincode.github.io/django-ninja-extra/
 
@@ -49,6 +52,16 @@ def register_urls(api: NinjaAPI) -> NinjaAPI:
 
 
 class NinjaAPIWithIOCapture(NinjaAPI):
+    def get_openapi_schema(self, *, path_prefix: str | None = None, path_params: dict[str, Any] | None = None) -> OpenAPISchema:
+        prefix = path_prefix if path_prefix is not None else self.get_root_path(path_params or {})
+        return self._schema_for_prefix(prefix)
+
+    @lru_cache(maxsize=8)
+    def _schema_for_prefix(self, path_prefix: str) -> OpenAPISchema:
+        # Routes are registered at startup. Server discovery can reuse their
+        # schema while preserving distinct URL prefixes for mounted instances.
+        return super().get_openapi_schema(path_prefix=path_prefix)
+
     def create_temporal_response(self, request: HttpRequest) -> HttpResponse:
         stdout, stderr = StringIO(), StringIO()
 

archivebox/api/v1_core.py

@@ -223,20 +223,20 @@ def get_archiveresults(request: HttpRequest, filters: Query[ArchiveResultFilterS
 
 def _uuid_ref_query(field_name: str, ref: str) -> Q:
     raw_ref = str(ref or "").strip()
-    query = Q(**{f"{field_name}__startswith": raw_ref})
-    if raw_ref:
-        query |= Q(**{f"{field_name}__icontains": raw_ref})
     try:
         parsed_uuid = UUID(raw_ref)
     except (TypeError, ValueError):
+        query = Q(**{f"{field_name}__startswith": raw_ref})
+        if raw_ref:
+            query |= Q(**{f"{field_name}__icontains": raw_ref})
         normalized_ref = raw_ref.replace("-", "")
         if normalized_ref and normalized_ref != raw_ref:
             query |= Q(**{f"{field_name}__startswith": normalized_ref})
             query |= Q(**{f"{field_name}__icontains": normalized_ref})
-    else:
-        query |= Q(**{field_name: parsed_uuid})
-        query |= Q(**{f"{field_name}__startswith": parsed_uuid.hex})
-    return query
+        return query
+    # An exact UUID cannot need a substring fallback. OR-ing it with LIKE
+    # prevents an indexed lookup and made Cabbage metadata reads take seconds.
+    return Q(**{field_name: parsed_uuid})
 
 
 @router.get("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema, url_name="get_archiveresult")
@@ -350,6 +350,12 @@ def _summarize_archiveresult_output_files(output_files: dict[str, dict[str, Any]
 
 def _get_snapshot_by_ref(snapshot_id: str):
     queryset = Snapshot.objects.select_related("crawl__created_by")
+    try:
+        full_id = UUID(snapshot_id)
+    except (TypeError, ValueError):
+        pass
+    else:
+        return queryset.get(pk=full_id)
     try:
         return queryset.get(_uuid_ref_query("id", snapshot_id) | Q(timestamp__startswith=snapshot_id))
     except Snapshot.DoesNotExist:
@@ -392,13 +398,13 @@ def _write_archiveresult_files(
     mime_types = _get_archiveresult_upload_form_values(request, "mime_types", "mime_type")
     chunk_output_path = _get_archiveresult_upload_form_value(request, "chunk_output_path")
 
+    output_files = dict(existing_output_files or {})
+    if not files:
+        return output_files
+
     snapshot_dir = snapshot.output_dir
     plugin_dir = snapshot_dir / plugin_name
     storage = FileSystemStorage(location=str(plugin_dir))
-    output_files = dict(existing_output_files or {})
-
-    if not files:
-        return output_files
 
     if chunk_output_path:
         if len(files) != 1:
@@ -630,6 +636,8 @@ def patch_archiveresult(
 class SnapshotSchema(Schema):
     TYPE: str = "core.models.Snapshot"
     id: UUID
+    crawl_id: UUID
+    persona: str | None
     created_by_id: str
     created_by_username: str
     created_at: datetime
@@ -648,6 +656,17 @@ class SnapshotSchema(Schema):
     num_archiveresults: int
     archiveresults: list[MinimalArchiveResultSchema]
 
+    @staticmethod
+    def resolve_archive_path(obj):
+        return obj.archive_path_from_db
+
+    @staticmethod
+    def resolve_persona(obj):
+        # A deduplicated submission can reuse a capture from a different
+        # persona. Report its actual DB-backed owner, not the new request's choice.
+        persona = obj.crawl.resolve_persona()
+        return persona.name if persona else None
+
     @staticmethod
     def resolve_created_by_id(obj):
         return str(obj.created_by.pk)
@@ -700,6 +719,7 @@ class SnapshotDeleteResponseSchema(Schema):
     snapshot_id: str
     crawl_id: str
     deleted_count: int
+    queued_count: int
 
 
 def normalize_tag_list(tags: list[str] | None = None) -> list[str]:
@@ -938,6 +958,15 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
     # lock. The unique insert recovery and CAS update below are the request-side
     # coordination boundary for this idempotent metadata sync.
     snapshot = Snapshot.objects.filter(url=data.url, crawl=crawl).first()
+    if snapshot is None and bool(crawl.get_current_config().get("ONLY_NEW", True)):
+        # Uploading browser metadata is part of the same submission as /add.
+        # It must honor the runner's deduplication policy, even if it arrives
+        # before the runner has processed that crawl. Report the actual owner
+        # so clients never mistake a reused capture for one they just created.
+        snapshot = Snapshot.objects.filter(url=data.url).exclude(status=Snapshot.DELETING_STATE).order_by("-created_at").first()
+        if snapshot is not None:
+            setattr(request, "with_archiveresults", False)
+            return snapshot
     if snapshot is None:
         try:
             snapshot = Snapshot.objects.create(
@@ -971,11 +1000,6 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
             created_by=request.user if isinstance(request.user, User) else None,
         )
 
-    try:
-        snapshot.ensure_crawl_symlink()
-    except Exception:
-        pass
-
     setattr(request, "with_archiveresults", False)
     return snapshot
 
@@ -1037,22 +1061,23 @@ def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateS
 
 @router.delete("/snapshot/{snapshot_id}", response=SnapshotDeleteResponseSchema, url_name="delete_snapshot")
 def delete_snapshot(request: HttpRequest, snapshot_id: str):
+    """Queue deletion; the normal server runner stops writers and removes files.
+
+    The snapshot remains readable with status=deleting until cleanup succeeds.
+    queued_count reports acceptance; deleted_count is zero in this response.
+    """
     snapshot = get_snapshot(request, snapshot_id, with_archiveresults=False)
     snapshot_id_str = str(snapshot.id)
-    crawl_id_str = str(snapshot.crawl.pk)
-    snapshot.cancel()
-
-    from archivebox.services.runner import run_pending_crawls
-
-    with crawl_lifecycle_lock(crawl_id_str):
-        run_pending_crawls(crawl_id=crawl_id_str, daemon=False)
-        snapshot = get_snapshot(request, snapshot_id_str, with_archiveresults=False)
-        deleted_count, _ = snapshot.delete()
+    crawl_id_str = str(snapshot.crawl_id)
+    # Use the same durable queue as the admin delete action. Stopping writers
+    # and removing archive files belongs to the runner, never the HTTP request.
+    queued_count = Snapshot.objects.filter(pk=snapshot.pk).request_delete()
     return {
         "success": True,
         "snapshot_id": snapshot_id_str,
         "crawl_id": crawl_id_str,
-        "deleted_count": deleted_count,
+        "deleted_count": 0,
+        "queued_count": queued_count,
     }
 
 

archivebox/api/v1_personas.py

@@ -5,9 +5,11 @@ from datetime import datetime
 from typing import Any, Literal
 from uuid import UUID
 
+from django.core.exceptions import ValidationError
 from django.db.models import Q
 from django.http import HttpRequest
 from ninja import Router, Schema
+from ninja.errors import HttpError
 from ninja.pagination import paginate
 from pydantic import Field
 
@@ -108,10 +110,9 @@ def browser_settings_to_config(extension_persona_id: str, settings: PersonaBrows
 
 
 def find_persona(extension_persona_id: str, name: str) -> Persona | None:
+    named = Persona.find_named(name)
     return (
-        Persona.objects.filter(
-            Q(config__BROWSER_EXTENSION_PERSONA_ID=extension_persona_id) | Q(name=name),
-        )
+        Persona.objects.filter(Q(config__BROWSER_EXTENSION_PERSONA_ID=extension_persona_id) | Q(pk=named.pk if named else None))
         .order_by("created_at")
         .first()
     )
@@ -138,7 +139,11 @@ def sync_persona(request: HttpRequest, payload: PersonaSyncSchema):
     if not is_valid:
         raise ValueError(error_message)
 
-    persona = find_persona(payload.extension_persona_id, name)
+    try:
+        persona = find_persona(payload.extension_persona_id, name)
+        (persona or Persona(name=name)).validate_name(persona.name if persona else name)
+    except ValidationError as err:
+        raise HttpError(409, "; ".join(err.messages)) from err
     created = persona is None
     if persona is None:
         persona = Persona(name=name)

archivebox/base_models/models.py

@@ -9,6 +9,7 @@ from typing import Any
 from archivebox.uuid_compat import CompactUUIDField, uuid7
 from pathlib import Path
 
+from django.core.exceptions import FieldDoesNotExist
 from django.db import models
 from django.db.models import F
 from django.db import transaction
@@ -23,6 +24,110 @@ from django_stubs_ext.db.models import TypedModelMeta
 from archivebox.config import CONSTANTS
 
 
+class PagedQuerySet(models.QuerySet):
+    def paged_iterator(self, chunk_size: int = 500):
+        """
+        Iterate rows using bounded keyset pages instead of one streaming cursor.
+
+        Django's iterator(chunk_size=...) still keeps a single SQLite SELECT
+        cursor open until the full queryset is exhausted. That is fine for
+        read-only exports, but update/migration code does filesystem work and
+        writes while iterating; a long-lived read cursor there can stretch lock
+        waits across thousands of rows. This respects the queryset's existing
+        filters, order_by(), select_related(), and prefetch_related() state; if
+        no ordering is defined, it falls back to primary-key order.
+        """
+        pk_field = self.model._meta.pk.name
+        raw_ordering = tuple(self.query.order_by or self.model._meta.ordering or (pk_field,))
+
+        if any(not isinstance(term, str) or term == "?" for term in raw_ordering):
+            offset = 0
+            while True:
+                batch = list(self[offset : offset + chunk_size])
+                if not batch:
+                    break
+                yield from batch
+                offset += chunk_size
+            return
+
+        ordering = []
+        for term in raw_ordering:
+            descending = term.startswith("-")
+            field_name = term[1:] if descending else term
+            if field_name == "pk":
+                field_name = pk_field
+            ordering.append(f"-{field_name}" if descending else field_name)
+
+        ordered_field_names = [term.removeprefix("-") for term in ordering]
+        try:
+            if any(self.model._meta.get_field(field_name).null for field_name in ordered_field_names):
+                offset = 0
+                while True:
+                    batch = list(self[offset : offset + chunk_size])
+                    if not batch:
+                        break
+                    yield from batch
+                    offset += chunk_size
+                return
+        except (AttributeError, FieldDoesNotExist):
+            offset = 0
+            while True:
+                batch = list(self[offset : offset + chunk_size])
+                if not batch:
+                    break
+                yield from batch
+                offset += chunk_size
+            return
+
+        unique_field_names = {pk_field, *(field.name for field in self.model._meta.fields if field.unique)}
+        if not any(field_name in unique_field_names for field_name in ordered_field_names):
+            offset = 0
+            while True:
+                batch = list(self[offset : offset + chunk_size])
+                if not batch:
+                    break
+                yield from batch
+                offset += chunk_size
+            return
+
+        last_values = None
+        value_field_names = tuple(dict.fromkeys([*ordered_field_names, pk_field]))
+        while True:
+            batch_qs = self.order_by(*ordering)
+            if last_values is not None:
+                page_filter = models.Q()
+                for idx, term in enumerate(ordering):
+                    descending = term.startswith("-")
+                    field_name = term[1:] if descending else term
+                    prefix = {ordered_field_names[i]: last_values[i] for i in range(idx)}
+                    comparison = "lt" if descending else "gt"
+                    page_filter |= models.Q(**prefix, **{f"{field_name}__{comparison}": last_values[idx]})
+                # Anchor the OR tie-breaks with an explicit leading range so
+                # SQLite/PG can seek the existing order index instead of
+                # rescanning its prefix for every keyset page.
+                first_comparison = "lte" if ordering[0].startswith("-") else "gte"
+                batch_qs = batch_qs.filter(
+                    page_filter,
+                    **{f"{ordered_field_names[0]}__{first_comparison}": last_values[0]},
+                )
+
+            batch_rows = list(batch_qs.values_list(*value_field_names)[:chunk_size])
+            if not batch_rows:
+                break
+
+            pk_idx = value_field_names.index(pk_field)
+            row_ids = [row[pk_idx] for row in batch_rows]
+            rows_by_id = {obj.pk: obj for obj in self.filter(pk__in=row_ids).order_by()}
+
+            for row in batch_rows:
+                row_id = row[pk_idx]
+                obj = rows_by_id.get(row_id)
+                if obj is not None:
+                    yield obj
+
+            last_values = batch_rows[-1][: len(ordered_field_names)]
+
+
 def normalize_config_json_values(config: Any) -> Any:
     if not isinstance(config, dict):
         return config
@@ -191,9 +296,21 @@ class ModelWithDeleteAfter(models.Model):
         return cls.objects.none()
 
     @classmethod
-    def delete_expired(cls, *, batch_size: int = 100, backfill_missing: bool = True) -> int:
+    def delete_expired(cls, *, batch_size: int = 100, backfill_missing: bool = True, backfill_cursors: dict | None = None) -> int:
         if backfill_missing:
-            missing_delete_at = list(cls.missing_delete_at_candidates().order_by("created_at", "pk")[:batch_size])
+            # Bound the scanned rows BEFORE evaluating JSON policy/parent
+            # joins. LIMIT on the policy query only bounded its output: finding
+            # no matches could scan the entire history with a SQLite read lock.
+            # The runner keeps a keyset cursor between idle passes, so disabled
+            # policies cannot starve older rows. Restarting safely repeats work.
+            page = cls.objects.order_by("-pk")
+            before = backfill_cursors.get(cls) if backfill_cursors is not None else None
+            if before is not None:
+                page = page.filter(pk__lt=before)
+            page_ids = list(page.values_list("pk", flat=True)[:batch_size])
+            if backfill_cursors is not None:
+                backfill_cursors[cls] = page_ids[-1] if len(page_ids) == batch_size else None
+            missing_delete_at = list(cls.missing_delete_at_candidates().filter(pk__in=page_ids).order_by())
             for obj in missing_delete_at:
                 if obj.set_delete_at_from_config():
                     cls.objects.filter(pk=obj.pk, delete_at__isnull=True).update(
@@ -231,12 +348,9 @@ class ModelWithOutputDir(ModelWithUUID):
 
     _delete_signal_registered = False
 
-    def save(self, *args, **kwargs):
-        super().save(*args, **kwargs)
-        output_dir = Path(self.output_dir)
-        # Avoid holding SQLite write transactions open across slow filesystem work.
-        transaction.on_commit(lambda: output_dir.mkdir(parents=True, exist_ok=True))
-        # Note: index.json is deprecated, models should use write_index_jsonl() for full data
+    # Saving metadata must not resolve or create output paths. The runner and
+    # explicit artifact writers create their own directories when writing files;
+    # on_commit would still make HTTP requests wait on remote archive storage.
 
     @property
     def output_dir_parent(self) -> str:
@@ -282,7 +396,14 @@ class ModelWithOutputDir(ModelWithUUID):
             if path.is_symlink() or path.is_file():
                 path.unlink(missing_ok=True)
             elif path.is_dir():
-                shutil.rmtree(path, ignore_errors=True)
+                shutil.rmtree(path)
+            # lstat also catches dangling symlinks and propagates mount and
+            # permission errors rather than mistaking them for absence.
+            try:
+                path.lstat()
+            except FileNotFoundError:
+                continue
+            raise OSError(f"Output path still exists after deletion: {path}")
 
     def schedule_delete_cleanup(self, *, using: str | None = None) -> None:
         """Capture output paths before DB deletion and remove them after commit."""
@@ -294,7 +415,9 @@ class ModelWithOutputDir(ModelWithUUID):
         if cls._delete_signal_registered:
             return
 
-        def schedule_output_dir_cleanup(sender, instance, using, **kwargs):
+        def schedule_output_dir_cleanup(sender, instance, using, origin=None, **kwargs):
+            if getattr(origin, "_output_files_deleted", False):
+                return
             if not isinstance(instance, ModelWithOutputDir):
                 return
             instance.schedule_delete_cleanup(using=using)

archivebox/cli/__init__.py

@@ -2,6 +2,7 @@ __package__ = "archivebox.cli"
 __command__ = "archivebox"
 import os
 import sys
+from contextlib import redirect_stdout
 from importlib import import_module
 
 import rich_click as click
@@ -219,6 +220,12 @@ def main(args=None, prog_name=None):
         raise SystemExit(130) from None
     finally:
         sys.unraisablehook = previous_unraisablehook
+        supervisor_util = sys.modules.get("archivebox.workers.supervisord_util")
+        if supervisor_util is not None:
+            # Short commands can start a foreground-owned search daemon too.
+            # Finish its cleanup before another CLI can borrow a dying stack.
+            with redirect_stdout(sys.stderr):
+                supervisor_util.stop_own_supervisord_process()
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_add.py

@@ -144,7 +144,6 @@ def add(
     persona_name = (persona or "Default").strip() or "Default"
     plugins = plugins or ""
     persona_obj = Persona.get_or_create_named(persona_name)
-    persona_obj.ensure_dirs()
     effective_persona_config = get_config(persona=persona_obj)
 
     crawl_config = {
@@ -217,9 +216,6 @@ def add(
         print(
             "[yellow]\\[*] URLs queued. The background runner will process them (run `archivebox server` or `archivebox run --daemon` if not already running).[/yellow]",
         )
-        from archivebox.services.runner import ensure_background_runner
-
-        ensure_background_runner()
     else:
         # Foreground mode: run full crawl runner until all work is done
         print("[green]\\[*] Starting crawl runner to process crawl...[/green]")
@@ -427,7 +423,23 @@ def main(**kwargs):
             existing = [token.strip() for token in (kwargs.get("plugins") or "").split(",") if token.strip()]
             kwargs["plugins"] = ",".join(dict.fromkeys([*existing, *selected]))
 
-        add(urls=urls, **kwargs)
+        from archivebox.machine.models import Process
+
+        # CLI commands own their process record and logs. Queue model saves
+        # must not register a process implicitly: HTTP add() shares that path.
+        process = Process.current()
+        exit_code = 0
+        try:
+            add(urls=urls, **kwargs)
+        except BaseException:
+            exit_code = 1
+            raise
+        finally:
+            # --bg/--index-only return without the foreground runner's cleanup.
+            # Leaving them RUNNING makes the active-process index accumulate
+            # historical CLI commands forever, defeating bounded recovery.
+            if process.status == Process.StatusChoices.RUNNING:
+                process.mark_exited(exit_code=exit_code)
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_init.py

@@ -116,6 +116,11 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     # Registration requires the final schema and reconciles legacy file config.
     Machine.current()
 
+    from archivebox.personas.models import Persona
+
+    # Initialize here, not when /add/ is rendered: a GET must not create profiles.
+    Persona.get_or_create_default().ensure_dirs()
+
     from archivebox.misc.db import database_display_location, is_postgres
 
     assert database_exists()

archivebox/cli/archivebox_install.py

@@ -58,7 +58,6 @@ def _install_raw_binary_names(binary_names: list[str], binproviders: str) -> Non
     from django.utils import timezone
 
     from archivebox.machine.models import Binary, Machine, _canonical_binary_name
-    from archivebox.services.runner import run_due_binary
 
     machine = Machine.current()
     for requested_name in binary_names:
@@ -80,7 +79,13 @@ def _install_raw_binary_names(binary_names: list[str], binproviders: str) -> Non
             binary.status = Binary.StatusChoices.QUEUED
             binary.retry_at = timezone.now()
             binary.save(update_fields=["binproviders", "overrides", "status", "retry_at", "modified_at"])
-        run_due_binary(binary, lock_seconds=60)
+        if binary.status == Binary.StatusChoices.INSTALLED:
+            continue
+        # The synchronous CLI needs the lifecycle's verified installation result,
+        # not the scheduler's "claimed work" result. Reuse its failure/requeue and
+        # health-stat handling so a failed install cannot exit successfully.
+        if not binary.install_claimed(lock_seconds=60):
+            raise click.ClickException(f"Binary {binary_name} is already being installed")
 
 
 def ensure_data_dir_lib_symlink(data_dir: Path, abxpkg_lib_dir: Path) -> Path | None:

archivebox/cli/archivebox_persona.py

@@ -163,8 +163,9 @@ def create_personas(
             rprint(f'[red]Invalid persona name "{name}": {error_msg}[/red]', file=sys.stderr)
             continue
 
-        persona, created = Persona.objects.get_or_create(
-            name=name,
+        created = Persona.find_named(name) is None
+        persona = Persona.get_or_create_named(
+            name,
             defaults={"config": {"PERMISSIONS": permissions}} if permissions else {},
         )
 
@@ -313,7 +314,9 @@ def update_personas(name: str | None = None) -> int:
             if persona_id:
                 persona = Persona.objects.get(id=persona_id)
             else:
-                persona = Persona.objects.get(name=old_name)
+                persona = Persona.find_named(old_name)
+                if persona is None:
+                    raise Persona.DoesNotExist
 
             # Apply updates from CLI flags
             if name:
@@ -325,7 +328,7 @@ def update_personas(name: str | None = None) -> int:
 
                 # Rename the persona directory too
                 old_path = persona.path
-                persona.name = name
+                persona.name = persona.validate_name(name)
                 new_path = persona.path
 
                 if old_path.exists() and old_path != new_path:

archivebox/cli/archivebox_remove.py

@@ -87,9 +87,8 @@ def remove(
     # Delete one snapshot at a time. Each ``.delete()`` is its own short
     # Django-atomic block, so the writer lock is released between rows and
     # an in-flight daemon transaction can interleave instead of deadlocking.
-    # Filesystem cleanup for each row is scheduled via ``transaction.on_commit``
-    # in ``base_models/models.py`` and runs AFTER its row's tx commits — so
-    # rmtree doesn't hold the lock either.
+    # Snapshot.delete() removes and verifies filesystem outputs first, outside
+    # the short database delete transaction. Failed cleanup retains the row.
     #
     deleted_snapshot_pks = []
     timed_out = False

archivebox/cli/archivebox_server.py

@@ -260,6 +260,12 @@ def server(
     bind_spec = next((arg for arg in runserver_args if arg), "")
     host, port = _parse_and_validate_bind_spec(bind_spec)
 
+    from archivebox.personas.models import Persona
+
+    # Older collections deferred this until /add/. Reconcile before serving any
+    # requests, and report ambiguous profile names before Chrome can use them.
+    Persona.get_or_create_default().ensure_dirs()
+
     if not User.objects.filter(is_superuser=True).exclude(username="system").exists():
         from archivebox.core.routes_util import build_admin_url
 

archivebox/cli/archivebox_snapshot.py

@@ -298,7 +298,7 @@ def list_snapshots(
     Exit codes:
         0: Success (even if no results)
     """
-    from archivebox.misc.jsonl import write_record
+    from archivebox.misc.jsonl import write_records
 
     output_formats = sum(bool(output_format) for output_format in (csv, as_json, as_html))
     if output_formats > 1:
@@ -351,9 +351,7 @@ def list_snapshots(
         return 0
 
     if not is_tty:
-        for snapshot_json in iter_snapshot_json(queryset):
-            write_record(snapshot_json)
-            count += 1
+        count = write_records(iter_snapshot_json(queryset))
         rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
         return 0
 

archivebox/cli/rescan.py

@@ -102,9 +102,19 @@ def _reconcile_directory(directory: Path) -> str:
     results = {(item.get("plugin"), item.get("hook_name", "")): item for item in records if item.get("type") == "ArchiveResult"}.values()
     pending = []
     for result in results:
-        result_id = uuid.UUID(result["id"])
         if uuid.UUID(result["snapshot_id"]) != snapshot_id:
             raise ValueError("ArchiveResult belongs to another snapshot")
+        # Legacy JSON conversion produced records without IDs. Recover these
+        # by hook identity while preserving strict collision checks for IDs.
+        if not result.get("id"):
+            if not ArchiveResult.objects.filter(
+                snapshot_id=snapshot_id,
+                plugin=result["plugin"],
+                hook_name=result.get("hook_name", ""),
+            ).exists():
+                pending.append(result)
+            continue
+        result_id = uuid.UUID(result["id"])
         existing = ArchiveResult.objects.filter(pk=result_id).first()
         identity = (snapshot_id, result["plugin"], result.get("hook_name", ""))
         if existing:
@@ -154,7 +164,10 @@ def _reconcile_directory(directory: Path) -> str:
     # Each insert is durable. An interruption after the Snapshot insert is not
     # completion: the next pass still imports missing result IDs and tags.
     for result in pending:
-        snapshot._create_archive_result_if_missing(result, {}, preserve_identity=True)
+        snapshot._create_archive_result_if_missing(result, {}, preserve_identity=bool(result.get("id")))
+    if any(not result.get("id") for result in pending):
+        snapshot.hydrate_archiveresult_output_metadata(snapshot_dir=directory)
+        snapshot.write_index_jsonl(output_dir=directory)
     tags_before = set(snapshot.tags.values_list("name", flat=True))
     snapshot._merge_tags_from_index(record)
     tags_changed = tags_before != set(snapshot.tags.values_list("name", flat=True))
@@ -182,6 +195,10 @@ def reconcile_known_snapshots(snapshots, wait_for_turn=None):
             snapshot.migrate_filesystem_to_current_version()
         directory = Path(snapshot.output_dir)
         if not directory.is_dir():
+            # Canceling queued metadata can seal a row before the runner ever
+            # creates storage. There is nothing on disk to reconcile for it.
+            if snapshot.downloaded_at is None and not snapshot.archiveresult_set.exists():
+                continue
             raise ValueError(f"Missing snapshot directory: {directory}")
         _reconcile_directory(directory)
         for result in snapshot.archiveresult_set.all():

archivebox/config/common.py

@@ -34,6 +34,25 @@ ConfigPayload = dict[str, object]
 PluginSchemaDocuments = dict[str, dict[str, Any]]
 LIVE_CONFIG_BASE_URL = "/admin/environment/config/"
 
+
+def _is_cookie_file_config_key(key: str) -> bool:
+    return key in {"COOKIES_FILE", "AUTH_STORAGE_FILE"} or key.endswith("_COOKIES_FILE")
+
+
+def _resolve_cookie_file_paths(config: dict[str, Any]) -> dict[str, Any]:
+    """Keep collection-relative auth paths valid after hooks change cwd."""
+    for key, value in config.items():
+        if _is_cookie_file_config_key(key):
+            if value is None or (isinstance(value, str) and not value.strip()):
+                config[key] = ""
+                continue
+            file_path = Path(value).expanduser()
+            if not file_path.is_absolute():
+                file_path = CONSTANTS.DATA_DIR / file_path
+            config[key] = str(file_path.absolute())
+    return config
+
+
 ###################### Config ##########################
 
 _STDOUT_CONSOLE = Console()
@@ -395,6 +414,36 @@ class ServerConfig(BaseConfigSet):
         )
 
 
+@lru_cache(maxsize=16)
+def _default_sqlite_journal_mode(database_name: str) -> str:
+    if not IN_DOCKER:
+        return "WAL"
+
+    # Native Linux bind mounts share the kernel's locks and mmap; Docker alone
+    # does not make WAL unsafe. DELETE there lets any slow reader block tiny
+    # writes. Desktop VM shares (virtiofs/9p/FUSE), network filesystems and
+    # unknown mounts keep rollback journaling: their host-side readers may not
+    # share SQLite's -shm locking domain. Inspect the DB mount, not archive/.
+    database_path = Path(database_name).expanduser().resolve()
+    try:
+        mounts = Path("/proc/self/mountinfo").read_text().splitlines()
+    except OSError:
+        return "DELETE"
+    filesystem = ""
+    mount_depth = -1
+    for line in mounts:
+        fields, separator, details = line.partition(" - ")
+        if not separator:
+            continue
+        mount_path = fields.split()[4]
+        mount_path = re.sub(r"\\([0-7]{3})", lambda match: chr(int(match[1], 8)), mount_path)
+        mount = Path(mount_path)
+        if database_path.is_relative_to(mount) and len(mount.parts) > mount_depth:
+            filesystem = details.split()[0]
+            mount_depth = len(mount.parts)
+    return "WAL" if filesystem in {"ext2", "ext3", "ext4", "xfs", "btrfs", "zfs", "overlay", "tmpfs", "f2fs"} else "DELETE"
+
+
 class DatabaseConfig(BaseConfigSet):
     toml_section_header: str = "DATABASE_CONFIG"
     _scope: str = PrivateAttr(default=_SCOPE_SERVER)
@@ -410,11 +459,7 @@ class DatabaseConfig(BaseConfigSet):
     DATABASE_USER: str = Field(default="archivebox", alias="ARCHIVEBOX_DATABASE_USER")
     DATABASE_PASSWORD: str = Field(default="", alias="ARCHIVEBOX_DATABASE_PASSWORD")
     SQLITE_JOURNAL_MODE: str = Field(
-        # Docker collections commonly live on a host bind mount. WAL's -shm
-        # locking is only safe when every SQLite process is on the same host;
-        # Docker Desktop/OrbStack place the container and host in different
-        # locking domains and a host-side reader can corrupt the live DB.
-        default="DELETE" if IN_DOCKER else "WAL",
+        default_factory=lambda data: _default_sqlite_journal_mode(str(data["DATABASE_NAME"])),
         alias="ARCHIVEBOX_SQLITE_JOURNAL_MODE",
         pattern=r"(?i)^(DELETE|TRUNCATE|PERSIST|MEMORY|WAL|OFF)$",
     )
@@ -428,11 +473,16 @@ class DatabaseConfig(BaseConfigSet):
     SQLITE_LOCK_RETRY_INTERVAL: float = Field(default=5.0, alias="ARCHIVEBOX_SQLITE_LOCK_RETRY_INTERVAL", gt=0)
 
     @model_validator(mode="after")
-    def reject_docker_sqlite_wal(self):
-        if IN_DOCKER and self.DATABASE_ENGINE.lower() == "sqlite" and self.SQLITE_JOURNAL_MODE.upper() == "WAL":
+    def reject_shared_filesystem_sqlite_wal(self):
+        if (
+            IN_DOCKER
+            and self.DATABASE_ENGINE.lower() == "sqlite"
+            and self.SQLITE_JOURNAL_MODE.upper() == "WAL"
+            and _default_sqlite_journal_mode(self.DATABASE_NAME) != "WAL"
+        ):
             raise ValueError(
-                "SQLITE_JOURNAL_MODE=WAL is unsafe for Docker collections because host bind mounts cross SQLite "
-                "locking domains; use DELETE (the Docker default) or PostgreSQL",
+                "SQLITE_JOURNAL_MODE=WAL requires a local filesystem with shared SQLite locking; "
+                "keep the database on a local Docker volume, use DELETE for host/VM shared mounts, or use PostgreSQL",
             )
         return self
 
@@ -468,6 +518,11 @@ class ArchivingConfig(BaseConfigSet):
     COOKIES_FILE: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
     AUTH_STORAGE_FILE: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
 
+    @field_validator("COOKIES_FILE", "AUTH_STORAGE_FILE", mode="before")
+    @classmethod
+    def empty_cookie_file_is_unset(cls, value):
+        return None if isinstance(value, str) and not value.strip() else value
+
     URL_DENYLIST: str = Field(
         default=(
             r"\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$"
@@ -498,7 +553,7 @@ class ArchivingConfig(BaseConfigSet):
             rprint("    (Setting it to somewhere between 30 and 3000 seconds is recommended)", file=sys.stderr)
             rprint(file=sys.stderr)
             rprint("    If you want to make ArchiveBox run faster, disable specific archive methods instead:", file=sys.stderr)
-            rprint("        https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#archive-method-toggles", file=sys.stderr)
+            rprint("        https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#plugins", file=sys.stderr)
             rprint(file=sys.stderr)
 
     @field_validator("CHECK_SSL_VALIDITY", mode="after")
@@ -825,6 +880,8 @@ class ArchiveBoxBaseConfig(
         if runtime_overrides:
             config.update(normalize_runtime_config(runtime_overrides, json_safe=False))
 
+        _resolve_cookie_file_paths(config)
+
         # Hooks should only see concrete plugin-local flags, never the
         # ArchiveBox selectors used to derive them.
         config.pop("PLUGINS", None)
@@ -849,6 +906,13 @@ class ArchiveBoxBaseConfig(
             lib_dir = CONSTANTS.DATA_DIR / lib_dir
         self.ABXPKG_LIB_DIR = lib_dir.resolve()
 
+        cookie_paths = {key: getattr(self, key) for key in type(self).model_fields if _is_cookie_file_config_key(key)}
+        for key, value in _resolve_cookie_file_paths(cookie_paths).items():
+            if key in {"COOKIES_FILE", "AUTH_STORAGE_FILE"}:
+                setattr(self, key, Path(value) if value else None)
+            else:
+                setattr(self, key, value)
+
         return self
 
     @model_validator(mode="after")
@@ -1156,11 +1220,15 @@ def get_config(
     scope_overrides: ConfigPayload = {}
 
     if include_machine and machine is not None and machine.config:
-        from archivebox.machine.models import _sanitize_machine_config
+        machine_config = machine.config
+        if resolve_plugins:
+            from archivebox.machine.models import _sanitize_machine_config
+
+            machine_config = _sanitize_machine_config(machine_config, lib_dir=config_data.get("ABXPKG_LIB_DIR"))
 
         scope_overrides.update(
             normalize_runtime_config(
-                _sanitize_machine_config(machine.config, lib_dir=config_data.get("ABXPKG_LIB_DIR")),
+                machine_config,
                 only_crawl_execution=crawl_config_base,
                 exclude_runtime_derived=True,
                 json_safe=False,

archivebox/config/views.py

@@ -150,9 +150,9 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
 
     rows = {
         "Binary Name": [],
-        "Found Version": [],
+        "Recorded Version": [],
         "Provided By": [],
-        "Found Abspath": [],
+        "Recorded Abspath": [],
     }
 
     db_binaries = get_db_binaries_by_name()
@@ -160,18 +160,10 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
 
     for name in all_binary_names:
         binary = db_binaries.get(name)
-        binary_is_valid = bool(binary and binary.is_valid)
-
         rows["Binary Name"].append(ItemLink(name, key=name))
-
-        if binary_is_valid:
-            rows["Found Version"].append(f"✅ {binary.version}" if binary.version else "✅ found")
-            rows["Provided By"].append(binary.binprovider or "-")
-            rows["Found Abspath"].append(binary.abspath or "-")
-        else:
-            rows["Found Version"].append("❌ missing")
-            rows["Provided By"].append("-")
-            rows["Found Abspath"].append("-")
+        rows["Recorded Version"].append(binary.version or "-")
+        rows["Provided By"].append(binary.binprovider or "-")
+        rows["Recorded Abspath"].append(binary.abspath or "-")
 
     return TableContext(
         title="Binaries",

archivebox/core/admin_snapshots.py

@@ -131,11 +131,11 @@ class SnapshotStatusListFilter(admin.SimpleListFilter):
     parameter_name = "snapshot_status"
 
     def lookups(self, request, model_admin):
-        return Snapshot.StatusChoices.choices
+        return Snapshot._meta.get_field("status").choices
 
     def queryset(self, request, queryset):
         value = self.value()
-        if value in Snapshot.StatusChoices.values:
+        if value in dict(Snapshot._meta.get_field("status").choices):
             return queryset.filter(status=value)
         return queryset
 
@@ -992,10 +992,14 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         return format_html(
             '<a class="snapshot-title-detail-hitbox" href="{}" aria-label="Open snapshot details"></a>'
             "{}"
+            "{}"
             '<div class="snapshot-title-url">'
             '<a class="snapshot-original-url" href="{}">{}<code style="user-select: all;">{}</code></a>'
             "</div>",
             detail_url,
+            format_html('<strong class="snapshot-delete-status">{}</strong>', "Deleting — verifying file removal")
+            if obj.status == Snapshot.DELETING_STATE
+            else "",
             title_html,
             url_raw or obj.url,
             favicon_html,
@@ -1535,25 +1539,14 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     @admin.action(
         description="🗑️ Delete",
+        permissions=["delete"],
     )
     def delete_snapshots(self, request, queryset):
-        """Delete snapshots in a single transaction to avoid SQLite concurrency issues."""
-        from django.db import transaction
-
-        total = queryset.count()
-
-        # Get list of IDs to delete first (outside transaction)
-        ids_to_delete = list(queryset.values_list("pk", flat=True))
-
-        # Delete everything in a single atomic transaction
-        with transaction.atomic():
-            deleted_count, _ = Snapshot.objects.filter(pk__in=ids_to_delete).delete()
-
+        """Keep the rows until the runner removes and verifies their files."""
+        total = queryset.request_delete()
         messages.success(
             request,
-            mark_safe(
-                f"Successfully deleted {total} Snapshots ({deleted_count} total objects including related records). Don't forget to scrub URLs from import logs (data/sources) and error logs (data/logs) if needed.",
-            ),
+            f"Queued {total} snapshots for deletion. They remain visible as Deleting until ArchiveBox verifies their files are removed.",
         )
 
     @admin.action(

archivebox/core/forms.py

@@ -304,13 +304,13 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             self.can_override_crawl_config = bool(user.is_authenticated and user.is_active and user.is_superuser)
         super().__init__(*args, **kwargs)
 
-        default_persona = Persona.get_or_create_default()
+        # Persona creation belongs to init/server startup, never a GET of /add/.
         persona_queryset = Persona.objects.order_by("name")
         if not self.can_override_crawl_config:
             persona_queryset = filter_personas_by_permissions(persona_queryset, {PERMISSIONS_PUBLIC})
         self.fields["persona"].queryset = persona_queryset
 
-        selected_persona = persona_queryset.filter(id=default_persona.id).first() or persona_queryset.first()
+        selected_persona = persona_queryset.filter(name__iexact="Default").first() or persona_queryset.first()
         default_config = get_config(persona=selected_persona) if selected_persona else get_config()
         if selected_persona:
             self.fields["persona"].initial = selected_persona.name
@@ -338,6 +338,12 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
                 get_choice_field(self, field_name).choices = [(plugin.name, plugin.name) for plugin in grouped_plugins.get(category, [])]
             self.plugin_groups = []
 
+    def clean_persona(self):
+        persona = self.cleaned_data.get("persona")
+        if persona:
+            persona.validate_name(persona.name)
+        return persona
+
     def clean(self):
         cleaned_data = super().clean() or {}
 

archivebox/core/migrations/0058_snapshot_deleting_status.py

@@ -0,0 +1,28 @@
+# Generated by Django 6.1 on 2026-09-30 03:32
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0057_repair_migrated_process_links"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="snapshot",
+            name="status",
+            field=models.CharField(
+                choices=[
+                    ("queued", "Queued"),
+                    ("started", "Started"),
+                    ("paused", "Paused"),
+                    ("sealed", "Sealed"),
+                    ("deleting", "Deleting"),
+                ],
+                db_index=True,
+                default="queued",
+                max_length=15,
+            ),
+        ),
+    ]

archivebox/core/models.py

@@ -1,6 +1,7 @@
 __package__ = "archivebox.core"
 
 import json
+import logging
 import os
 import uuid
 from collections.abc import Iterable, Mapping, Sequence
@@ -11,7 +12,7 @@ from urllib.parse import urlparse
 
 from django.conf import settings
 from django.contrib import admin
-from django.core.exceptions import FieldDoesNotExist, ObjectDoesNotExist, ValidationError
+from django.core.exceptions import ObjectDoesNotExist, ValidationError
 from django.db import IntegrityError, models, transaction
 from django.db.models import F, Q, QuerySet, Sum, Value
 from django.db.models.fields.json import KT
@@ -23,6 +24,7 @@ from django.utils.safestring import mark_safe
 from django.utils.text import slugify
 
 from archivebox.base_models.models import (
+    PagedQuerySet,
     ModelWithConfig,
     ModelWithDeleteAfter,
     ModelWithHealthStats,
@@ -172,9 +174,28 @@ class SnapshotTag(models.Model):
         unique_together: ClassVar[list[tuple[str, str]]] = [("snapshot", "tag")]
 
 
-class SnapshotQuerySet(models.QuerySet):
+class SnapshotQuerySet(PagedQuerySet):
     """Custom QuerySet for Snapshot model with export methods that persist through .filter() etc."""
 
+    def request_delete(self) -> int:
+        # Admin and API deletion share this durable command. Keep the row until
+        # the runner stops its writers and verifies removal from remote storage;
+        # RETRY_AT_MAX prevents capture scheduling while cleanup is pending.
+        return self.update(status=self.model.DELETING_STATE, retry_at=RETRY_AT_MAX, modified_at=timezone.now())
+
+    def delete(self):
+        # Finish the SELECT before filesystem work, so no read cursor or SQLite
+        # transaction is held while deleting remote archive payloads.
+        snapshots = list(self.select_related("crawl__created_by"))
+        total = 0
+        counts = {}
+        for snapshot in snapshots:
+            deleted, details = snapshot.delete(using=self.db)
+            total += deleted
+            for label, count in details.items():
+                counts[label] = counts.get(label, 0) + count
+        return total, counts
+
     def bulk_create(self, objs, *args, **kwargs):
         objs = list(objs)
         missing_crawl_ids = set()
@@ -206,101 +227,6 @@ class SnapshotQuerySet(models.QuerySet):
                 truncate_overlong_charfields(obj)
         return super().bulk_create(objs, *args, **kwargs)
 
-    def paged_iterator(self, chunk_size: int = 500):
-        """
-        Iterate snapshots using bounded keyset pages instead of one streaming cursor.
-
-        Django's iterator(chunk_size=...) still keeps a single SQLite SELECT
-        cursor open until the full queryset is exhausted. That is fine for
-        read-only exports, but update/migration code does filesystem work and
-        writes while iterating; a long-lived read cursor there can stretch lock
-        waits across thousands of rows. This respects the queryset's existing
-        filters, order_by(), select_related(), and prefetch_related() state; if
-        no ordering is defined, it falls back to primary-key order.
-        """
-        pk_field = self.model._meta.pk.name
-        raw_ordering = tuple(self.query.order_by or self.model._meta.ordering or (pk_field,))
-
-        if any(not isinstance(term, str) or term == "?" for term in raw_ordering):
-            offset = 0
-            while True:
-                batch = list(self[offset : offset + chunk_size])
-                if not batch:
-                    break
-                yield from batch
-                offset += chunk_size
-            return
-
-        ordering = []
-        for term in raw_ordering:
-            descending = term.startswith("-")
-            field_name = term[1:] if descending else term
-            if field_name == "pk":
-                field_name = pk_field
-            ordering.append(f"-{field_name}" if descending else field_name)
-
-        ordered_field_names = [term.removeprefix("-") for term in ordering]
-        try:
-            if any(self.model._meta.get_field(field_name).null for field_name in ordered_field_names):
-                offset = 0
-                while True:
-                    batch = list(self[offset : offset + chunk_size])
-                    if not batch:
-                        break
-                    yield from batch
-                    offset += chunk_size
-                return
-        except (AttributeError, FieldDoesNotExist):
-            offset = 0
-            while True:
-                batch = list(self[offset : offset + chunk_size])
-                if not batch:
-                    break
-                yield from batch
-                offset += chunk_size
-            return
-
-        unique_field_names = {pk_field, *(field.name for field in self.model._meta.fields if field.unique)}
-        if not any(field_name in unique_field_names for field_name in ordered_field_names):
-            offset = 0
-            while True:
-                batch = list(self[offset : offset + chunk_size])
-                if not batch:
-                    break
-                yield from batch
-                offset += chunk_size
-            return
-
-        last_values = None
-        value_field_names = tuple(dict.fromkeys([*ordered_field_names, pk_field]))
-        while True:
-            batch_qs = self.order_by(*ordering)
-            if last_values is not None:
-                page_filter = models.Q()
-                for idx, term in enumerate(ordering):
-                    descending = term.startswith("-")
-                    field_name = term[1:] if descending else term
-                    prefix = {ordered_field_names[i]: last_values[i] for i in range(idx)}
-                    comparison = "lt" if descending else "gt"
-                    page_filter |= models.Q(**prefix, **{f"{field_name}__{comparison}": last_values[idx]})
-                batch_qs = batch_qs.filter(page_filter)
-
-            batch_rows = list(batch_qs.values_list(*value_field_names)[:chunk_size])
-            if not batch_rows:
-                break
-
-            pk_idx = value_field_names.index(pk_field)
-            snapshot_ids = [row[pk_idx] for row in batch_rows]
-            snapshots_by_id = {snapshot.pk: snapshot for snapshot in self.filter(pk__in=snapshot_ids).order_by()}
-
-            for row in batch_rows:
-                snapshot_id = row[pk_idx]
-                snapshot = snapshots_by_id.get(snapshot_id)
-                if snapshot is not None:
-                    yield snapshot
-
-            last_values = batch_rows[-1][: len(ordered_field_names)]
-
     # =========================================================================
     # Filtering Methods
     # =========================================================================
@@ -541,6 +467,7 @@ class SnapshotManager(models.Manager.from_queryset(SnapshotQuerySet)):  # ty: ig
 
 class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHealthStats, ModelWithQueue):
     BROWSER_EXTENSION_UPLOAD_HOOK_NAME = "on_Snapshot__archivebox_browser_extension_upload"
+    DELETING_STATE = "deleting"
 
     id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
@@ -574,7 +501,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
     )
     retry_at = ModelWithQueue.RetryAtField(default=timezone.now)
     status = ModelWithQueue.StatusField(
-        choices=ModelWithQueue.StatusChoices,
+        choices=[*ModelWithQueue.StatusChoices.choices, (DELETING_STATE, "Deleting")],
         default=ModelWithQueue.StatusChoices.QUEUED,
     )
     config = models.JSONField(default=dict, null=False, blank=False, editable=True)
@@ -661,6 +588,49 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
     def __str__(self):
         return f"[{self.id}] {self.url[:64]}"
 
+    def output_paths_for_delete(self) -> tuple[Path, ...]:
+        current = self.get_storage_path_for_version(self.fs_version)
+        paths = [Path(self.output_dir), current, CONSTANTS.ARCHIVE_DIR / self.timestamp]
+        if self.fs_version in ("0.9.0", "0.9.1", "0.9.2", "0.9.3", "0.9.4", "1.0.0"):
+            paths.append(current.with_name(str(uuid.UUID(hex=self.id.hex))))
+        if self.crawl_id:
+            domain_dir = self.crawl.output_dir / CONSTANTS.SNAPSHOTS_DIR_NAME / self.extract_domain_from_url(self.url)
+            paths.extend((domain_dir / str(self.id), domain_dir / str(uuid.UUID(hex=self.id.hex))))
+        return tuple(dict.fromkeys(paths))
+
+    def delete(self, using=None, keep_parents=False):
+        from django.db import connections
+
+        using = using or self._state.db or "default"
+        if connections[using].in_atomic_block:
+            raise RuntimeError("Snapshot filesystem deletion must run outside a database transaction")
+        type(self).objects.using(using).filter(pk=self.pk).request_delete()
+        self.status = self.DELETING_STATE
+        from archivebox.machine.models import Process
+
+        process_ids = self.archiveresult_set.filter(process__status=Process.StatusChoices.RUNNING).values_list("process_id", flat=True)
+        for process in list(Process.objects.using(using).filter(pk__in=process_ids)):
+            process.kill_tree()
+            if process.is_running:
+                raise OSError(f"Snapshot output writer is still running: {process.pk}")
+        paths = self.output_paths_for_delete()
+        self.delete_output_paths(paths)
+        # Cascaded ArchiveResults share this directory. Their normal post-commit
+        # cleanup would repeat filesystem work and update an already-deleted row.
+        self._output_files_deleted = True
+        return super().delete(using=using, keep_parents=keep_parents)
+
+    @classmethod
+    def delete_requested(cls, *, batch_size=100):
+        # Finish the bounded read before touching files: even an open SELECT
+        # cursor can hold a SQLite read lock for the duration of slow cleanup.
+        pending = list(cls.objects.filter(status=cls.DELETING_STATE).select_related("crawl__created_by")[:batch_size])
+        for snapshot in pending:
+            try:
+                snapshot.delete()
+            except OSError:
+                logging.getLogger(__name__).exception("Snapshot %s file deletion failed; keeping its database row", snapshot.pk)
+
     @classmethod
     def crawl_count_subquery(cls, *, status: str | None = None, outer_ref: str = "pk") -> QuerySet:
         """Return a scalar subquery counting Snapshots for one outer Crawl."""
@@ -911,8 +881,11 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
     @classmethod
     def missing_delete_at_candidates(cls):
-        return cls.objects.filter(delete_at__isnull=True).filter(
-            Q(config__has_key="DELETE_AFTER") | Q(crawl__config__has_key="DELETE_AFTER"),
+        return (
+            cls.objects.filter(delete_at__isnull=True)
+            .filter(Q(config__has_key="DELETE_AFTER") | Q(crawl__config__has_key="DELETE_AFTER"))
+            .select_related("crawl")
+            .only("id", "created_at", "delete_at", "config", "crawl__config")
         )
 
     @classmethod
@@ -1073,7 +1046,6 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         from django.db import transaction
 
         def finish_snapshot_save():
-            self.reconcile_filesystem_links()
             crawl = Crawl.objects.filter(pk=self.crawl_id).first()
             if crawl is None:
                 return
@@ -1093,8 +1065,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             # Crawl.urls remains the original submitted source. Snapshot rows
             # are the normalized work queue and discovery projection.
 
-        # get_or_create/update_or_create wrap save() in atomic(); defer filesystem
-        # work and crawl maintenance so SQLite commits before touching the disk.
+        # get_or_create/update_or_create wrap save() in atomic(); attach inherited
+        # tags after that write commits. Filesystem projections belong to the runner.
         transaction.on_commit(finish_snapshot_save)
 
     # =========================================================================
@@ -1198,6 +1170,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
         target_dir = self.get_storage_path_for_version(target)
         if target_dir.exists():
+            self.reconcile_with_index(output_dir=target_dir, update_existing_archive_results=False)
             self.hydrate_archiveresult_output_metadata(snapshot_dir=target_dir)
         if cleanup:
             old_dir, new_dir = cleanup
@@ -1230,7 +1203,9 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
     def hydrate_archiveresult_output_metadata(self, snapshot_dir: Path | None = None) -> int:
         """Populate missing ArchiveResult file metadata from existing outputs."""
         hydrated = 0
-        for result in self.archiveresult_set.filter(output_files={}).iterator():
+        # Files may live on slow remote storage: finish the bounded read before
+        # inspecting outputs or writing their metadata back to the database.
+        for result in self.archiveresult_set.filter(output_files={}).order_by("pk").paged_iterator():
             hydrated += int(result.update_output_metadata_from_filesystem(snapshot_dir=snapshot_dir))
         return hydrated
 
@@ -3975,6 +3950,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
 
 class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
+    objects = PagedQuerySet.as_manager()
+
     class StatusChoices(models.TextChoices):
         QUEUED = "queued", "Queued"
         STARTED = "started", "Started"
@@ -4196,8 +4173,11 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
 
     @classmethod
     def missing_delete_at_candidates(cls):
-        return cls.objects.filter(delete_at__isnull=True).filter(
-            Q(snapshot__config__has_key="DELETE_AFTER") | Q(snapshot__crawl__config__has_key="DELETE_AFTER"),
+        return (
+            cls.objects.filter(delete_at__isnull=True)
+            .filter(Q(snapshot__config__has_key="DELETE_AFTER") | Q(snapshot__crawl__config__has_key="DELETE_AFTER"))
+            .select_related("snapshot__crawl")
+            .only("id", "created_at", "delete_at", "snapshot__config", "snapshot__crawl__config")
         )
 
     @property
@@ -4773,9 +4753,13 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
 
     @property
     def pwd(self) -> str:
-        """Working directory, derived from the snapshot/plugin path if the Process row is gone."""
+        """Working directory metadata without probing archive storage."""
         process = self.process_record
-        return process.pwd if process and process.pwd else str(self.output_dir)
+        return (
+            process.pwd
+            if process and process.pwd
+            else str(self.snapshot.get_storage_path_for_version(self.snapshot.fs_version) / self.plugin)
+        )
 
     @property
     def cmd(self) -> list:

archivebox/core/snapshot_status.py

@@ -6,7 +6,7 @@ from django.db.models import QuerySet
 def snapshot_status_values() -> tuple[str, ...]:
     from archivebox.core.models import Snapshot
 
-    return tuple(Snapshot.StatusChoices.values)
+    return tuple(value for value, _label in Snapshot._meta.get_field("status").choices)
 
 
 def normalize_snapshot_status(status: str | None) -> str | None:

archivebox/core/takeover_util.py

@@ -62,7 +62,7 @@ def runtime_stack_owner(*, data_dir: str | Path, exclude_id=None):
         # not keep stealing leadership from the next foreground command.
         base_qs.filter(process_type=Process.TypeChoices.ORCHESTRATOR).exclude(parent__process_type=Process.TypeChoices.SUPERVISORD),
     ):
-        for proc in qs.order_by("-created_at", "-modified_at").iterator(chunk_size=50):
+        for proc in qs.order_by("-created_at", "-modified_at").paged_iterator(chunk_size=50):
             if proc.is_running:
                 return proc
             proc.mark_exited(exit_code=proc.exit_code if proc.exit_code is not None else 0)
@@ -95,7 +95,7 @@ def foreground_runner_owner(*, data_dir: str | Path, exclude_id=None):
     )
     if exclude_id is not None:
         qs = qs.exclude(id=exclude_id)
-    for proc in qs.order_by("-created_at", "-modified_at").iterator(chunk_size=50):
+    for proc in qs.order_by("-created_at", "-modified_at").paged_iterator(chunk_size=50):
         if proc.is_running:
             return proc
         proc.mark_exited(exit_code=proc.exit_code if proc.exit_code is not None else 0)
@@ -178,7 +178,7 @@ def live_runner_processes(*, data_dir: str | Path):
     qs = qs.filter(machine=machine)
     live = []
     foreign_namespace_ids = []
-    for process in qs.order_by("started_at", "created_at").iterator(chunk_size=20):
+    for process in qs.order_by("started_at", "created_at").paged_iterator(chunk_size=20):
         if not process.shares_pid_namespace:
             foreign_namespace_ids.append(process.id)
             continue

archivebox/core/views.py

@@ -122,7 +122,7 @@ def _find_snapshot_by_ref(snapshot_ref: str) -> Snapshot | None:
     if len(lookup) == 12 and "-" not in lookup:
         # Resolve suffixes using the covering primary-key index before fetching
         # wide snapshot rows and joining their crawl/user data.
-        matches = list(Snapshot.objects.filter(id__endswith=lookup).order_by().values_list("id", flat=True))
+        matches = Snapshot.objects.filter(id__endswith=lookup).order_by().values_list("id", flat=True)
         return snapshots.filter(pk__in=matches).order_by("-created_at", "-downloaded_at").first()
 
     try:
@@ -506,7 +506,7 @@ class SnapshotView(View):
         def _resolve_snapshots_for_slug(slug: str):
             # full URLs go straight to the url-only path (fast, indexed)
             if "://" in slug:
-                return SnapshotView.find_snapshots_for_url(slug)
+                return SnapshotView.find_snapshots_for_url(slug, allow_fallback=False)
             # short uuid-shaped slugs (>=8 hex chars after stripping non-hex) try id matching first
             id_qs = SnapshotView.find_snapshots_for_id(slug)
             if id_qs.exists():
@@ -517,6 +517,12 @@ class SnapshotView(View):
         try:
             if "://" in path:
                 snapshot = snapshots.order_by("-bookmarked_at").first()
+                if snapshot is None:
+                    # Fetch the exact match once before trying legacy URL
+                    # fallbacks; an exists() probe followed by the same lookup
+                    # needlessly doubled reads on every normal redirect.
+                    snapshots = direct_snapshots_queryset(request, self.find_snapshots_for_url(path)).select_related("crawl__created_by")
+                    snapshot = snapshots.order_by("-bookmarked_at").first()
                 if snapshot is None:
                     raise Snapshot.DoesNotExist
             else:
@@ -1228,7 +1234,6 @@ class PublicIndexView(ListView):
     model = Snapshot
     ordering: ClassVar[list[str]] = ["-bookmarked_at", "-created_at"]
     paginator_class = AcceleratedPaginator
-    public_page_scan_chunk_size = 50
 
     def get_paginate_by(self, queryset):
         runtime_config = self.__dict__.get("runtime_config")
@@ -1256,31 +1261,31 @@ class PublicIndexView(ListView):
         )
 
     def _ordered_public_page_from_order_index(self, *, page_number: int, page_size: int) -> list[Snapshot]:
-        # One extra row determines Next without a full collection count. Cached
-        # totals can be stale while the runner adds/removes snapshots, so they
-        # must never decide whether the requested page exists.
-        target_count = page_number * page_size + 1
-        public_snapshots: list[Snapshot] = []
-        scanned = 0
-        chunk_size = max(self.public_page_scan_chunk_size, page_size + 1)
-        # Read in snapshot_public_order_idx order and filter visibility in small
-        # batches. Combining the permissions predicate with ORDER BY can make
-        # SQLite select the permissions index and sort the whole public archive
-        # before LIMIT. This favors the common early pages without a new index;
-        # deep pages or mostly private collections can still require more reads.
-        ordered_snapshots = (
-            Snapshot.objects.select_related("crawl__created_by").order_by(*self.ordering).only(*self._base_public_snapshot_fields())
-        )
-
-        while len(public_snapshots) < target_count:
-            chunk = list(ordered_snapshots[scanned : scanned + chunk_size])
-            if not chunk:
-                break
-            scanned += len(chunk)
-            public_snapshots.extend(snapshot for snapshot in chunk if snapshot.permissions == PERMISSIONS_PUBLIC)
-
+        # Walk the existing chronological index in bounded keyset pages,
+        # reading only identity/visibility until reaching the requested page.
+        # Repeated OFFSET batches used to revisit every earlier row AND join
+        # owners each time, making deep pages quadratic. Filtering visibility
+        # in SQL instead made SQLite sort the entire permissions index first.
+        # Each helper page closes its cursor before yielding; only the final
+        # displayed IDs load wide rows/owners. No new index or SQLite hint.
         start = (page_number - 1) * page_size
-        return public_snapshots[start:target_count]
+        visible_count = 0
+        page_ids = []
+        candidates = Snapshot.objects.only("id", "permissions").order_by(*self.ordering, "id")
+        for snapshot in candidates.paged_iterator(chunk_size=max(100, page_size + 1)):
+            if snapshot.permissions != PERMISSIONS_PUBLIC:
+                continue
+            if visible_count >= start:
+                page_ids.append(snapshot.pk)
+                if len(page_ids) > page_size:
+                    break
+            visible_count += 1
+        return list(
+            Snapshot.objects.filter(pk__in=page_ids, permissions=PERMISSIONS_PUBLIC)
+            .select_related("crawl__created_by")
+            .order_by(*self.ordering, "id")
+            .only(*self._base_public_snapshot_fields()),
+        )
 
     def paginate_queryset(self, queryset, page_size):
         if self.request.GET.get("q", default="").strip():

archivebox/crawls/admin.py

@@ -947,7 +947,7 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         updated = 0
         batch = []
         crawls_to_update = []
-        for crawl in queryset.only("id", "config", "permissions").iterator(chunk_size=500):
+        for crawl in queryset.only("id", "config", "permissions").order_by("pk").paged_iterator(chunk_size=500):
             old_permissions = crawl.permissions
             config = dict(crawl.config or {})
             config["PERMISSIONS"] = permissions

archivebox/crawls/models.py

@@ -23,6 +23,7 @@ from archivebox.config.common import rprint as print
 from archivebox.core.permissions import PERMISSIONS_VALUES, normalize_permissions
 
 from archivebox.base_models.models import (
+    PagedQuerySet,
     ModelWithUUID,
     ModelWithDeleteAfter,
     ModelWithOutputDir,
@@ -148,6 +149,8 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes):
 
 
 class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWithQueue):
+    objects = PagedQuerySet.as_manager()
+
     id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
@@ -229,7 +232,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         if paused and save and self.pk:
             from archivebox.core.models import Snapshot
 
-            for snapshot in self.snapshot_set.exclude(status__in=Snapshot.FINAL_STATES).iterator():
+            for snapshot in self.snapshot_set.exclude(status__in=Snapshot.FINAL_STATES).order_by("pk").paged_iterator():
                 snapshot.pause()
         return paused
 
@@ -301,7 +304,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
 
     @classmethod
     def missing_delete_at_candidates(cls):
-        return cls.objects.filter(delete_at__isnull=True, config__has_key="DELETE_AFTER")
+        return cls.objects.filter(delete_at__isnull=True, config__has_key="DELETE_AFTER").only("id", "created_at", "delete_at", "config")
 
     def save(self, *args, **kwargs):
         update_fields = kwargs.get("update_fields")
@@ -368,7 +371,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         batch = []
         updated = 0
         queryset = self.snapshot_set.filter(Q(permissions=old_permissions) | Q(permissions__isnull=True)).only("id", "config")
-        for snapshot in queryset.iterator(chunk_size=500):
+        for snapshot in queryset.order_by("pk").paged_iterator(chunk_size=500):
             config = dict(snapshot.config or {})
             config["PERMISSIONS"] = normalized_new_permissions
             snapshot.config = config
@@ -427,7 +430,11 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
                 tags_by_name = {tag.name: tag for tag in Tag.objects.filter(name__in=added_names)}
 
             tag_ids = [tag.pk for tag_name in added_names if (tag := tags_by_name.get(tag_name))]
-            snapshot_ids = Snapshot.objects.filter(crawl=self).values_list("id", flat=True).iterator(chunk_size=5000)
+            # Fetch each page before writing M2M rows. iterator(chunk_size=...)
+            # would keep one read cursor open across every bulk write.
+            snapshot_ids = (
+                snapshot.pk for snapshot in Snapshot.objects.filter(crawl=self).only("id").order_by("pk").paged_iterator(chunk_size=500)
+            )
             while True:
                 batch_snapshot_ids = list(islice(snapshot_ids, 5000))
                 if not batch_snapshot_ids:

archivebox/machine/models.py

@@ -20,7 +20,7 @@ from django.utils.functional import cached_property
 
 from archivebox.config import CONSTANTS
 from archivebox.config.common import rprint
-from archivebox.base_models.models import ModelWithDeleteAfter, ModelWithHealthStats, normalize_config_json_values
+from archivebox.base_models.models import PagedQuerySet, ModelWithDeleteAfter, ModelWithHealthStats, normalize_config_json_values
 from archivebox.workers.models import ModelWithQueue
 from .detect import get_host_guid, get_os_info, get_vm_info, get_host_network, get_host_stats
 
@@ -48,7 +48,6 @@ _CURRENT_PROCESS: Process | None = None
 MACHINE_RECHECK_INTERVAL = 7 * 24 * 60 * 60
 NETWORK_INTERFACE_RECHECK_INTERVAL = 1 * 60 * 60
 BINARY_RECHECK_INTERVAL = 1 * 30 * 60
-PROCESS_RECHECK_INTERVAL = 60  # Re-validate every 60 seconds
 PID_REUSE_WINDOW = timedelta(hours=24)  # Max age for considering a PID match valid
 PROCESS_TIMEOUT_GRACE = timedelta(seconds=30)  # Extra margin before force-cleaning timed-out RUNNING rows
 START_TIME_TOLERANCE = 5.0  # Seconds tolerance for start time matching
@@ -255,40 +254,40 @@ class Machine(ModelWithHealthStats):
                 **get_vm_info(),
                 stats=get_host_stats(),
             )
-        else:
-            if timezone.now() >= machine.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL):
-                for key, value in {
-                    "hostname": socket.gethostname(),
-                    **get_os_info(),
-                    **get_vm_info(),
-                    "stats": get_host_stats(),
-                }.items():
-                    setattr(machine, key, value)
-                machine.save(
-                    update_fields=[
-                        "hostname",
-                        "hw_in_docker",
-                        "hw_in_vm",
-                        "hw_manufacturer",
-                        "hw_product",
-                        "hw_uuid",
-                        "os_arch",
-                        "os_family",
-                        "os_platform",
-                        "os_release",
-                        "os_kernel",
-                        "stats",
-                        "modified_at",
-                    ],
-                )
-        machine = cls._sanitize_config(machine)
-        # Reconcile once, on the first Machine.current() call in this process.
+        needs_recheck = timezone.now() >= machine.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL)
+        # Reconcile before any save can mirror stale DB config over file edits.
         try:
             from archivebox.config.collection import sync_machine_and_file
 
             sync_machine_and_file(machine)
         except Exception:
             pass
+        if needs_recheck:
+            for key, value in {
+                "hostname": socket.gethostname(),
+                **get_os_info(),
+                **get_vm_info(),
+                "stats": get_host_stats(),
+            }.items():
+                setattr(machine, key, value)
+            machine.save(
+                update_fields=[
+                    "hostname",
+                    "hw_in_docker",
+                    "hw_in_vm",
+                    "hw_manufacturer",
+                    "hw_product",
+                    "hw_uuid",
+                    "os_arch",
+                    "os_family",
+                    "os_platform",
+                    "os_release",
+                    "os_kernel",
+                    "stats",
+                    "modified_at",
+                ],
+            )
+        machine = cls._sanitize_config(machine)
         # Publish only after initialization is complete.
         _CURRENT_MACHINE = machine
         return machine
@@ -890,7 +889,7 @@ class Binary(ModelWithHealthStats, ModelWithQueue):
 # =============================================================================
 
 
-class ProcessManager(models.Manager):
+class ProcessManager(models.Manager.from_queryset(PagedQuerySet)):
     """Manager for Process model."""
 
     def current(self) -> Process:
@@ -1209,8 +1208,11 @@ class Process(ModelWithDeleteAfter, models.Model):
 
     @classmethod
     def missing_delete_at_candidates(cls):
-        return cls.objects.filter(delete_at__isnull=True).filter(
-            Q(env__has_key="DELETE_AFTER") | Q(machine__config__has_key="DELETE_AFTER"),
+        return (
+            cls.objects.filter(delete_at__isnull=True)
+            .filter(Q(env__has_key="DELETE_AFTER") | Q(machine__config__has_key="DELETE_AFTER"))
+            .select_related("machine")
+            .only("id", "created_at", "delete_at", "env", "machine__config")
         )
 
     # Properties that delegate to related objects
@@ -1410,6 +1412,13 @@ class Process(ModelWithDeleteAfter, models.Model):
     # Process.current() and hierarchy methods
     # =========================================================================
 
+    @classmethod
+    def current_readonly(cls) -> Process | None:
+        """Return this process's registered identity without registration or probes."""
+        if _CURRENT_PROCESS is not None and _CURRENT_PROCESS.pid == os.getpid():
+            return _CURRENT_PROCESS
+        return None
+
     @classmethod
     def current(cls) -> Process:
         """
@@ -1427,40 +1436,19 @@ class Process(ModelWithDeleteAfter, models.Model):
 
         current_pid = os.getpid()
 
-        # Fast path used by model save diagnostics and hot runner loops. A
-        # cached Process object is valid when the immutable identity we wrote
-        # at creation time still describes this Python process. PID reuse cannot
-        # happen while this process is alive, so pid + present started_at/cmd is
-        # enough here; the slower psutil validation below remains the fallback
-        # for missing/stale cache.
-        if (
-            _CURRENT_PROCESS
-            and _CURRENT_PROCESS.pid == current_pid
-            and _CURRENT_PROCESS.status == cls.StatusChoices.RUNNING
-            and timezone.now() < _CURRENT_PROCESS.modified_at + timedelta(seconds=PROCESS_RECHECK_INTERVAL)
-            and _CURRENT_PROCESS.started_at is not None
-            and bool(_CURRENT_PROCESS.cmd)
-        ):
-            return _CURRENT_PROCESS
+        # This is process identity, not a health check. A live Python process
+        # cannot reuse its own PID, and a fork fails current_readonly's PID
+        # check. Expiring this cache by modified_at made every call after 60s
+        # query the database and touch logs until something else saved the row.
+        # Explicit lifecycle/recovery code owns liveness and interface refresh.
+        cached = cls.current_readonly()
+        if cached is not None:
+            return cached
+        _CURRENT_PROCESS = None
 
         machine = Machine.current()
         iface = NetworkInterface.current()
 
-        # Check cache validity
-        if _CURRENT_PROCESS:
-            # Verify: same PID, same machine, cache not expired
-            if (
-                _CURRENT_PROCESS.pid == current_pid
-                and _CURRENT_PROCESS.machine_id == machine.id
-                and timezone.now() < _CURRENT_PROCESS.modified_at + timedelta(seconds=PROCESS_RECHECK_INTERVAL)
-            ):
-                if _CURRENT_PROCESS.iface_id != iface.id:
-                    _CURRENT_PROCESS.iface = iface
-                    _CURRENT_PROCESS.save(update_fields=["iface", "modified_at"])
-                _CURRENT_PROCESS.ensure_log_files()
-                return _CURRENT_PROCESS
-            _CURRENT_PROCESS = None
-
         # Get actual process start time from OS for validation
         os_start_time = None
         if PSUTIL_AVAILABLE:
@@ -1640,9 +1628,8 @@ class Process(ModelWithDeleteAfter, models.Model):
         if machine is not None:
             stale = stale.filter(machine=machine)
 
-        # Recovery can run against damaged DB state; stream rows so a large
-        # stale Process backlog cannot be materialized in memory at once.
-        for proc in stale.iterator(chunk_size=100):
+        # Close each bounded read before inspecting processes or writing their state.
+        for proc in stale.order_by("pk").paged_iterator(chunk_size=100):
             shares_pid_namespace = proc.shares_pid_namespace
             if shares_pid_namespace and proc.poll() is not None:
                 cleaned += 1
@@ -2582,14 +2569,17 @@ class Process(ModelWithDeleteAfter, models.Model):
         """
         cleaned = 0
 
+        # Select active IDs through the existing status index first. Combining
+        # type/status directly let SQLite prefer the historical type index and
+        # scan 350k processes to return zero workers (blocking writes for 12s).
+        active_ids = cls.objects.filter(status=cls.StatusChoices.RUNNING).order_by().values("pk")
         running_children = cls.objects.filter(
+            pk__in=active_ids,
             process_type__in=[cls.TypeChoices.WORKER, cls.TypeChoices.HOOK],
-            status=cls.StatusChoices.RUNNING,
         )
 
-        # Recovery can run against damaged DB state; stream rows so a large
-        # orphaned Process backlog cannot be materialized in memory at once.
-        for proc in running_children.iterator(chunk_size=100):
+        # Close each bounded read before inspecting processes or writing their state.
+        for proc in running_children.order_by("pk").paged_iterator(chunk_size=100):
             if not proc.is_running:
                 proc.mark_exited(
                     exit_code=proc.exit_code if proc.exit_code is not None else _default_exit_code_for_unowned_process(proc.process_type),

archivebox/misc/checks.py

@@ -194,7 +194,10 @@ def check_not_root():
     if os.geteuid() == 0 and not (is_getting_help or is_getting_version):
         print("[yellow][!] Running ArchiveBox as root is not recommended.[/yellow]", file=sys.stderr)
         print("    Root-owned DATA_DIR files may be inaccessible to non-root users later.", file=sys.stderr)
-        print("        https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#do-not-run-as-root", file=sys.stderr)
+        print(
+            "        https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#run-archivebox-as-an-unprivileged-user",
+            file=sys.stderr,
+        )
 
 
 def is_archivebox_source_root(path: Path | str | None = None) -> bool:
@@ -255,7 +258,7 @@ def check_data_dir_permissions(config=None, **config_kwargs):
             "    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#permissions]https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#permissions[/link]",
         )
         STDERR.print(
-            "    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#puid--pgid]https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#puid--pgid[/link]",
+            "    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#docker-permissions-issues]https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#docker-permissions-issues[/link]",
         )
         STDERR.print(
             "    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#filesystem-doesnt-support-fsync-eg-network-mounts]https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#filesystem-doesnt-support-fsync-eg-network-mounts[/link]",

archivebox/misc/db.py

@@ -240,89 +240,38 @@ def drop_models_on_postgres(apps, schema_editor, app_label: str, model_names: li
             schema_editor.delete_model(model)
 
 
-def run_db_analyze_batch(
-    remaining: list[str] | None,
-    *,
-    max_seconds_per_table: float = 120.0,
-) -> list[str]:
-    """Advance one step of a batched SQLite ``ANALYZE`` sweep.
+def run_db_analyze_batch(remaining: list[str] | None) -> list[str]:
+    """Refresh approximate SQLite planner stats for one table per idle tick.
 
-    Without periodic ANALYZE the optimizer's table stats go stale as
-    snapshot/archiveresult tables grow, causing it to start large joins from
-    ``auth_user`` instead of using the indexed url column and blowing snapshot
-    detail page render time from ~50ms to ~500ms+.
-
-    The whole sweep is spread across many calls instead of running as one
-    blocking ``ANALYZE``: pass ``None`` to start a fresh sweep (this call
-    enumerates user tables and runs ``ANALYZE`` on the first one); pass the
-    returned list to advance one more table on each subsequent call. An
-    empty return value means the sweep is complete (or has been aborted) and
-    the next caller should pass ``None`` again. Caller is responsible for
-    throttling new sweeps (orchestrator starts at most one per 24hr while
-    idle) and enforcing a hard upper bound on total sweep wall time.
-
-    Safety guarantees:
-
-    - **Never raises**: every database call is wrapped; on any failure the
-      function returns ``[]`` (abandoning the rest of the sweep) so the
-      orchestrator never crashes on maintenance errors.
-    - **Bounded per-call wall time**: a SQLite progress handler aborts the
-      current ``ANALYZE`` statement once ``max_seconds_per_table`` is
-      exceeded, so a single pathological table cannot wedge the call.
-    - **Never leaves the db locked**: each ``ANALYZE`` runs as a single
-      statement transaction that auto-commits (or rolls back on
-      abort/error). The cursor and progress handler are always cleaned up
-      in ``finally`` blocks even if Python raises mid-call.
-    - Silent no-op on non-SQLite backends.
-
-    WAL journal mode (set in Django settings) keeps readers fully unblocked
-    throughout; the writer lock is only held for the brief ``sqlite_stat*``
-    flush after each table completes.
+    A full ANALYZE scans every index while holding a write transaction; moving
+    it into the runner did not prevent it from blocking submissions. Sampling
+    at most about 1000 entries per index keeps maintenance independent of
+    collection size. Approximate stats are sufficient for choosing indexes.
+    PostgreSQL owns its statistics through autovacuum/autoanalyze.
     """
     from django.db import connection
 
     if connection.vendor != "sqlite":
         return []
-
-    if remaining is None:
-        try:
-            with connection.cursor() as cursor:
-                cursor.execute(
-                    "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name",
-                )
-                remaining = [row[0] for row in cursor.fetchall()]
-        except Exception:
-            return []
-
-    if not remaining:
-        return []
-
-    next_table, *rest = remaining
-    raw_conn = connection.connection
-    progress_handler_set = False
-    if raw_conn is not None and max_seconds_per_table > 0:
-        deadline = time.monotonic() + max_seconds_per_table
-        try:
-            raw_conn.set_progress_handler(lambda: 1 if time.monotonic() > deadline else 0, 10000)
-            progress_handler_set = True
-        except Exception:
-            progress_handler_set = False
-
     try:
         with connection.cursor() as cursor:
-            cursor.execute(f'ANALYZE "{next_table}"')
-    except Exception:
-        # Aborted by progress handler, locked db, or any other failure — skip
-        # this table and continue the sweep. ANALYZE is idempotent so we can
-        # retry on the next 24hr sweep.
-        pass
-    finally:
-        if progress_handler_set and raw_conn is not None:
+            if remaining is None:
+                cursor.execute("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name")
+                remaining = [row[0] for row in cursor.fetchall()]
+            if not remaining:
+                return []
+            next_table, *rest = remaining
+            cursor.execute("PRAGMA analysis_limit")
+            previous_limit = int(cursor.fetchone()[0])
             try:
-                raw_conn.set_progress_handler(None, 0)
-            except Exception:
-                pass
-    return rest
+                cursor.execute("PRAGMA analysis_limit = 1000")
+                cursor.execute(f"ANALYZE {connection.ops.quote_name(next_table)}")
+            finally:
+                cursor.execute(f"PRAGMA analysis_limit = {previous_limit}")
+            return rest
+    except Exception:
+        # Optional planner maintenance must not prevent queued captures.
+        return []
 
 
 def compact_command(cmdline: list[str] | None, fallback: str = "") -> str:

archivebox/misc/jsonl.py

@@ -142,3 +142,27 @@ def write_record(record: dict[str, Any], stream: TextIO | None = None) -> None:
     active_stream: TextIO = sys.stdout if stream is None else stream
     active_stream.write(json.dumps(record) + "\n")
     active_stream.flush()
+
+
+def write_records(records: Iterable[dict[str, Any]], stream: TextIO | None = None) -> int:
+    """Stream bulk JSONL output with bounded buffering, returning its row count."""
+    active_stream: TextIO = sys.stdout if stream is None else stream
+    lines: list[str] = []
+    pending_size = 0
+    count = 0
+    for record in records:
+        line = json.dumps(record) + "\n"
+        lines.append(line)
+        pending_size += len(line)
+        count += 1
+        # CLI stdout can be unbuffered. A syscall per snapshot makes large
+        # exports IO-bound; small batches keep memory and pipe latency bounded.
+        if pending_size >= 64 * 1024:
+            active_stream.write("".join(lines))
+            active_stream.flush()
+            lines.clear()
+            pending_size = 0
+    if lines:
+        active_stream.write("".join(lines))
+        active_stream.flush()
+    return count

archivebox/misc/shell_welcome_message.py

@@ -48,10 +48,10 @@ if __name__ == "__main__":
 
     prnt("[i] :heavy_dollar_sign: Welcome to the ArchiveBox Shell!")
     prnt(
-        "    [deep_sky_blue4]Docs:[/deep_sky_blue4] [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Shell-Usage]https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Shell-Usage[/link]",
+        "    [deep_sky_blue4]Docs:[/deep_sky_blue4] [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage]https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage[/link]",
     )
     prnt(
-        "          [link=https://docs.archivebox.io/en/dev/apidocs/archivebox/archivebox.html]https://docs.archivebox.io/en/dev/apidocs/archivebox/archivebox.html[/link]",
+        "          [link=https://docs.archivebox.io/dev/apidocs/archivebox/archivebox.html]https://docs.archivebox.io/dev/apidocs/archivebox/archivebox.html[/link]",
     )
     prnt()
     prnt(" :grey_question: [violet]Hint[/] [i]Here are some examples to get started:[/]")

archivebox/personas/forms.py

@@ -122,14 +122,14 @@ class PersonaAdminForm(PluginConfigFormMixin, forms.ModelForm):
                 "Use the custom path/CDP option if the browser data lives elsewhere."
             )
 
-        self.build_plugin_groups(get_config(persona=self.instance) if self.instance and self.instance.pk else get_config())
+        self.build_plugin_groups(get_config(persona=self.instance) if not self.instance._state.adding else get_config())
 
     def clean_name(self) -> str:
         name = str(self.cleaned_data.get("name") or "").strip()
         is_valid, error_message = validate_persona_name(name)
         if not is_valid:
             raise forms.ValidationError(error_message)
-        return name
+        return self.instance.validate_name(name)
 
     def clean(self) -> dict[str, Any]:
         cleaned_data = super().clean()

archivebox/personas/models.py

@@ -18,6 +18,7 @@ from typing import TYPE_CHECKING, Any
 from collections.abc import Mapping
 
 from django.db import IntegrityError, models
+from django.core.exceptions import ValidationError
 from django.db.models.fields.json import KT
 from django.conf import settings
 from django.utils import timezone
@@ -74,8 +75,12 @@ def derive_persona_config(*, name: str, config: Mapping[str, Any] | None, person
         derived["COOKIES_FILE"] = str(cookies_path)
 
     auth_path = persona_dir / "auth.json"
-    if "AUTH_STORAGE_FILE" not in derived and auth_path.exists():
-        derived["AUTH_STORAGE_FILE"] = str(auth_path)
+    if "AUTH_STORAGE_FILE" not in derived:
+        try:
+            if auth_path.stat().st_size > 0:
+                derived["AUTH_STORAGE_FILE"] = str(auth_path)
+        except FileNotFoundError:
+            pass
 
     derived["ACTIVE_PERSONA"] = name
     return derived
@@ -117,6 +122,13 @@ class Persona(ModelWithConfig):
         app_label = "personas"
 
     def save(self, *args, **kwargs):
+        # Adopt the spelling on disk, never rename a Chrome profile just to fix case.
+        self.__dict__.pop("_persona_path", None)
+        canonical_name = self.validate_name(self.name)
+        if canonical_name != self.name:
+            self.name = canonical_name
+            if kwargs.get("update_fields") is not None:
+                kwargs["update_fields"] = tuple(dict.fromkeys([*kwargs["update_fields"], "name"]))
         config = dict(self.config or {})
         if str(config.get("PERMISSIONS") or "").strip().lower() not in PERMISSIONS_VALUES:
             from archivebox.config.common import get_config
@@ -133,10 +145,59 @@ class Persona(ModelWithConfig):
 
     @property
     def path(self) -> Path:
-        """Path to persona directory under PERSONAS_DIR."""
+        """Reuse the directory's spelling without renaming a Chrome profile."""
         from archivebox.config.constants import CONSTANTS
 
-        return CONSTANTS.PERSONAS_DIR / self.name
+        cached = self.__dict__.get("_persona_path")
+        if cached is None or cached[0] != self.name:
+            directory_name = self.directory_name(self.name)
+            cached = (self.name, CONSTANTS.PERSONAS_DIR / directory_name)
+            self.__dict__["_persona_path"] = cached
+        return cached[1]
+
+    @staticmethod
+    def directory_name(name: str) -> str:
+        from archivebox.config.constants import CONSTANTS
+        from archivebox.personas.importers import validate_persona_name
+
+        valid, error = validate_persona_name(name)
+        if not valid:
+            raise ValidationError(error)
+        # Linux can hold both Default/default; macOS usually aliases them. Reuse a
+        # single match on either filesystem, but never guess which conflicting
+        # profile contains the user's login state. Only list names, not profile trees.
+        try:
+            with os.scandir(CONSTANTS.PERSONAS_DIR) as entries:
+                matches = sorted(entry.name for entry in entries if entry.name.casefold() == name.casefold())
+        except FileNotFoundError:
+            matches = []
+        if len(matches) > 1:
+            raise ValidationError(
+                f"Conflicting persona directories: {'; '.join(str(CONSTANTS.PERSONAS_DIR / match) for match in matches)}. "
+                "Stop Chrome and rename the conflicting directories manually to distinct names; no profile directories were renamed.",
+            )
+        return matches[0] if matches else name
+
+    @classmethod
+    def find_named(cls, name: str) -> "Persona | None":
+        # SQLite's iexact is ASCII-only; Python casefold gives every entry point
+        # the same Unicode matching rules as the filesystem-name lookup above.
+        matches = [(pk, existing) for pk, existing in cls.objects.values_list("pk", "name") if existing.casefold() == name.casefold()]
+        if len(matches) > 1:
+            raise ValidationError(
+                f"Conflicting persona records: {'; '.join(f'{existing!r} (ID {pk})' for pk, existing in matches)}. "
+                "Rename the conflicting personas manually to distinct names; profile files have not been changed.",
+            )
+        return cls.objects.get(pk=matches[0][0]) if matches else None
+
+    def validate_name(self, name: str) -> str:
+        existing = self.find_named(name)
+        if existing is not None and existing.pk != self.pk:
+            raise ValidationError(
+                f"Name conflicts with persona {existing.name!r} (ID {existing.pk}, directory {existing.path}). "
+                "Choose a distinct name or rename that persona record manually.",
+            )
+        return self.directory_name(name)
 
     @property
     def CHROME_USER_DATA_DIR(self) -> str:
@@ -156,9 +217,12 @@ class Persona(ModelWithConfig):
 
     @property
     def AUTH_STORAGE_FILE(self) -> str:
-        """Derived path to auth.json for this persona (if it exists)."""
+        """Derived path to a nonempty auth.json export for this persona."""
         auth_path = self.path / "auth.json"
-        return str(auth_path) if auth_path.exists() else ""
+        try:
+            return str(auth_path) if auth_path.stat().st_size > 0 else ""
+        except FileNotFoundError:
+            return ""
 
     def get_derived_config(self) -> dict:
         """
@@ -171,10 +235,13 @@ class Persona(ModelWithConfig):
         - AUTH_STORAGE_FILE (derived from persona path, if file exists)
         - ACTIVE_PERSONA (set to this persona's name)
         """
-        return derive_persona_config(name=self.name, config=self.config, persona_dir=self.path)
+        return derive_persona_config(name=self.path.name, config=self.config, persona_dir=self.path)
 
     def ensure_dirs(self) -> None:
         """Create persona directories if they don't exist."""
+        # Recheck at profile preparation, including changes made outside this process.
+        self.__dict__.pop("_persona_path", None)
+        self.validate_name(self.name)
         self.path.mkdir(parents=True, exist_ok=True)
         (self.path / "chrome_profile").mkdir(parents=True, exist_ok=True)
         (self.path / "chrome_downloads").mkdir(parents=True, exist_ok=True)
@@ -246,16 +313,26 @@ class Persona(ModelWithConfig):
                     fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
 
     @classmethod
-    def get_or_create_named(cls, name: str) -> "Persona":
+    def get_or_create_named(cls, name: str, *, defaults: dict | None = None) -> "Persona":
+        # Init/startup must reuse a user-created 'default', not create a second
+        # DB row pointing at a different (or shared) profile (issue #1896).
         persona_name = (name or "Default").strip() or "Default"
-        persona = cls.objects.filter(name=persona_name).first()
+        persona = cls.find_named(persona_name)
         if persona is not None:
+            canonical_name = persona.validate_name(persona.name)
+            if canonical_name != persona.name:
+                persona.name = canonical_name
+                persona.save(update_fields=["name"])
             return persona
 
         try:
-            return cls.objects.create(name=persona_name)
-        except IntegrityError:
-            return cls.objects.get(name=persona_name)
+            return cls.objects.create(name=persona_name, **(defaults or {}))
+        except (IntegrityError, ValidationError):
+            persona = cls.find_named(persona_name)
+            if persona is None:
+                raise
+            persona.validate_name(persona.name)
+            return persona
 
     def runtime_root_for_crawl(self, crawl) -> Path:
         return Path(crawl.output_dir) / ".persona" / self.name
@@ -315,20 +392,9 @@ class Persona(ModelWithConfig):
             if chrome_binary:
                 (runtime_root / "chrome_binary.txt").write_text(chrome_binary)
 
-        return {
-            # Hooks derive CHROME_USER_DATA_DIR/CHROME_DOWNLOADS_DIR from
-            # PERSONAS_DIR + ACTIVE_PERSONA. Point PERSONAS_DIR at the
-            # per-crawl runtime root here so CHROME_ISOLATION=crawl never
-            # leaks or reuses the template profile while keeping Chrome path
-            # derivation centralized in the Chrome plugin helpers.
-            "PERSONAS_DIR": str(runtime_root.parent),
-            "ACTIVE_PERSONA": self.name,
-            **{
-                key: str(runtime_root / filename)
-                for key, filename in (("COOKIES_FILE", "cookies.txt"), ("AUTH_STORAGE_FILE", "auth.json"))
-                if (runtime_root / filename).is_file()
-            },
-        }
+        # Hooks derive Chrome profile paths from the runtime persona root.
+        # Use the same optional auth discovery as the template persona.
+        return derive_persona_config(name=self.name, config={}, persona_dir=runtime_root)
 
     def prepare_runtime_for_snapshot(self, snapshot, chrome_binary: str = "") -> dict[str, str]:
         crawl_runtime_profile_dir = self.runtime_profile_dir_for_crawl(snapshot.crawl)
@@ -358,18 +424,7 @@ class Persona(ModelWithConfig):
         if chrome_binary:
             (runtime_root / "chrome_binary.txt").write_text(chrome_binary)
 
-        return {
-            # See prepare_runtime_for_crawl(): snapshot isolation changes the
-            # persona root, not individual CHROME_* config keys, so standalone
-            # Chrome hooks and ArchiveBox-driven hooks resolve paths the same way.
-            "PERSONAS_DIR": str(runtime_root.parent),
-            "ACTIVE_PERSONA": self.name,
-            **{
-                key: str(runtime_root / filename)
-                for key, filename in (("COOKIES_FILE", "cookies.txt"), ("AUTH_STORAGE_FILE", "auth.json"))
-                if (runtime_root / filename).is_file()
-            },
-        }
+        return derive_persona_config(name=self.name, config={}, persona_dir=runtime_root)
 
     def cleanup_runtime_for_crawl(self, crawl) -> None:
         shutil.rmtree(Path(crawl.output_dir) / ".persona", ignore_errors=True)

archivebox/services/archive_result_service.py

@@ -210,6 +210,12 @@ def _save_archiveresult_event_to_db(
 
     with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.diff_fields"):
         update_fields = []
+        # This runner already loaded the Snapshot/Crawl for the event. Resolve
+        # retention here without extra parent reads so new results do not wait
+        # for the bounded legacy-repair sweep to traverse historical rows.
+        result.snapshot = snapshot
+        if result.set_delete_at_from_config():
+            update_fields.append("delete_at")
         for field, value in defaults.items():
             if result.__dict__[field] != value:
                 setattr(result, field, value)

archivebox/services/runner.py

@@ -65,6 +65,7 @@ from archivebox.services.resource_admission import RESOURCE_RECHECK_SECONDS, res
 from archivebox.config.common import (
     ArchiveBoxBaseConfig,
     normalize_runtime_config,
+    _is_cookie_file_config_key,
     _plugin_enabled_config_keys,
 )
 from archivebox.misc.db import run_db_analyze_batch
@@ -738,8 +739,16 @@ class CrawlRunner:
             current_process.save(update_fields=["iface", "machine", "modified_at"])
         self.persona = self.crawl.resolve_persona()
         self.base_config = get_config(crawl=self.crawl, overrides=self.config_overrides)
-        self.derived_config = dict(machine.config or {})
+        # The merged user payload owns auth paths and binary selections. The
+        # install phase publishes resolved binaries; stale machine paths must
+        # not override explicit runtime settings before that phase runs.
+        self.derived_config = {
+            key: value
+            for key, value in (machine.config or {}).items()
+            if not _is_cookie_file_config_key(key) and (not key.endswith("_BINARY") or value == self.base_config.get(key))
+        }
         self.crawl_output_dir = str(self.crawl.output_dir)
+        Path(self.crawl_output_dir).mkdir(parents=True, exist_ok=True)
         if self.persona:
             self.base_config.update(
                 self.persona.prepare_runtime_for_crawl(
@@ -948,6 +957,7 @@ class CrawlRunner:
                     },
                 )
         snapshot_output_dir = str(snapshot.output_dir)
+        Path(snapshot_output_dir).mkdir(parents=True, exist_ok=True)
         tags = snapshot.tags_str()
         config = self.base_config.for_crawl_runtime(
             crawl=snapshot.crawl,
@@ -1245,6 +1255,7 @@ class CrawlRunner:
             user_config_event = MachineEvent(config=config, config_type="user")
             user_config_event.event_parent_id = crawl_start_event.event_id
             await self.bus.emit(user_config_event).now()
+            derived_config_event = None
             if derived_config:
                 derived_config_event = MachineEvent(config=derived_config, config_type="derived")
                 derived_config_event.event_parent_id = crawl_start_event.event_id
@@ -1260,6 +1271,7 @@ class CrawlRunner:
                 snapshot_cleanup_phase_timeout=snapshot_phase_timeout,
                 abort_requested=self.crawl_is_cancelled,
             )
+            projected = False
             try:
                 snapshot_event = SnapshotEvent(
                     url=snapshot["url"],
@@ -1283,6 +1295,7 @@ class CrawlRunner:
                     raise RuntimeError(f"Snapshot {snapshot_id} did not complete")
                 await completed_snapshot.wait(timeout=snapshot_phase_timeout)
                 await completed_snapshot.event_results_list()
+                projected = True
                 if self.execution_interrupted():
                     return
                 if snapshot["status"] == "sealed":
@@ -1306,6 +1319,16 @@ class CrawlRunner:
                 await sync_to_async(_seal_when_last_snapshot_finished, thread_sensitive=True)()
             finally:
                 snapshot_service.close()
+                # Projection and snapshot cleanup have finished. Historical
+                # capture results now live in the database; only active work
+                # needs its event ancestry available to the downloader.
+                expired = [snapshot_event, user_config_event] if projected else []
+                if projected and derived_config_event is not None:
+                    expired.append(derived_config_event)
+                if projected:
+                    expired.extend(await self.bus.filter("*", child_of=snapshot_event, past=True, future=False))
+                for event in expired:
+                    event.event_ttl = 0
 
     def seal_snapshot_due_to_limit(self, snapshot_id: str) -> None:
         from archivebox.core.models import Snapshot
@@ -1670,7 +1693,9 @@ def _run_due_snapshot_locked(snapshot, *, lock_seconds: int, interactive_interru
         # every URL, even though crawl isolation owns one browser for the whole
         # crawl. The crawl lifecycle lock already serializes all of its child
         # work, so let one runner drain the due siblings and discoveries.
-        if any(process.is_running for process in snapshot.process_set.filter(status="running").iterator()):
+        if any(
+            process.is_running for process in snapshot.process_set.filter(status="running").order_by("pk").paged_iterator(chunk_size=100)
+        ):
             snapshot.update_and_requeue(retry_at=timezone.now() + timedelta(seconds=lock_seconds))
             return True
         if snapshot.fs_migration_needed:
@@ -1704,7 +1729,7 @@ def _run_due_snapshot_locked(snapshot, *, lock_seconds: int, interactive_interru
     if not snapshot.claim_processing_lock(lock_seconds=lock_seconds):
         return False
     snapshot.refresh_from_db()
-    if any(process.is_running for process in snapshot.process_set.filter(status="running").iterator()):
+    if any(process.is_running for process in snapshot.process_set.filter(status="running").order_by("pk").paged_iterator(chunk_size=100)):
         # The Snapshot lease may have expired while an abx-dl hook process is
         # still alive. Preserve the snapshot-level ownership boundary and do
         # not launch a second sequence; ArchiveResult status is irrelevant.
@@ -1981,14 +2006,36 @@ def run_pending_crawls(
     last_recovery_at = 0.0
     last_retention_at = 0.0
     last_retention_repair_at = 0.0
+    retention_backfill_cursors: dict = {}
+    deletion_thread: threading.Thread | None = None
     last_analyze_at = 0.0
     analyze_queue: list[str] | None = None
     analyze_sweep_started_at = 0.0
     orchestrator_started_at = time.monotonic()
+
+    def delete_in_background() -> None:
+        from django.db import connections
+
+        try:
+            Snapshot.delete_requested(batch_size=100)
+        finally:
+            connections.close_all()
+
     while True:
         raise_if_shutdown_requested()
         now_monotonic = time.monotonic()
         if crawl_id is None and now_monotonic - last_retention_at >= (60.0 if daemon else 1.0):
+            if daemon:
+                # At most one batch can be doing filesystem I/O. A stuck mount
+                # must not block capture scheduling or runner shutdown; rows
+                # remain pending until cleanup succeeds, including on restart.
+                if (deletion_thread is None or not deletion_thread.is_alive()) and Snapshot.objects.filter(
+                    status=Snapshot.DELETING_STATE,
+                ).exists():
+                    deletion_thread = threading.Thread(target=delete_in_background, name="snapshot-delete", daemon=True)
+                    deletion_thread.start()
+            else:
+                Snapshot.delete_requested(batch_size=100)
             for model in (ArchiveResult, Snapshot, Crawl, Process):
                 # Keep the tight scheduler loop anchored on indexed delete_at
                 # columns only. Backfilling missing delete_at values has to read
@@ -2156,7 +2203,7 @@ def run_pending_crawls(
                 # delete_at in the plugin-result hot path. Running it here keeps
                 # DELETE_AFTER resolution fresh without making every hook event
                 # load parent Snapshot/Crawl config.
-                model.delete_expired(batch_size=100, backfill_missing=True)
+                model.delete_expired(batch_size=100, backfill_missing=True, backfill_cursors=retention_backfill_cursors)
             last_retention_repair_at = now_monotonic
 
         if daemon:
@@ -2172,12 +2219,10 @@ def run_pending_crawls(
             # snapshot detail page out to ~500ms. Refresh stats at most once per
             # 24hr while the queue is idle, and only after the orchestrator has
             # been alive for at least an hour so short server boots / one-off work
-            # never pay the cost. The sweep is batched one table per idle tick;
-            # individual table ANALYZE statements abort after 2min (progress
-            # handler) and the whole sweep is hard-capped at 5min so a
-            # pathological table cannot wedge maintenance forever. Any failure
-            # inside the maintenance hook is swallowed — orchestrator must never
-            # be taken down by stats refresh.
+            # never pay the cost. Each idle tick samples a bounded number of
+            # index entries for one table; never scan the full history while
+            # holding ANALYZE's write transaction. Optional stats refresh must
+            # not prevent queued captures.
             try:
                 if (
                     analyze_queue is None

archivebox/services/snapshot_service.py

@@ -146,7 +146,7 @@ class SnapshotService(BaseService):
             elif snapshot.status not in (Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.SEALED):
                 return
             if snapshot.status == Snapshot.StatusChoices.STARTED:
-                await sync_to_async(snapshot.ensure_crawl_symlink, thread_sensitive=True)()
+                await sync_to_async(snapshot.reconcile_filesystem_links, thread_sensitive=True)()
             retry_plugins = [str(name).strip() for name in (snapshot.config or {}).get("RETRY_PLUGINS", []) if str(name).strip()]
             async with self._ownership_lock:
                 self._run_ownership[str(event.snapshot_id)] = (str(event.event_id), snapshot.retry_at, was_sealed, retry_plugins)

archivebox/templates/core/setup_wizard.html

@@ -35,7 +35,7 @@
       <div class="abx-question-options" id="archivebox-setup-dns-options">
         <label class="abx-question-option"><input type="radio" name="archivebox-dns-mode" value="localhost"> <span><b>⭐ Localhost</b><small>No setup needed. <code id="archivebox-setup-localhost-dns-example">*.archivebox.localhost</code> works directly in modern browsers with all features.</small></span></label>
         <label class="abx-question-option"><input type="radio" name="archivebox-dns-mode" value="single"> <span><b>⚠️ Single-domain DNS</b><small>Register a DNS record like <code>A archivebox.example.com</code> → <code>your server's IP</code>, or add an equivalent <code>/etc/hosts</code> entry pointing to your server. Some archived content that depends on JavaScript may not display properly in this mode. JavaScript still runs during capture, but viewing HTML captured by <code>wget</code>, <code>dom</code>, <code>responses</code>, <code>staticfile</code>, and similar plugins will not replay JavaScript unless wildcard DNS is used.</small></span></label>
-        <label class="abx-question-option"><input type="radio" name="archivebox-dns-mode" value="wildcard"> <span><b>⭐ Wildcard DNS</b><small>Point both the bare <code>BASE_URL</code> hostname and <code>*.BASE_URL</code> to this server using A/AAAA records or CNAME records. Ideal: safest isolation with all replay features. <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Modes" target="_blank" rel="noopener noreferrer">Setup instructions</a>.</small></span></label>
+        <label class="abx-question-option"><input type="radio" name="archivebox-dns-mode" value="wildcard"> <span><b>⭐ Wildcard DNS</b><small>Point both the bare <code>BASE_URL</code> hostname and <code>*.BASE_URL</code> to this server using A/AAAA records or CNAME records. Ideal: safest isolation with all replay features. <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#server_security_mode" target="_blank" rel="noopener noreferrer">Setup instructions</a>.</small></span></label>
       </div>
       <div class="abx-question-status" id="archivebox-setup-dns-status">Choose a DNS mode to begin testing.</div>
     </fieldset>

archivebox/templates/core/system_warnings_banner.html

@@ -27,7 +27,7 @@
       Only <code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">{{ free_gb }} GiB</code>
       free on <code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">DATA_DIR</code>
       — new archive jobs will fail until space is reclaimed.
-      <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Disk-Usage" target="_blank" rel="noopener noreferrer" style="color:#fff;text-decoration:underline;margin-left:6px;">docs</a>
+      <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#purging-entries" target="_blank" rel="noopener noreferrer" style="color:#fff;text-decoration:underline;margin-left:6px;">docs</a>
     {% elif mode == "high_memory" %}
       Virtual memory at <code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">{{ mem_pct }}%</code>
       — the host is close to OOM. Consider stopping crawls or scaling up.
@@ -35,7 +35,7 @@
       15-min loadavg <code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">{{ load_15 }}</code>
     {% elif mode == "unsafe" %}
       ArchiveBox single-domain mode — archived pages share an origin with this site
-      <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Modes" target="_blank" rel="noopener noreferrer" style="color:#fff;text-decoration:underline;margin-left:6px;">why?</a>
+      <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#server_security_mode" target="_blank" rel="noopener noreferrer" style="color:#fff;text-decoration:underline;margin-left:6px;">why?</a>
     {% elif mode == "base_url_mismatch" %}
       Editing disabled from unrecognized URL. Visit
       <a href="{{ configured_base_url }}" style="color:#fff;text-decoration:underline;text-transform:none;">{{ configured_base_url }}</a>

archivebox/tests/conftest.py

@@ -32,6 +32,13 @@ PYTEST_BASETEMP_ROOT = (REPO_ROOT / "tests" / "out").resolve()
 SESSION_DATA_DIR = Path(
     os.environ.get("ARCHIVEBOX_PYTEST_SESSION_DATA_DIR") or tempfile.mkdtemp(prefix="archivebox-pytest-session-"),
 ).resolve()
+if worker_id := os.environ.get("PYTEST_XDIST_WORKER"):
+    # Workers inherit the coordinator's environment. Give each one its own
+    # collection and cleanup root so a finishing worker cannot delete another's.
+    SESSION_DATA_DIR /= worker_id
+    for runtime_key in ("ABXPKG_LIB_DIR", "PERSONAS_DIR", "CHROME_USER_DATA_DIR"):
+        if runtime_path := os.environ.get(runtime_key):
+            os.environ[runtime_key] = str(Path(runtime_path) / worker_id)
 
 os.environ["ARCHIVEBOX_PYTEST_SESSION_DATA_DIR"] = str(SESSION_DATA_DIR)
 os.environ["DATA_DIR"] = str(SESSION_DATA_DIR)
@@ -323,6 +330,18 @@ def pytest_configure():
     if not apps.ready:
         django.setup()
 
+    if os.environ.get("PYTEST_XDIST_WORKER"):
+        from django.conf import settings
+        from archivebox.config import django as django_config
+        from archivebox.config.common import get_config
+
+        # pytest-django may import startup config before this conftest isolates
+        # the worker's paths. Refresh that real startup config once so HTTP
+        # requests resolve the same installed binaries as worker subprocesses;
+        # otherwise a service restart searches the coordinator's empty lib dir.
+        django_config.CONFIG = get_config(include_machine=False)
+        settings.CONFIG = django_config.CONFIG
+
 
 @pytest.fixture(scope="session")
 def django_db_modify_db_settings(
@@ -437,7 +456,7 @@ def hermetic_lib_dir(tmp_path):
 @pytest.fixture
 def cached_abxpkg_lib_dir():
     """Reuse one real abxpkg installation cache for the current pytest session."""
-    lib_dir = SESSION_DATA_DIR / "lib"
+    lib_dir = Path(os.environ["ABXPKG_LIB_DIR"]) if os.environ.get("ABXPKG_LIB_DIR") else SESSION_DATA_DIR / "lib"
     lib_dir.mkdir(parents=True, exist_ok=True)
     original_lib_dir = os.environ.get("ABXPKG_LIB_DIR")
     os.environ["ABXPKG_LIB_DIR"] = str(lib_dir)

archivebox/tests/test_api_discovery_performance.py

@@ -0,0 +1,54 @@
+"""Server discovery must not repeat schema generation or probe installed tools."""
+
+import sys
+import threading
+import os
+from pathlib import Path
+
+import pytest
+from django.test.utils import override_script_prefix
+
+from archivebox.config import VERSION
+from archivebox.machine.models import Machine
+from archivebox.tests.conftest import install_real_binary, resolve_abxpkg_binary_env
+
+
+@pytest.mark.django_db(transaction=True)
+@pytest.mark.parametrize("script_name", ["", "/mounted"])
+def test_repeated_discovery_reuses_schema_without_binary_filesystem_probes(client, hermetic_lib_dir, script_name):
+    install_real_binary("node", binproviders="env,apt,brew")
+    resolve_abxpkg_binary_env(hermetic_lib_dir, "node")
+    machine = Machine.current()
+    machine.config = {**machine.config, "NODE_BINARY": str(hermetic_lib_dir / "env" / "bin" / "node")}
+    machine.save(update_fields=["config"])
+    request = {"HTTP_HOST": "api.archivebox.localhost:5797", "SCRIPT_NAME": script_name}
+    with override_script_prefix(script_name):
+        first = client.get("/api/v1/openapi.json", **request)
+    assert first.status_code == 200, first.content
+    schema_calls = []
+    filesystem_calls = []
+
+    def observe(frame, event, arg):
+        if event == "call" and frame.f_code.co_name == "model_json_schema":
+            schema_calls.append(frame.f_code.co_filename)
+        if (event == "call" and frame.f_code.co_name in {"stat", "lstat", "open", "iterdir", "glob", "rglob"}) or (
+            event == "c_call" and arg in (os.stat, os.lstat)
+        ):
+            path = frame.f_locals.get("self", frame.f_locals.get("path"))
+            if isinstance(path, (str, Path)) and Path(path).is_relative_to(hermetic_lib_dir):
+                filesystem_calls.append(str(path))
+
+    previous = sys.getprofile()
+    previous_thread = threading.getprofile()
+    threading.setprofile_all_threads(observe)
+    try:
+        with override_script_prefix(script_name):
+            second = client.get("/api/v1/openapi.json", **request)
+    finally:
+        threading.setprofile_all_threads(previous_thread)
+        sys.setprofile(previous)
+    assert second.status_code == 200, second.content
+    assert second.json() == first.json()
+    assert second.json()["info"]["version"] == VERSION
+    assert f"{script_name}/api/v1/core/snapshots" in second.json()["paths"]
+    assert (len(schema_calls), filesystem_calls) == (0, [])

archivebox/tests/test_api_v1_cli_add.py

@@ -478,3 +478,40 @@ def test_api_cli_add_rejects_file_path_and_shell_injection_payloads(tmp_path):
     with use_archivebox_db(tmp_path):
         tag_names = set(SnapshotTag.objects.filter(snapshot=snapshot).values_list("tag__name", flat=True))
     assert "api-security" in tag_names
+
+
+def test_api_add_only_queues_work_without_storage_or_worker_control(client, api_headers):
+    import sys
+    from archivebox.config import CONSTANTS
+
+    calls = []
+
+    def observe(frame, event, arg):
+        if event != "call":
+            return
+        name = frame.f_code.co_name
+        if name == "ensure_background_runner":
+            calls.append(name)
+        if name in {"stat", "lstat", "exists", "is_dir", "is_file", "is_symlink", "mkdir", "iterdir", "symlink_to"}:
+            path = frame.f_locals.get("self")
+            if isinstance(path, Path) and path.is_relative_to(CONSTANTS.ARCHIVE_DIR):
+                calls.append((name, str(path)))
+
+    previous = sys.getprofile()
+    sys.setprofile(observe)
+    try:
+        response = client.post(
+            "/api/v1/cli/add",
+            data={"urls": ["https://example.com/queue-only"], "tag": "queue-only"},
+            content_type="application/json",
+            **api_headers,
+        )
+    finally:
+        sys.setprofile(previous)
+    assert response.status_code == 200, response.content
+    crawl = Crawl.objects.get(pk=response.json()["result"]["crawl_id"])
+    assert crawl.urls == "https://example.com/queue-only"
+    assert crawl.status == Crawl.StatusChoices.QUEUED
+    assert crawl.retry_at is not None
+    assert not crawl.snapshot_set.exists()
+    assert calls == []

archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py

@@ -477,25 +477,38 @@ def test_paused_snapshot_survives_server_restart_and_resumes_via_api(client, tmp
         stop_server(tmp_path)
 
 
-def test_rest_snapshot_delete_removes_output_dir(client, api_headers):
-    url = "https://example.com/delete-path-snapshot"
+@pytest.mark.timeout(180)
+def test_rest_snapshot_delete_removes_output_dir(tmp_path, recursive_test_site):
+    import time
+    from .test_snapshot_service import _snapshot_state
+    from .conftest import create_admin_and_token, live_api_request, start_archivebox_server, stop_archivebox_process
 
-    response = api_client_request(
-        client,
-        "post",
-        "/api/v1/core/snapshots",
-        payload={"url": url, "depth": 0, "status": Snapshot.StatusChoices.QUEUED},
-        headers=api_headers,
-    )
-    assert response.status_code == 200, response.content.decode()
-
-    snapshot = Snapshot.objects.get(url=url)
-    snapshot_dir = Path(snapshot.output_dir)
-    snapshot_dir.mkdir(parents=True, exist_ok=True)
-    (snapshot_dir / "delete-path-test.txt").write_text("snapshot output")
-    assert snapshot_dir.exists()
-
-    response = client.delete(f"/api/v1/core/snapshot/{snapshot.id}", **api_headers)
-    assert response.status_code == 200, response.content.decode()
-    assert not Snapshot.objects.filter(pk=snapshot.pk).exists()
-    assert not snapshot_dir.exists()
+    init_archive(tmp_path)
+    port = get_free_port()
+    env = cli_env(port=port, server=True, PLUGINS="wget")
+    url = recursive_test_site["root_url"]
+    captured = run_archivebox_cmd(["add", "--plugins=wget", url], cwd=tmp_path, env=env)
+    assert captured.returncode == 0, captured.stdout + captured.stderr
+    state = _snapshot_state(tmp_path, url)
+    assert any("Root" in path.read_text(errors="ignore") for path in (state["snapshot_dir"] / "wget").rglob("*.html"))
+    with use_archivebox_db(tmp_path):
+        snapshot_id = str(Snapshot.objects.get(url=url).pk)
+    token = create_admin_and_token(tmp_path)
+    server = start_archivebox_server(tmp_path, port=port, env=env, log_name="api-delete-handoff.log")
+    try:
+        response = live_api_request(port, "delete", f"/api/v1/core/snapshot/{snapshot_id}", api_token=token)
+        assert response.status_code == 200, response.text
+        assert response.json()["queued_count"] == 1
+        assert response.json()["deleted_count"] == 0
+        deadline = time.monotonic() + 75
+        with use_archivebox_db(tmp_path):
+            while Snapshot.objects.filter(pk=snapshot_id).exists() and time.monotonic() < deadline:
+                assert server.poll() is None
+                time.sleep(0.1)
+            assert not Snapshot.objects.filter(pk=snapshot_id).exists()
+        assert not state["snapshot_dir"].exists()
+        assert not state["crawl_link"].is_symlink()
+        missing = live_api_request(port, "get", f"/api/v1/core/snapshot/{snapshot_id}", api_token=token)
+        assert missing.status_code == 404
+    finally:
+        stop_archivebox_process(server)

archivebox/tests/test_api_v1_core_snapshots.py

@@ -13,6 +13,65 @@ from archivebox.crawls.models import Crawl
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
+def test_browser_metadata_respects_only_new_and_reports_capture_owner(client, api_headers):
+    url = "https://example.com/browser-deduplication"
+    for name in ("Original", "Replacement"):
+        persona = client.post(
+            "/api/v1/personas/sync",
+            data={"extension_persona_id": f"dedupe-{name}", "name": name},
+            content_type="application/json",
+            **api_headers,
+        )
+        assert persona.status_code == 200, persona.content
+
+    def submit(only_new, persona):
+        queued = client.post(
+            "/api/v1/cli/add",
+            data={"urls": [url], "only_new": only_new, "persona": persona},
+            content_type="application/json",
+            **api_headers,
+        )
+        assert queued.status_code == 200, queued.content
+        crawl_id = queued.json()["result"]["crawl_id"]
+        metadata = client.post(
+            "/api/v1/core/snapshots",
+            data={"url": url, "crawl_id": crawl_id},
+            content_type="application/json",
+            **api_headers,
+        )
+        assert metadata.status_code == 200, metadata.content
+        return crawl_id, metadata.json()
+
+    first_crawl, first = submit(True, "Original")
+    second_crawl, reused = submit(True, "Replacement")
+    assert second_crawl != first_crawl
+    assert reused["id"] == first["id"]
+    assert reused["crawl_id"] == first_crawl
+    assert reused["persona"] == "Original"
+    assert Snapshot.objects.filter(url=url).count() == 1
+    forced_crawl, forced = submit(False, "Replacement")
+    assert forced["id"] != first["id"]
+    assert forced["crawl_id"] == forced_crawl
+    assert forced["persona"] == "Replacement"
+    assert Snapshot.objects.filter(url=url).count() == 2
+
+
+def test_snapshot_delete_only_queues_cleanup(client, api_headers):
+    created = client.post(
+        "/api/v1/core/snapshots",
+        data={"url": "https://example.com/queued-delete"},
+        content_type="application/json",
+        **api_headers,
+    )
+    assert created.status_code == 200, created.content
+    snapshot_id = created.json()["id"]
+    response = client.delete(f"/api/v1/core/snapshot/{snapshot_id}", **api_headers)
+    assert response.status_code == 200, response.content
+    assert response.json()["queued_count"] == 1
+    assert response.json()["deleted_count"] == 0
+    assert Snapshot.objects.get(pk=snapshot_id).status == Snapshot.DELETING_STATE
+
+
 def test_snapshots_api_filters_status_column(client, api_admin_user, api_headers):
     crawl = Crawl.objects.create(
         urls="https://example.com",
@@ -150,3 +209,81 @@ def test_new_snapshot_creation_does_not_wait_for_active_crawl(client, api_admin_
     assert response.status_code == 200, response.content
     assert Snapshot.objects.filter(url=url, crawl=crawl).count() == 1
     assert elapsed < 1
+
+
+def test_snapshot_metadata_requests_do_not_touch_archive_storage(client, api_admin_user, api_headers):
+    """Metadata writes and serialization must work without replay storage access."""
+    import sys
+    import threading
+    from pathlib import Path
+    from archivebox.config import CONSTANTS
+
+    url = "https://example.com/metadata-without-storage"
+    crawl = Crawl.objects.create(urls=url, created_by=api_admin_user)
+    storage_calls = []
+
+    def observe(frame, event, arg):
+        if event == "call" and frame.f_code.co_name in {
+            "stat",
+            "lstat",
+            "exists",
+            "is_dir",
+            "is_file",
+            "is_symlink",
+            "open",
+            "mkdir",
+            "iterdir",
+            "glob",
+            "rglob",
+            "readlink",
+            "symlink_to",
+            "unlink",
+        }:
+            path = frame.f_locals.get("self", frame.f_locals.get("path"))
+            if isinstance(path, (str, Path)) and Path(path).is_relative_to(CONSTANTS.ARCHIVE_DIR):
+                storage_calls.append((frame.f_code.co_name, str(path)))
+
+    previous = sys.getprofile()
+    thread_profile = threading.getprofile()
+    threading.setprofile_all_threads(observe)
+    try:
+        response = client.post(
+            "/api/v1/core/snapshots",
+            data={"url": url, "crawl_id": str(crawl.id), "title": "Browser title", "tags": ["browser"]},
+            content_type="application/json",
+            **api_headers,
+        )
+        assert response.status_code == 200, response.content
+        snapshot_id = response.json()["id"]
+        updated = client.post(
+            "/api/v1/core/snapshots",
+            data={"url": url, "crawl_id": str(crawl.id), "title": "Updated title", "tags": ["browser"]},
+            content_type="application/json",
+            **api_headers,
+        )
+        assert updated.status_code == 200, updated.content
+        result = client.post(
+            "/api/v1/core/archiveresults",
+            data={"snapshot_id": snapshot_id, "plugin": "chrome_extension_viewport", "status": "started"},
+            **api_headers,
+        )
+        assert result.status_code == 200, result.content
+        result_id = result.json()["id"]
+        result_get = client.get(f"/api/v1/core/archiveresult/{result_id}", **api_headers)
+        assert result_get.status_code == 200, result_get.content
+        assert result_get.json()["output_files"] == {}
+        fetched = client.get(f"/api/v1/core/snapshot/{snapshot_id}", **api_headers)
+        assert fetched.status_code == 200, fetched.content
+        assert fetched.json()["archiveresults"][0]["id"] == result_id
+        deleted = client.delete(f"/api/v1/core/snapshot/{snapshot_id}", **api_headers)
+        assert deleted.status_code == 200, deleted.content
+        assert deleted.json()["queued_count"] == 1
+    finally:
+        threading.setprofile_all_threads(thread_profile)
+        sys.setprofile(previous)
+    snapshot = Snapshot.objects.get(pk=snapshot_id)
+    assert snapshot.title == "Updated title"
+    assert snapshot.status == Snapshot.DELETING_STATE
+    assert list(snapshot.tags.values_list("name", flat=True)) == ["browser"]
+    assert fetched.json()["title"] == "Updated title"
+    assert storage_calls == []

archivebox/tests/test_archive_result_service.py

@@ -51,7 +51,7 @@ def _run_shipped_snapshot_hook(
 
     from abx_dl.services.process_service import ProcessService as HookProcessService
     from abx_dl.services.archive_result_service import ArchiveResultService as HookArchiveResultService
-    from abx_plugins.plugins.base.utils import _collect_required_schema_paths, get_hydrated_required_binaries
+    from abx_plugins.plugins.base.utils import _collect_required_schema_paths
     from abxpkg import prepare_script_exec_plan
     from archivebox.core.models import ArchiveResult
     from archivebox.machine.models import Process
@@ -66,15 +66,9 @@ def _run_shipped_snapshot_hook(
     hook_path = matching_hooks[0].path
     projected_hook_name = event_hook_name or hook_name
     hook_config = hook_path.parent / "config.json"
-    for required_binary in get_hydrated_required_binaries(
-        hook_config,
-        environ={**os.environ, "ABXPKG_LIB_DIR": str(lib_dir)},
-    ):
-        install_real_binary(
-            required_binary["name"],
-            binproviders=required_binary["binproviders"],
-            overrides=required_binary.get("overrides"),
-        )
+    # This helper runs standalone hooks. Resolve their dependencies through the
+    # public abxpkg CLI below; creating a new ArchiveBox Binary row on every
+    # invocation makes a second run collide with the already-installed identity.
     binary_env = resolve_abxpkg_binary_env(lib_dir, deps_from=_collect_required_schema_paths(hook_config))
     hook_env = {
         **binary_env,

archivebox/tests/test_binary_service.py

@@ -198,21 +198,26 @@ def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized
     assert Path(relibbed.abspath).resolve() == Path(host_binary).resolve()
 
 
-def test_missing_binary_request_stays_queued_then_recovers_when_provider_can_resolve(initialized_archive, tmp_path):
-    name = "http"
+def test_missing_binary_request_stays_queued_then_recovers_when_provider_can_resolve(initialized_archive):
+    name = "httpie"
     provider_bin_dir = initialized_archive / "lib" / "pip" / "venv" / "bin"
     runtime_env = _runtime_env(initialized_archive)
 
-    _cmd_result = _run_real_binary_lifecycle(initialized_archive, name=name, binproviders="env", env=runtime_env)
-    stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
-
-    assert returncode != 0, stdout + stderr
-    assert "Binary http installation failed" in stderr
+    failed = run_archivebox_cmd(
+        ["install", "--binproviders=env", name],
+        cwd=initialized_archive,
+        timeout=120,
+        env=runtime_env,
+        default_cli_env=True,
+        disable_extractors=True,
+    )
+    assert failed.returncode != 0, failed.stdout + failed.stderr
+    assert f"Binary {name} installation failed" in failed.stderr
 
     with use_archivebox_db(initialized_archive):
         queued = Binary.objects.get(name=name)
         queued_id = str(queued.id)
-        failed_process = Process.objects.filter(process_type=Process.TypeChoices.BINARY).latest("created_at")
+        failed_process = Process.objects.filter(binary_id=queued_id).latest("created_at")
         machine_config = Machine.objects.get(pk=queued.machine_id).config or {}
 
     assert queued.status == Binary.StatusChoices.QUEUED
@@ -223,28 +228,42 @@ def test_missing_binary_request_stays_queued_then_recovers_when_provider_can_res
     assert f"{name.upper().replace('-', '_')}_BINARY" not in machine_config
     assert not (provider_bin_dir / name).exists()
 
-    with use_archivebox_db(initialized_archive):
-        queued = Binary.objects.get(pk=queued_id)
-        queued.binproviders = "pip"
-        queued.overrides = {"pip": {"install_args": ["httpie>=3.2.4"]}}
-        queued.save(update_fields=["binproviders", "overrides", "modified_at"])
-    recovered_runtime_env = _runtime_env(initialized_archive)
-
-    _cmd_result = run_archivebox_cmd(
-        ["run", f"--binary-id={queued_id}"],
+    recovered_install = run_archivebox_cmd(
+        ["install", "--binproviders=pip", name],
         cwd=initialized_archive,
         timeout=120,
-        env=recovered_runtime_env,
+        env=runtime_env,
         default_cli_env=True,
         disable_extractors=True,
     )
-    recover_stdout, recover_stderr, recover_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
-
-    assert recover_code == 0, recover_stdout + recover_stderr
+    assert recovered_install.returncode == 0, recovered_install.stdout + recovered_install.stderr
+    repeated_install = run_archivebox_cmd(
+        ["install", "--binproviders=pip", name],
+        cwd=initialized_archive,
+        timeout=120,
+        env=runtime_env,
+        default_cli_env=True,
+        disable_extractors=True,
+    )
+    assert repeated_install.returncode == 0, repeated_install.stdout + repeated_install.stderr
+    # Changing the provider policy runs a real installation request against the
+    # now-warm cache. Its fast result must still finalize the DB projection.
+    cached_install = run_archivebox_cmd(
+        ["install", "--binproviders=pip,env", name],
+        cwd=initialized_archive,
+        timeout=120,
+        env=runtime_env,
+        default_cli_env=True,
+        disable_extractors=True,
+    )
+    assert cached_install.returncode == 0, cached_install.stdout + cached_install.stderr
     with use_archivebox_db(initialized_archive):
         recovered = Binary.objects.get(pk=queued_id)
+        # Providers may resolve their own dependencies during installation. Those
+        # Process rows are not attempts to install this binary; track its exact
+        # identity instead of assuming the last two global rows belong to it.
         process_exit_codes = list(
-            Process.objects.filter(process_type=Process.TypeChoices.BINARY).order_by("created_at").values_list("exit_code", flat=True),
+            Process.objects.filter(binary_id=queued_id).order_by("created_at").values_list("exit_code", flat=True),
         )
 
     assert recovered.status == Binary.StatusChoices.INSTALLED
@@ -253,4 +272,4 @@ def test_missing_binary_request_stays_queued_then_recovers_when_provider_can_res
     assert Path(recovered.abspath) == provider_bin_dir / name
     assert recovered.binprovider == "pip"
     assert Path(recovered.abspath).is_file()
-    assert process_exit_codes[-2:] == [1, 0]
+    assert process_exit_codes == [1, 0, 0]

archivebox/tests/test_cli_add_4.py

@@ -325,11 +325,15 @@ def test_add_duplicate_url_creates_separate_crawls(initialized_archive):
     assert snapshots == [("https://example.com", 0), ("https://example.com", 0)]
 
 
-def test_snapshot_create_creates_current_output_directory(initialized_archive):
-    """Test the user-facing snapshot creation path creates an output directory."""
-    env = cli_env(disable_extractors=True)
+def test_snapshot_create_server_creates_current_output_directory(initialized_archive, recursive_test_site):
+    """Queued metadata gets its output directory from the normal server worker."""
+    from .conftest import start_archivebox_server, stop_archivebox_process, get_free_port, wait_for_snapshot_capture
+
+    port = get_free_port()
+    env = cli_env(port=port, server=True, PLUGINS="wget")
+    url = recursive_test_site["root_url"]
     run_archivebox_cmd(
-        ["snapshot", "create", "https://example.com"],
+        ["snapshot", "create", url],
         cwd=initialized_archive,
         env=env,
         check=True,
@@ -338,6 +342,12 @@ def test_snapshot_create_creates_current_output_directory(initialized_archive):
     with use_archivebox_db(initialized_archive):
         snapshot_id = str(Snapshot.objects.values_list("id", flat=True).get())
 
+    assert find_snapshot_dir(initialized_archive, snapshot_id) is None
+    server = start_archivebox_server(initialized_archive, env=env, port=port, log_name="snapshot-output-server.log")
+    try:
+        assert "Root" in wait_for_snapshot_capture(initialized_archive, url, timeout=120)
+    finally:
+        stop_archivebox_process(server)
     snapshot_dir = find_snapshot_dir(initialized_archive, snapshot_id)
     assert snapshot_dir is not None, f"Snapshot output directory not found for {snapshot_id}"
     assert snapshot_dir.is_dir()

archivebox/tests/test_cli_background_screenshot.py

@@ -0,0 +1,39 @@
+"""Exercise the browser target lifecycle through a real multi-page CLI crawl."""
+
+import os
+import sqlite3
+import subprocess
+
+import pytest
+
+
+@pytest.mark.timeout(180)
+def test_cli_crawl_captures_background_screenshots(tmp_path):
+    env = {**os.environ, "SCREENSHOT_TIMEOUT": "5"}
+    initialized = subprocess.run(
+        ["archivebox", "init", "--quick"],
+        cwd=tmp_path,
+        env=env,
+        capture_output=True,
+        text=True,
+        timeout=60,
+    )
+    assert initialized.returncode == 0, initialized.stderr
+    captured = subprocess.run(
+        ["archivebox", "add", "--plugins=title,wget,screenshot", "https://example.com", "https://example.org"],
+        cwd=tmp_path,
+        env=env,
+        capture_output=True,
+        text=True,
+        timeout=120,
+    )
+    assert captured.returncode == 0, captured.stderr
+    with sqlite3.connect(tmp_path / "index.sqlite3") as db:
+        results = db.execute("SELECT status, output_str FROM core_archiveresult WHERE plugin='screenshot'").fetchall()
+    assert len(results) == 2, captured.stdout
+    if any(status != "succeeded" for status, _ in results):
+        print(captured.stdout, captured.stderr)
+    assert all(status == "succeeded" for status, _ in results), (results, captured.stdout, captured.stderr)
+    images = list((tmp_path / "archive" / "users").glob("*/snapshots/**/screenshot/screenshot.png"))
+    assert len(images) == 2
+    assert all(image.read_bytes().startswith(b"\x89PNG\r\n\x1a\n") and image.stat().st_size > 1000 for image in images)

archivebox/tests/test_cli_config.py

@@ -8,37 +8,20 @@ from archivebox.tests.conftest import run_archivebox_cmd
 from archivebox.config.configset import read_ini_config
 
 
-def test_config_displays_all_config(initialized_archive):
-    """Test that config without args displays all configuration."""
-    result = run_archivebox_cmd(["config"])
-
-    assert result.returncode == 0
-    output = result.stdout
-    # Should show config sections
-    assert len(output) > 100
-    # Should show at least some standard config keys
-    assert "TIMEOUT" in output or "OUTPUT_PERMISSIONS" in output
-
-
-def test_config_shows_derived_collection_paths_but_not_runtime_dirs(initialized_archive):
-    """The CLI should expose collection paths, not per-crawl/per-snapshot runtime dirs."""
-    run_archivebox_cmd(["init"], cwd=initialized_archive, check=True)
-
+def test_config_read_get_and_search_cover_core_plugin_alias_and_path_options(initialized_archive):
+    """Exercise every read-only CLI branch against one unchanged collection."""
     result = run_archivebox_cmd(["config"], cwd=initialized_archive)
-
     assert result.returncode == 0, result.stderr
-    unwrapped_output = result.stdout.replace("\n", "")
-    assert "DATA_DIR" in result.stdout
-    assert result.stdout.count("\nDATA_DIR =") == 1
-    assert str(initialized_archive) in unwrapped_output
-    assert "PERSONAS_DIR" in result.stdout
-    assert result.stdout.count("\nPERSONAS_DIR =") == 1
-    assert "SNAP_DIR" not in result.stdout
-    assert "CRAWL_DIR" not in result.stdout
-
-
-def test_config_get_derived_path_but_rejects_runtime_dir(initialized_archive):
-    run_archivebox_cmd(["init"], cwd=initialized_archive, check=True)
+    output = result.stdout
+    assert len(output) > 100
+    assert "TIMEOUT" in output or "OUTPUT_PERMISSIONS" in output
+    assert "DATA_DIR" in output
+    assert output.count("\nDATA_DIR =") == 1
+    assert str(initialized_archive) in output.replace("\n", "")
+    assert "PERSONAS_DIR" in output
+    assert output.count("\nPERSONAS_DIR =") == 1
+    assert "SNAP_DIR" not in output
+    assert "CRAWL_DIR" not in output
 
     data_dir = run_archivebox_cmd(["config", "--get", "DATA_DIR"], cwd=initialized_archive)
     snap_dir = run_archivebox_cmd(["config", "--get", "SNAP_DIR"], cwd=initialized_archive)
@@ -49,10 +32,21 @@ def test_config_get_derived_path_but_rejects_runtime_dir(initialized_archive):
     assert snap_dir.returncode != 0
     assert "SNAP_DIR =" not in snap_dir.stdout
 
+    result = run_archivebox_cmd(
+        ["config", "--get", "TIMEOUT"],
+    )
 
-def test_config_set_rejects_readonly_and_runtime_dirs(initialized_archive):
-    run_archivebox_cmd(["init"], cwd=initialized_archive, check=True)
+    assert result.returncode == 0
+    assert "TIMEOUT" in result.stdout
 
+    for query, expected in (("TIMEOUT", "TIMEOUT"), ("wget", "WGET_BINARY"), ("URL_BLACK", "URL_DENYLIST")):
+        result = run_archivebox_cmd(["config", "--search", query])
+        assert result.returncode == 0, result.stderr
+        assert expected in result.stdout, query
+
+
+def test_config_rejects_readonly_runtime_unknown_and_malformed_keys(initialized_archive):
+    """Each rejected input must fail through its own CLI validation branch."""
     data_dir = run_archivebox_cmd(
         ["config", "--set", f"DATA_DIR={initialized_archive / 'other'}"],
         cwd=initialized_archive,
@@ -68,87 +62,12 @@ def test_config_set_rejects_readonly_and_runtime_dirs(initialized_archive):
     assert "DATA_DIR" not in content
     assert "CRAWL_DIR" not in content
 
-
-def test_config_get_specific_key(initialized_archive):
-    """Test that config --get KEY retrieves specific value."""
-    result = run_archivebox_cmd(
-        ["config", "--get", "TIMEOUT"],
-    )
-
-    assert result.returncode == 0
-    assert "TIMEOUT" in result.stdout
-
-
-def test_config_set_writes_to_file(initialized_archive):
-    """Test that config --set KEY=VALUE writes to ArchiveBox.conf."""
-
-    result = run_archivebox_cmd(
-        ["config", "--set", "TIMEOUT=120"],
-    )
-
-    assert result.returncode == 0
-    assert "TIMEOUT=120" in result.stdout
-
-    # Verify config file was updated
-    config_file = initialized_archive / "ArchiveBox.conf"
-    assert config_file.exists()
-
-    content = config_file.read_text()
-    assert "TIMEOUT" in content or "120" in content
-
-
-def test_config_set_and_get_roundtrip(initialized_archive):
-    """Test that set value can be retrieved with get."""
-
-    # Set a unique value
-    run_archivebox_cmd(
-        ["config", "--set", "TIMEOUT=987"],
-    )
-
-    updated = run_archivebox_cmd(
-        ["config", "--set", "TIMEOUT=654"],
-    )
-
-    assert updated.returncode == 0
-    assert "TIMEOUT=654" in updated.stdout
-
-    # Get the value back
-    result = run_archivebox_cmd(
-        ["config", "--get", "TIMEOUT"],
-    )
-
-    assert "654" in result.stdout
-
-
-def test_config_set_multiple_values(initialized_archive):
-    """Test setting multiple config values at once."""
-
-    result = run_archivebox_cmd(
-        ["config", "--set", "TIMEOUT=111", "YTDLP_TIMEOUT=222"],
-    )
-
-    assert result.returncode == 0
-
-    # Verify both were written
-    config_file = initialized_archive / "ArchiveBox.conf"
-    content = config_file.read_text()
-    assert "111" in content
-    assert "222" in content
-
-
-def test_config_set_invalid_key_fails(initialized_archive):
-    """Test that setting invalid config key fails."""
-
     result = run_archivebox_cmd(
         ["config", "--set", "TOTALLY_INVALID_KEY_XYZ=value"],
     )
 
     assert result.returncode != 0
 
-
-def test_config_set_requires_equals_sign(initialized_archive):
-    """Test that set requires KEY=VALUE format."""
-
     result = run_archivebox_cmd(
         ["config", "--set", "TIMEOUT"],
     )
@@ -156,93 +75,46 @@ def test_config_set_requires_equals_sign(initialized_archive):
     assert result.returncode != 0
 
 
-def test_config_search_finds_keys(initialized_archive):
-    """Test that config --search finds matching keys."""
-
-    result = run_archivebox_cmd(
-        ["config", "--search", "TIMEOUT"],
-    )
-
-    # Should find timeout-related config
-    assert "TIMEOUT" in result.stdout
-
-
-def test_config_search_finds_plugin_options(initialized_archive):
-    """Test that config --search finds plugin keys and descriptions."""
-
-    result = run_archivebox_cmd(
-        ["config", "--search", "wget"],
-    )
-
-    assert result.returncode == 0
-    assert "WGET_BINARY" in result.stdout
-
-
-def test_config_search_finds_core_aliases(initialized_archive):
-    """Test that config --search finds core options by partial alias."""
-
-    result = run_archivebox_cmd(
-        ["config", "--search", "URL_BLACK"],
-    )
-
-    assert result.returncode == 0
-    assert "URL_DENYLIST" in result.stdout
-
-
-def test_config_preserves_existing_values(initialized_archive):
-    """Test that setting new values preserves existing ones."""
-
-    # Set first value
-    run_archivebox_cmd(
-        ["config", "--set", "TIMEOUT=100"],
-    )
-
-    # Set second value
-    run_archivebox_cmd(
-        ["config", "--set", "YTDLP_TIMEOUT=200"],
-    )
-
-    # Verify both are in config file
+def test_config_set_update_get_preserves_other_keys(initialized_archive):
+    """Exercise each write path against one real config, checking each transition."""
+    result = run_archivebox_cmd(["config", "--set", "TIMEOUT=120"])
+    assert result.returncode == 0, result.stderr
+    assert "TIMEOUT=120" in result.stdout
     config_file = initialized_archive / "ArchiveBox.conf"
+    assert config_file.is_file()
+    content = config_file.read_text()
+    assert "TIMEOUT" in content and "120" in content
+    assert "[" in content or "=" in content
+    assert read_ini_config(config_file)["TIMEOUT"] == "120"
+
+    result = run_archivebox_cmd(["config", "--set", "YTDLP_TIMEOUT=200"])
+    assert result.returncode == 0, result.stderr
     content = config_file.read_text()
     assert "TIMEOUT" in content
     assert "YTDLP_TIMEOUT" in content
+    assert read_ini_config(config_file)["TIMEOUT"] == "120"
+    assert read_ini_config(config_file)["YTDLP_TIMEOUT"] == "200"
+
+    updated = run_archivebox_cmd(["config", "--set", "TIMEOUT=654"])
+    assert updated.returncode == 0, updated.stderr
+    assert "TIMEOUT=654" in updated.stdout
+    result = run_archivebox_cmd(["config", "--get", "TIMEOUT"])
+    assert result.returncode == 0, result.stderr
+    assert "654" in result.stdout
+    assert read_ini_config(config_file)["TIMEOUT"] == "654"
+    assert read_ini_config(config_file)["YTDLP_TIMEOUT"] == "200"
 
 
-def test_config_file_is_valid_toml(initialized_archive):
-    """Test that config file remains valid TOML after set."""
-
-    run_archivebox_cmd(
-        ["config", "--set", "TIMEOUT=150"],
-    )
-
+def test_config_set_multiple_values_in_fresh_sections(initialized_archive):
+    """Keep first writes into new sections distinct from updating existing keys."""
+    result = run_archivebox_cmd(["config", "--set", "TIMEOUT=111", "YTDLP_TIMEOUT=222"])
+    assert result.returncode == 0, result.stderr
     config_file = initialized_archive / "ArchiveBox.conf"
     content = config_file.read_text()
-
-    # Basic TOML validation - should have sections and key=value pairs
-    assert "[" in content or "=" in content
-
-
-def test_config_updates_existing_value(initialized_archive):
-    """Test that setting same key twice updates the value."""
-
-    # Set initial value
-    run_archivebox_cmd(
-        ["config", "--set", "TIMEOUT=100"],
-    )
-
-    # Update to new value
-    run_archivebox_cmd(
-        ["config", "--set", "TIMEOUT=200"],
-    )
-
-    # Get current value
-    result = run_archivebox_cmd(
-        ["config", "--get", "TIMEOUT"],
-    )
-
-    # Should show updated value
-    assert "200" in result.stdout
+    assert "111" in content
+    assert "222" in content
+    assert read_ini_config(config_file)["TIMEOUT"] == "111"
+    assert read_ini_config(config_file)["YTDLP_TIMEOUT"] == "222"
 
 
 def test_config_ignores_legacy_unknown_keys(tmp_path, initialized_archive):

archivebox/tests/test_cli_init.py

@@ -95,110 +95,33 @@ def test_cli_refuses_source_root_without_side_effects_and_allows_separate_data_d
     assert (data_dir / "logs").is_dir()
 
 
-def test_init_creates_database_file(tmp_path):
-    """Test that init creates index.sqlite3 database file."""
+def test_init_creates_collection_schema_directories_config_and_permissions(tmp_path):
+    """Inspect all outputs of one real fresh init; upgrade cases stay separate."""
     result = run_archivebox_cmd(["init"])
+    assert result.returncode == 0, result.stderr
 
-    assert result.returncode == 0
     db_path = tmp_path / "index.sqlite3"
-    assert db_path.exists()
     assert db_path.is_file()
-
-
-def test_init_creates_archive_directory(tmp_path):
-    """Test that init creates archive directory."""
-    run_archivebox_cmd(["init"])
-
-    archive_dir = tmp_path / "archive"
-    assert archive_dir.exists()
-    assert archive_dir.is_dir()
-
-
-def test_init_uses_cwd_archive_and_users_dirs(tmp_path):
-    """Test that init creates archive/users storage roots under cwd."""
-
-    result = run_archivebox_cmd(["init"])
-
-    assert result.returncode == 0
-    assert (tmp_path / "archive").is_dir()
-    assert (tmp_path / "archive" / "users").is_dir()
-
-
-def test_init_creates_sources_directory(tmp_path):
-    """Test that init creates sources directory."""
-    run_archivebox_cmd(["init"])
-
-    sources_dir = tmp_path / "sources"
-    assert sources_dir.exists()
-    assert sources_dir.is_dir()
-
-
-def test_init_creates_logs_directory(tmp_path):
-    """Test that init creates logs directory."""
-    run_archivebox_cmd(["init"])
-
-    logs_dir = tmp_path / "logs"
-    assert logs_dir.exists()
-    assert logs_dir.is_dir()
-
-
-def test_init_creates_config_file(tmp_path):
-    """Test that init creates ArchiveBox.conf config file."""
-    run_archivebox_cmd(["init"])
-
     config_file = tmp_path / "ArchiveBox.conf"
-    assert config_file.exists()
     assert config_file.is_file()
-
-
-def test_init_runs_migrations(tmp_path):
-    """Test that init runs Django migrations and creates core tables."""
-    run_archivebox_cmd(["init"])
+    for relative_path in ("archive", "archive/users", "sources", "logs"):
+        assert (tmp_path / relative_path).is_dir(), relative_path
 
     with use_archivebox_db(tmp_path):
-        migration_count = MigrationRecorder.Migration.objects.count()
-
-    assert migration_count > 0
-
-
-def test_init_creates_core_snapshot_table(tmp_path):
-    """Test that init creates core_snapshot table."""
-    run_archivebox_cmd(["init"])
-
-    assert Snapshot._meta.db_table == "core_snapshot"
-    with use_archivebox_db(tmp_path):
+        assert MigrationRecorder.Migration.objects.count() > 0
+        assert Snapshot._meta.db_table == "core_snapshot"
         assert Snapshot.objects.count() == 0
-
-
-def test_init_creates_crawls_crawl_table(tmp_path):
-    """Test that init creates crawls_crawl table."""
-    run_archivebox_cmd(["init"])
-
-    assert Crawl._meta.db_table == "crawls_crawl"
-    with use_archivebox_db(tmp_path):
+        assert Crawl._meta.db_table == "crawls_crawl"
         assert Crawl.objects.count() == 0
-
-
-def test_init_creates_core_archiveresult_table(tmp_path):
-    """Test that init creates core_archiveresult table."""
-    run_archivebox_cmd(["init"])
-
-    assert ArchiveResult._meta.db_table == "core_archiveresult"
-    with use_archivebox_db(tmp_path):
+        assert ArchiveResult._meta.db_table == "core_archiveresult"
         assert ArchiveResult.objects.count() == 0
+        assert Machine._meta.db_table == "machine_machine"
+        Machine.objects.count()
 
-
-def test_init_sets_correct_file_permissions(tmp_path):
-    """Test that init sets correct permissions on created files."""
-    run_archivebox_cmd(["init"])
-
-    # Check database permissions
-    db_path = tmp_path / "index.sqlite3"
     assert oct(db_path.stat().st_mode)[-3:] in (get_config().OUTPUT_PERMISSIONS, DIR_PERMISSIONS)
-
-    # Check directory permissions
-    archive_dir = tmp_path / "archive"
-    assert oct(archive_dir.stat().st_mode)[-3:] in (get_config().OUTPUT_PERMISSIONS, DIR_PERMISSIONS)
+    assert oct((tmp_path / "archive").stat().st_mode)[-3:] in (get_config().OUTPUT_PERMISSIONS, DIR_PERMISSIONS)
+    output = result.stdout
+    assert "ArchiveBox" in output or "collection" in output.lower() or "Initializing" in output
 
 
 def test_init_creates_database_with_archivebox_permissions_despite_permissive_umask(tmp_path):
@@ -325,24 +248,6 @@ def test_init_quick_flag_skips_checks(tmp_path):
     assert db_path.exists()
 
 
-def test_init_creates_machine_table(tmp_path):
-    """Test that init creates the machine_machine table."""
-    run_archivebox_cmd(["init"])
-
-    assert Machine._meta.db_table == "machine_machine"
-    with use_archivebox_db(tmp_path):
-        Machine.objects.count()
-
-
-def test_init_output_shows_collection_info(tmp_path):
-    """Test that init output shows helpful collection information."""
-    result = run_archivebox_cmd(["init"])
-
-    output = result.stdout
-    # Should show some helpful info about the collection
-    assert "ArchiveBox" in output or "collection" in output.lower() or "Initializing" in output
-
-
 def test_init_ignores_unrecognized_archive_directories(initialized_archive):
     """Test that init upgrades existing dirs without choking on extra folders."""
     env = cli_env(disable_extractors=True)

archivebox/tests/test_cli_install.py

@@ -22,52 +22,14 @@ from archivebox.tests.test_orm_helpers import use_archivebox_db
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
-def test_install_runs_successfully(initialized_archive):
-    """Test that install command runs without error."""
-    result = run_archivebox_cmd(
-        ["install", "--dry-run"],
-        timeout=60,
-    )
-
-    assert result.returncode == 0, result.stderr or result.stdout
-    assert "Dry run - would detect ArchiveBox dependencies" in result.stdout
-
-
-def test_install_creates_binary_records_in_db(initialized_archive):
-    """Test that install --dry-run does not create Binary records in database."""
-
-    result = run_archivebox_cmd(
-        ["install", "--dry-run"],
-        timeout=60,
-    )
-    assert result.returncode == 0, result.stderr or result.stdout
-
-    with use_archivebox_db(initialized_archive):
-        assert Binary.objects.count() == 0
-
-
-def test_install_dry_run_does_not_install(initialized_archive):
-    """Test that --dry-run doesn't actually install anything."""
-
-    result = run_archivebox_cmd(
-        ["install", "--dry-run"],
-        timeout=60,
-    )
-
+def test_install_dry_run_reports_plan_without_creating_binary_records(initialized_archive):
+    """Inspect one dry run; real installs and empty-dir bootstrap stay separate."""
+    result = run_archivebox_cmd(["install", "--dry-run"], timeout=60)
     assert result.returncode == 0, result.stderr or result.stdout
     assert result.stdout.strip() == "Dry run - would detect ArchiveBox dependencies and run the abx-dl install flow"
-
-
-def test_install_detects_system_binaries(initialized_archive):
-    """Test that install detects existing system binaries."""
-
-    result = run_archivebox_cmd(
-        ["install", "--dry-run"],
-        timeout=60,
-    )
-
-    assert result.returncode == 0, result.stderr or result.stdout
-    assert "ArchiveBox dependencies" in result.stdout
+    assert "dry run" in result.stdout.lower()
+    with use_archivebox_db(initialized_archive):
+        assert Binary.objects.count() == 0
 
 
 def test_install_target_resolver_uses_declared_binary_aliases():
@@ -95,29 +57,6 @@ def test_default_dependency_install_memory_preflight_fails_before_installers(cap
     assert "No plugin dependency installers were started" in output
 
 
-def test_install_shows_binary_status(initialized_archive):
-    """Test that install shows status of binaries."""
-
-    result = run_archivebox_cmd(
-        ["install", "--dry-run"],
-        timeout=60,
-    )
-
-    assert result.returncode == 0, result.stderr or result.stdout
-    assert result.stdout.strip() == "Dry run - would detect ArchiveBox dependencies and run the abx-dl install flow"
-
-
-def test_install_dry_run_prints_dry_run_message(initialized_archive):
-    """Test that install --dry-run clearly reports that no changes will be made."""
-    result = run_archivebox_cmd(
-        ["install", "--dry-run"],
-        timeout=60,
-    )
-
-    assert result.returncode == 0
-    assert "dry run" in result.stdout.lower()
-
-
 def test_install_help_lists_dry_run_flag(tmp_path):
     """Test that install --help documents the dry-run option."""
     result = run_archivebox_cmd(

archivebox/tests/test_cli_list.py

@@ -26,9 +26,7 @@ def test_static_export_creates_detail_page_for_unarchived_snapshot(snapshot):
     from archivebox.config import CONSTANTS
 
     snapshot_dir = Path(snapshot.output_dir)
-    assert snapshot_dir.is_dir()
-    assert not any(snapshot_dir.iterdir())
-    snapshot_dir.rmdir()
+    # Queued metadata has no output directory until an explicit writer runs.
     assert not snapshot_dir.exists()
 
     html = Snapshot.objects.filter(pk=snapshot.pk).to_html(with_headers=True)

archivebox/tests/test_cli_persona.py

@@ -50,12 +50,14 @@ def test_persona_import_real_browser_session(initialized_archive, tmp_path, http
     """Import an actual Chromium-created session and archive its private page."""
     import hashlib
     import json
+    import shlex
     import subprocess
 
     from werkzeug.wrappers import Response
     from archivebox.tests.conftest import resolve_abxpkg_chrome_env
 
     browser_env = resolve_abxpkg_chrome_env(tmp_path / "lib")
+    browser_env["ABXPKG_LIB_DIR"] = str(tmp_path / "lib")
     source = tmp_path / "browser"
     httpserver.expect_request("/login").respond_with_data(
         '<html><title>Signed in</title><script>localStorage.setItem("persona-preference", "forest-green")</script>Signed in</html>',
@@ -90,6 +92,7 @@ const puppeteer = require(process.argv[1]);
         const page = await browser.newPage();
         await page.goto(process.argv[4]);
         console.log(await page.title());
+        console.log(await page.evaluate(() => localStorage.getItem('persona-preference')));
     } finally { await browser.close(); }
 })().catch(error => { console.error(error); process.exit(1); });
 """,
@@ -104,6 +107,7 @@ const puppeteer = require(process.argv[1]);
     )
     assert seeded.returncode == 0, seeded.stderr
     assert "Signed in" in seeded.stdout
+    assert "forest-green" in seeded.stdout
     preferences = source / "Profile 2" / "Preferences"
     original_preferences = preferences.read_bytes()
 
@@ -161,6 +165,9 @@ const puppeteer = require(process.argv[1]);
         env={**browser_env, "CHROME_SANDBOX": "false"},
     )
     assert archived.returncode == 0, archived.stderr
+    launch_commands = list((initialized_archive / "archive" / "users").glob("*/crawls/**/chrome/cmd.sh"))
+    assert len(launch_commands) == 1
+    assert shlex.split(launch_commands[0].read_text().splitlines()[1])[0] == browser_env["CHROME_BINARY"]
     outputs = list((initialized_archive / "archive" / "users").glob("*/snapshots/**/dom/output.html"))
     assert len(outputs) == 1
     html = outputs[0].read_text()

archivebox/tests/test_cli_remove.py

@@ -29,8 +29,8 @@ print(json.dumps([
     return json.loads(result.stdout.strip().splitlines()[-1])
 
 
-def test_remove_deletes_snapshot_from_db(initialized_archive):
-    """Test that remove command deletes snapshot from database."""
+def test_remove_yes_deletes_database_row_and_directory_without_confirmation(initialized_archive):
+    """Verify the DB, filesystem, exit code and summary of one confirmed removal."""
     env = cli_env(disable_extractors=True)
 
     # Add a snapshot
@@ -46,62 +46,17 @@ def test_remove_deletes_snapshot_from_db(initialized_archive):
     snapshot_dir = find_snapshot_dir(initialized_archive, snapshot_id)
     assert snapshot_dir is not None, f"Snapshot output directory not found for {snapshot_id}"
 
-    # Remove it
-    run_archivebox_cmd(
-        ["remove", "https://example.com", "--yes"],
-        env=env,
-    )
-
-    assert len(_snapshot_rows(initialized_archive, env)) == 0
-    assert not snapshot_dir.exists()
-
-
-def test_remove_deletes_archive_directory(initialized_archive):
-    """Test that remove --yes removes the current snapshot output directory."""
-    env = cli_env(disable_extractors=True)
-
-    # Add a snapshot
-    run_archivebox_cmd(
-        ["add", "--index-only", "--depth=0", "https://example.com"],
-        env=env,
-    )
-    run_queued_crawls(initialized_archive, env)
-
-    rows = _snapshot_rows(initialized_archive, env)
-    assert len(rows) == 1
-    snapshot_id = rows[0]["id"]
-
-    snapshot_dir = find_snapshot_dir(initialized_archive, snapshot_id)
-    assert snapshot_dir is not None, f"Snapshot output directory not found for {snapshot_id}"
-
-    run_archivebox_cmd(
-        ["remove", "https://example.com", "--yes"],
-        env=env,
-    )
-
-    assert not snapshot_dir.exists()
-
-
-def test_remove_yes_flag_skips_confirmation(initialized_archive):
-    """Test that --yes flag skips confirmation prompt."""
-    env = cli_env(disable_extractors=True)
-
-    run_archivebox_cmd(
-        ["add", "--index-only", "--depth=0", "https://example.com"],
-        env=env,
-    )
-    run_queued_crawls(initialized_archive, env)
-
-    # Remove with --yes should complete without interaction
+    # Remove with --yes must finish without interaction.
     result = run_archivebox_cmd(
         ["remove", "https://example.com", "--yes"],
         env=env,
         timeout=30,
     )
-
     assert result.returncode == 0
-    output = result.stdout + result.stderr
-    assert "Index now contains 0 links." in output
+    assert "Index now contains 0 links." in result.stdout + result.stderr
+
+    assert len(_snapshot_rows(initialized_archive, env)) == 0
+    assert not snapshot_dir.exists()
 
 
 def test_remove_without_yes_prompts_and_keeps_snapshot(initialized_archive):

archivebox/tests/test_cli_server.py

@@ -177,70 +177,41 @@ def test_sqlite_connections_use_explicit_busy_timeout():
     assert "PRAGMA journal_mode = WAL;" in SQLITE_CONNECTION_OPTIONS["OPTIONS"]["init_command"]
 
 
-def test_docker_sqlite_never_uses_wal_on_host_shared_collection(tmp_path):
-    """Docker collections must not expose a WAL database through /data.
+def test_sqlite_add_commits_while_read_cursor_is_open(tmp_path):
+    """A real read snapshot must not hold up a new public CLI submission."""
+    import sqlite3
 
-    Docker Desktop/OrbStack bind mounts cross the Linux VM/host locking
-    boundary. A container WAL writer plus a host-side sqlite reader can make
-    the writer SIGBUS and leave index.sqlite3 malformed; the reader does not
-    need to write. Run a fresh real settings process with Docker identity so a
-    host test cannot accidentally pass by reusing this process's non-Docker
-    config imports.
-    """
-    env = os.environ.copy()
-    env["IN_DOCKER"] = "True"
-    env["DJANGO_SETTINGS_MODULE"] = "archivebox.core.settings"
-    result = subprocess.run(
-        [
-            sys.executable,
-            "-c",
-            (
-                "import django;"
-                "django.setup();"
-                "from django.db import connection;"
-                "cursor=connection.cursor();"
-                "cursor.execute('CREATE TABLE journal_probe (value INTEGER)');"
-                "cursor.execute('INSERT INTO journal_probe VALUES (1)');"
-                "cursor.execute('PRAGMA journal_mode');"
-                "print(cursor.fetchone()[0]);"
-                "cursor.close();"
-                "connection.close()"
-            ),
-        ],
-        cwd=tmp_path,
-        env=env,
-        capture_output=True,
-        text=True,
-        check=True,
-    )
+    env = {"SQLITE_JOURNAL_MODE": "WAL"}
+    initialized = run_archivebox_cmd(["init", "--quick"], cwd=tmp_path, env=env, timeout=90)
+    assert initialized.returncode == 0, initialized.stderr or initialized.stdout
+    for url in ("https://example.com/read-one", "https://example.com/read-two"):
+        initial = run_archivebox_cmd(["add", "--index-only", url], cwd=tmp_path, env=env, timeout=90)
+        assert initial.returncode == 0, initial.stderr or initial.stdout
 
-    assert result.stdout.strip().lower() == "delete"
-    assert not tmp_path.joinpath("index.sqlite3-wal").exists()
-    assert not tmp_path.joinpath("index.sqlite3-shm").exists()
-
-
-def test_docker_rejects_explicit_wal_override(tmp_path):
-    """An old config or environment override must not bypass the invariant."""
-    env = os.environ.copy()
-    env["IN_DOCKER"] = "True"
-    result = subprocess.run(
-        [
-            sys.executable,
-            "-c",
-            ("from archivebox.config.common import DatabaseConfig;DatabaseConfig(SQLITE_JOURNAL_MODE='WAL')"),
-        ],
-        cwd=tmp_path,
-        env=env,
-        capture_output=True,
-        text=True,
-    )
-
-    assert result.returncode != 0
-    assert "WAL is unsafe for Docker collections" in result.stderr
+    reader = sqlite3.connect(f"file:{tmp_path / 'index.sqlite3'}?mode=ro", uri=True)
+    try:
+        assert reader.execute("PRAGMA journal_mode").fetchone()[0] == "wal"
+        cursor = reader.execute("SELECT urls FROM crawls_crawl ORDER BY urls")
+        first = cursor.fetchone()
+        assert first == ("https://example.com/read-one",)
+        # Do not exhaust/close the actual cursor until add commits. This is the
+        # contention that rollback journaling turns into seconds of waiting.
+        added = run_archivebox_cmd(
+            ["add", "--index-only", "https://example.com/write-during-read"],
+            cwd=tmp_path,
+            env=env,
+            timeout=30,
+        )
+        assert added.returncode == 0, added.stderr or added.stdout
+        assert cursor.fetchall() == [("https://example.com/read-two",)]
+        cursor.close()
+        assert reader.execute("SELECT urls FROM crawls_crawl WHERE urls = ?", ("https://example.com/write-during-read",)).fetchone()
+    finally:
+        reader.close()
 
 
 def test_docker_postgres_ignores_irrelevant_sqlite_wal_override(tmp_path):
-    """The Docker SQLite safety invariant must not reject PostgreSQL.
+    """SQLite filesystem constraints must not reject PostgreSQL.
 
     Operators can switch an existing deployment to PostgreSQL while an old
     SQLITE_JOURNAL_MODE setting remains in ArchiveBox.conf or the environment.
@@ -448,7 +419,10 @@ def test_port_compatibility_does_not_add_listeners_outside_docker_defaults(tmp_p
         check=True,
     )
     workers = json.loads(result.stdout)
-    assert [worker["name"] for worker in workers] == ["worker_runserver" if debug else "worker_daphne", "worker_runner"]
+    assert [worker["name"] for worker in workers] == [
+        "worker_runserver" if debug else "worker_daphne",
+        "worker_runner",
+    ]
     command = shlex.split(workers[0]["command"])
     assert "--endpoint" not in command
     assert (f"127.0.0.1:{port}" if debug else f"--port={port}") in command

archivebox/tests/test_cli_snapshot.py

@@ -20,6 +20,10 @@ from archivebox.tests.conftest import (
     create_test_url,
     parse_jsonl_output,
     run_archivebox_cmd,
+    start_archivebox_server,
+    stop_archivebox_process,
+    get_free_port,
+    wait_for_snapshot_capture,
 )
 from archivebox.tests.test_orm_helpers import use_archivebox_db
 
@@ -380,19 +384,30 @@ class TestSnapshotDelete:
         assert "Would delete" in stderr
 
 
-def test_snapshot_creates_snapshot_with_correct_url(tmp_path, initialized_archive):
+def test_snapshot_creates_snapshot_with_correct_url(tmp_path, initialized_archive, recursive_test_site):
     """Test that snapshot stores the exact URL in the database."""
-    env = cli_env(disable_extractors=True)
+    port = get_free_port()
+    env = cli_env(port=port, server=True, PLUGINS="wget")
+    url = recursive_test_site["root_url"]
 
-    run_archivebox_cmd(
-        ["snapshot", "create", "https://example.com"],
+    result = run_archivebox_cmd(
+        ["snapshot", "create", url],
         cwd=tmp_path,
         env=env,
     )
+    assert result.returncode == 0, result.stderr
 
     with use_archivebox_db(tmp_path):
-        snapshot = Snapshot.objects.select_related("crawl__created_by").get(url="https://example.com")
+        snapshot = Snapshot.objects.select_related("crawl__created_by").get(url=url)
         username = snapshot.crawl.created_by.username
+        assert snapshot.status == Snapshot.StatusChoices.QUEUED
+        assert not snapshot.output_dir.exists()
+
+    server = start_archivebox_server(tmp_path, env=env, port=port, log_name="snapshot-create-server.log")
+    try:
+        assert "Root" in wait_for_snapshot_capture(tmp_path, url, timeout=120)
+    finally:
+        stop_archivebox_process(server)
 
     # Verify the crawl tree contains a relative symlink to the user-scoped snapshot output.
     snapshots_root = tmp_path / "archive" / "users" / username / "snapshots"

archivebox/tests/test_cli_status.py

@@ -24,21 +24,17 @@ def _create_snapshot_rows(initialized_archive, env, *urls):
     return result
 
 
-def test_status_runs_successfully(initialized_archive):
-    """Test that status command runs without error."""
+def test_status_reports_empty_collection_size_user_index_and_path(initialized_archive):
+    """All empty-collection fields come from the same real status response."""
     result = run_archivebox_cmd(["status"], cwd=initialized_archive)
-
     assert result.returncode == 0
-    assert len(result.stdout) > 100
-
-
-def test_status_shows_zero_snapshots_in_empty_archive(initialized_archive):
-    """Test status shows 0 snapshots in empty archive."""
-    result = run_archivebox_cmd(["status"], cwd=initialized_archive)
-
     output = result.stdout
-    # Should indicate empty/zero state
+    assert len(output) > 100
     assert "0" in output
+    assert "Size" in output or "size" in output
+    assert "user" in output.lower() or "login" in output.lower()
+    assert "index" in output.lower() or "Index" in output
+    assert "archive" in output.lower() or str(initialized_archive) in output
 
 
 def test_status_shows_correct_snapshot_count(initialized_archive):
@@ -58,36 +54,16 @@ def test_status_shows_correct_snapshot_count(initialized_archive):
     assert "3" in result.stdout
 
 
-def test_status_shows_archived_count(initialized_archive):
-    """Test status distinguishes archived vs unarchived snapshots."""
+def test_status_reports_database_snapshot_count_and_archive_categories(initialized_archive):
+    """One uncaptured snapshot exercises the DB count and filesystem categories."""
     env = cli_env(disable_extractors=True)
-
     _create_snapshot_rows(initialized_archive, env, "https://example.com")
-
+    with use_archivebox_db(initialized_archive):
+        assert Snapshot.objects.count() == 1
     result = run_archivebox_cmd(["status"], cwd=initialized_archive)
-
-    # Should show archived/unarchived categories
+    assert result.returncode == 0, result.stderr
+    assert "1" in result.stdout
     assert "archived" in result.stdout.lower() or "queued" in result.stdout.lower()
-
-
-def test_status_shows_archive_directory_size(initialized_archive):
-    """Test status reports archive directory size."""
-    result = run_archivebox_cmd(["status"], cwd=initialized_archive)
-
-    output = result.stdout
-    # Should show size info
-    assert "Size" in output or "size" in output
-
-
-def test_status_counts_archive_directories(initialized_archive):
-    """Test status counts directories in archive/ folder."""
-    env = cli_env(disable_extractors=True)
-
-    _create_snapshot_rows(initialized_archive, env, "https://example.com")
-
-    result = run_archivebox_cmd(["status"], cwd=initialized_archive)
-
-    # Should show directory count
     assert "present" in result.stdout.lower() or "directories" in result.stdout
 
 
@@ -107,22 +83,22 @@ def test_status_detects_orphaned_directories(initialized_archive):
     assert "archivebox update" in result.stdout
 
 
-def test_status_counts_new_snapshot_output_dirs_as_archived(initialized_archive):
+def test_status_counts_new_snapshot_output_dirs_as_archived(initialized_archive, recursive_test_site):
     """Test status reads archived/present counts from the current snapshot output layout."""
     env = cli_env(disable_extractors=True)
     env = env.copy()
     env["ARCHIVEBOX_ALLOW_NO_UNIX_SOCKETS"] = "true"
 
-    _create_snapshot_rows(initialized_archive, env, "https://example.com")
+    url = recursive_test_site["root_url"]
+    captured = run_archivebox_cmd(["add", "--plugins=wget", url], cwd=initialized_archive, env=env, timeout=120)
+    assert captured.returncode == 0, captured.stdout + captured.stderr
 
     with use_archivebox_db(initialized_archive):
-        snapshot_id = Snapshot.objects.values_list("id", flat=True).get(url="https://example.com")
+        snapshot_id = Snapshot.objects.values_list("id", flat=True).get(url=url)
 
     snapshot_dir = find_snapshot_dir(initialized_archive, str(snapshot_id))
     assert snapshot_dir is not None, f"Snapshot output directory not found for {snapshot_id}"
-    title_dir = snapshot_dir / "title"
-    title_dir.mkdir(parents=True, exist_ok=True)
-    (title_dir / "title.txt").write_text("Example Domain")
+    assert any("Root" in path.read_text(errors="ignore") for path in (snapshot_dir / "wget").rglob("*.html"))
 
     result = run_archivebox_cmd(["status"], cwd=initialized_archive, env=env)
 
@@ -131,40 +107,6 @@ def test_status_counts_new_snapshot_output_dirs_as_archived(initialized_archive)
     assert "present: 1" in result.stdout
 
 
-def test_status_shows_user_info(initialized_archive):
-    """Test status shows user/login information."""
-    result = run_archivebox_cmd(["status"], cwd=initialized_archive)
-
-    output = result.stdout
-    # Should show user section
-    assert "user" in output.lower() or "login" in output.lower()
-
-
-def test_status_reads_from_db_not_filesystem(initialized_archive):
-    """Test that status uses DB as source of truth, not filesystem."""
-    env = cli_env(disable_extractors=True)
-
-    _create_snapshot_rows(initialized_archive, env, "https://example.com")
-
-    # Verify DB has snapshot
-    with use_archivebox_db(initialized_archive):
-        db_count = Snapshot.objects.count()
-
-    assert db_count == 1
-
-    # Status should reflect DB count
-    result = run_archivebox_cmd(["status"], cwd=initialized_archive)
-    assert "1" in result.stdout
-
-
-def test_status_shows_index_file_info(initialized_archive):
-    """Test status shows index file information."""
-    result = run_archivebox_cmd(["status"], cwd=initialized_archive)
-
-    # Should mention index
-    assert "index" in result.stdout.lower() or "Index" in result.stdout
-
-
 def test_status_help_lists_available_options(initialized_archive):
     """Test that status --help works and documents the command."""
     result = run_archivebox_cmd(
@@ -174,10 +116,3 @@ def test_status_help_lists_available_options(initialized_archive):
 
     assert result.returncode == 0
     assert "status" in result.stdout.lower() or "statistic" in result.stdout.lower()
-
-
-def test_status_shows_data_directory_path(initialized_archive):
-    """Test that status reports which collection directory it is inspecting."""
-    result = run_archivebox_cmd(["status"], cwd=initialized_archive)
-
-    assert "archive" in result.stdout.lower() or str(initialized_archive) in result.stdout

archivebox/tests/test_cli_update.py

@@ -19,6 +19,50 @@ from archivebox.tests.test_orm_helpers import use_archivebox_db
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
+def test_rescan_recovers_already_migrated_idless_result(initialized_archive, tmp_path):
+    """Repair collections already migrated by versions affected by #1899."""
+    env = cli_env(disable_extractors=True)
+    run_archivebox_cmd(["add", "--index-only", "https://example.com/legacy-result"], env=env, check=True)
+    run_queued_crawls(initialized_archive, env)
+    with use_archivebox_db(initialized_archive):
+        snapshot = Snapshot.objects.get(url="https://example.com/legacy-result")
+        snapshot.write_index_jsonl()
+        output_dir = snapshot.output_dir
+        snapshot_id = snapshot.pk
+    payload = b"<html>Recovered after a previous migration</html>"
+    (output_dir / "singlefile.html").write_bytes(payload)
+    with (output_dir / "index.jsonl").open("a") as index:
+        index.write(
+            json.dumps(
+                {
+                    "type": "ArchiveResult",
+                    "snapshot_id": str(snapshot_id),
+                    "plugin": "singlefile",
+                    "hook_name": "",
+                    "status": "succeeded",
+                    "output_str": "singlefile.html",
+                    "start_ts": "2021-03-07T23:56:16-08:00",
+                },
+            )
+            + "\n",
+        )
+    result_id = None
+    for expected in ("Repaired: 1", "Unchanged: 1"):
+        result = run_archivebox_cmd(["update", "--rescan", "--migrate-only"], env=env, timeout=60)
+        assert result.returncode == 0, result.stdout + result.stderr
+        assert expected in result.stdout
+        with use_archivebox_db(initialized_archive):
+            archived = ArchiveResult.objects.get(snapshot_id=snapshot_id, plugin="singlefile")
+            assert archived.status == "succeeded"
+            assert archived.output_size == len(payload)
+            assert archived.output_files
+            assert archived.snapshot.output_size == len(payload)
+            if result_id is not None:
+                assert archived.pk == result_id
+            result_id = archived.pk
+        assert (output_dir / "singlefile.html").read_bytes() == payload
+
+
 def test_rescan_imports_current_layout_and_is_idempotent(initialized_archive, tmp_path):
     env = cli_env(disable_extractors=True)
     run_archivebox_cmd(["add", "--index-only", "https://example.com/rescan"], cwd=initialized_archive, env=env, check=True)
@@ -284,3 +328,39 @@ def test_update_seals_migrated_snapshots(initialized_archive):
         status = Snapshot.objects.values_list("status", flat=True).get()
 
     assert status == "sealed"
+
+
+@pytest.mark.parametrize("captured", [False, True])
+def test_index_only_distinguishes_uncaptured_metadata_from_missing_archive(initialized_archive, recursive_test_site, captured):
+    url = recursive_test_site["root_url"]
+    env = cli_env(disable_extractors=True)
+    if captured:
+        result = run_archivebox_cmd(["add", "--plugins=wget", url], cwd=initialized_archive, env=env, timeout=120)
+        assert result.returncode == 0, result.stdout + result.stderr
+    else:
+        created = run_archivebox_cmd(["snapshot", "create", url], cwd=initialized_archive, env=env, check=True)
+        sealed = run_archivebox_cmd(
+            ["snapshot", "update", "--status=sealed"],
+            cwd=initialized_archive,
+            env=env,
+            input=created.stdout,
+        )
+        assert sealed.returncode == 0, sealed.stdout + sealed.stderr
+    with use_archivebox_db(initialized_archive):
+        snapshot = Snapshot.objects.get(url=url)
+        directory = snapshot.output_dir
+        assert snapshot.status == Snapshot.StatusChoices.SEALED
+        if captured:
+            assert snapshot.downloaded_at is not None
+            assert snapshot.archiveresult_set.filter(plugin="wget", status="succeeded").exists()
+            directory.rename(directory.with_name(directory.name + ".removed"))
+        else:
+            assert snapshot.downloaded_at is None
+            assert not snapshot.archiveresult_set.exists()
+        assert not directory.exists()
+    result = run_archivebox_cmd(["update", "--index-only"], cwd=initialized_archive, env=env, timeout=120)
+    if captured:
+        assert result.returncode != 0
+        assert "Missing snapshot directory" in result.stdout + result.stderr
+    else:
+        assert result.returncode == 0, result.stdout + result.stderr

archivebox/tests/test_cli_update_reindex_snapshots.py

@@ -15,6 +15,49 @@ from archivebox.tests.test_orm_helpers import use_archivebox_db
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
+@pytest.mark.parametrize("index_format", ["history", "archive_results"])
+def test_rescan_imports_legacy_result_rows_and_saved_content(tmp_path, initialized_archive, index_format):
+    """Issue #1899: orphan metadata must become visible results on the first import."""
+    env = cli_env(disable_extractors=True)
+    legacy_dir = tmp_path / "archive" / "1615190176"
+    legacy_dir.mkdir(parents=True)
+    payload = b"<html><title>Recovered capture</title><body>saved content</body></html>"
+    (legacy_dir / "singlefile.html").write_bytes(payload)
+    entry = {
+        "plugin": "singlefile",
+        "status": "succeeded",
+        "output": "singlefile.html",
+        "start_ts": "2021-03-07T23:56:16-08:00",
+        "end_ts": "2021-03-07T23:56:16-08:00",
+    }
+    metadata = {
+        "url": "https://example.com/recovered",
+        "timestamp": legacy_dir.name,
+        "title": "Recovered capture",
+        index_format: {"singlefile": [entry]} if index_format == "history" else [entry],
+    }
+    original_json = json.dumps(metadata)
+    (legacy_dir / "index.json").write_text(original_json)
+    result_id = None
+    for _ in range(2):
+        command = run_archivebox_cmd(["update", "--rescan", "--migrate-only"], env=env, timeout=60)
+        assert command.returncode == 0, command.stdout + command.stderr
+        with use_archivebox_db(tmp_path):
+            snapshot = Snapshot.objects.get(url=metadata["url"])
+            result = snapshot.archiveresult_set.get(plugin="singlefile")
+            assert result.status == "succeeded"
+            assert result.output_size == len(payload)
+            assert result.output_files
+            assert snapshot.status == "sealed"
+            assert snapshot.archiveresult_set.count() == 1
+            assert snapshot.output_size == len(payload)
+            assert (snapshot.output_dir / "singlefile.html").read_bytes() == payload
+            assert (snapshot.output_dir / "index.json").read_text() == original_json
+            if result_id is not None:
+                assert result.pk == result_id
+            result_id = result.pk
+
+
 def test_update_imports_orphaned_snapshots(tmp_path, initialized_archive):
     """Test that archivebox update imports real legacy archive directories."""
     env = cli_env(disable_extractors=True)

archivebox/tests/test_config_DELETE_AFTER.py

@@ -160,6 +160,10 @@ print(json.dumps({{
 
 
 def test_delete_after_real_add_page_and_rest_create_paths(client):
+    from archivebox.personas.models import Persona
+
+    # Match init/server startup; visiting /add/ must not create persona records.
+    Persona.get_or_create_default()
     User = get_user_model()
     admin_user = User.objects.create_superuser(
         username="retentionadmin",

archivebox/tests/test_config_MAX_limits.py

@@ -203,6 +203,7 @@ def test_seal_snapshot_cancels_queued_descendants_after_crawl_max_size():
 
     Snapshot.objects.filter(pk=root.pk).update(output_size=32)
 
+    root_output_dir = str(root.output_dir)
     bus = create_bus(name=f"test_snapshot_limit_cancel_{str(crawl.id).replace('-', '_')}")
     SnapshotService(bus, crawl_id=str(crawl.id))
     try:
@@ -212,14 +213,14 @@ def test_seal_snapshot_cancels_queued_descendants_after_crawl_max_size():
                 SnapshotEvent(
                     url=root.url,
                     snapshot_id=str(root.id),
-                    output_dir=str(root.output_dir),
+                    output_dir=root_output_dir,
                 ),
             )
             await snapshot_event.now()
             completed_event = SnapshotCompletedEvent(
                 url=root.url,
                 snapshot_id=str(root.id),
-                output_dir=str(root.output_dir),
+                output_dir=root_output_dir,
             )
             completed_event.event_parent_id = snapshot_event.event_id
             await bus.emit(completed_event).now()

archivebox/tests/test_config_ONLY_NEW.py

@@ -93,4 +93,6 @@ def test_create_discovered_snapshots_respects_only_new_exact_url_matches(admin_u
         crawl_link = (
             Path(crawl.output_dir) / CONSTANTS.SNAPSHOTS_DIR_NAME / snapshot.extract_domain_from_url(snapshot.url) / str(snapshot.id)
         )
-        assert crawl_link.is_symlink()
+        # Queued metadata stays in the DB; the normal runner creates links.
+        assert not crawl_link.is_symlink()
+        assert not snapshot.output_dir.exists()

archivebox/tests/test_core_config.py

@@ -1,4 +1,5 @@
 import os
+from pathlib import Path
 
 import pytest
 from pydantic import ValidationError
@@ -95,3 +96,40 @@ def test_resolving_scoped_config_does_not_mutate_process_environment(tmp_path):
             os.environ.pop("ABXPKG_LIB_DIR", None)
         else:
             os.environ["ABXPKG_LIB_DIR"] = previous_lib_dir
+
+
+@pytest.mark.parametrize("key", ["COOKIES_FILE", "AUTH_STORAGE_FILE", "WGET_COOKIES_FILE", "YTDLP_COOKIES_FILE", "GALLERYDL_COOKIES_FILE"])
+def test_cookie_paths_remain_collection_relative_after_runtime_overlays(key, tmp_path):
+    from archivebox.config.common import ArchiveBoxConfig
+
+    config = ArchiveBoxConfig(**{key: "auth/cookies.txt"})
+    assert str(config[key]) == str(CONSTANTS.DATA_DIR / "auth/cookies.txt")
+    runtime = config.for_crawl_runtime(runtime_overrides={key: "other/cookies.txt"})
+    assert runtime[key] == str(CONSTANTS.DATA_DIR / "other/cookies.txt")
+    absolute = tmp_path / "absolute-cookies.txt"
+    config = ArchiveBoxConfig(**{key: str(absolute)})
+    assert str(config[key]) == str(absolute)
+
+
+@pytest.mark.parametrize("key", ["COOKIES_FILE", "AUTH_STORAGE_FILE"])
+@pytest.mark.parametrize("value", ["", "  ", None])
+def test_empty_cookie_path_disables_import(key, value):
+    from archivebox.config.common import ArchiveBoxConfig
+
+    config = ArchiveBoxConfig(**{key: value})
+    assert config[key] is None
+    assert key not in config.for_crawl_runtime()
+
+
+def test_cookie_runtime_path_preserves_configured_symlink(tmp_path):
+    from archivebox.config.common import ArchiveBoxConfig
+
+    export = tmp_path / "export.txt"
+    export.write_text("# Netscape HTTP Cookie File\n")
+    configured = tmp_path / "cookies.txt"
+    configured.symlink_to(export)
+    config = ArchiveBoxConfig(COOKIES_FILE=configured)
+    assert str(config.COOKIES_FILE) == str(configured)
+    runtime = config.for_crawl_runtime()
+    assert runtime["COOKIES_FILE"] == str(configured)
+    assert Path(runtime["COOKIES_FILE"]).read_bytes() == export.read_bytes()

archivebox/tests/test_crawl_history.py

@@ -0,0 +1,111 @@
+"""Completed captures belong in the database, not a crawl-long event history."""
+
+import asyncio
+import gc
+import json
+import threading
+import weakref
+from collections import Counter
+
+import pytest
+
+from abx_dl.events import ProcessCompletedEvent, SnapshotCompletedEvent, SnapshotEvent
+from werkzeug import Response
+from archivebox.base_models.models import get_or_create_system_user_pk
+from archivebox.core.models import ArchiveResult, Snapshot
+from archivebox.crawls.models import Crawl
+from archivebox.services.runner import CrawlRunner, run_install
+
+
+@pytest.mark.django_db(transaction=True)
+@pytest.mark.parametrize("timeout_page", [False, True])
+def test_completed_snapshots_expire_after_database_projection(tmp_path, httpserver, hermetic_lib_dir, timeout_page):
+    urls = []
+    response_started = threading.Event()
+    release_response = threading.Event()
+
+    def stalled_response(_request):
+        def body():
+            response_started.set()
+            yield b"<html><head><title>Saved page 3</title></head><body>"
+            release_response.wait()
+            yield b"Completed response after the browser timed out.</body></html>"
+
+        return Response(body(), content_type="text/html")
+
+    httpserver.expect_request("/favicon.ico").respond_with_data("", status=204)
+    for index in range(8):
+        path = f"/page/{index}"
+        if timeout_page and index == 3:
+            httpserver.expect_request(path).respond_with_handler(stalled_response)
+            urls.append(httpserver.url_for(path))
+            continue
+        httpserver.expect_request(path).respond_with_data(
+            f"<html><head><title>Saved page {index}</title></head><body>"
+            "This real page is part of a multi-snapshot crawl whose completed captures "
+            "must remain available in the database after their events expire.</body></html>",
+            content_type="text/html",
+        )
+        urls.append(httpserver.url_for(path))
+    run_install(plugin_names=["title"])
+    crawl = Crawl.objects.create(
+        urls="\n".join(urls),
+        config={
+            "ABXPKG_LIB_DIR": str(hermetic_lib_dir),
+            "PLUGINS": "title",
+            "CRAWL_MAX_CONCURRENT_SNAPSHOTS": 1,
+            "CHROME_PAGELOAD_TIMEOUT": 5,
+        },
+        created_by_id=get_or_create_system_user_pk(),
+    )
+    runner = CrawlRunner(crawl, selected_plugins=["title"], show_progress=False)
+    runner.bus.event_history.max_history_size = 120
+    sizes = []
+    references = []
+    counts = Counter()
+    retired_ids = set()
+    retained_completed = []
+
+    def count_event(event):
+        counts[event.event_type] += 1
+
+    def observe_snapshot(event: SnapshotEvent):
+        sizes.append(sum(isinstance(previous, SnapshotEvent) for previous in runner.bus.event_history.values()))
+        retained_completed.append(len(retired_ids.intersection(runner.bus.event_history)))
+        references.append(weakref.ref(event))
+
+    async def observe_completed(event: SnapshotCompletedEvent):
+        root = await runner.bus.find(SnapshotEvent, snapshot_id=event.snapshot_id, past=True, future=False)
+        assert root is not None
+        retired_ids.add(root.event_id)
+        retired_ids.update(item.event_id for item in await runner.bus.filter("*", child_of=root, past=True, future=False))
+
+    def release_timed_out_response(event: ProcessCompletedEvent):
+        if response_started.is_set() and event.hook_name == "on_Snapshot__30_chrome_navigate" and event.status == "failed":
+            release_response.set()
+
+    runner.bus.on("*", count_event)
+    runner.bus.on(SnapshotEvent, observe_snapshot)
+    runner.bus.on(SnapshotCompletedEvent, observe_completed)
+    runner.bus.on(ProcessCompletedEvent, release_timed_out_response)
+    try:
+        asyncio.run(runner.run())
+    finally:
+        release_response.set()
+    (tmp_path / "event-counts.json").write_text(json.dumps({"events": counts, "snapshots_in_history": sizes}))
+    snapshots = list(Snapshot.objects.filter(crawl=crawl).order_by("url"))
+    assert len(snapshots) == len(urls)
+    assert all(snapshot.status == "sealed" for snapshot in snapshots)
+    for index, snapshot in enumerate(snapshots):
+        if timeout_page and index == 3:
+            assert response_started.is_set()
+            result = ArchiveResult.objects.get(snapshot=snapshot, hook_name="on_Snapshot__30_chrome_navigate", status="failed")
+            assert "timeout" in result.output_str.lower()
+            continue
+        result = ArchiveResult.objects.get(snapshot=snapshot, plugin="title", status="succeeded")
+        assert result.output_str == f"Saved page {index}"
+        assert (snapshot.output_dir / "title" / "title.txt").read_text().strip() == f"Saved page {index}"
+    assert sizes == [1] * len(urls), sizes
+    assert retained_completed == [0] * len(urls), retained_completed
+    gc.collect()
+    assert all(reference() is None for reference in references)

archivebox/tests/test_crawl_runner.py

@@ -71,6 +71,7 @@ def test_cancelled_crawl_projection_emits_abort_event_from_runner_bus():
         crawl=crawl,
         status=Snapshot.StatusChoices.STARTED,
     )
+    crawl_output_dir = str(crawl.output_dir)
     runner = CrawlRunner(crawl)
 
     async def run() -> CrawlAbortEvent | None:
@@ -90,7 +91,7 @@ def test_cancelled_crawl_projection_emits_abort_event_from_runner_bus():
             CrawlEvent(
                 url=snapshot.url,
                 snapshot_id=str(snapshot.id),
-                output_dir=str(crawl.output_dir),
+                output_dir=crawl_output_dir,
             ),
         ).now()
         await runner.bus.wait_until_idle()
@@ -323,6 +324,7 @@ def test_crawl_start_event_keeps_retry_at_lease():
         status=Snapshot.StatusChoices.QUEUED,
         retry_at=before,
     )
+    crawl_output_dir = str(crawl.output_dir)
     bus = create_bus(name="test_crawl_start_event_keeps_retry_at_lease")
     CrawlService(bus, crawl_id=str(crawl.id))
 
@@ -332,7 +334,7 @@ def test_crawl_start_event_keeps_retry_at_lease():
                 CrawlStartEvent(
                     url=snapshot.url,
                     snapshot_id=str(snapshot.id),
-                    output_dir=str(crawl.output_dir),
+                    output_dir=crawl_output_dir,
                 ),
             ).now()
             await bus.wait_until_idle()
@@ -979,6 +981,7 @@ def test_crawl_completed_event_requeues_active_snapshots():
         retry_at=None,
     )
 
+    crawl_output_dir = str(crawl.output_dir)
     bus = create_bus(name=f"test_crawl_completed_active_snapshots_{str(crawl.id).replace('-', '_')}")
     service = CrawlService(bus, crawl_id=str(crawl.id))
     assert service is not None
@@ -988,7 +991,7 @@ def test_crawl_completed_event_requeues_active_snapshots():
             event = CrawlCompletedEvent(
                 url="https://example.com",
                 snapshot_id="",
-                output_dir=str(crawl.output_dir),
+                output_dir=crawl_output_dir,
             )
             emitted = bus.emit(event)
             await emitted.wait()
@@ -1022,6 +1025,7 @@ def test_crawl_start_event_does_not_resurrect_cancelled_crawl():
         retry_at=now,
     )
 
+    crawl_output_dir = str(crawl.output_dir)
     bus = create_bus(name=f"test_crawl_start_cancelled_{str(crawl.id).replace('-', '_')}")
     service = CrawlService(bus, crawl_id=str(crawl.id))
     assert service is not None
@@ -1031,7 +1035,7 @@ def test_crawl_start_event_does_not_resurrect_cancelled_crawl():
             event = CrawlStartEvent(
                 url="https://example.com",
                 snapshot_id="",
-                output_dir=str(crawl.output_dir),
+                output_dir=crawl_output_dir,
             )
             emitted = bus.emit(event)
             await emitted.wait()
@@ -1069,6 +1073,7 @@ def test_crawl_cleanup_event_requeues_unfinished_crawl():
         retry_at=None,
     )
 
+    crawl_output_dir = str(crawl.output_dir)
     bus = create_bus(name=f"test_crawl_cleanup_requeues_unfinished_{str(crawl.id).replace('-', '_')}")
     service = CrawlService(bus, crawl_id=str(crawl.id))
     assert service is not None
@@ -1078,7 +1083,7 @@ def test_crawl_cleanup_event_requeues_unfinished_crawl():
             event = CrawlCleanupEvent(
                 url="https://example.com",
                 snapshot_id=str(snapshot.id),
-                output_dir=str(crawl.output_dir),
+                output_dir=crawl_output_dir,
             )
             emitted = bus.emit(event)
             await emitted.wait()
@@ -1117,6 +1122,7 @@ def test_crawl_completed_event_seals_finished_crawl():
         retry_at=None,
     )
 
+    crawl_output_dir = str(crawl.output_dir)
     bus = create_bus(name=f"test_crawl_completed_finished_crawl_{str(crawl.id).replace('-', '_')}")
     service = CrawlService(bus, crawl_id=str(crawl.id))
     assert service is not None
@@ -1126,7 +1132,7 @@ def test_crawl_completed_event_seals_finished_crawl():
             event = CrawlCompletedEvent(
                 url="https://example.com",
                 snapshot_id=str(snapshot.id),
-                output_dir=str(crawl.output_dir),
+                output_dir=crawl_output_dir,
             )
             emitted = bus.emit(event)
             await emitted.wait()
@@ -1165,6 +1171,7 @@ def test_snapshot_completed_event_bus_defers_finished_crawl_seal():
         retry_at=None,
     )
 
+    snapshot_output_dir = str(snapshot.output_dir)
     bus = create_bus(name=f"test_snapshot_completed_bus_finished_crawl_{str(crawl.id).replace('-', '_')}")
     SnapshotService(bus, crawl_id=str(crawl.id))
     try:
@@ -1174,14 +1181,14 @@ def test_snapshot_completed_event_bus_defers_finished_crawl_seal():
                 SnapshotEvent(
                     url="https://example.com",
                     snapshot_id=str(snapshot.id),
-                    output_dir=str(snapshot.output_dir),
+                    output_dir=snapshot_output_dir,
                 ),
             )
             await snapshot_event.now()
             completed_event = SnapshotCompletedEvent(
                 url="https://example.com",
                 snapshot_id=str(snapshot.id),
-                output_dir=str(snapshot.output_dir),
+                output_dir=snapshot_output_dir,
             )
             completed_event.event_parent_id = snapshot_event.event_id
             await bus.emit(completed_event).now()
@@ -1221,24 +1228,25 @@ def test_delayed_snapshot_completion_cannot_seal_new_run():
         retry_at=old_retry_at,
     )
 
+    snapshot_output_dir = str(snapshot.output_dir)
     bus = create_bus(name=f"test_delayed_snapshot_completion_{str(crawl.id).replace('-', '_')}")
     SnapshotService(bus, crawl_id=str(crawl.id))
     try:
 
         async def emit_runs() -> None:
             old_event = bus.emit(
-                SnapshotEvent(url=snapshot.url, snapshot_id=str(snapshot.id), output_dir=str(snapshot.output_dir)),
+                SnapshotEvent(url=snapshot.url, snapshot_id=str(snapshot.id), output_dir=snapshot_output_dir),
             )
             await old_event.now()
             await sync_to_async(Snapshot.objects.filter(pk=snapshot.pk).update, thread_sensitive=True)(retry_at=new_retry_at)
             new_event = bus.emit(
-                SnapshotEvent(url=snapshot.url, snapshot_id=str(snapshot.id), output_dir=str(snapshot.output_dir)),
+                SnapshotEvent(url=snapshot.url, snapshot_id=str(snapshot.id), output_dir=snapshot_output_dir),
             )
             await new_event.now()
             completed_event = SnapshotCompletedEvent(
                 url=snapshot.url,
                 snapshot_id=str(snapshot.id),
-                output_dir=str(snapshot.output_dir),
+                output_dir=snapshot_output_dir,
             )
             completed_event.event_parent_id = old_event.event_id
             await bus.emit(completed_event).now()

archivebox/tests/test_hook_result_lifecycle.py

@@ -70,6 +70,11 @@ def test_cancelled_hook_discards_result_without_deleting_partial_files(snapshot,
 
 @pytest.mark.parametrize("stop_mode", ["crash", "cancel"])
 def test_stopped_hook_reconciles_its_earlier_success(initialized_archive, stop_mode):
+    env = cli_env(CHROME_DELAY_AFTER_LOAD="60", CHROME_TIMEOUT="120", CHROME_HEADLESS="True")
+    # The deadline below measures a running hook's lifecycle, not a cold Chrome
+    # download/install. Install through the public CLI before starting that clock.
+    installed = run_archivebox_cmd(["install", "chrome"], cwd=initialized_archive, env=env, timeout=600)
+    assert installed.returncode == 0, installed.stdout + installed.stderr
     output = initialized_archive / "crash-check.log"
     cli = None
     with output.open("w") as log:
@@ -77,7 +82,7 @@ def test_stopped_hook_reconciles_its_earlier_success(initialized_archive, stop_m
             cli = run_archivebox_cmd(
                 ["add", "--plugins=chrome", "https://example.com"],
                 cwd=initialized_archive,
-                env=cli_env(CHROME_DELAY_AFTER_LOAD="60", CHROME_TIMEOUT="120", CHROME_HEADLESS="True"),
+                env=env,
                 stdin=subprocess.DEVNULL,
                 stdout=log,
                 stderr=log,

archivebox/tests/test_hooks.py

@@ -218,13 +218,6 @@ class TestRequiredBinaryConfigHandling:
         assert result.returncode == 0, result.stderr
         assert "Wget" in result.stdout
 
-    def test_binary_env_var_empty_default(self):
-        """The shipped wget schema should retain wget as its required binary."""
-        config = json.loads(files("abx_plugins.plugins.wget").joinpath("config.json").read_text())
-
-        assert config["required_binaries"][0]["name"] == "{WGET_BINARY}"
-        assert config["properties"]["WGET_BINARY"]["default"] == "wget"
-
 
 class TestHookDiscovery:
     """Test hook discovery functions."""
@@ -455,38 +448,6 @@ class TestHookExecution:
         assert process.exit_code == 0, process.stderr
         assert "chrome zombies" in process.stdout
 
-    def test_hook_receives_cli_args(self, tmp_path):
-        """Hook should receive CLI arguments."""
-        snap_dir = tmp_path / "snapshot"
-        output_dir = snap_dir / "hashes"
-        output_dir.mkdir(parents=True)
-        (snap_dir / "source.txt").write_text("real CLI argument input", encoding="utf-8")
-        hook_path = Path(
-            str(
-                files("abx_plugins.plugins.hashes").joinpath(
-                    "on_Snapshot__93_hashes.py",
-                ),
-            ),
-        )
-
-        result = subprocess.run(
-            [str(hook_path), "--url=https://example.com/real-hook-argument"],
-            cwd=output_dir,
-            capture_output=True,
-            text=True,
-            env={**os.environ, "SNAP_DIR": str(snap_dir)},
-            timeout=30,
-        )
-
-        assert result.returncode == 0, result.stderr
-        from archivebox.machine.models import Process
-
-        records = Process.parse_records_from_text(result.stdout)
-        source_hash = hashlib.sha256(b"real CLI argument input").hexdigest()
-        assert records == [{"type": "ArchiveResult", "status": "succeeded", "output_str": f"0.0MB {source_hash[:12]}"}]
-        hashes = json.loads((output_dir / "hashes.json").read_text())
-        assert hashes["files"][0]["hash"] == source_hash
-
 
 class TestDependencyRecordOutput:
     """Test Binary JSONL emitted by the real CLI and persisted model."""

archivebox/tests/test_issue_1896.py

@@ -0,0 +1,193 @@
+"""Regression coverage for collection cookie paths and config edits on upgrade."""
+
+import json
+import sqlite3
+from contextlib import closing
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+import pytest
+
+from archivebox.config.configset import read_ini_config
+from archivebox.tests.conftest import run_archivebox_cmd
+
+
+@pytest.mark.parametrize("persona_name", ["Default", "default"])
+def test_add_reuses_persona_ignoring_case(initialized_archive, httpserver, persona_name):
+    """Either spelling reuses the same authenticated persona without moving files."""
+    run_archivebox_cmd(["persona", "create", "Default", "default"], cwd=initialized_archive, check=True)
+    persona_dir = initialized_archive / "personas" / "Default"
+    (persona_dir / "auth.json").touch()
+    (persona_dir / "cookies.txt").write_text(
+        "# Netscape HTTP Cookie File\n127.0.0.1\tFALSE\t/\tFALSE\t2147483647\tissue1896\timported\n",
+    )
+    original_inode = (persona_dir / "cookies.txt").stat().st_ino
+    httpserver.expect_request("/").respond_with_data(
+        "<html><head><title>Existing persona capture</title></head><body>"
+        '<output id="cookies"></output><script>document.getElementById("cookies").textContent = document.cookie;</script>'
+        "</body></html>",
+        content_type="text/html",
+    )
+    result = run_archivebox_cmd(
+        ["add", "--persona", persona_name, "--plugins=dom", httpserver.url_for("/").replace("localhost", "127.0.0.1")],
+        cwd=initialized_archive,
+        timeout=180,
+        env={"CHROME_SANDBOX": "false"},
+    )
+    assert result.returncode == 0, result.stdout + result.stderr
+    outputs = list((initialized_archive / "archive" / "users").glob("*/snapshots/**/dom/output.html"))
+    assert len(outputs) == 1, result.stdout + result.stderr
+    assert "issue1896=imported" in outputs[0].read_text()
+    assert (persona_dir / "cookies.txt").stat().st_ino == original_inode
+    with closing(sqlite3.connect(initialized_archive / "index.sqlite3")) as db:
+        assert db.execute("SELECT name FROM personas_persona ORDER BY name").fetchall() == [("Default",)]
+        assert db.execute("SELECT status FROM core_archiveresult WHERE plugin='dom'").fetchall() == [("succeeded",)]
+        hook_envs = db.execute("SELECT env FROM machine_process WHERE process_type='hook'").fetchall()
+        assert hook_envs
+        assert all(json.loads(row[0])["ACTIVE_PERSONA"] == "Default" for row in hook_envs)
+
+
+@pytest.mark.parametrize("contents,explicit", [("", True), ("not JSON", True), ("not JSON", False)])
+def test_add_rejects_invalid_persona_auth_with_file_path(initialized_archive, httpserver, contents, explicit):
+    persona_dir = initialized_archive / "personas" / "Default"
+    persona_dir.mkdir(parents=True, exist_ok=True)
+    auth_file = persona_dir / "auth.json"
+    auth_file.write_text(contents)
+    if explicit:
+        run_archivebox_cmd(
+            ["config", "--set", "AUTH_STORAGE_FILE=personas/Default/auth.json"],
+            cwd=initialized_archive,
+            check=True,
+        )
+    result = run_archivebox_cmd(
+        ["add", "--plugins=dom", httpserver.url_for("/")],
+        cwd=initialized_archive,
+        timeout=180,
+        env={"CHROME_SANDBOX": "false"},
+    )
+    assert result.returncode == 1, result.stdout + result.stderr
+    logs = list((initialized_archive / "archive" / "users").glob("*/crawls/**/chrome/*.stderr.log"))
+    assert logs
+    assert f"Invalid JSON cookie export: {auth_file}" in "\n".join(log.read_text() for log in logs)
+    assert not list((initialized_archive / "archive" / "users").glob("*/snapshots/**/dom/output.html"))
+    assert auth_file.read_text() == contents
+
+
+@pytest.mark.parametrize("cookie_file_exists", [True, False])
+@pytest.mark.parametrize("isolation", ["crawl", "snapshot"])
+def test_add_with_empty_autodiscovered_persona_auth(initialized_archive, httpserver, cookie_file_exists, isolation):
+    """An empty auth placeholder must not block a crawl or hide Netscape cookies."""
+    persona_dir = initialized_archive / "personas" / "Default"
+    persona_dir.mkdir(parents=True, exist_ok=True)
+    auth_file = persona_dir / "auth.json"
+    auth_file.touch()
+    cookies_file = persona_dir / "cookies.txt"
+    cookies_file.write_text(
+        "# Netscape HTTP Cookie File\n127.0.0.1\tFALSE\t/\tFALSE\t2147483647\tissue1896\timported\n" if cookie_file_exists else "",
+    )
+    httpserver.expect_request("/").respond_with_data(
+        '<html><title>Empty persona auth</title><body><output id="cookies"></output>'
+        '<script>document.getElementById("cookies").textContent = document.cookie;</script></body></html>',
+        content_type="text/html",
+    )
+    result = run_archivebox_cmd(
+        ["add", "--plugins=dom", httpserver.url_for("/").replace("localhost", "127.0.0.1")],
+        cwd=initialized_archive,
+        timeout=180,
+        env={"CHROME_SANDBOX": "false", "CHROME_ISOLATION": isolation},
+    )
+    assert result.returncode == 0, result.stdout + result.stderr
+    outputs = list((initialized_archive / "archive" / "users").glob("*/snapshots/**/dom/output.html"))
+    assert len(outputs) == 1, result.stdout + result.stderr
+    assert ("issue1896=imported" in outputs[0].read_text()) == cookie_file_exists
+    assert auth_file.read_bytes() == b""
+    with closing(sqlite3.connect(initialized_archive / "index.sqlite3")) as db:
+        assert db.execute("SELECT status FROM core_snapshot").fetchall() == [("sealed",)]
+        assert db.execute("SELECT status FROM core_archiveresult WHERE plugin='dom'").fetchall() == [("succeeded",)]
+        hook_rows = db.execute("SELECT cmd, env, exit_code FROM machine_process WHERE process_type='hook'").fetchall()
+        assert hook_rows
+        for cmd, env, exit_code in hook_rows:
+            hook_env = json.loads(env)
+            assert not hook_env.get("AUTH_STORAGE_FILE")
+            expected_cookies = cookies_file if isolation == "crawl" else Path(hook_env["PERSONAS_DIR"]) / "Default" / "cookies.txt"
+            assert hook_env["COOKIES_FILE"] == str(expected_cookies)
+            hook_name = Path(json.loads(cmd)[0]).name
+            # The crawl wait hook explicitly skips when each snapshot owns Chrome.
+            assert exit_code == (10 if isolation == "snapshot" and hook_name == "on_CrawlSetup__91_chrome_wait.js" else 0)
+
+
+@pytest.mark.parametrize("cookie_file_exists", [True, False])
+@pytest.mark.parametrize("absolute_path", [False, True], ids=["collection-relative", "absolute"])
+def test_add_with_collection_relative_cookies(initialized_archive, httpserver, cookie_file_exists, absolute_path):
+    """Real hooks resolve relative paths against DATA_DIR and preserve absolute paths."""
+    cookies_file = (
+        initialized_archive.parent / f"{initialized_archive.name}-cookies.txt" if absolute_path else initialized_archive / "cookies.txt"
+    )
+    configured_path = str(cookies_file) if absolute_path else "cookies.txt"
+    if cookie_file_exists:
+        cookies_file.write_text(
+            "# Netscape HTTP Cookie File\n127.0.0.1\tFALSE\t/\tFALSE\t2147483647\tissue1896\timported\n",
+        )
+    httpserver.expect_request("/").respond_with_data(
+        "<html><title>Issue 1896</title><body><p>Collection cookie import regression: "
+        "this page must still be archived when the optional cookie file is absent.</p>"
+        '<output id="cookies"></output><script>document.getElementById("cookies").textContent = document.cookie;</script></body></html>',
+        content_type="text/html",
+    )
+    run_archivebox_cmd(
+        ["config", "--set", f"COOKIES_FILE={configured_path}"],
+        cwd=initialized_archive,
+        check=True,
+    )
+    result = run_archivebox_cmd(
+        ["add", "--plugins=dom", httpserver.url_for("/").replace("localhost", "127.0.0.1")],
+        cwd=initialized_archive,
+        timeout=180,
+        env={"CHROME_SANDBOX": "false"},
+    )
+    assert result.returncode == 0, result.stdout + result.stderr
+    outputs = list((initialized_archive / "archive" / "users").glob("*/snapshots/**/dom/output.html"))
+    assert len(outputs) == 1, result.stdout + result.stderr
+    html = outputs[0].read_text()
+    assert ("issue1896=imported" in html) == cookie_file_exists
+    launch_logs = list((initialized_archive / "archive" / "users").glob("*/crawls/**/chrome/*.stderr.log"))
+    logs = "\n".join(log.read_text() for log in launch_logs)
+    if cookie_file_exists:
+        assert f"from {cookies_file}" in logs
+    else:
+        assert "Cookies file not found" in logs
+        assert "continuing without cookie import" in logs
+    with closing(sqlite3.connect(initialized_archive / "index.sqlite3")) as db:
+        assert db.execute("SELECT status FROM core_snapshot").fetchall() == [("sealed",)]
+        assert db.execute("SELECT status FROM core_archiveresult WHERE plugin='dom'").fetchall() == [("succeeded",)]
+        hook_rows = db.execute("SELECT env, status, exit_code FROM machine_process WHERE process_type='hook'").fetchall()
+        assert hook_rows
+        for env, status, exit_code in hook_rows:
+            assert json.loads(env)["COOKIES_FILE"] == str(cookies_file)
+            assert (status, exit_code) == ("exited", 0)
+
+
+@pytest.mark.parametrize("enabled_key", ["USE_CHROME", "CHROME_ENABLED"])
+@pytest.mark.parametrize("startup_save", ["binary_sanitation", "host_refresh"])
+def test_init_preserves_edited_use_chrome_before_startup_save(initialized_archive, enabled_key, startup_save):
+    """Startup saves must not overwrite a newer file before config sync."""
+    config_path = initialized_archive / "ArchiveBox.conf"
+    config_path.write_text(config_path.read_text() + f"\n[SERVER_CONFIG]\n{enabled_key} = True\n")
+    run_archivebox_cmd(
+        ["config", "--set", f"WGET_BINARY={initialized_archive / 'removed-wget'}"]
+        if startup_save == "binary_sanitation"
+        else ["config", "--set", "TIMEOUT=37"],
+        cwd=initialized_archive,
+        check=True,
+    )
+    if startup_save == "host_refresh":
+        with closing(sqlite3.connect(initialized_archive / "index.sqlite3")) as db:
+            db.execute("UPDATE machine_machine SET modified_at=?", [(datetime.now(timezone.utc) - timedelta(days=8)).isoformat()])
+            db.commit()
+    config_path.write_text(config_path.read_text().replace(f"{enabled_key} = True", f"{enabled_key} = False"))
+    assert read_ini_config(config_path)[enabled_key] == "False"
+    run_archivebox_cmd(["init"], cwd=initialized_archive, timeout=120, check=True)
+    assert read_ini_config(config_path)[enabled_key] == "False"
+    result = run_archivebox_cmd(["config", "--get", "CHROME_ENABLED"], cwd=initialized_archive)
+    assert result.returncode == 0, result.stderr
+    assert "CHROME_ENABLED = false" in result.stdout

archivebox/tests/test_machine_models.py

@@ -1005,3 +1005,58 @@ class TestProcessClassMethods:
 
 if __name__ == "__main__":
     pytest.main([__file__, "-v"])
+
+
+@pytest.mark.parametrize("cleanup", ["cleanup_stale_running", "cleanup_orphaned_workers"])
+def test_process_cleanup_releases_reads_before_process_checks(cleanup, tmp_path):
+    """Another real connection can write while recovery inspects live processes."""
+    import sqlite3
+    from django.db import connection
+
+    from archivebox.tests.conftest import init_archive
+    from archivebox.tests.test_orm_helpers import use_archivebox_db
+
+    init_archive(tmp_path)
+    with use_archivebox_db(tmp_path):
+        machine = Machine.current()
+        processes = [
+            Process(
+                machine=machine,
+                process_type=Process.TypeChoices.HOOK,
+                status=Process.StatusChoices.RUNNING,
+                pid=os.getpid(),
+                started_at=_current_process_started_at(),
+            )
+            for _ in range(105)
+        ]
+        Process.objects.bulk_create(processes)
+        with connection.cursor() as cursor:
+            cursor.execute("PRAGMA journal_mode=DELETE")
+        blocked = []
+        checks = []
+        writer = sqlite3.connect(connection.settings_dict["NAME"], timeout=0.01, uri=True)
+
+        def observe(frame, event, arg):
+            if (
+                event == "call"
+                and frame.f_code.co_name == ("poll" if cleanup == "cleanup_stale_running" else "is_running")
+                and isinstance(frame.f_locals.get("self"), Process)
+            ):
+                checks.append(1)
+                try:
+                    writer.execute("UPDATE machine_machine SET hostname=hostname WHERE id=?", [machine.pk.hex])
+                    writer.commit()
+                except sqlite3.OperationalError as error:
+                    writer.rollback()
+                    blocked.append(str(error))
+
+        previous = sys.getprofile()
+        sys.setprofile(observe)
+        try:
+            getattr(Process, cleanup)()
+        finally:
+            sys.setprofile(previous)
+            writer.close()
+        assert len(checks) >= len(processes)
+        assert blocked == []
+        assert Process.objects.filter(pk__in=[p.pk for p in processes], status=Process.StatusChoices.RUNNING).count() == len(processes)

archivebox/tests/test_opencode_agent_1.py

@@ -185,10 +185,9 @@ def test_opencode_starts_with_isolated_state(admin_client, live_opencode):
     assert not (Path(workdir) / ".git").exists()
     agent = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
     assert agent.status_code == 200
-    from abx_plugins.plugins.opencode import runtime
 
     frame = admin_client.get(
-        runtime._project_route(live_opencode.config.data_dir),
+        agent.context["proxy_url"],
         HTTP_HOST=ADMIN_TEST_HOST,
     )
     assert frame.status_code == 302

archivebox/tests/test_opencode_agent_2.py

@@ -1,13 +1,8 @@
 import asyncio
-import json
 import os
 import re
-from concurrent.futures import ThreadPoolExecutor
-from concurrent.futures import TimeoutError as FutureTimeoutError
 from urllib.parse import parse_qs, urlsplit
 
-import pytest
-import requests
 from asgiref.testing import ApplicationCommunicator
 
 from archivebox.tests.conftest import ADMIN_TEST_HOST
@@ -60,10 +55,10 @@ def test_opencode_agent_superuser_gets_admin_wrapper(admin_client, live_opencode
     from abx_plugins.plugins.opencode import runtime
 
     response = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
-    frame_path = runtime._project_route(live_opencode.config.data_dir)
+    frame_path = response.context["proxy_url"]
 
     assert response.status_code == 200
-    assert f'<iframe src="{frame_path}"'.encode() in response.content
+    assert f'<iframe data-src="{frame_path}"'.encode() in response.content
     assert b'id="opencode-agent-welcome"' in response.content
     assert b'id="header"' in response.content
     assert b'id="progress-monitor"' in response.content
@@ -94,45 +89,6 @@ def test_opencode_agent_superuser_gets_admin_wrapper(admin_client, live_opencode
     assert session.headers["Content-Security-Policy"] == "frame-ancestors 'self'"
 
 
-def test_opencode_oauth_callback_waits_for_user_and_preserves_cancellation(live_opencode):
-    from abx_plugins.plugins.opencode import runtime
-
-    settings = {**live_opencode.settings, "timeout": 1}
-    headers = {"Content-Type": "application/json"}
-    status, _, body = runtime.proxy(settings, "POST", "provider/openai/oauth/authorize", (), headers, b'{"method":0}')
-    assert status == 200
-    authorization = json.loads(body)
-    assert urlsplit(authorization["url"]).hostname == "auth.openai.com"
-    redirect = parse_qs(urlsplit(authorization["url"]).query)["redirect_uri"][0]
-    callback_origin = urlsplit(redirect)
-    assert callback_origin.hostname == "localhost"
-
-    with ThreadPoolExecutor(max_workers=1) as executor:
-        callback = executor.submit(runtime.proxy, settings, "POST", "provider/openai/oauth/callback", (), headers, b'{"method":0}')
-        try:
-            # A real pending authorization must outlive the ordinary API read
-            # timeout. No tokens or substituted provider responses are used.
-            with pytest.raises(FutureTimeoutError):
-                callback.result(timeout=2)
-        finally:
-            cancelled = requests.get(f"{callback_origin.scheme}://{callback_origin.netloc}/cancel", timeout=5)
-            assert cancelled.status_code == 200
-            assert cancelled.text == "Login cancelled"
-        status, response_headers, body = callback.result(timeout=5)
-    assert status == 500
-    assert response_headers["Content-Type"].startswith("application/json")
-    error = json.loads(body)
-    assert error["name"] == "UnknownError"
-    assert error["data"]["ref"].startswith("err_")
-    runtime._stop_owned_process()
-    # The pnpm launcher can exit before its server child. Stopping the owned
-    # process must release both listeners so another collection can authorize.
-    with pytest.raises(requests.ConnectionError):
-        requests.get(settings["origin"] + "/global/health", timeout=2)
-    with pytest.raises(requests.ConnectionError):
-        requests.get(f"{callback_origin.scheme}://{callback_origin.netloc}/cancel", timeout=2)
-
-
 def test_opencode_static_assets_cache_privately_and_revalidate(admin_client, live_opencode):
     headers = {"HTTP_HOST": ADMIN_TEST_HOST, "HTTP_SEC_FETCH_SITE": "same-origin"}
     page = admin_client.get("/admin/agent/opencode/", **headers)
@@ -230,7 +186,7 @@ def test_opencode_invalid_state_does_not_break_archivebox(admin_client, live_ope
     wrapper = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
     assert wrapper.status_code == 200
     for url in (
-        runtime._project_route(live_opencode.config.data_dir),
+        wrapper.context["proxy_url"],
         "/admin/agent/opencode/global/health",
     ):
         response = admin_client.get(url, HTTP_HOST=ADMIN_TEST_HOST)

archivebox/tests/test_opencode_agent_3.py

@@ -44,7 +44,7 @@ def test_opencode_cold_agent_wrapper_returns_before_server_starts(admin_client,
     assert time.monotonic() - started < 3
     assert not runtime._owned_process_running()
     assert b'id="opencode-agent-welcome"' in response.content
-    assert f'<iframe src="{runtime._project_route(installed_opencode.config.data_dir)}"'.encode() in response.content
+    assert f'<iframe data-src="{response.context["proxy_url"]}"'.encode() in response.content
 
 
 def test_opencode_proxy_serves_real_project_and_session(admin_client, live_opencode):
@@ -53,9 +53,7 @@ def test_opencode_proxy_serves_real_project_and_session(admin_client, live_openc
 
     agent = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
     assert agent.status_code == 200
-    from abx_plugins.plugins.opencode import runtime
-
-    frame_path = runtime._project_route(live_opencode.config.data_dir)
+    frame_path = agent.context["proxy_url"]
     frame = admin_client.get(frame_path, HTTP_HOST=ADMIN_TEST_HOST)
     assert frame.status_code == 302
     session_id = frame.headers["Location"].rsplit("/", 1)[-1]
@@ -99,39 +97,17 @@ def test_opencode_proxy_restarts_server_for_an_existing_agent_page(admin_client,
         HTTP_SEC_FETCH_SITE="same-origin",
     )
 
-    assert response.status_code == 200
+    assert response.status_code == 200, {
+        "installed_lib": live_opencode.settings["config"].get("ABXPKG_LIB_DIR"),
+        "request_lib": response.wsgi_request.archivebox_config.ABXPKG_LIB_DIR,
+        "installed_binary": live_opencode.settings["binary"],
+        "request_binary": getattr(response.wsgi_request.archivebox_config, "OPENCODE_BINARY", None),
+    }
     assert runtime._PROCESS is not None
     assert runtime._PROCESS is not old_process
     assert runtime._PROCESS.poll() is None
 
 
-def test_concurrent_opencode_startup_waits_until_server_is_ready(live_opencode):
-    from abx_plugins.plugins.opencode import runtime
-
-    runtime._stop_owned_process()
-
-    with ThreadPoolExecutor(max_workers=2) as executor:
-        results = list(executor.map(runtime._ensure_opencode, [live_opencode.settings] * 2))
-
-    assert results == [(True, ""), (True, "")]
-    assert runtime._health(live_opencode.settings)
-
-
-def test_opencode_does_not_probe_or_replace_a_ready_owned_process(live_opencode):
-    from abx_plugins.plugins.opencode import runtime
-
-    process = runtime._PROCESS
-    settings = {**live_opencode.settings, "port": _free_port()}
-    settings["origin"] = f"http://{settings['host']}:{settings['port']}"
-
-    ok, error = runtime._ensure_opencode(settings)
-
-    assert ok, error
-    assert process is not None
-    assert runtime._PROCESS is process
-    assert process.poll() is None
-
-
 def test_opencode_proxy_does_not_wait_for_recovery_lock(admin_client, live_opencode):
     from abx_plugins.plugins.opencode import runtime
 

archivebox/tests/test_opencode_browser_1.py

@@ -88,25 +88,43 @@ const puppeteer = require('puppeteer');
 const config = JSON.parse(require('node:fs').readFileSync(0, 'utf8'));
 (async () => {
   const browser = await puppeteer.launch({executablePath: config.chrome, headless: true,
-    args: ['--no-sandbox', '--disable-frame-rate-limit']});
+    args: [...config.chromeArgs, '--no-sandbox']});
+  const requestTimings = new Map();
+  const browserErrors = [];
+  let page;
   try {
-    const page = await browser.newPage();
+    page = await browser.newPage();
     const escapedRequests = [];
+    const agentRequests = [];
     page.on('request', request => {
       const url = new URL(request.url());
+      requestTimings.set(request, {url: url.pathname, started: performance.now()});
+      if (url.pathname.startsWith('/admin/agent/opencode/')) agentRequests.push(url.pathname);
       if (request.frame()?.parentFrame() && url.origin === config.url &&
           !url.pathname.startsWith('/admin/agent/opencode/')) {
         escapedRequests.push(request.method() + ' ' + url.pathname);
       }
     });
+    page.on('requestfinished', request => {
+      const timing = requestTimings.get(request);
+      if (timing) Object.assign(timing, {status: request.response()?.status(), elapsed: performance.now() - timing.started});
+    });
+    page.on('requestfailed', request => {
+      const timing = requestTimings.get(request);
+      if (timing) Object.assign(timing, {error: request.failure(), elapsed: performance.now() - timing.started});
+    });
+    page.on('pageerror', error => browserErrors.push(String(error)));
     await page.setViewport({width: 1440, height: 900});
     await page.goto(config.url + '/admin/login/', {waitUntil: 'domcontentloaded'});
     await page.locator('#login-form input[name="username"]').fill('agent-browser-test');
     await page.locator('#login-form input[name="password"]').fill('test-password');
     await Promise.all([page.waitForNavigation({waitUntil: 'domcontentloaded'}),
       page.locator('#login-form input[type="submit"]').click()]);
-    assert.equal((await page.goto(config.url + '/admin/agent', {waitUntil: 'domcontentloaded'})).status(), 200);
-    await page.locator('#opencode-agent-welcome-dismiss').click();
+    assert.equal((await page.goto(config.url + '/admin/agent', {waitUntil: 'load'})).status(), 200);
+    // The welcome dialog must not start a hidden OpenCode app: its cold startup
+    // and asset load compete with rendering the very dialog the user must read.
+    assert.deepEqual(agentRequests, [], 'OpenCode started before the user dismissed its welcome dialog');
+    await page.locator('::-p-aria(Start using Agent[role="button"])').click();
     const frame = await (await page.waitForSelector('iframe')).contentFrame();
     await frame.waitForSelector('a[href*="/session"]');
     const links = await frame.$$eval('a[href*="/session"]', nodes => nodes.map(node => node.getAttribute('href')));
@@ -264,13 +282,32 @@ const config = JSON.parse(require('node:fs').readFileSync(0, 'utf8'));
     assert.ok(terminal.includes('ABX_TERMINAL_OK'), terminal);
     assert.deepEqual(escapedRequests, [], 'OpenCode requests must stay inside its proxy mount');
     assert.equal((await page.goto(config.url + '/add/', {waitUntil: 'domcontentloaded'})).status(), 200);
+    // A real click persists the dismissal: returning users load the Agent
+    // directly, without another click or seeded browser preferences.
+    assert.equal((await page.goto(config.url + '/admin/agent', {waitUntil: 'load'})).status(), 200);
+    await page.waitForSelector('::-p-aria(Start using Agent[role="button"])', {hidden: true});
+    const returningFrame = await (await page.waitForSelector('iframe[title="OpenCode Agent"]')).contentFrame();
+    await returningFrame.waitForSelector('::-p-aria(New session[role="button"])');
     console.log('AGENT_NAVIGATION_OK');
+  } catch (error) {
+    const frames = await Promise.all((page?.frames() || []).map(async frame => ({
+      url: frame.url(), body: await frame.$eval('body', node => node.innerText.slice(0, 4000)).catch(String),
+    })));
+    console.error(JSON.stringify({browserErrors, frames, requests: [...requestTimings.values()]}));
+    throw error;
   } finally { await browser.close(); }
 })().catch(error => { console.error(error); process.exitCode = 1; });
 """
     result = subprocess.run(
         [str(browser_runtime["node_binary"]), "-e", script],
-        input=json.dumps({"chrome": str(browser_runtime["chrome_binary"]), "url": server_url, "filename": filename}),
+        input=json.dumps(
+            {
+                "chrome": str(browser_runtime["chrome_binary"]),
+                "chromeArgs": browser_runtime["chrome_args"],
+                "url": server_url,
+                "filename": filename,
+            },
+        ),
         env={**os.environ, "NODE_PATH": browser_runtime["node_path"]},
         text=True,
         capture_output=True,

archivebox/tests/test_opencode_browser_3.py

@@ -62,8 +62,7 @@ const config = JSON.parse(require('node:fs').readFileSync(0, 'utf8'));
     const browser = await puppeteer.launch({
       executablePath: config.chrome,
       headless: true,
-      // Headless interaction must not depend on the desktop display clock.
-      args: ['--no-sandbox', '--disable-frame-rate-limit', ...(disabled ? ['--disable-local-storage'] : [])],
+      args: [...config.chromeArgs, '--no-sandbox', ...(disabled ? ['--disable-local-storage'] : [])],
     });
     try {
       const page = await browser.newPage();
@@ -149,6 +148,7 @@ const config = JSON.parse(require('node:fs').readFileSync(0, 'utf8'));
         input=json.dumps(
             {
                 "chrome": str(browser_runtime["chrome_binary"]),
+                "chromeArgs": browser_runtime["chrome_args"],
                 "url": f"{server_url}/admin/agent",
                 "workdir": str(data_dir.resolve()),
             },

archivebox/tests/test_persona_casing.py

@@ -0,0 +1,237 @@
+"""Persona names have the same identity on case-sensitive and insensitive disks."""
+
+import json
+import sqlite3
+import subprocess
+import sys
+from contextlib import closing
+
+import pytest
+from django.urls import reverse
+
+from archivebox.personas.models import Persona
+from archivebox.tests.conftest import ADMIN_TEST_HOST, api_client_request, get_free_port, run_archivebox_cmd, stop_server
+
+
+def test_cli_create_reuses_case_insensitive_persona(initialized_archive):
+    first = run_archivebox_cmd(["persona", "create", "--permissions=private", "personal"], cwd=initialized_archive, check=True)
+    second = run_archivebox_cmd(["persona", "create", "--permissions=public", "PERSONAL"], cwd=initialized_archive, check=True)
+    assert json.loads(second.stdout)["id"] == json.loads(first.stdout)["id"]
+    with closing(sqlite3.connect(initialized_archive / "index.sqlite3")) as db:
+        rows = db.execute("SELECT name, config FROM personas_persona WHERE name != 'Default'").fetchall()
+    assert len(rows) == 1
+    assert rows[0][0] == "personal"
+    assert json.loads(rows[0][1])["PERMISSIONS"] == "private"
+
+
+@pytest.mark.django_db
+def test_add_form_reuses_existing_lowercase_default(admin_client, admin_user):
+    original = Persona.objects.create(name="default", created_by=admin_user)
+    response = admin_client.get("/add/", HTTP_HOST=ADMIN_TEST_HOST)
+    assert response.status_code == 200
+    assert list(Persona.objects.values_list("id", "name")) == [(original.id, "default")]
+
+
+@pytest.mark.django_db
+def test_add_form_does_not_create_personas(admin_client):
+    assert not Persona.objects.exists()
+    response = admin_client.get("/add/", HTTP_HOST=ADMIN_TEST_HOST)
+    assert response.status_code == 200
+    assert not Persona.objects.exists()
+
+
+def test_init_creates_default_persona(initialized_archive):
+    with closing(sqlite3.connect(initialized_archive / "index.sqlite3")) as db:
+        assert db.execute("SELECT name FROM personas_persona").fetchall() == [("Default",)]
+    assert (initialized_archive / "personas" / "Default" / "chrome_profile").is_dir()
+
+
+def test_server_initializes_default_persona_for_existing_collections(initialized_archive):
+    from archivebox.tests.conftest import cli_env, get_free_port, start_archivebox_server, stop_server
+
+    # Pre-fix collections may have no persona row until the first visit to /add/.
+    run_archivebox_cmd(
+        ["manage", "shell", "--no-imports", "-c", "from archivebox.personas.models import Persona; Persona.objects.all().delete()"],
+        cwd=initialized_archive,
+        check=True,
+    )
+    port = get_free_port()
+    try:
+        start_archivebox_server(initialized_archive, port=port, env=cli_env(port=port, server=True))
+        with closing(sqlite3.connect(initialized_archive / "index.sqlite3")) as db:
+            assert db.execute("SELECT name FROM personas_persona").fetchall() == [("Default",)]
+    finally:
+        stop_server(initialized_archive)
+
+
+@pytest.mark.django_db(transaction=True)
+def test_admin_delete_preserves_files_and_recreation_adopts_directory_case(admin_client, admin_user):
+    persona = Persona.objects.create(name="personal", created_by=admin_user)
+    persona.ensure_dirs()
+    profile = persona.path / "chrome_profile" / "Preferences"
+    profile.write_text('{"profile":{"name":"Preserve me"}}')
+    cookies = persona.path / "cookies.txt"
+    cookies.write_text("# Netscape HTTP Cookie File\n")
+    before = (profile.stat().st_ino, cookies.stat().st_ino)
+    response = admin_client.post(reverse("admin:personas_persona_delete", args=[persona.pk]), {"post": "yes"}, HTTP_HOST=ADMIN_TEST_HOST)
+    assert response.status_code == 302
+    assert not Persona.objects.filter(pk=persona.pk).exists()
+    assert profile.read_text() == '{"profile":{"name":"Preserve me"}}'
+    response = admin_client.post(
+        reverse("admin:personas_persona_add"),
+        {
+            "name": "PERSONAL",
+            "created_by": str(admin_user.pk),
+            "permissions": "private",
+            "config": "{}",
+            "import_mode": "none",
+            "_save": "Save",
+        },
+        HTTP_HOST=ADMIN_TEST_HOST,
+    )
+    assert response.status_code == 302, response.content
+    recreated = Persona.objects.get()
+    assert recreated.name == "personal"
+    assert recreated.path == profile.parent.parent
+    assert (profile.stat().st_ino, cookies.stat().st_ino) == before
+
+
+@pytest.mark.django_db(transaction=True)
+def test_extension_sync_reuses_persona_case(client, api_headers):
+    original = Persona.objects.create(name="personal", config={"TIMEOUT": 37})
+    response = api_client_request(
+        client,
+        "post",
+        "/api/v1/personas/sync",
+        payload={"extension_persona_id": "case-test", "name": "PERSONAL", "settings": {}},
+        headers=api_headers,
+    )
+    assert response.status_code == 200, response.content
+    assert response.json()["created"] is False
+    assert response.json()["persona"]["id"] == str(original.pk)
+    assert Persona.objects.count() == 1
+    original.refresh_from_db()
+    assert original.config["TIMEOUT"] == 37
+
+
+@pytest.mark.django_db(transaction=True)
+def test_admin_duplicate_name_identifies_existing_record_and_directory(admin_client, admin_user):
+    original = Persona.objects.create(name="personal", created_by=admin_user)
+    original.ensure_dirs()
+    response = admin_client.post(
+        reverse("admin:personas_persona_add"),
+        {
+            "name": "PERSONAL",
+            "created_by": str(admin_user.pk),
+            "permissions": "private",
+            "config": "{}",
+            "import_mode": "none",
+            "_save": "Save",
+        },
+        HTTP_HOST=ADMIN_TEST_HOST,
+    )
+    assert response.status_code == 200
+    assert str(original.pk).encode() in response.content
+    assert str(original.path).encode() in response.content
+    assert list(Persona.objects.values_list("id", "name")) == [(original.id, "personal")]
+
+
+@pytest.mark.django_db(transaction=True)
+def test_admin_case_only_edit_preserves_profile_directory(admin_client, admin_user):
+    persona = Persona.objects.create(name="personal", created_by=admin_user)
+    persona.ensure_dirs()
+    profile = persona.path / "chrome_profile" / "Preferences"
+    profile.write_text('{"profile":{"name":"Keep this profile"}}')
+    inode = profile.stat().st_ino
+    response = admin_client.post(
+        reverse("admin:personas_persona_change", args=[persona.pk]),
+        {"name": "PERSONAL", "created_by": str(admin_user.pk), "permissions": "private", "config": "{}", "_save": "Save"},
+        HTTP_HOST=ADMIN_TEST_HOST,
+    )
+    assert response.status_code == 302, response.content
+    persona.refresh_from_db()
+    assert persona.name == "personal"
+    assert profile.stat().st_ino == inode
+    assert profile.read_text() == '{"profile":{"name":"Keep this profile"}}'
+
+
+@pytest.mark.django_db(transaction=True)
+def test_extension_sync_rejects_legacy_case_conflicts_before_writing(client, api_headers):
+    # Real rows representing collections created before case-insensitive matching.
+    Persona.objects.bulk_create([Persona(name="personal"), Persona(name="PERSONAL")])
+    response = api_client_request(
+        client,
+        "post",
+        "/api/v1/personas/sync",
+        payload={"extension_persona_id": "case-test-conflict", "name": "Personal", "settings": {}, "cookies_txt": "must not be written"},
+        headers=api_headers,
+    )
+    assert response.status_code == 409, response.content
+    assert "rename" in response.content.decode().lower()
+    assert "manually" in response.content.decode().lower()
+    assert Persona.objects.count() == 2
+    assert all(str(persona.pk).encode() in response.content for persona in Persona.objects.all())
+    assert all(not (persona.path / "cookies.txt").exists() for persona in Persona.objects.all())
+
+
+@pytest.fixture
+def case_sensitive_collection(tmp_path):
+    """Exercise actual Linux-style directory collisions, including on macOS."""
+    mount = tmp_path / "case-sensitive"
+    mount.mkdir()
+    if sys.platform == "darwin":
+        image = tmp_path / "personas.sparseimage"
+        subprocess.run(
+            ["hdiutil", "create", "-size", "128m", "-type", "SPARSE", "-fs", "Case-sensitive APFS", "-volname", "PersonaTest", str(image)],
+            check=True,
+            capture_output=True,
+        )
+        subprocess.run(["hdiutil", "attach", "-nobrowse", "-mountpoint", str(mount), str(image)], check=True, capture_output=True)
+    try:
+        yield mount
+    finally:
+        if sys.platform == "darwin":
+            subprocess.run(["hdiutil", "detach", str(mount)], check=True, capture_output=True)
+
+
+def test_cli_rejects_case_conflicting_directories_without_mutating(case_sensitive_collection):
+    collection = case_sensitive_collection
+    run_archivebox_cmd(["init", "--quick"], cwd=collection, check=True)
+    for name in ("Default", "default"):
+        profile = collection / "personas" / name / "chrome_profile"
+        profile.mkdir(parents=True, exist_ok=True)
+        (profile / "Preferences").write_text(name)
+    commands = (["persona", "create", "Default"], ["init", "--quick"], ["server", "--daemonize", f"127.0.0.1:{get_free_port()}"])
+    try:
+        for command in commands:
+            result = run_archivebox_cmd(command, cwd=collection)
+            assert result.returncode != 0
+            assert "rename" in result.stderr.lower()
+            assert "manually" in result.stderr.lower()
+            assert all(str(collection / "personas" / name) in result.stderr for name in ("Default", "default"))
+    finally:
+        stop_server(collection)
+    with closing(sqlite3.connect(collection / "index.sqlite3")) as db:
+        assert db.execute("SELECT name FROM personas_persona").fetchall() == [("Default",)]
+    for name in ("Default", "default"):
+        assert (collection / "personas" / name / "chrome_profile" / "Preferences").read_text() == name
+
+
+def test_cli_adopts_existing_directory_spelling_on_case_sensitive_disk(case_sensitive_collection):
+    collection = case_sensitive_collection
+    run_archivebox_cmd(["init", "--quick"], cwd=collection, check=True)
+    profile = collection / "personas" / "personal" / "chrome_profile" / "Preferences"
+    profile.parent.mkdir(parents=True)
+    profile.write_text('{"profile":{"name":"Keep me"}}')
+    inode = profile.stat().st_ino
+    run_archivebox_cmd(["persona", "create", "PERSONAL"], cwd=collection, check=True)
+    with closing(sqlite3.connect(collection / "index.sqlite3")) as db:
+        assert db.execute("SELECT name FROM personas_persona ORDER BY name").fetchall() == [("Default",), ("personal",)]
+        # A legacy database can disagree with an existing directory's spelling.
+        db.execute("UPDATE personas_persona SET name='PERSONAL' WHERE name='personal'")
+        db.commit()
+    result = run_archivebox_cmd(["persona", "create", "Personal"], cwd=collection, check=True)
+    assert json.loads(result.stdout)["name"] == "personal"
+    assert not (collection / "personas" / "PERSONAL").exists()
+    assert profile.stat().st_ino == inode
+    assert profile.read_text() == '{"profile":{"name":"Keep me"}}'

archivebox/tests/test_process_runtime_paths.py

@@ -51,6 +51,8 @@ class TestProcessRuntimePaths:
             )
 
         expected_dir = Path(process.pwd)
+        assert process.status == Process.StatusChoices.EXITED
+        assert process.exit_code == 0
         assert process.runtime_dir == expected_dir
         assert process.stdout_file == expected_dir / "stdout.log"
         assert process.stderr_file == expected_dir / "stderr.log"

archivebox/tests/test_recursive_crawl_1.py

@@ -33,6 +33,7 @@ def run_add_until(args, env, condition, timeout=120):
 @pytest.mark.timeout(1200)
 def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(
     initialized_archive,
+    cached_abxpkg_lib_dir,
     free_tcp_port_factory,
     recursive_test_site,
 ):
@@ -59,8 +60,8 @@ def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(
             "USE_COLOR": "false",
             "SHOW_PROGRESS": "false",
             "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*",
-            "ABXPKG_LIB_DIR": str(initialized_archive / "lib"),
-            "CHROMEWEBSTORE_EXTENSIONS_DIR": str(initialized_archive / "lib/chromewebstore/extensions"),
+            "ABXPKG_LIB_DIR": str(cached_abxpkg_lib_dir),
+            "CHROMEWEBSTORE_EXTENSIONS_DIR": str(cached_abxpkg_lib_dir / "chromewebstore/extensions"),
             "TIMEOUT": "90",
             "CRAWL_MAX_CONCURRENT_SNAPSHOTS": "3",
             "SEARCH_BACKEND_SONIC_HOST_NAME": "127.0.0.1",

archivebox/tests/test_recursive_crawl_2.py

@@ -18,7 +18,11 @@ from .test_recursive_crawl_1 import (
 
 
 @pytest.mark.timeout(600)
-def test_direct_url_crawl_measures_first_snapshot_before_parallel_admission(initialized_archive, recursive_test_site):
+def test_direct_url_crawl_measures_first_snapshot_before_parallel_admission(
+    initialized_archive,
+    recursive_test_site,
+    cached_abxpkg_lib_dir,
+):
     """A real three-URL capture must not launch all Chrome hook trees at once."""
     urls = [recursive_test_site["root_url"], *recursive_test_site["child_urls"][:2]]
     env = os.environ.copy()
@@ -27,7 +31,7 @@ def test_direct_url_crawl_measures_first_snapshot_before_parallel_admission(init
             "USE_COLOR": "false",
             "SHOW_PROGRESS": "false",
             "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*",
-            "ABXPKG_LIB_DIR": str(initialized_archive / "lib"),
+            "ABXPKG_LIB_DIR": str(cached_abxpkg_lib_dir),
             "CHROME_HEADLESS": "true",
             "CHROME_SANDBOX": "false",
             "CHROME_ISOLATION": "crawl",

archivebox/tests/test_release_workflow.py

@@ -185,13 +185,22 @@ def test_release_uses_registered_publisher_and_authorized_tag_credentials():
 
     assert all(step.get("name") != "Verify published PyPI package installs and runs" for step in python_release["steps"])
     ci = yaml.safe_load(CI_WORKFLOW.read_text())
-    screenshots = yaml.safe_load((REPO_ROOT / ".github" / "workflows" / "screenshots.yml").read_text())
+    screenshots = yaml.load((REPO_ROOT / ".github" / "workflows" / "screenshots.yml").read_text(), Loader=yaml.BaseLoader)
     prepare_path = ci["jobs"]["prepare"]["uses"]
-    assert screenshots["jobs"]["prepare"]["uses"] == prepare_path
     prepare = yaml.safe_load((REPO_ROOT / prepare_path).read_text())
     assert prepare["jobs"]["prepare"]["steps"][-1]["uses"] == "ArchiveBox/monorepo/.github/actions/prepare-release-version@main"
-    assert screenshots["jobs"]["deploy"]["needs"] == "prepare"
-    assert screenshots["jobs"]["deploy"]["if"] == "needs.prepare.outputs.run_jobs == 'true'"
+    assert set(screenshots["on"]) == {"schedule", "workflow_dispatch"}
+    _, hours, *days = screenshots["on"]["schedule"][0]["cron"].split()
+    assert days == ["*", "*", "*"]
+    assert len(hours.split(",")) == 2
+    first, second = map(int, hours.split(","))
+    assert second - first == 12
+    assert "prepare" not in screenshots["jobs"]
+    assert screenshots["jobs"]["changes"]["steps"][0]["uses"] == "ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main"
+    assert screenshots["jobs"]["deploy"]["needs"] == "changes"
+    assert screenshots["jobs"]["deploy"]["if"] == "needs.changes.outputs.changed == 'true'"
+    assert screenshots["jobs"]["complete"]["needs"] == ["changes", "deploy"]
+    assert screenshots["jobs"]["complete"]["name"] == "Scheduled inputs: ${{ needs.changes.outputs.key }}"
     for name, job in ci["jobs"].items():
         if name not in {"prepare", "required"}:
             assert job["needs"] == "prepare"

archivebox/tests/test_runner_deletion.py

@@ -0,0 +1,182 @@
+"""Exercise queued deletion in the existing runner with a real server."""
+
+import json
+import textwrap
+import time
+from pathlib import Path
+
+import pytest
+
+from archivebox.tests.conftest import (
+    cli_env,
+    get_free_port,
+    kill_processes_for_data_dir,
+    resolve_abxpkg_chrome_env,
+    run_archivebox_cmd,
+    start_archivebox_server,
+    stop_archivebox_process,
+    wait_for_log,
+)
+from archivebox.tests.test_orm_helpers import use_archivebox_db
+
+
+@pytest.mark.django_db(transaction=True)
+@pytest.mark.parametrize("restart", [False, True])
+def test_runner_retries_failed_deletion_without_blocking_captures(initialized_archive, recursive_test_site, restart):
+    from archivebox.core.models import ArchiveResult, Snapshot
+    from archivebox.machine.models import Process
+
+    port = get_free_port()
+    env = cli_env(
+        live=True,
+        server=True,
+        port=port,
+        PLUGINS="title",
+        SAVE_TITLE="True",
+        SEARCH_BACKEND_ENGINE="ripgrep",
+    )
+    # Measure captures while deletion retries, not a cold browser installation.
+    # Without setup, only the first case spends its capture deadline installing
+    # Chrome; later cases silently benefit from the shared binary cache.
+    env.update(resolve_abxpkg_chrome_env(Path(env["ABXPKG_LIB_DIR"]), env))
+    prepared = run_archivebox_cmd(
+        [
+            "manage",
+            "shell",
+            "-c",
+            "import json; from archivebox.crawls.models import Crawl; "
+            "from archivebox.core.models import Snapshot, ArchiveResult; "
+            "from archivebox.base_models.models import get_or_create_system_user_pk; "
+            "c=Crawl.objects.create(urls='https://example.com/delete',created_by_id=get_or_create_system_user_pk(),status='sealed'); "
+            "s=Snapshot.objects.create(url='https://example.com/delete',crawl=c,status='deleting'); "
+            "p=s.output_dir/'wget'; p.mkdir(parents=True,exist_ok=True); "
+            "(p/'index.html').write_text('keep this until cleanup succeeds'); "
+            "ArchiveResult.objects.create(snapshot=s,plugin='wget',hook_name='on_Snapshot__06_wget',status='succeeded'); "
+            "p.chmod(0o500); print(json.dumps({'id':str(s.pk),'path':str(p)}))",
+        ],
+        cwd=initialized_archive,
+        env=env,
+    )
+    assert prepared.returncode == 0, prepared.stderr or prepared.stdout
+    fixture = json.loads(prepared.stdout.strip().splitlines()[-1])
+    protected = Path(fixture["path"])
+    worker_log = initialized_archive / "logs" / "worker_runner.log"
+    server = None
+    try:
+        server = start_archivebox_server(initialized_archive, port=port, env=env, log_name="delete-worker-server.log")
+        wait_for_log(worker_log, "PermissionError", timeout=30)
+        with use_archivebox_db(initialized_archive):
+            assert Process.objects.filter(worker_type="worker_runner", status="running").exists()
+            assert not Process.objects.filter(worker_type="worker_snapshot_delete").exists()
+            assert Snapshot.objects.filter(pk=fixture["id"], status="deleting").exists()
+            assert ArchiveResult.objects.filter(snapshot_id=fixture["id"]).exists()
+        assert (protected / "index.html").is_file()
+        added = run_archivebox_cmd(
+            ["add", "--bg", "--plugins=title", recursive_test_site["root_url"]],
+            cwd=initialized_archive,
+            env=env,
+        )
+        assert added.returncode == 0, added.stderr or added.stdout
+        deadline = time.monotonic() + 60
+        with use_archivebox_db(initialized_archive):
+            while time.monotonic() < deadline:
+                captured = Snapshot.objects.filter(url=recursive_test_site["root_url"], status="sealed").first()
+                if captured is not None:
+                    break
+                assert server.poll() is None
+                time.sleep(0.1)
+            else:
+                pytest.fail((initialized_archive / "logs" / "worker_runner.log").read_text()[-12000:])
+            result = ArchiveResult.objects.get(snapshot=captured, plugin="title", status="succeeded")
+            assert result.output_files
+            assert captured.title == "Root"
+            assert Snapshot.objects.filter(pk=fixture["id"], status="deleting").exists()
+        if restart:
+            stop_archivebox_process(server)
+            server = None
+        protected.chmod(0o700)
+        if restart:
+            server = start_archivebox_server(initialized_archive, port=port, env=env, log_name="delete-restarted-server.log")
+        deadline = time.monotonic() + 75
+        with use_archivebox_db(initialized_archive):
+            while Snapshot.objects.filter(pk=fixture["id"]).exists() and time.monotonic() < deadline:
+                time.sleep(0.1)
+            assert not Snapshot.objects.filter(pk=fixture["id"]).exists(), worker_log.read_text()
+            assert not ArchiveResult.objects.filter(snapshot_id=fixture["id"]).exists()
+        assert not protected.parent.exists()
+    finally:
+        if protected.exists():
+            protected.chmod(0o700)
+        if server is not None:
+            stop_archivebox_process(server)
+        kill_processes_for_data_dir(initialized_archive)
+
+
+def test_deletion_releases_database_before_filesystem_io(initialized_archive):
+    result = run_archivebox_cmd(
+        [
+            "manage",
+            "shell",
+            "-c",
+            textwrap.dedent("""
+                import json
+                import sqlite3
+                import sys
+                from pathlib import Path
+                from django.db import connection
+                from archivebox.base_models.models import get_or_create_system_user_pk
+                from archivebox.core.models import ArchiveResult, Snapshot
+                from archivebox.crawls.models import Crawl
+                from archivebox.services.runner import run_pending_crawls
+
+                # DELETE mode exposes both read-cursor and transaction locks.
+                with connection.cursor() as cursor:
+                    cursor.execute('PRAGMA journal_mode=DELETE')
+                    assert cursor.fetchone()[0] == 'delete'
+                crawl = Crawl.objects.create(
+                    urls='https://example.com/delete',
+                    created_by_id=get_or_create_system_user_pk(), status='sealed',
+                )
+                paths = set()
+                snapshot_ids = []
+                for i in range(2):
+                    snapshot = Snapshot.objects.create(url=f'https://example.com/delete/{i}', crawl=crawl, status='deleting')
+                    path = snapshot.output_dir
+                    path.mkdir(parents=True, exist_ok=True)
+                    (path / 'saved.html').write_text('real saved output')
+                    ArchiveResult.objects.create(snapshot=snapshot, plugin='wget', status='succeeded')
+                    paths.add(str(path))
+                    snapshot_ids.append(snapshot.pk)
+                checked = set()
+
+                def inspect_filesystem_operation(event, args):
+                    if event != 'shutil.rmtree' or str(args[0]) not in paths:
+                        return
+                    assert not connection.in_atomic_block
+                    # Observe the real rmtree call without replacing it. An
+                    # independent writer must acquire its lock immediately.
+                    writer = sqlite3.connect(connection.settings_dict['NAME'], timeout=0, isolation_level=None)
+                    try:
+                        writer.execute('BEGIN EXCLUSIVE')
+                        writer.execute('UPDATE crawls_crawl SET notes=? WHERE id=?', ('concurrent write succeeded', crawl.pk.hex))
+                        writer.execute('COMMIT')
+                    finally:
+                        writer.close()
+                    checked.add(str(args[0]))
+
+                sys.addaudithook(inspect_filesystem_operation)
+                run_pending_crawls(maintenance_only=True)
+                assert checked == paths
+                assert not Snapshot.objects.filter(crawl=crawl).exists()
+                assert not ArchiveResult.objects.filter(snapshot_id__in=snapshot_ids).exists()
+                assert all(not Path(path).exists() for path in paths)
+                crawl.refresh_from_db()
+                assert crawl.notes == 'concurrent write succeeded'
+                print(json.dumps({'verified_deletions': len(checked)}))
+            """),
+        ],
+        cwd=initialized_archive,
+        env=cli_env(disable_extractors=True),
+    )
+    assert result.returncode == 0, result.stderr or result.stdout
+    assert json.loads(result.stdout.strip().splitlines()[-1]) == {"verified_deletions": 2}

archivebox/tests/test_server_security_browser.py

@@ -386,8 +386,14 @@ main().catch(async (error) => {
 
 
 @pytest.fixture
-def browser_runtime(initialized_archive: Path):
-    shared_lib = initialized_archive / "lib"
+def browser_runtime(cached_abxpkg_lib_dir):
+    from abx_plugins import get_plugins_dir
+    from abx_plugins.plugins.base.utils import get_config
+
+    # Security modes need isolated collections, but the browser installation is
+    # shared. Rebuilding it for every mode adds unrelated registry dependencies
+    # and discards the real binary cache already provided by the test helpers.
+    shared_lib = cached_abxpkg_lib_dir
     env = cli_env(
         ABXPKG_LIB_DIR=str(shared_lib),
         CHROME_HEADLESS="True",
@@ -395,15 +401,12 @@ def browser_runtime(initialized_archive: Path):
         CHROME_ISOLATION="snapshot",
     )
     env.pop("CHROME_BINARY", None)
-    install_result = run_archivebox_cmd(
-        ["install", "chrome"],
-        cwd=initialized_archive,
-        env=env,
-        timeout=900,
-    )
-    assert install_result.returncode == 0, install_result.stderr or install_result.stdout
-
     resolved_env = resolve_abxpkg_chrome_env(shared_lib, env)
+    chrome_config = get_config(
+        Path(get_plugins_dir()) / "chrome" / "config.json",
+        environ={**env, **resolved_env},
+        hydrate_binaries=False,
+    )
 
     return {
         "lib_dir": shared_lib,
@@ -411,6 +414,7 @@ def browser_runtime(initialized_archive: Path):
         "node_path": resolved_env["NODE_PATH"],
         "node_binary": Path(resolved_env["NODE_BINARY"]),
         "chrome_binary": Path(resolved_env["CHROME_BINARY"]),
+        "chrome_args": [*chrome_config.CHROME_ARGS, *chrome_config.CHROME_ARGS_EXTRA],
     }
 
 

archivebox/tests/test_shutdown_util.py

@@ -37,8 +37,10 @@ def test_foreground_shutdown_second_signal_exits_immediately():
         from archivebox.core.shutdown_util import foreground_shutdown_signals
 
         with foreground_shutdown_signals(first_signal_message=None) as state:
-            print("READY", flush=True)
             try:
+                # The sender may signal as soon as READY is flushed, before
+                # signal.pause(); the handler must already be inside this try.
+                print("READY", flush=True)
                 signal.pause()
             except KeyboardInterrupt:
                 print(f"FIRST:{state.signal_name}", flush=True)

archivebox/tests/test_snapshot_service.py

@@ -198,3 +198,62 @@ def test_snapshot_service_cli_add_seals_snapshot_and_writes_indexes(tmp_path, re
     assert wget_files
     assert any("Root" in path.read_text(encoding="utf-8", errors="ignore") for path in wget_files if path.suffix in (".html", ".txt"))
     assert any(result["plugin"] == "wget" and result["status"] == ArchiveResult.StatusChoices.SUCCEEDED for result in state["results"])
+
+
+@pytest.mark.timeout(180)
+@pytest.mark.parametrize("depth", [0, 1])
+def test_api_queue_hands_storage_creation_to_runner(tmp_path, recursive_test_site, depth):
+    from .conftest import (
+        create_admin_and_token,
+        live_api_request,
+        stop_archivebox_process,
+        start_archivebox_server,
+        wait_for_snapshot_capture,
+    )
+
+    init_archive(tmp_path)
+    port = get_free_port()
+    env = cli_env(port=port, server=True, PLUGINS="wget,parse_html_urls")
+    token = create_admin_and_token(tmp_path)
+    url = recursive_test_site["root_url"]
+    server = start_archivebox_server(tmp_path, env=env, port=port, log_name="api-storage-handoff.log")
+    try:
+        response = live_api_request(
+            port,
+            "post",
+            "/api/v1/cli/add",
+            api_token=token,
+            json={"urls": [url], "plugins": "wget,parse_html_urls", "tag": "handoff", "depth": depth},
+        )
+        assert response.status_code == 200, response.text
+        crawl_id = response.json()["result"]["crawl_id"]
+        metadata = live_api_request(
+            port,
+            "post",
+            "/api/v1/core/snapshots",
+            api_token=token,
+            json={"crawl_id": crawl_id, "url": url, "title": "Browser title"},
+        )
+        assert metadata.status_code == 200, metadata.text
+        assert "Root" in wait_for_snapshot_capture(tmp_path, url, timeout=120)
+        if depth:
+            assert "About" in wait_for_snapshot_capture(tmp_path, url + "about", timeout=120)
+            child_state = _snapshot_state(tmp_path, url + "about")
+            assert child_state["status"] == Snapshot.StatusChoices.SEALED
+            assert child_state["crawl_link"].is_symlink()
+            assert child_state["crawl_link"].resolve() == child_state["snapshot_dir"].resolve()
+            assert (child_state["snapshot_dir"] / "index.jsonl").is_file()
+        with use_archivebox_db(tmp_path):
+            snapshot = Snapshot.objects.get(pk=metadata.json()["id"])
+            assert snapshot.tags.filter(name="handoff").exists()
+    finally:
+        stop_archivebox_process(server)
+    state = _snapshot_state(tmp_path, url)
+    assert state["status"] == Snapshot.StatusChoices.SEALED
+    assert state["crawl_dir"].is_dir()
+    assert state["snapshot_dir"].is_dir()
+    assert state["crawl_link"].is_symlink()
+    assert state["crawl_link"].resolve() == state["snapshot_dir"].resolve()
+    assert (state["snapshot_dir"] / "index.jsonl").is_file()
+    assert any(row["plugin"] == "wget" and row["status"] == "succeeded" for row in state["results"])
+    assert any("Root" in path.read_text(errors="ignore") for path in (state["snapshot_dir"] / "wget").rglob("*.html"))

archivebox/tests/test_takeover_util_2.py

@@ -175,6 +175,10 @@ def test_explicit_sonic_binary_is_used_by_real_search_worker(initialized_archive
     worker_text = worker_config.read_text(encoding="utf-8")
     command = next(line.partition("=")[2] for line in worker_text.splitlines() if line.startswith("command="))
     assert shlex.split(command)[0] == str(sonic_binary), worker_text
+    supervisor_pids = re.findall(r"Supervisord connected \(pid=(\d+)\)", result.stderr)
+    assert supervisor_pids, result.stderr
+    assert all(not pid_is_alive(int(pid)) for pid in supervisor_pids), "Search CLI returned while its owned supervisord was still alive"
+    assert result.stdout == "", result.stdout
     from archivebox.workers.supervisord_util import _sonic_worker_bind_target
 
     assert _sonic_worker_bind_target({"name": "worker_sonic", "command": command}) == ("127.0.0.1", sonic_port)
@@ -198,6 +202,7 @@ def test_explicit_sonic_binary_is_used_by_real_search_worker(initialized_archive
         timeout=60,
     )
     assert relative_result.returncode == 0, relative_result.stderr or relative_result.stdout
+    assert relative_result.stdout == "", relative_result.stdout
     relative_worker_text = worker_config.read_text(encoding="utf-8")
     relative_command = next(line.partition("=")[2] for line in relative_worker_text.splitlines() if line.startswith("command="))
     assert Path(shlex.split(relative_command)[0]).name == relative_binary.name, relative_worker_text

archivebox/tests/test_ui_add_view.py

@@ -37,6 +37,12 @@ ADD_FORM_DEFAULTS = {
 }
 
 
+@pytest.fixture(autouse=True)
+def initialized_persona(db):
+    # Match init/server startup; rendering /add/ must no longer create DB rows.
+    return Persona.get_or_create_default()
+
+
 @pytest.fixture
 def admin_user(db):
     return User.objects.create_superuser(

archivebox/tests/test_ui_add_view_runtime.py

@@ -190,7 +190,7 @@ def assert_no_file_or_shell_payload_snapshots(cwd: Path, *, canary: Path) -> Non
 
 
 @pytest.mark.timeout(180)
-def test_add_view_restarts_stopped_supervisord_runner(tmp_path, recursive_test_site):
+def test_add_view_queues_work_without_restarting_stopped_runner(tmp_path, recursive_test_site):
     init_archive(tmp_path)
 
     port = get_free_port()
@@ -239,15 +239,30 @@ def test_add_view_restarts_stopped_supervisord_runner(tmp_path, recursive_test_s
         )
         assert response.status_code in (302, 303), response.text
 
-        assert _worker_state(tmp_path, "worker_runner") == "RUNNING"
+        assert _worker_state(tmp_path, "worker_runner") != "RUNNING"
         with use_archivebox_db(tmp_path):
             crawl = Crawl.objects.order_by("-created_at").first()
             assert crawl is not None
             assert crawl.tags_str == "restart-supervised-runner"
             assert crawl.urls == recursive_test_site["root_url"]
+            assert crawl.status == Crawl.StatusChoices.QUEUED
+            assert not crawl.output_dir.exists()
     finally:
         stop_server(tmp_path)
 
+    from .conftest import stop_archivebox_process, wait_for_snapshot_capture
+
+    server = start_archivebox_server(tmp_path, env=env, port=port, log_name="resumed-add-server.log")
+    try:
+        assert "Root" in wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=120)
+    finally:
+        stop_archivebox_process(server)
+    with use_archivebox_db(tmp_path):
+        snapshot = Snapshot.objects.get(crawl_id=crawl.id, url=recursive_test_site["root_url"])
+        assert snapshot.status == Snapshot.StatusChoices.SEALED
+        assert snapshot.archiveresult_set.filter(plugin="wget", status=ArchiveResult.StatusChoices.SUCCEEDED).exists()
+        assert any("Root" in path.read_text(errors="ignore") for path in (snapshot.output_dir / "wget").rglob("*.html"))
+
 
 def _login_to_add_view(port: int) -> tuple[requests.Session, requests.Response, str]:
     # Browsers resolve *.localhost themselves; system DNS in stock containers