Bump release version to 0.9.72rc70

Resolve relative cookie files before hooks change cwd, reconcile file edits before machine startup saves, and prevent stale cached binaries overriding runtime selections. Cover issue #1896 and real persona localStorage capture through CLI regressions.

.github/CONTRIBUTING.md

@@ -12,7 +12,7 @@
 - https://github.com/ArchiveBox/ArchiveBox/issues
 - https://github.com/ArchiveBox/ArchiveBox/pulls
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Install#manual-setup
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-c-bare-metal-setup
 
 ### Development Setup
 

.github/STAGING.md

@@ -0,0 +1,42 @@
+# Stable releases require live dev acceptance
+
+The order is **dev CI → dev image publication → automatic Cabbage deployment →
+real capture and browser acceptance → stable publication**. Never
+substitute a published PyPI wheel, healthy container, or local build for this
+acceptance. The release workflow blocks *before* a stable upload unless Cabbage
+has successful evidence from `staging-acceptance.yml` matching all tracked
+source and dependency pins. Only ArchiveBox's own version fields may change
+between dev and stable.
+
+The staging workflow observes the existing image watcher; it cannot deploy or
+restart the application. On Cabbage, install `bin/staging-acceptance-host.sh`
+as root-owned mode 0755 `/usr/local/sbin/archivebox-staging-acceptance`. Its hash
+is checked against the candidate. A dedicated SSH key must use
+`restrict,command="/usr/local/sbin/archivebox-staging-acceptance"` in authorized
+keys. It accepts only `inspect <source-sha>` and `capture <source-sha>`; it cannot
+run caller-provided commands or archive arbitrary URLs.
+
+The `staging` GitHub environment is restricted to dev/main branches. It contains
+`STAGING_SSH_KEY` and `STAGING_KNOWN_HOSTS`, with independently verified host keys.
+Only trusted release events use the private ugNAS runner route. Never expose
+these credentials to PR jobs or copy an operator's existing root key.
+
+Acceptance creates one tagged background crawl per source through the normal
+add/runner path, enabling every snapshot extractor, including TLSNotary. It
+checks installed pins, completed outputs and their files, unexpected/missing
+plugin results, host OOM kills, and real rendered snapshot pages. For this
+release, the explicitly accepted TLSNotary failures and exact missing-Claude-auth
+diagnostics are retained in the report's `limitations` list; they do not disable
+those plugins or excuse missing/truncated successful outputs. Remove the
+TLSNotary exception when its follow-up fix ships. Logs, result inventories and
+screenshots are retained as Actions artifacts. Other failures remain
+failures; do not publish a success deployment record to bypass them. No code
+mounts are allowed. The Cabbage job must pass before its record authorizes a
+stable upload. Queued capture progress is polled; failed tests are not retried.
+
+Browser acceptance follows the default WACZ viewer into the captured document,
+matching its replay base and original URL and checking that its body is visible
+and contains substantial text. An attached viewer iframe can still be
+`about:blank`, so its presence alone cannot prove replay works. Extracted display
+titles and the captured document's `<title>` may legitimately differ; neither
+title equality nor output thumbnails substitute for checking the actual page.

.github/actionlint.yaml

@@ -0,0 +1,3 @@
+self-hosted-runner:
+  labels:
+    - ugnas-linux

.github/dependabot.yml

@@ -15,7 +15,7 @@ updates:
         patterns:
           - "*"
   - package-ecosystem: "npm"
-    directory: "/"
+    directory: "/etc"
     target-branch: "dev"
     schedule:
       interval: "monthly"

.github/lychee.toml

@@ -0,0 +1,38 @@
+# Documentation destinations only: do not crawl archived URLs, app routes, or image assets.
+# Manual run from the repository root with Lychee 0.24.2:
+# git ls-files -- '*.md' '*.html' '*.py' '*.rst' '*.yml' '*.yaml' '*.sh' '*.toml' > /tmp/archivebox-link-inputs.txt
+# lychee --config .github/lychee.toml --root-dir "$PWD" --files-from /tmp/archivebox-link-inputs.txt
+# Explicit tracked inputs include README symlinks and let errors in one file leave others checkable.
+include_fragments = "anchor-only"
+include_verbatim = true
+max_retries = 0
+max_concurrency = 4
+timeout = 20
+no_progress = true
+hidden = true
+extensions = ["md", "html", "py", "rst", "yml", "yaml", "sh", "toml"]
+exclude_path = ['(^|/)(tests|data|apidocs|node_modules|\.git)(/|$)']
+# Includes override this exclude; this is a documentation-link allowlist.
+exclude = ['.*']
+include = [
+ '^https://github\.com/ArchiveBox/ArchiveBox/blob/[^/]+/.*\.md(?:#|$)',
+ '^https://raw\.githubusercontent\.com/wiki/ArchiveBox/ArchiveBox/',
+ '^https?://github\.com/(?:ArchiveBox|pirate)/ArchiveBox(?:/wiki(?:/|#|$)|/?#)',
+ '^https?://docs\.archivebox\.io(?:/|$)',
+ '^file:.*\.md(?:#|$)',
+ '^file:.*/docs/[^/.#]+(?:#|$)',
+]
+fallback_extensions = ["md"]
+remap = [
+ # Lychee's Markdown slugger drops variation selectors. Check these against rendered HTML.
+
+ '^file:.*/README\.md(#%EF%B8%8F-cli-usage)$ https://github.com/ArchiveBox/ArchiveBox${1}',
+ '^file:.*/docs/Install\.md(#option-a-docker--docker-compose-setup-%EF%B8%8F)$ https://github.com/ArchiveBox/ArchiveBox/wiki/Install${1}',
+ # The wiki accepts this case alias, but raw file paths are case-sensitive.
+ '^https?://github\.com/(?:ArchiveBox|pirate)/ArchiveBox/wiki/Setting-up-Storage$ https://raw.githubusercontent.com/wiki/ArchiveBox/ArchiveBox/Setting-Up-Storage.md',
+
+ # Bare missing wiki pages silently redirect to Home; raw pages return an actual 404.
+ # Keep fragment URLs on the rendered wiki to catch published heading drift.
+ '^https?://github\.com/(?:ArchiveBox|pirate)/ArchiveBox/wiki/([^#/?]+)$ https://raw.githubusercontent.com/wiki/ArchiveBox/ArchiveBox/${1}.md',
+]
+preprocess = { command = "bin/lychee_preprocess.py" }

.github/pages/artifacts.py

@@ -14,7 +14,7 @@ def trusted(run, repo, workflow, branch, successful=True):
     return (
         (run.get("head_repository") or {}).get("full_name", "").lower() == repo.lower()
         and run["head_branch"] == branch
-        and run["event"] in ("push", "workflow_dispatch")
+        and run["event"] in ("push", "workflow_dispatch", "schedule")
         and run["path"] == f".github/workflows/{workflow}"
         and (not successful or run["conclusion"] == "success")
     )

.github/pages/base/footer-start.html

@@ -10,4 +10,4 @@ Also review the inline navbar/footer in abx-dl/website/index.html and
 -->
 <footer class="abx-footer" aria-label="ArchiveBox ecosystem">
   <div class="abx-footer-intro"><div><a class="abx-brand" href="https://archivebox.io/"><svg class="abx-logo" viewBox="0 0 40 40" aria-hidden="true"><rect width="40" height="40" rx="10" fill="#9b2854"/><path d="M10 14h20v17H10z" fill="#fff4ed"/><path d="M8 9h24v6H8z" fill="#fff4ed"/><path d="M16 19h8v3h-8z" fill="#9b2854"/><path d="M12 31V17" stroke="#9b2854" stroke-width="2"/></svg><span>ArchiveBox</span></a><p class="abx-tagline">Open-source tools for keeping a lasting copy of the web.<br>Your data. Your devices. Your archive.</p></div><p class="abx-footer-motto">A home for the web<br>you want to keep.</p></div>
-  <div class="abx-footer-grid"><section class="abx-footer-column"><h2>Apps &amp; tools</h2><ul><li><a href="https://archivebox.io/">ArchiveBox Server</a><a class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" aria-label="ArchiveBox Server source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://app.archivebox.io/#get-started">ArchiveBox for macOS &amp; iOS</a><a class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" aria-label="ArchiveBox for macOS &amp; iOS source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://android.archivebox.io/">ArchiveBox for Android</a><a class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" aria-label="ArchiveBox for Android source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox Desktop</a><a class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" aria-label="ArchiveBox Desktop source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://extension.archivebox.io/">ArchiveBox Browser Extension</a><a class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" aria-label="ArchiveBox Browser Extension source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://plugins.archivebox.io/">ArchiveBox Plugin Library</a><a class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" aria-label="ArchiveBox Plugin Library source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://abx-dl.archivebox.io/">One-shot abx-Line truncated
+  <div class="abx-footer-grid"><section class="abx-footer-column"><h2>Apps &amp; tools</h2><ul><li><a href="https://archivebox.io/">ArchiveBox Server</a><a class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" aria-label="ArchiveBox Server source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://app.archivebox.io/#get-started">ArchiveBox for macOS &amp; iOS</a><a class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" aria-label="ArchiveBox for macOS &amp; iOS source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://android.archivebox.io/">ArchiveBox for Android</a><a class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" aria-label="ArchiveBox for Android source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox Desktop</a><a class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" aria-label="ArchiveBox Desktop source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://extension.archivebox.io/">ArchiveBox Browser Extension</a><a class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" aria-label="ArchiveBox Browser Extension source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://plugins.archivebox.io/">ArchiveBox Plugin Library</a><a class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" aria-label="ArchiveBox Plugin Library source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://abx-dl.archivebox.io/">One-shot abx-Line truncated

.github/scripts/discover_test_matrix.py

@@ -1,37 +1,79 @@
 #!/usr/bin/env python3
-"""Discover every ArchiveBox test file and expose it as one CI job."""
+"""Discover every test once, batching measured short files."""
 
 import json
+import os
 from pathlib import Path
 
 
 def main() -> None:
-    root = Path.cwd().resolve()
+    root = Path(__file__).resolve().parents[2]
     archivebox_tests = sorted((root / "archivebox/tests").glob("test_*.py"))
 
     if not archivebox_tests:
         raise SystemExit("No ArchiveBox tests discovered")
 
-    matrix: list[dict[str, object]] = []
+    # These are observed per-file pytest seconds, including fixture/startup
+    # time. Shared job setup is paid once per batch. Unknown/long files, optional
+    # dependencies and hosted-only requirements keep independent jobs.
+    durations = json.loads((root / ".github/test-durations.json").read_text())["seconds"]
+    hosted = {path for path in archivebox_tests if "# ci-runner: hosted" in path.read_text().splitlines()[:5]}
+    requested = os.environ.get("TEST_FILE", "")
+    batches: list[list[str]] = []
+    batch: list[str] = []
+    batch_seconds = 0
     for path in archivebox_tests:
         test_path = path.relative_to(root).as_posix()
+        duration = durations.get(test_path, 60)
+        if duration >= 60 or path in hosted or path.stem == "test_auth_ldap" or test_path == requested:
+            batches.append([test_path])
+            continue
+        if batch and (len(batch) == 8 or batch_seconds + duration > 180):
+            batches.append(batch)
+            batch = []
+            batch_seconds = 0
+        batch.append(test_path)
+        batch_seconds += duration
+    if batch:
+        batches.append(batch)
+
+    matrix: list[dict[str, object]] = []
+    for paths in batches:
         matrix.append(
             {
-                "name": f"main/{path.stem}",
-                "paths": [test_path],
-                "paths_arg": test_path,
-                "extra": "ldap" if test_path.endswith("/test_auth_ldap.py") else "",
-                "count": 1,
+                "name": f"main/{Path(paths[0]).stem}" if len(paths) == 1 else f"batch/{Path(paths[0]).stem}",
+                "paths": paths,
+                "paths_arg": " ".join(paths),
+                "extra": "ldap" if paths == ["archivebox/tests/test_auth_ldap.py"] else "",
+                "count": len(paths),
+                "ugnas": False,
             },
         )
 
+    # Ordinary Linux tests can use either pool. Only genuine hosted-only
+    # requirements need a file header; filenames never enter runner policy.
+    # Keep a bounded NAS share so hosted capacity continues working in parallel.
+    # This is queued work, not a worker count: several waves keep the capped
+    # runners occupied while hosted jobs run in parallel.
+    nas_job_limit = int(os.environ.get("UGNAS_CI_MAX_JOBS", "3"))
+    # Keep long jobs early in the hosted queue. Giving the longest jobs to the
+    # CPU-capped NAS instead created a tail after hosted work had finished; use
+    # its extra slots for a bounded share of shorter work without filename rules.
+    matrix.sort(key=lambda item: sum(durations.get(path, 60) for path in item["paths"]), reverse=True)
+    eligible = [item for item in reversed(matrix) if not any(root / path in hosted for path in item["paths"])]
+    for item in eligible[:nas_job_limit]:
+        item["ugnas"] = True
+
     discovered_paths = [path for entry in matrix for path in entry["paths"]]
     if len(discovered_paths) != len(set(discovered_paths)):
         raise SystemExit("Tests were not discovered exactly once")
     if sorted(discovered_paths) != [path.relative_to(root).as_posix() for path in archivebox_tests]:
         raise SystemExit("Discovered test shard coverage does not match test files")
 
-    print(f"Discovered {len(discovered_paths)} test files exactly once across {len(matrix)} jobs")
+    print(
+        f"Discovered {len(discovered_paths)} test files exactly once across {len(matrix)} jobs; "
+        f"{sum(item['ugnas'] for item in matrix)} selected for ugNAS, subject to the availability check",
+    )
     print(json.dumps(matrix, separators=(",", ":")))
 
 

.github/test-durations.json

@@ -0,0 +1,174 @@
+{
+  "source_runs": [
+    "https://github.com/ArchiveBox/ArchiveBox/actions/runs/37190995942"
+  ],
+  "measurement": "Per-file pytest wall seconds from JUnit suites, including fixture setup, plus 9 seconds for process startup outside JUnit (observed median 8.1 seconds across 69 successful unbatched jobs). Failed files retain the larger prior successful estimate. Unknown files stay independent. Queue and shared job setup are excluded.",
+  "seconds": {
+    "archivebox/tests/test_api_discovery_performance.py": 28,
+    "archivebox/tests/test_api_v1_auth_browser_session.py": 17,
+    "archivebox/tests/test_api_v1_auth_check_api_token.py": 15,
+    "archivebox/tests/test_api_v1_auth_get_api_token.py": 16,
+    "archivebox/tests/test_api_v1_cli_add.py": 84,
+    "archivebox/tests/test_api_v1_cli_remove.py": 33,
+    "archivebox/tests/test_api_v1_cli_schedule.py": 29,
+    "archivebox/tests/test_api_v1_cli_search.py": 15,
+    "archivebox/tests/test_api_v1_cli_update.py": 175,
+    "archivebox/tests/test_api_v1_cli_workflow_add_search_update_remove.py": 81,
+    "archivebox/tests/test_api_v1_core_any_id.py": 13,
+    "archivebox/tests/test_api_v1_core_archiveresult_archiveresult_id.py": 24,
+    "archivebox/tests/test_api_v1_core_archiveresults.py": 19,
+    "archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py": 106,
+    "archivebox/tests/test_api_v1_core_snapshots.py": 17,
+    "archivebox/tests/test_api_v1_core_snapshots_rss.py": 16,
+    "archivebox/tests/test_api_v1_core_tag_tag_id.py": 15,
+    "archivebox/tests/test_api_v1_core_tag_tag_id_rename.py": 16,
+    "archivebox/tests/test_api_v1_core_tag_tag_id_snapshots_jsonl.py": 16,
+    "archivebox/tests/test_api_v1_core_tag_tag_id_urls_txt.py": 16,
+    "archivebox/tests/test_api_v1_core_tags.py": 15,
+    "archivebox/tests/test_api_v1_core_tags_add_to_snapshot.py": 16,
+    "archivebox/tests/test_api_v1_core_tags_autocomplete.py": 15,
+    "archivebox/tests/test_api_v1_core_tags_create.py": 15,
+    "archivebox/tests/test_api_v1_core_tags_remove_from_snapshot.py": 15,
+    "archivebox/tests/test_api_v1_core_tags_search.py": 17,
+    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py": 131,
+    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_filename.py": 15,
+    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_filename.py": 15,
+    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_subfolder_filename.py": 16,
+    "archivebox/tests/test_api_v1_crawls_crawls.py": 16,
+    "archivebox/tests/test_api_v1_machine_binaries.py": 16,
+    "archivebox/tests/test_api_v1_machine_binary_binary_id.py": 16,
+    "archivebox/tests/test_api_v1_machine_binary_by_name_name.py": 16,
+    "archivebox/tests/test_api_v1_machine_machine_current.py": 15,
+    "archivebox/tests/test_api_v1_machine_machine_machine_id.py": 16,
+    "archivebox/tests/test_api_v1_machine_machines.py": 15,
+    "archivebox/tests/test_api_v1_personas_personas.py": 15,
+    "archivebox/tests/test_api_v1_personas_sync.py": 16,
+    "archivebox/tests/test_api_v1_workflow_core_token_auth_side_effects.py": 27,
+    "archivebox/tests/test_api_v1_workflow_frozen_crawl_config_sources.py": 19,
+    "archivebox/tests/test_archive_result_service.py": 49,
+    "archivebox/tests/test_auth_ldap.py": 32,
+    "archivebox/tests/test_binary_service.py": 109,
+    "archivebox/tests/test_cli_add_1.py": 107,
+    "archivebox/tests/test_cli_add_2.py": 86,
+    "archivebox/tests/test_cli_add_3.py": 108,
+    "archivebox/tests/test_cli_add_4.py": 105,
+    "archivebox/tests/test_cli_archiveresult.py": 86,
+    "archivebox/tests/test_cli_background_screenshot.py": 49,
+    "archivebox/tests/test_cli_binary.py": 12,
+    "archivebox/tests/test_cli_config.py": 75,
+    "archivebox/tests/test_cli_crawl.py": 98,
+    "archivebox/tests/test_cli_extract.py": 63,
+    "archivebox/tests/test_cli_extract_input.py": 124,
+    "archivebox/tests/test_cli_help.py": 16,
+    "archivebox/tests/test_cli_init.py": 72,
+    "archivebox/tests/test_cli_install.py": 38,
+    "archivebox/tests/test_cli_list.py": 141,
+    "archivebox/tests/test_cli_machine.py": 19,
+    "archivebox/tests/test_cli_manage.py": 32,
+    "archivebox/tests/test_cli_mcp.py": 30,
+    "archivebox/tests/test_cli_oneshot.py": 15,
+    "archivebox/tests/test_cli_persona.py": 147,
+    "archivebox/tests/test_cli_piping.py": 60,
+    "archivebox/tests/test_cli_pluginmap.py": 12,
+    "archivebox/tests/test_cli_process.py": 12,
+    "archivebox/tests/test_cli_remove.py": 102,
+    "archivebox/tests/test_cli_run_1.py": 117,
+    "archivebox/tests/test_cli_run_2.py": 112,
+    "archivebox/tests/test_cli_run_3.py": 80,
+    "archivebox/tests/test_cli_run_4.py": 111,
+    "archivebox/tests/test_cli_schedule.py": 147,
+    "archivebox/tests/test_cli_search.py": 25,
+    "archivebox/tests/test_cli_server.py": 143,
+    "archivebox/tests/test_cli_shell.py": 17,
+    "archivebox/tests/test_cli_snapshot.py": 109,
+    "archivebox/tests/test_cli_status.py": 46,
+    "archivebox/tests/test_cli_tag.py": 12,
+    "archivebox/tests/test_cli_update.py": 114,
+    "archivebox/tests/test_cli_update_reindex_snapshots.py": 100,
+    "archivebox/tests/test_cli_version.py": 119,
+    "archivebox/tests/test_config_DELETE_AFTER.py": 187,
+    "archivebox/tests/test_config_MAX_limits.py": 25,
+    "archivebox/tests/test_config_ONLY_NEW.py": 15,
+    "archivebox/tests/test_config_SAVE_TITLE.py": 59,
+    "archivebox/tests/test_config_URL_filters.py": 15,
+    "archivebox/tests/test_core_config.py": 11,
+    "archivebox/tests/test_core_forms.py": 14,
+    "archivebox/tests/test_crawl_history.py": 162,
+    "archivebox/tests/test_crawl_pause.py": 11,
+    "archivebox/tests/test_crawl_runner.py": 22,
+    "archivebox/tests/test_crawl_runner_2.py": 27,
+    "archivebox/tests/test_crawl_service.py": 19,
+    "archivebox/tests/test_frozen_crawl_config.py": 26,
+    "archivebox/tests/test_hook_result_lifecycle.py": 83,
+    "archivebox/tests/test_hooks.py": 54,
+    "archivebox/tests/test_issue_1896.py": 107,
+    "archivebox/tests/test_machine_models.py": 26,
+    "archivebox/tests/test_machine_service.py": 87,
+    "archivebox/tests/test_migrations_04_to_09.py": 25,
+    "archivebox/tests/test_migrations_07_to_09.py": 107,
+    "archivebox/tests/test_migrations_08_to_09.py": 141,
+    "archivebox/tests/test_migrations_fresh.py": 55,
+    "archivebox/tests/test_misc_checks.py": 12,
+    "archivebox/tests/test_misc_monkey_patches.py": 12,
+    "archivebox/tests/test_opencode_agent_1.py": 92,
+    "archivebox/tests/test_opencode_agent_2.py": 60,
+    "archivebox/tests/test_opencode_agent_3.py": 154,
+    "archivebox/tests/test_opencode_browser_1.py": 268,
+    "archivebox/tests/test_opencode_browser_2.py": 77,
+    "archivebox/tests/test_opencode_browser_3.py": 74,
+    "archivebox/tests/test_orm_helpers.py": 14,
+    "archivebox/tests/test_permissions.py": 12,
+    "archivebox/tests/test_persona_casing.py": 96,
+    "archivebox/tests/test_persona_runtime.py": 30,
+    "archivebox/tests/test_postgres_backend.py": 76,
+    "archivebox/tests/test_process_runtime_paths.py": 20,
+    "archivebox/tests/test_process_service.py": 40,
+    "archivebox/tests/test_proof_previews.py": 24,
+    "archivebox/tests/test_recursive_crawl_1.py": 374,
+    "archivebox/tests/test_recursive_crawl_2.py": 55,
+    "archivebox/tests/test_recursive_crawl_3.py": 82,
+    "archivebox/tests/test_release_notes.py": 12,
+    "archivebox/tests/test_release_source_fingerprint.py": 13,
+    "archivebox/tests/test_release_workflow.py": 15,
+    "archivebox/tests/test_resource_admission.py": 13,
+    "archivebox/tests/test_runner_deletion.py": 135,
+    "archivebox/tests/test_runner_watch.py": 104,
+    "archivebox/tests/test_search.py": 151,
+    "archivebox/tests/test_serve_static.py": 19,
+    "archivebox/tests/test_server_automatic_url.py": 22,
+    "archivebox/tests/test_server_security_browser.py": 137,
+    "archivebox/tests/test_settings_signal_webhooks.py": 13,
+    "archivebox/tests/test_setup_script.py": 12,
+    "archivebox/tests/test_shutdown_util.py": 13,
+    "archivebox/tests/test_snapshot_filesystem_migration.py": 16,
+    "archivebox/tests/test_snapshot_output_stacks.py": 76,
+    "archivebox/tests/test_snapshot_ref_lookup.py": 15,
+    "archivebox/tests/test_snapshot_service.py": 48,
+    "archivebox/tests/test_snapshot_url_length.py": 16,
+    "archivebox/tests/test_tag_service.py": 15,
+    "archivebox/tests/test_takeover_util_1.py": 100,
+    "archivebox/tests/test_takeover_util_2.py": 86,
+    "archivebox/tests/test_takeover_util_3.py": 98,
+    "archivebox/tests/test_takeover_util_4.py": 96,
+    "archivebox/tests/test_test_harness.py": 12,
+    "archivebox/tests/test_ui_add_view.py": 26,
+    "archivebox/tests/test_ui_add_view_runtime.py": 121,
+    "archivebox/tests/test_ui_admin_apitoken.py": 16,
+    "archivebox/tests/test_ui_admin_archiveresult.py": 18,
+    "archivebox/tests/test_ui_admin_config_widget.py": 13,
+    "archivebox/tests/test_ui_admin_crawl.py": 25,
+    "archivebox/tests/test_ui_admin_links.py": 23,
+    "archivebox/tests/test_ui_admin_machine.py": 18,
+    "archivebox/tests/test_ui_admin_persona.py": 19,
+    "archivebox/tests/test_ui_admin_snapshot.py": 116,
+    "archivebox/tests/test_ui_admin_tag.py": 16,
+    "archivebox/tests/test_ui_admin_user.py": 16,
+    "archivebox/tests/test_ui_config_views.py": 20,
+    "archivebox/tests/test_ui_live_progress.py": 140,
+    "archivebox/tests/test_ui_public_snapshot.py": 95,
+    "archivebox/tests/test_ui_savepagenow.py": 31,
+    "archivebox/tests/test_urls.py": 132,
+    "archivebox/tests/test_util.py": 13,
+    "archivebox/tests/test_version_metadata.py": 13
+  }
+}

.github/workflows/ci.yml

@@ -18,51 +18,64 @@ concurrency:
   cancel-in-progress: ${{ github.event_name != 'workflow_dispatch' }}
 
 jobs:
+  prepare:
+    permissions:
+      contents: write
+    uses: ./.github/workflows/prepare-release.yml
+    secrets: inherit
+
   lint:
+    needs: prepare
+    if: needs.prepare.outputs.run_jobs == 'true'
     name: Linters
     uses: ./.github/workflows/lint.yml
     secrets: inherit
 
   install-cli-platform:
+    needs: prepare
+    if: needs.prepare.outputs.run_jobs == 'true'
     name: Install and CLI platform compatibility
     uses: ./.github/workflows/test.yml
     secrets: inherit
 
   tests:
+    needs: prepare
+    if: needs.prepare.outputs.run_jobs == 'true'
     name: Discovered test matrix
     uses: ./.github/workflows/test-parallel.yml
     secrets: inherit
 
-  documentation:
-    name: Documentation and root tests
-    uses: ./.github/workflows/docs.yml
-    secrets: inherit
-
   codeql:
+    needs: prepare
+    if: needs.prepare.outputs.run_jobs == 'true'
     name: CodeQL
     uses: ./.github/workflows/codeql.yml
     secrets: inherit
 
   python-artifacts:
+    needs: prepare
+    if: needs.prepare.outputs.run_jobs == 'true'
     name: Tested Python artifacts
     uses: ./.github/workflows/pip.yml
     secrets: inherit
 
   docker-digests:
+    needs: prepare
+    if: needs.prepare.outputs.run_jobs == 'true'
     name: Tested Docker images
     uses: ./.github/workflows/docker.yml
     with:
-      push_digests: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
+      push_digests: ${{ github.event_name == 'push' && (github.ref_name == 'dev' || github.ref_name == 'main') }}
     secrets: inherit
 
   required:
     name: All required CI lanes
-    if: ${{ always() }}
+    if: ${{ always() && needs.prepare.outputs.run_jobs == 'true' }}
     needs:
+      - prepare
       - lint
       - install-cli-platform
       - tests
-      - documentation
       - codeql
       - python-artifacts
       - docker-digests
@@ -73,7 +86,6 @@ jobs:
           LINT_RESULT: ${{ needs.lint.result }}
           INSTALL_CLI_PLATFORM_RESULT: ${{ needs.install-cli-platform.result }}
           TESTS_RESULT: ${{ needs.tests.result }}
-          DOCUMENTATION_RESULT: ${{ needs.documentation.result }}
           CODEQL_RESULT: ${{ needs.codeql.result }}
           PYTHON_ARTIFACTS_RESULT: ${{ needs.python-artifacts.result }}
           DOCKER_DIGESTS_RESULT: ${{ needs.docker-digests.result }}
@@ -82,7 +94,6 @@ jobs:
             "$LINT_RESULT" \
             "$INSTALL_CLI_PLATFORM_RESULT" \
             "$TESTS_RESULT" \
-            "$DOCUMENTATION_RESULT" \
             "$CODEQL_RESULT" \
             "$PYTHON_ARTIFACTS_RESULT" \
             "$DOCKER_DIGESTS_RESULT"

.github/workflows/docker.yml

@@ -65,7 +65,7 @@ jobs:
         if: github.event_name == 'workflow_dispatch'
         shell: bash
         env:
-          SOURCE_REF: ${{ inputs.source_ref }}
+          SOURCE_REF: ${{ inputs.source_ref || github.sha }}
         run: |
           set -Eeuo pipefail
           [[ "$SOURCE_REF" =~ ^[0-9a-f]{40}$ ]]
@@ -295,7 +295,7 @@ jobs:
             '
 
       - name: Validate exact built image version and commit
-        if: ${{ !inputs.push_digests }}
+        if: inputs.full_tests
         shell: bash
         run: |
           set -Eeuo pipefail
@@ -350,6 +350,7 @@ jobs:
           (( COMPRESSED_SIZE < LIMIT ))
 
       - name: Resolve real Docker CLI test dependencies through abxpkg
+        id: mount_dependencies
         if: inputs.full_tests && matrix.platform == 'linux/amd64'
         shell: bash
         run: |
@@ -371,6 +372,15 @@ jobs:
             test -x "$binary"
           done
 
+      - name: Diagnose failed mount dependency installation
+        if: failure() && steps.mount_dependencies.conclusion == 'failure'
+        shell: bash
+        run: |
+          sudo tail -n 120 /var/log/apt/term.log /var/log/dpkg.log
+          sudo journalctl --since '-10 minutes' --no-pager \
+            -u smbd -u nmbd -u samba-ad-dc -u nfs-server
+          ps -eo pid,ppid,stat,etimes,wchan:32,comm
+
       - name: Validate Docker UID and GID behavior on real NFS and SMB mounts
         if: inputs.full_tests && matrix.platform == 'linux/amd64'
         shell: bash
@@ -552,6 +562,9 @@ jobs:
 
       - name: Export digest
         if: inputs.push_digests
+        env:
+          ABX_DL_IMAGE_REF: ${{ steps.abx_dl_image.outputs.image }}
+          ARTIFACT_NAME: ${{ matrix.artifact_name }}
         shell: bash
         run: |
           set -Eeuo pipefail
@@ -559,6 +572,7 @@ jobs:
           digest="${{ steps.docker_build.outputs.digest }}"
           touch "/tmp/digests/${digest#sha256:}"
           printf '%s\n' '${{ inputs.source_ref || github.sha }}' > /tmp/digests/source-ref
+          printf '%s\n' "$ABX_DL_IMAGE_REF" > "/tmp/digests/abx-dl-image-$ARTIFACT_NAME"
 
       - name: Upload digest
         if: inputs.push_digests

.github/workflows/docs-links.yml

@@ -0,0 +1,38 @@
+name: Documentation link checks
+
+on:
+  schedule:
+    - cron: '17 10 * * *' # Daily at 10:17 UTC, away from the busy top of the hour.
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+env:
+  PYTHONIOENCODING: utf-8
+
+jobs:
+  links:
+    name: Documentation links and anchors
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+        with:
+          version: "0.11.3"
+          enable-cache: false
+      - name: List tracked documentation and source inputs
+        run: |
+          git ls-files -- '*.md' '*.html' '*.py' '*.rst' '*.yml' '*.yaml' '*.sh' '*.toml' > "$RUNNER_TEMP/link-inputs.txt"
+      - uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2
+        with:
+          lycheeVersion: v0.24.2
+          args: >-
+            --config .github/lychee.toml
+            --root-dir "${{ github.workspace }}"
+            --files-from "${{ runner.temp }}/link-inputs.txt"
+          fail: true
+          failIfEmpty: true

.github/workflows/docs.yml

@@ -1,7 +1,17 @@
 name: Documentation tests
 
 on:
-  workflow_call:
+  schedule:
+    - cron: '7 5,17 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  actions: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
 
 env:
   PYTHONIOENCODING: utf-8
@@ -9,7 +19,18 @@ env:
   SHOW_PROGRESS: "False"
 
 jobs:
+  changes:
+    runs-on: ubuntu-24.04
+    outputs:
+      changed: ${{ steps.check.outputs.changed }}
+      key: ${{ steps.check.outputs.key }}
+    steps:
+      - id: check
+        uses: ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main
+
   discover:
+    needs: changes
+    if: needs.changes.outputs.changed == 'true'
     name: Discover documentation examples
     runs-on: ubuntu-24.04
     timeout-minutes: 5
@@ -67,7 +88,7 @@ jobs:
     timeout-minutes: 15
     strategy:
       fail-fast: false
-      max-parallel: 4
+      max-parallel: 8
       matrix: ${{ fromJSON(needs.discover.outputs.matrix) }}
     steps:
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
@@ -158,3 +179,17 @@ jobs:
           "$SUDO_BINARY" \
             --preserve-env=PATH,ABXPKG_LIB_DIR,BASH_BINARY,PYTHON_BINARY,SUDO_BINARY,UV_BINARY \
             "$UV_BINARY" run --no-cache --no-sync python docs/test_codeblocks_manifest.py run-group '${{ matrix.snippet_ids }}'
+      - name: Diagnose failed Linux package installation
+        if: failure() && runner.os == 'Linux'
+        run: |
+          sudo tail -n 120 /var/log/apt/term.log /var/log/dpkg.log
+          apt-config dump Acquire::Retries Acquire::http::Timeout Acquire::https::Timeout
+          ps -eo pid,ppid,stat,etimes,wchan:32,comm
+
+  complete:
+    name: 'Scheduled inputs: ${{ needs.changes.outputs.key }}'
+    needs: [changes, discover, examples]
+    if: needs.changes.outputs.changed == 'true'
+    runs-on: ubuntu-24.04
+    steps:
+      - run: echo 'Scheduled work completed successfully.'

.github/workflows/macos.yml

@@ -0,0 +1,48 @@
+name: macOS compatibility
+
+on:
+  schedule:
+    - cron: '51 2,10,18 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  actions: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  changes:
+    runs-on: ubuntu-24.04
+    outputs:
+      changed: ${{ steps.check.outputs.changed }}
+      key: ${{ steps.check.outputs.key }}
+    steps:
+      - id: check
+        uses: ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main
+
+  install-cli:
+    needs: changes
+    if: needs.changes.outputs.changed == 'true'
+    uses: ./.github/workflows/test.yml
+    with:
+      platform: macos
+    secrets: inherit
+
+  python-artifacts:
+    needs: changes
+    if: needs.changes.outputs.changed == 'true'
+    uses: ./.github/workflows/pip.yml
+    with:
+      platforms: '["macos-15"]'
+    secrets: inherit
+
+  complete:
+    name: 'Scheduled inputs: ${{ needs.changes.outputs.key }}'
+    needs: [changes, install-cli, python-artifacts]
+    if: needs.changes.outputs.changed == 'true'
+    runs-on: ubuntu-24.04
+    steps:
+      - run: echo 'Scheduled macOS coverage completed successfully.'

.github/workflows/pip.yml

@@ -3,6 +3,11 @@ name: Build Pip package
 on:
   workflow_call:
     inputs:
+      platforms:
+        description: Runner matrix for package compatibility
+        required: false
+        default: '["ubuntu-24.04"]'
+        type: string
       full_tests:
         description: Run the full package compatibility matrix instead of one release smoke check
         required: false
@@ -95,7 +100,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-24.04, macos-15]
+        os: ${{ fromJSON(inputs.platforms) }}
         python: ["3.13", "3.14"]
         artifact: [wheel, sdist]
 

.github/workflows/prepare-release.yml

@@ -0,0 +1,55 @@
+name: Prepare release version
+
+on:
+  workflow_call:
+    outputs:
+      run_jobs:
+        description: Whether this revision owns its release version and should run expensive jobs
+        value: ${{ jobs.prepare.outputs.run_jobs }}
+
+permissions:
+  contents: write
+
+jobs:
+  prepare:
+    name: Prepare the version before starting expensive jobs
+    permissions:
+      contents: write
+    runs-on: ubuntu-24.04
+    outputs:
+      run_jobs: ${{ github.event_name != 'push' || steps.channel.outputs.stable == 'true' || steps.version.outputs.action == 'ready' || steps.version.outputs.action == 'reserved' }}
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          fetch-depth: 0
+          submodules: true
+          token: ${{ github.event_name == 'push' && secrets.RELEASE_GH_TOKEN || github.token }}
+
+      - name: Keep synchronized stable branches on the released version
+        id: channel
+        if: github.event_name == 'push'
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          VERSION="$(sed -nE 's/^version = "([^"]+)".*/\1/p' pyproject.toml | head -n 1)"
+          git fetch --no-tags origin main
+          MAIN_SHA="$(git rev-parse origin/main)"
+          # Stable reconciliation may preserve dev history with a merge commit.
+          # Its tree still matches main; bumping it would undo synchronization
+          # before the stable publisher can advance the Docker :dev tag.
+          if [[ "$GITHUB_REF_NAME" == dev && "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] &&
+             git merge-base --is-ancestor "$MAIN_SHA" "$GITHUB_SHA" &&
+             git diff --quiet "$MAIN_SHA" "$GITHUB_SHA"; then
+            echo 'stable=true' >> "$GITHUB_OUTPUT"
+          else
+            echo 'stable=false' >> "$GITHUB_OUTPUT"
+          fi
+
+      - id: version
+        if: github.event_name == 'push' && steps.channel.outputs.stable != 'true'
+        uses: ArchiveBox/monorepo/.github/actions/prepare-release-version@main
+        with:
+          package: archivebox
+          branch: ${{ github.ref_name }}
+          tag-prefix: v
+          version-scheme: ${{ github.ref_name == 'main' && 'patch' || 'rc' }}

.github/workflows/release-candidate.yml

@@ -1,79 +0,0 @@
-name: Release candidate
-
-on:
-  push:
-    branches: [dev, main]
-
-permissions:
-  actions: read
-  contents: write
-  packages: write
-
-concurrency:
-  group: release-candidate-${{ github.ref_name }}
-  cancel-in-progress: true
-
-jobs:
-  prepare:
-    runs-on: ubuntu-24.04
-    outputs:
-      action: ${{ steps.version.outputs.action }}
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 0
-          submodules: true
-          token: ${{ secrets.RELEASE_GH_TOKEN }}
-
-      - name: Keep synchronized stable branches on the released version
-        id: channel
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          VERSION="$(sed -nE 's/^version = "([^"]+)".*/\1/p' pyproject.toml | head -n 1)"
-          MAIN_SHA="$(git ls-remote origin refs/heads/main | cut -f1)"
-          if [[ "$GITHUB_REF_NAME" == dev && "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ && "$MAIN_SHA" == "$GITHUB_SHA" ]]; then
-            echo 'stable=true' >> "$GITHUB_OUTPUT"
-          else
-            echo 'stable=false' >> "$GITHUB_OUTPUT"
-          fi
-
-      - id: version
-        if: steps.channel.outputs.stable != 'true'
-        uses: ArchiveBox/monorepo/.github/actions/prepare-release-version@main
-        with:
-          package: archivebox
-          branch: ${{ github.ref_name }}
-          tag-prefix: v
-          version-scheme: ${{ github.ref_name == 'main' && 'patch' || 'rc' }}
-
-  python-artifacts:
-    needs: prepare
-    if: github.ref_name == 'dev' && (needs.prepare.outputs.action == 'ready' || needs.prepare.outputs.action == 'reserved')
-    uses: ./.github/workflows/pip.yml
-    with:
-      full_tests: false
-    secrets: inherit
-
-  docker-digests:
-    needs: prepare
-    if: github.ref_name == 'dev' && (needs.prepare.outputs.action == 'ready' || needs.prepare.outputs.action == 'reserved')
-    uses: ./.github/workflows/docker.yml
-    with:
-      push_digests: true
-      full_tests: false
-    secrets: inherit
-
-  opencode:
-    needs: prepare
-    if: github.ref_name == 'dev' && (needs.prepare.outputs.action == 'ready' || needs.prepare.outputs.action == 'reserved')
-    strategy:
-      fail-fast: false
-      matrix:
-        test_file:
-          - archivebox/tests/test_opencode_agent.py
-          - archivebox/tests/test_opencode_browser.py
-    uses: ./.github/workflows/test-parallel.yml
-    with:
-      test_file: ${{ matrix.test_file }}
-    secrets: inherit

.github/workflows/release.yml

@@ -2,7 +2,7 @@ name: Release
 
 on:
   workflow_run:
-    workflows: [CI, Release candidate]
+    workflows: [CI]
     types: [completed]
     branches: [dev, main]
   workflow_dispatch:
@@ -68,6 +68,15 @@ jobs:
           fi
           VERSION="$(sed -nE 's/^version = "([^"]+)".*/\1/p' pyproject.toml | head -n 1)"
           if [[ "$RELEASE_BRANCH" == main ]]; then
+            [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || {
+              echo "Refusing to publish non-stable version $VERSION as a main release" >&2
+              exit 1
+            }
+            git fetch --no-tags origin main
+            git merge-base --is-ancestor "$RELEASE_SHA" origin/main || {
+              echo "Refusing to publish $RELEASE_SHA: it is not on main" >&2
+              exit 1
+            }
             TAG_TARGET="$(git ls-remote origin "refs/tags/v$VERSION" | cut -f1)"
             if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch && -n "$TAG_TARGET" && "$TAG_TARGET" != "$RELEASE_SHA" ]]; then
               echo 'ready=false' >> "$GITHUB_OUTPUT"
@@ -102,26 +111,14 @@ jobs:
               --jq '[.status, .conclusion, .headSha, .event, .name] | @tsv')"
             [[ "$CI_STATUS" == "$(printf 'completed\tsuccess\t%s\tpush\tCI' "$RELEASE_SHA")" ]]
           else
-            CANDIDATE_RUN_ID="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow release-candidate.yml \
-              --commit "$RELEASE_SHA" --branch "$RELEASE_BRANCH" --event push --limit 1 \
-              --json databaseId,status,conclusion \
-              --jq '.[0] | select(.status == "completed" and .conclusion == "success") | .databaseId // empty')"
             CI_RUN_ID="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow ci.yml \
               --commit "$RELEASE_SHA" --branch "$RELEASE_BRANCH" --event push --limit 1 \
               --json databaseId,status,conclusion \
               --jq '.[0] | select(.status == "completed" and .conclusion == "success") | .databaseId // empty')"
-            [[ -n "$CANDIDATE_RUN_ID" && -n "$CI_RUN_ID" ]] || exit 0
-            if [[ "$RELEASE_BRANCH" == dev ]]; then
-              DIGEST_RUN_ID="$CANDIDATE_RUN_ID"
-            else
-              DIGEST_RUN_ID="$CI_RUN_ID"
-            fi
-          fi
-          if [[ "$RELEASE_BRANCH" == dev && "$GITHUB_EVENT_NAME" != workflow_dispatch ]]; then
-            ARTIFACT_RUN_ID="$CANDIDATE_RUN_ID"
-          else
-            ARTIFACT_RUN_ID="$CI_RUN_ID"
+            [[ -n "$CI_RUN_ID" ]] || exit 0
+            DIGEST_RUN_ID="$CI_RUN_ID"
           fi
+          ARTIFACT_RUN_ID="$CI_RUN_ID"
           {
             echo 'ready=true'
             echo "artifact_run_id=$ARTIFACT_RUN_ID"
@@ -135,10 +132,12 @@ jobs:
     runs-on: ubuntu-24.04
     env:
       CI_RUN_ID: ${{ needs.candidate.outputs.artifact_run_id }}
+      DOCKER_DIGEST_RUN_ID: ${{ needs.candidate.outputs.digest_run_id }}
     environment: pypi
     permissions:
       actions: read
       contents: write
+      deployments: read
       id-token: write
     outputs:
       version: ${{ steps.version.outputs.version }}
@@ -162,6 +161,26 @@ jobs:
           version: "0.11.3"
           enable-cache: false
 
+      # This precedes every stable upload, including workflow_dispatch. A green
+      # build is not evidence that our automatically updated test servers ran it.
+      - name: Download tested Docker base-image metadata
+        if: env.RELEASE_BRANCH == 'main'
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        with:
+          path: ${{ runner.temp }}/tested-docker-artifacts
+          pattern: digest-*
+          merge-multiple: true
+          github-token: ${{ github.token }}
+          repository: ${{ github.repository }}
+          run-id: ${{ env.DOCKER_DIGEST_RUN_ID }}
+
+      - name: Require matching acceptance on Cabbage
+        if: env.RELEASE_BRANCH == 'main'
+        env:
+          GH_TOKEN: ${{ github.token }}
+          STAGING_DOCKER_ARTIFACT_DIR: ${{ runner.temp }}/tested-docker-artifacts
+        run: uv run --no-project python bin/verify-staging-release.py
+
       - name: Resolve release binaries through abxpkg
         env:
           ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
@@ -210,6 +229,10 @@ jobs:
           VERSION='${{ steps.version.outputs.version }}'
           case "$RELEASE_BRANCH" in
             main)
+              [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || {
+                echo "Refusing to publish non-stable version $VERSION from main" >&2
+                exit 1
+              }
               ;;
             dev)
               [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+rc[0-9]+$ ]] || {
@@ -266,7 +289,7 @@ jobs:
           ref: ${{ env.RELEASE_SHA }}
           token: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
 
-      - name: Fast-forward dev to the tested stable release
+      - name: Reconcile dev to the tested stable release
         if: env.RELEASE_BRANCH == 'main' && needs.python-release.outputs.release_ready == 'true'
         shell: bash
         run: |
@@ -274,10 +297,65 @@ jobs:
           git fetch --no-tags origin main dev
           MAIN_SHA="$(git rev-parse origin/main)"
           DEV_SHA="$(git rev-parse origin/dev)"
-          if [[ "$MAIN_SHA" == "$RELEASE_SHA" && "$DEV_SHA" != "$RELEASE_SHA" ]] &&
-             git merge-base --is-ancestor "$DEV_SHA" "$RELEASE_SHA"; then
-            git push origin "$RELEASE_SHA:refs/heads/dev"
+          if [[ "$MAIN_SHA" != "$RELEASE_SHA" || "$DEV_SHA" == "$RELEASE_SHA" ]]; then
+            exit 0
           fi
+          push_dev() {
+            if ! git push origin "$1:refs/heads/dev"; then
+              CURRENT_DEV="$(git ls-remote origin refs/heads/dev | cut -f1)"
+              if [[ "$CURRENT_DEV" == "$DEV_SHA" ]]; then
+                echo "Failed to push dev although it still points to $DEV_SHA" >&2
+                exit 1
+              fi
+              echo "Leaving dev at $CURRENT_DEV: it advanced during release reconciliation"
+            fi
+          }
+          if git merge-base --is-ancestor "$DEV_SHA" "$RELEASE_SHA"; then
+            push_dev "$RELEASE_SHA"
+            exit 0
+          else
+            STATUS=$?
+            [[ "$STATUS" -eq 1 ]] || exit "$STATUS"
+          fi
+          BASE_SHA="$(git merge-base "$MAIN_SHA" "$DEV_SHA")"
+          if git diff --quiet "$BASE_SHA" "$DEV_SHA" -- . ':!pyproject.toml' ':!uv.lock' ':!etc/package.json'; then
+            :
+          else
+            STATUS=$?
+            [[ "$STATUS" -eq 1 ]] || exit "$STATUS"
+            echo "Leaving dev at $DEV_SHA: it contains changes beyond release metadata"
+            exit 0
+          fi
+          normalized_metadata() {
+            case "$2" in
+              pyproject.toml)
+                git show "$1:$2" | awk '
+                  /^\[/ {section = $0}
+                  section == "[project]" && /^version = "[^"]+"$/ {print "version = \"__VERSION__\""; next}
+                  section == "[tool.bumpver]" && /^current_version = "[^"]+"$/ {print "current_version = \"__VERSION__\""; next}
+                  {print}
+                ' ;;
+              uv.lock)
+                git show "$1:$2" | awk 'previous == "name = \"archivebox\"" && /^version = / {print "version = \"__VERSION__\""; previous = $0; next} {print; previous = $0}' ;;
+              etc/package.json)
+                git show "$1:$2" | sed -E 's/^(  "version": )"[^"]+"(,?)$/\1"__VERSION__"\2/' ;;
+            esac
+          }
+          for path in pyproject.toml uv.lock etc/package.json; do
+            BASE_METADATA="$(normalized_metadata "$BASE_SHA" "$path")"
+            DEV_METADATA="$(normalized_metadata "$DEV_SHA" "$path")"
+            if [[ "$BASE_METADATA" != "$DEV_METADATA" ]]; then
+              echo "Leaving dev at $DEV_SHA: $path contains changes beyond release version metadata"
+              exit 0
+            fi
+          done
+          # A concurrent release bot may bump only dev's own version metadata.
+          # Keep that commit as a parent while preserving the exact tested stable tree.
+          git switch --detach "$MAIN_SHA"
+          git -c user.name='github-actions[bot]' -c user.email='41898282+github-actions[bot]@users.noreply.github.com' \
+            merge -s ours --no-ff -m "Merge tested stable release into dev" "$DEV_SHA"
+          MERGE_SHA="$(git rev-parse HEAD)"
+          push_dev "$MERGE_SHA"
 
       - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
@@ -369,6 +447,26 @@ jobs:
             TAG_TARGET="$($GIT_BINARY ls-remote origin "refs/tags/v${VERSION}")"
             TAG_TARGET="${TAG_TARGET%%[[:space:]]*}"
           fi
+          if [[ "$RELEASE_BRANCH" == main ]]; then
+            [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || {
+              echo "Refusing to tag :latest with prerelease $VERSION" >&2
+              exit 1
+            }
+            [[ "$TAG_TARGET" == "$RELEASE_SHA" ]] || {
+              echo "Refusing to tag :latest: v$VERSION does not point to $RELEASE_SHA" >&2
+              exit 1
+            }
+            $GIT_BINARY fetch --no-tags origin main
+            $GIT_BINARY merge-base --is-ancestor "$RELEASE_SHA" origin/main || {
+              echo "Refusing to tag :latest: $RELEASE_SHA is not on main" >&2
+              exit 1
+            }
+            MAIN_VERSION="$($GIT_BINARY show origin/main:pyproject.toml | sed -nE 's/^version = "([^"]+)".*/\1/p' | head -n 1)"
+            [[ "$MAIN_VERSION" == "$VERSION" ]] || {
+              echo "Refusing to tag :latest: main is now $MAIN_VERSION, not $VERSION" >&2
+              exit 1
+            }
+          fi
           SYNC_DEV=false
           if [[ "$RELEASE_BRANCH" == main ]]; then
             $GIT_BINARY fetch --no-tags origin main dev
@@ -376,7 +474,8 @@ jobs:
             DEV_TARGET="$($GIT_BINARY rev-parse origin/dev)"
             # Recovery can add CI/docs fixes after the immutable release tag.
             # Both branches and every shipped application file must agree.
-            if [[ "$MAIN_TARGET" == "$DEV_TARGET" ]] &&
+            if $GIT_BINARY merge-base --is-ancestor "$MAIN_TARGET" "$DEV_TARGET" &&
+               $GIT_BINARY diff --quiet "$MAIN_TARGET" "$DEV_TARGET" &&
                $GIT_BINARY diff --quiet "$RELEASE_SHA" "$MAIN_TARGET" -- . ':!.github' ':!README.md'; then
               SYNC_DEV=true
             fi
@@ -438,6 +537,26 @@ jobs:
             test "$INSTALLED_VERSION" = "$VERSION"
           done
 
+      - name: Record published image source
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          # workflow_run can report the default branch's head even when this
+          # run published a different branch/SHA. Preserve the verified target.
+          [[ "$RELEASE_BRANCH" == dev || "$RELEASE_BRANCH" == main ]]
+          [[ "$RELEASE_SHA" =~ ^[0-9a-f]{40}$ ]]
+          mkdir -p "$RUNNER_TEMP/published-image-source"
+          # shellcheck disable=SC2016
+          "$JQ_BINARY" -n --arg branch "$RELEASE_BRANCH" --arg sha "$RELEASE_SHA" \
+            '{branch: $branch, sha: $sha}' > "$RUNNER_TEMP/published-image-source/source.json"
+
+      - name: Upload published image source
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: published-image-source
+          path: ${{ runner.temp }}/published-image-source/source.json
+          if-no-files-found: error
+
       - name: Update Docker Hub README
         uses: peter-evans/dockerhub-description@432a30c9e07499fd01da9f8a49f0faf9e0ca5b77 # v4
         with:

.github/workflows/screenshots.yml

@@ -1,29 +1,31 @@
 name: Capture publicsite screenshots
 
 on:
-  push:
-    branches:
-      - dev
-    paths:
-      - archivebox/**
-      - bin/collect_ui_screenshots.*
-      - bin/generate_ui_screenshot_gallery.py
-      - bin/take_screenshot.js
-      - docs/Screenshots.md
-      - pyproject.toml
-      - uv.lock
-      - .github/workflows/screenshots.yml
+  schedule:
+    - cron: '37 5,17 * * *'
   workflow_dispatch:
 
 permissions:
   contents: read
+  actions: read
 
 concurrency:
-  group: ${{ github.event_name == 'workflow_dispatch' && format('publicsite-captures-manual-{0}', github.run_id) || 'publicsite-captures' }}
+  group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: false
 
 jobs:
+  changes:
+    runs-on: ubuntu-24.04
+    outputs:
+      changed: ${{ steps.check.outputs.changed }}
+      key: ${{ steps.check.outputs.key }}
+    steps:
+      - id: check
+        uses: ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main
+
   deploy:
+    needs: changes
+    if: needs.changes.outputs.changed == 'true'
     runs-on: ubuntu-24.04
     steps:
       - name: Checkout
@@ -69,3 +71,11 @@ jobs:
           path: publicsite/screenshots
           if-no-files-found: error
           retention-days: 90
+
+  complete:
+    name: 'Scheduled inputs: ${{ needs.changes.outputs.key }}'
+    needs: [changes, deploy]
+    if: needs.changes.outputs.changed == 'true'
+    runs-on: ubuntu-24.04
+    steps:
+      - run: echo 'Scheduled work completed successfully.'

.github/workflows/staging-acceptance.yml

@@ -0,0 +1,109 @@
+name: Staging acceptance
+
+on:
+  workflow_run:
+    workflows: [Release]
+    types: [completed]
+
+permissions: {}
+
+concurrency:
+  group: staging-acceptance
+  cancel-in-progress: false
+
+jobs:
+  published:
+    if: github.event.workflow_run.conclusion == 'success'
+    runs-on: ubuntu-24.04
+    permissions:
+      actions: read
+    outputs:
+      ready: ${{ steps.source.outputs.ready }}
+      sha: ${{ steps.source.outputs.sha }}
+    steps:
+      - id: image
+        env:
+          GH_TOKEN: ${{ github.token }}
+          RELEASE_RUN_ID: ${{ github.event.workflow_run.id }}
+        run: |
+          set -Eeuo pipefail
+          published=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$RELEASE_RUN_ID/jobs?per_page=100" --jq 'any(.jobs[]; .name == "Publish tested multiarch image" and .conclusion == "success")')
+          echo "published=$published" >> "$GITHUB_OUTPUT"
+      - name: Download published image source
+        if: steps.image.outputs.published == 'true'
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        with:
+          name: published-image-source
+          path: ${{ runner.temp }}/published-image-source
+          github-token: ${{ github.token }}
+          repository: ${{ github.repository }}
+          run-id: ${{ github.event.workflow_run.id }}
+      - name: Select published dev image
+        id: source
+        if: steps.image.outputs.published == 'true'
+        run: |
+          set -Eeuo pipefail
+          # The triggering run may report the default branch as its head;
+          # this artifact names the image source verified by the publisher.
+          source="$RUNNER_TEMP/published-image-source/source.json"
+          jq -e 'type == "object" and (keys | sort) == ["branch", "sha"]' "$source" >/dev/null
+          branch=$(jq -er '.branch | select(type == "string")' "$source")
+          sha=$(jq -er '.sha | select(type == "string")' "$source")
+          [[ "$branch" == dev || "$branch" == main ]]
+          [[ "$sha" =~ ^[0-9a-f]{40}$ ]]
+          echo "sha=$sha" >> "$GITHUB_OUTPUT"
+          if [[ "$branch" == dev ]]; then
+            echo 'ready=true' >> "$GITHUB_OUTPUT"
+          else
+            echo 'ready=false' >> "$GITHUB_OUTPUT"
+          fi
+  acceptance:
+    # The privileged key is never exposed to PRs or arbitrary dispatch refs.
+    needs: published
+    if: needs.published.outputs.ready == 'true'
+    runs-on: [self-hosted, Linux, X64, ugnas-linux]
+    environment: staging
+    timeout-minutes: 110
+    permissions:
+      contents: read
+      deployments: write
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [cabbage]
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
+        with:
+          ref: ${{ needs.published.outputs.sha }}
+          fetch-depth: 0
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e
+        with:
+          version: '0.11.3'
+      - name: Prepare browser and restricted staging identity
+        env:
+          STAGING_SSH_KEY: ${{ secrets.STAGING_SSH_KEY }}
+          STAGING_KNOWN_HOSTS: ${{ secrets.STAGING_KNOWN_HOSTS }}
+        run: |
+          set -Eeuo pipefail
+          umask 077
+          printf '%s\n' "$STAGING_SSH_KEY" > "$RUNNER_TEMP/staging-key"
+          printf '%s\n' "$STAGING_KNOWN_HOSTS" > "$RUNNER_TEMP/staging-known-hosts"
+          uv run --no-project --with playwright==1.63.0 playwright install chromium
+      - name: Verify actual deployment, full captures and rendered snapshots
+        env:
+          GH_TOKEN: ${{ github.token }}
+          STAGING_TARGET: ${{ matrix.target }}
+          STAGING_SHA: ${{ needs.published.outputs.sha }}
+          STAGING_KEY_FILE: ${{ runner.temp }}/staging-key
+          STAGING_KNOWN_HOSTS_FILE: ${{ runner.temp }}/staging-known-hosts
+        run: uv run --no-project --with playwright==1.63.0 python bin/check-staging-deployment.py
+      - name: Remove staging identity
+        if: always()
+        run: rm -f "$RUNNER_TEMP/staging-key" "$RUNNER_TEMP/staging-known-hosts"
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
+        if: always()
+        with:
+          name: staging-${{ matrix.target }}-${{ needs.published.outputs.sha }}
+          path: staging-evidence/
+          if-no-files-found: error

.github/workflows/test-parallel.yml

@@ -1,6 +1,13 @@
 name: Parallel Tests
 
 on:
+  workflow_dispatch:
+    inputs:
+      test_file:
+        description: Run one discovered test file
+        required: true
+        default: archivebox/tests/test_cli_list.py
+        type: string
   workflow_call:
     inputs:
       test_file:
@@ -19,11 +26,20 @@ env:
 jobs:
   discover-tests:
     name: Discover every test
+    # Selection must run on GitHub so an offline NAS cannot block the fallback.
     runs-on: ubuntu-24.04
     outputs:
       tests: ${{ steps.discover.outputs.tests }}
+      runner: ${{ steps.runner.outputs.use-runner || '["ubuntu-24.04"]' }}
 
     steps:
+      - name: Select available CI capacity
+        id: runner
+        if: vars.UGNAS_CI_ENABLED == 'true' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main')
+        uses: ArchiveBox/monorepo/.github/actions/select-ci-runner@main
+        with:
+          github-token: ${{ secrets.RUNNER_STATUS_TOKEN }}
+
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
           submodules: true
@@ -66,6 +82,7 @@ jobs:
         id: discover
         env:
           TEST_FILE: ${{ inputs.test_file }}
+          UGNAS_CI_MAX_JOBS: ${{ vars.UGNAS_CI_MAX_JOBS || 3 }}
         run: |
           set -Eeuo pipefail
           mapfile -t output < <("$PYTHON_BINARY" .github/scripts/discover_test_matrix.py)
@@ -80,13 +97,13 @@ jobs:
   tests:
     name: ${{ matrix.test.name }}
     needs: discover-tests
-    runs-on: ubuntu-24.04
+    # Opt in trusted branches only; PRs always use GitHub-hosted runners.
+    # Ordinary Linux tests are eligible; # ci-runner: hosted declares exceptions.
+    # The hosted discovery job falls back when workers are offline or the
+    # availability API fails. UGNAS_CI_ENABLED=false disables the pool entirely.
+    runs-on: ${{ fromJSON(matrix.test.ugnas && needs.discover-tests.outputs.runner || '["ubuntu-24.04"]') }}
     env:
-      ABXPKG_LIB_DIR: /tmp/archivebox-lib
       CHROME_HEADLESS: "true"
-      DATA_DIR: /tmp/archivebox-test-data
-      PERSONAS_DIR: /tmp/abx-personas
-      CHROME_USER_DATA_DIR: /tmp/abx-personas/Default/chrome_profile
 
     strategy:
       fail-fast: false
@@ -94,6 +111,25 @@ jobs:
         test: ${{ fromJson(needs.discover-tests.outputs.tests) }}
 
     steps:
+      - name: Set job-local runtime paths
+        env:
+          CI_RUNNER_ENVIRONMENT: ${{ runner.environment }}
+        run: |
+          workers=2
+          cache_suffix=''
+          if [[ "$CI_RUNNER_ENVIRONMENT" == github-hosted ]]; then
+            workers="$(nproc)"
+            cache_suffix="-workers-$workers"
+          fi
+          {
+            echo "CI_TEST_WORKERS=$workers"
+            echo "ABXPKG_CACHE_WORKERS_SUFFIX=$cache_suffix"
+            echo "ABXPKG_LIB_DIR=$RUNNER_TEMP/archivebox-lib"
+            echo "DATA_DIR=$RUNNER_TEMP/archivebox-test-data"
+            echo "PERSONAS_DIR=$RUNNER_TEMP/abx-personas"
+            echo "CHROME_USER_DATA_DIR=$RUNNER_TEMP/abx-personas/Default/chrome_profile"
+          } >> "$GITHUB_ENV"
+
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
           submodules: true
@@ -103,6 +139,7 @@ jobs:
         with:
           python-version: ${{ env.CI_PYTHON_VERSION }}
           architecture: x64
+        id: python
 
       - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
         with:
@@ -110,6 +147,41 @@ jobs:
           enable-cache: false
           cache-dependency-glob: uv.lock
 
+      # Self-hosted tool storage survives jobs. Normal abxpkg validation and
+      # install locks allow reuse across files; cold-install tests use tmp_path.
+      - name: Reuse persistent runner tools
+        if: runner.environment == 'self-hosted'
+        env:
+          PERSISTENT_ABXPKG_LIB: ${{ runner.tool_cache }}/abxpkg/${{ github.repository }}/${{ steps.python.outputs.python-version }}
+        run: |
+          mkdir -p "$PERSISTENT_ABXPKG_LIB"
+          echo "ABXPKG_LIB_DIR=$PERSISTENT_ABXPKG_LIB" >> "$GITHUB_ENV"
+
+      # Restored binaries still pass normal abxpkg resolution/version checks.
+      # Installation and isolation tests explicitly choose fresh temporary libs.
+      - name: Identify runtime dependency cache
+        shell: bash
+        run: |
+          uv run --no-project python - <<'PYTHON' >> "$GITHUB_ENV"
+          import hashlib
+          import json
+          import tomllib
+          from pathlib import Path
+
+          packages = tomllib.loads(Path("uv.lock").read_text())["package"]
+          versions = {p["name"]: p["version"] for p in packages if p["name"] in {"abxpkg", "abx-plugins"} and "editable" not in p["source"]}
+          print("ABXPKG_CACHE_DEPS=" + hashlib.sha256(json.dumps(versions, sort_keys=True).encode()).hexdigest())
+          PYTHON
+
+      - name: Reuse validated runtime dependencies
+        if: runner.environment != 'self-hosted'
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+        with:
+          path: ${{ runner.temp }}/archivebox-lib
+          key: abxpkg-v1-${{ runner.os }}-${{ runner.arch }}-${{ runner.environment }}-${{ steps.python.outputs.python-version }}-${{ matrix.test.name }}-${{ env.ABXPKG_CACHE_DEPS }}${{ env.ABXPKG_CACHE_WORKERS_SUFFIX }}
+          restore-keys: |
+            abxpkg-v1-${{ runner.os }}-${{ runner.arch }}-${{ runner.environment }}-${{ steps.python.outputs.python-version }}-${{ matrix.test.name }}-
+
       - name: Resolve optional build dependencies through abxpkg
         if: matrix.test.extra != '' || contains(matrix.test.paths_arg, 'test_postgres_backend')
         env:
@@ -150,12 +222,15 @@ jobs:
       - name: Run ${{ matrix.test.name }}
         env:
           TEST_PATHS: ${{ matrix.test.paths_arg }}
+          CI_RUNNER_ENVIRONMENT: ${{ runner.environment }}
         run: |
           set -Eeuo pipefail
+          echo "Running pytest with $CI_TEST_WORKERS workers on $CI_RUNNER_ENVIRONMENT"
           read -r -a test_paths <<< "$TEST_PATHS"
           for test_path in "${test_paths[@]}"; do
             test -f "$test_path"
           done
+          test_status=0
           for test_path in "${test_paths[@]}"; do
             test_name="$(basename "$test_path" .py)"
             test_slug="${test_name//[^A-Za-z0-9_.-]/_}"
@@ -163,6 +238,24 @@ jobs:
             export PERSONAS_DIR="$RUNNER_TEMP/abx-personas/$test_slug"
             export CHROME_USER_DATA_DIR="$PERSONAS_DIR/Default/chrome_profile"
             mkdir -p "$DATA_DIR" "$PERSONAS_DIR" "$RUNNER_TEMP/pytest/$test_slug"
-            uv run --no-cache --no-sync --no-sources pytest -vs "$test_path" \
-              --basetemp="$RUNNER_TEMP/pytest/$test_slug"
+            uv run --no-cache --no-sync --no-sources pytest -vs -n "$CI_TEST_WORKERS" "$GITHUB_WORKSPACE/$test_path" --durations=20 \
+              --basetemp="$RUNNER_TEMP/pytest/$test_slug" \
+              --junitxml="$RUNNER_TEMP/test-results/$test_slug.xml" || test_status=1
           done
+          exit "$test_status"
+      - name: Save test results
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: test-results-${{ strategy.job-index }}
+          path: ${{ runner.temp }}/test-results/*.xml
+          if-no-files-found: error
+          retention-days: 14
+      - name: Save failed runtime diagnostics
+        if: failure()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: runtime-logs-${{ strategy.job-index }}
+          path: ${{ runner.temp }}/pytest/**/*.log
+          if-no-files-found: ignore
+          retention-days: 14

.github/workflows/test.yml

@@ -2,6 +2,12 @@ name: Install and CLI Compatibility
 
 on:
   workflow_call:
+    inputs:
+      platform:
+        description: Platform for install and CLI compatibility
+        required: false
+        default: linux
+        type: string
 
 env:
   DOCKER_IMAGE: archivebox-ci
@@ -19,16 +25,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - os: ubuntu-24.04
-            os_name: linux
-            python: "3.14"
-          - os: macos-15
-            os_name: macOS
-            python: "3.13"
-          - os: macos-15
-            os_name: macOS
-            python: "3.14"
+        include: ${{ fromJSON(inputs.platform == 'macos' && '[{"os":"macos-15","os_name":"macOS","python":"3.13"},{"os":"macos-15","os_name":"macOS","python":"3.14"}]' || '[{"os":"ubuntu-24.04","os_name":"linux","python":"3.14"}]') }}
 
     steps:
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4

AGENTS.md

@@ -77,11 +77,7 @@ archivebox run
 
 ## Verification
 
-Use targeted tests for focused work:
-
-```bash
-uv run pytest archivebox/tests/test_cli_add.py::test_add_help_shows_depth_and_tag_options -q
-uv run prek run --all-files
-```
+Run the test suite with `uv run pytest archivebox/tests -x` and repository checks
+with `uv run prek run --all-files`. For focused work, select the relevant tests.
 
 Releases are published only by `.github/workflows/release.yml` after complete CI succeeds for the exact commit. Pushes to `main` prepare stable patch releases automatically; `dev` prepares release candidates. Version selection and dependency repinning are owned by CI and the monorepo coordinator. Local development and deployment commands must not publish packages, images, tags, or GitHub releases.

Dockerfile

@@ -125,6 +125,7 @@ FROM archivebox-runtime-base AS archivebox-builder
 WORKDIR "$CODE_DIR"
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
     --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
+    --mount=type=bind,source=uv.lock,target=/app/uv.lock \
     <<'EOF'
 echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..."
 export UV_NO_CACHE=true
@@ -137,48 +138,14 @@ apt-get install -qq -y --no-install-recommends x11-utils
 # Do not rewrite inherited native libraries merely to strip them again.
 /usr/bin/find /venv/lib/python3.*/site-packages -type f -name '*.so' -print > /tmp/archivebox-inherited-libraries
 
-mkdir -p /tmp/archivebox-uv-project
-/venv/bin/python - <<'PY'
-from pathlib import Path
-import json
-import re
-import urllib.request
-
-source = Path("/app/pyproject.toml")
-target = Path("/tmp/archivebox-uv-project/pyproject.toml")
-text = source.read_text()
-text = text.replace(
-    'environments = ["sys_platform == \'darwin\'", "sys_platform == \'linux\'"]',
-    'environments = ["sys_platform == \'linux\'"]',
-)
-
-# Docker builds need the just-published internal abx wheels immediately, but
-# PyPI simple can lag the version JSON endpoints by tens of minutes. Generate a
-# Docker-only dependency view from the version JSON so the published package
-# metadata stays normal while image builds remain resumable after a release.
-for package in ("abxbus", "abxpkg", "abx-plugins", "abx-dl"):
-    match = re.search(
-        rf'"{re.escape(package)}(?P<operator>==|>=)(?P<version>[^"]+)"',
-        text,
-    )
-    if not match:
-        continue
-    version = match.group("version")
-    with urllib.request.urlopen(f"https://pypi.org/pypi/{package}/{version}/json", timeout=20) as response:
-        data = json.load(response)
-    wheel_url = next(url["url"] for url in data["urls"] if url["filename"].endswith(".whl"))
-    text = re.sub(
-        rf'"{re.escape(package)}(?:==|>=)[^"]+"',
-        f'"{package} @ {wheel_url}"',
-        text,
-        count=1,
-    )
-
-target.write_text(text)
-PY
-
+# The old Docker-only dependency rewrite predated this checkout's lockfile:
+# it fetched version JSON to avoid stale PyPI Simple listings. That extra API
+# can itself lag or return 503s. The verified lock already records exact public
+# artifact URLs and hashes, including extras and platform markers. Consume it
+# directly, retaining the freshness check instead of resolving a second graph.
 /usr/bin/uv sync \
-    --project /tmp/archivebox-uv-project \
+    --project /app \
+    --locked \
     --no-cache \
     --no-dev \
     --inexact \
@@ -209,7 +176,7 @@ rm -rf "$builder_abxpkg_lib_dir"
 rm -rf /venv/lib/python3.*/site-packages/pip* \
     /venv/lib/python3.*/site-packages/wheel* \
     /venv/bin/pip /venv/bin/pip3 /venv/bin/pip3.* /venv/bin/wheel
-rm -rf /var/lib/apt/lists/* /tmp/archivebox-uv-project
+rm -rf /var/lib/apt/lists/*
 EOF
 
 COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
@@ -291,7 +258,13 @@ RUN --mount=type=cache,target=/var/tmp/abxpkg-cache,sharing=locked,mode=1777,id=
         setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abx-dl install $ABX_DOCKER_PLUGINS \
     # pnpm includes musl variants that Debian's glibc runtime cannot use.
     && find "$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules" -type l -name 'opencode-linux-*-musl' -delete \
-    && find "$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.pnpm" -maxdepth 1 -type d -name 'opencode-linux-*-musl@*' -exec rm -rf {} +
+    && find "$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.pnpm" -maxdepth 1 -type d -name 'opencode-linux-*-musl@*' -exec rm -rf {} + \
+    # pnpm's explicit store lives under ABXPKG_LIB_DIR, regardless of XDG_CACHE_HOME.
+    # Retaining its download copies made image size depend on whether the build
+    # preserved hardlinks to installed files. Remove the store in this layer;
+    # installed files retain their contents, and the offline check below proves
+    # the complete catalog still works without the disposable package payloads.
+    && rm -rf "$ABXPKG_LIB_DIR/cache/pnpm"
 RUN --network=none export ABX_DOCKER_PLUGINS="$(/venv/bin/python3 -c 'from abx_dl.catalog import PluginCatalog; print(" ".join(PluginCatalog.discover(runtime="archivebox")))')" \
     && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups \
     bash -c '"$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" --version && abx-dl install $ABX_DOCKER_PLUGINS'

README.md

@@ -105,12 +105,11 @@ curl -fsSL 'https://get.archivebox.io' | bash
 </details>
 <br/>
 
-
 <div align="center" style="text-align: center">
 <br/><br/>
 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/5a7d95f2-6977-4de6-9f08-42851a1fe1d2" height="70px" alt="bookshelf graphic"> &nbsp; <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/b2765a33-0d1e-4019-a1db-920c7e00e20e" height="75px" alt="logo" align="top"/> &nbsp; <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/5a7d95f2-6977-4de6-9f08-42851a1fe1d2" height="70px" alt="bookshelf graphic">
 <br/><br/>
-<small><a href="https://demo.archivebox.io">Demo</a> | <a href="#screenshots">Screenshots</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a></small>
+<small><a href="https://demo.archivebox.io">Demo</a> | <a href="https://archivebox.io/screenshots">Screenshots</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a></small>
 <br/>
 <sub>. . . . . . . . . . . . . . . . . . . . . . . . . . . .</sub>
 <br/><br/>
@@ -124,13 +123,13 @@ curl -fsSL 'https://get.archivebox.io' | bash
 ## Key Features
 
 - [**Free & open source**](https://github.com/ArchiveBox/ArchiveBox/blob/dev/LICENSE), own your own data & maintain your privacy by self-hosting
-- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) with [modular dependencies](#dependencies) and [support for Google Drive/NFS/SMB/S3/B2/etc.](https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage)
+- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage) with [modular dependencies](#dependencies) and [support for Google Drive/NFS/SMB/S3/B2/etc.](https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage)
 - [**Comprehensive documentation**](https://github.com/ArchiveBox/ArchiveBox/wiki), [active development](https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap), and [rich community](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community)
 - [**Extracts a wide variety of content out-of-the-box**](https://github.com/ArchiveBox/ArchiveBox/issues/51): [media (yt-dlp), articles (readability), code (git), etc.](#output-formats)
 - [**Supports scheduled/realtime importing**](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving) from [many types of sources](#input-formats)
 - [**Uses standard, durable, long-term formats**](#output-formats) like HTML, JSON, PDF, PNG, MP4, TXT, and WARC
-- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [Python API](https://docs.archivebox.io/en/dev/apidocs/archivebox/archivebox.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://electron.archivebox.io/)
-- [**Saves all pages to archive.org as well**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#save_archive_dot_org) by default for redundancy (can be [disabled](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#stealth-mode) for local-only mode)
+- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#ui-usage), [Python API](https://docs.archivebox.io/dev/apidocs/archivebox/archivebox.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://electron.archivebox.io/)
+- [**Saves all pages to archive.org as well**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#archivedotorg_enabled) by default for redundancy (can be [disabled](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#archivedotorg_enabled) for local-only mode)
 - Support for archiving [content requiring login/paywall/cookies](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir) (see wiki security caveats!)
 - Support for running [JS during archiving](https://github.com/ArchiveBox/ArchiveBox/issues/51) to adblock, [infiniscroll](https://github.com/ArchiveBox/ArchiveBox/issues/80), [modal-hide](https://github.com/ArchiveBox/ArchiveBox/issues/175), [thread-expand](https://github.com/ArchiveBox/ArchiveBox/issues/345)
 
@@ -221,7 +220,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <ol>
 <li>Run the shortcut for the <code>uv</code> install method. It installs <code>uv</code> first when needed.
 <pre lang="bash"><code style="white-space: pre-line">curl -fsSL 'https://get.archivebox.io' | bash</code></pre>
-<i>For more info, see <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-b-automatic-setup-script">Install: Bare Metal</a> in the Wiki. ➡️</i>
+<i>For more info, see <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-b-uv-install-shortcut">Install: Bare Metal</a> in the Wiki. ➡️</i>
 </li>
 </ol>
 
@@ -381,7 +380,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for usage examples using the CLI, W
 > *Warning: These are contributed by external volunteers and may lag behind the official Docker and `uv` channels.*
 
 <ul>
-<li><s>TrueNAS: <a href="https://truecharts.org/charts/stable/archivebox/">Official ArchiveBox TrueChart</a> / <a href="https://dev.to/finloop/setting-up-archivebox-on-truenas-scale-1788">Custom App Guide</a></s> (<a href="https://truecharts.org/news/scale-deprecation/">TrueCharts is discontinued</a>, wait for <a href="https://forums.truenas.com/t/the-future-of-electric-eel-and-apps/5409/">Electric Eel</a>)</li>
+<li><a href="https://apps.truenas.com/catalog/archivebox_community/">TrueNAS</a></li>
 <li><a href="https://unraid.net/community/apps?q=archivebox#r">UnRaid</a></li>
 <li><a href="https://community-scripts.github.io/ProxmoxVE/scripts?id=archivebox">Proxmox</a></li>
 <li><a href="https://github.com/YunoHost-Apps/archivebox_ynh">Yunohost</a></li>
@@ -432,7 +431,7 @@ For more discussion on third-party hosting options see here: <a href="https://gi
 #### ➡️&nbsp; Next Steps
 
 - Import URLs from some of the supported [Input Formats](#input-formats) or view the supported [Output Formats](#output-formats)...
-- (Optional) [Import browser cookies and settings into a persona](docs/Chromium-Install.md#import-an-existing-browser-profile).
+- (Optional) [Import browser cookies and settings into a persona](https://github.com/ArchiveBox/ArchiveBox/blob/dev/docs/Chromium-Install.md#import-an-existing-browser-profile).
 - Tweak your UI or archiving behavior [Configuration](#configuration), read about some of the [Caveats](#caveats), or [Troubleshoot](https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting)
 - Read about the [Dependencies](#dependencies) used for archiving, the [Upgrading Process](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives), or the [Archive Layout](#archive-layout) on disk...
 - Or check out our full [Documentation](#documentation) or [Community Wiki](#internet-archiving-ecosystem)...
@@ -614,7 +613,7 @@ docker run -it -v $PWD:/data archivebox/archivebox:dev add --depth=1 'https://ex
 <sub>. . . . . . . . . . . . . . . . . . . . . . . . . . . .</sub>
 <br/><br/>
 <a href="https://demo.archivebox.io">DEMO: <code>https://demo.archivebox.io</code></a><br/>
-<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration">Configuration</a> | <a href="#Caveats">Caveats</a>
+<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration">Configuration</a> | <a href="#caveats">Caveats</a>
 <br/>
 </div>
 
@@ -642,7 +641,7 @@ docker run -it -v $PWD:/data archivebox/archivebox:dev add --depth=1 'https://ex
   <i>Save directly from the Share menu or Siri/Shortcuts to your server.</i>
 
 - <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/64078483-21d7-4eb1-aa6e-9ad55afe45b8" height="22px"/> From manual imports of URLs from RSS, JSON, CSV, TXT, SQL, HTML, Markdown, etc. files  
-  <i>ArchiveBox supports injesting URLs in [any text-based format](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Import-a-list-of-URLs-from-a-text-file).</i>
+  <i>ArchiveBox supports injesting URLs in [any text-based format](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#import-a-list-of-urls-from-a-text-file).</i>
 
 - <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/32b494e6-4de1-4984-8d88-dc02f18e5c34" height="22px"/> From manually exported [browser history](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) or [browser bookmarks](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) (in Netscape format)  
   <i>Instructions: <a href="https://support.google.com/chrome/answer/96816?hl=en">Chrome</a>, <a href="https://support.mozilla.org/en-US/kb/export-firefox-bookmarks-to-backup-or-transfer">Firefox</a>, <a href="https://github.com/ArchiveBox/ArchiveBox/assets/511499/24ad068e-0fa6-41f4-a7ff-4c26fc91f71a">Safari</a>, <a href="https://support.microsoft.com/en-us/help/211089/how-to-import-and-export-the-internet-explorer-favorites-folder-to-a-32-bit-version-of-windows">IE</a>, <a href="https://help.opera.com/en/latest/features/#bookmarks:~:text=Click%20the%20import/-,export%20button,-on%20the%20bottom">Opera</a>, <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive">and more...</a></i>
@@ -671,7 +670,7 @@ echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox a
 # echo 'https://example.com' | docker compose run --rm -T archivebox add
 ```
 
-See the [Usage: CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) page for documentation and examples.
+See the [Usage: CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage) page for documentation and examples.
 
 It also includes a built-in scheduled import feature with `archivebox schedule`, handled by the same orchestrator that powers `archivebox server`, so you can pull in URLs from RSS feeds and websites regularly without a separate cron container.
 
@@ -812,7 +811,7 @@ Installing directly on <strong>Windows without Docker or WSL/WSL2/Cygwin is not
 
 <h4>Learn More</h4>
 <ul>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#dependencies">Wiki: Install (Dependencies)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#3-install-runtime-dependencies-using-archivebox-install">Wiki: Install (Dependencies)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install">Wiki: Chromium Install</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives">Wiki: Upgrading or Merging Archives</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#installing">Wiki: Troubleshooting (Installing)</a></li>
@@ -866,7 +865,7 @@ Each snapshot subfolder includes static metadata and plain extractor output file
 <h4>Learn More</h4>
 <ul>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage">Wiki: Setting Up Storage (SMB, NFS, S3, B2, Google Drive, etc.)</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">Wiki: Usage (Disk Layout)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout">Wiki: Usage (Disk Layout)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#large-archives">Wiki: Usage (Large Archives)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#output-folder">Wiki: Security Overview (Output Folder)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive">Wiki: Publishing Your Archive</a></li>
@@ -952,7 +951,7 @@ archivebox manage createsuperuser
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive">Wiki: Publishing Your Archive</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview">Wiki: Security Overview</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile">Wiki: Chromium Install (Setting Up a User Profile)</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#import-an-existing-browser-profile">Wiki: Chrome profiles and cookies</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/docs/Chromium-Install.md#import-an-existing-browser-profile">Wiki: Chrome profiles and cookies</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#default_persona">Wiki: Configuration (<code>DEFAULT_PERSONA</code>)</a></li>
 </ul>
 
@@ -1005,7 +1004,7 @@ For various reasons, many large sites (Reddit, Twitter, Cloudflare, etc.) active
 
 
 <ul>
-<li>Set up a logged-in browser session for archiving by <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#import-an-existing-browser-profile">importing a chrome profile into a persona</a></li>
+<li>Set up a logged-in browser session for archiving by <a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/docs/Chromium-Install.md#import-an-existing-browser-profile">importing a chrome profile into a persona</a></li>
 <li>Rewrite your URLs before archiving to swap in alternative frontends that are more bot-friendly e.g.<br>
 <code>reddit.com/some/url</code> -&gt; <code>teddit.net/some/url</code>: <a href="https://github.com/mendel5/alternative-front-ends">https://github.com/mendel5/alternative-front-ends</a></li>
 </ul>
@@ -1066,7 +1065,7 @@ Because ArchiveBox is designed to ingest a large volume of URLs with multiple co
 <h4>Learn More</h4>
 
 <ul>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">Wiki: Usage (Disk Layout)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout">Wiki: Usage (Disk Layout)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#output-folder">Wiki: Security Overview (Output-Folder)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#large-archives">Wiki: Usage (Large Archives)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#run-archivebox-as-an-unprivileged-user">Wiki: Security Overview (Privilege Dropping)</a></li>
@@ -1191,7 +1190,7 @@ By having users store their content locally or within their organizations, we ca
 
 <h3>Comparison With Other Self-Hosted Archiving Options</h3>
 
-ArchiveBox differentiates itself from [similar self-hosted projects](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#Web-Archiving-Projects) by providing both a comprehensive CLI interface for managing your archive, a Web UI that can be used either independently or together with the CLI, and a simple on-disk data format that can be used without either.
+ArchiveBox differentiates itself from [similar self-hosted projects](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#web-archiving-projects) by providing both a comprehensive CLI interface for managing your archive, a Web UI that can be used either independently or together with the CLI, and a simple on-disk data format that can be used without either.
 
 
 *If you want better fidelity for very complex interactive pages with heavy JS/streams/API requests, check out [ArchiveWeb.page](https://archiveweb.page) and [ReplayWeb.page](https://replayweb.page).*
@@ -1200,7 +1199,7 @@ ArchiveBox differentiates itself from [similar self-hosted projects](https://git
 
 *If you need more advanced recursive spider/crawling ability beyond `--depth=1`, check out [Browsertrix](https://github.com/webrecorder/browsertrix-crawler), [Photon](https://github.com/s0md3v/Photon), or [Scrapy](https://scrapy.org/) and pipe the outputted URLs into ArchiveBox.*
 
-For more alternatives, see our [list here](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#Web-Archiving-Projects)...
+For more alternatives, see our [list here](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#web-archiving-projects)...
 
 ArchiveBox is neither the highest fidelity nor the simplest tool available for self-hosted archiving, rather it's a jack-of-all-trades that tries to do most things well by default. We encourage you to try these other tools made by our friends if ArchiveBox isn't suited to your needs.
 
@@ -1255,7 +1254,7 @@ We use the [ArchiveBox GitHub Wiki](https://github.com/ArchiveBox/ArchiveBox/wik
 - [Usage](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage)
 - [Configuration](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration)
 - [Supported Sources](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive)
-- [Supported Outputs](https://github.com/ArchiveBox/ArchiveBox/wiki#can-save-these-things-for-each-site)
+- [Supported Outputs](https://github.com/ArchiveBox/ArchiveBox#output-formats)
 - [Scheduled Archiving](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving)
 
 ## Advanced

archivebox/api/apps.py

@@ -7,6 +7,11 @@ class APIConfig(AppConfig):
     name = "archivebox.api"
     label = "api"
 
+    def ready(self):
+        from archivebox.api.webhooks import register_delete_webhook_field_loader
+
+        register_delete_webhook_field_loader()
+
 
 def register_admin(admin_site):
     from archivebox.api.admin import register_admin

archivebox/api/v1_api.py

@@ -2,6 +2,8 @@ __package__ = "archivebox.api"
 
 
 from io import StringIO
+from functools import lru_cache
+from typing import Any
 from traceback import format_exception
 from contextlib import redirect_stdout, redirect_stderr
 
@@ -10,6 +12,7 @@ from django.core.exceptions import ObjectDoesNotExist, EmptyResultSet, Permissio
 from django.contrib.auth.models import User
 
 from ninja import NinjaAPI, Swagger
+from ninja.openapi.schema import OpenAPISchema
 
 # TODO: explore adding https://eadwincode.github.io/django-ninja-extra/
 
@@ -49,6 +52,16 @@ def register_urls(api: NinjaAPI) -> NinjaAPI:
 
 
 class NinjaAPIWithIOCapture(NinjaAPI):
+    def get_openapi_schema(self, *, path_prefix: str | None = None, path_params: dict[str, Any] | None = None) -> OpenAPISchema:
+        prefix = path_prefix if path_prefix is not None else self.get_root_path(path_params or {})
+        return self._schema_for_prefix(prefix)
+
+    @lru_cache(maxsize=8)
+    def _schema_for_prefix(self, path_prefix: str) -> OpenAPISchema:
+        # Routes are registered at startup. Server discovery can reuse their
+        # schema while preserving distinct URL prefixes for mounted instances.
+        return super().get_openapi_schema(path_prefix=path_prefix)
+
     def create_temporal_response(self, request: HttpRequest) -> HttpResponse:
         stdout, stderr = StringIO(), StringIO()
 

archivebox/api/v1_core.py

@@ -223,20 +223,20 @@ def get_archiveresults(request: HttpRequest, filters: Query[ArchiveResultFilterS
 
 def _uuid_ref_query(field_name: str, ref: str) -> Q:
     raw_ref = str(ref or "").strip()
-    query = Q(**{f"{field_name}__startswith": raw_ref})
-    if raw_ref:
-        query |= Q(**{f"{field_name}__icontains": raw_ref})
     try:
         parsed_uuid = UUID(raw_ref)
     except (TypeError, ValueError):
+        query = Q(**{f"{field_name}__startswith": raw_ref})
+        if raw_ref:
+            query |= Q(**{f"{field_name}__icontains": raw_ref})
         normalized_ref = raw_ref.replace("-", "")
         if normalized_ref and normalized_ref != raw_ref:
             query |= Q(**{f"{field_name}__startswith": normalized_ref})
             query |= Q(**{f"{field_name}__icontains": normalized_ref})
-    else:
-        query |= Q(**{field_name: parsed_uuid})
-        query |= Q(**{f"{field_name}__startswith": parsed_uuid.hex})
-    return query
+        return query
+    # An exact UUID cannot need a substring fallback. OR-ing it with LIKE
+    # prevents an indexed lookup and made Cabbage metadata reads take seconds.
+    return Q(**{field_name: parsed_uuid})
 
 
 @router.get("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema, url_name="get_archiveresult")
@@ -350,6 +350,12 @@ def _summarize_archiveresult_output_files(output_files: dict[str, dict[str, Any]
 
 def _get_snapshot_by_ref(snapshot_id: str):
     queryset = Snapshot.objects.select_related("crawl__created_by")
+    try:
+        full_id = UUID(snapshot_id)
+    except (TypeError, ValueError):
+        pass
+    else:
+        return queryset.get(pk=full_id)
     try:
         return queryset.get(_uuid_ref_query("id", snapshot_id) | Q(timestamp__startswith=snapshot_id))
     except Snapshot.DoesNotExist:
@@ -392,13 +398,13 @@ def _write_archiveresult_files(
     mime_types = _get_archiveresult_upload_form_values(request, "mime_types", "mime_type")
     chunk_output_path = _get_archiveresult_upload_form_value(request, "chunk_output_path")
 
+    output_files = dict(existing_output_files or {})
+    if not files:
+        return output_files
+
     snapshot_dir = snapshot.output_dir
     plugin_dir = snapshot_dir / plugin_name
     storage = FileSystemStorage(location=str(plugin_dir))
-    output_files = dict(existing_output_files or {})
-
-    if not files:
-        return output_files
 
     if chunk_output_path:
         if len(files) != 1:
@@ -630,6 +636,8 @@ def patch_archiveresult(
 class SnapshotSchema(Schema):
     TYPE: str = "core.models.Snapshot"
     id: UUID
+    crawl_id: UUID
+    persona: str | None
     created_by_id: str
     created_by_username: str
     created_at: datetime
@@ -648,6 +656,17 @@ class SnapshotSchema(Schema):
     num_archiveresults: int
     archiveresults: list[MinimalArchiveResultSchema]
 
+    @staticmethod
+    def resolve_archive_path(obj):
+        return obj.archive_path_from_db
+
+    @staticmethod
+    def resolve_persona(obj):
+        # A deduplicated submission can reuse a capture from a different
+        # persona. Report its actual DB-backed owner, not the new request's choice.
+        persona = obj.crawl.resolve_persona()
+        return persona.name if persona else None
+
     @staticmethod
     def resolve_created_by_id(obj):
         return str(obj.created_by.pk)
@@ -700,6 +719,7 @@ class SnapshotDeleteResponseSchema(Schema):
     snapshot_id: str
     crawl_id: str
     deleted_count: int
+    queued_count: int
 
 
 def normalize_tag_list(tags: list[str] | None = None) -> list[str]:
@@ -938,6 +958,15 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
     # lock. The unique insert recovery and CAS update below are the request-side
     # coordination boundary for this idempotent metadata sync.
     snapshot = Snapshot.objects.filter(url=data.url, crawl=crawl).first()
+    if snapshot is None and bool(crawl.get_current_config().get("ONLY_NEW", True)):
+        # Uploading browser metadata is part of the same submission as /add.
+        # It must honor the runner's deduplication policy, even if it arrives
+        # before the runner has processed that crawl. Report the actual owner
+        # so clients never mistake a reused capture for one they just created.
+        snapshot = Snapshot.objects.filter(url=data.url).exclude(status=Snapshot.DELETING_STATE).order_by("-created_at").first()
+        if snapshot is not None:
+            setattr(request, "with_archiveresults", False)
+            return snapshot
     if snapshot is None:
         try:
             snapshot = Snapshot.objects.create(
@@ -971,11 +1000,6 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
             created_by=request.user if isinstance(request.user, User) else None,
         )
 
-    try:
-        snapshot.ensure_crawl_symlink()
-    except Exception:
-        pass
-
     setattr(request, "with_archiveresults", False)
     return snapshot
 
@@ -1037,22 +1061,23 @@ def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateS
 
 @router.delete("/snapshot/{snapshot_id}", response=SnapshotDeleteResponseSchema, url_name="delete_snapshot")
 def delete_snapshot(request: HttpRequest, snapshot_id: str):
+    """Queue deletion; the normal server runner stops writers and removes files.
+
+    The snapshot remains readable with status=deleting until cleanup succeeds.
+    queued_count reports acceptance; deleted_count is zero in this response.
+    """
     snapshot = get_snapshot(request, snapshot_id, with_archiveresults=False)
     snapshot_id_str = str(snapshot.id)
-    crawl_id_str = str(snapshot.crawl.pk)
-    snapshot.cancel()
-
-    from archivebox.services.runner import run_pending_crawls
-
-    with crawl_lifecycle_lock(crawl_id_str):
-        run_pending_crawls(crawl_id=crawl_id_str, daemon=False)
-        snapshot = get_snapshot(request, snapshot_id_str, with_archiveresults=False)
-        deleted_count, _ = snapshot.delete()
+    crawl_id_str = str(snapshot.crawl_id)
+    # Use the same durable queue as the admin delete action. Stopping writers
+    # and removing archive files belongs to the runner, never the HTTP request.
+    queued_count = Snapshot.objects.filter(pk=snapshot.pk).request_delete()
     return {
         "success": True,
         "snapshot_id": snapshot_id_str,
         "crawl_id": crawl_id_str,
-        "deleted_count": deleted_count,
+        "deleted_count": 0,
+        "queued_count": queued_count,
     }
 
 

archivebox/api/v1_crawls.py

@@ -182,6 +182,14 @@ def crawl_file(request: HttpRequest, crawl_id: str, path: str):
     if not file_path.is_file() or crawl_root not in file_path.parents:
         raise HttpError(404, "Crawl file not found")
 
+    if path == "chrome_screencast/latest.jpg" and request.GET.get("after"):
+        try:
+            started_at = float(request.GET["after"])
+        except ValueError:
+            raise HttpError(400, "Invalid screencast start time")
+        if file_path.stat().st_mtime < started_at:
+            raise HttpError(404, "No frame for this capture yet")
+
     response = FileResponse(file_path.open("rb"))
     response["Cache-Control"] = "no-store, no-cache, max-age=0, must-revalidate"
     response["Pragma"] = "no-cache"

archivebox/api/v1_personas.py

@@ -5,9 +5,11 @@ from datetime import datetime
 from typing import Any, Literal
 from uuid import UUID
 
+from django.core.exceptions import ValidationError
 from django.db.models import Q
 from django.http import HttpRequest
 from ninja import Router, Schema
+from ninja.errors import HttpError
 from ninja.pagination import paginate
 from pydantic import Field
 
@@ -108,10 +110,9 @@ def browser_settings_to_config(extension_persona_id: str, settings: PersonaBrows
 
 
 def find_persona(extension_persona_id: str, name: str) -> Persona | None:
+    named = Persona.find_named(name)
     return (
-        Persona.objects.filter(
-            Q(config__BROWSER_EXTENSION_PERSONA_ID=extension_persona_id) | Q(name=name),
-        )
+        Persona.objects.filter(Q(config__BROWSER_EXTENSION_PERSONA_ID=extension_persona_id) | Q(pk=named.pk if named else None))
         .order_by("created_at")
         .first()
     )
@@ -138,7 +139,11 @@ def sync_persona(request: HttpRequest, payload: PersonaSyncSchema):
     if not is_valid:
         raise ValueError(error_message)
 
-    persona = find_persona(payload.extension_persona_id, name)
+    try:
+        persona = find_persona(payload.extension_persona_id, name)
+        (persona or Persona(name=name)).validate_name(persona.name if persona else name)
+    except ValidationError as err:
+        raise HttpError(409, "; ".join(err.messages)) from err
     created = persona is None
     if persona is None:
         persona = Persona(name=name)

archivebox/api/webhooks.py

@@ -5,12 +5,40 @@ from collections.abc import Callable
 import logging
 
 from django.db import transaction
+from django.db.models.signals import pre_delete
 from signal_webhooks.handlers import sync_task_handler
 
 
 logger = logging.getLogger(__name__)
 
 
+def load_deferred_fields_for_delete_webhook(sender, instance, using, **kwargs) -> None:
+    """Materialize deferred fields while DELETE webhook rows still exist.
+
+    Admin lists defer large fields, but signal_webhooks serializes all concrete
+    fields in post_delete. Loading one then would query an already-deleted row
+    and drop the event. Hydrate only deferred fields on webhook-enabled models
+    here, preserving the normal list-query optimization.
+    """
+    from signal_webhooks.handlers import find_hook_handler
+    from signal_webhooks.utils import reference_for_model
+
+    ref = reference_for_model(type(instance))
+    deferred_fields = instance.get_deferred_fields()
+    if not deferred_fields or find_hook_handler(ref, "DELETE") is None:
+        return
+
+    instance.refresh_from_db(using=using, fields=deferred_fields)
+
+
+def register_delete_webhook_field_loader() -> None:
+    pre_delete.connect(
+        load_deferred_fields_for_delete_webhook,
+        dispatch_uid="archivebox.load_deferred_fields_for_delete_webhook",
+        weak=False,
+    )
+
+
 def warning_error_handler(hook: Any, error: Exception | None) -> None:
     if error is not None:
         logger.warning("Outbound webhook %r failed: %s", hook.name, error)

archivebox/base_models/models.py

@@ -9,6 +9,7 @@ from typing import Any
 from archivebox.uuid_compat import CompactUUIDField, uuid7
 from pathlib import Path
 
+from django.core.exceptions import FieldDoesNotExist
 from django.db import models
 from django.db.models import F
 from django.db import transaction
@@ -23,6 +24,110 @@ from django_stubs_ext.db.models import TypedModelMeta
 from archivebox.config import CONSTANTS
 
 
+class PagedQuerySet(models.QuerySet):
+    def paged_iterator(self, chunk_size: int = 500):
+        """
+        Iterate rows using bounded keyset pages instead of one streaming cursor.
+
+        Django's iterator(chunk_size=...) still keeps a single SQLite SELECT
+        cursor open until the full queryset is exhausted. That is fine for
+        read-only exports, but update/migration code does filesystem work and
+        writes while iterating; a long-lived read cursor there can stretch lock
+        waits across thousands of rows. This respects the queryset's existing
+        filters, order_by(), select_related(), and prefetch_related() state; if
+        no ordering is defined, it falls back to primary-key order.
+        """
+        pk_field = self.model._meta.pk.name
+        raw_ordering = tuple(self.query.order_by or self.model._meta.ordering or (pk_field,))
+
+        if any(not isinstance(term, str) or term == "?" for term in raw_ordering):
+            offset = 0
+            while True:
+                batch = list(self[offset : offset + chunk_size])
+                if not batch:
+                    break
+                yield from batch
+                offset += chunk_size
+            return
+
+        ordering = []
+        for term in raw_ordering:
+            descending = term.startswith("-")
+            field_name = term[1:] if descending else term
+            if field_name == "pk":
+                field_name = pk_field
+            ordering.append(f"-{field_name}" if descending else field_name)
+
+        ordered_field_names = [term.removeprefix("-") for term in ordering]
+        try:
+            if any(self.model._meta.get_field(field_name).null for field_name in ordered_field_names):
+                offset = 0
+                while True:
+                    batch = list(self[offset : offset + chunk_size])
+                    if not batch:
+                        break
+                    yield from batch
+                    offset += chunk_size
+                return
+        except (AttributeError, FieldDoesNotExist):
+            offset = 0
+            while True:
+                batch = list(self[offset : offset + chunk_size])
+                if not batch:
+                    break
+                yield from batch
+                offset += chunk_size
+            return
+
+        unique_field_names = {pk_field, *(field.name for field in self.model._meta.fields if field.unique)}
+        if not any(field_name in unique_field_names for field_name in ordered_field_names):
+            offset = 0
+            while True:
+                batch = list(self[offset : offset + chunk_size])
+                if not batch:
+                    break
+                yield from batch
+                offset += chunk_size
+            return
+
+        last_values = None
+        value_field_names = tuple(dict.fromkeys([*ordered_field_names, pk_field]))
+        while True:
+            batch_qs = self.order_by(*ordering)
+            if last_values is not None:
+                page_filter = models.Q()
+                for idx, term in enumerate(ordering):
+                    descending = term.startswith("-")
+                    field_name = term[1:] if descending else term
+                    prefix = {ordered_field_names[i]: last_values[i] for i in range(idx)}
+                    comparison = "lt" if descending else "gt"
+                    page_filter |= models.Q(**prefix, **{f"{field_name}__{comparison}": last_values[idx]})
+                # Anchor the OR tie-breaks with an explicit leading range so
+                # SQLite/PG can seek the existing order index instead of
+                # rescanning its prefix for every keyset page.
+                first_comparison = "lte" if ordering[0].startswith("-") else "gte"
+                batch_qs = batch_qs.filter(
+                    page_filter,
+                    **{f"{ordered_field_names[0]}__{first_comparison}": last_values[0]},
+                )
+
+            batch_rows = list(batch_qs.values_list(*value_field_names)[:chunk_size])
+            if not batch_rows:
+                break
+
+            pk_idx = value_field_names.index(pk_field)
+            row_ids = [row[pk_idx] for row in batch_rows]
+            rows_by_id = {obj.pk: obj for obj in self.filter(pk__in=row_ids).order_by()}
+
+            for row in batch_rows:
+                row_id = row[pk_idx]
+                obj = rows_by_id.get(row_id)
+                if obj is not None:
+                    yield obj
+
+            last_values = batch_rows[-1][: len(ordered_field_names)]
+
+
 def normalize_config_json_values(config: Any) -> Any:
     if not isinstance(config, dict):
         return config
@@ -191,9 +296,21 @@ class ModelWithDeleteAfter(models.Model):
         return cls.objects.none()
 
     @classmethod
-    def delete_expired(cls, *, batch_size: int = 100, backfill_missing: bool = True) -> int:
+    def delete_expired(cls, *, batch_size: int = 100, backfill_missing: bool = True, backfill_cursors: dict | None = None) -> int:
         if backfill_missing:
-            missing_delete_at = list(cls.missing_delete_at_candidates().order_by("created_at", "pk")[:batch_size])
+            # Bound the scanned rows BEFORE evaluating JSON policy/parent
+            # joins. LIMIT on the policy query only bounded its output: finding
+            # no matches could scan the entire history with a SQLite read lock.
+            # The runner keeps a keyset cursor between idle passes, so disabled
+            # policies cannot starve older rows. Restarting safely repeats work.
+            page = cls.objects.order_by("-pk")
+            before = backfill_cursors.get(cls) if backfill_cursors is not None else None
+            if before is not None:
+                page = page.filter(pk__lt=before)
+            page_ids = list(page.values_list("pk", flat=True)[:batch_size])
+            if backfill_cursors is not None:
+                backfill_cursors[cls] = page_ids[-1] if len(page_ids) == batch_size else None
+            missing_delete_at = list(cls.missing_delete_at_candidates().filter(pk__in=page_ids).order_by())
             for obj in missing_delete_at:
                 if obj.set_delete_at_from_config():
                     cls.objects.filter(pk=obj.pk, delete_at__isnull=True).update(
@@ -231,12 +348,9 @@ class ModelWithOutputDir(ModelWithUUID):
 
     _delete_signal_registered = False
 
-    def save(self, *args, **kwargs):
-        super().save(*args, **kwargs)
-        output_dir = Path(self.output_dir)
-        # Avoid holding SQLite write transactions open across slow filesystem work.
-        transaction.on_commit(lambda: output_dir.mkdir(parents=True, exist_ok=True))
-        # Note: index.json is deprecated, models should use write_index_jsonl() for full data
+    # Saving metadata must not resolve or create output paths. The runner and
+    # explicit artifact writers create their own directories when writing files;
+    # on_commit would still make HTTP requests wait on remote archive storage.
 
     @property
     def output_dir_parent(self) -> str:
@@ -282,7 +396,14 @@ class ModelWithOutputDir(ModelWithUUID):
             if path.is_symlink() or path.is_file():
                 path.unlink(missing_ok=True)
             elif path.is_dir():
-                shutil.rmtree(path, ignore_errors=True)
+                shutil.rmtree(path)
+            # lstat also catches dangling symlinks and propagates mount and
+            # permission errors rather than mistaking them for absence.
+            try:
+                path.lstat()
+            except FileNotFoundError:
+                continue
+            raise OSError(f"Output path still exists after deletion: {path}")
 
     def schedule_delete_cleanup(self, *, using: str | None = None) -> None:
         """Capture output paths before DB deletion and remove them after commit."""
@@ -294,7 +415,9 @@ class ModelWithOutputDir(ModelWithUUID):
         if cls._delete_signal_registered:
             return
 
-        def schedule_output_dir_cleanup(sender, instance, using, **kwargs):
+        def schedule_output_dir_cleanup(sender, instance, using, origin=None, **kwargs):
+            if getattr(origin, "_output_files_deleted", False):
+                return
             if not isinstance(instance, ModelWithOutputDir):
                 return
             instance.schedule_delete_cleanup(using=using)

archivebox/cli/__init__.py

@@ -2,6 +2,7 @@ __package__ = "archivebox.cli"
 __command__ = "archivebox"
 import os
 import sys
+from contextlib import redirect_stdout
 from importlib import import_module
 
 import rich_click as click
@@ -174,8 +175,12 @@ def cli(ctx, help=False):
                 archivebox_init(quick=True)
                 os.environ.pop("ARCHIVEBOX_WANTS_INIT", None)
             check_data_folder()
-            if subcommand != "update":
-                check_migrations(auto_apply=True)
+            if subcommand != "update" and not check_migrations(auto_apply=True):
+                from archivebox.machine.models import Machine
+
+                # Host registration and config reconciliation belong to CLI
+                # startup; get_config() is also used by read-only HTTP views.
+                Machine.current()
         except Exception as e:
             STDERR.print(f"[red][X] Error setting up Django or checking data folder: {e}[/red]")
             if subcommand not in ("manage", "shell"):  # not all management commands need django to be setup beforehand
@@ -215,6 +220,12 @@ def main(args=None, prog_name=None):
         raise SystemExit(130) from None
     finally:
         sys.unraisablehook = previous_unraisablehook
+        supervisor_util = sys.modules.get("archivebox.workers.supervisord_util")
+        if supervisor_util is not None:
+            # Short commands can start a foreground-owned search daemon too.
+            # Finish its cleanup before another CLI can borrow a dying stack.
+            with redirect_stdout(sys.stderr):
+                supervisor_util.stop_own_supervisord_process()
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_add.py

@@ -144,7 +144,6 @@ def add(
     persona_name = (persona or "Default").strip() or "Default"
     plugins = plugins or ""
     persona_obj = Persona.get_or_create_named(persona_name)
-    persona_obj.ensure_dirs()
     effective_persona_config = get_config(persona=persona_obj)
 
     crawl_config = {
@@ -217,9 +216,6 @@ def add(
         print(
             "[yellow]\\[*] URLs queued. The background runner will process them (run `archivebox server` or `archivebox run --daemon` if not already running).[/yellow]",
         )
-        from archivebox.services.runner import ensure_background_runner
-
-        ensure_background_runner()
     else:
         # Foreground mode: run full crawl runner until all work is done
         print("[green]\\[*] Starting crawl runner to process crawl...[/green]")
@@ -427,7 +423,23 @@ def main(**kwargs):
             existing = [token.strip() for token in (kwargs.get("plugins") or "").split(",") if token.strip()]
             kwargs["plugins"] = ",".join(dict.fromkeys([*existing, *selected]))
 
-        add(urls=urls, **kwargs)
+        from archivebox.machine.models import Process
+
+        # CLI commands own their process record and logs. Queue model saves
+        # must not register a process implicitly: HTTP add() shares that path.
+        process = Process.current()
+        exit_code = 0
+        try:
+            add(urls=urls, **kwargs)
+        except BaseException:
+            exit_code = 1
+            raise
+        finally:
+            # --bg/--index-only return without the foreground runner's cleanup.
+            # Leaving them RUNNING makes the active-process index accumulate
+            # historical CLI commands forever, defeating bounded recovery.
+            if process.status == Process.StatusChoices.RUNNING:
+                process.mark_exited(exit_code=exit_code)
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_init.py

@@ -111,6 +111,16 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
         else:
             os.environ["ARCHIVEBOX_WANTS_INIT"] = previous_wants_init
 
+    from archivebox.machine.models import Machine
+
+    # Registration requires the final schema and reconciles legacy file config.
+    Machine.current()
+
+    from archivebox.personas.models import Persona
+
+    # Initialize here, not when /add/ is rendered: a GET must not create profiles.
+    Persona.get_or_create_default().ensure_dirs()
+
     from archivebox.misc.db import database_display_location, is_postgres
 
     assert database_exists()

archivebox/cli/archivebox_install.py

@@ -58,7 +58,6 @@ def _install_raw_binary_names(binary_names: list[str], binproviders: str) -> Non
     from django.utils import timezone
 
     from archivebox.machine.models import Binary, Machine, _canonical_binary_name
-    from archivebox.services.runner import run_due_binary
 
     machine = Machine.current()
     for requested_name in binary_names:
@@ -80,7 +79,13 @@ def _install_raw_binary_names(binary_names: list[str], binproviders: str) -> Non
             binary.status = Binary.StatusChoices.QUEUED
             binary.retry_at = timezone.now()
             binary.save(update_fields=["binproviders", "overrides", "status", "retry_at", "modified_at"])
-        run_due_binary(binary, lock_seconds=60)
+        if binary.status == Binary.StatusChoices.INSTALLED:
+            continue
+        # The synchronous CLI needs the lifecycle's verified installation result,
+        # not the scheduler's "claimed work" result. Reuse its failure/requeue and
+        # health-stat handling so a failed install cannot exit successfully.
+        if not binary.install_claimed(lock_seconds=60):
+            raise click.ClickException(f"Binary {binary_name} is already being installed")
 
 
 def ensure_data_dir_lib_symlink(data_dir: Path, abxpkg_lib_dir: Path) -> Path | None:

archivebox/cli/archivebox_persona.py

@@ -163,8 +163,9 @@ def create_personas(
             rprint(f'[red]Invalid persona name "{name}": {error_msg}[/red]', file=sys.stderr)
             continue
 
-        persona, created = Persona.objects.get_or_create(
-            name=name,
+        created = Persona.find_named(name) is None
+        persona = Persona.get_or_create_named(
+            name,
             defaults={"config": {"PERMISSIONS": permissions}} if permissions else {},
         )
 
@@ -313,7 +314,9 @@ def update_personas(name: str | None = None) -> int:
             if persona_id:
                 persona = Persona.objects.get(id=persona_id)
             else:
-                persona = Persona.objects.get(name=old_name)
+                persona = Persona.find_named(old_name)
+                if persona is None:
+                    raise Persona.DoesNotExist
 
             # Apply updates from CLI flags
             if name:
@@ -325,7 +328,7 @@ def update_personas(name: str | None = None) -> int:
 
                 # Rename the persona directory too
                 old_path = persona.path
-                persona.name = name
+                persona.name = persona.validate_name(name)
                 new_path = persona.path
 
                 if old_path.exists() and old_path != new_path:

archivebox/cli/archivebox_remove.py

@@ -87,9 +87,8 @@ def remove(
     # Delete one snapshot at a time. Each ``.delete()`` is its own short
     # Django-atomic block, so the writer lock is released between rows and
     # an in-flight daemon transaction can interleave instead of deadlocking.
-    # Filesystem cleanup for each row is scheduled via ``transaction.on_commit``
-    # in ``base_models/models.py`` and runs AFTER its row's tx commits — so
-    # rmtree doesn't hold the lock either.
+    # Snapshot.delete() removes and verifies filesystem outputs first, outside
+    # the short database delete transaction. Failed cleanup retains the row.
     #
     deleted_snapshot_pks = []
     timed_out = False

archivebox/cli/archivebox_server.py

@@ -260,6 +260,12 @@ def server(
     bind_spec = next((arg for arg in runserver_args if arg), "")
     host, port = _parse_and_validate_bind_spec(bind_spec)
 
+    from archivebox.personas.models import Persona
+
+    # Older collections deferred this until /add/. Reconcile before serving any
+    # requests, and report ambiguous profile names before Chrome can use them.
+    Persona.get_or_create_default().ensure_dirs()
+
     if not User.objects.filter(is_superuser=True).exclude(username="system").exists():
         from archivebox.core.routes_util import build_admin_url
 

archivebox/cli/archivebox_snapshot.py

@@ -298,7 +298,7 @@ def list_snapshots(
     Exit codes:
         0: Success (even if no results)
     """
-    from archivebox.misc.jsonl import write_record
+    from archivebox.misc.jsonl import write_records
 
     output_formats = sum(bool(output_format) for output_format in (csv, as_json, as_html))
     if output_formats > 1:
@@ -351,9 +351,7 @@ def list_snapshots(
         return 0
 
     if not is_tty:
-        for snapshot_json in iter_snapshot_json(queryset):
-            write_record(snapshot_json)
-            count += 1
+        count = write_records(iter_snapshot_json(queryset))
         rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
         return 0
 

archivebox/cli/rescan.py

@@ -102,9 +102,19 @@ def _reconcile_directory(directory: Path) -> str:
     results = {(item.get("plugin"), item.get("hook_name", "")): item for item in records if item.get("type") == "ArchiveResult"}.values()
     pending = []
     for result in results:
-        result_id = uuid.UUID(result["id"])
         if uuid.UUID(result["snapshot_id"]) != snapshot_id:
             raise ValueError("ArchiveResult belongs to another snapshot")
+        # Legacy JSON conversion produced records without IDs. Recover these
+        # by hook identity while preserving strict collision checks for IDs.
+        if not result.get("id"):
+            if not ArchiveResult.objects.filter(
+                snapshot_id=snapshot_id,
+                plugin=result["plugin"],
+                hook_name=result.get("hook_name", ""),
+            ).exists():
+                pending.append(result)
+            continue
+        result_id = uuid.UUID(result["id"])
         existing = ArchiveResult.objects.filter(pk=result_id).first()
         identity = (snapshot_id, result["plugin"], result.get("hook_name", ""))
         if existing:
@@ -154,7 +164,10 @@ def _reconcile_directory(directory: Path) -> str:
     # Each insert is durable. An interruption after the Snapshot insert is not
     # completion: the next pass still imports missing result IDs and tags.
     for result in pending:
-        snapshot._create_archive_result_if_missing(result, {}, preserve_identity=True)
+        snapshot._create_archive_result_if_missing(result, {}, preserve_identity=bool(result.get("id")))
+    if any(not result.get("id") for result in pending):
+        snapshot.hydrate_archiveresult_output_metadata(snapshot_dir=directory)
+        snapshot.write_index_jsonl(output_dir=directory)
     tags_before = set(snapshot.tags.values_list("name", flat=True))
     snapshot._merge_tags_from_index(record)
     tags_changed = tags_before != set(snapshot.tags.values_list("name", flat=True))
@@ -182,6 +195,10 @@ def reconcile_known_snapshots(snapshots, wait_for_turn=None):
             snapshot.migrate_filesystem_to_current_version()
         directory = Path(snapshot.output_dir)
         if not directory.is_dir():
+            # Canceling queued metadata can seal a row before the runner ever
+            # creates storage. There is nothing on disk to reconcile for it.
+            if snapshot.downloaded_at is None and not snapshot.archiveresult_set.exists():
+                continue
             raise ValueError(f"Missing snapshot directory: {directory}")
         _reconcile_directory(directory)
         for result in snapshot.archiveresult_set.all():

archivebox/config/common.py

@@ -34,6 +34,25 @@ ConfigPayload = dict[str, object]
 PluginSchemaDocuments = dict[str, dict[str, Any]]
 LIVE_CONFIG_BASE_URL = "/admin/environment/config/"
 
+
+def _is_cookie_file_config_key(key: str) -> bool:
+    return key in {"COOKIES_FILE", "AUTH_STORAGE_FILE"} or key.endswith("_COOKIES_FILE")
+
+
+def _resolve_cookie_file_paths(config: dict[str, Any]) -> dict[str, Any]:
+    """Keep collection-relative auth paths valid after hooks change cwd."""
+    for key, value in config.items():
+        if _is_cookie_file_config_key(key):
+            if value is None or (isinstance(value, str) and not value.strip()):
+                config[key] = ""
+                continue
+            file_path = Path(value).expanduser()
+            if not file_path.is_absolute():
+                file_path = CONSTANTS.DATA_DIR / file_path
+            config[key] = str(file_path.absolute())
+    return config
+
+
 ###################### Config ##########################
 
 _STDOUT_CONSOLE = Console()
@@ -395,6 +414,36 @@ class ServerConfig(BaseConfigSet):
         )
 
 
+@lru_cache(maxsize=16)
+def _default_sqlite_journal_mode(database_name: str) -> str:
+    if not IN_DOCKER:
+        return "WAL"
+
+    # Native Linux bind mounts share the kernel's locks and mmap; Docker alone
+    # does not make WAL unsafe. DELETE there lets any slow reader block tiny
+    # writes. Desktop VM shares (virtiofs/9p/FUSE), network filesystems and
+    # unknown mounts keep rollback journaling: their host-side readers may not
+    # share SQLite's -shm locking domain. Inspect the DB mount, not archive/.
+    database_path = Path(database_name).expanduser().resolve()
+    try:
+        mounts = Path("/proc/self/mountinfo").read_text().splitlines()
+    except OSError:
+        return "DELETE"
+    filesystem = ""
+    mount_depth = -1
+    for line in mounts:
+        fields, separator, details = line.partition(" - ")
+        if not separator:
+            continue
+        mount_path = fields.split()[4]
+        mount_path = re.sub(r"\\([0-7]{3})", lambda match: chr(int(match[1], 8)), mount_path)
+        mount = Path(mount_path)
+        if database_path.is_relative_to(mount) and len(mount.parts) > mount_depth:
+            filesystem = details.split()[0]
+            mount_depth = len(mount.parts)
+    return "WAL" if filesystem in {"ext2", "ext3", "ext4", "xfs", "btrfs", "zfs", "overlay", "tmpfs", "f2fs"} else "DELETE"
+
+
 class DatabaseConfig(BaseConfigSet):
     toml_section_header: str = "DATABASE_CONFIG"
     _scope: str = PrivateAttr(default=_SCOPE_SERVER)
@@ -410,11 +459,7 @@ class DatabaseConfig(BaseConfigSet):
     DATABASE_USER: str = Field(default="archivebox", alias="ARCHIVEBOX_DATABASE_USER")
     DATABASE_PASSWORD: str = Field(default="", alias="ARCHIVEBOX_DATABASE_PASSWORD")
     SQLITE_JOURNAL_MODE: str = Field(
-        # Docker collections commonly live on a host bind mount. WAL's -shm
-        # locking is only safe when every SQLite process is on the same host;
-        # Docker Desktop/OrbStack place the container and host in different
-        # locking domains and a host-side reader can corrupt the live DB.
-        default="DELETE" if IN_DOCKER else "WAL",
+        default_factory=lambda data: _default_sqlite_journal_mode(str(data["DATABASE_NAME"])),
         alias="ARCHIVEBOX_SQLITE_JOURNAL_MODE",
         pattern=r"(?i)^(DELETE|TRUNCATE|PERSIST|MEMORY|WAL|OFF)$",
     )
@@ -428,11 +473,16 @@ class DatabaseConfig(BaseConfigSet):
     SQLITE_LOCK_RETRY_INTERVAL: float = Field(default=5.0, alias="ARCHIVEBOX_SQLITE_LOCK_RETRY_INTERVAL", gt=0)
 
     @model_validator(mode="after")
-    def reject_docker_sqlite_wal(self):
-        if IN_DOCKER and self.DATABASE_ENGINE.lower() == "sqlite" and self.SQLITE_JOURNAL_MODE.upper() == "WAL":
+    def reject_shared_filesystem_sqlite_wal(self):
+        if (
+            IN_DOCKER
+            and self.DATABASE_ENGINE.lower() == "sqlite"
+            and self.SQLITE_JOURNAL_MODE.upper() == "WAL"
+            and _default_sqlite_journal_mode(self.DATABASE_NAME) != "WAL"
+        ):
             raise ValueError(
-                "SQLITE_JOURNAL_MODE=WAL is unsafe for Docker collections because host bind mounts cross SQLite "
-                "locking domains; use DELETE (the Docker default) or PostgreSQL",
+                "SQLITE_JOURNAL_MODE=WAL requires a local filesystem with shared SQLite locking; "
+                "keep the database on a local Docker volume, use DELETE for host/VM shared mounts, or use PostgreSQL",
             )
         return self
 
@@ -468,6 +518,11 @@ class ArchivingConfig(BaseConfigSet):
     COOKIES_FILE: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
     AUTH_STORAGE_FILE: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
 
+    @field_validator("COOKIES_FILE", "AUTH_STORAGE_FILE", mode="before")
+    @classmethod
+    def empty_cookie_file_is_unset(cls, value):
+        return None if isinstance(value, str) and not value.strip() else value
+
     URL_DENYLIST: str = Field(
         default=(
             r"\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$"
@@ -498,7 +553,7 @@ class ArchivingConfig(BaseConfigSet):
             rprint("    (Setting it to somewhere between 30 and 3000 seconds is recommended)", file=sys.stderr)
             rprint(file=sys.stderr)
             rprint("    If you want to make ArchiveBox run faster, disable specific archive methods instead:", file=sys.stderr)
-            rprint("        https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#archive-method-toggles", file=sys.stderr)
+            rprint("        https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#plugins", file=sys.stderr)
             rprint(file=sys.stderr)
 
     @field_validator("CHECK_SSL_VALIDITY", mode="after")
@@ -825,6 +880,8 @@ class ArchiveBoxBaseConfig(
         if runtime_overrides:
             config.update(normalize_runtime_config(runtime_overrides, json_safe=False))
 
+        _resolve_cookie_file_paths(config)
+
         # Hooks should only see concrete plugin-local flags, never the
         # ArchiveBox selectors used to derive them.
         config.pop("PLUGINS", None)
@@ -849,6 +906,13 @@ class ArchiveBoxBaseConfig(
             lib_dir = CONSTANTS.DATA_DIR / lib_dir
         self.ABXPKG_LIB_DIR = lib_dir.resolve()
 
+        cookie_paths = {key: getattr(self, key) for key in type(self).model_fields if _is_cookie_file_config_key(key)}
+        for key, value in _resolve_cookie_file_paths(cookie_paths).items():
+            if key in {"COOKIES_FILE", "AUTH_STORAGE_FILE"}:
+                setattr(self, key, Path(value) if value else None)
+            else:
+                setattr(self, key, value)
+
         return self
 
     @model_validator(mode="after")
@@ -1113,7 +1177,7 @@ def get_config(
             if apps.ready:
                 from archivebox.machine.models import Machine
 
-                machine = Machine.current()
+                machine = Machine.current_readonly()
         except (DatabaseError, ImportError, RuntimeError, TypeError, ValueError):
             machine = None
 
@@ -1156,11 +1220,15 @@ def get_config(
     scope_overrides: ConfigPayload = {}
 
     if include_machine and machine is not None and machine.config:
-        from archivebox.machine.models import _sanitize_machine_config
+        machine_config = machine.config
+        if resolve_plugins:
+            from archivebox.machine.models import _sanitize_machine_config
+
+            machine_config = _sanitize_machine_config(machine_config, lib_dir=config_data.get("ABXPKG_LIB_DIR"))
 
         scope_overrides.update(
             normalize_runtime_config(
-                _sanitize_machine_config(machine.config, lib_dir=config_data.get("ABXPKG_LIB_DIR")),
+                machine_config,
                 only_crawl_execution=crawl_config_base,
                 exclude_runtime_derived=True,
                 json_safe=False,

archivebox/config/django.py

@@ -63,7 +63,14 @@ def setup_django(check_db=False) -> None:
             os.chown(CONSTANTS.DATA_DIR, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, follow_symlinks=False)
             if CONSTANTS.DATA_DIR.exists():
                 for child in CONSTANTS.DATA_DIR.iterdir():
-                    os.chown(child, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, follow_symlinks=False)
+                    try:
+                        os.chown(child, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, follow_symlinks=False)
+                    except FileNotFoundError:
+                        # A concurrent SQLite transaction can remove its journal
+                        # between directory enumeration and chown. There is no
+                        # ownership left to fix; other permission errors must
+                        # still fail startup rather than being hidden.
+                        continue
 
     # Suppress the "database access during app initialization" warning
     # This warning can be triggered during django.setup() but is safe to ignore

archivebox/config/paths.py

@@ -85,23 +85,31 @@ def get_collection_id(DATA_DIR=DATA_DIR) -> str:
 
 
 @cache
-def get_machine_id() -> str:
-    """Get a short, stable, unique ID for the current machine (e.g. abc45678)"""
+def get_machine_id(length: int = 8) -> str:
+    """Get a machine ID, using the network identity when no OS ID is available."""
+    import machineid
 
-    MACHINE_ID = "unknown"
     try:
-        import machineid
+        return machineid.hashed_id("archivebox")[:length]
+    except machineid.MachineIdNotFound:
+        import uuid
 
-        MACHINE_ID = machineid.hashed_id("archivebox")[:8]
-    except Exception:
-        try:
-            import uuid
-            import hashlib
+        import psutil
 
-            MACHINE_ID = hashlib.sha256(str(uuid.getnode()).encode()).hexdigest()[:8]
-        except Exception:
-            pass
-    return MACHINE_ID
+        node = uuid.getnode()
+        identity = str(node)
+        if node & (1 << 40):
+            # uuid.getnode() silently generates a per-process random value when
+            # tools such as ip/ifconfig are absent. Read interfaces directly so
+            # separate CLI processes agree even in a minimal container.
+            addresses = sorted(
+                address.address
+                for interface in psutil.net_if_addrs().values()
+                for address in interface
+                if address.family == psutil.AF_LINK and address.address not in ("", "00:00:00:00:00:00")
+            )
+            identity = str(int(addresses[0].replace(":", "").replace("-", ""), 16)) if addresses else f"hostname:{socket.gethostname()}"
+        return hashlib.sha256(identity.encode()).hexdigest()[:length]
 
 
 @cache

archivebox/config/views.py

@@ -150,9 +150,9 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
 
     rows = {
         "Binary Name": [],
-        "Found Version": [],
+        "Recorded Version": [],
         "Provided By": [],
-        "Found Abspath": [],
+        "Recorded Abspath": [],
     }
 
     db_binaries = get_db_binaries_by_name()
@@ -160,18 +160,10 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
 
     for name in all_binary_names:
         binary = db_binaries.get(name)
-        binary_is_valid = bool(binary and binary.is_valid)
-
         rows["Binary Name"].append(ItemLink(name, key=name))
-
-        if binary_is_valid:
-            rows["Found Version"].append(f"✅ {binary.version}" if binary.version else "✅ found")
-            rows["Provided By"].append(binary.binprovider or "-")
-            rows["Found Abspath"].append(binary.abspath or "-")
-        else:
-            rows["Found Version"].append("❌ missing")
-            rows["Provided By"].append("-")
-            rows["Found Abspath"].append("-")
+        rows["Recorded Version"].append(binary.version or "-")
+        rows["Provided By"].append(binary.binprovider or "-")
+        rows["Recorded Abspath"].append(binary.abspath or "-")
 
     return TableContext(
         title="Binaries",

archivebox/core/admin_archiveresults.py

@@ -2,11 +2,9 @@ __package__ = "archivebox.core"
 
 import html
 import json
-import os
 import shlex
 from functools import reduce
 from operator import and_
-from pathlib import Path
 from urllib.parse import quote
 
 from django.contrib import admin
@@ -29,6 +27,7 @@ from archivebox.core.routes_util import (
     build_snapshot_files_url,
     build_snapshot_url,
     build_snapshot_zip_url,
+    get_snapshot_output_anchor,
 )
 from archivebox.core.widgets import InlineTagEditorWidget
 from archivebox.machine.env_util import env_to_shell_exports
@@ -670,16 +669,19 @@ class ArchiveResultAdmin(BaseModelAdmin):
     def get_output_view_url(self, result: ArchiveResult) -> str:
         request = self.request
         config = request.archivebox_config
-        output_path = result.embed_path()
-        if not output_path:
-            output_path = result.plugin or ""
-        return build_snapshot_url(str(result.snapshot_id), output_path, request=request, config=config)
+        output_path = get_snapshot_output_anchor(result.plugin, result.embed_path() or "")
+        return build_snapshot_detail_url(
+            result.snapshot.archive_path_from_db,
+            output_path=output_path,
+            request=request,
+            config=config,
+        )
 
     def get_output_files_url(self, result: ArchiveResult) -> str:
         request = self.request
         return build_snapshot_files_url(
             str(result.snapshot_id),
-            result.plugin,
+            result.output_folder_path(),
             request=request,
             config=request.archivebox_config,
         )
@@ -688,7 +690,7 @@ class ArchiveResultAdmin(BaseModelAdmin):
         request = self.request
         return build_snapshot_zip_url(
             str(result.snapshot_id),
-            result.plugin,
+            result.output_folder_path(),
             request=request,
             config=request.archivebox_config,
         )
@@ -885,50 +887,16 @@ class ArchiveResultAdmin(BaseModelAdmin):
         )
 
     def output_summary(self, result):
-        snapshot_dir = Path(result.snapshot.output_dir)
-        output_html = format_html(
-            '<pre style="display: inline-block">{}</pre><br/>',
-            result.output_str_for_display(),
-        )
-        request = self.request
-        output_html += format_html(
-            '<a href="{}">See result files ...</a><br/><pre><code>',
-            build_snapshot_detail_url(
-                result.snapshot.archive_path_from_db,
-                output_path="all",
-                request=request,
-                config=request.archivebox_config,
-            ),
-        )
-        embed_path = result.embed_path() or ""
-        path_from_embed = snapshot_dir / (embed_path or "")
-        output_html += format_html(
-            '<i style="padding: 1px">{}</i><b style="padding-right: 20px">/</b><i>{}</i><br/><hr/>',
-            str(snapshot_dir),
-            str(embed_path),
-        )
-        if os.access(path_from_embed, os.R_OK):
-            root_dir = str(path_from_embed)
-        else:
-            root_dir = str(snapshot_dir)
-
-        for root, dirs, files in os.walk(root_dir):
-            depth = root.replace(root_dir, "").count(os.sep) + 1
-            if depth > 2:
-                continue
-            indent = " " * 4 * (depth)
-            output_html += format_html('<b style="padding: 1px">{}{}/</b><br/>', indent, os.path.basename(root))
-            indentation_str = " " * 4 * (depth + 1)
-            for filename in sorted(files):
-                is_hidden = filename.startswith(".")
-                output_html += format_html(
-                    '<span style="opacity: {}.2">{}{}</span><br/>',
-                    int(not is_hidden),
-                    indentation_str,
-                    filename.strip(),
-                )
-
-        return output_html + mark_safe("</code></pre>")
+        output_html = format_html('<pre style="display: inline-block">{}</pre>', result.output_str_for_display())
+        file_map = result.output_file_map()
+        paths = sorted({path for key in file_map if (path := result.output_file_path(key, output_file_map=file_map))})
+        if paths:
+            output_html += format_html(
+                '<br/><a href="{}">See result files ...</a><pre><code>{}</code></pre>',
+                self.get_output_files_url(result),
+                "\n".join(paths),
+            )
+        return output_html
 
 
 def register_admin(admin_site):

archivebox/core/admin_snapshots.py

@@ -131,11 +131,11 @@ class SnapshotStatusListFilter(admin.SimpleListFilter):
     parameter_name = "snapshot_status"
 
     def lookups(self, request, model_admin):
-        return Snapshot.StatusChoices.choices
+        return Snapshot._meta.get_field("status").choices
 
     def queryset(self, request, queryset):
         value = self.value()
-        if value in Snapshot.StatusChoices.values:
+        if value in dict(Snapshot._meta.get_field("status").choices):
             return queryset.filter(status=value)
         return queryset
 
@@ -992,10 +992,14 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         return format_html(
             '<a class="snapshot-title-detail-hitbox" href="{}" aria-label="Open snapshot details"></a>'
             "{}"
+            "{}"
             '<div class="snapshot-title-url">'
             '<a class="snapshot-original-url" href="{}">{}<code style="user-select: all;">{}</code></a>'
             "</div>",
             detail_url,
+            format_html('<strong class="snapshot-delete-status">{}</strong>', "Deleting — verifying file removal")
+            if obj.status == Snapshot.DELETING_STATE
+            else "",
             title_html,
             url_raw or obj.url,
             favicon_html,
@@ -1535,25 +1539,14 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     @admin.action(
         description="🗑️ Delete",
+        permissions=["delete"],
     )
     def delete_snapshots(self, request, queryset):
-        """Delete snapshots in a single transaction to avoid SQLite concurrency issues."""
-        from django.db import transaction
-
-        total = queryset.count()
-
-        # Get list of IDs to delete first (outside transaction)
-        ids_to_delete = list(queryset.values_list("pk", flat=True))
-
-        # Delete everything in a single atomic transaction
-        with transaction.atomic():
-            deleted_count, _ = Snapshot.objects.filter(pk__in=ids_to_delete).delete()
-
+        """Keep the rows until the runner removes and verifies their files."""
+        total = queryset.request_delete()
         messages.success(
             request,
-            mark_safe(
-                f"Successfully deleted {total} Snapshots ({deleted_count} total objects including related records). Don't forget to scrub URLs from import logs (data/sources) and error logs (data/logs) if needed.",
-            ),
+            f"Queued {total} snapshots for deletion. They remain visible as Deleting until ArchiveBox verifies their files are removed.",
         )
 
     @admin.action(

archivebox/core/forms.py

@@ -304,13 +304,13 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             self.can_override_crawl_config = bool(user.is_authenticated and user.is_active and user.is_superuser)
         super().__init__(*args, **kwargs)
 
-        default_persona = Persona.get_or_create_default()
+        # Persona creation belongs to init/server startup, never a GET of /add/.
         persona_queryset = Persona.objects.order_by("name")
         if not self.can_override_crawl_config:
             persona_queryset = filter_personas_by_permissions(persona_queryset, {PERMISSIONS_PUBLIC})
         self.fields["persona"].queryset = persona_queryset
 
-        selected_persona = persona_queryset.filter(id=default_persona.id).first() or persona_queryset.first()
+        selected_persona = persona_queryset.filter(name__iexact="Default").first() or persona_queryset.first()
         default_config = get_config(persona=selected_persona) if selected_persona else get_config()
         if selected_persona:
             self.fields["persona"].initial = selected_persona.name
@@ -338,6 +338,12 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
                 get_choice_field(self, field_name).choices = [(plugin.name, plugin.name) for plugin in grouped_plugins.get(category, [])]
             self.plugin_groups = []
 
+    def clean_persona(self):
+        persona = self.cleaned_data.get("persona")
+        if persona:
+            persona.validate_name(persona.name)
+        return persona
+
     def clean(self):
         cleaned_data = super().clean() or {}
 

archivebox/core/migrations/0024_assign_default_crawl.py

@@ -100,11 +100,12 @@ def create_default_crawl_and_assign_snapshots(apps, schema_editor):
     if schema_editor.connection.vendor != "sqlite":
         return
 
-    from django.db import connection
+    from django.utils import timezone
     import uuid as uuid_lib
-    from datetime import datetime
 
+    connection = schema_editor.connection
     cursor = connection.cursor()
+    now = connection.ops.adapt_datetimefield_value(timezone.now())
 
     # Check if there are any snapshots without a crawl
     cursor.execute("SELECT COUNT(*) FROM core_snapshot WHERE crawl_id IS NULL")
@@ -125,14 +126,13 @@ def create_default_crawl_and_assign_snapshots(apps, schema_editor):
             INSERT INTO auth_user (id, password, is_superuser, username, first_name, last_name, email, is_staff, is_active, date_joined)
             VALUES (1, '!', 1, 'system', '', '', '', 1, 1, %s)
         """,
-            [datetime.now().isoformat()],
+            [now],
         )
 
     # Create a default crawl for migrated snapshots.
     # Depending on migration graph order, later crawls migrations may already
     # have removed output_dir by the time this data migration runs.
     crawl_id = uuid_lib.uuid4().hex
-    now = datetime.now().isoformat()
     cursor.execute("PRAGMA table_info(crawls_crawl)")
     crawl_columns = {row[1] for row in cursor.fetchall()}
     default_columns: list[str] = []
@@ -155,9 +155,9 @@ def create_default_crawl_and_assign_snapshots(apps, schema_editor):
             status, retry_at, created_by_id, schedule_id, config, persona_id
         ) VALUES (%s, %s, %s, 0, 0, %s, 0, '', 'Migrated from v0.7.2/v0.8.6',
                   'Auto-created crawl for migrated snapshots', {default_values_sql}
-                  'sealed', %s, 1, NULL, '{{}}', NULL)
+                  'sealed', NULL, 1, NULL, '{{}}', NULL)
     """,
-        [crawl_id, now, now, crawl_urls, now],
+        [crawl_id, now, now, crawl_urls],
     )
 
     # Assign all snapshots without a crawl to the default crawl

archivebox/core/migrations/0055_repair_legacy_retry_timestamps.py

@@ -0,0 +1,60 @@
+"""Repair SQLite queue timestamps written as raw ISO strings by old migrations."""
+
+from django.db import migrations
+from django.utils.dateparse import parse_datetime
+
+
+def repair_legacy_retry_timestamps(apps, schema_editor):
+    connection = schema_editor.connection
+    if connection.vendor != "sqlite":
+        return
+
+    with connection.cursor() as cursor:
+        # 0024 created this Crawl already sealed, but left a due retry marker.
+        # Its orphaned snapshots have their own scheduler rows for maintenance.
+        cursor.execute(
+            """
+            UPDATE crawls_crawl SET retry_at = NULL
+            WHERE status = 'sealed'
+              AND label = 'Migrated from v0.7.2/v0.8.6'
+              AND notes = 'Auto-created crawl for migrated snapshots'
+              AND substr(retry_at, 11, 1) = 'T'
+            """,
+        )
+
+        # SQLite compares stored date strings lexically. A raw `T` timestamp
+        # from a legacy migration sorts differently from Django's space-separated
+        # parameter, so a due row can be selected but never claimed. Preserve
+        # the datetime Django already reads from each row while rewriting its
+        # storage representation through the active database adapter.
+        for table in ("crawls_crawl", "core_snapshot", "machine_binary", "machine_process"):
+            last_id = ""
+            while True:
+                cursor.execute(
+                    f"SELECT id, CAST(retry_at AS TEXT) FROM {table} WHERE id > %s AND substr(retry_at, 11, 1) = 'T' ORDER BY id LIMIT 500",
+                    [last_id],
+                )
+                rows = cursor.fetchall()
+                if not rows:
+                    break
+                for row_id, raw_retry_at in rows:
+                    try:
+                        parsed = parse_datetime(raw_retry_at)
+                    except ValueError:
+                        continue
+                    if parsed is None:
+                        continue
+                    normalized = connection.ops.adapt_datetimefield_value(parsed)
+                    cursor.execute(
+                        f"UPDATE {table} SET retry_at = %s WHERE id = %s AND retry_at = %s",
+                        [normalized, row_id, raw_retry_at],
+                    )
+                last_id = rows[-1][0]
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0054_snapshot_fs_version_index"),
+    ]
+
+    operations = [migrations.RunPython(repair_legacy_retry_timestamps, migrations.RunPython.noop)]

archivebox/core/migrations/0057_repair_migrated_process_links.py

@@ -0,0 +1,25 @@
+"""Repair process links after both core's legacy copy and machine migrations."""
+
+from importlib import import_module
+
+from django.db import migrations
+
+
+repair_process_binary_iface_links = import_module(
+    "archivebox.machine.migrations.0020_repair_process_binary_iface_links",
+).repair_process_binary_iface_links
+
+
+class Migration(migrations.Migration):
+    atomic = False
+
+    dependencies = [
+        ("core", "0055_repair_legacy_retry_timestamps"),
+        ("machine", "0022_networkinterface_identity_without_mac"),
+    ]
+
+    operations = [
+        # The original repair can run before core.0027 creates legacy Process
+        # rows. Repair again after the copy, including already upgraded DBs.
+        migrations.RunPython(repair_process_binary_iface_links, migrations.RunPython.noop),
+    ]

archivebox/core/migrations/0058_snapshot_deleting_status.py

@@ -0,0 +1,28 @@
+# Generated by Django 6.1 on 2026-09-30 03:32
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0057_repair_migrated_process_links"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="snapshot",
+            name="status",
+            field=models.CharField(
+                choices=[
+                    ("queued", "Queued"),
+                    ("started", "Started"),
+                    ("paused", "Paused"),
+                    ("sealed", "Sealed"),
+                    ("deleting", "Deleting"),
+                ],
+                db_index=True,
+                default="queued",
+                max_length=15,
+            ),
+        ),
+    ]

archivebox/core/models.py

@@ -1,6 +1,7 @@
 __package__ = "archivebox.core"
 
 import json
+import logging
 import os
 import uuid
 from collections.abc import Iterable, Mapping, Sequence
@@ -11,7 +12,7 @@ from urllib.parse import urlparse
 
 from django.conf import settings
 from django.contrib import admin
-from django.core.exceptions import FieldDoesNotExist, ObjectDoesNotExist, ValidationError
+from django.core.exceptions import ObjectDoesNotExist, ValidationError
 from django.db import IntegrityError, models, transaction
 from django.db.models import F, Q, QuerySet, Sum, Value
 from django.db.models.fields.json import KT
@@ -23,6 +24,7 @@ from django.utils.safestring import mark_safe
 from django.utils.text import slugify
 
 from archivebox.base_models.models import (
+    PagedQuerySet,
     ModelWithConfig,
     ModelWithDeleteAfter,
     ModelWithHealthStats,
@@ -172,9 +174,28 @@ class SnapshotTag(models.Model):
         unique_together: ClassVar[list[tuple[str, str]]] = [("snapshot", "tag")]
 
 
-class SnapshotQuerySet(models.QuerySet):
+class SnapshotQuerySet(PagedQuerySet):
     """Custom QuerySet for Snapshot model with export methods that persist through .filter() etc."""
 
+    def request_delete(self) -> int:
+        # Admin and API deletion share this durable command. Keep the row until
+        # the runner stops its writers and verifies removal from remote storage;
+        # RETRY_AT_MAX prevents capture scheduling while cleanup is pending.
+        return self.update(status=self.model.DELETING_STATE, retry_at=RETRY_AT_MAX, modified_at=timezone.now())
+
+    def delete(self):
+        # Finish the SELECT before filesystem work, so no read cursor or SQLite
+        # transaction is held while deleting remote archive payloads.
+        snapshots = list(self.select_related("crawl__created_by"))
+        total = 0
+        counts = {}
+        for snapshot in snapshots:
+            deleted, details = snapshot.delete(using=self.db)
+            total += deleted
+            for label, count in details.items():
+                counts[label] = counts.get(label, 0) + count
+        return total, counts
+
     def bulk_create(self, objs, *args, **kwargs):
         objs = list(objs)
         missing_crawl_ids = set()
@@ -206,101 +227,6 @@ class SnapshotQuerySet(models.QuerySet):
                 truncate_overlong_charfields(obj)
         return super().bulk_create(objs, *args, **kwargs)
 
-    def paged_iterator(self, chunk_size: int = 500):
-        """
-        Iterate snapshots using bounded keyset pages instead of one streaming cursor.
-
-        Django's iterator(chunk_size=...) still keeps a single SQLite SELECT
-        cursor open until the full queryset is exhausted. That is fine for
-        read-only exports, but update/migration code does filesystem work and
-        writes while iterating; a long-lived read cursor there can stretch lock
-        waits across thousands of rows. This respects the queryset's existing
-        filters, order_by(), select_related(), and prefetch_related() state; if
-        no ordering is defined, it falls back to primary-key order.
-        """
-        pk_field = self.model._meta.pk.name
-        raw_ordering = tuple(self.query.order_by or self.model._meta.ordering or (pk_field,))
-
-        if any(not isinstance(term, str) or term == "?" for term in raw_ordering):
-            offset = 0
-            while True:
-                batch = list(self[offset : offset + chunk_size])
-                if not batch:
-                    break
-                yield from batch
-                offset += chunk_size
-            return
-
-        ordering = []
-        for term in raw_ordering:
-            descending = term.startswith("-")
-            field_name = term[1:] if descending else term
-            if field_name == "pk":
-                field_name = pk_field
-            ordering.append(f"-{field_name}" if descending else field_name)
-
-        ordered_field_names = [term.removeprefix("-") for term in ordering]
-        try:
-            if any(self.model._meta.get_field(field_name).null for field_name in ordered_field_names):
-                offset = 0
-                while True:
-                    batch = list(self[offset : offset + chunk_size])
-                    if not batch:
-                        break
-                    yield from batch
-                    offset += chunk_size
-                return
-        except (AttributeError, FieldDoesNotExist):
-            offset = 0
-            while True:
-                batch = list(self[offset : offset + chunk_size])
-                if not batch:
-                    break
-                yield from batch
-                offset += chunk_size
-            return
-
-        unique_field_names = {pk_field, *(field.name for field in self.model._meta.fields if field.unique)}
-        if not any(field_name in unique_field_names for field_name in ordered_field_names):
-            offset = 0
-            while True:
-                batch = list(self[offset : offset + chunk_size])
-                if not batch:
-                    break
-                yield from batch
-                offset += chunk_size
-            return
-
-        last_values = None
-        value_field_names = tuple(dict.fromkeys([*ordered_field_names, pk_field]))
-        while True:
-            batch_qs = self.order_by(*ordering)
-            if last_values is not None:
-                page_filter = models.Q()
-                for idx, term in enumerate(ordering):
-                    descending = term.startswith("-")
-                    field_name = term[1:] if descending else term
-                    prefix = {ordered_field_names[i]: last_values[i] for i in range(idx)}
-                    comparison = "lt" if descending else "gt"
-                    page_filter |= models.Q(**prefix, **{f"{field_name}__{comparison}": last_values[idx]})
-                batch_qs = batch_qs.filter(page_filter)
-
-            batch_rows = list(batch_qs.values_list(*value_field_names)[:chunk_size])
-            if not batch_rows:
-                break
-
-            pk_idx = value_field_names.index(pk_field)
-            snapshot_ids = [row[pk_idx] for row in batch_rows]
-            snapshots_by_id = {snapshot.pk: snapshot for snapshot in self.filter(pk__in=snapshot_ids).order_by()}
-
-            for row in batch_rows:
-                snapshot_id = row[pk_idx]
-                snapshot = snapshots_by_id.get(snapshot_id)
-                if snapshot is not None:
-                    yield snapshot
-
-            last_values = batch_rows[-1][: len(ordered_field_names)]
-
     # =========================================================================
     # Filtering Methods
     # =========================================================================
@@ -477,7 +403,9 @@ class SnapshotQuerySet(models.QuerySet):
         for snapshot in snapshot_list:
             # Unarchived snapshots still need an export destination before discovery.
             snapshot.output_dir.mkdir(parents=True, exist_ok=True)
-            outputs = snapshot.discover_outputs(include_filesystem_fallback=True)
+            # The list icon is part of the portable export too; an incomplete
+            # hash manifest must not hide a file that is still on disk.
+            outputs = snapshot.discover_outputs(include_filesystem_fallback=True, filesystem_index={})
             snapshot._public_list_icon_paths = [
                 str(output.get("path") or "") for output in outputs if output.get("name") in list_icon_plugins
             ]
@@ -539,6 +467,7 @@ class SnapshotManager(models.Manager.from_queryset(SnapshotQuerySet)):  # ty: ig
 
 class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHealthStats, ModelWithQueue):
     BROWSER_EXTENSION_UPLOAD_HOOK_NAME = "on_Snapshot__archivebox_browser_extension_upload"
+    DELETING_STATE = "deleting"
 
     id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
@@ -572,7 +501,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
     )
     retry_at = ModelWithQueue.RetryAtField(default=timezone.now)
     status = ModelWithQueue.StatusField(
-        choices=ModelWithQueue.StatusChoices,
+        choices=[*ModelWithQueue.StatusChoices.choices, (DELETING_STATE, "Deleting")],
         default=ModelWithQueue.StatusChoices.QUEUED,
     )
     config = models.JSONField(default=dict, null=False, blank=False, editable=True)
@@ -659,6 +588,49 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
     def __str__(self):
         return f"[{self.id}] {self.url[:64]}"
 
+    def output_paths_for_delete(self) -> tuple[Path, ...]:
+        current = self.get_storage_path_for_version(self.fs_version)
+        paths = [Path(self.output_dir), current, CONSTANTS.ARCHIVE_DIR / self.timestamp]
+        if self.fs_version in ("0.9.0", "0.9.1", "0.9.2", "0.9.3", "0.9.4", "1.0.0"):
+            paths.append(current.with_name(str(uuid.UUID(hex=self.id.hex))))
+        if self.crawl_id:
+            domain_dir = self.crawl.output_dir / CONSTANTS.SNAPSHOTS_DIR_NAME / self.extract_domain_from_url(self.url)
+            paths.extend((domain_dir / str(self.id), domain_dir / str(uuid.UUID(hex=self.id.hex))))
+        return tuple(dict.fromkeys(paths))
+
+    def delete(self, using=None, keep_parents=False):
+        from django.db import connections
+
+        using = using or self._state.db or "default"
+        if connections[using].in_atomic_block:
+            raise RuntimeError("Snapshot filesystem deletion must run outside a database transaction")
+        type(self).objects.using(using).filter(pk=self.pk).request_delete()
+        self.status = self.DELETING_STATE
+        from archivebox.machine.models import Process
+
+        process_ids = self.archiveresult_set.filter(process__status=Process.StatusChoices.RUNNING).values_list("process_id", flat=True)
+        for process in list(Process.objects.using(using).filter(pk__in=process_ids)):
+            process.kill_tree()
+            if process.is_running:
+                raise OSError(f"Snapshot output writer is still running: {process.pk}")
+        paths = self.output_paths_for_delete()
+        self.delete_output_paths(paths)
+        # Cascaded ArchiveResults share this directory. Their normal post-commit
+        # cleanup would repeat filesystem work and update an already-deleted row.
+        self._output_files_deleted = True
+        return super().delete(using=using, keep_parents=keep_parents)
+
+    @classmethod
+    def delete_requested(cls, *, batch_size=100):
+        # Finish the bounded read before touching files: even an open SELECT
+        # cursor can hold a SQLite read lock for the duration of slow cleanup.
+        pending = list(cls.objects.filter(status=cls.DELETING_STATE).select_related("crawl__created_by")[:batch_size])
+        for snapshot in pending:
+            try:
+                snapshot.delete()
+            except OSError:
+                logging.getLogger(__name__).exception("Snapshot %s file deletion failed; keeping its database row", snapshot.pk)
+
     @classmethod
     def crawl_count_subquery(cls, *, status: str | None = None, outer_ref: str = "pk") -> QuerySet:
         """Return a scalar subquery counting Snapshots for one outer Crawl."""
@@ -893,7 +865,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             return False
         if self.status == self.StatusChoices.QUEUED:
             return bool(self.url) and self.start_processing()
-        # abx-dl emits SnapshotCompletedEvent after the complete hook sequence;
+        # SnapshotCompletedEvent seals completed or explicitly aborted attempts;
+        # runner takeover preserves the active capture for its replacement.
         # ArchiveResult projection state never drives Snapshot completion.
         return False
 
@@ -908,8 +881,11 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
     @classmethod
     def missing_delete_at_candidates(cls):
-        return cls.objects.filter(delete_at__isnull=True).filter(
-            Q(config__has_key="DELETE_AFTER") | Q(crawl__config__has_key="DELETE_AFTER"),
+        return (
+            cls.objects.filter(delete_at__isnull=True)
+            .filter(Q(config__has_key="DELETE_AFTER") | Q(crawl__config__has_key="DELETE_AFTER"))
+            .select_related("crawl")
+            .only("id", "created_at", "delete_at", "config", "crawl__config")
         )
 
     @classmethod
@@ -1070,7 +1046,6 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         from django.db import transaction
 
         def finish_snapshot_save():
-            self.reconcile_filesystem_links()
             crawl = Crawl.objects.filter(pk=self.crawl_id).first()
             if crawl is None:
                 return
@@ -1090,8 +1065,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             # Crawl.urls remains the original submitted source. Snapshot rows
             # are the normalized work queue and discovery projection.
 
-        # get_or_create/update_or_create wrap save() in atomic(); defer filesystem
-        # work and crawl maintenance so SQLite commits before touching the disk.
+        # get_or_create/update_or_create wrap save() in atomic(); attach inherited
+        # tags after that write commits. Filesystem projections belong to the runner.
         transaction.on_commit(finish_snapshot_save)
 
     # =========================================================================
@@ -1195,6 +1170,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
         target_dir = self.get_storage_path_for_version(target)
         if target_dir.exists():
+            self.reconcile_with_index(output_dir=target_dir, update_existing_archive_results=False)
             self.hydrate_archiveresult_output_metadata(snapshot_dir=target_dir)
         if cleanup:
             old_dir, new_dir = cleanup
@@ -1227,7 +1203,9 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
     def hydrate_archiveresult_output_metadata(self, snapshot_dir: Path | None = None) -> int:
         """Populate missing ArchiveResult file metadata from existing outputs."""
         hydrated = 0
-        for result in self.archiveresult_set.filter(output_files={}).iterator():
+        # Files may live on slow remote storage: finish the bounded read before
+        # inspecting outputs or writing their metadata back to the database.
+        for result in self.archiveresult_set.filter(output_files={}).order_by("pk").paged_iterator():
             hydrated += int(result.update_output_metadata_from_filesystem(snapshot_dir=snapshot_dir))
         return hydrated
 
@@ -3583,8 +3561,6 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             entry = Path(path)
             if len(entry.parts) != 1 or entry.name in ("index.html", "index.json", "favicon.ico"):
                 continue
-            if filesystem_index is not None and entry.name == "index.jsonl":
-                continue
             if entry.suffix.lstrip(".").lower() not in embeddable_exts or entry.stem in seen:
                 continue
             outputs.append(
@@ -3707,7 +3683,9 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         self.__dict__["num_outputs_cached"] = sum(result.status == ArchiveResult.StatusChoices.SUCCEEDED for result in archive_results)
         self.__dict__["num_failures_cached"] = sum(result.status == ArchiveResult.StatusChoices.FAILED for result in archive_results)
 
-        filesystem_index = {} if discover_files else None
+        # Static exports must reflect files that still exist even when the
+        # saved hash manifest is incomplete or stale.
+        filesystem_index = {} if discover_files or static_export_dir is not None else None
         outputs = [
             output
             for output in self.discover_outputs(
@@ -3738,7 +3716,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             # A snapshot's index.jsonl is a manifest, not another captured
             # output card. Preserve it as the preview fallback only when the
             # manifest is the snapshot's sole available output.
-            if any(output.get("result") is not None for output in outputs):
+            if any(output.get("path") != "index.jsonl" for output in outputs):
                 outputs = [output for output in outputs if output.get("path") != "index.jsonl"]
         outputs_by_name: dict[str, dict[str, Any]] = {}
         result_ids_by_name: dict[str, list[str]] = {}
@@ -3753,6 +3731,11 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         output_sizes = plugin_output_sizes(archive_results)
         for name, output in outputs_by_name.items():
             output["result_ids"] = ",".join(result_ids_by_name.get(name, ()))
+            output["folder_path"] = (
+                output["result"].output_folder_path() if output.get("result") is not None else str(Path(str(output["path"])).parent)
+            )
+            if output["folder_path"] == ".":
+                output["folder_path"] = ""
             if name in output_sizes:
                 output["size"] = output_sizes[name]
 
@@ -3967,6 +3950,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
 
 class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
+    objects = PagedQuerySet.as_manager()
+
     class StatusChoices(models.TextChoices):
         QUEUED = "queued", "Queued"
         STARTED = "started", "Started"
@@ -4188,8 +4173,11 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
 
     @classmethod
     def missing_delete_at_candidates(cls):
-        return cls.objects.filter(delete_at__isnull=True).filter(
-            Q(snapshot__config__has_key="DELETE_AFTER") | Q(snapshot__crawl__config__has_key="DELETE_AFTER"),
+        return (
+            cls.objects.filter(delete_at__isnull=True)
+            .filter(Q(snapshot__config__has_key="DELETE_AFTER") | Q(snapshot__crawl__config__has_key="DELETE_AFTER"))
+            .select_related("snapshot__crawl")
+            .only("id", "created_at", "delete_at", "snapshot__config", "snapshot__crawl__config")
         )
 
     @property
@@ -4429,6 +4417,10 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
             return 0
 
     def output_file_map(self) -> dict[str, dict[str, Any]]:
+        # List views opt into this cache on their read-only, request-local rows.
+        # Runner-owned results must still reflect metadata changed during a hook.
+        if "_render_output_file_map" in self.__dict__:
+            return self._render_output_file_map
         return self._normalize_output_files(self.output_files)
 
     def output_file_paths(self) -> list[str]:
@@ -4583,11 +4575,20 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
         output_file_map: dict[str, dict[str, Any]] | None = None,
     ) -> str | None:
         ignored = {"stdout.log", "stderr.log", "hook.pid", "listener.pid"}
-        candidates = [
-            path
-            for path in output_file_paths
-            if Path(path).name not in ignored and Path(path).suffix.lower() not in (".pid", ".log", ".sh")
-        ]
+        # Responses/wget manifests can contain thousands of paths. Parse each
+        # once instead of rebuilding Path objects for every preferred filename
+        # and extension group, each time a list row selects a preview.
+        candidates = []
+        names = {}
+        extensions = {}
+        for path in output_file_paths:
+            parsed_path = Path(path)
+            name, extension = parsed_path.name, parsed_path.suffix.lower()
+            if name in ignored or extension in (".pid", ".log", ".sh"):
+                continue
+            candidates.append(path)
+            names.setdefault(name.lower(), path)
+            extensions[path] = extension
         if not candidates:
             return None
 
@@ -4610,9 +4611,8 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
             "article.json",
         ]
         for preferred_name in preferred_names:
-            for candidate in candidates:
-                if Path(candidate).name.lower() == preferred_name:
-                    return candidate
+            if preferred_name in names:
+                return names[preferred_name]
 
         from archivebox.plugins.discovery import get_plugin_output_extension_preference
 
@@ -4623,7 +4623,7 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
             (".mp4", ".webm", ".mp3", ".opus", ".ogg", ".wav"),
         )
         for ext_group in ext_groups:
-            group_candidates = [candidate for candidate in candidates if Path(candidate).suffix.lower() in ext_group]
+            group_candidates = [candidate for candidate in candidates if extensions[candidate] in ext_group]
             if group_candidates:
                 return max(
                     group_candidates,
@@ -4718,6 +4718,14 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
         """
         return self.embed_path_db()
 
+    def output_folder_path(self) -> str:
+        """Return the physical folder for this result's files, using saved metadata."""
+        output_path = self.embed_path_db()
+        if output_path and not output_path.startswith(f"{self.plugin}/"):
+            parent = Path(output_path).parent.as_posix()
+            return "" if parent == "." else parent
+        return self.plugin
+
     @property
     def output_dir_name(self) -> str:
         return self.plugin
@@ -4745,9 +4753,13 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
 
     @property
     def pwd(self) -> str:
-        """Working directory, derived from the snapshot/plugin path if the Process row is gone."""
+        """Working directory metadata without probing archive storage."""
         process = self.process_record
-        return process.pwd if process and process.pwd else str(self.output_dir)
+        return (
+            process.pwd
+            if process and process.pwd
+            else str(self.snapshot.get_storage_path_for_version(self.snapshot.fs_version) / self.plugin)
+        )
 
     @property
     def cmd(self) -> list:

archivebox/core/recovery_util.py

@@ -74,8 +74,9 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
         **snapshot_filter,
     ).update(retry_at=now, modified_at=now)
 
+    # Find rare uploads through hook_name's index before checking their parents;
+    # a correlated upload probe otherwise visits every sealed snapshot.
     extension_upload_results = ArchiveResult.objects.filter(
-        snapshot_id=OuterRef("pk"),
         hook_name=Snapshot.BROWSER_EXTENSION_UPLOAD_HOOK_NAME,
     )
     server_results = ArchiveResult.objects.filter(snapshot_id=OuterRef("pk")).exclude(
@@ -83,15 +84,15 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
     )
     extension_only_snapshots = (
         Snapshot.objects.filter(
+            id__in=extension_upload_results.values("snapshot_id"),
             status=Snapshot.StatusChoices.SEALED,
             crawl__status__in=[Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED, Crawl.StatusChoices.SEALED],
             **snapshot_filter,
         )
         .annotate(
-            has_extension_upload=Exists(extension_upload_results),
             has_server_result=Exists(server_results),
         )
-        .filter(has_extension_upload=True, has_server_result=False)
+        .filter(has_server_result=False)
     )
     # Older browser-extension uploads could win a race with runner startup:
     # their successful external rows made the fresh Snapshot look finished

archivebox/core/shutdown_util.py

@@ -1,3 +1,28 @@
+"""Shared foreground signal and child-process cleanup contracts.
+
+Shutdown intent and process exit are separate: takeover must preserve the
+replacement owner's work, while a user abort must not later resume the crawl.
+Callers own that policy; this module preserves the received signal and stops
+only the process objects they explicitly supply.
+
+Cooperative boundaries are essential, not just quieter exception handling.
+A real Linux CI regression interrupted Popen.poll() with SIGINT and stranded
+its wait lock. SIGKILL stopped the child (it became a zombie), but Popen.wait()
+still timed out. Raising a different exception or waiting longer cannot repair
+that interrupted critical section. Polling loops must record the signal first
+and honor it between operations; async owners may cancel their own task while
+their loop is running. Once exiting, the last-resort process cleanup must stay
+synchronous: the original loop may be closed, and creating a replacement loop
+cannot recover tasks or resources owned by it.
+
+Keep immediate signal notices and repeated-signal force exit independent of
+the event bus: it may already be draining or closed during shutdown. Resumable
+pause/retry/skip belongs to the existing crawl controller and must not set
+sticky command-shutdown state. See foreground_shutdown_signals below for both
+cooperative and legacy blocking callers; do not change all callers' policy at
+once without supplying their interruption boundaries.
+"""
+
 from __future__ import annotations
 
 import os
@@ -23,7 +48,7 @@ _active_shutdown_state: ShutdownSignalState | None = None
 
 
 def raise_if_shutdown_requested() -> None:
-    """Let long foreground loops honor a signal even if Python ignored it once."""
+    """Honor sticky shutdown intent at a caller-owned interruption boundary."""
 
     if _active_shutdown_state and _active_shutdown_state.signal_name:
         raise KeyboardInterrupt
@@ -48,14 +73,37 @@ def wait_popen_and_kill_children(
     timeout: float,
     kill_timeout: float = 2.0,
 ) -> None:
-    """Wait for a Popen parent and then hard-kill any surviving descendants."""
+    """Reap our own Popen child and stop its explicitly captured descendants.
+
+    A dead child still needs its parent to reap it. Keep wait failures visible;
+    bypassing Popen's lock would hide the interrupted-owner bug described above.
+    Descendants must still be cleaned up when that wait fails.
+    """
 
     try:
-        proc.wait(timeout=timeout)
-    except subprocess.TimeoutExpired:
-        proc.kill()
-        proc.wait(timeout=kill_timeout)
-    kill_remaining_processes(children, timeout=kill_timeout)
+        try:
+            proc.wait(timeout=timeout)
+        except subprocess.TimeoutExpired:
+            proc.kill()
+            proc.wait(timeout=kill_timeout)
+    except subprocess.TimeoutExpired as err:
+        try:
+            state = psutil.Process(proc.pid).as_dict(attrs=["pid", "ppid", "status", "num_threads", "cpu_times"])
+        except psutil.NoSuchProcess:
+            state = {"pid": proc.pid, "status": "exited"}
+        err.add_note(f"Process state after SIGKILL: {state}")
+        # A locked Popen wait can time out even after the OS process exits.
+        # Inspect CPython's lock only for diagnostics; never release it here.
+        waitpid_lock = getattr(proc, "_waitpid_lock", None)
+        err.add_note(
+            f"Popen state: returncode={proc.returncode}, "
+            f"waitpid_lock_locked={waitpid_lock.locked() if waitpid_lock is not None else None}, "
+            f"threads={[thread.name for thread in threading.enumerate()]}",
+        )
+        raise
+    finally:
+        # A parent that cannot be reaped must not leave its descendants running.
+        kill_remaining_processes(children, timeout=kill_timeout)
 
 
 def wait_psutil_and_kill_children(
@@ -104,11 +152,14 @@ def foreground_shutdown_signals(
     interrupt_handlers: dict[signal.Signals, Callable[[], None]] | None = None,
     raise_on_first_signal: bool = True,
 ) -> Iterator[ShutdownSignalState]:
-    """Install foreground signal handlers that print an immediate exit notice.
+    """Keep shutdown intent visible until the owning foreground command unwinds.
 
-    Some log-tail loops intentionally swallow KeyboardInterrupt so that callers
-    can centralize cleanup in finally blocks. The handler writes the signal name
-    immediately, then raises KeyboardInterrupt to break out of the blocking read.
+    Signal state was added because inner loops can swallow KeyboardInterrupt
+    while their caller still needs to stop its children. Cooperative callers use
+    raise_on_first_signal=False and check raise_if_shutdown_requested() between
+    operations, or cancel their owned async task through on_signal. Raising from
+    the handler can otherwise interrupt library locks or partially sent I/O.
+    Blocking callers without a cooperative boundary retain immediate exceptions.
     """
 
     global _active_shutdown_state

archivebox/core/snapshot_status.py

@@ -6,7 +6,7 @@ from django.db.models import QuerySet
 def snapshot_status_values() -> tuple[str, ...]:
     from archivebox.core.models import Snapshot
 
-    return tuple(Snapshot.StatusChoices.values)
+    return tuple(value for value, _label in Snapshot._meta.get_field("status").choices)
 
 
 def normalize_snapshot_status(status: str | None) -> str | None:

archivebox/core/takeover_util.py

@@ -62,7 +62,7 @@ def runtime_stack_owner(*, data_dir: str | Path, exclude_id=None):
         # not keep stealing leadership from the next foreground command.
         base_qs.filter(process_type=Process.TypeChoices.ORCHESTRATOR).exclude(parent__process_type=Process.TypeChoices.SUPERVISORD),
     ):
-        for proc in qs.order_by("-created_at", "-modified_at").iterator(chunk_size=50):
+        for proc in qs.order_by("-created_at", "-modified_at").paged_iterator(chunk_size=50):
             if proc.is_running:
                 return proc
             proc.mark_exited(exit_code=proc.exit_code if proc.exit_code is not None else 0)
@@ -95,7 +95,7 @@ def foreground_runner_owner(*, data_dir: str | Path, exclude_id=None):
     )
     if exclude_id is not None:
         qs = qs.exclude(id=exclude_id)
-    for proc in qs.order_by("-created_at", "-modified_at").iterator(chunk_size=50):
+    for proc in qs.order_by("-created_at", "-modified_at").paged_iterator(chunk_size=50):
         if proc.is_running:
             return proc
         proc.mark_exited(exit_code=proc.exit_code if proc.exit_code is not None else 0)
@@ -178,7 +178,7 @@ def live_runner_processes(*, data_dir: str | Path):
     qs = qs.filter(machine=machine)
     live = []
     foreign_namespace_ids = []
-    for process in qs.order_by("started_at", "created_at").iterator(chunk_size=20):
+    for process in qs.order_by("started_at", "created_at").paged_iterator(chunk_size=20):
         if not process.shares_pid_namespace:
             foreign_namespace_ids.append(process.id)
             continue

archivebox/core/templatetags/core_tags.py

@@ -775,7 +775,9 @@ def snapshot_index_row(context, link) -> str:
     title = unescape(getattr(link, "title", "") or "")
     title_text = title or url
     tags_str = link.tags_str() if callable(getattr(link, "tags_str", None)) else getattr(link, "tags_str", "")
-    tag_html = "".join(f'<span class="snapshot-tag">{escape(tag)}</span>' for tag in (tags_str or "").split(",") if tag)
+    tag_html = "".join(
+        f'<span class="snapshot-tag" title="{escape(tag)}">{escape(tag)}</span>' for tag in (tags_str or "").split(",") if tag
+    )
     if tag_html:
         tag_cell = f'<span class="snapshot-tags">{tag_html}</span>'
     else:

archivebox/core/views.py

@@ -17,7 +17,6 @@ from django.contrib.auth import HASH_SESSION_KEY, SESSION_KEY, get_user_model
 from django.contrib.auth.mixins import UserPassesTestMixin
 from django.contrib.sessions.models import Session
 from django.core import signing
-from django.core.paginator import InvalidPage
 from django.db.models import Case, IntegerField, Q, Value, When
 from django.http import Http404, HttpRequest, HttpResponse, HttpResponseForbidden, QueryDict
 from django.shortcuts import redirect, render
@@ -48,7 +47,6 @@ from archivebox.config.configset import BaseConfigSet
 from archivebox.core.forms import AddLinkForm
 from archivebox.core.models import ArchiveResult, Snapshot, SnapshotTag
 from archivebox.core.permissions import (
-    PERMISSIONS_PRIVATE,
     PERMISSIONS_PUBLIC,
     PERMISSIONS_UNLISTED,
     can_view_snapshot,
@@ -92,6 +90,7 @@ from archivebox.plugins.discovery import (
 from archivebox.plugins.forms import get_plugin_config_binary_urls
 from archivebox.plugins.views import get_config_definition_link
 from archivebox.progressmonitor.views import live_progress_view
+from archivebox.progressmonitor.collection import collection_summary
 from archivebox.search.config import (
     get_search_mode,
     get_search_mode_backend,
@@ -123,7 +122,7 @@ def _find_snapshot_by_ref(snapshot_ref: str) -> Snapshot | None:
     if len(lookup) == 12 and "-" not in lookup:
         # Resolve suffixes using the covering primary-key index before fetching
         # wide snapshot rows and joining their crawl/user data.
-        matches = list(Snapshot.objects.filter(id__endswith=lookup).order_by().values_list("id", flat=True))
+        matches = Snapshot.objects.filter(id__endswith=lookup).order_by().values_list("id", flat=True)
         return snapshots.filter(pk__in=matches).order_by("-created_at", "-downloaded_at").first()
 
     try:
@@ -507,7 +506,7 @@ class SnapshotView(View):
         def _resolve_snapshots_for_slug(slug: str):
             # full URLs go straight to the url-only path (fast, indexed)
             if "://" in slug:
-                return SnapshotView.find_snapshots_for_url(slug)
+                return SnapshotView.find_snapshots_for_url(slug, allow_fallback=False)
             # short uuid-shaped slugs (>=8 hex chars after stripping non-hex) try id matching first
             id_qs = SnapshotView.find_snapshots_for_id(slug)
             if id_qs.exists():
@@ -518,6 +517,12 @@ class SnapshotView(View):
         try:
             if "://" in path:
                 snapshot = snapshots.order_by("-bookmarked_at").first()
+                if snapshot is None:
+                    # Fetch the exact match once before trying legacy URL
+                    # fallbacks; an exists() probe followed by the same lookup
+                    # needlessly doubled reads on every normal redirect.
+                    snapshots = direct_snapshots_queryset(request, self.find_snapshots_for_url(path)).select_related("crawl__created_by")
+                    snapshot = snapshots.order_by("-bookmarked_at").first()
                 if snapshot is None:
                     raise Snapshot.DoesNotExist
             else:
@@ -1213,11 +1218,22 @@ class OriginalDomainReplayView(View):
 
 
 class PublicIndexView(ListView):
+    """Keep browsing responsive while the same small server is archiving.
+
+    Performance takes priority over perfect initial totals: on a 1 vCPU / 1 GB
+    host with hundreds of thousands of snapshots, even an indexed count can
+    consume the entire roughly 0.5-0.7 second page-load budget. Benchmark with
+    the runner active, since it shares the CPU, memory, and SQLite database.
+
+    The unfiltered list therefore renders rows first and accepts missing/stale
+    display totals. A later ordinary reload can show the background-cached
+    count; do not restore a synchronous COUNT or add polling just to fill it in.
+    """
+
     template_name = "public_index.html"
     model = Snapshot
     ordering: ClassVar[list[str]] = ["-bookmarked_at", "-created_at"]
     paginator_class = AcceleratedPaginator
-    public_page_scan_chunk_size = 50
 
     def get_paginate_by(self, queryset):
         runtime_config = self.__dict__.get("runtime_config")
@@ -1238,43 +1254,76 @@ class PublicIndexView(ListView):
             "status",
             "output_size",
             "permissions",
+            # Modern replay URLs need the filesystem version and owner's name.
+            # Deferring these caused four extra SELECTs per displayed snapshot.
+            "fs_version",
+            "crawl__created_by__username",
         )
 
-    def _ordered_public_page_from_order_index(self, *, page_number: int, page_size: int) -> list[Snapshot] | None:
-        target_count = page_number * page_size
-        public_snapshots: list[Snapshot] = []
-        scanned = 0
-        chunk_size = max(self.public_page_scan_chunk_size, page_size)
-        ordered_snapshots = Snapshot.objects.order_by(*self.ordering).only(*self._base_public_snapshot_fields())
-
-        while len(public_snapshots) < target_count:
-            chunk = list(ordered_snapshots[scanned : scanned + chunk_size])
-            if not chunk:
-                break
-            scanned += len(chunk)
-            public_snapshots.extend(snapshot for snapshot in chunk if snapshot.permissions == PERMISSIONS_PUBLIC)
-
+    def _ordered_public_page_from_order_index(self, *, page_number: int, page_size: int) -> list[Snapshot]:
+        # Walk the existing chronological index in bounded keyset pages,
+        # reading only identity/visibility until reaching the requested page.
+        # Repeated OFFSET batches used to revisit every earlier row AND join
+        # owners each time, making deep pages quadratic. Filtering visibility
+        # in SQL instead made SQLite sort the entire permissions index first.
+        # Each helper page closes its cursor before yielding; only the final
+        # displayed IDs load wide rows/owners. No new index or SQLite hint.
         start = (page_number - 1) * page_size
-        return public_snapshots[start:target_count]
+        visible_count = 0
+        page_ids = []
+        candidates = Snapshot.objects.only("id", "permissions").order_by(*self.ordering, "id")
+        for snapshot in candidates.paged_iterator(chunk_size=max(100, page_size + 1)):
+            if snapshot.permissions != PERMISSIONS_PUBLIC:
+                continue
+            if visible_count >= start:
+                page_ids.append(snapshot.pk)
+                if len(page_ids) > page_size:
+                    break
+            visible_count += 1
+        return list(
+            Snapshot.objects.filter(pk__in=page_ids, permissions=PERMISSIONS_PUBLIC)
+            .select_related("crawl__created_by")
+            .order_by(*self.ordering, "id")
+            .only(*self._base_public_snapshot_fields()),
+        )
 
     def paginate_queryset(self, queryset, page_size):
         if self.request.GET.get("q", default="").strip():
             return super().paginate_queryset(queryset, page_size)
 
-        public_count = self.get_exact_public_snapshot_count()
-        paginator = self.get_paginator(range(public_count), page_size)
         page_kwarg = self.kwargs.get(self.page_kwarg)
         page_query = self.request.GET.get(self.page_kwarg)
         page_number = page_kwarg or page_query or 1
 
         try:
-            page = paginator.page(page_number)
-        except InvalidPage as err:
+            page_number = int(page_number)
+            if page_number < 1:
+                raise ValueError("Page must be positive")
+        except (ValueError, TypeError) as err:
             raise Http404(f"Invalid page ({page_number}): {err}") from err
 
-        object_list = self._ordered_public_page_from_order_index(page_number=page.number, page_size=page_size)
-        page.object_list = object_list
-        return paginator, page, object_list, page.has_other_pages()
+        snapshots = self._ordered_public_page_from_order_index(page_number=page_number, page_size=page_size)
+        if not snapshots and page_number > 1:
+            raise Http404(f"Invalid page ({page_number}): No snapshots")
+        # Exhausting the rows gives an exact total for this read at no extra
+        # cost. Otherwise the lookahead is only a lower bound: an absent count
+        # is preferable to an on-request scan or a cached total smaller than
+        # the rows we just observed. Cached overestimates are acceptable here.
+        observed_count = (page_number - 1) * page_size + len(snapshots)
+        self.public_snapshot_count = observed_count
+        if len(snapshots) > page_size:
+            summary = collection_summary(self.request.user, public=True)
+            cached_count = summary["snapshots"] if summary is not None else None
+            self.public_snapshot_count = cached_count if cached_count is not None and cached_count >= observed_count else None
+
+        # Give Django an in-memory lower bound, never the collection QuerySet
+        # whose Paginator.count would issue COUNT(*). Navigation uses this read's
+        # lookahead even when a stale total would hide a newly added page.
+        # The template must use the separate display totals, not num_pages here.
+        paginator = self.get_paginator(range(observed_count), page_size)
+        page = paginator.page(page_number)
+        page.object_list = snapshots[:page_size]
+        return paginator, page, page.object_list, page.has_other_pages()
 
     def get_context_data(self, **kwargs):
         runtime_config = self.__dict__.get("runtime_config")
@@ -1303,10 +1352,16 @@ class PublicIndexView(ListView):
             and search_mode_backend
             and context["paginator"].count == 0,
         )
+        context["public_snapshot_count"] = context["paginator"].count if query else self.public_snapshot_count
+        count = context["public_snapshot_count"]
+        context["public_page_count"] = (
+            max(1, (count + context["paginator"].per_page - 1) // context["paginator"].per_page) if count is not None else None
+        )
         snapshots = list(context.get("object_list") or ())
         from archivebox.plugins.output_groups import plugin_output_sizes
 
         all_results_by_snapshot = {str(snapshot.id): [] for snapshot in snapshots}
+        snapshots_by_id = {str(snapshot.id): snapshot for snapshot in snapshots}
         icons_by_snapshot: dict[str, dict[str, ArchiveResult]] = {str(snapshot.id): {} for snapshot in snapshots}
         tag_names_by_snapshot: dict[str, list[str]] = {str(snapshot.id): [] for snapshot in snapshots}
         list_icon_paths_by_snapshot: dict[str, list[str]] = {str(snapshot.id): [] for snapshot in snapshots}
@@ -1343,8 +1398,17 @@ class PublicIndexView(ListView):
                     "output_size",
                 )
             )
-            for result in results.iterator(chunk_size=1000):
+            # This is already bounded by the displayed snapshot page. Finish
+            # the SELECT before parsing output manifests: a streaming cursor
+            # otherwise keeps SQLite's read lock throughout rendering.
+            for result in list(results):
                 snapshot_key = str(result.snapshot_id)
+                # Thumbnail URLs traverse result.snapshot too; share the page's
+                # already-loaded parent rather than fetching it once per result.
+                result.snapshot = snapshots_by_id[snapshot_key]
+                # These read-only page objects are shared by icons, sizes, and
+                # thumbnails. Normalize each manifest once for this render.
+                result._render_output_file_map = result.output_file_map()
                 all_results_by_snapshot[snapshot_key].append(result)
                 progress = progress_by_snapshot[snapshot_key]
                 progress["total"] += 1
@@ -1376,13 +1440,12 @@ class PublicIndexView(ListView):
         context["object_list"] = snapshots
         return context
 
-    def get_exact_public_snapshot_count(self) -> int:
-        hidden_count = Snapshot.objects.filter(permissions=PERMISSIONS_PRIVATE).count()
-        hidden_count += Snapshot.objects.filter(permissions=PERMISSIONS_UNLISTED).count()
-        return Snapshot.objects.count() - hidden_count
-
     def get_queryset(self, **kwargs):
-        qs = public_snapshots_queryset(super().get_queryset(**kwargs)).only(*self._base_public_snapshot_fields())
+        qs = (
+            public_snapshots_queryset(super().get_queryset(**kwargs))
+            .select_related("crawl__created_by")
+            .only(*self._base_public_snapshot_fields())
+        )
         query = self.request.GET.get("q", default="").strip()
 
         if not query:

archivebox/core/widgets.py

@@ -717,7 +717,7 @@ class InlineTagEditorWidget(TagEditorWidget):
                 )
             pills_html += f'''
                 <span class="tag-pill" data-tag="{self._escape(td["name"])}" data-tag-id="{td["id"]}" style="{self._tag_style(td["name"])}">
-                    <a href="/admin/core/snapshot/?tags__id__exact={td["id"]}" class="tag-link">{self._escape(td["name"])}</a>
+                    <a href="/admin/core/snapshot/?tags__id__exact={td["id"]}" class="tag-link" title="{self._escape(td["name"])}">{self._escape(td["name"])}</a>
                     {remove_button}
                 </span>
             '''

archivebox/crawls/admin.py

@@ -947,7 +947,7 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         updated = 0
         batch = []
         crawls_to_update = []
-        for crawl in queryset.only("id", "config", "permissions").iterator(chunk_size=500):
+        for crawl in queryset.only("id", "config", "permissions").order_by("pk").paged_iterator(chunk_size=500):
             old_permissions = crawl.permissions
             config = dict(crawl.config or {})
             config["PERMISSIONS"] = permissions

archivebox/crawls/models.py

@@ -23,6 +23,7 @@ from archivebox.config.common import rprint as print
 from archivebox.core.permissions import PERMISSIONS_VALUES, normalize_permissions
 
 from archivebox.base_models.models import (
+    PagedQuerySet,
     ModelWithUUID,
     ModelWithDeleteAfter,
     ModelWithOutputDir,
@@ -148,6 +149,8 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes):
 
 
 class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWithQueue):
+    objects = PagedQuerySet.as_manager()
+
     id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
@@ -229,7 +232,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         if paused and save and self.pk:
             from archivebox.core.models import Snapshot
 
-            for snapshot in self.snapshot_set.exclude(status__in=Snapshot.FINAL_STATES).iterator():
+            for snapshot in self.snapshot_set.exclude(status__in=Snapshot.FINAL_STATES).order_by("pk").paged_iterator():
                 snapshot.pause()
         return paused
 
@@ -301,7 +304,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
 
     @classmethod
     def missing_delete_at_candidates(cls):
-        return cls.objects.filter(delete_at__isnull=True, config__has_key="DELETE_AFTER")
+        return cls.objects.filter(delete_at__isnull=True, config__has_key="DELETE_AFTER").only("id", "created_at", "delete_at", "config")
 
     def save(self, *args, **kwargs):
         update_fields = kwargs.get("update_fields")
@@ -368,7 +371,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         batch = []
         updated = 0
         queryset = self.snapshot_set.filter(Q(permissions=old_permissions) | Q(permissions__isnull=True)).only("id", "config")
-        for snapshot in queryset.iterator(chunk_size=500):
+        for snapshot in queryset.order_by("pk").paged_iterator(chunk_size=500):
             config = dict(snapshot.config or {})
             config["PERMISSIONS"] = normalized_new_permissions
             snapshot.config = config
@@ -427,7 +430,11 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
                 tags_by_name = {tag.name: tag for tag in Tag.objects.filter(name__in=added_names)}
 
             tag_ids = [tag.pk for tag_name in added_names if (tag := tags_by_name.get(tag_name))]
-            snapshot_ids = Snapshot.objects.filter(crawl=self).values_list("id", flat=True).iterator(chunk_size=5000)
+            # Fetch each page before writing M2M rows. iterator(chunk_size=...)
+            # would keep one read cursor open across every bulk write.
+            snapshot_ids = (
+                snapshot.pk for snapshot in Snapshot.objects.filter(crawl=self).only("id").order_by("pk").paged_iterator(chunk_size=500)
+            )
             while True:
                 batch_snapshot_ids = list(islice(snapshot_ids, 5000))
                 if not batch_snapshot_ids:

archivebox/machine/detect.py

@@ -57,7 +57,10 @@ def get_vm_info():
 
     hw_manufacturer = "Docker" if hw_in_docker else "Unknown"
     hw_product = "Container" if hw_in_docker else "Unknown"
-    hw_uuid = machineid.id()
+    try:
+        hw_uuid = machineid.id()
+    except machineid.MachineIdNotFound:
+        hw_uuid = "Unknown"
 
     if platform.system().lower() == "darwin":
         # Get macOS machine info
@@ -301,7 +304,9 @@ def get_host_stats() -> dict[str, Any]:
 
 
 def get_host_guid() -> str:
-    return machineid.hashed_id("archivebox")
+    from archivebox.config.paths import get_machine_id
+
+    return get_machine_id(length=64)
 
 
 # Example usage

archivebox/machine/models.py

@@ -20,7 +20,7 @@ from django.utils.functional import cached_property
 
 from archivebox.config import CONSTANTS
 from archivebox.config.common import rprint
-from archivebox.base_models.models import ModelWithDeleteAfter, ModelWithHealthStats, normalize_config_json_values
+from archivebox.base_models.models import PagedQuerySet, ModelWithDeleteAfter, ModelWithHealthStats, normalize_config_json_values
 from archivebox.workers.models import ModelWithQueue
 from .detect import get_host_guid, get_os_info, get_vm_info, get_host_network, get_host_stats
 
@@ -48,7 +48,6 @@ _CURRENT_PROCESS: Process | None = None
 MACHINE_RECHECK_INTERVAL = 7 * 24 * 60 * 60
 NETWORK_INTERFACE_RECHECK_INTERVAL = 1 * 60 * 60
 BINARY_RECHECK_INTERVAL = 1 * 30 * 60
-PROCESS_RECHECK_INTERVAL = 60  # Re-validate every 60 seconds
 PID_REUSE_WINDOW = timedelta(hours=24)  # Max age for considering a PID match valid
 PROCESS_TIMEOUT_GRACE = timedelta(seconds=30)  # Extra margin before force-cleaning timed-out RUNNING rows
 START_TIME_TOLERANCE = 5.0  # Seconds tolerance for start time matching
@@ -221,35 +220,21 @@ class Machine(ModelWithHealthStats):
         app_label = "machine"
 
     @classmethod
-    def current(cls, refresh: bool = False) -> Machine:
-        global _CURRENT_MACHINE
-        if refresh:
-            _CURRENT_MACHINE = None
-        if _CURRENT_MACHINE:
-            if timezone.now() < _CURRENT_MACHINE.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL):
-                # Reconciliation can save() and clear the shared cache; keep
-                # this call's instance so we never return None after that save.
-                machine = _CURRENT_MACHINE
-                # One-time-per-process reconciliation between ArchiveBox.conf
-                # and Machine.config. Fast-path: bool check + early-return when
-                # the sync has already run, so the cached-machine return path
-                # stays sub-microsecond.
-                try:
-                    from archivebox.config.collection import sync_machine_and_file
+    def current_readonly(cls) -> Machine | None:
+        """Read the current machine without registration, refresh, or config writes."""
+        if _CURRENT_MACHINE is not None:
+            return _CURRENT_MACHINE
+        return cls.objects.filter(guid=get_host_guid()).first()
 
-                    sync_machine_and_file(machine)
-                except Exception:
-                    pass
-                return machine
-            else:
-                _CURRENT_MACHINE = None
+    @classmethod
+    def current(cls) -> Machine:
+        global _CURRENT_MACHINE
+        if _CURRENT_MACHINE is not None:
+            return _CURRENT_MACHINE
 
         host_guid = get_host_guid()
-        # save() intentionally invalidates _CURRENT_MACHINE so config edits
-        # made through other instances become visible in this process. Host
-        # refresh (after seven days), sanitation, and reconciliation can all
-        # save here, so the working instance must not live only in that cache.
-        # Otherwise a routine refresh leaves startup dereferencing None.
+        # Host detection, sanitation, and config reconciliation run only on
+        # initial registration, never on cached reads.
         try:
             machine = cls.objects.get(guid=host_guid)
         except cls.DoesNotExist:
@@ -269,44 +254,41 @@ class Machine(ModelWithHealthStats):
                 **get_vm_info(),
                 stats=get_host_stats(),
             )
-        else:
-            if timezone.now() >= machine.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL):
-                for key, value in {
-                    "hostname": socket.gethostname(),
-                    **get_os_info(),
-                    **get_vm_info(),
-                    "stats": get_host_stats(),
-                }.items():
-                    setattr(machine, key, value)
-                machine.save(
-                    update_fields=[
-                        "hostname",
-                        "hw_in_docker",
-                        "hw_in_vm",
-                        "hw_manufacturer",
-                        "hw_product",
-                        "hw_uuid",
-                        "os_arch",
-                        "os_family",
-                        "os_platform",
-                        "os_release",
-                        "os_kernel",
-                        "stats",
-                        "modified_at",
-                    ],
-                )
-        machine = cls._sanitize_config(machine)
-        # Same one-time sync as the cached-return path. Triggers here on the
-        # very first ``Machine.current()`` call in a process before any
-        # cached return can occur.
+        needs_recheck = timezone.now() >= machine.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL)
+        # Reconcile before any save can mirror stale DB config over file edits.
         try:
             from archivebox.config.collection import sync_machine_and_file
 
             sync_machine_and_file(machine)
         except Exception:
             pass
-        # Publish only after all save-capable steps; publishing earlier would
-        # let those steps invalidate the cache we are in the middle of filling.
+        if needs_recheck:
+            for key, value in {
+                "hostname": socket.gethostname(),
+                **get_os_info(),
+                **get_vm_info(),
+                "stats": get_host_stats(),
+            }.items():
+                setattr(machine, key, value)
+            machine.save(
+                update_fields=[
+                    "hostname",
+                    "hw_in_docker",
+                    "hw_in_vm",
+                    "hw_manufacturer",
+                    "hw_product",
+                    "hw_uuid",
+                    "os_arch",
+                    "os_family",
+                    "os_platform",
+                    "os_release",
+                    "os_kernel",
+                    "stats",
+                    "modified_at",
+                ],
+            )
+        machine = cls._sanitize_config(machine)
+        # Publish only after initialization is complete.
         _CURRENT_MACHINE = machine
         return machine
 
@@ -377,18 +359,15 @@ class Machine(ModelWithHealthStats):
             if update_fields is not None:
                 kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "config"]))
 
-        # Drop the ``Machine.current()`` module-level cache on every save so
-        # config edits (admin form, from_json, etc.) become live in the same
-        # process without waiting out the 7-day ``MACHINE_RECHECK_INTERVAL``.
-        # The cache reads ``_CURRENT_MACHINE.modified_at`` and that value
-        # never moves forward on the cached object even when the row is
-        # updated in the DB, so without this we'd keep serving stale
-        # ``machine.config`` (incl. ``BASE_URL``) until the worker restarts.
+        # Apply explicit edits to the cached instance without making the next
+        # read repeat host detection, sanitation, or config reconciliation.
         update_fields = kwargs.get("update_fields")
         super().save(*args, **kwargs)
         global _CURRENT_MACHINE
         if _CURRENT_MACHINE is not None and _CURRENT_MACHINE.pk == self.pk:
-            _CURRENT_MACHINE = None
+            for field in self._meta.concrete_fields:
+                if field.attname in self.__dict__ and (update_fields is None or field.name in update_fields):
+                    setattr(_CURRENT_MACHINE, field.attname, self.__dict__[field.attname])
 
         # Mirror Machine.config into ArchiveBox.conf so the two stores stay
         # 1:1. Skipped when ``update_fields`` is set and doesn't touch
@@ -443,7 +422,7 @@ class NetworkInterface(ModelWithHealthStats):
     @classmethod
     def current(cls, refresh: bool = False) -> NetworkInterface:
         global _CURRENT_INTERFACE
-        machine = Machine.current(refresh=refresh)
+        machine = Machine.current()
         if _CURRENT_INTERFACE and _CURRENT_INTERFACE.machine_id == machine.id:
             if not refresh:
                 # Callers that pass refresh=False are asking for attribution to
@@ -910,7 +889,7 @@ class Binary(ModelWithHealthStats, ModelWithQueue):
 # =============================================================================
 
 
-class ProcessManager(models.Manager):
+class ProcessManager(models.Manager.from_queryset(PagedQuerySet)):
     """Manager for Process model."""
 
     def current(self) -> Process:
@@ -1229,8 +1208,11 @@ class Process(ModelWithDeleteAfter, models.Model):
 
     @classmethod
     def missing_delete_at_candidates(cls):
-        return cls.objects.filter(delete_at__isnull=True).filter(
-            Q(env__has_key="DELETE_AFTER") | Q(machine__config__has_key="DELETE_AFTER"),
+        return (
+            cls.objects.filter(delete_at__isnull=True)
+            .filter(Q(env__has_key="DELETE_AFTER") | Q(machine__config__has_key="DELETE_AFTER"))
+            .select_related("machine")
+            .only("id", "created_at", "delete_at", "env", "machine__config")
         )
 
     # Properties that delegate to related objects
@@ -1430,6 +1412,13 @@ class Process(ModelWithDeleteAfter, models.Model):
     # Process.current() and hierarchy methods
     # =========================================================================
 
+    @classmethod
+    def current_readonly(cls) -> Process | None:
+        """Return this process's registered identity without registration or probes."""
+        if _CURRENT_PROCESS is not None and _CURRENT_PROCESS.pid == os.getpid():
+            return _CURRENT_PROCESS
+        return None
+
     @classmethod
     def current(cls) -> Process:
         """
@@ -1447,40 +1436,19 @@ class Process(ModelWithDeleteAfter, models.Model):
 
         current_pid = os.getpid()
 
-        # Fast path used by model save diagnostics and hot runner loops. A
-        # cached Process object is valid when the immutable identity we wrote
-        # at creation time still describes this Python process. PID reuse cannot
-        # happen while this process is alive, so pid + present started_at/cmd is
-        # enough here; the slower psutil validation below remains the fallback
-        # for missing/stale cache.
-        if (
-            _CURRENT_PROCESS
-            and _CURRENT_PROCESS.pid == current_pid
-            and _CURRENT_PROCESS.status == cls.StatusChoices.RUNNING
-            and timezone.now() < _CURRENT_PROCESS.modified_at + timedelta(seconds=PROCESS_RECHECK_INTERVAL)
-            and _CURRENT_PROCESS.started_at is not None
-            and bool(_CURRENT_PROCESS.cmd)
-        ):
-            return _CURRENT_PROCESS
+        # This is process identity, not a health check. A live Python process
+        # cannot reuse its own PID, and a fork fails current_readonly's PID
+        # check. Expiring this cache by modified_at made every call after 60s
+        # query the database and touch logs until something else saved the row.
+        # Explicit lifecycle/recovery code owns liveness and interface refresh.
+        cached = cls.current_readonly()
+        if cached is not None:
+            return cached
+        _CURRENT_PROCESS = None
 
         machine = Machine.current()
         iface = NetworkInterface.current()
 
-        # Check cache validity
-        if _CURRENT_PROCESS:
-            # Verify: same PID, same machine, cache not expired
-            if (
-                _CURRENT_PROCESS.pid == current_pid
-                and _CURRENT_PROCESS.machine_id == machine.id
-                and timezone.now() < _CURRENT_PROCESS.modified_at + timedelta(seconds=PROCESS_RECHECK_INTERVAL)
-            ):
-                if _CURRENT_PROCESS.iface_id != iface.id:
-                    _CURRENT_PROCESS.iface = iface
-                    _CURRENT_PROCESS.save(update_fields=["iface", "modified_at"])
-                _CURRENT_PROCESS.ensure_log_files()
-                return _CURRENT_PROCESS
-            _CURRENT_PROCESS = None
-
         # Get actual process start time from OS for validation
         os_start_time = None
         if PSUTIL_AVAILABLE:
@@ -1660,9 +1628,8 @@ class Process(ModelWithDeleteAfter, models.Model):
         if machine is not None:
             stale = stale.filter(machine=machine)
 
-        # Recovery can run against damaged DB state; stream rows so a large
-        # stale Process backlog cannot be materialized in memory at once.
-        for proc in stale.iterator(chunk_size=100):
+        # Close each bounded read before inspecting processes or writing their state.
+        for proc in stale.order_by("pk").paged_iterator(chunk_size=100):
             shares_pid_namespace = proc.shares_pid_namespace
             if shares_pid_namespace and proc.poll() is not None:
                 cleaned += 1
@@ -1778,7 +1745,8 @@ class Process(ModelWithDeleteAfter, models.Model):
             return None
 
         # Can't validate processes on other machines
-        if self.machine_id != Machine.current().id:
+        machine = Machine.current_readonly()
+        if machine is None or self.machine_id != machine.id:
             return None
 
         try:
@@ -2601,14 +2569,17 @@ class Process(ModelWithDeleteAfter, models.Model):
         """
         cleaned = 0
 
+        # Select active IDs through the existing status index first. Combining
+        # type/status directly let SQLite prefer the historical type index and
+        # scan 350k processes to return zero workers (blocking writes for 12s).
+        active_ids = cls.objects.filter(status=cls.StatusChoices.RUNNING).order_by().values("pk")
         running_children = cls.objects.filter(
+            pk__in=active_ids,
             process_type__in=[cls.TypeChoices.WORKER, cls.TypeChoices.HOOK],
-            status=cls.StatusChoices.RUNNING,
         )
 
-        # Recovery can run against damaged DB state; stream rows so a large
-        # orphaned Process backlog cannot be materialized in memory at once.
-        for proc in running_children.iterator(chunk_size=100):
+        # Close each bounded read before inspecting processes or writing their state.
+        for proc in running_children.order_by("pk").paged_iterator(chunk_size=100):
             if not proc.is_running:
                 proc.mark_exited(
                     exit_code=proc.exit_code if proc.exit_code is not None else _default_exit_code_for_unowned_process(proc.process_type),

archivebox/misc/checks.py

@@ -194,7 +194,10 @@ def check_not_root():
     if os.geteuid() == 0 and not (is_getting_help or is_getting_version):
         print("[yellow][!] Running ArchiveBox as root is not recommended.[/yellow]", file=sys.stderr)
         print("    Root-owned DATA_DIR files may be inaccessible to non-root users later.", file=sys.stderr)
-        print("        https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#do-not-run-as-root", file=sys.stderr)
+        print(
+            "        https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#run-archivebox-as-an-unprivileged-user",
+            file=sys.stderr,
+        )
 
 
 def is_archivebox_source_root(path: Path | str | None = None) -> bool:
@@ -255,7 +258,7 @@ def check_data_dir_permissions(config=None, **config_kwargs):
             "    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#permissions]https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#permissions[/link]",
         )
         STDERR.print(
-            "    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#puid--pgid]https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#puid--pgid[/link]",
+            "    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#docker-permissions-issues]https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#docker-permissions-issues[/link]",
         )
         STDERR.print(
             "    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#filesystem-doesnt-support-fsync-eg-network-mounts]https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#filesystem-doesnt-support-fsync-eg-network-mounts[/link]",

archivebox/misc/db.py

@@ -240,89 +240,38 @@ def drop_models_on_postgres(apps, schema_editor, app_label: str, model_names: li
             schema_editor.delete_model(model)
 
 
-def run_db_analyze_batch(
-    remaining: list[str] | None,
-    *,
-    max_seconds_per_table: float = 120.0,
-) -> list[str]:
-    """Advance one step of a batched SQLite ``ANALYZE`` sweep.
+def run_db_analyze_batch(remaining: list[str] | None) -> list[str]:
+    """Refresh approximate SQLite planner stats for one table per idle tick.
 
-    Without periodic ANALYZE the optimizer's table stats go stale as
-    snapshot/archiveresult tables grow, causing it to start large joins from
-    ``auth_user`` instead of using the indexed url column and blowing snapshot
-    detail page render time from ~50ms to ~500ms+.
-
-    The whole sweep is spread across many calls instead of running as one
-    blocking ``ANALYZE``: pass ``None`` to start a fresh sweep (this call
-    enumerates user tables and runs ``ANALYZE`` on the first one); pass the
-    returned list to advance one more table on each subsequent call. An
-    empty return value means the sweep is complete (or has been aborted) and
-    the next caller should pass ``None`` again. Caller is responsible for
-    throttling new sweeps (orchestrator starts at most one per 24hr while
-    idle) and enforcing a hard upper bound on total sweep wall time.
-
-    Safety guarantees:
-
-    - **Never raises**: every database call is wrapped; on any failure the
-      function returns ``[]`` (abandoning the rest of the sweep) so the
-      orchestrator never crashes on maintenance errors.
-    - **Bounded per-call wall time**: a SQLite progress handler aborts the
-      current ``ANALYZE`` statement once ``max_seconds_per_table`` is
-      exceeded, so a single pathological table cannot wedge the call.
-    - **Never leaves the db locked**: each ``ANALYZE`` runs as a single
-      statement transaction that auto-commits (or rolls back on
-      abort/error). The cursor and progress handler are always cleaned up
-      in ``finally`` blocks even if Python raises mid-call.
-    - Silent no-op on non-SQLite backends.
-
-    WAL journal mode (set in Django settings) keeps readers fully unblocked
-    throughout; the writer lock is only held for the brief ``sqlite_stat*``
-    flush after each table completes.
+    A full ANALYZE scans every index while holding a write transaction; moving
+    it into the runner did not prevent it from blocking submissions. Sampling
+    at most about 1000 entries per index keeps maintenance independent of
+    collection size. Approximate stats are sufficient for choosing indexes.
+    PostgreSQL owns its statistics through autovacuum/autoanalyze.
     """
     from django.db import connection
 
     if connection.vendor != "sqlite":
         return []
-
-    if remaining is None:
-        try:
-            with connection.cursor() as cursor:
-                cursor.execute(
-                    "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name",
-                )
-                remaining = [row[0] for row in cursor.fetchall()]
-        except Exception:
-            return []
-
-    if not remaining:
-        return []
-
-    next_table, *rest = remaining
-    raw_conn = connection.connection
-    progress_handler_set = False
-    if raw_conn is not None and max_seconds_per_table > 0:
-        deadline = time.monotonic() + max_seconds_per_table
-        try:
-            raw_conn.set_progress_handler(lambda: 1 if time.monotonic() > deadline else 0, 10000)
-            progress_handler_set = True
-        except Exception:
-            progress_handler_set = False
-
     try:
         with connection.cursor() as cursor:
-            cursor.execute(f'ANALYZE "{next_table}"')
-    except Exception:
-        # Aborted by progress handler, locked db, or any other failure — skip
-        # this table and continue the sweep. ANALYZE is idempotent so we can
-        # retry on the next 24hr sweep.
-        pass
-    finally:
-        if progress_handler_set and raw_conn is not None:
+            if remaining is None:
+                cursor.execute("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name")
+                remaining = [row[0] for row in cursor.fetchall()]
+            if not remaining:
+                return []
+            next_table, *rest = remaining
+            cursor.execute("PRAGMA analysis_limit")
+            previous_limit = int(cursor.fetchone()[0])
             try:
-                raw_conn.set_progress_handler(None, 0)
-            except Exception:
-                pass
-    return rest
+                cursor.execute("PRAGMA analysis_limit = 1000")
+                cursor.execute(f"ANALYZE {connection.ops.quote_name(next_table)}")
+            finally:
+                cursor.execute(f"PRAGMA analysis_limit = {previous_limit}")
+            return rest
+    except Exception:
+        # Optional planner maintenance must not prevent queued captures.
+        return []
 
 
 def compact_command(cmdline: list[str] | None, fallback: str = "") -> str:
@@ -445,7 +394,10 @@ def migration_lock(stdout: TextIO | None = None):
 
 
 # Migration names that previously existed in ArchiveBox's source tree but have
-# since been deleted (squashed away, renamed, moved between apps, etc.). DBs
+# since been deleted (squashed away, renamed, moved between apps, etc.), plus
+# exact older migrations generated in the installed package by v0.6.2's
+# startup makemigrations. Those runtime-generated names were never in Git, so
+# the deleted-file history alone cannot cover them. DBs
 # upgraded incrementally through 0.8.x → 0.9.x dev rcs accumulate rows for
 # these in ``django_migrations``; the newer-DB guard added in 65dc2521 would
 # otherwise refuse to start with "applied migrations missing from this build"
@@ -454,13 +406,18 @@ def migration_lock(stdout: TextIO | None = None):
 # splits the migration graph when only a *subset* of the replaces list is
 # applied (which is exactly what happens for users at different intermediate
 # dev branch states). This set is the authoritative compat list — extend it
-# when squashing more migrations away. Generated from
-# ``git log --diff-filter=D --name-only`` over each app's migrations/ tree.
+# when squashing more migrations away. The source-tracked names were generated
+# from ``git log --diff-filter=D --name-only`` over each app's migrations/ tree.
 HISTORICAL_GHOST_MIGRATIONS: frozenset[tuple[str, str]] = frozenset(
     {
         # core: 0023→0075 sequence plus every transient dev rename
         ("core", "0002_auto_20190417_0739"),
         ("core", "0006_auto_20200915_2006"),
+        # Reported v0.6.2 runtime-generated 0021 migrations; the checked-in
+        # replacement only changes ArchiveResult.extractor choices.
+        ("core", "0021_auto_20220510_0644"),
+        ("core", "0021_auto_20220724_1254"),
+        ("core", "0021_auto_20221128_1116"),
         ("core", "0023_alter_archiveresult_options_archiveresult_abid_and_more"),
         ("core", "0023_new_schema"),
         ("core", "0024_auto_20240513_1143"),
@@ -518,6 +475,8 @@ HISTORICAL_GHOST_MIGRATIONS: frozenset[tuple[str, str]] = frozenset(
         ("core", "0053_remove_snapshottag_snapshot_old"),
         ("core", "0054_alter_snapshot_timestamp"),
         ("core", "0055_alter_tag_slug"),
+        # Dev-only progress indexes removed in favor of existing-index queries.
+        ("core", "0056_progress_covering_indexes"),
         ("core", "0056_remove_tag_uuid"),
         ("core", "0057_rename_id_tag_old_id"),
         ("core", "0058_alter_tag_old_id"),

archivebox/misc/jsonl.py

@@ -142,3 +142,27 @@ def write_record(record: dict[str, Any], stream: TextIO | None = None) -> None:
     active_stream: TextIO = sys.stdout if stream is None else stream
     active_stream.write(json.dumps(record) + "\n")
     active_stream.flush()
+
+
+def write_records(records: Iterable[dict[str, Any]], stream: TextIO | None = None) -> int:
+    """Stream bulk JSONL output with bounded buffering, returning its row count."""
+    active_stream: TextIO = sys.stdout if stream is None else stream
+    lines: list[str] = []
+    pending_size = 0
+    count = 0
+    for record in records:
+        line = json.dumps(record) + "\n"
+        lines.append(line)
+        pending_size += len(line)
+        count += 1
+        # CLI stdout can be unbuffered. A syscall per snapshot makes large
+        # exports IO-bound; small batches keep memory and pipe latency bounded.
+        if pending_size >= 64 * 1024:
+            active_stream.write("".join(lines))
+            active_stream.flush()
+            lines.clear()
+            pending_size = 0
+    if lines:
+        active_stream.write("".join(lines))
+        active_stream.flush()
+    return count

archivebox/misc/shell_welcome_message.py

@@ -48,10 +48,10 @@ if __name__ == "__main__":
 
     prnt("[i] :heavy_dollar_sign: Welcome to the ArchiveBox Shell!")
     prnt(
-        "    [deep_sky_blue4]Docs:[/deep_sky_blue4] [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Shell-Usage]https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Shell-Usage[/link]",
+        "    [deep_sky_blue4]Docs:[/deep_sky_blue4] [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage]https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage[/link]",
     )
     prnt(
-        "          [link=https://docs.archivebox.io/en/dev/apidocs/archivebox/archivebox.html]https://docs.archivebox.io/en/dev/apidocs/archivebox/archivebox.html[/link]",
+        "          [link=https://docs.archivebox.io/dev/apidocs/archivebox/archivebox.html]https://docs.archivebox.io/dev/apidocs/archivebox/archivebox.html[/link]",
     )
     prnt()
     prnt(" :grey_question: [violet]Hint[/] [i]Here are some examples to get started:[/]")

archivebox/misc/util.py

@@ -3,7 +3,7 @@ __package__ = "archivebox.misc"
 # Bootable utility functions (URL parsing, date parsing, JSON encoding, decorators).
 # MUST NOT import archivebox.config, archivebox.core, or Django — this module is
 # loaded by hooks.py and other early code paths. Only depends on stdlib and the
-# bootable .logging module.
+# bootable .logging module and standalone abx_plugins URL helpers.
 
 import re
 import json as pyjson
@@ -19,6 +19,9 @@ from urllib.parse import urlparse, quote, unquote
 from html import escape, unescape
 from datetime import datetime, timezone
 
+from abx_plugins.plugins.base.url_cleaning import URL_REGEX, fix_url_from_markdown
+from abx_plugins.plugins.base.url_cleaning import sanitize_extracted_url as sanitize_plugin_url
+
 from .logging import COLOR_DICT
 
 
@@ -90,18 +93,6 @@ def ts_to_date_str(ts: Any) -> str | None:
 COLOR_REGEX = re.compile(r"\[(?P<arg_1>\d+)(;(?P<arg_2>\d+)(;(?P<arg_3>\d+))?)?m")
 
 
-# https://mathiasbynens.be/demo/url-regex
-URL_REGEX = re.compile(
-    r"(?=("
-    r"http[s]?://"  # start matching from allowed schemes
-    r"(?:[a-zA-Z]|[0-9]"  # followed by allowed alphanum characters
-    r"|[-_$@.&+!*\(\),]"  #   or allowed symbols (keep hyphen first to match literal hyphen)
-    r"|[^\u0000-\u007F])+"  #   or allowed unicode bytes
-    r'[^\]\[<>"\'\s]+'  # stop parsing at these symbols
-    r"))",
-    re.IGNORECASE | re.UNICODE,
-)
-
 # Maximum supported URL length. Very long URLs are rare but must be supported correctly
 # (e.g. data: URLs, deeply nested query strings). The Snapshot.url column is stored as a
 # variable-length TextField (so short URLs don't reserve space and very long URLs still
@@ -109,29 +100,6 @@ URL_REGEX = re.compile(
 # all keep working.
 MAX_URL_LENGTH = 65535
 
-QUOTE_DELIMITERS = (
-    '"',
-    "'",
-    "`",
-    "“",
-    "”",
-    "‘",
-    "’",
-)
-QUOTE_ENTITY_DELIMITERS = (
-    "&quot;",
-    "&#34;",
-    "&#x22;",
-    "&apos;",
-    "&#39;",
-    "&#x27;",
-)
-URL_ENTITY_REPLACEMENTS = (
-    ("&amp;", "&"),
-    ("&#38;", "&"),
-    ("&#x26;", "&"),
-)
-
 FILESIZE_UNITS: dict[str, int] = {
     "": 1,
     "b": 1,
@@ -153,36 +121,9 @@ FILESIZE_UNITS: dict[str, int] = {
 
 
 def sanitize_extracted_url(url: str) -> str:
-    """Trim quote garbage and dangling prose punctuation from an extracted URL candidate."""
-    cleaned = (url or "").strip()
-    if not cleaned:
-        return cleaned
-
-    lower_cleaned = cleaned.lower()
-    cut_index = len(cleaned)
-
-    for delimiter in QUOTE_DELIMITERS:
-        found_index = cleaned.find(delimiter)
-        if found_index != -1:
-            cut_index = min(cut_index, found_index)
-
-    for delimiter in QUOTE_ENTITY_DELIMITERS:
-        found_index = lower_cleaned.find(delimiter)
-        if found_index != -1:
-            cut_index = min(cut_index, found_index)
-
-    cleaned = cleaned[:cut_index].strip()
-    lower_cleaned = cleaned.lower()
-    for entity, replacement in URL_ENTITY_REPLACEMENTS:
-        while entity in lower_cleaned:
-            entity_index = lower_cleaned.find(entity)
-            cleaned = cleaned[:entity_index] + replacement + cleaned[entity_index + len(entity) :]
-            lower_cleaned = cleaned.lower()
-
-    cleaned = cleaned.rstrip(".,;:!?\\'\"")
-    cleaned = cleaned.rstrip('"')
-
-    return cleaned
+    """Use the standalone parser's quote rules, then trim dangling prose punctuation."""
+    cleaned = sanitize_plugin_url(url)
+    return cleaned.rstrip('.,;:!?\\"')
 
 
 def validate_url_length(url: str) -> str:
@@ -215,39 +156,6 @@ def parens_are_matched(string: str, open_char="(", close_char=")"):
     return count == 0
 
 
-def fix_url_from_markdown(url_str: str) -> str:
-    """
-    cleanup a regex-parsed url that may contain dangling trailing parens from markdown link syntax
-    helpful to fix URLs parsed from markdown e.g.
-      input:  https://wikipedia.org/en/some_article_(Disambiguation).html?abc=def).somemoretext
-      result: https://wikipedia.org/en/some_article_(Disambiguation).html?abc=def
-
-    IMPORTANT ASSUMPTION: valid urls wont have unbalanced or incorrectly nested parentheses
-    e.g. this will fail the user actually wants to ingest a url like 'https://example.com/some_wei)(rd_url'
-         in that case it will return https://example.com/some_wei (truncated up to the first unbalanced paren)
-    This assumption is true 99.9999% of the time, and for the rare edge case the user can use url_list parser.
-    """
-    trimmed_url = url_str
-    if len(trimmed_url) > 2048:
-        return trimmed_url
-
-    # cut off one trailing character at a time
-    # until parens are balanced e.g. /a(b)c).x(y)z -> /a(b)c
-    trim_attempts = 0
-    while trimmed_url and not parens_are_matched(trimmed_url) and trim_attempts < 256:
-        trimmed_url = trimmed_url[:-1]
-        trim_attempts += 1
-
-    if not trimmed_url or not parens_are_matched(trimmed_url):
-        return url_str
-
-    # make sure trimmed url is still valid
-    if any(match == trimmed_url for match in re.findall(URL_REGEX, trimmed_url)):
-        return trimmed_url
-
-    return url_str
-
-
 def split_comma_separated_urls(url: str):
     offset = 0
     while True:
@@ -274,7 +182,8 @@ def find_all_urls(urls_str: str):
         if match.start() in skipped_starts:
             continue
 
-        cleaned_match = sanitize_extracted_url(fix_url_from_markdown(match.group(1)))
+        cleaned_match = fix_url_from_markdown(match.group(1), preceding_text=urls_str[max(0, match.start(1) - 6) : match.start(1)])
+        cleaned_match = sanitize_extracted_url(cleaned_match)
         for offset, url in split_comma_separated_urls(cleaned_match):
             if offset:
                 skipped_starts.add(match.start() + offset)

archivebox/personas/forms.py

@@ -122,14 +122,14 @@ class PersonaAdminForm(PluginConfigFormMixin, forms.ModelForm):
                 "Use the custom path/CDP option if the browser data lives elsewhere."
             )
 
-        self.build_plugin_groups(get_config(persona=self.instance) if self.instance and self.instance.pk else get_config())
+        self.build_plugin_groups(get_config(persona=self.instance) if not self.instance._state.adding else get_config())
 
     def clean_name(self) -> str:
         name = str(self.cleaned_data.get("name") or "").strip()
         is_valid, error_message = validate_persona_name(name)
         if not is_valid:
             raise forms.ValidationError(error_message)
-        return name
+        return self.instance.validate_name(name)
 
     def clean(self) -> dict[str, Any]:
         cleaned_data = super().clean()

archivebox/personas/models.py

@@ -18,6 +18,7 @@ from typing import TYPE_CHECKING, Any
 from collections.abc import Mapping
 
 from django.db import IntegrityError, models
+from django.core.exceptions import ValidationError
 from django.db.models.fields.json import KT
 from django.conf import settings
 from django.utils import timezone
@@ -74,8 +75,12 @@ def derive_persona_config(*, name: str, config: Mapping[str, Any] | None, person
         derived["COOKIES_FILE"] = str(cookies_path)
 
     auth_path = persona_dir / "auth.json"
-    if "AUTH_STORAGE_FILE" not in derived and auth_path.exists():
-        derived["AUTH_STORAGE_FILE"] = str(auth_path)
+    if "AUTH_STORAGE_FILE" not in derived:
+        try:
+            if auth_path.stat().st_size > 0:
+                derived["AUTH_STORAGE_FILE"] = str(auth_path)
+        except FileNotFoundError:
+            pass
 
     derived["ACTIVE_PERSONA"] = name
     return derived
@@ -117,6 +122,13 @@ class Persona(ModelWithConfig):
         app_label = "personas"
 
     def save(self, *args, **kwargs):
+        # Adopt the spelling on disk, never rename a Chrome profile just to fix case.
+        self.__dict__.pop("_persona_path", None)
+        canonical_name = self.validate_name(self.name)
+        if canonical_name != self.name:
+            self.name = canonical_name
+            if kwargs.get("update_fields") is not None:
+                kwargs["update_fields"] = tuple(dict.fromkeys([*kwargs["update_fields"], "name"]))
         config = dict(self.config or {})
         if str(config.get("PERMISSIONS") or "").strip().lower() not in PERMISSIONS_VALUES:
             from archivebox.config.common import get_config
@@ -133,10 +145,59 @@ class Persona(ModelWithConfig):
 
     @property
     def path(self) -> Path:
-        """Path to persona directory under PERSONAS_DIR."""
+        """Reuse the directory's spelling without renaming a Chrome profile."""
         from archivebox.config.constants import CONSTANTS
 
-        return CONSTANTS.PERSONAS_DIR / self.name
+        cached = self.__dict__.get("_persona_path")
+        if cached is None or cached[0] != self.name:
+            directory_name = self.directory_name(self.name)
+            cached = (self.name, CONSTANTS.PERSONAS_DIR / directory_name)
+            self.__dict__["_persona_path"] = cached
+        return cached[1]
+
+    @staticmethod
+    def directory_name(name: str) -> str:
+        from archivebox.config.constants import CONSTANTS
+        from archivebox.personas.importers import validate_persona_name
+
+        valid, error = validate_persona_name(name)
+        if not valid:
+            raise ValidationError(error)
+        # Linux can hold both Default/default; macOS usually aliases them. Reuse a
+        # single match on either filesystem, but never guess which conflicting
+        # profile contains the user's login state. Only list names, not profile trees.
+        try:
+            with os.scandir(CONSTANTS.PERSONAS_DIR) as entries:
+                matches = sorted(entry.name for entry in entries if entry.name.casefold() == name.casefold())
+        except FileNotFoundError:
+            matches = []
+        if len(matches) > 1:
+            raise ValidationError(
+                f"Conflicting persona directories: {'; '.join(str(CONSTANTS.PERSONAS_DIR / match) for match in matches)}. "
+                "Stop Chrome and rename the conflicting directories manually to distinct names; no profile directories were renamed.",
+            )
+        return matches[0] if matches else name
+
+    @classmethod
+    def find_named(cls, name: str) -> "Persona | None":
+        # SQLite's iexact is ASCII-only; Python casefold gives every entry point
+        # the same Unicode matching rules as the filesystem-name lookup above.
+        matches = [(pk, existing) for pk, existing in cls.objects.values_list("pk", "name") if existing.casefold() == name.casefold()]
+        if len(matches) > 1:
+            raise ValidationError(
+                f"Conflicting persona records: {'; '.join(f'{existing!r} (ID {pk})' for pk, existing in matches)}. "
+                "Rename the conflicting personas manually to distinct names; profile files have not been changed.",
+            )
+        return cls.objects.get(pk=matches[0][0]) if matches else None
+
+    def validate_name(self, name: str) -> str:
+        existing = self.find_named(name)
+        if existing is not None and existing.pk != self.pk:
+            raise ValidationError(
+                f"Name conflicts with persona {existing.name!r} (ID {existing.pk}, directory {existing.path}). "
+                "Choose a distinct name or rename that persona record manually.",
+            )
+        return self.directory_name(name)
 
     @property
     def CHROME_USER_DATA_DIR(self) -> str:
@@ -156,9 +217,12 @@ class Persona(ModelWithConfig):
 
     @property
     def AUTH_STORAGE_FILE(self) -> str:
-        """Derived path to auth.json for this persona (if it exists)."""
+        """Derived path to a nonempty auth.json export for this persona."""
         auth_path = self.path / "auth.json"
-        return str(auth_path) if auth_path.exists() else ""
+        try:
+            return str(auth_path) if auth_path.stat().st_size > 0 else ""
+        except FileNotFoundError:
+            return ""
 
     def get_derived_config(self) -> dict:
         """
@@ -171,10 +235,13 @@ class Persona(ModelWithConfig):
         - AUTH_STORAGE_FILE (derived from persona path, if file exists)
         - ACTIVE_PERSONA (set to this persona's name)
         """
-        return derive_persona_config(name=self.name, config=self.config, persona_dir=self.path)
+        return derive_persona_config(name=self.path.name, config=self.config, persona_dir=self.path)
 
     def ensure_dirs(self) -> None:
         """Create persona directories if they don't exist."""
+        # Recheck at profile preparation, including changes made outside this process.
+        self.__dict__.pop("_persona_path", None)
+        self.validate_name(self.name)
         self.path.mkdir(parents=True, exist_ok=True)
         (self.path / "chrome_profile").mkdir(parents=True, exist_ok=True)
         (self.path / "chrome_downloads").mkdir(parents=True, exist_ok=True)
@@ -246,16 +313,26 @@ class Persona(ModelWithConfig):
                     fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
 
     @classmethod
-    def get_or_create_named(cls, name: str) -> "Persona":
+    def get_or_create_named(cls, name: str, *, defaults: dict | None = None) -> "Persona":
+        # Init/startup must reuse a user-created 'default', not create a second
+        # DB row pointing at a different (or shared) profile (issue #1896).
         persona_name = (name or "Default").strip() or "Default"
-        persona = cls.objects.filter(name=persona_name).first()
+        persona = cls.find_named(persona_name)
         if persona is not None:
+            canonical_name = persona.validate_name(persona.name)
+            if canonical_name != persona.name:
+                persona.name = canonical_name
+                persona.save(update_fields=["name"])
             return persona
 
         try:
-            return cls.objects.create(name=persona_name)
-        except IntegrityError:
-            return cls.objects.get(name=persona_name)
+            return cls.objects.create(name=persona_name, **(defaults or {}))
+        except (IntegrityError, ValidationError):
+            persona = cls.find_named(persona_name)
+            if persona is None:
+                raise
+            persona.validate_name(persona.name)
+            return persona
 
     def runtime_root_for_crawl(self, crawl) -> Path:
         return Path(crawl.output_dir) / ".persona" / self.name
@@ -315,20 +392,9 @@ class Persona(ModelWithConfig):
             if chrome_binary:
                 (runtime_root / "chrome_binary.txt").write_text(chrome_binary)
 
-        return {
-            # Hooks derive CHROME_USER_DATA_DIR/CHROME_DOWNLOADS_DIR from
-            # PERSONAS_DIR + ACTIVE_PERSONA. Point PERSONAS_DIR at the
-            # per-crawl runtime root here so CHROME_ISOLATION=crawl never
-            # leaks or reuses the template profile while keeping Chrome path
-            # derivation centralized in the Chrome plugin helpers.
-            "PERSONAS_DIR": str(runtime_root.parent),
-            "ACTIVE_PERSONA": self.name,
-            **{
-                key: str(runtime_root / filename)
-                for key, filename in (("COOKIES_FILE", "cookies.txt"), ("AUTH_STORAGE_FILE", "auth.json"))
-                if (runtime_root / filename).is_file()
-            },
-        }
+        # Hooks derive Chrome profile paths from the runtime persona root.
+        # Use the same optional auth discovery as the template persona.
+        return derive_persona_config(name=self.name, config={}, persona_dir=runtime_root)
 
     def prepare_runtime_for_snapshot(self, snapshot, chrome_binary: str = "") -> dict[str, str]:
         crawl_runtime_profile_dir = self.runtime_profile_dir_for_crawl(snapshot.crawl)
@@ -358,18 +424,7 @@ class Persona(ModelWithConfig):
         if chrome_binary:
             (runtime_root / "chrome_binary.txt").write_text(chrome_binary)
 
-        return {
-            # See prepare_runtime_for_crawl(): snapshot isolation changes the
-            # persona root, not individual CHROME_* config keys, so standalone
-            # Chrome hooks and ArchiveBox-driven hooks resolve paths the same way.
-            "PERSONAS_DIR": str(runtime_root.parent),
-            "ACTIVE_PERSONA": self.name,
-            **{
-                key: str(runtime_root / filename)
-                for key, filename in (("COOKIES_FILE", "cookies.txt"), ("AUTH_STORAGE_FILE", "auth.json"))
-                if (runtime_root / filename).is_file()
-            },
-        }
+        return derive_persona_config(name=self.name, config={}, persona_dir=runtime_root)
 
     def cleanup_runtime_for_crawl(self, crawl) -> None:
         shutil.rmtree(Path(crawl.output_dir) / ".persona", ignore_errors=True)

archivebox/progressmonitor/collection.py

@@ -1,4 +1,15 @@
-"""Non-blocking, database-only collection totals for the progress monitor."""
+"""Cached display totals for progress polling and the public snapshot list.
+
+Missing/stale statistics are intentional: returning useful rows and progress
+quickly matters more than exact initial counts. Each viewer may poll once per
+second; repeating aggregates per request would consume the small server's CPU
+and hold SQLite read locks needed by the archiving runner. Callers must accept
+None on a cold cache and must never wait for the refresh or count synchronously.
+
+Background reads still take normal SQLite read locks. Existing indexes, short
+batches where needed, and throttled refreshes limit their cost; moving work to
+a thread does not make a scan free or require switching the database to WAL.
+"""
 
 import logging
 from threading import Lock, Thread
@@ -9,14 +20,77 @@ from django.db import close_old_connections, connections
 from django.db.models import Count, Sum
 
 
+# This is only a process-local gate for background work, not a DB/file lock.
+# Every request tries it without waiting; a busy refresh means returning the
+# cached value (or None). Serializing refreshes also avoids competing scans for
+# different crawls/users when many viewer tabs open at once.
 _refresh_lock = Lock()
 
 
-def collection_summary(user):
-    user_id = None if user.is_superuser else user.pk
-    key = f"progress-collection:{'all' if user_id is None else user_id}"
+def crawl_summary(crawl_id, *, snapshot_id=None):
+    """Sample sealed snapshot metadata without adding table scans to polls."""
+    key = f"progress-crawl:{crawl_id}:{snapshot_id or 'all'}"
     summary = cache.get(key)
-    # Even an indexed SUM scans the index. Never make an HTTP request wait for
+    if (summary is None or time() - summary["sampled_at"] >= 30) and _refresh_lock.acquire(blocking=False):
+        if not cache.add(f"{key}:refresh", True, timeout=30):
+            _refresh_lock.release()
+            return summary
+
+        def refresh():
+            try:
+                close_old_connections()
+                from archivebox.core.models import Snapshot
+
+                snapshots = Snapshot.objects.filter(crawl_id=crawl_id, status=Snapshot.StatusChoices.SEALED)
+                if snapshot_id is not None:
+                    snapshots = snapshots.filter(pk=snapshot_id)
+                snapshots = snapshots.order_by("modified_at").values_list("modified_at", "downloaded_at", "output_size")
+                totals = {"snapshots": 0, "cancelled": 0, "bytes": 0}
+                modified_at, offset = None, 0
+                while True:
+                    batch_scope = snapshots.filter(modified_at__gte=modified_at) if modified_at is not None else snapshots
+                    # Use (crawl, status, modified_at), releasing the SQLite read
+                    # cursor after each small batch. Never hold a cursor over the
+                    # whole crawl, even in this background worker.
+                    batch = list(batch_scope[offset : offset + 100])
+                    for _, downloaded_at, size in batch:
+                        totals["snapshots"] += 1
+                        totals["cancelled"] += downloaded_at is None
+                        totals["bytes"] += size or 0
+                    if len(batch) < 100:
+                        break
+                    last_modified_at = batch[-1][0]
+                    # Only offset within ties; otherwise advance the indexed
+                    # timestamp range instead of rescanning previous pages.
+                    offset = (offset if last_modified_at == modified_at else 0) + sum(row[0] == last_modified_at for row in batch)
+                    modified_at = last_modified_at
+                totals["sampled_at"] = time()
+                cache.set(key, totals, timeout=3600)
+            except Exception:
+                logging.getLogger(__name__).exception("Could not refresh crawl totals")
+            finally:
+                connections.close_all()
+                _refresh_lock.release()
+
+        try:
+            Thread(target=refresh, name="crawl-summary", daemon=True).start()
+        except RuntimeError:
+            _refresh_lock.release()
+            logging.getLogger(__name__).exception("Could not start crawl totals refresh")
+    return summary
+
+
+def collection_summary(user, *, public=False):
+    """Return cached totals immediately and, when due, start one refresh.
+
+    Public pages need no extra AJAX/polling for these totals: an ordinary later
+    page load reads the populated cache. Keep this best-effort display cache
+    separate from pagination/authorization decisions, which need actual rows.
+    """
+    user_id = None if user.is_superuser else user.pk
+    key = "progress-collection:public" if public else f"progress-collection:{'all' if user_id is None else user_id}"
+    summary = cache.get(key)
+    # Even indexed COUNT/SUM aggregates scan the index. Never make HTTP wait for
     # that scan on a large collection. Serve stale totals while one worker refreshes;
     # a single non-blocking lock prevents a queue of scans from concurrent clients.
     if (summary is None or time() - summary["sampled_at"] >= 60) and _refresh_lock.acquire(blocking=False):
@@ -32,13 +106,29 @@ def collection_summary(user):
                 from archivebox.core.models import Snapshot
 
                 snapshots = Snapshot.objects.all()
-                if user_id is not None:
-                    snapshots = snapshots.filter(crawl__created_by_id=user_id)
-                # COUNT(*) + SUM(output_size) can use Snapshot's covering size index.
-                # Snapshot.output_size is maintained when ArchiveResult sizes change.
-                totals = snapshots.aggregate(snapshots=Count("*"), bytes=Sum("output_size"))
-                totals["bytes"] = totals["bytes"] or 0
+                if public:
+                    from archivebox.core.permissions import PERMISSIONS_PUBLIC
+
+                    # SQLite can choose a table scan for equality after ANALYZE
+                    # on an all-public archive. Equal range bounds keep the count
+                    # on the existing permissions index. It is still O(n), so
+                    # public pages share this background cache, never await it.
+                    # Count the indexed, non-null permissions column so this
+                    # needs no Snapshot table reads or new covering-index migration.
+                    totals = snapshots.filter(permissions__gte=PERMISSIONS_PUBLIC, permissions__lte=PERMISSIONS_PUBLIC).aggregate(
+                        snapshots=Count("permissions"),
+                    )
+                else:
+                    if user_id is not None:
+                        snapshots = snapshots.filter(crawl__created_by_id=user_id)
+                    # COUNT(*) + SUM(output_size) can use Snapshot's covering size index.
+                    # Snapshot.output_size is maintained when ArchiveResult sizes change.
+                    totals = snapshots.aggregate(snapshots=Count("*"), bytes=Sum("output_size"))
+                totals["bytes"] = totals.get("bytes") or 0
                 totals["sampled_at"] = time()
+                # Retain old totals longer than the 60s refresh interval so a
+                # slow/failed refresh does not force every tab back to a cold
+                # count. Staleness is acceptable; blocking page loads is not.
                 cache.set(key, totals, timeout=3600)
             except Exception:
                 logging.getLogger(__name__).exception("Could not refresh collection totals")

archivebox/progressmonitor/templates/progressmonitor/progress_monitor.html

@@ -1904,12 +1904,22 @@
     const progressEndpoint = monitor.dataset.progressEndpoint || '/progress.json';
 
     function fetchProgress() {
+        // Opening a replay replaces the queued snapshot's progress panel.
+        // Its interval and already-running fetch must stop using that old DOM.
+        if (!monitor.isConnected) {
+            stopPolling();
+            return;
+        }
         fetch(progressEndpoint, { credentials: 'same-origin' })
             .then(response => {
                 if (!response.ok) throw new Error(`HTTP ${response.status}`);
                 return response.json();
             })
             .then(data => {
+                if (!monitor.isConnected) {
+                    stopPolling();
+                    return;
+                }
                 if (data.error) {
                     console.error('Progress API error:', data.error, data.traceback);
                     setOrchestratorState('error', 'Backend not responding');
@@ -1921,6 +1931,10 @@
                 updateProgress(data);
             })
             .catch(error => {
+                if (!monitor.isConnected) {
+                    stopPolling();
+                    return;
+                }
                 console.error('Progress fetch error:', error);
                 setOrchestratorState('error', 'Backend not responding');
                 idleMessage.textContent = 'Fetch Error: ' + error.message;
@@ -1929,7 +1943,7 @@
     }
 
     function startPolling() {
-        if (pollInterval) return;
+        if (pollInterval || !monitor.isConnected) return;
         fetchProgress();
         pollInterval = setInterval(fetchProgress, pollDelayMs);
     }
@@ -2033,7 +2047,13 @@
 
     // Start polling when page loads
     startPolling();
-    setInterval(updateDurationBadges, 1000);
+    const durationInterval = setInterval(() => {
+        if (!monitor.isConnected) {
+            clearInterval(durationInterval);
+            return;
+        }
+        updateDurationBadges();
+    }, 1000);
 
     // Pause polling when tab is hidden
     document.addEventListener('visibilitychange', function() {

archivebox/progressmonitor/views.py

@@ -7,12 +7,11 @@ from typing import Literal
 from abx_dl.events import PROCESS_EXIT_SKIPPED
 from django.conf import settings
 from django.db import DatabaseError
-from django.db.models import CharField, Count, Q, Sum
+from django.db.models import CharField, Count, Q
 from django.db.models.functions import Cast
 from django.http import HttpResponse, JsonResponse
 from django.utils import timezone
 
-from archivebox.config.common import get_config
 from archivebox.core.permissions import can_view_snapshot, is_admin_user
 from archivebox.core.routes_util import build_snapshot_detail_url, build_snapshot_url, get_api_base_url
 from archivebox.misc.logging_util import printable_filesize
@@ -182,53 +181,37 @@ def live_progress_view(request, *, authorized_snapshot=None):
 
         running_screencasts = {}
 
-        def screencast_frame_url(crawl_id: str, crawl_dir: Path) -> str:
-            frame_path = crawl_dir / "chrome_screencast" / "latest.jpg"
-            try:
-                frame_stat = frame_path.stat()
-            except OSError:
-                return ""
-            if frame_stat.st_size <= 0:
-                return ""
+        def screencast_frame_url(crawl_id: str) -> str:
             started_at = running_screencasts.get(crawl_id)
-            if started_at is None or frame_stat.st_mtime < started_at.timestamp():
+            if started_at is None:
                 return ""
-            rel = f"/api/v1/crawls/crawl/{crawl_id}/files/chrome_screencast/latest.jpg?v={frame_stat.st_mtime_ns}"
+            # File existence/freshness belongs to the image request, never to
+            # the one-second JSON poll (archive storage may be remote).
+            rel = f"/api/v1/crawls/crawl/{crawl_id}/files/chrome_screencast/latest.jpg?v={int(now.timestamp())}&after={started_at.timestamp()}"
             return f"{api_base}{rel}" if api_base else rel
 
-        machine_id = Machine.current().id
+        machine = Machine.current_readonly()
+        machine_id = machine.id if machine is not None else None
         orchestrator_proc = (
             Process.objects.filter(
                 machine_id=machine_id,
                 process_type=Process.TypeChoices.ORCHESTRATOR,
                 status=Process.StatusChoices.RUNNING,
             )
-            .only("id", "pid", "started_at", "machine_id", "process_type", "status")
+            .only("id", "pid", "started_at", "machine_id", "process_type", "status", "env", "cmd")
             .order_by("-started_at")
             .first()
             if machine_id is not None
             else None
         )
-        runner_worker = None
         orchestrator_proc_running = bool(orchestrator_proc and orchestrator_proc.is_running)
-        if not orchestrator_proc_running:
-            try:
-                from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker
-
-                supervisor = get_existing_supervisord_process(quiet=True)
-                runner_worker = get_worker(supervisor, "worker_runner") if supervisor else None
-            except (OSError, RuntimeError, TimeoutError):
-                runner_worker = None
-
-        runner_worker_running = bool(runner_worker and runner_worker.get("statename") in ("STARTING", "RUNNING"))
-        runner_worker_pid = runner_worker.get("pid") if runner_worker else None
-        orchestrator_running = orchestrator_proc_running or runner_worker_running
-        orchestrator_pid = orchestrator_proc.pid if orchestrator_proc_running and orchestrator_proc else runner_worker_pid
+        orchestrator_running = orchestrator_proc_running
+        orchestrator_pid = orchestrator_proc.pid if orchestrator_proc_running and orchestrator_proc else None
 
         # Get model counts by status
         crawl_status_counts = Crawl.status_counts(
             crawl_scope,
-            (Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED, Crawl.StatusChoices.PAUSED),
+            (Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED),
         )
         crawls_queued = crawl_status_counts.get(Crawl.StatusChoices.QUEUED, 0)
         crawls_active = crawl_status_counts.get(Crawl.StatusChoices.STARTED, 0)
@@ -242,7 +225,7 @@ def live_progress_view(request, *, authorized_snapshot=None):
 
         snapshot_status_counts = Snapshot.status_counts(
             snapshot_scope,
-            Snapshot.OPEN_STATES,
+            Snapshot.RUNNABLE_STATES,
         )
         snapshots_queued = snapshot_status_counts.get(Snapshot.StatusChoices.QUEUED, 0)
         snapshots_active = snapshot_status_counts.get(Snapshot.StatusChoices.STARTED, 0)
@@ -252,15 +235,24 @@ def live_progress_view(request, *, authorized_snapshot=None):
             ArchiveResult.StatusChoices.QUEUED,
             ArchiveResult.StatusChoices.STARTED,
         )
-        archiveresult_status_counts = ArchiveResult.status_counts(archiveresult_scope, result_statuses)
+        # Start from open results using the existing status index. Filtering by
+        # plugin first scans historical runs, including all completed results.
+        result_counts = list(
+            archiveresult_scope.filter(status__in=result_statuses).order_by().values("status", "plugin").annotate(count=Count("*")),
+        )
+        archiveresult_status_counts = {
+            status: sum(row["count"] for row in result_counts if row["status"] == status) for status in result_statuses
+        }
         download_scope = archiveresult_scope.filter(
             plugin__in=download_plugin_names,
             snapshot__status__in=Snapshot.RUNNABLE_STATES,
             snapshot__crawl__status__in=Crawl.RUNNABLE_STATES,
         )
-        indexing_scope = archiveresult_scope.filter(plugin__in=indexing_plugin_names)
         download_status_counts = ArchiveResult.status_counts(download_scope, result_statuses)
-        indexing_status_counts = ArchiveResult.status_counts(indexing_scope, result_statuses)
+        indexing_status_counts = {
+            status: sum(row["count"] for row in result_counts if row["status"] == status and row["plugin"] in indexing_plugin_names)
+            for status in result_statuses
+        }
         archiveresults_queued = archiveresult_status_counts.get(ArchiveResult.StatusChoices.QUEUED, 0)
         archiveresults_active = archiveresult_status_counts.get(ArchiveResult.StatusChoices.STARTED, 0)
 
@@ -296,21 +288,27 @@ def live_progress_view(request, *, authorized_snapshot=None):
             .values(*active_crawl_fields)
             .order_by("-modified_at")[:max_active_crawls],
         )
-        paused_crawls = list(
-            crawl_scope.filter(
-                Q(status=Crawl.StatusChoices.PAUSED, created_at__gte=paused_crawl_cutoff)
-                | Q(
-                    status=Crawl.StatusChoices.PAUSED,
-                    snapshot_set__status__in=Snapshot.RUNNABLE_STATES,
-                    snapshot_set__retry_at__lte=now,
-                )
-                | Q(
-                    status=Crawl.StatusChoices.PAUSED,
-                    snapshot_set__archiveresult__status=ArchiveResult.StatusChoices.QUEUED,
-                ),
+        # Start with runnable children, not every historical snapshot belonging
+        # to a paused crawl. These uncorrelated subqueries run once per poll.
+        due_paused_snapshots = (
+            Snapshot.objects.filter(
+                status__in=Snapshot.RUNNABLE_STATES,
+                retry_at__lte=now,
             )
+            .order_by()
+            .values("crawl_id")
+        )
+        queued_paused_results = (
+            ArchiveResult.objects.filter(
+                status=ArchiveResult.StatusChoices.QUEUED,
+            )
+            .order_by()
+            .values("snapshot__crawl_id")
+        )
+        paused_crawls = list(
+            crawl_scope.filter(status=Crawl.StatusChoices.PAUSED)
+            .filter(Q(created_at__gte=paused_crawl_cutoff) | Q(id__in=due_paused_snapshots) | Q(id__in=queued_paused_results))
             .values(*active_crawl_fields)
-            .distinct()
             .order_by("-modified_at")[:max_active_crawls],
         )
         queued_crawls = list(
@@ -324,52 +322,29 @@ def live_progress_view(request, *, authorized_snapshot=None):
                 crawl["persona_id"] = str(crawl["persona_id"])
         persona_details_by_id: dict[str, dict[str, str]] = {}
         persona_details_by_name: dict[str, dict[str, str]] = {}
-        persona_objects_by_id = {}
-        persona_objects_by_name = {}
         persona_ids = {crawl["persona_id"] for crawl in active_crawls_list if crawl["persona_id"]}
         persona_names = {"Default"} if any(not crawl["persona_id"] for crawl in active_crawls_list) else set()
         if persona_ids or persona_names:
             from archivebox.personas.models import Persona
 
-            for persona in Persona.objects.filter(Q(id__in=persona_ids) | Q(name__in=persona_names)).only("id", "name", "config"):
+            for persona in Persona.objects.filter(Q(id__in=persona_ids) | Q(name__in=persona_names)).only("id", "name"):
                 persona_details = {
                     "name": persona.name,
                     "admin_url": f"/admin/personas/persona/{persona.pk}/change/",
                 }
                 persona_details_by_id[str(persona.id)] = persona_details
                 persona_details_by_name[persona.name] = persona_details
-                persona_objects_by_id[str(persona.id)] = persona
-                persona_objects_by_name[persona.name] = persona
         active_crawl_ids = [crawl["id"] for crawl in active_crawls_list]
-        active_crawl_objects = {}
-        if active_crawl_ids:
-            for crawl_obj in Crawl.objects.filter(id__in=active_crawl_ids).select_related("created_by", "persona"):
-                crawl_obj._runtime_config = request_config
-                active_crawl_objects[str(crawl_obj.id)] = crawl_obj
         snapshot_counts_by_crawl: dict[str, dict[str, int]] = {str(crawl_id): {} for crawl_id in active_crawl_ids}
-        cancelled_snapshot_counts_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in active_crawl_ids}
-        crawl_output_sizes_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in active_crawl_ids}
         queued_snapshot_overflow_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in active_crawl_ids}
         active_snapshot_scope = snapshot_scope.filter(crawl_id__in=active_crawl_ids)
         if active_crawl_ids:
-            for row in active_snapshot_scope.values("crawl_id", "status").annotate(count=Count("id")):
-                snapshot_counts_by_crawl.setdefault(str(row["crawl_id"]), {})[row["status"]] = row["count"]
-
-            for row in (
-                active_snapshot_scope.filter(status=Snapshot.StatusChoices.SEALED, downloaded_at__isnull=True)
-                .values("crawl_id")
-                .annotate(count=Count("id"))
-            ):
-                cancelled_snapshot_counts_by_crawl[str(row["crawl_id"])] = row["count"]
-
-            for row in (
-                active_snapshot_scope.filter(
-                    status=Snapshot.StatusChoices.SEALED,
-                )
-                .values("crawl_id")
-                .annotate(size=Sum("output_size"))
-            ):
-                crawl_output_sizes_by_crawl[str(row["crawl_id"])] = int(row["size"] or 0)
+            # The existing (crawl, status, modified_at) index covers these counts.
+            # Byte totals and cancellation details require table reads, so sample
+            # those separately and share cached values between viewer tabs.
+            for row in active_snapshot_scope.order_by().values("crawl_id", "status").annotate(count=Count("*")):
+                crawl_id = str(row["crawl_id"])
+                snapshot_counts_by_crawl[crawl_id][row["status"]] = row["count"]
 
         crawl_process_pids: dict[str, int] = {}
         snapshot_process_pids: dict[str, int] = {}
@@ -588,7 +563,14 @@ def live_progress_view(request, *, authorized_snapshot=None):
             completed_snapshots = crawl_snapshot_counts.get(Snapshot.StatusChoices.SEALED, 0)
             started_snapshots = crawl_snapshot_counts.get(Snapshot.StatusChoices.STARTED, 0)
             pending_snapshots = crawl_snapshot_counts.get(Snapshot.StatusChoices.QUEUED, 0)
-            cancelled_snapshots = cancelled_snapshot_counts_by_crawl.get(crawl_id, 0)
+            from archivebox.progressmonitor.collection import crawl_summary
+
+            totals = (
+                crawl_summary(crawl_id, snapshot_id=scoped_snapshot.pk if scoped_snapshot is not None else None)
+                if completed_snapshots
+                else {"snapshots": 0, "cancelled": 0, "bytes": 0}
+            )
+            cancelled_snapshots = totals["cancelled"] if totals is not None else None
 
             # Count URLs in the crawl (for when snapshots haven't been created yet)
             urls_count = 0
@@ -607,7 +589,7 @@ def live_progress_view(request, *, authorized_snapshot=None):
             crawl_setup_completed = sum(1 for item in crawl_setup_plugins if item.get("status") == "succeeded")
             crawl_setup_failed = sum(1 for item in crawl_setup_plugins if item.get("status") == "failed")
             crawl_setup_pending = sum(1 for item in crawl_setup_plugins if item.get("status") == "queued")
-            crawl_screencast_url = screencast_frame_url(crawl_id, active_crawl_objects[crawl_id].output_dir)
+            crawl_screencast_url = screencast_frame_url(crawl_id)
             crawl_screencast_link = f"/admin/crawls/crawl/{crawl_id.replace('-', '')}/change/" if crawl_screencast_url else ""
 
             # Get active snapshots for this crawl (already prefetched)
@@ -661,7 +643,7 @@ def live_progress_view(request, *, authorized_snapshot=None):
                     snapshot_preview_url = snapshot_favicon_url
 
                 if snapshot["status"] == Snapshot.StatusChoices.STARTED:
-                    snapshot_screencast_url = screencast_frame_url(crawl_id, active_crawl_objects[crawl_id].output_dir)
+                    snapshot_screencast_url = screencast_frame_url(crawl_id)
                     snapshot_screencast_link = snapshot_view_url(snapshot) if snapshot_screencast_url else ""
 
                 def plugin_sort_key(ar):
@@ -809,18 +791,18 @@ def live_progress_view(request, *, authorized_snapshot=None):
             persona_details = persona_details_by_id.get(str(crawl["persona_id"])) if crawl["persona_id"] else None
             persona_name = persona_details["name"] if persona_details else "Default"
             persona_details = persona_details or persona_details_by_name.get(persona_name)
-            crawl_output_size = crawl_output_sizes_by_crawl.get(crawl_id, 0)
-            avg_snapshot_size = int(crawl_output_size / completed_snapshots) if completed_snapshots else 0
-            crawl_obj = active_crawl_objects[crawl_id]
-            effective_crawl_config = get_config(crawl=crawl_obj, resolve_plugins=False)
-            max_urls = int(effective_crawl_config.CRAWL_MAX_URLS or 0)
-            crawl_max_size = int(effective_crawl_config.CRAWL_MAX_SIZE or 0)
-            crawl_timeout = int(effective_crawl_config.CRAWL_TIMEOUT or 0)
-            snapshot_max_size = int(effective_crawl_config.SNAPSHOT_MAX_SIZE or 0)
+            crawl_output_size = totals["bytes"] if totals is not None else None
+            avg_snapshot_size = int(totals["bytes"] / max(totals["snapshots"], 1)) if totals is not None else None
+            # These crawl-scope limits are frozen in Crawl.config. Rendering
+            # them must not resolve personas, probe binaries, or reread config.
+            max_urls = int(crawl["config"].get("CRAWL_MAX_URLS", request_config.CRAWL_MAX_URLS) or 0)
+            crawl_max_size = int(crawl["config"].get("CRAWL_MAX_SIZE", request_config.CRAWL_MAX_SIZE) or 0)
+            crawl_timeout = int(crawl["config"].get("CRAWL_TIMEOUT", request_config.CRAWL_TIMEOUT) or 0)
+            snapshot_max_size = int(crawl["config"].get("SNAPSHOT_MAX_SIZE", request_config.SNAPSHOT_MAX_SIZE) or 0)
 
             # Check if retry_at is in the future (would prevent worker from claiming)
             retry_at_future = crawl["retry_at"] > now if crawl["retry_at"] else False
-            is_paused = crawl_obj.is_paused
+            is_paused = crawl["status"] == Crawl.StatusChoices.PAUSED
             seconds_until_retry = (
                 0 if is_paused else int((crawl["retry_at"] - now).total_seconds()) if crawl["retry_at"] and retry_at_future else 0
             )
@@ -860,8 +842,8 @@ def live_progress_view(request, *, authorized_snapshot=None):
                     "max_snapshot_size_display": printable_filesize(snapshot_max_size) if snapshot_max_size else "unlimited",
                     "crawl_output_size": crawl_output_size,
                     "avg_snapshot_size": avg_snapshot_size,
-                    "crawl_output_size_display": printable_filesize(crawl_output_size) if crawl_output_size else "0 B",
-                    "avg_snapshot_size_display": printable_filesize(avg_snapshot_size) if avg_snapshot_size else "0 B",
+                    "crawl_output_size_display": printable_filesize(crawl_output_size or 0) if totals is not None else "…",
+                    "avg_snapshot_size_display": printable_filesize(avg_snapshot_size or 0) if totals is not None else "…",
                     "tags": crawl_tags,
                     "urls_count": urls_count,
                     "total_snapshots": total_snapshots,

archivebox/search/backends.py

@@ -12,8 +12,11 @@ _search_backends_cache: dict | None = None
 
 def search_backend_command_env(config: dict[str, Any] | None = None, **config_kwargs: Any) -> dict[str, str]:
     """Serialize resolved application config for a standalone plugin command."""
+    from abx_dl.models import PluginEnv
+    from abxpkg import BinProvider
+
     config = config or get_config(**config_kwargs)
-    env = os.environ.copy()
+    env: dict[str, str] = {}
     for key, value in config.items():
         key = str(key)
         if value is None:
@@ -24,7 +27,9 @@ def search_backend_command_env(config: dict[str, Any] | None = None, **config_kw
             env[key] = json.dumps(value)
         elif isinstance(value, (str, int, float, os.PathLike)):
             env[key] = str(value)
-    return env
+    # Use the same runtime bootstrap as hooks so the installation's abxpkg
+    # entrypoint remains available even without an activated virtualenv.
+    return BinProvider.build_exec_env(base_env=PluginEnv(**os.environ).to_env(), extra_env=env)
 
 
 def normalize_search_backend_name(backend_name: str | None) -> str:

archivebox/services/archive_result_service.py

@@ -210,6 +210,12 @@ def _save_archiveresult_event_to_db(
 
     with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.diff_fields"):
         update_fields = []
+        # This runner already loaded the Snapshot/Crawl for the event. Resolve
+        # retention here without extra parent reads so new results do not wait
+        # for the bounded legacy-repair sweep to traverse historical rows.
+        result.snapshot = snapshot
+        if result.set_delete_at_from_config():
+            update_fields.append("delete_at")
         for field, value in defaults.items():
             if result.__dict__[field] != value:
                 setattr(result, field, value)

archivebox/services/resource_admission.py

@@ -2,9 +2,14 @@
 
 import math
 import os
+import re
+import stat
+import sys
 import time
 from pathlib import Path
 
+import psutil
+
 
 RESOURCE_RECHECK_SECONDS = 2.0
 # Scheduling policy, not a minimum-RAM claim: stop increasing extraction
@@ -12,6 +17,65 @@ RESOURCE_RECHECK_SECONDS = 2.0
 # stalls. Require both recent and rolling pressure so transient reclaim does
 # not indefinitely starve otherwise viable work on small hosts.
 MEMORY_FULL_STALL_PERCENT = 10.0
+ZRAM_DEVICE_NAME = re.compile(r"zram\d+")
+
+
+def zram_swap_identifiers(sys_block: Path = Path("/sys/block")) -> set[str]:
+    """Return zram block names and major:minor IDs known to sysfs."""
+    identifiers = set()
+    for device in sys_block.glob("zram*"):
+        if ZRAM_DEVICE_NAME.fullmatch(device.name):
+            identifiers.add(device.name)
+            try:
+                identifiers.add((device / "dev").read_text().strip())
+            except OSError:
+                pass
+    return identifiers
+
+
+def swap_source_identifiers(filename: str) -> set[str]:
+    """Identify a swap source by its visible name and, for block devices, ID."""
+    identifiers = {Path(filename).name}
+    try:
+        source = os.stat(filename)
+        if stat.S_ISBLK(source.st_mode):
+            identifiers.add(f"{os.major(source.st_rdev)}:{os.minor(source.st_rdev)}")
+    except OSError:
+        pass
+    return identifiers
+
+
+def disk_backed_swap_free_bytes(swaps: str, zram_identifiers: set[str]) -> int:
+    """Sum free active swap except zram, whose capacity is already backed by RAM."""
+    lines = swaps.splitlines()
+    free_kib = 0
+    for line in lines[1:]:  # /proc/swaps starts with a column header
+        fields = line.split()
+        if len(fields) < 5:
+            continue
+        filename, _, size_kib, used_kib, _ = fields[:5]
+        source_ids = swap_source_identifiers(filename)
+        if source_ids & zram_identifiers or any(ZRAM_DEVICE_NAME.fullmatch(name) for name in source_ids):
+            continue
+        try:
+            free_kib += max(0, int(size_kib) - int(used_kib))
+        except ValueError:
+            continue
+    return free_kib * 1024
+
+
+def host_swap_free_bytes(meminfo_swap_free: int) -> int:
+    """Use only independently backed swap as extra capacity on Linux."""
+    if not sys.platform.startswith("linux"):
+        return psutil.swap_memory().free
+    try:
+        swaps = Path("/proc/swaps").read_text()
+    except OSError:
+        # If Linux swap sources cannot be classified, fail closed rather than
+        # treating potentially RAM-backed zram capacity as extra headroom.
+        return 0
+    disk_free = disk_backed_swap_free_bytes(swaps, zram_swap_identifiers())
+    return min(max(0, meminfo_swap_free), disk_free)
 
 
 class ResourceAdmission:
@@ -44,10 +108,73 @@ class ResourceAdmission:
                     cpus = min(cpus, max(1, math.ceil(int(quota) / int(period))))
             except (OSError, ValueError, ZeroDivisionError):
                 continue
-        # CPU quota limits parallel admission, not forward progress. A host
-        # with a fractional CPU must still be able to run one snapshot.
+        # CPU quota bounds parallel work; measured memory admission is done
+        # only after the first complete snapshot establishes its actual cost.
         return max(1, min(configured, cpus))
 
+    def additional_snapshot_slots(self, configured: int, active: int, observed_cost: int | None) -> int:
+        if not observed_cost:
+            # Stage one complete real capture before spending memory on parallel
+            # hook trees. A zero-cost/failed probe cannot justify fanout.
+            return 1 if active == 0 and configured > 0 else 0
+        headroom = self.memory_headroom()
+        return self.slots_for_headroom(configured, active, observed_cost, headroom[1] if headroom else 0)
+
+    @staticmethod
+    def slots_for_headroom(configured: int, active: int, observed_cost: int, available: int) -> int:
+        slots = max(0, configured - active)
+        if observed_cost > 0:
+            # Active captures can still grow to the measured peak. Reserve for
+            # that growth before promising the remaining headroom to new work.
+            unpromised = max(0, available - active * observed_cost)
+            slots = min(slots, unpromised // observed_cost)
+        return max(1, slots) if active == 0 and configured else slots
+
+    def memory_headroom(self) -> tuple[int, int] | None:
+        """Return workload RAM+swap usage and effective available headroom."""
+        try:
+            meminfo = {
+                name: int(value.split()[0]) * 1024
+                for name, value in (line.split(":", 1) for line in Path("/proc/meminfo").read_text().splitlines() if ":" in line)
+            }
+            host_swap_free = host_swap_free_bytes(meminfo.get("SwapFree", 0))
+            host_available = meminfo["MemAvailable"] + host_swap_free
+        except (OSError, ValueError, KeyError):
+            host_swap_free = 0 if sys.platform.startswith("linux") else psutil.swap_memory().free
+            host_available = psutil.virtual_memory().available + host_swap_free
+
+        available = host_available
+        used_bytes = None
+        for path in self.cgroup_paths():
+            try:
+                memory_max = (path / "memory.max").read_text().strip()
+                if memory_max == "max":
+                    continue
+                current = int((path / "memory.current").read_text())
+                swap_max = (path / "memory.swap.max").read_text().strip()
+                swap_current = int((path / "memory.swap.current").read_text())
+                swap_room = host_swap_free if swap_max == "max" else min(host_swap_free, max(0, int(swap_max) - swap_current))
+                available = min(available, max(0, int(memory_max) - current) + swap_room)
+                if used_bytes is None:
+                    used_bytes = current + swap_current
+            except (OSError, ValueError, KeyError):
+                continue
+        if used_bytes is None:
+            # Bare-metal and non-Linux installs have no finite cgroup. Count
+            # the real runner process tree rather than the entire host.
+            try:
+                process = psutil.Process()
+                processes = [process, *process.children(recursive=True)]
+            except (psutil.Error, OSError):
+                return None
+            used_bytes = 0
+            for child in processes:
+                try:
+                    used_bytes += child.memory_info().rss
+                except (psutil.Error, OSError):
+                    continue
+        return used_bytes, available
+
     def observe_memory_stalls(self, source: str, total: int, avg10: float, sampled_at: float) -> bool:
         previous = self._memory_stalls.get(source)
         self._memory_stalls[source] = (total, sampled_at)

archivebox/services/runner.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import asyncio
 import contextvars
 import json
+import logging
 import os
 import signal
 import shutil
@@ -64,6 +65,7 @@ from archivebox.services.resource_admission import RESOURCE_RECHECK_SECONDS, res
 from archivebox.config.common import (
     ArchiveBoxBaseConfig,
     normalize_runtime_config,
+    _is_cookie_file_config_key,
     _plugin_enabled_config_keys,
 )
 from archivebox.misc.db import run_db_analyze_batch
@@ -223,6 +225,7 @@ class CrawlRunner:
         self.snapshot_service = SnapshotService(
             self.bus,
             crawl_id=str(crawl.id),
+            interrupted=self.execution_interrupted,
         )
         HookArchiveResultService(self.bus, emit_jsonl=False)
         ArchiveResultService(self.bus)
@@ -232,6 +235,10 @@ class CrawlRunner:
         self.snapshot_tasks: dict[str, asyncio.Task[None]] = {}
         self.snapshot_semaphore = asyncio.Semaphore(1)
         self.max_concurrent_snapshots = 1
+        self._warmup_snapshot_id: str | None = None
+        self._memory_baseline: int | None = None
+        self._memory_peak_cost = 0
+        self._observed_snapshot_cost: int | None = None
         self._resource_deferred = False
         self._initial_snapshot_admitted = False
         self.persona = None
@@ -253,6 +260,10 @@ class CrawlRunner:
         self._signal_abort_requested = False
         self._last_lease_heartbeat_at = 0.0
 
+    def execution_interrupted(self) -> bool:
+        """Owner loss preserves work; explicit user abort keeps its sealing policy."""
+        return self._signal_abort_requested and not self._user_aborted
+
     async def on_CrawlAbortEvent(self, event: CrawlAbortEvent) -> None:
         self._signal_abort_requested = True
         # An abort can arrive while the prompt transport is being established,
@@ -498,10 +509,15 @@ class CrawlRunner:
         if task is not None and not task.done():
             return
         current_event = crawl_start_event or get_current_event()
+        if not isinstance(current_event, CrawlStartEvent) and in_handler_context():
+            return
+        if self._observed_snapshot_cost is None and self._warmup_snapshot_id is None:
+            observation = resource_admission.memory_headroom()
+            self._memory_baseline = observation[0] if observation is not None else None
+            self._memory_peak_cost = 0
+            self._warmup_snapshot_id = snapshot_id
         if isinstance(current_event, CrawlStartEvent):
             task = asyncio.create_task(self.run_snapshot(snapshot_id, current_event), context=_runner_task_context())
-        elif in_handler_context():
-            return
         else:
             task = asyncio.create_task(self.run_snapshot(snapshot_id), context=_runner_task_context())
         self._initial_snapshot_admitted = False
@@ -530,6 +546,7 @@ class CrawlRunner:
                 if task.done():
                     if self.snapshot_tasks.get(snapshot_id) is task:
                         self.snapshot_tasks.pop(snapshot_id, None)
+                    self._finish_warmup_snapshot(snapshot_id)
                     try:
                         task.result()
                     except asyncio.CancelledError as err:
@@ -554,13 +571,16 @@ class CrawlRunner:
                     return
                 continue
             await self.heartbeat_active_leases()
-            done, _pending = await asyncio.wait(pending_tasks, timeout=10.0, return_when=asyncio.FIRST_COMPLETED)
+            self._observe_snapshot_memory()
+            done, _pending = await asyncio.wait(pending_tasks, timeout=RESOURCE_RECHECK_SECONDS, return_when=asyncio.FIRST_COMPLETED)
+            self._observe_snapshot_memory()
             if not done:
                 continue
             for task in done:
                 for snapshot_id, tracked_task in list(self.snapshot_tasks.items()):
                     if tracked_task is task:
                         self.snapshot_tasks.pop(snapshot_id, None)
+                        self._finish_warmup_snapshot(snapshot_id)
                         break
                 try:
                     task.result()
@@ -578,6 +598,41 @@ class CrawlRunner:
             if not stop_scheduling and enqueue_projected:
                 await self.enqueue_pending_snapshots_from_projection()
 
+    def _observe_snapshot_memory(self) -> None:
+        if self._memory_baseline is None:
+            return
+        observation = resource_admission.memory_headroom()
+        if observation is not None:
+            used, _available = observation
+            # Only attribute this workload's growth to a capture. Other jobs on
+            # the host reduce available headroom, but are not a per-capture cost
+            # to multiply again for every snapshot we want to admit.
+            self._memory_peak_cost = max(
+                self._memory_peak_cost,
+                used - self._memory_baseline,
+            )
+            if self._observed_snapshot_cost is not None:
+                self._observed_snapshot_cost = max(self._observed_snapshot_cost, self._memory_peak_cost)
+
+    def _finish_warmup_snapshot(self, snapshot_id: str) -> None:
+        if snapshot_id != self._warmup_snapshot_id or self._observed_snapshot_cost is not None:
+            return
+        self._observe_snapshot_memory()
+        if self._memory_peak_cost > 0:
+            self._observed_snapshot_cost = self._memory_peak_cost
+            headroom = resource_admission.memory_headroom()
+            logging.getLogger(__name__).info(
+                "Snapshot admission: concurrency limit=%s, observed memory growth=%s bytes, available=%s bytes",
+                self.max_concurrent_snapshots,
+                self._observed_snapshot_cost,
+                headroom[1] if headroom else None,
+            )
+        else:
+            # No observed cost cannot justify parallel admission. Probe the
+            # next real snapshot while the queue keeps moving one at a time.
+            self._memory_baseline = None
+        self._warmup_snapshot_id = None
+
     async def heartbeat_active_leases(self) -> None:
         # These are resumable work-item leases, not orchestrator-election
         # heartbeats. Each update is a short autocommit statement; network and
@@ -645,7 +700,11 @@ class CrawlRunner:
         self.max_concurrent_snapshots = resource_admission.snapshot_limit(int(config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"]))
 
         active_snapshot_ids = [snapshot_id for snapshot_id, task in self.snapshot_tasks.items() if not task.done()]
-        available_slots = max(0, self.max_concurrent_snapshots - len(active_snapshot_ids))
+        available_slots = resource_admission.additional_snapshot_slots(
+            self.max_concurrent_snapshots,
+            len(active_snapshot_ids),
+            self._observed_snapshot_cost,
+        )
         if available_slots <= 0:
             return
         pending_snapshot_ids = await sync_to_async(
@@ -669,6 +728,9 @@ class CrawlRunner:
         from archivebox.machine.models import Machine, NetworkInterface, Process
 
         self.primary_url = self.crawl.get_urls_list()[0] if self.crawl.get_urls_list() else ""
+        # Validate saved binary paths at the execution boundary, including
+        # config edited since this process first registered its Machine.
+        machine = Machine._sanitize_config(Machine.current())
         current_iface = NetworkInterface.current(refresh=not self.allow_maintenance_on_inactive_crawl)
         current_process = Process.current()
         if current_process.iface_id != current_iface.id or current_process.machine_id != current_iface.machine_id:
@@ -677,8 +739,16 @@ class CrawlRunner:
             current_process.save(update_fields=["iface", "machine", "modified_at"])
         self.persona = self.crawl.resolve_persona()
         self.base_config = get_config(crawl=self.crawl, overrides=self.config_overrides)
-        self.derived_config = dict(Machine.current().config or {})
+        # The merged user payload owns auth paths and binary selections. The
+        # install phase publishes resolved binaries; stale machine paths must
+        # not override explicit runtime settings before that phase runs.
+        self.derived_config = {
+            key: value
+            for key, value in (machine.config or {}).items()
+            if not _is_cookie_file_config_key(key) and (not key.endswith("_BINARY") or value == self.base_config.get(key))
+        }
         self.crawl_output_dir = str(self.crawl.output_dir)
+        Path(self.crawl_output_dir).mkdir(parents=True, exist_ok=True)
         if self.persona:
             self.base_config.update(
                 self.persona.prepare_runtime_for_crawl(
@@ -887,6 +957,7 @@ class CrawlRunner:
                     },
                 )
         snapshot_output_dir = str(snapshot.output_dir)
+        Path(snapshot_output_dir).mkdir(parents=True, exist_ok=True)
         tags = snapshot.tags_str()
         config = self.base_config.for_crawl_runtime(
             crawl=snapshot.crawl,
@@ -997,7 +1068,8 @@ class CrawlRunner:
             if self.initial_snapshot_ids is not None:
                 remaining_snapshot_ids = iter(snapshot_ids)
                 while True:
-                    batch = list(zip(range(self.max_concurrent_snapshots), remaining_snapshot_ids, strict=False))
+                    slots = resource_admission.additional_snapshot_slots(self.max_concurrent_snapshots, 0, self._observed_snapshot_cost)
+                    batch = list(zip(range(slots), remaining_snapshot_ids, strict=False))
                     if not batch:
                         return
                     for _slot, snapshot_id in batch:
@@ -1008,7 +1080,8 @@ class CrawlRunner:
                         await self.enqueue_snapshot(snapshot_id)
                     await self.wait_for_snapshot_tasks(enqueue_projected=False)
             else:
-                for snapshot_id in snapshot_ids[: self.max_concurrent_snapshots]:
+                slots = resource_admission.additional_snapshot_slots(self.max_concurrent_snapshots, 0, self._observed_snapshot_cost)
+                for snapshot_id in snapshot_ids[:slots]:
                     if await self.crawl_is_cancelled():
                         break
                     if await self.crawl_is_paused():
@@ -1073,18 +1146,17 @@ class CrawlRunner:
             finally:
                 cancel_watcher.cancel()
                 await asyncio.gather(cancel_watcher, return_exceptions=True)
-            completed_event = event.emit(
-                CrawlCompletedEvent(
-                    url=snapshot["url"],
-                    snapshot_id=snapshot["id"],
-                    output_dir=str(output_dir),
-                ),
-            )
-            # Same signal lifecycle as CrawlCleanupEvent above: completion is a
-            # normal bus event unless the interpreter is already unwinding from
-            # SIGINT/SIGTERM/SIGHUP, where synchronous bus delivery is no
-            # longer a dependable shutdown primitive.
-            if not self._signal_abort_requested:
+            # emit() schedules handlers immediately; guarding only the await
+            # still lets completion projectors run during takeover. An aborted
+            # execution owner must leave durable work for its replacement.
+            if not self.execution_interrupted():
+                completed_event = event.emit(
+                    CrawlCompletedEvent(
+                        url=snapshot["url"],
+                        snapshot_id=snapshot["id"],
+                        output_dir=str(output_dir),
+                    ),
+                )
                 await _run_event_now(completed_event, CrawlCompletedEvent.model_fields["event_timeout"].default)
 
         on_archivebox_CrawlStartEvent.__name__ = "on_archivebox_CrawlStartEvent__run_snapshots"
@@ -1183,6 +1255,7 @@ class CrawlRunner:
             user_config_event = MachineEvent(config=config, config_type="user")
             user_config_event.event_parent_id = crawl_start_event.event_id
             await self.bus.emit(user_config_event).now()
+            derived_config_event = None
             if derived_config:
                 derived_config_event = MachineEvent(config=derived_config, config_type="derived")
                 derived_config_event.event_parent_id = crawl_start_event.event_id
@@ -1198,6 +1271,7 @@ class CrawlRunner:
                 snapshot_cleanup_phase_timeout=snapshot_phase_timeout,
                 abort_requested=self.crawl_is_cancelled,
             )
+            projected = False
             try:
                 snapshot_event = SnapshotEvent(
                     url=snapshot["url"],
@@ -1221,6 +1295,9 @@ class CrawlRunner:
                     raise RuntimeError(f"Snapshot {snapshot_id} did not complete")
                 await completed_snapshot.wait(timeout=snapshot_phase_timeout)
                 await completed_snapshot.event_results_list()
+                projected = True
+                if self.execution_interrupted():
+                    return
                 if snapshot["status"] == "sealed":
                     await sync_to_async(run_snapshot_maintenance, thread_sensitive=True)(snapshot_id, output_dir=output_dir)
                     return
@@ -1242,6 +1319,16 @@ class CrawlRunner:
                 await sync_to_async(_seal_when_last_snapshot_finished, thread_sensitive=True)()
             finally:
                 snapshot_service.close()
+                # Projection and snapshot cleanup have finished. Historical
+                # capture results now live in the database; only active work
+                # needs its event ancestry available to the downloader.
+                expired = [snapshot_event, user_config_event] if projected else []
+                if projected and derived_config_event is not None:
+                    expired.append(derived_config_event)
+                if projected:
+                    expired.extend(await self.bus.filter("*", child_of=snapshot_event, past=True, future=False))
+                for event in expired:
+                    event.event_ttl = 0
 
     def seal_snapshot_due_to_limit(self, snapshot_id: str) -> None:
         from archivebox.core.models import Snapshot
@@ -1471,7 +1558,9 @@ def _run_due_crawl_locked(crawl, *, lock_seconds: int, interactive_interrupts: b
             # avoids hot-looping on the parent while child work is ready without
             # resurrecting a user cancellation that sealed the crawl after
             # selection.
-            crawl.safe_update(
+            if crawl.status != crawl.StatusChoices.STARTED:
+                return False
+            return crawl.safe_update(
                 {
                     "status": crawl.StatusChoices.STARTED,
                     "retry_at": now + timedelta(seconds=ACTIVE_STATE_LEASE_SECONDS),
@@ -1480,7 +1569,6 @@ def _run_due_crawl_locked(crawl, *, lock_seconds: int, interactive_interrupts: b
                 refresh=False,
                 extra_filter={"status": crawl.StatusChoices.STARTED},
             )
-            return True
         if snapshot_count and not due_active_snapshots:
             if crawl.is_finished():
                 if not crawl.claim_processing_lock(lock_seconds=lock_seconds):
@@ -1605,7 +1693,9 @@ def _run_due_snapshot_locked(snapshot, *, lock_seconds: int, interactive_interru
         # every URL, even though crawl isolation owns one browser for the whole
         # crawl. The crawl lifecycle lock already serializes all of its child
         # work, so let one runner drain the due siblings and discoveries.
-        if any(process.is_running for process in snapshot.process_set.filter(status="running").iterator()):
+        if any(
+            process.is_running for process in snapshot.process_set.filter(status="running").order_by("pk").paged_iterator(chunk_size=100)
+        ):
             snapshot.update_and_requeue(retry_at=timezone.now() + timedelta(seconds=lock_seconds))
             return True
         if snapshot.fs_migration_needed:
@@ -1639,7 +1729,7 @@ def _run_due_snapshot_locked(snapshot, *, lock_seconds: int, interactive_interru
     if not snapshot.claim_processing_lock(lock_seconds=lock_seconds):
         return False
     snapshot.refresh_from_db()
-    if any(process.is_running for process in snapshot.process_set.filter(status="running").iterator()):
+    if any(process.is_running for process in snapshot.process_set.filter(status="running").order_by("pk").paged_iterator(chunk_size=100)):
         # The Snapshot lease may have expired while an abx-dl hook process is
         # still alive. Preserve the snapshot-level ownership boundary and do
         # not launch a second sequence; ArchiveResult status is irrelevant.
@@ -1838,12 +1928,11 @@ def _run_due_crawl_status(
     due_crawl = Crawl.objects.filter(id=due_crawl_id).first()
     if due_crawl is None:
         return True
-    run_due_crawl(
+    return run_due_crawl(
         due_crawl,
         lock_seconds=lock_seconds,
         interactive_interrupts=interactive_interrupts,
     )
-    return True
 
 
 def _run_due_snapshot_query(
@@ -1897,8 +1986,7 @@ def _run_due_binary() -> bool:
     due_binary = Binary.objects.filter(id=due_binary_id).first()
     if due_binary is None:
         return True
-    run_due_binary(due_binary, lock_seconds=60)
-    return True
+    return run_due_binary(due_binary, lock_seconds=60)
 
 
 def run_pending_crawls(
@@ -1918,14 +2006,36 @@ def run_pending_crawls(
     last_recovery_at = 0.0
     last_retention_at = 0.0
     last_retention_repair_at = 0.0
+    retention_backfill_cursors: dict = {}
+    deletion_thread: threading.Thread | None = None
     last_analyze_at = 0.0
     analyze_queue: list[str] | None = None
     analyze_sweep_started_at = 0.0
     orchestrator_started_at = time.monotonic()
+
+    def delete_in_background() -> None:
+        from django.db import connections
+
+        try:
+            Snapshot.delete_requested(batch_size=100)
+        finally:
+            connections.close_all()
+
     while True:
         raise_if_shutdown_requested()
         now_monotonic = time.monotonic()
         if crawl_id is None and now_monotonic - last_retention_at >= (60.0 if daemon else 1.0):
+            if daemon:
+                # At most one batch can be doing filesystem I/O. A stuck mount
+                # must not block capture scheduling or runner shutdown; rows
+                # remain pending until cleanup succeeds, including on restart.
+                if (deletion_thread is None or not deletion_thread.is_alive()) and Snapshot.objects.filter(
+                    status=Snapshot.DELETING_STATE,
+                ).exists():
+                    deletion_thread = threading.Thread(target=delete_in_background, name="snapshot-delete", daemon=True)
+                    deletion_thread.start()
+            else:
+                Snapshot.delete_requested(batch_size=100)
             for model in (ArchiveResult, Snapshot, Crawl, Process):
                 # Keep the tight scheduler loop anchored on indexed delete_at
                 # columns only. Backfilling missing delete_at values has to read
@@ -2093,7 +2203,7 @@ def run_pending_crawls(
                 # delete_at in the plugin-result hot path. Running it here keeps
                 # DELETE_AFTER resolution fresh without making every hook event
                 # load parent Snapshot/Crawl config.
-                model.delete_expired(batch_size=100, backfill_missing=True)
+                model.delete_expired(batch_size=100, backfill_missing=True, backfill_cursors=retention_backfill_cursors)
             last_retention_repair_at = now_monotonic
 
         if daemon:
@@ -2109,12 +2219,10 @@ def run_pending_crawls(
             # snapshot detail page out to ~500ms. Refresh stats at most once per
             # 24hr while the queue is idle, and only after the orchestrator has
             # been alive for at least an hour so short server boots / one-off work
-            # never pay the cost. The sweep is batched one table per idle tick;
-            # individual table ANALYZE statements abort after 2min (progress
-            # handler) and the whole sweep is hard-capped at 5min so a
-            # pathological table cannot wedge maintenance forever. Any failure
-            # inside the maintenance hook is swallowed — orchestrator must never
-            # be taken down by stats refresh.
+            # never pay the cost. Each idle tick samples a bounded number of
+            # index entries for one table; never scan the full history while
+            # holding ANALYZE's write transaction. Optional stats refresh must
+            # not prevent queued captures.
             try:
                 if (
                     analyze_queue is None

archivebox/services/snapshot_service.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import asyncio
+from collections.abc import Callable
 from pathlib import Path
 
 from asgiref.sync import sync_to_async
@@ -73,10 +74,11 @@ def finalize_completed_snapshot(
             modified_at=timezone.now(),
         )
 
-    # SnapshotCompletedEvent is abx-dl's authoritative signal that the complete
-    # snapshot hook sequence (including cleanup) finished. ArchiveResult rows
-    # are projections of that work, never prerequisites used to decide whether
-    # the Snapshot may seal.
+    # SnapshotCompletedEvent finishes an attempt, including abort cleanup. An
+    # explicit user abort deliberately seals that point-in-time capture; the
+    # runner's interrupted-owner guard below keeps takeover work resumable.
+    # ArchiveResult rows are projections of that work, never prerequisites used
+    # to decide whether the Snapshot may seal.
     if not was_sealed and snapshot.status == Snapshot.StatusChoices.STARTED and snapshot.retry_at == owned_retry_at:
         snapshot.seal()
         snapshot.refresh_from_db()
@@ -120,8 +122,9 @@ class SnapshotService(BaseService):
     LISTENS_TO = [SnapshotEvent, SnapshotCompletedEvent]
     EMITS = []
 
-    def __init__(self, bus, *, crawl_id: str):
+    def __init__(self, bus, *, crawl_id: str, interrupted: Callable[[], bool] | None = None):
         self.crawl_id = crawl_id
+        self.interrupted = interrupted
         self._run_ownership: dict[str, tuple[str, object, bool, list[str]]] = {}
         self._ownership_lock = asyncio.Lock()
         super().__init__(bus)
@@ -143,7 +146,7 @@ class SnapshotService(BaseService):
             elif snapshot.status not in (Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.SEALED):
                 return
             if snapshot.status == Snapshot.StatusChoices.STARTED:
-                await sync_to_async(snapshot.ensure_crawl_symlink, thread_sensitive=True)()
+                await sync_to_async(snapshot.reconcile_filesystem_links, thread_sensitive=True)()
             retry_plugins = [str(name).strip() for name in (snapshot.config or {}).get("RETRY_PLUGINS", []) if str(name).strip()]
             async with self._ownership_lock:
                 self._run_ownership[str(event.snapshot_id)] = (str(event.event_id), snapshot.retry_at, was_sealed, retry_plugins)
@@ -175,6 +178,15 @@ class SnapshotService(BaseService):
             if ownership is None or ownership[0] != str(event.event_parent_id):
                 return
             self._run_ownership.pop(snapshot_id, None)
+        # abx-dl also emits completion after abort cleanup, even before the
+        # first hook. Explicit user aborts seal a point-in-time capture so later
+        # work cannot silently mix observations taken days apart; reopening it
+        # is a separate user action. Takeover only replaces its execution owner.
+        # Consult the runner's synchronous signal state: an exiting owner cannot
+        # rely on another bus event arriving before this completion projector runs.
+        # Leave its durable lease for the next elected runner's startup recovery.
+        if self.interrupted is not None and self.interrupted():
+            return
         _, owned_retry_at, was_sealed, retry_plugins = ownership
         await sync_to_async(finalize_completed_snapshot, thread_sensitive=True)(
             event.snapshot_id,

archivebox/templates/admin/base.html

@@ -1527,6 +1527,9 @@
                 display: inline-flex;
                 flex-wrap: wrap;
                 align-items: center;
+                min-width: 0;
+                max-width: 100%;
+                box-sizing: border-box;
                 gap: 4px;
                 padding: 2px 4px;
                 background: transparent;
@@ -1540,15 +1543,27 @@
                 flex-wrap: wrap;
                 gap: 4px;
                 align-items: center;
+                min-width: 0;
+                max-width: 100%;
             }
 
             .tag-editor-inline .tag-pill {
+                min-width: 0;
+                max-width: min(160px, 100%);
+                box-sizing: border-box;
                 padding: 2px 6px 2px 8px;
                 font-size: 11px;
                 border-radius: 12px;
             }
 
+            .tag-editor-inline .tag-link {
+                min-width: 0;
+                overflow: hidden;
+                text-overflow: ellipsis;
+            }
+
             .tag-editor-inline .tag-remove-btn {
+                flex-shrink: 0;
                 width: 14px;
                 height: 14px;
                 font-size: 12px;

archivebox/templates/admin/snapshots_grid.html

@@ -79,6 +79,13 @@
   }
   .cards .card .card-tags span {
       display: inline-block;
+      max-width: min(160px, 100%);
+      box-sizing: border-box;
+      overflow: hidden;
+      text-overflow: ellipsis;
+      white-space: nowrap;
+      vertical-align: middle;
+      pointer-events: auto;
       padding: 2px 5px;
       border-radius: 5px;
       opacity: 0.95;
@@ -275,7 +282,7 @@
                 {% else %}
                     <img src="{% static 'spinner.gif' %}" onerror="this.style.display='none'" class="link-favicon" decoding="async"/>
                 {% endif %}
-                <span class="title-text">{{obj.title|default:'Pending...' }}</span>
+                <span class="title-text">{{ obj.title|default:obj.domain }}</span>
               </h4>
             </a>
           </div>
@@ -283,16 +290,16 @@
           <div class="card-media">
             <a href="{% snapshot_detail_url obj %}" class="card-thumbnail">
               {% snapshot_thumbnail obj admin=True width="100%" height="245px" %}
-            </a>
               {% if obj.tags_str %}
                 <div class="card-tags">
                   {% for tag in obj.tags_str|split:',' %}
-                    <span>
+                    <span title="{{tag}}">
                       {{tag}}
                     </span>
                   {% endfor %}
                 </div>
               {% endif %}
+            </a>
           </div>
           <div class="card-outputs">
             {% if obj.status == 'started' %}

archivebox/templates/core/public_index.html

@@ -198,21 +198,29 @@
     }
 
     .snapshot-preview-cell {
+        position: relative;
         width: 110px;
         min-width: 116px;
         vertical-align: middle;
     }
 
     .snapshot-preview-cell a {
+        position: absolute;
+        top: 10px;
+        bottom: 10px;
         display: inline-flex;
         align-items: center;
         justify-content: center;
         width: 102px;
-        height: 102px;
         line-height: 0;
         vertical-align: middle;
     }
 
+    .snapshot-preview-cell .snapshot-thumbnail {
+        width: 100% !important;
+        height: 100% !important;
+    }
+
     #content .public-snapshot-list .snapshot-preview-cell .snapshot-preview,
     #content .public-snapshot-list .snapshot-preview-cell img.snapshot-preview.screenshot {
         box-sizing: border-box;
@@ -330,9 +338,10 @@
     }
 
     .snapshot-tag {
-        display: inline-flex;
-        align-items: center;
-        max-width: 160px;
+        display: block;
+        min-width: 0;
+        max-width: min(160px, 100%);
+        box-sizing: border-box;
         padding: 2px 7px;
         border: 1px solid #bfdbfe;
         border-radius: 999px;
@@ -585,7 +594,8 @@
 
         .snapshot-preview-cell a {
             width: 82px;
-            height: 82px;
+            top: 7px;
+            bottom: 7px;
         }
 
         #content .public-snapshot-list .snapshot-preview-cell .snapshot-preview,
@@ -644,12 +654,19 @@
                 </span>
                 <input type="submit" value="Search">
             </form>
+            {% comment %}
+                Missing/stale totals are intentional on large collections: show rows
+                immediately and let a later normal reload pick up the cached count.
+                Do not fill this in with an extra count request or paginator.count:
+                the unfiltered paginator only knows the rows needed for navigation.
+                Use the optional public_* totals here and in the footer below.
+            {% endcomment %}
             <div class="public-snapshot-count">
                 {% if public_search_stream_pending %}
                     <span class="archivebox-search-stream-status" role="status" aria-live="polite">Searching...</span>
                 {% else %}
-                    {{ page_obj.start_index }}-{{ page_obj.end_index }} of {{ page_obj.paginator.count }}
-                    <span>(page {{ page_obj.number }} of {{ page_obj.paginator.num_pages }})</span>
+                    {{ page_obj.start_index }}-{{ page_obj.end_index }}{% if public_snapshot_count != None %} of {{ public_snapshot_count }}{% endif %}
+                    <span>(page {{ page_obj.number }}{% if public_page_count != None %} of {{ public_page_count }}{% endif %})</span>
                 {% endif %}
             </div>
         </div>
@@ -659,7 +676,7 @@
                     <tr>
                         <th class="snapshot-time">Saved</th>
                         <th class="snapshot-preview-cell">Preview</th>
-                        <th>Snapshot ({{ page_obj.paginator.count }})</th>
+                        <th>Snapshot{% if public_snapshot_count != None %} ({{ public_snapshot_count }}){% endif %}</th>
                         <th class="snapshot-tags-cell">Tags</th>
                         <th class="snapshot-status-cell">Status</th>
                         <th class="snapshot-files-cell">Files</th>
@@ -687,7 +704,7 @@
             </table>
         </div>
         <div class="public-pagination">
-            Showing {{ page_obj.start_index }}-{{ page_obj.end_index }} of {{ page_obj.paginator.count }} total
+            Showing {{ page_obj.start_index }}-{{ page_obj.end_index }}{% if public_snapshot_count != None %} of {{ public_snapshot_count }} total{% endif %}
             <br>
             <span class="step-links">
                 {% if page_obj.has_previous %}
@@ -696,14 +713,14 @@
                 {% endif %}
 
                 <span class="current">
-                    Page {{ page_obj.number }} of {{ page_obj.paginator.num_pages }}
+                    Page {{ page_obj.number }}{% if public_page_count != None %} of {{ public_page_count }}{% endif %}
                 </span>
 
                 {% if page_obj.has_next %}
                     <a href="?{% url_replace page=page_obj.next_page_number %}">next</a>
                 {% endif %}
-                {% if page_obj.has_next %}
-                    <a href="?{% url_replace page=page_obj.paginator.num_pages %}">last &raquo;</a>
+                {% if page_obj.has_next and public_page_count != None %}
+                    <a href="?{% url_replace page=public_page_count %}">last &raquo;</a>
                 {% endif %}
             </span>
         </div>

archivebox/templates/core/setup_wizard.html

@@ -35,7 +35,7 @@
       <div class="abx-question-options" id="archivebox-setup-dns-options">
         <label class="abx-question-option"><input type="radio" name="archivebox-dns-mode" value="localhost"> <span><b>⭐ Localhost</b><small>No setup needed. <code id="archivebox-setup-localhost-dns-example">*.archivebox.localhost</code> works directly in modern browsers with all features.</small></span></label>
         <label class="abx-question-option"><input type="radio" name="archivebox-dns-mode" value="single"> <span><b>⚠️ Single-domain DNS</b><small>Register a DNS record like <code>A archivebox.example.com</code> → <code>your server's IP</code>, or add an equivalent <code>/etc/hosts</code> entry pointing to your server. Some archived content that depends on JavaScript may not display properly in this mode. JavaScript still runs during capture, but viewing HTML captured by <code>wget</code>, <code>dom</code>, <code>responses</code>, <code>staticfile</code>, and similar plugins will not replay JavaScript unless wildcard DNS is used.</small></span></label>
-        <label class="abx-question-option"><input type="radio" name="archivebox-dns-mode" value="wildcard"> <span><b>⭐ Wildcard DNS</b><small>Point both the bare <code>BASE_URL</code> hostname and <code>*.BASE_URL</code> to this server using A/AAAA records or CNAME records. Ideal: safest isolation with all replay features. <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Modes" target="_blank" rel="noopener noreferrer">Setup instructions</a>.</small></span></label>
+        <label class="abx-question-option"><input type="radio" name="archivebox-dns-mode" value="wildcard"> <span><b>⭐ Wildcard DNS</b><small>Point both the bare <code>BASE_URL</code> hostname and <code>*.BASE_URL</code> to this server using A/AAAA records or CNAME records. Ideal: safest isolation with all replay features. <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#server_security_mode" target="_blank" rel="noopener noreferrer">Setup instructions</a>.</small></span></label>
       </div>
       <div class="abx-question-status" id="archivebox-setup-dns-status">Choose a DNS mode to begin testing.</div>
     </fieldset>

archivebox/templates/core/snapshot.html

@@ -1671,6 +1671,13 @@
                     return false
                 }
 
+                // A new snapshot has only the synthetic "Browse all files" card.
+                // Selecting it would replace the live monitor with an empty index.
+                // Keep progress/empty state until the user explicitly opens a file.
+                if (document.getElementById('main-frame-wrapper')?.dataset.hasOutputs !== '1') {
+                    return false
+                }
+
                 const selectedCard = document.querySelector('.thumb-card.selected-card') || document.querySelector('.thumb-card')
                 const selectedLink = selectedCard && (selectedCard.querySelector('a[target=preview]') || selectedCard.querySelector('a'))
                 if (selectedCard && selectedLink) {

archivebox/templates/core/snapshot_output_cards.html

@@ -6,7 +6,7 @@
                         <div class="thumb-card{% if forloop.first %} selected-card{% endif %}" data-plugin-name="{{result.name|plugin_name}}" data-output-group="{{result.output_group}}" data-output-size="{{result.size|default:0}}"{% if preview_url %} data-preview-url="{{preview_url}}"{% endif %}{% if display_path %} data-output-path="{{display_path}}"{% endif %}>
                                 <div class="thumb-body">
                                     <div class="thumb-actions">
-                                        {% if not STATIC_EXPORT %}<a href="{% snapshot_files_url snapshot result.folder_path|default:result.name %}" data-no-preview="1" title="Open output folder" target="_blank" rel="noopener">📁</a>{% endif %}
+                                        {% if not STATIC_EXPORT %}<a href="{% snapshot_files_url snapshot result.folder_path %}" data-no-preview="1" title="Open output folder" target="_blank" rel="noopener">📁</a>{% endif %}
                                         {% if display_path %}
                                             <a href="{{display_url}}" data-no-preview="1" title="Download output file" download>⬇️</a>
                                         {% endif %}

archivebox/templates/core/system_warnings_banner.html

@@ -27,7 +27,7 @@
       Only <code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">{{ free_gb }} GiB</code>
       free on <code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">DATA_DIR</code>
       — new archive jobs will fail until space is reclaimed.
-      <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Disk-Usage" target="_blank" rel="noopener noreferrer" style="color:#fff;text-decoration:underline;margin-left:6px;">docs</a>
+      <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#purging-entries" target="_blank" rel="noopener noreferrer" style="color:#fff;text-decoration:underline;margin-left:6px;">docs</a>
     {% elif mode == "high_memory" %}
       Virtual memory at <code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">{{ mem_pct }}%</code>
       — the host is close to OOM. Consider stopping crawls or scaling up.
@@ -35,7 +35,7 @@
       15-min loadavg <code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">{{ load_15 }}</code>
     {% elif mode == "unsafe" %}
       ArchiveBox single-domain mode — archived pages share an origin with this site
-      <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Modes" target="_blank" rel="noopener noreferrer" style="color:#fff;text-decoration:underline;margin-left:6px;">why?</a>
+      <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#server_security_mode" target="_blank" rel="noopener noreferrer" style="color:#fff;text-decoration:underline;margin-left:6px;">why?</a>
     {% elif mode == "base_url_mismatch" %}
       Editing disabled from unrecognized URL. Visit
       <a href="{{ configured_base_url }}" style="color:#fff;text-decoration:underline;text-transform:none;">{{ configured_base_url }}</a>

archivebox/templates/static/admin-inline-tags.js

@@ -85,6 +85,7 @@
             link.href = '/admin/core/snapshot/?tags__id__exact=' + td.id;
             link.className = 'tag-link';
             link.textContent = td.name;
+            link.title = td.name;
             pill.appendChild(link);
 
             var removeBtn = document.createElement('button');

archivebox/templates/static/admin.css

@@ -1155,7 +1155,19 @@ body.change-list:not(.model-snapshot) #changelist #toolbar,
     opacity: 0.75;
 }
 
+#result_list tbody td.field-preview_icon {
+    position: relative;
+}
+
+#result_list tbody td.field-preview_icon > a {
+    position: absolute;
+    inset: 4px 2px;
+    display: block;
+}
+
 #result_list tbody td.field-preview_icon .snapshot-thumbnail {
+    width: 100% !important;
+    height: 100% !important;
     transition: box-shadow 140ms ease, transform 140ms ease;
 }
 
@@ -2226,7 +2238,7 @@ tbody .output-link:hover {opacity: 1;}
     .model-snapshot.change-list #result_list tbody th.field-created_at_display > a,
     .model-snapshot.change-list #result_list tbody td.field-created_at_display > a {
         display: inline-flex;
-        max-width: none;
+        max-width: 100%;
         white-space: nowrap !important;
     }
 
@@ -2234,8 +2246,10 @@ tbody .output-link:hover {opacity: 1;}
     .model-snapshot.change-list #result_list tbody td.field-created_at_display .snapshot-created-at {
         display: inline-flex;
         flex-direction: row !important;
+        flex-wrap: wrap;
         justify-content: center;
         gap: 5px;
+        row-gap: 0;
         white-space: nowrap !important;
     }
 
@@ -2250,13 +2264,10 @@ tbody .output-link:hover {opacity: 1;}
         grid-area: preview;
         align-self: stretch;
         place-items: center;
-        min-height: 110px;
         overflow: hidden;
     }
 
     .model-snapshot.change-list #result_list tbody td.field-preview_icon .snapshot-thumbnail {
-        width: 80px !important;
-        height: 110px !important;
         max-width: 100%;
         box-sizing: border-box;
     }
@@ -2278,7 +2289,7 @@ tbody .output-link:hover {opacity: 1;}
         padding-left: 0;
     }
 
-    .model-snapshot.change-list #result_list tbody td.field-title_str a {
+    .model-snapshot.change-list #result_list tbody td.field-title_str > a {
         display: block;
         max-width: 100%;
         overflow-wrap: anywhere;
@@ -2514,26 +2525,12 @@ tbody .output-link:hover {opacity: 1;}
     }
 
     .model-snapshot.change-list #result_list tbody tr {
-        grid-template-columns: 24px 84px minmax(0, 1fr);
         gap: 2px 4px;
-        grid-template-areas:
-            "check preview date"
-            "check preview title"
-            "check preview tags"
-            "check preview status"
-            "check preview files"
-            "check preview size";
     }
 
     .model-snapshot.change-list #result_list tbody td.field-files .files-icons {
         gap: 0;
     }
-
-    .model-snapshot.change-list #result_list tbody td.field-status_with_progress,
-    .model-snapshot.change-list #result_list tbody td.field-size_with_stats {
-        justify-self: start;
-        text-align: left;
-    }
 }
 
 body.embedded-change-list #content {

archivebox/tests/conftest.py

@@ -32,6 +32,13 @@ PYTEST_BASETEMP_ROOT = (REPO_ROOT / "tests" / "out").resolve()
 SESSION_DATA_DIR = Path(
     os.environ.get("ARCHIVEBOX_PYTEST_SESSION_DATA_DIR") or tempfile.mkdtemp(prefix="archivebox-pytest-session-"),
 ).resolve()
+if worker_id := os.environ.get("PYTEST_XDIST_WORKER"):
+    # Workers inherit the coordinator's environment. Give each one its own
+    # collection and cleanup root so a finishing worker cannot delete another's.
+    SESSION_DATA_DIR /= worker_id
+    for runtime_key in ("ABXPKG_LIB_DIR", "PERSONAS_DIR", "CHROME_USER_DATA_DIR"):
+        if runtime_path := os.environ.get(runtime_key):
+            os.environ[runtime_key] = str(Path(runtime_path) / worker_id)
 
 os.environ["ARCHIVEBOX_PYTEST_SESSION_DATA_DIR"] = str(SESSION_DATA_DIR)
 os.environ["DATA_DIR"] = str(SESSION_DATA_DIR)
@@ -323,6 +330,18 @@ def pytest_configure():
     if not apps.ready:
         django.setup()
 
+    if os.environ.get("PYTEST_XDIST_WORKER"):
+        from django.conf import settings
+        from archivebox.config import django as django_config
+        from archivebox.config.common import get_config
+
+        # pytest-django may import startup config before this conftest isolates
+        # the worker's paths. Refresh that real startup config once so HTTP
+        # requests resolve the same installed binaries as worker subprocesses;
+        # otherwise a service restart searches the coordinator's empty lib dir.
+        django_config.CONFIG = get_config(include_machine=False)
+        settings.CONFIG = django_config.CONFIG
+
 
 @pytest.fixture(scope="session")
 def django_db_modify_db_settings(
@@ -437,7 +456,7 @@ def hermetic_lib_dir(tmp_path):
 @pytest.fixture
 def cached_abxpkg_lib_dir():
     """Reuse one real abxpkg installation cache for the current pytest session."""
-    lib_dir = SESSION_DATA_DIR / "lib"
+    lib_dir = Path(os.environ["ABXPKG_LIB_DIR"]) if os.environ.get("ABXPKG_LIB_DIR") else SESSION_DATA_DIR / "lib"
     lib_dir.mkdir(parents=True, exist_ok=True)
     original_lib_dir = os.environ.get("ABXPKG_LIB_DIR")
     os.environ["ABXPKG_LIB_DIR"] = str(lib_dir)
@@ -1010,11 +1029,15 @@ def kill_processes_for_data_dir(data_dir: Path) -> None:
             continue
         if pid != os.getpid():
             try:
-                os.kill(pid, signal.SIGTERM)
-            except ProcessLookupError:
+                # Retain the process identity before signalling: a fast, clean
+                # exit can remove the PID before a subsequent Process lookup.
+                # terminate() also checks that the PID has not been reused.
+                process = psutil.Process(pid)
+                process.terminate()
+            except psutil.NoSuchProcess:
                 pass
             else:
-                processes.append(psutil.Process(pid))
+                processes.append(process)
     _gone, alive = psutil.wait_procs(processes, timeout=10)
     assert not alive, f"processes did not stop after SIGTERM: {[proc.pid for proc in alive]}"
 
@@ -1659,6 +1682,8 @@ def resolve_abxpkg_binary_env(
     deps_from: Path | list[Path] | tuple[Path, ...] | None = None,
 ) -> dict[str, str]:
     """Resolve required test dependencies through abxpkg's normal install path."""
+    from abxpkg.config import apply_exec_env
+
     command_env = dict(env) if env is not None else os.environ.copy()
     command_env["ABXPKG_LIB_DIR"] = str(lib_dir)
     command = [
@@ -1679,7 +1704,11 @@ def resolve_abxpkg_binary_env(
     )
     assert result.returncode == 0, result.stderr or result.stdout
     payload = json.loads(result.stdout)
-    return {str(key): str(value) for key, value in payload.items()}
+    # abxpkg env emits prepend/append deltas, not a replacement environment.
+    # Applying a PATH ending in ':' verbatim hides the caller's abxpkg CLI.
+    delta = {str(key): str(value) for key, value in payload.items()}
+    apply_exec_env(delta, command_env)
+    return {key: command_env[key] for key in delta}
 
 
 def run_test_hook(
@@ -1777,7 +1806,7 @@ def install_real_binary(
     from archivebox.machine.models import Binary, Machine
 
     binary = Binary.objects.create(
-        machine=machine or Machine.current(refresh=True),
+        machine=machine or Machine.current(),
         name=name,
         binproviders=binproviders,
         overrides=overrides or {},

archivebox/tests/test_api_discovery_performance.py

@@ -0,0 +1,54 @@
+"""Server discovery must not repeat schema generation or probe installed tools."""
+
+import sys
+import threading
+import os
+from pathlib import Path
+
+import pytest
+from django.test.utils import override_script_prefix
+
+from archivebox.config import VERSION
+from archivebox.machine.models import Machine
+from archivebox.tests.conftest import install_real_binary, resolve_abxpkg_binary_env
+
+
+@pytest.mark.django_db(transaction=True)
+@pytest.mark.parametrize("script_name", ["", "/mounted"])
+def test_repeated_discovery_reuses_schema_without_binary_filesystem_probes(client, hermetic_lib_dir, script_name):
+    install_real_binary("node", binproviders="env,apt,brew")
+    resolve_abxpkg_binary_env(hermetic_lib_dir, "node")
+    machine = Machine.current()
+    machine.config = {**machine.config, "NODE_BINARY": str(hermetic_lib_dir / "env" / "bin" / "node")}
+    machine.save(update_fields=["config"])
+    request = {"HTTP_HOST": "api.archivebox.localhost:5797", "SCRIPT_NAME": script_name}
+    with override_script_prefix(script_name):
+        first = client.get("/api/v1/openapi.json", **request)
+    assert first.status_code == 200, first.content
+    schema_calls = []
+    filesystem_calls = []
+
+    def observe(frame, event, arg):
+        if event == "call" and frame.f_code.co_name == "model_json_schema":
+            schema_calls.append(frame.f_code.co_filename)
+        if (event == "call" and frame.f_code.co_name in {"stat", "lstat", "open", "iterdir", "glob", "rglob"}) or (
+            event == "c_call" and arg in (os.stat, os.lstat)
+        ):
+            path = frame.f_locals.get("self", frame.f_locals.get("path"))
+            if isinstance(path, (str, Path)) and Path(path).is_relative_to(hermetic_lib_dir):
+                filesystem_calls.append(str(path))
+
+    previous = sys.getprofile()
+    previous_thread = threading.getprofile()
+    threading.setprofile_all_threads(observe)
+    try:
+        with override_script_prefix(script_name):
+            second = client.get("/api/v1/openapi.json", **request)
+    finally:
+        threading.setprofile_all_threads(previous_thread)
+        sys.setprofile(previous)
+    assert second.status_code == 200, second.content
+    assert second.json() == first.json()
+    assert second.json()["info"]["version"] == VERSION
+    assert f"{script_name}/api/v1/core/snapshots" in second.json()["paths"]
+    assert (len(schema_calls), filesystem_calls) == (0, [])

archivebox/tests/test_api_v1_cli_add.py

@@ -478,3 +478,40 @@ def test_api_cli_add_rejects_file_path_and_shell_injection_payloads(tmp_path):
     with use_archivebox_db(tmp_path):
         tag_names = set(SnapshotTag.objects.filter(snapshot=snapshot).values_list("tag__name", flat=True))
     assert "api-security" in tag_names
+
+
+def test_api_add_only_queues_work_without_storage_or_worker_control(client, api_headers):
+    import sys
+    from archivebox.config import CONSTANTS
+
+    calls = []
+
+    def observe(frame, event, arg):
+        if event != "call":
+            return
+        name = frame.f_code.co_name
+        if name == "ensure_background_runner":
+            calls.append(name)
+        if name in {"stat", "lstat", "exists", "is_dir", "is_file", "is_symlink", "mkdir", "iterdir", "symlink_to"}:
+            path = frame.f_locals.get("self")
+            if isinstance(path, Path) and path.is_relative_to(CONSTANTS.ARCHIVE_DIR):
+                calls.append((name, str(path)))
+
+    previous = sys.getprofile()
+    sys.setprofile(observe)
+    try:
+        response = client.post(
+            "/api/v1/cli/add",
+            data={"urls": ["https://example.com/queue-only"], "tag": "queue-only"},
+            content_type="application/json",
+            **api_headers,
+        )
+    finally:
+        sys.setprofile(previous)
+    assert response.status_code == 200, response.content
+    crawl = Crawl.objects.get(pk=response.json()["result"]["crawl_id"])
+    assert crawl.urls == "https://example.com/queue-only"
+    assert crawl.status == Crawl.StatusChoices.QUEUED
+    assert crawl.retry_at is not None
+    assert not crawl.snapshot_set.exists()
+    assert calls == []

archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py

@@ -477,25 +477,38 @@ def test_paused_snapshot_survives_server_restart_and_resumes_via_api(client, tmp
         stop_server(tmp_path)
 
 
-def test_rest_snapshot_delete_removes_output_dir(client, api_headers):
-    url = "https://example.com/delete-path-snapshot"
+@pytest.mark.timeout(180)
+def test_rest_snapshot_delete_removes_output_dir(tmp_path, recursive_test_site):
+    import time
+    from .test_snapshot_service import _snapshot_state
+    from .conftest import create_admin_and_token, live_api_request, start_archivebox_server, stop_archivebox_process
 
-    response = api_client_request(
-        client,
-        "post",
-        "/api/v1/core/snapshots",
-        payload={"url": url, "depth": 0, "status": Snapshot.StatusChoices.QUEUED},
-        headers=api_headers,
-    )
-    assert response.status_code == 200, response.content.decode()
-
-    snapshot = Snapshot.objects.get(url=url)
-    snapshot_dir = Path(snapshot.output_dir)
-    snapshot_dir.mkdir(parents=True, exist_ok=True)
-    (snapshot_dir / "delete-path-test.txt").write_text("snapshot output")
-    assert snapshot_dir.exists()
-
-    response = client.delete(f"/api/v1/core/snapshot/{snapshot.id}", **api_headers)
-    assert response.status_code == 200, response.content.decode()
-    assert not Snapshot.objects.filter(pk=snapshot.pk).exists()
-    assert not snapshot_dir.exists()
+    init_archive(tmp_path)
+    port = get_free_port()
+    env = cli_env(port=port, server=True, PLUGINS="wget")
+    url = recursive_test_site["root_url"]
+    captured = run_archivebox_cmd(["add", "--plugins=wget", url], cwd=tmp_path, env=env)
+    assert captured.returncode == 0, captured.stdout + captured.stderr
+    state = _snapshot_state(tmp_path, url)
+    assert any("Root" in path.read_text(errors="ignore") for path in (state["snapshot_dir"] / "wget").rglob("*.html"))
+    with use_archivebox_db(tmp_path):
+        snapshot_id = str(Snapshot.objects.get(url=url).pk)
+    token = create_admin_and_token(tmp_path)
+    server = start_archivebox_server(tmp_path, port=port, env=env, log_name="api-delete-handoff.log")
+    try:
+        response = live_api_request(port, "delete", f"/api/v1/core/snapshot/{snapshot_id}", api_token=token)
+        assert response.status_code == 200, response.text
+        assert response.json()["queued_count"] == 1
+        assert response.json()["deleted_count"] == 0
+        deadline = time.monotonic() + 75
+        with use_archivebox_db(tmp_path):
+            while Snapshot.objects.filter(pk=snapshot_id).exists() and time.monotonic() < deadline:
+                assert server.poll() is None
+                time.sleep(0.1)
+            assert not Snapshot.objects.filter(pk=snapshot_id).exists()
+        assert not state["snapshot_dir"].exists()
+        assert not state["crawl_link"].is_symlink()
+        missing = live_api_request(port, "get", f"/api/v1/core/snapshot/{snapshot_id}", api_token=token)
+        assert missing.status_code == 404
+    finally:
+        stop_archivebox_process(server)

archivebox/tests/test_api_v1_core_snapshots.py

@@ -13,6 +13,65 @@ from archivebox.crawls.models import Crawl
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
+def test_browser_metadata_respects_only_new_and_reports_capture_owner(client, api_headers):
+    url = "https://example.com/browser-deduplication"
+    for name in ("Original", "Replacement"):
+        persona = client.post(
+            "/api/v1/personas/sync",
+            data={"extension_persona_id": f"dedupe-{name}", "name": name},
+            content_type="application/json",
+            **api_headers,
+        )
+        assert persona.status_code == 200, persona.content
+
+    def submit(only_new, persona):
+        queued = client.post(
+            "/api/v1/cli/add",
+            data={"urls": [url], "only_new": only_new, "persona": persona},
+            content_type="application/json",
+            **api_headers,
+        )
+        assert queued.status_code == 200, queued.content
+        crawl_id = queued.json()["result"]["crawl_id"]
+        metadata = client.post(
+            "/api/v1/core/snapshots",
+            data={"url": url, "crawl_id": crawl_id},
+            content_type="application/json",
+            **api_headers,
+        )
+        assert metadata.status_code == 200, metadata.content
+        return crawl_id, metadata.json()
+
+    first_crawl, first = submit(True, "Original")
+    second_crawl, reused = submit(True, "Replacement")
+    assert second_crawl != first_crawl
+    assert reused["id"] == first["id"]
+    assert reused["crawl_id"] == first_crawl
+    assert reused["persona"] == "Original"
+    assert Snapshot.objects.filter(url=url).count() == 1
+    forced_crawl, forced = submit(False, "Replacement")
+    assert forced["id"] != first["id"]
+    assert forced["crawl_id"] == forced_crawl
+    assert forced["persona"] == "Replacement"
+    assert Snapshot.objects.filter(url=url).count() == 2
+
+
+def test_snapshot_delete_only_queues_cleanup(client, api_headers):
+    created = client.post(
+        "/api/v1/core/snapshots",
+        data={"url": "https://example.com/queued-delete"},
+        content_type="application/json",
+        **api_headers,
+    )
+    assert created.status_code == 200, created.content
+    snapshot_id = created.json()["id"]
+    response = client.delete(f"/api/v1/core/snapshot/{snapshot_id}", **api_headers)
+    assert response.status_code == 200, response.content
+    assert response.json()["queued_count"] == 1
+    assert response.json()["deleted_count"] == 0
+    assert Snapshot.objects.get(pk=snapshot_id).status == Snapshot.DELETING_STATE
+
+
 def test_snapshots_api_filters_status_column(client, api_admin_user, api_headers):
     crawl = Crawl.objects.create(
         urls="https://example.com",
@@ -150,3 +209,81 @@ def test_new_snapshot_creation_does_not_wait_for_active_crawl(client, api_admin_
     assert response.status_code == 200, response.content
     assert Snapshot.objects.filter(url=url, crawl=crawl).count() == 1
     assert elapsed < 1
+
+
+def test_snapshot_metadata_requests_do_not_touch_archive_storage(client, api_admin_user, api_headers):
+    """Metadata writes and serialization must work without replay storage access."""
+    import sys
+    import threading
+    from pathlib import Path
+    from archivebox.config import CONSTANTS
+
+    url = "https://example.com/metadata-without-storage"
+    crawl = Crawl.objects.create(urls=url, created_by=api_admin_user)
+    storage_calls = []
+
+    def observe(frame, event, arg):
+        if event == "call" and frame.f_code.co_name in {
+            "stat",
+            "lstat",
+            "exists",
+            "is_dir",
+            "is_file",
+            "is_symlink",
+            "open",
+            "mkdir",
+            "iterdir",
+            "glob",
+            "rglob",
+            "readlink",
+            "symlink_to",
+            "unlink",
+        }:
+            path = frame.f_locals.get("self", frame.f_locals.get("path"))
+            if isinstance(path, (str, Path)) and Path(path).is_relative_to(CONSTANTS.ARCHIVE_DIR):
+                storage_calls.append((frame.f_code.co_name, str(path)))
+
+    previous = sys.getprofile()
+    thread_profile = threading.getprofile()
+    threading.setprofile_all_threads(observe)
+    try:
+        response = client.post(
+            "/api/v1/core/snapshots",
+            data={"url": url, "crawl_id": str(crawl.id), "title": "Browser title", "tags": ["browser"]},
+            content_type="application/json",
+            **api_headers,
+        )
+        assert response.status_code == 200, response.content
+        snapshot_id = response.json()["id"]
+        updated = client.post(
+            "/api/v1/core/snapshots",
+            data={"url": url, "crawl_id": str(crawl.id), "title": "Updated title", "tags": ["browser"]},
+            content_type="application/json",
+            **api_headers,
+        )
+        assert updated.status_code == 200, updated.content
+        result = client.post(
+            "/api/v1/core/archiveresults",
+            data={"snapshot_id": snapshot_id, "plugin": "chrome_extension_viewport", "status": "started"},
+            **api_headers,
+        )
+        assert result.status_code == 200, result.content
+        result_id = result.json()["id"]
+        result_get = client.get(f"/api/v1/core/archiveresult/{result_id}", **api_headers)
+        assert result_get.status_code == 200, result_get.content
+        assert result_get.json()["output_files"] == {}
+        fetched = client.get(f"/api/v1/core/snapshot/{snapshot_id}", **api_headers)
+        assert fetched.status_code == 200, fetched.content
+        assert fetched.json()["archiveresults"][0]["id"] == result_id
+        deleted = client.delete(f"/api/v1/core/snapshot/{snapshot_id}", **api_headers)
+        assert deleted.status_code == 200, deleted.content
+        assert deleted.json()["queued_count"] == 1
+    finally:
+        threading.setprofile_all_threads(thread_profile)
+        sys.setprofile(previous)
+    snapshot = Snapshot.objects.get(pk=snapshot_id)
+    assert snapshot.title == "Updated title"
+    assert snapshot.status == Snapshot.DELETING_STATE
+    assert list(snapshot.tags.values_list("name", flat=True)) == ["browser"]
+    assert fetched.json()["title"] == "Updated title"
+    assert storage_calls == []

archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_filename.py

@@ -1,4 +1,6 @@
 import pytest
+import os
+import time
 
 from archivebox.crawls.models import Crawl
 
@@ -15,3 +17,19 @@ def test_basic_success_case_request(client, tmp_path, api_admin_user, api_header
     response = client.get(f"/api/v1/crawls/crawl/{crawl.id}/files/folder/basic.txt", **api_headers)
 
     assert response.status_code == 200, response.content
+
+
+def test_screencast_frame_is_checked_by_image_request(client, api_admin_user, api_headers):
+    crawl = Crawl.objects.create(urls="https://example.com/screencast", created_by=api_admin_user)
+    frame = crawl.output_dir / "chrome_screencast" / "latest.jpg"
+    frame.parent.mkdir(parents=True, exist_ok=True)
+    frame.write_bytes(b"stored frame")
+    started_at = time.time()
+    os.utime(frame, (started_at - 10, started_at - 10))
+    url = f"/api/v1/crawls/crawl/{crawl.id}/files/chrome_screencast/latest.jpg?after={started_at}"
+    assert client.get(url, **api_headers).status_code == 404
+    os.utime(frame, (started_at + 1, started_at + 1))
+    response = client.get(url, **api_headers)
+    assert response.status_code == 200
+    assert b"".join(response.streaming_content) == b"stored frame"
+    assert client.get(url.replace(str(started_at), "invalid"), **api_headers).status_code == 400

archivebox/tests/test_api_v1_machine_binaries.py

@@ -8,7 +8,7 @@ pytestmark = pytest.mark.django_db(transaction=True)
 
 
 def test_basic_success_case_request(client, tmp_path, api_headers):
-    machine = Machine.current(refresh=True)
+    machine = Machine.current()
     binary = install_real_binary("python3", machine=machine)
 
     response = client.get("/api/v1/machine/binaries", **api_headers)

archivebox/tests/test_api_v1_machine_binary_binary_id.py

@@ -8,7 +8,7 @@ pytestmark = pytest.mark.django_db(transaction=True)
 
 
 def test_basic_success_case_request(client, tmp_path, api_headers):
-    machine = Machine.current(refresh=True)
+    machine = Machine.current()
     binary = install_real_binary("python3", machine=machine)
 
     response = client.get(f"/api/v1/machine/binary/{binary.id}", **api_headers)

archivebox/tests/test_api_v1_machine_binary_by_name_name.py

@@ -8,7 +8,7 @@ pytestmark = pytest.mark.django_db(transaction=True)
 
 
 def test_basic_success_case_request(client, tmp_path, api_headers):
-    machine = Machine.current(refresh=True)
+    machine = Machine.current()
     binary = install_real_binary("python3", machine=machine)
 
     response = client.get("/api/v1/machine/binary/by-name/python3", **api_headers)

archivebox/tests/test_api_v1_machine_machine_machine_id.py

@@ -7,7 +7,7 @@ pytestmark = pytest.mark.django_db(transaction=True)
 
 
 def test_basic_success_case_request(client, tmp_path, api_headers):
-    machine = Machine.current(refresh=True)
+    machine = Machine.current()
 
     response = client.get(f"/api/v1/machine/machine/{machine.id}", **api_headers)
 

archivebox/tests/test_api_v1_machine_machines.py

@@ -7,7 +7,7 @@ pytestmark = pytest.mark.django_db(transaction=True)
 
 
 def test_basic_success_case_request(client, tmp_path, api_headers):
-    Machine.current(refresh=True)
+    Machine.current()
 
     response = client.get("/api/v1/machine/machines", **api_headers)
 

archivebox/tests/test_archive_result_service.py

@@ -51,7 +51,7 @@ def _run_shipped_snapshot_hook(
 
     from abx_dl.services.process_service import ProcessService as HookProcessService
     from abx_dl.services.archive_result_service import ArchiveResultService as HookArchiveResultService
-    from abx_plugins.plugins.base.utils import _collect_required_schema_paths, get_hydrated_required_binaries
+    from abx_plugins.plugins.base.utils import _collect_required_schema_paths
     from abxpkg import prepare_script_exec_plan
     from archivebox.core.models import ArchiveResult
     from archivebox.machine.models import Process
@@ -66,15 +66,9 @@ def _run_shipped_snapshot_hook(
     hook_path = matching_hooks[0].path
     projected_hook_name = event_hook_name or hook_name
     hook_config = hook_path.parent / "config.json"
-    for required_binary in get_hydrated_required_binaries(
-        hook_config,
-        environ={**os.environ, "ABXPKG_LIB_DIR": str(lib_dir)},
-    ):
-        install_real_binary(
-            required_binary["name"],
-            binproviders=required_binary["binproviders"],
-            overrides=required_binary.get("overrides"),
-        )
+    # This helper runs standalone hooks. Resolve their dependencies through the
+    # public abxpkg CLI below; creating a new ArchiveBox Binary row on every
+    # invocation makes a second run collide with the already-installed identity.
     binary_env = resolve_abxpkg_binary_env(lib_dir, deps_from=_collect_required_schema_paths(hook_config))
     hook_env = {
         **binary_env,
@@ -296,9 +290,11 @@ def test_archiveresult_event_create_uses_one_result_lookup():
     assert len(result_lookups) == 1
 
 
-def test_process_completed_projects_failed_archiveresult_from_shipped_hook(tmp_path, hermetic_lib_dir):
+def test_failed_shipped_title_hook_projects_failure_without_overwriting_title(tmp_path, hermetic_lib_dir):
     from archivebox.core.models import ArchiveResult
 
+    # Result linkage and the unchanged title are consequences of the same failed
+    # hook. Check them together so this isolated runtime is installed only once.
     snapshot = _create_snapshot()
     process, result = _run_shipped_snapshot_hook(
         snapshot,
@@ -310,22 +306,6 @@ def test_process_completed_projects_failed_archiveresult_from_shipped_hook(tmp_p
     assert result.status == ArchiveResult.StatusChoices.FAILED
     assert result.process_id == process.id
     assert "Chrome session" in result.output_str
-    _cleanup_machine_process_rows()
-
-
-def test_failed_title_archiveresult_does_not_overwrite_snapshot_title(tmp_path, hermetic_lib_dir):
-    from archivebox.core.models import ArchiveResult
-
-    snapshot = _create_snapshot()
-    _, result = _run_shipped_snapshot_hook(
-        snapshot,
-        plugin="title",
-        hook_name="on_Snapshot__54_title.js",
-        lib_dir=hermetic_lib_dir,
-        expected_exit_codes=(1,),
-    )
-    assert result.status == ArchiveResult.StatusChoices.FAILED
-    assert "Chrome session" in result.output_str
     snapshot.refresh_from_db()
     assert snapshot.title in (None, "")
     assert snapshot.resolved_title == ""